diff --git a/.gitignore b/.gitignore index 8a22bc67f..cc1c664a6 100644 --- a/.gitignore +++ b/.gitignore @@ -30,5 +30,4 @@ Makefile.main .ci/ _example/main _example/*.exe - -go.sum \ No newline at end of file +test-server diff --git a/.travis.yml b/.travis.yml index 930d32726..e06f94818 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,27 +1,20 @@ language: go +go_import_path: github.com/src-d/go-mysql-server -go_import_path: gopkg.in/src-d/go-mysql-server.v0 +env: + global: + - LD_LIBRARY_PATH="/usr/local/lib":${LD_LIBRARY_PATH} + - GO111MODULE=on + - GOPROXY=https://proxy.golang.org addons: apt: packages: - - libonig-dev - libmysqlclient-dev matrix: fast_finish: true -sudo: required - -services: - - docker - -install: - - go get -u github.com/golang/dep/cmd/dep - - touch Gopkg.toml - - dep ensure -v -add "github.com/pilosa/pilosa@f62dbc00b96f596a1f2ef8b4e87ba8ec847eda37" "github.com/moovweb/rubex@b3d9ff6ad7d9b14f94a91c8271cd9ad9e77132e5" - - make dependencies - before_script: - sudo service mysql stop @@ -30,65 +23,70 @@ script: jobs: include: - - go: 1.10.x - go: 1.11.x + name: 'Unit tests Go 1.11' + - go: 1.12.x + name: 'Unit tests Go 1.12' # Integration test builds for 3rd party clients - - go: 1.11.x + - go: 1.12.x + name: 'Integration test go' script: - make TEST=go integration - language: python python: '3.6' before_install: - - eval "$(gimme 1.11)" - install: - - go get ./... + - eval "$(gimme 1.12.4)" + name: 'Integration test python-pymysql' script: - make TEST=python-pymysql integration - language: python python: '3.6' before_install: - - eval "$(gimme 1.11)" - install: - - go get ./... + - eval "$(gimme 1.12.4)" + name: 'Integration test python-mysql' script: - make TEST=python-mysql integration + - language: python + python: '3.6' + before_install: + - eval "$(gimme 1.12.4)" + name: 'Integration test python-sqlalchemy' + script: + - make TEST=python-sqlalchemy integration + - language: php php: '7.1' before_install: - - eval "$(gimme 1.11)" - install: - - go get ./... + - eval "$(gimme 1.12.4)" + name: 'Integration test php' script: - make TEST=php integration - language: ruby ruby: '2.3' before_install: - - eval "$(gimme 1.11)" - install: - - go get ./... + - eval "$(gimme 1.12.4)" + name: 'Integration test ruby' script: - make TEST=ruby integration - language: java - jdk: oraclejdk9 + jdk: openjdk8 before_install: - - eval "$(gimme 1.11)" - install: - - go get ./... + - eval "$(gimme 1.12.4)" + name: 'Integration test jdbc-mariadb' script: - make TEST=jdbc-mariadb integration - language: node_js - node_js: '7' + node_js: '12' before_install: - - eval "$(gimme 1.11)" - install: - - go get ./... + - eval "$(gimme 1.12.4)" + name: 'Integration test javascript' script: - make TEST=javascript integration @@ -96,17 +94,15 @@ jobs: mono: none dotnet: '2.1' before_install: - - eval "$(gimme 1.11)" - install: - - go get ./... + - eval "$(gimme 1.12.4)" + name: 'Integration test dotnet' script: - make TEST=dotnet integration - language: c compiler: clang before_install: - - eval "$(gimme 1.11)" - install: - - go get ./... + - eval "$(gimme 1.12.4)" + name: 'Integration test c' script: - - make TEST=c integration \ No newline at end of file + - make TEST=c integration diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 000000000..24f3aebe8 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,142 @@ +# Architecture overview + +This document provides an overview of all parts and pieces of the project as well as how they fit together. It is meant to help new contributors understand where things may be, and how changes in some components may interact with other components of the system. + +## Root package (`sqle`) + +This is where the engine lives. The engine is the piece that coordinates and makes all other pieces work together as well as the main API users of the system will use to create and configure an engine and perform queries. + +Because this is the point where all components fit together, it is also where integration tests are. Those integration tests can be found in `engine_test.go`. +A test should be added here, plus in any specific place where the feature/issue belonged, if needed. + +**How to add integration tests** + +The test executing all integration test is `TestQueries`, which you can run with the following command: + +``` +go test -run=TestQueries +``` + +This test is just executing all the queries in a loop. New test cases should be added to the `queries` package variable at the top of `engine_test.go`. +Simply add a new element to the slice with the query and the expected result. + +## `sql` + +This package is probably the most important of the project. It has several main roles: +- Defines the main interfaces used in the rest of the packages `Node`, `Expression`, ... +- Provides implementations of components used in the rest of the packages `Row`, `Context`, `ProcessList`, `Catalog`, ... +- Defines the `information_schema` table, which is a special table available in all databases and contains some data about the schemas of other tables. + +### `sql/analyzer` + +The analyzer is the more complex component of the project. It contains a main component, which is the `Analyzer`, in charge of executing its registered rules on execution trees for resolving some parts, removing redundant data, optimizing things for performance, etc. + +There are several phases on the analyzer, because some rules need to be run before others, some need to be executed several times, other just once, etc. +Inside `rules.go` are all the default rules and the phases in which they're executed. + +On top of that, all available rules are defined in this package. Each rule has a specific role in the analyzer. Rules should be as small and atomic as possible and try to do only one job and always produce a tree that is as resolved as the one it received or more. + +### `sql/expression` + +This package includes the implementation of all the SQL expressions available in go-mysql-server, except functions. Arithmetic operators, logic operators, conversions, etc are implemented here. + +Inside `registry.go` there is a registry of all the default functions, even if they're not defined here. + +`Inspect` and `Walk` utility functions are provided to inspect expressions. + +### `sql/expression/function` + +Implementation of all the functions available in go-mysql-server. + +### `sql/expression/function/aggregation` + +Implementation of all the aggregation functions available in go-mysql-server. + +### `sql/index` + +Contains the index driver configuration file implementation and other utilities for dealing with index drivers. + +### `sql/index/pilosa` + +Actual implementation of an index driver. Underneath, it's using a bitmap database called pilosa (hence the name) to implement bitmap indexes. + +### `sql/parse` + +This package exposes the `Parse` function, which parses a SQL query and translates it into an execution plan. + +Parsing is done using `vitess` parser, but sometimes there are queries vitess cannot parse. In this case, custom parsers are used. Otherwise, vitess is used to parse the query and then converted to a go-mysql-server execution plan. + +### `sql/plan` + +All the different nodes of the execution plan (except for very specific nodes used in some optimisation rules) are defined here. + +For example, `SELECT foo FROM bar` is translated into the following plan: + +``` +Project(foo) + |- Table(bar) +``` + +Which means, the execution plan is a `Project` node projecting `foo` and has a `ResolvedTable`, which is `bar` as its children. + +Each node inside this package implements at least the `sql.Node` interface, but it can implement more. `sql.Expressioner`, for example. + +Along with the nodes, `Inspect` and `Walk` functions are provided as utilities to inspect an execution tree. + +## `server` + +Contains all the code to turn an engine into a runnable server that can communicate using the MySQL wire protocol. + +## `auth` + +This package contains all the code related to the audit log, authentication and permission management in go-mysql-server. + +There are two authentication methods: +- **None:** no authentication needed. +- **Native:** authentication performed with user and password. Read, write or all permissions can be specified for those users. It can also be configured using a JSON file. + +## `internal/similartext` + +Contains a function to `Find` the most similar name from an +array to a given one using the Levenshtein distance algorithm. Used for suggestions on errors. + +## `internal/regex` + +go-mysql-server has multiple regular expression engines, such as oniguruma and the standard Go regexp engine. In this package, a common interface for regular expression engines is defined. +This means, Go standard library `regexp` package should not be used in any user-facing feature, instead this package should be used. + +The default engine is oniguruma, but the Go standard library engine can be used using the `mysql_go_regex` build tag. + +## `test` + +Test contains pieces that are only used for tests, such as an opentracing tracer that stores spans in memory to be inspected later in the tests. + +## `_integration` + +To ensure compatibility with some clients, there is a small example connecting and querying a go-mysql-server server from those clients. Each folder corresponds to a different client. + +For more info about supported clients see [SUPPORTED_CLIENTS.md](/SUPPORTED_CLIENTS.md). + +These integrations tests can be run using this command: + +``` +make TEST=${CLIENT FOLDER NAME} integration +``` + +It will take care of setting up the test server and shutting it down. + +## `_example` + +A small example of how to use go-mysql-server to create a server and run it. + +# Connecting the dots + +`server` uses the engine defined in `sql`. + +Engine uses audit logs and authentication defined in `auth`, parses using `sql/parse` to convert a query into an execution plan, with nodes defined in `sql/plan` and expressions defined in `sql/expression`, `sql/expression/function` and `sql/expression/function/aggregation`. + +After parsing, the obtained execution plan is analyzed using the analyzer defined in `sql/analyzer` and its rules to resolve tables, fields, databases, apply optimisation rules, etc. + +If indexes can be used, the analyzer will transform the query so it uses indexes reading from the drivers in `sql/index` (in this case `sql/index/pilosa` because there is only one driver). + +Once the plan is analyzed, it will be executed recursively from the top of the tree to the bottom to obtain the results and they will be sent back to the client using the MySQL wire protocol. diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 000000000..f48cffb3a --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +* @src-d/data-processing diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3dc0dd69f..71a98af05 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,8 +17,6 @@ the following line: `Signed-off-by: John Doe `, using your This can be done easily using the [`-s`](https://github.com/git/git/blob/b2c150d3aa82f6583b9aadfecc5f8fa1c74aca09/Documentation/git-commit.txt#L154-L161) flag on the `git commit`. - - ## Support Channels The official support channels, for both users and contributors, are: @@ -30,7 +28,6 @@ The official support channels, for both users and contributors, are: search the project - it's likely that another user has already reported the issue you're facing, or it's a known issue that we're already aware of. - ## How to Contribute Pull Requests (PRs) are the main and exclusive way to contribute code to source{d} projects. @@ -45,6 +42,10 @@ In order for a PR to be accepted it needs to pass a list of requirements: - If the PR is a new feature, it has to come with a suite of unit tests, that tests the new functionality. - In any case, all the PRs have to pass the personal evaluation of at least one of the [maintainers](MAINTAINERS) of the project. +### Getting started + +If you are a new contributor to the project, reading [ARCHITECTURE.md](/ARCHITECTURE.md) is highly recommended, as it contains all the details about the architecture of go-mysql-server and its components. + ### Format of the commit message diff --git a/MAINTAINERS b/MAINTAINERS index dc6817245..8d8de2618 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1 +1,2 @@ -Antonio Navarro Perez (@ajnavarro) +Miguel Molina (@erizocosmico) +Juanjo Álvarez Martinez (@juanjux) diff --git a/Makefile b/Makefile index 6aa923435..81c0e52e0 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Package configuration PROJECT = go-mysql-server COMMANDS = -GO_TAGS = oniguruma +UNAME_S := $(shell uname -s) # Including ci Makefile CI_REPOSITORY ?= https://github.com/src-d/ci.git @@ -15,4 +15,13 @@ $(MAKEFILE): integration: ./_integration/run ${TEST} +oniguruma: +ifeq ($(UNAME_S),Linux) + $(shell apt-get install libonig-dev) +endif + +ifeq ($(UNAME_S),Darwin) + $(shell brew install oniguruma) +endif + .PHONY: integration \ No newline at end of file diff --git a/README.md b/README.md index 035935397..1ab7e9ee5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ +_**Notice: This repository is no longer actively maintained, and no further updates will be done, nor issues/PRs will be answered or attended. An alternative actively maintained can be found at https://github.com/dolthub/go-mysql-server repository.**_ + # go-mysql-server [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -Build Status +Build Status codecov GoDoc [![Issues](http://img.shields.io/github/issues/src-d/go-mysql-server.svg)](https://github.com/src-d/go-mysql-server/issues) @@ -33,12 +35,12 @@ Having data in another format that you want as tabular data to query using SQL, ## Installation -The import path for the package is `gopkg.in/src-d/go-mysql-server.v0`. +The import path for the package is `github.com/src-d/go-mysql-server`. To install it, run: ``` -go get gopkg.in/src-d/go-mysql-server.v0 +go get github.com/src-d/go-mysql-server ``` ## Documentation @@ -54,46 +56,98 @@ We are continuously adding more functionality to go-mysql-server. We support a s We support and actively test against certain third-party clients to ensure compatibility between them and go-mysql-server. You can check out the list of supported third party clients in the [SUPPORTED_CLIENTS](./SUPPORTED_CLIENTS.md) file along with some examples on how to connect to go-mysql-server using them. -## Custom functions - -- `COUNT(expr)`: Returns a count of the number of non-NULL values of expr in the rows retrieved by a SELECT statement. -- `MIN(expr)`: Returns the minimum value of expr. -- `MAX(expr)`: Returns the maximum value of expr. -- `AVG(expr)`: Returns the average value of expr. -- `SUM(expr)`: Returns the sum of expr. -- `IS_BINARY(blob)`: Returns whether a BLOB is a binary file or not. -- `SUBSTRING(str, pos)`, `SUBSTRING(str, pos, len)` : Return a substring from the provided string. -- `SUBSTR(str, pos)`, `SUBSTR(str, pos, len)` : Return a substring from the provided string. -- `MID(str, pos)`, `MID(str, pos, len)` : Return a substring from the provided string. -- Date and Timestamp functions: `YEAR(date)`, `MONTH(date)`, `DAY(date)`, `WEEKDAY(date)`, `HOUR(date)`, `MINUTE(date)`, `SECOND(date)`, `DAYOFWEEK(date)`, `DAYOFYEAR(date)`, `NOW()`. -- `ARRAY_LENGTH(json)`: If the json representation is an array, this function returns its size. -- `SPLIT(str,sep)`: Receives a string and a separator and returns the parts of the string split by the separator as a JSON array of strings. -- `CONCAT(...)`: Concatenate any group of fields into a single string. -- `CONCAT_WS(sep, ...)`: Concatenate any group of fields into a single string. The first argument is the separator for the rest of the arguments. The separator is added between the strings to be concatenated. The separator can be a string, as can the rest of the arguments. If the separator is NULL, the result is NULL. -- `COALESCE(...)`: The function returns the first non-null value in a list. -- `LOWER(str)`, `UPPER(str)`: Receives a string and modify it changing all the chars to upper or lower case. -- `CEILING(number)`, `CEIL(number)`: Return the smallest integer value that is greater than or equal to `number`. -- `FLOOR(number)`: Return the largest integer value that is less than or equal to `number`. -- `ROUND(number, decimals)`: Round the `number` to `decimals` decimal places. -- `CONNECTION_ID()`: Return the current connection ID. -- `SOUNDEX(str)`: Returns the soundex of a string. -- `JSON_EXTRACT(json_doc, path, ...)`: Extracts data from a json document using json paths. -- `LN(X)`: Return the natural logarithm of X. -- `LOG2(X)`: Returns the base-2 logarithm of X. -- `LOG10(X)`: Returns the base-10 logarithm of X. -- `LOG(X), LOG(B, X)`: If called with one parameter, this function returns the natural logarithm of X. If called with two parameters, this function returns the logarithm of X to the base B. If X is less than or equal to 0, or if B is less than or equal to 1, then NULL is returned. -- `RPAD(str, len, padstr)`: Returns the string str, right-padded with the string padstr to a length of len characters. -- `LPAD(str, len, padstr)`: Return the string argument, left-padded with the specified string. -- `SQRT(X)`: Returns the square root of a nonnegative number X. -- `POW(X, Y)`, `POWER(X, Y)`: Returns the value of X raised to the power of Y. -- `TRIM(str)`: Returns the string str with all spaces removed. -- `LTRIM(str)`: Returns the string str with leading space characters removed. -- `RTRIM(str)`: Returns the string str with trailing space characters removed. -- `REVERSE(str)`: Returns the string str with the order of the characters reversed. -- `REPEAT(str, count)`: Returns a string consisting of the string str repeated count times. -- `REPLACE(str,from_str,to_str)`: Returns the string str with all occurrences of the string from_str replaced by the string to_str. -- `IFNULL(expr1, expr2)`: If expr1 is not NULL, IFNULL() returns expr1; otherwise it returns expr2. -- `NULLIF(expr1, expr2)`: Returns NULL if expr1 = expr2 is true, otherwise returns expr1. +## Available functions + + +| Name | Description | +|:-------------|:-------------------------------------------------------------------------------------------------------------------------------| +|`ARRAY_LENGTH(json)`|if the json representation is an array, this function returns its size.| +|`AVG(expr)`| returns the average value of expr in all rows.| +|`CEIL(number)`| returns the smallest integer value that is greater than or equal to `number`.| +|`CEILING(number)`| returns the smallest integer value that is greater than or equal to `number`.| +|`CHAR_LENGTH(str)`| returns the length of the string in characters.| +|`COALESCE(...)`| returns the first non-null value in a list.| +|`CONCAT(...)`| concatenates any group of fields into a single string.| +|`CONCAT_WS(sep, ...)`| concatenates any group of fields into a single string. The first argument is the separator for the rest of the arguments. The separator is added between the strings to be concatenated. The separator can be a string, as can the rest of the arguments. If the separator is NULL, the result is NULL.| +|`CONNECTION_ID()`| returns the current connection ID.| +|`COUNT(expr)`| returns a count of the number of non-NULL values of expr in the rows retrieved by a SELECT statement.| +|`DATE_ADD(date, interval)`| adds the interval to the given `date`.| +|`DATE_SUB(date, interval)`| subtracts the interval from the given `date`.| +|`DAY(date)`| is a synonym for DAYOFMONTH().| +|`DATE(date)`| returns the date part of the given `date`.| +|`DAYOFMONTH(date)`| returns the day of the month (0-31).| +|`DAYOFWEEK(date)`| returns the day of the week of the given `date`.| +|`DAYOFYEAR(date)`| returns the day of the year of the given `date`.| +|`FIRST(expr)`| returns the first value in a sequence of elements of an aggregation.| +|`FLOOR(number)`| returns the largest integer value that is less than or equal to `number`.| +|`FROM_BASE64(str)`| decodes the base64-encoded string `str`.| +|`GREATEST(...)`| returns the greatest numeric or string value.| +|`HOUR(date)`| returns the hours of the given `date`.| +|`IFNULL(expr1, expr2)`| if `expr1` is not NULL, it returns `expr1`; otherwise it returns `expr2`.| +|`IS_BINARY(blob)`| returns whether a `blob` is a binary file or not.| +|`JSON_EXTRACT(json_doc, path, ...)`| extracts data from a json document using json paths. Extracting a string will result in that string being quoted. To avoid this, use `JSON_UNQUOTE(JSON_EXTRACT(json_doc, path, ...))`.| +|`JSON_UNQUOTE(json)`| unquotes JSON value and returns the result as a utf8mb4 string.| +|`LAST(expr)`| returns the last value in a sequence of elements of an aggregation.| +|`LEAST(...)`| returns the smaller numeric or string value.| +|`LENGTH(str)`| returns the length of the string in bytes.| +|`LN(X)`| returns the natural logarithm of `X`.| +|`LOG(X), LOG(B, X)`| if called with one parameter, this function returns the natural logarithm of `X`. If called with two parameters, this function returns the logarithm of `X` to the base `B`. If `X` is less than or equal to 0, or if `B` is less than or equal to 1, then NULL is returned.| +|`LOG10(X)`| returns the base-10 logarithm of `X`.| +|`LOG2(X)`| returns the base-2 logarithm of `X`.| +|`LOWER(str)`| returns the string `str` with all characters in lower case.| +|`LPAD(str, len, padstr)`| returns the string `str`, left-padded with the string `padstr` to a length of `len` characters.| +|`LTRIM(str)`| returns the string `str` with leading space characters removed.| +|`MAX(expr)`| returns the maximum value of `expr` in all rows.| +|`MID(str, pos, [len])`| returns a substring from the provided string starting at `pos` with a length of `len` characters. If no `len` is provided, all characters from `pos` until the end will be taken.| +|`MIN(expr)`| returns the minimum value of `expr` in all rows.| +|`MINUTE(date)`| returns the minutes of the given `date`.| +|`MONTH(date)`| returns the month of the given `date`.| +|`NOW()`| returns the current timestamp.| +|`NULLIF(expr1, expr2)`| returns NULL if `expr1 = expr2` is true, otherwise returns `expr1`.| +|`POW(X, Y)`| returns the value of `X` raised to the power of `Y`.| +|`REGEXP_MATCHES(text, pattern, [flags])`| returns an array with the matches of the `pattern` in the given `text`. Flags can be given to control certain behaviours of the regular expression. Currently, only the `i` flag is supported, to make the comparison case insensitive.| +|`REPEAT(str, count)`| returns a string consisting of the string `str` repeated `count` times.| +|`REPLACE(str,from_str,to_str)`| returns the string `str` with all occurrences of the string `from_str` replaced by the string `to_str`.| +|`REVERSE(str)`| returns the string `str` with the order of the characters reversed.| +|`ROUND(number, decimals)`| rounds the `number` to `decimals` decimal places.| +|`RPAD(str, len, padstr)`| returns the string `str`, right-padded with the string `padstr` to a length of `len` characters.| +|`RTRIM(str)`| returns the string `str` with trailing space characters removed.| +|`SECOND(date)`| returns the seconds of the given `date`.| +|`SLEEP(seconds)`| waits for the specified number of seconds (can be fractional).| +|`SOUNDEX(str)`| returns the soundex of a string.| +|`SPLIT(str,sep)`| returns the parts of the string `str` split by the separator `sep` as a JSON array of strings.| +|`SQRT(X)`| returns the square root of a nonnegative number `X`.| +|`SUBSTR(str, pos, [len])`| returns a substring from the string `str` starting at `pos` with a length of `len` characters. If no `len` is provided, all characters from `pos` until the end will be taken.| +|`SUBSTRING(str, pos, [len])`| returns a substring from the string `str` starting at `pos` with a length of `len` characters. If no `len` is provided, all characters from `pos` until the end will be taken.| +|`SUM(expr)`| returns the sum of `expr` in all rows.| +|`TO_BASE64(str)`| encodes the string `str` in base64 format.| +|`TRIM(str)`| returns the string `str` with all spaces removed.| +|`UPPER(str)`| returns the string `str` with all characters in upper case.| +|`WEEKDAY(date)`| returns the weekday of the given `date`.| +|`YEAR(date)`| returns the year of the given `date`.| +|`YEARWEEK(date, mode)`| returns year and week for a date. The year in the result may be different from the year in the date argument for the first and the last week of the year.| + + +## Configuration + +The behaviour of certain parts of go-mysql-server can be configured using either environment variables or session variables. + +Session variables are set using the following SQL queries: + +```sql +SET = +``` + + +| Name | Type | Description | +|:-----|:-----|:------------| +|`INMEMORY_JOINS`|environment|If set it will perform all joins in memory. Default is off.| +|`inmemory_joins`|session|If set it will perform all joins in memory. Default is off. This has precedence over `INMEMORY_JOINS`.| +|`MAX_MEMORY`|environment|The maximum number of memory, in megabytes, that can be consumed by go-mysql-server. Any in-memory caches or computations will no longer try to use memory when the limit is reached. Note that this may cause certain queries to fail if there is not enough memory available, such as queries using DISTINCT, ORDER BY or GROUP BY with groupings.| +|`DEBUG_ANALYZER`|environment|If set, the analyzer will print debug messages. Default is off.| +|`PILOSA_INDEX_THREADS`|environment|Number of threads used in index creation. Default is the number of cores available in the machine.| +|`pilosa_index_threads`|environment|Number of threads used in index creation. Default is the number of cores available in the machine. This has precedence over `PILOSA_INDEX_THREADS`.| + ## Example @@ -109,15 +163,10 @@ func main() { driver := sqle.NewDefault() driver.AddDatabase(createTestDatabase()) - auth := mysql.NewAuthServerStatic() - auth.Entries["user"] = []*mysql.AuthServerStaticEntry{{ - Password: "pass", - }} - config := server.Config{ Protocol: "tcp", Address: "localhost:3306", - Auth: auth, + Auth: auth.NewNativeSingle("user", "pass", auth.AllPermissions), } s, err := server.NewDefaultServer(config, driver) @@ -128,14 +177,14 @@ func main() { s.Start() } -func createTestDatabase() *mem.Database { +func createTestDatabase() *memory.Database { const ( dbName = "test" tableName = "mytable" ) - db := mem.NewDatabase(dbName) - table := mem.NewTable(tableName, sql.Schema{ + db := memory.NewDatabase(dbName) + table := memory.NewTable(tableName, sql.Schema{ {Name: "name", Type: sql.Text, Nullable: false, Source: tableName}, {Name: "email", Type: sql.Text, Nullable: false, Source: tableName}, {Name: "phone_numbers", Type: sql.JSON, Nullable: false, Source: tableName}, @@ -259,11 +308,137 @@ CREATE INDEX foo ON table USING driverid (col1, col2) WITH (async = true) ### Old `pilosalib` driver -`pilosalib` driver was renamed to `pilosa` and now `pilosa` does not require an external pilosa server. +`pilosalib` driver was renamed to `pilosa` and now `pilosa` does not require an external pilosa server. `pilosa` is not supported on Windows. + +### Metrics + +`go-mysql-server` utilizes `github.com/go-kit/kit/metrics` module to expose metrics (counters, gauges, histograms) for certain packages (so far for `engine`, `analyzer`, `regex`, `pilosa`). If you already have metrics server (prometheus, statsd/statsite, influxdb, etc.) and you want to gather metrics also from `go-mysql-server` components, you will need to initialize some global variables by particular implementations to satisfy following interfaces: +```go +// Counter describes a metric that accumulates values monotonically. +type Counter interface { + With(labelValues ...string) Counter + Add(delta float64) +} + +// Gauge describes a metric that takes specific values over time. +type Gauge interface { + With(labelValues ...string) Gauge + Set(value float64) + Add(delta float64) +} + +// Histogram describes a metric that takes repeated observations of the same +// kind of thing, and produces a statistical summary of those observations, +// typically expressed as quantiles or buckets. +type Histogram interface { + With(labelValues ...string) Histogram + Observe(value float64) +} +``` + +You can use one of `go-kit` implementations or try your own. +For instance, we want to expose metrics for _prometheus_ server. So, before we start _mysql engine_, we have to set up the following variables: +```go + +import( + "github.com/go-kit/kit/metrics/prometheus" + promopts "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +//.... + +// engine metrics +sqle.QueryCounter = prometheus.NewCounterFrom(promopts.CounterOpts{ + Namespace: "go_mysql_server", + Subsystem: "engine", + Name: "query_counter", + }, []string{ + "query", + }) +sqle.QueryErrorCounter = prometheus.NewCounterFrom(promopts.CounterOpts{ + Namespace: "go_mysql_server", + Subsystem: "engine", + Name: "query_error_counter", +}, []string{ + "query", + "error", +}) +sqle.QueryHistogram = prometheus.NewHistogramFrom(promopts.HistogramOpts{ + Namespace: "go_mysql_server", + Subsystem: "engine", + Name: "query_histogram", +}, []string{ + "query", + "duration", +}) + +// analyzer metrics +analyzer.ParallelQueryCounter = prometheus.NewCounterFrom(promopts.CounterOpts{ + Namespace: "go_mysql_server", + Subsystem: "analyzer", + Name: "parallel_query_counter", +}, []string{ + "parallelism", +}) + +// regex metrics +regex.CompileHistogram = prometheus.NewHistogramFrom(promopts.HistogramOpts{ + Namespace: "go_mysql_server", + Subsystem: "regex", + Name: "compile_histogram", +}, []string{ + "regex", + "duration", +}) +regex.MatchHistogram = prometheus.NewHistogramFrom(promopts.HistogramOpts{ + Namespace: "go_mysql_server", + Subsystem: "regex", + Name: "match_histogram", +}, []string{ + "string", + "duration", +}) + +// pilosa index driver metrics +pilosa.RowsGauge = prometheus.NewGaugeFrom(promopts.GaugeOpts{ + Namespace: "go_mysql_server", + Subsystem: "index", + Name: "indexed_rows_gauge", +}, []string{ + "driver", +}) +pilosa.TotalHistogram = prometheus.NewHistogramFrom(promopts.HistogramOpts{ + Namespace: "go_mysql_server", + Subsystem: "index", + Name: "index_created_total_histogram", +}, []string{ + "driver", + "duration", +}) +pilosa.MappingHistogram = prometheus.NewHistogramFrom(promopts.HistogramOpts{ + Namespace: "go_mysql_server", + Subsystem: "index", + Name: "index_created_mapping_histogram", +}, []string{ + "driver", + "duration", +}) +pilosa.BitmapHistogram = prometheus.NewHistogramFrom(promopts.HistogramOpts{ + Namespace: "go_mysql_server", + Subsystem: "index", + Name: "index_created_bitmap_histogram", +}, []string{ + "driver", + "duration", +}) +``` +One _important note_ - internally we set some _labels_ for metrics, that's why have to pass those keys like "duration", "query", "driver", ... when we register metrics in `prometheus`. Other systems may have different requirements. ## Powered by go-mysql-server * [gitbase](https://github.com/src-d/gitbase) +* [dolt](https://github.com/liquidata-inc/dolt) ## License diff --git a/SUPPORTED.md b/SUPPORTED.md index 84d33a0e0..87841ac72 100644 --- a/SUPPORTED.md +++ b/SUPPORTED.md @@ -18,7 +18,7 @@ ## Grouping expressions - AVG -- COUNT +- COUNT and COUNT(DISTINCT) - MAX - MIN - SUM (always returns DOUBLE) @@ -27,7 +27,6 @@ - ALIAS (AS) - CAST/CONVERT - CREATE TABLE -- DESCRIBE/DESC/EXPLAIN [table name] - DESCRIBE/DESC/EXPLAIN FORMAT=TREE [query] - DISTINCT - FILTER (WHERE) @@ -50,6 +49,7 @@ - USE - SHOW DATABASES - SHOW WARNINGS +- INTERVALS ## Index expressions - CREATE INDEX (an index can be created using either column names or a single arbitrary expression). @@ -67,8 +67,8 @@ - OR ## Arithmetic expressions -- \+ -- \- +- \+ (including between dates and intervals) +- \- (including between dates and intervals) - \* - \\ - << @@ -79,45 +79,56 @@ - div - % -## Subqueries -- supported only as tables, not as expressions. - ## Functions - ARRAY_LENGTH +- CEIL +- CEILING +- COALESCE - CONCAT - CONCAT_WS +- CONNECTION_ID +- DATABASE +- FLOOR +- FROM_BASE64 +- GREATEST - IS_BINARY -- SPLIT -- SUBSTRING - IS_BINARY -- LOWER -- UPPER -- CEILING -- CEIL -- FLOOR -- ROUND -- COALESCE -- CONNECTION_ID -- SOUNDEX - JSON_EXTRACT -- DATABASE -- SQRT +- JSON_UNQUOTE +- LEAST +- LN +- LOG10 +- LOG2 +- LOWER +- LPAD - POW - POWER +- ROUND - RPAD -- LPAD -- LN -- LOG2 -- LOG10 +- SLEEP +- SOUNDEX +- SPLIT +- SQRT +- SUBSTRING +- TO_BASE64 +- UPPER ## Time functions +- DATE +- DATE_ADD +- DATE_SUB - DAY -- WEEKDAY +- DAYOFMONTH - DAYOFWEEK - DAYOFYEAR - HOUR - MINUTE - MONTH +- NOW - SECOND +- WEEKDAY - YEAR -- NOW +- YEARWEEK + +## Subqueries +Supported both as a table and as expressions but they can't access the parent query scope. diff --git a/SUPPORTED_CLIENTS.md b/SUPPORTED_CLIENTS.md index e30e928e9..f0f4f8adb 100644 --- a/SUPPORTED_CLIENTS.md +++ b/SUPPORTED_CLIENTS.md @@ -1,6 +1,6 @@ # Supported clients -These are the clients we actively test against to check are compatible with go-mysql-server. Other clients may also work, but we don't check on every build if we remain compatible with them. +These are the clients we actively test against to check that they are compatible with go-mysql-server. Other clients may also work, but we don't check on every build if we remain compatible with them. - Python - [pymysql](#pymysql) @@ -16,7 +16,7 @@ These are the clients we actively test against to check are compatible with go-m - Java/JVM - [mariadb-java-client](#mariadb-java-client) - Go - - [go-mysql-driver/mysql](#go-mysql-driver-mysql) + - [go-mysql-driver/mysql](#go-sql-drivermysql) - C - [mysql-connector-c](#mysql-connector-c) - Grafana @@ -218,7 +218,7 @@ func main() { } ``` -### #mysql-connector-c +### mysql-connector-c ```c #include @@ -271,4 +271,4 @@ int main(int argc, char **argv) return 0; } -``` \ No newline at end of file +``` diff --git a/_example/main.go b/_example/main.go index 7f245f3cf..55a007247 100644 --- a/_example/main.go +++ b/_example/main.go @@ -3,11 +3,11 @@ package main import ( "time" - "gopkg.in/src-d/go-mysql-server.v0" - "gopkg.in/src-d/go-mysql-server.v0/auth" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/server" - "gopkg.in/src-d/go-mysql-server.v0/sql" + sqle "github.com/src-d/go-mysql-server" + "github.com/src-d/go-mysql-server/auth" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/server" + "github.com/src-d/go-mysql-server/sql" ) // Example of how to implement a MySQL server based on a Engine: @@ -42,14 +42,14 @@ func main() { s.Start() } -func createTestDatabase() *mem.Database { +func createTestDatabase() *memory.Database { const ( dbName = "mydb" tableName = "mytable" ) - db := mem.NewDatabase(dbName) - table := mem.NewTable(tableName, sql.Schema{ + db := memory.NewDatabase(dbName) + table := memory.NewTable(tableName, sql.Schema{ {Name: "name", Type: sql.Text, Nullable: false, Source: tableName}, {Name: "email", Type: sql.Text, Nullable: false, Source: tableName}, {Name: "phone_numbers", Type: sql.JSON, Nullable: false, Source: tableName}, diff --git a/_integration/go/go.mod b/_integration/go/go.mod index 197918374..01e7159d1 100644 --- a/_integration/go/go.mod +++ b/_integration/go/go.mod @@ -1,4 +1,4 @@ -module gopkg.in/src-d/go-mysql-server.v0/integration/go +module github.com/src-d/go-mysql-server/integration/go require ( github.com/go-sql-driver/mysql v1.4.0 diff --git a/_integration/go/mysql_test.go b/_integration/go/mysql_test.go index 73ec4af71..926477b06 100644 --- a/_integration/go/mysql_test.go +++ b/_integration/go/mysql_test.go @@ -74,7 +74,7 @@ func TestGrafana(t *testing.T) { }, { `select @@version_comment limit 1`, - [][]string{{"NULL"}}, + [][]string{{""}}, }, { `describe table mytable`, diff --git a/_integration/javascript/package-lock.json b/_integration/javascript/package-lock.json index f5586593f..b37fc5abe 100644 --- a/_integration/javascript/package-lock.json +++ b/_integration/javascript/package-lock.json @@ -5,87 +5,74 @@ "requires": true, "dependencies": { "@ava/babel-plugin-throws-helper": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@ava/babel-plugin-throws-helper/-/babel-plugin-throws-helper-3.0.0.tgz", - "integrity": "sha512-mN9UolOs4WX09QkheU1ELkVy2WPnwonlO3XMdN8JF8fQqRVgVTR21xDbvEOUsbwz6Zwjq7ji9yzyjuXqDPalxg==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@ava/babel-plugin-throws-helper/-/babel-plugin-throws-helper-4.0.0.tgz", + "integrity": "sha512-3diBLIVBPPh3j4+hb5lo0I1D+S/O/VDJPI4Y502apBxmwEqjyXG4gTSPFUlm41sSZeZzMarT/Gzovw9kV7An0w==", "dev": true }, "@ava/babel-preset-stage-4": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@ava/babel-preset-stage-4/-/babel-preset-stage-4-2.0.0.tgz", - "integrity": "sha512-OWqMYeTSZ16AfLx0Vn0Uj7tcu+uMRlbKmks+DVCFlln7vomVsOtst+Oz+HCussDSFGpE+30VtHAUHLy6pLDpHQ==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@ava/babel-preset-stage-4/-/babel-preset-stage-4-4.0.0.tgz", + "integrity": "sha512-lZEV1ZANzfzSYBU6WHSErsy7jLPbD1iIgAboASPMcKo7woVni5/5IKWeT0RxC8rY802MFktur3OKEw2JY1Tv2w==", "dev": true, "requires": { - "@babel/plugin-proposal-async-generator-functions": "^7.0.0", - "@babel/plugin-proposal-object-rest-spread": "^7.0.0", - "@babel/plugin-proposal-optional-catch-binding": "^7.0.0", - "@babel/plugin-transform-async-to-generator": "^7.0.0", - "@babel/plugin-transform-dotall-regex": "^7.0.0", - "@babel/plugin-transform-exponentiation-operator": "^7.0.0", - "@babel/plugin-transform-modules-commonjs": "^7.0.0" + "@babel/plugin-proposal-async-generator-functions": "^7.2.0", + "@babel/plugin-proposal-dynamic-import": "^7.5.0", + "@babel/plugin-proposal-optional-catch-binding": "^7.2.0", + "@babel/plugin-transform-dotall-regex": "^7.4.4", + "@babel/plugin-transform-modules-commonjs": "^7.5.0" } }, "@ava/babel-preset-transform-test-files": { - "version": "4.0.0-beta.9", - "resolved": "https://registry.npmjs.org/@ava/babel-preset-transform-test-files/-/babel-preset-transform-test-files-4.0.0-beta.9.tgz", - "integrity": "sha512-22KVcr0Xr5q5TXlb5WVuEB5krG69XKOUHSrWkkq26O400ZYAlnh7FKB1Xf5hAiCQHX228gRp6yyYKG+OqeNgaw==", - "dev": true, - "requires": { - "@ava/babel-plugin-throws-helper": "^3.0.0", - "babel-plugin-espower": "3.0.0-beta.2" - } - }, - "@ava/write-file-atomic": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@ava/write-file-atomic/-/write-file-atomic-2.2.0.tgz", - "integrity": "sha512-BTNB3nGbEfJT+69wuqXFr/bQH7Vr7ihx2xGOMNqPgDGhwspoZhiWumDDZNjBy7AScmqS5CELIOGtPVXESyrnDA==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@ava/babel-preset-transform-test-files/-/babel-preset-transform-test-files-6.0.0.tgz", + "integrity": "sha512-8eKhFzZp7Qcq1VLfoC75ggGT8nQs9q8fIxltU47yCB7Wi7Y8Qf6oqY1Bm0z04fIec24vEgr0ENhDHEOUGVDqnA==", "dev": true, "requires": { - "graceful-fs": "^4.1.11", - "imurmurhash": "^0.1.4", - "slide": "^1.1.5" + "@ava/babel-plugin-throws-helper": "^4.0.0", + "babel-plugin-espower": "^3.0.1" } }, "@babel/code-frame": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.0.0.tgz", - "integrity": "sha512-OfC2uemaknXr87bdLUkWog7nYuliM9Ij5HUcajsVcMCpQrcLmtxRbVFTIqmcSkSeYRBFBRxs2FiUqFJDLdiebA==", + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.5.5.tgz", + "integrity": "sha512-27d4lZoomVyo51VegxI20xZPuSHusqbQag/ztrBC7wegWoQ1nLREPVSKSW8byhTlzTKyNE4ifaTA6lCp7JjpFw==", "dev": true, "requires": { "@babel/highlight": "^7.0.0" } }, "@babel/core": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.1.2.tgz", - "integrity": "sha512-IFeSSnjXdhDaoysIlev//UzHZbdEmm7D0EIH2qtse9xK7mXEZQpYjs2P00XlP1qYsYvid79p+Zgg6tz1mp6iVw==", - "dev": true, - "requires": { - "@babel/code-frame": "^7.0.0", - "@babel/generator": "^7.1.2", - "@babel/helpers": "^7.1.2", - "@babel/parser": "^7.1.2", - "@babel/template": "^7.1.2", - "@babel/traverse": "^7.1.0", - "@babel/types": "^7.1.2", + "version": "7.6.0", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.6.0.tgz", + "integrity": "sha512-FuRhDRtsd6IptKpHXAa+4WPZYY2ZzgowkbLBecEDDSje1X/apG7jQM33or3NdOmjXBKWGOg4JmSiRfUfuTtHXw==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.5.5", + "@babel/generator": "^7.6.0", + "@babel/helpers": "^7.6.0", + "@babel/parser": "^7.6.0", + "@babel/template": "^7.6.0", + "@babel/traverse": "^7.6.0", + "@babel/types": "^7.6.0", "convert-source-map": "^1.1.0", - "debug": "^3.1.0", - "json5": "^0.5.0", - "lodash": "^4.17.10", + "debug": "^4.1.0", + "json5": "^2.1.0", + "lodash": "^4.17.13", "resolve": "^1.3.2", "semver": "^5.4.1", "source-map": "^0.5.0" } }, "@babel/generator": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.1.2.tgz", - "integrity": "sha512-70A9HWLS/1RHk3Ck8tNHKxOoKQuSKocYgwDN85Pyl/RBduss6AKxUR7RIZ/lzduQMSYfWEM4DDBu6A+XGbkFig==", + "version": "7.6.0", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.6.0.tgz", + "integrity": "sha512-Ms8Mo7YBdMMn1BYuNtKuP/z0TgEIhbcyB8HVR6PPNYp4P61lMsABiS4A3VG1qznjXVCf3r+fVHhm4efTYVsySA==", "dev": true, "requires": { - "@babel/types": "^7.1.2", + "@babel/types": "^7.6.0", "jsesc": "^2.5.1", - "lodash": "^4.17.10", + "lodash": "^4.17.13", "source-map": "^0.5.0", "trim-right": "^1.0.1" } @@ -99,26 +86,6 @@ "@babel/types": "^7.0.0" } }, - "@babel/helper-builder-binary-assignment-operator-visitor": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/@babel/helper-builder-binary-assignment-operator-visitor/-/helper-builder-binary-assignment-operator-visitor-7.1.0.tgz", - "integrity": "sha512-qNSR4jrmJ8M1VMM9tibvyRAHXQs2PmaksQF7c1CGJNipfe3D8p+wgNwgso/P2A2r2mdgBWAXljNWR0QRZAMW8w==", - "dev": true, - "requires": { - "@babel/helper-explode-assignable-expression": "^7.1.0", - "@babel/types": "^7.0.0" - } - }, - "@babel/helper-explode-assignable-expression": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.1.0.tgz", - "integrity": "sha512-NRQpfHrJ1msCHtKjbzs9YcMmJZOg6mQMmGRB+hbamEdG5PNpaSm95275VD92DvJKuyl0s2sFiDmMZ+EnnvufqA==", - "dev": true, - "requires": { - "@babel/traverse": "^7.1.0", - "@babel/types": "^7.0.0" - } - }, "@babel/helper-function-name": { "version": "7.1.0", "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.1.0.tgz", @@ -149,17 +116,17 @@ } }, "@babel/helper-module-transforms": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.1.0.tgz", - "integrity": "sha512-0JZRd2yhawo79Rcm4w0LwSMILFmFXjugG3yqf+P/UsKsRS1mJCmMwwlHDlMg7Avr9LrvSpp4ZSULO9r8jpCzcw==", + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.5.5.tgz", + "integrity": "sha512-jBeCvETKuJqeiaCdyaheF40aXnnU1+wkSiUs/IQg3tB85up1LyL8x77ClY8qJpuRJUcXQo+ZtdNESmZl4j56Pw==", "dev": true, "requires": { "@babel/helper-module-imports": "^7.0.0", "@babel/helper-simple-access": "^7.1.0", - "@babel/helper-split-export-declaration": "^7.0.0", - "@babel/template": "^7.1.0", - "@babel/types": "^7.0.0", - "lodash": "^4.17.10" + "@babel/helper-split-export-declaration": "^7.4.4", + "@babel/template": "^7.4.4", + "@babel/types": "^7.5.5", + "lodash": "^4.17.13" } }, "@babel/helper-plugin-utils": { @@ -169,12 +136,12 @@ "dev": true }, "@babel/helper-regex": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/@babel/helper-regex/-/helper-regex-7.0.0.tgz", - "integrity": "sha512-TR0/N0NDCcUIUEbqV6dCO+LptmmSQFQ7q70lfcEB4URsjD0E1HzicrwUH+ap6BAQ2jhCX9Q4UqZy4wilujWlkg==", + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/@babel/helper-regex/-/helper-regex-7.5.5.tgz", + "integrity": "sha512-CkCYQLkfkiugbRDO8eZn6lRuR8kzZoGXCg3149iTk5se7g6qykSpy3+hELSwquhu+TgHn8nkLiBwHvNX8Hofcw==", "dev": true, "requires": { - "lodash": "^4.17.10" + "lodash": "^4.17.13" } }, "@babel/helper-remap-async-to-generator": { @@ -201,41 +168,41 @@ } }, "@babel/helper-split-export-declaration": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.0.0.tgz", - "integrity": "sha512-MXkOJqva62dfC0w85mEf/LucPPS/1+04nmmRMPEBUB++hiiThQ2zPtX/mEWQ3mtzCEjIJvPY8nuwxXtQeQwUag==", + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.4.4.tgz", + "integrity": "sha512-Ro/XkzLf3JFITkW6b+hNxzZ1n5OQ80NvIUdmHspih1XAhtN3vPTuUFT4eQnela+2MaZ5ulH+iyP513KJrxbN7Q==", "dev": true, "requires": { - "@babel/types": "^7.0.0" + "@babel/types": "^7.4.4" } }, "@babel/helper-wrap-function": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/@babel/helper-wrap-function/-/helper-wrap-function-7.1.0.tgz", - "integrity": "sha512-R6HU3dete+rwsdAfrOzTlE9Mcpk4RjU3aX3gi9grtmugQY0u79X7eogUvfXA5sI81Mfq1cn6AgxihfN33STjJA==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@babel/helper-wrap-function/-/helper-wrap-function-7.2.0.tgz", + "integrity": "sha512-o9fP1BZLLSrYlxYEYyl2aS+Flun5gtjTIG8iln+XuEzQTs0PLagAGSXUcqruJwD5fM48jzIEggCKpIfWTcR7pQ==", "dev": true, "requires": { "@babel/helper-function-name": "^7.1.0", "@babel/template": "^7.1.0", "@babel/traverse": "^7.1.0", - "@babel/types": "^7.0.0" + "@babel/types": "^7.2.0" } }, "@babel/helpers": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.1.2.tgz", - "integrity": "sha512-Myc3pUE8eswD73aWcartxB16K6CGmHDv9KxOmD2CeOs/FaEAQodr3VYGmlvOmog60vNQ2w8QbatuahepZwrHiA==", + "version": "7.6.0", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.6.0.tgz", + "integrity": "sha512-W9kao7OBleOjfXtFGgArGRX6eCP0UEcA2ZWEWNkJdRZnHhW4eEbeswbG3EwaRsnQUAEGWYgMq1HsIXuNNNy2eQ==", "dev": true, "requires": { - "@babel/template": "^7.1.2", - "@babel/traverse": "^7.1.0", - "@babel/types": "^7.1.2" + "@babel/template": "^7.6.0", + "@babel/traverse": "^7.6.0", + "@babel/types": "^7.6.0" } }, "@babel/highlight": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.0.0.tgz", - "integrity": "sha512-UFMC4ZeFC48Tpvj7C8UgLvtkaUuovQX+5xNWrsIoMG8o2z+XFKjKaN9iVmS84dPwVN00W4wPmqvYoZF3EGAsfw==", + "version": "7.5.0", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.5.0.tgz", + "integrity": "sha512-7dV4eu9gBxoM0dAnj/BCFDW9LFU0zvTrkq0ugM7pnHEgguOEeOz1so2ZghEdzviYzQEED0r4EAgpsBChKy1TRQ==", "dev": true, "requires": { "chalk": "^2.0.0", @@ -244,198 +211,295 @@ } }, "@babel/parser": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.1.2.tgz", - "integrity": "sha512-x5HFsW+E/nQalGMw7hu+fvPqnBeBaIr0lWJ2SG0PPL2j+Pm9lYvCrsZJGIgauPIENx0v10INIyFjmSNUD/gSqQ==", + "version": "7.6.0", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.6.0.tgz", + "integrity": "sha512-+o2q111WEx4srBs7L9eJmcwi655eD8sXniLqMB93TBK9GrNzGrxDWSjiqz2hLU0Ha8MTXFIP0yd9fNdP+m43ZQ==", "dev": true }, "@babel/plugin-proposal-async-generator-functions": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.1.0.tgz", - "integrity": "sha512-Fq803F3Jcxo20MXUSDdmZZXrPe6BWyGcWBPPNB/M7WaUYESKDeKMOGIxEzQOjGSmW/NWb6UaPZrtTB2ekhB/ew==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.2.0.tgz", + "integrity": "sha512-+Dfo/SCQqrwx48ptLVGLdE39YtWRuKc/Y9I5Fy0P1DDBB9lsAHpjcEJQt+4IifuSOSTLBKJObJqMvaO1pIE8LQ==", "dev": true, "requires": { "@babel/helper-plugin-utils": "^7.0.0", "@babel/helper-remap-async-to-generator": "^7.1.0", - "@babel/plugin-syntax-async-generators": "^7.0.0" + "@babel/plugin-syntax-async-generators": "^7.2.0" } }, - "@babel/plugin-proposal-object-rest-spread": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.0.0.tgz", - "integrity": "sha512-14fhfoPcNu7itSen7Py1iGN0gEm87hX/B+8nZPqkdmANyyYWYMY2pjA3r8WXbWVKMzfnSNS0xY8GVS0IjXi/iw==", + "@babel/plugin-proposal-dynamic-import": { + "version": "7.5.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-dynamic-import/-/plugin-proposal-dynamic-import-7.5.0.tgz", + "integrity": "sha512-x/iMjggsKTFHYC6g11PL7Qy58IK8H5zqfm9e6hu4z1iH2IRyAp9u9dL80zA6R76yFovETFLKz2VJIC2iIPBuFw==", "dev": true, "requires": { "@babel/helper-plugin-utils": "^7.0.0", - "@babel/plugin-syntax-object-rest-spread": "^7.0.0" + "@babel/plugin-syntax-dynamic-import": "^7.2.0" } }, "@babel/plugin-proposal-optional-catch-binding": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-catch-binding/-/plugin-proposal-optional-catch-binding-7.0.0.tgz", - "integrity": "sha512-JPqAvLG1s13B/AuoBjdBYvn38RqW6n1TzrQO839/sIpqLpbnXKacsAgpZHzLD83Sm8SDXMkkrAvEnJ25+0yIpw==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-catch-binding/-/plugin-proposal-optional-catch-binding-7.2.0.tgz", + "integrity": "sha512-mgYj3jCcxug6KUcX4OBoOJz3CMrwRfQELPQ5560F70YQUBZB7uac9fqaWamKR1iWUzGiK2t0ygzjTScZnVz75g==", "dev": true, "requires": { "@babel/helper-plugin-utils": "^7.0.0", - "@babel/plugin-syntax-optional-catch-binding": "^7.0.0" + "@babel/plugin-syntax-optional-catch-binding": "^7.2.0" } }, "@babel/plugin-syntax-async-generators": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.0.0.tgz", - "integrity": "sha512-im7ged00ddGKAjcZgewXmp1vxSZQQywuQXe2B1A7kajjZmDeY/ekMPmWr9zJgveSaQH0k7BcGrojQhcK06l0zA==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.2.0.tgz", + "integrity": "sha512-1ZrIRBv2t0GSlcwVoQ6VgSLpLgiN/FVQUzt9znxo7v2Ov4jJrs8RY8tv0wvDmFN3qIdMKWrmMMW6yZ0G19MfGg==", "dev": true, "requires": { "@babel/helper-plugin-utils": "^7.0.0" } }, - "@babel/plugin-syntax-object-rest-spread": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.0.0.tgz", - "integrity": "sha512-5A0n4p6bIiVe5OvQPxBnesezsgFJdHhSs3uFSvaPdMqtsovajLZ+G2vZyvNe10EzJBWWo3AcHGKhAFUxqwp2dw==", + "@babel/plugin-syntax-dynamic-import": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-dynamic-import/-/plugin-syntax-dynamic-import-7.2.0.tgz", + "integrity": "sha512-mVxuJ0YroI/h/tbFTPGZR8cv6ai+STMKNBq0f8hFxsxWjl94qqhsb+wXbpNMDPU3cfR1TIsVFzU3nXyZMqyK4w==", "dev": true, "requires": { "@babel/helper-plugin-utils": "^7.0.0" } }, "@babel/plugin-syntax-optional-catch-binding": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.0.0.tgz", - "integrity": "sha512-Wc+HVvwjcq5qBg1w5RG9o9RVzmCaAg/Vp0erHCKpAYV8La6I94o4GQAmFYNmkzoMO6gzoOSulpKeSSz6mPEoZw==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.2.0.tgz", + "integrity": "sha512-bDe4xKNhb0LI7IvZHiA13kff0KEfaGX/Hv4lMA9+7TEc63hMNvfKo6ZFpXhKuEp+II/q35Gc4NoMeDZyaUbj9w==", "dev": true, "requires": { "@babel/helper-plugin-utils": "^7.0.0" } }, - "@babel/plugin-transform-async-to-generator": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-async-to-generator/-/plugin-transform-async-to-generator-7.1.0.tgz", - "integrity": "sha512-rNmcmoQ78IrvNCIt/R9U+cixUHeYAzgusTFgIAv+wQb9HJU4szhpDD6e5GCACmj/JP5KxuCwM96bX3L9v4ZN/g==", - "dev": true, - "requires": { - "@babel/helper-module-imports": "^7.0.0", - "@babel/helper-plugin-utils": "^7.0.0", - "@babel/helper-remap-async-to-generator": "^7.1.0" - } - }, "@babel/plugin-transform-dotall-regex": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.0.0.tgz", - "integrity": "sha512-00THs8eJxOJUFVx1w8i1MBF4XH4PsAjKjQ1eqN/uCH3YKwP21GCKfrn6YZFZswbOk9+0cw1zGQPHVc1KBlSxig==", + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.4.4.tgz", + "integrity": "sha512-P05YEhRc2h53lZDjRPk/OektxCVevFzZs2Gfjd545Wde3k+yFDbXORgl2e0xpbq8mLcKJ7Idss4fAg0zORN/zg==", "dev": true, "requires": { "@babel/helper-plugin-utils": "^7.0.0", - "@babel/helper-regex": "^7.0.0", - "regexpu-core": "^4.1.3" - } - }, - "@babel/plugin-transform-exponentiation-operator": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-exponentiation-operator/-/plugin-transform-exponentiation-operator-7.1.0.tgz", - "integrity": "sha512-uZt9kD1Pp/JubkukOGQml9tqAeI8NkE98oZnHZ2qHRElmeKCodbTZgOEUtujSCSLhHSBWbzNiFSDIMC4/RBTLQ==", - "dev": true, - "requires": { - "@babel/helper-builder-binary-assignment-operator-visitor": "^7.1.0", - "@babel/helper-plugin-utils": "^7.0.0" + "@babel/helper-regex": "^7.4.4", + "regexpu-core": "^4.5.4" } }, "@babel/plugin-transform-modules-commonjs": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.1.0.tgz", - "integrity": "sha512-wtNwtMjn1XGwM0AXPspQgvmE6msSJP15CX2RVfpTSTNPLhKhaOjaIfBaVfj4iUZ/VrFSodcFedwtPg/NxwQlPA==", + "version": "7.6.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.6.0.tgz", + "integrity": "sha512-Ma93Ix95PNSEngqomy5LSBMAQvYKVe3dy+JlVJSHEXZR5ASL9lQBedMiCyVtmTLraIDVRE3ZjTZvmXXD2Ozw3g==", "dev": true, "requires": { - "@babel/helper-module-transforms": "^7.1.0", + "@babel/helper-module-transforms": "^7.4.4", "@babel/helper-plugin-utils": "^7.0.0", - "@babel/helper-simple-access": "^7.1.0" + "@babel/helper-simple-access": "^7.1.0", + "babel-plugin-dynamic-import-node": "^2.3.0" } }, "@babel/template": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.1.2.tgz", - "integrity": "sha512-SY1MmplssORfFiLDcOETrW7fCLl+PavlwMh92rrGcikQaRq4iWPVH0MpwPpY3etVMx6RnDjXtr6VZYr/IbP/Ag==", + "version": "7.6.0", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.6.0.tgz", + "integrity": "sha512-5AEH2EXD8euCk446b7edmgFdub/qfH1SN6Nii3+fyXP807QRx9Q73A2N5hNwRRslC2H9sNzaFhsPubkS4L8oNQ==", "dev": true, "requires": { "@babel/code-frame": "^7.0.0", - "@babel/parser": "^7.1.2", - "@babel/types": "^7.1.2" + "@babel/parser": "^7.6.0", + "@babel/types": "^7.6.0" } }, "@babel/traverse": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.1.0.tgz", - "integrity": "sha512-bwgln0FsMoxm3pLOgrrnGaXk18sSM9JNf1/nHC/FksmNGFbYnPWY4GYCfLxyP1KRmfsxqkRpfoa6xr6VuuSxdw==", + "version": "7.6.0", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.6.0.tgz", + "integrity": "sha512-93t52SaOBgml/xY74lsmt7xOR4ufYvhb5c5qiM6lu4J/dWGMAfAh6eKw4PjLes6DI6nQgearoxnFJk60YchpvQ==", "dev": true, "requires": { - "@babel/code-frame": "^7.0.0", - "@babel/generator": "^7.0.0", + "@babel/code-frame": "^7.5.5", + "@babel/generator": "^7.6.0", "@babel/helper-function-name": "^7.1.0", - "@babel/helper-split-export-declaration": "^7.0.0", - "@babel/parser": "^7.1.0", - "@babel/types": "^7.0.0", - "debug": "^3.1.0", + "@babel/helper-split-export-declaration": "^7.4.4", + "@babel/parser": "^7.6.0", + "@babel/types": "^7.6.0", + "debug": "^4.1.0", "globals": "^11.1.0", - "lodash": "^4.17.10" + "lodash": "^4.17.13" } }, "@babel/types": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.1.2.tgz", - "integrity": "sha512-pb1I05sZEKiSlMUV9UReaqsCPUpgbHHHu2n1piRm7JkuBkm6QxcaIzKu6FMnMtCbih/cEYTR+RGYYC96Yk9HAg==", + "version": "7.6.1", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.6.1.tgz", + "integrity": "sha512-X7gdiuaCmA0uRjCmRtYJNAVCc/q+5xSgsfKJHqMN4iNLILX39677fJE1O40arPMh0TTtS9ItH67yre6c7k6t0g==", "dev": true, "requires": { "esutils": "^2.0.2", - "lodash": "^4.17.10", + "lodash": "^4.17.13", "to-fast-properties": "^2.0.0" } }, "@concordance/react": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@concordance/react/-/react-1.0.0.tgz", - "integrity": "sha512-htrsRaQX8Iixlsek8zQU7tE8wcsTQJ5UhZkSPEA8slCDAisKpC/2VgU/ucPn32M5/LjGGXRaUEKvEw1Wiuu4zQ==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@concordance/react/-/react-2.0.0.tgz", + "integrity": "sha512-huLSkUuM2/P+U0uy2WwlKuixMsTODD8p4JVQBI4VKeopkiN0C7M3N9XYVawb4M+4spN5RrO/eLhk7KoQX6nsfA==", "dev": true, "requires": { "arrify": "^1.0.1" + }, + "dependencies": { + "arrify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", + "integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=", + "dev": true + } + } + }, + "@nodelib/fs.scandir": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.2.tgz", + "integrity": "sha512-wrIBsjA5pl13f0RN4Zx4FNWmU71lv03meGKnqRUoCyan17s4V3WL92f3w3AIuWbNnpcrQyFBU5qMavJoB8d27w==", + "dev": true, + "requires": { + "@nodelib/fs.stat": "2.0.2", + "run-parallel": "^1.1.9" + } + }, + "@nodelib/fs.stat": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.2.tgz", + "integrity": "sha512-z8+wGWV2dgUhLqrtRYa03yDx4HWMvXKi1z8g3m2JyxAx8F7xk74asqPk5LAETjqDSGLFML/6CDl0+yFunSYicw==", + "dev": true + }, + "@nodelib/fs.walk": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.3.tgz", + "integrity": "sha512-l6t8xEhfK9Sa4YO5mIRdau7XSOADfmh3jCr0evNHdY+HNkW6xuQhgMH7D73VV6WpZOagrW0UludvMTiifiwTfA==", + "dev": true, + "requires": { + "@nodelib/fs.scandir": "2.1.2", + "fastq": "^1.6.0" + } + }, + "@sindresorhus/is": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz", + "integrity": "sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ==", + "dev": true + }, + "@szmarczak/http-timer": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-1.1.2.tgz", + "integrity": "sha512-XIB2XbzHTN6ieIjfIMV9hlVcfPU26s2vafYWQcZHWXHOxiaRZYEDKEwdl129Zyg50+foYV2jCgtrqSA6qNuNSA==", + "dev": true, + "requires": { + "defer-to-connect": "^1.0.1" + } + }, + "@types/events": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/events/-/events-3.0.0.tgz", + "integrity": "sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g==", + "dev": true + }, + "@types/glob": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.1.1.tgz", + "integrity": "sha512-1Bh06cbWJUHMC97acuD6UMG29nMt0Aqz1vF3guLfG+kHHJhy3AyohZFFxYk2f7Q1SQIrNwvncxAE0N/9s70F2w==", + "dev": true, + "requires": { + "@types/events": "*", + "@types/minimatch": "*", + "@types/node": "*" } }, + "@types/minimatch": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz", + "integrity": "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==", + "dev": true + }, + "@types/node": { + "version": "12.7.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-12.7.5.tgz", + "integrity": "sha512-9fq4jZVhPNW8r+UYKnxF1e2HkDWOWKM5bC2/7c9wPV835I0aOrVbS/Hw/pWPk2uKrNXQqg9Z959Kz+IYDd5p3w==", + "dev": true + }, "ansi-align": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-2.0.0.tgz", - "integrity": "sha1-w2rsy6VjuJzrVW82kPCx2eNUf38=", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-3.0.0.tgz", + "integrity": "sha512-ZpClVKqXN3RGBmKibdfWzqCY4lnjEuoNzU5T0oEFpfd/z5qJHVarukridD4juLO2FXMiwUQxr9WqQtaYa8XRYw==", "dev": true, "requires": { - "string-width": "^2.0.0" + "string-width": "^3.0.0" + }, + "dependencies": { + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + } } }, "ansi-escapes": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.1.0.tgz", - "integrity": "sha512-UgAb8H9D41AQnu/PbWlCofQVcnV4Gs2bBJi9eZPxfU/hgglFh3SMDMENRIqdr7H6XFnXdoknctFByVsCOotTVw==", - "dev": true + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.2.1.tgz", + "integrity": "sha512-Cg3ymMAdN10wOk/VYfLV7KCQyv7EDirJ64500sU7n9UlmioEtDuU5Gd+hj73hXSU/ex7tHJSssmyftDdkMLO8Q==", + "dev": true, + "requires": { + "type-fest": "^0.5.2" + } }, "ansi-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", - "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", "dev": true }, "ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.1.0.tgz", + "integrity": "sha512-Qts4KCLKG+waHc9C4m07weIY8qyeixoS0h6RnbsNVD6Fw+pEZGW3vTyObL3WXpE09Mq4Oi7/lBEyLmOiLtlYWQ==", "dev": true, "requires": { - "color-convert": "^1.9.0" + "color-convert": "^2.0.1" + }, + "dependencies": { + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + } } }, "anymatch": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-2.0.0.tgz", - "integrity": "sha512-5teOsQWABXHHBFP9y3skS5P3d/WfWXpv3FUpy+LorMrNYaT9pI4oLMQX7jzQ2KklNpGpWHzdCXTDT2Y3XGlZBw==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.0.tgz", + "integrity": "sha512-Ozz7l4ixzI7Oxj2+cw+p0tVUt27BpaJ+1+q1TCeANWxHpvyn2+Un+YamBdfKu0uh8xLodGhoa1v7595NhKDAuA==", "dev": true, "requires": { - "micromatch": "^3.1.4", - "normalize-path": "^2.1.1" + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" } }, "argparse": { @@ -447,30 +511,12 @@ "sprintf-js": "~1.0.2" } }, - "arr-diff": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz", - "integrity": "sha1-1kYQdP6/7HHn4VI1dhoyml3HxSA=", - "dev": true - }, "arr-flatten": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/arr-flatten/-/arr-flatten-1.1.0.tgz", "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg==", "dev": true }, - "arr-union": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz", - "integrity": "sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ=", - "dev": true - }, - "array-differ": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/array-differ/-/array-differ-1.0.0.tgz", - "integrity": "sha1-7/UuN1gknTO+QCuLuOVkuytdQDE=", - "dev": true - }, "array-find-index": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/array-find-index/-/array-find-index-1.0.2.tgz", @@ -478,153 +524,127 @@ "dev": true }, "array-union": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/array-union/-/array-union-1.0.2.tgz", - "integrity": "sha1-mjRBDk9OPaI96jdb5b5w8kd47Dk=", - "dev": true, - "requires": { - "array-uniq": "^1.0.1" - }, - "dependencies": { - "array-uniq": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.3.tgz", - "integrity": "sha1-r2rId6Jcx/dOBYiUdThY39sk/bY=", - "dev": true - } - } - }, - "array-uniq": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-2.0.0.tgz", - "integrity": "sha512-O3QZEr+3wDj7otzF7PjNGs6CA3qmYMLvt5xGkjY/V0VxS+ovvqVo/5wKM/OVOAyuX4DTh9H31zE/yKtO66hTkg==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", + "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", "dev": true }, - "array-unique": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz", - "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg=", + "array-uniq": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-2.1.0.tgz", + "integrity": "sha512-bdHxtev7FN6+MXI1YFW0Q8mQ8dTJc2S8AMfju+ZR77pbg2yAdVyDlwkaUI7Har0LyOMRFPHrJ9lYdyjZZswdlQ==", "dev": true }, "arrify": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", - "integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", "dev": true }, - "assign-symbols": { + "astral-regex": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz", - "integrity": "sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c=", - "dev": true - }, - "async-each": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/async-each/-/async-each-1.0.1.tgz", - "integrity": "sha1-GdOGodntxufByF04iu28xW0zYC0=", - "dev": true - }, - "atob": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", - "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", + "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-1.0.0.tgz", + "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", "dev": true }, "ava": { - "version": "1.0.0-beta.8", - "resolved": "https://registry.npmjs.org/ava/-/ava-1.0.0-beta.8.tgz", - "integrity": "sha512-aEG/JoBOP/iMC+0vGbfFnvQzTUKgpYm5i17j+VRLEy/qnGmynQJfPW4Hot/Cv1VaiRbpZ/S52O4BAI4N2HHpbA==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/ava/-/ava-2.3.0.tgz", + "integrity": "sha512-4VaaSnl13vpTZmqW3aMqioSolT0/ozRkjQxTLi3p8wtyRONuX/uLKL3uF0j50w2BNRoLsJqztnkX2h8xeVp2lg==", "dev": true, "requires": { - "@ava/babel-preset-stage-4": "^2.0.0", - "@ava/babel-preset-transform-test-files": "4.0.0-beta.9", - "@ava/write-file-atomic": "^2.2.0", - "@babel/core": "^7.0.0", - "@babel/generator": "^7.0.0", - "@babel/plugin-syntax-async-generators": "^7.0.0", - "@babel/plugin-syntax-object-rest-spread": "^7.0.0", - "@babel/plugin-syntax-optional-catch-binding": "^7.0.0", - "@concordance/react": "^1.0.0", - "ansi-escapes": "^3.1.0", - "ansi-styles": "^3.2.1", + "@ava/babel-preset-stage-4": "^4.0.0", + "@ava/babel-preset-transform-test-files": "^6.0.0", + "@babel/core": "^7.5.5", + "@babel/generator": "^7.5.5", + "@concordance/react": "^2.0.0", + "ansi-escapes": "^4.2.1", + "ansi-styles": "^4.0.0", "arr-flatten": "^1.1.0", - "array-union": "^1.0.1", - "array-uniq": "^2.0.0", - "arrify": "^1.0.0", - "bluebird": "^3.5.1", - "chalk": "^2.4.1", - "chokidar": "^2.0.4", + "array-union": "^2.1.0", + "array-uniq": "^2.1.0", + "arrify": "^2.0.1", + "bluebird": "^3.5.5", + "chalk": "^2.4.2", + "chokidar": "^3.0.2", + "chunkd": "^1.0.0", "ci-parallel-vars": "^1.0.0", - "clean-stack": "^1.1.1", + "clean-stack": "^2.2.0", "clean-yaml-object": "^0.1.0", - "cli-cursor": "^2.1.0", - "cli-truncate": "^1.1.0", + "cli-cursor": "^3.1.0", + "cli-truncate": "^2.0.0", "code-excerpt": "^2.1.1", "common-path-prefix": "^1.0.0", - "concordance": "^3.0.0", - "convert-source-map": "^1.5.1", + "concordance": "^4.0.0", + "convert-source-map": "^1.6.0", "currently-unhandled": "^0.4.1", - "debug": "^3.1.0", - "del": "^3.0.0", - "dot-prop": "^4.2.0", + "debug": "^4.1.1", + "del": "^4.1.1", + "dot-prop": "^5.1.0", "emittery": "^0.4.1", "empower-core": "^1.2.0", "equal-length": "^1.0.0", - "escape-string-regexp": "^1.0.5", - "esm": "^3.0.80", - "figures": "^2.0.0", - "get-port": "^4.0.0", - "globby": "^7.1.1", + "escape-string-regexp": "^2.0.0", + "esm": "^3.2.25", + "figures": "^3.0.0", + "find-up": "^4.1.0", + "get-port": "^5.0.0", + "globby": "^10.0.1", "ignore-by-default": "^1.0.0", - "import-local": "^1.0.0", - "indent-string": "^3.2.0", - "is-ci": "^1.2.0", - "is-error": "^2.2.1", - "is-observable": "^1.1.0", - "is-plain-object": "^2.0.4", + "import-local": "^3.0.2", + "indent-string": "^4.0.0", + "is-ci": "^2.0.0", + "is-error": "^2.2.2", + "is-observable": "^2.0.0", + "is-plain-object": "^3.0.0", "is-promise": "^2.1.0", - "lodash.clone": "^4.5.0", - "lodash.clonedeep": "^4.5.0", - "lodash.clonedeepwith": "^4.5.0", - "lodash.debounce": "^4.0.3", - "lodash.difference": "^4.3.0", - "lodash.flatten": "^4.2.0", - "loud-rejection": "^1.2.0", - "make-dir": "^1.3.0", - "matcher": "^1.1.1", - "md5-hex": "^2.0.0", + "lodash": "^4.17.15", + "loud-rejection": "^2.1.0", + "make-dir": "^3.0.0", + "matcher": "^2.0.0", + "md5-hex": "^3.0.1", "meow": "^5.0.0", - "ms": "^2.1.1", - "multimatch": "^2.1.0", - "observable-to-promise": "^0.5.0", - "ora": "^3.0.0", - "package-hash": "^2.0.0", - "pkg-conf": "^2.1.0", - "plur": "^3.0.1", - "pretty-ms": "^3.2.0", + "micromatch": "^4.0.2", + "ms": "^2.1.2", + "observable-to-promise": "^1.0.0", + "ora": "^3.4.0", + "package-hash": "^4.0.0", + "pkg-conf": "^3.1.0", + "plur": "^3.1.1", + "pretty-ms": "^5.0.0", "require-precompiled": "^0.1.0", - "resolve-cwd": "^2.0.0", - "slash": "^2.0.0", - "source-map-support": "^0.5.9", - "stack-utils": "^1.0.1", - "strip-ansi": "^4.0.0", - "strip-bom-buf": "^1.0.0", + "resolve-cwd": "^3.0.0", + "slash": "^3.0.0", + "source-map-support": "^0.5.13", + "stack-utils": "^1.0.2", + "strip-ansi": "^5.2.0", + "strip-bom-buf": "^2.0.0", "supertap": "^1.0.0", - "supports-color": "^5.5.0", + "supports-color": "^7.0.0", "trim-off-newlines": "^1.0.1", "trim-right": "^1.0.1", "unique-temp-dir": "^1.0.0", - "update-notifier": "^2.5.0" + "update-notifier": "^3.0.1", + "write-file-atomic": "^3.0.0" + } + }, + "babel-plugin-dynamic-import-node": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/babel-plugin-dynamic-import-node/-/babel-plugin-dynamic-import-node-2.3.0.tgz", + "integrity": "sha512-o6qFkpeQEBxcqt0XYlWzAVxNCSCZdUgcR8IRlhD/8DylxjjO4foPcvTW0GGKa/cVt3rvxZ7o5ippJ+/0nvLhlQ==", + "dev": true, + "requires": { + "object.assign": "^4.1.0" } }, "babel-plugin-espower": { - "version": "3.0.0-beta.2", - "resolved": "https://registry.npmjs.org/babel-plugin-espower/-/babel-plugin-espower-3.0.0-beta.2.tgz", - "integrity": "sha512-oK85tacH/SRebgt+f3NCu/CSGhutlgSXiJ87UMDN4pBTW1rgHrlYoqcKCdQZv23SztVkUy7o7bBEAtH/NyRpJQ==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/babel-plugin-espower/-/babel-plugin-espower-3.0.1.tgz", + "integrity": "sha512-Ms49U7VIAtQ/TtcqRbD6UBmJBUCSxiC3+zPc+eGqxKUIFO1lTshyEDRUjhoAbd2rWfwYf3cZ62oXozrd8W6J0A==", "dev": true, "requires": { - "@babel/generator": "^7.0.0-beta.54", - "@babel/parser": "^7.0.0-beta.54", + "@babel/generator": "^7.0.0", + "@babel/parser": "^7.0.0", "call-matcher": "^1.0.0", "core-js": "^2.0.0", "espower-location-detector": "^1.0.0", @@ -638,91 +658,69 @@ "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", "dev": true }, - "base": { - "version": "0.11.2", - "resolved": "https://registry.npmjs.org/base/-/base-0.11.2.tgz", - "integrity": "sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg==", - "dev": true, - "requires": { - "cache-base": "^1.0.1", - "class-utils": "^0.3.5", - "component-emitter": "^1.2.1", - "define-property": "^1.0.0", - "isobject": "^3.0.1", - "mixin-deep": "^1.2.0", - "pascalcase": "^0.1.1" - }, - "dependencies": { - "define-property": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", - "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=", - "dev": true, - "requires": { - "is-descriptor": "^1.0.0" - } - }, - "is-accessor-descriptor": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", - "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", - "dev": true, - "requires": { - "kind-of": "^6.0.0" - } - }, - "is-data-descriptor": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", - "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", - "dev": true, - "requires": { - "kind-of": "^6.0.0" - } - }, - "is-descriptor": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", - "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", - "dev": true, - "requires": { - "is-accessor-descriptor": "^1.0.0", - "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.2" - } - } - } - }, - "bignumber.js": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-4.1.0.tgz", - "integrity": "sha512-eJzYkFYy9L4JzXsbymsFn3p54D+llV27oTQ+ziJG7WFRheJcNZilgVXMG0LoZtlQSKBsJdWtLFqOD0u+U0jZKA==" - }, "binary-extensions": { - "version": "1.12.0", - "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.12.0.tgz", - "integrity": "sha512-DYWGk01lDcxeS/K9IHPGWfT8PsJmbXRtRd2Sx72Tnb8pcYZQFF1oSDb8hJtS1vhp212q1Rzi5dUf9+nq0o9UIg==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.0.0.tgz", + "integrity": "sha512-Phlt0plgpIIBOGTT/ehfFnbNlfsDEiqmzE2KRXoX1bLIlir4X/MR+zSyBEkL05ffWgnRSf/DXv+WrUAVr93/ow==", "dev": true }, "bluebird": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.2.tgz", - "integrity": "sha512-dhHTWMI7kMx5whMQntl7Vr9C6BvV10lFXDAasnqnrMYhXVCzzk6IO9Fo2L75jXHT07WrOngL1WDXOp+yYS91Yg==", + "version": "3.5.5", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.5.tgz", + "integrity": "sha512-5am6HnnfN+urzt4yfg7IgTbotDjIT/u8AJpEt0sIU9FtXfVeezXAPKswrG+xKUCOYAINpSdgZVDU6QFh+cuH3w==", + "dev": true + }, + "blueimp-md5": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/blueimp-md5/-/blueimp-md5-2.12.0.tgz", + "integrity": "sha512-zo+HIdIhzojv6F1siQPqPFROyVy7C50KzHv/k/Iz+BtvtVzSHXiMXOpq2wCfNkeBqdCv+V8XOV96tsEt2W/3rQ==", "dev": true }, "boxen": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/boxen/-/boxen-1.3.0.tgz", - "integrity": "sha512-TNPjfTr432qx7yOjQyaXm3dSR0MH9vXp7eT1BFSl/C51g+EFnOR9hTg1IreahGBmDNCehscshe45f+C1TBZbLw==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/boxen/-/boxen-3.2.0.tgz", + "integrity": "sha512-cU4J/+NodM3IHdSL2yN8bqYqnmlBTidDR4RC7nJs61ZmtGz8VZzM3HLQX0zY5mrSmPtR3xWwsq2jOUQqFZN8+A==", "dev": true, "requires": { - "ansi-align": "^2.0.0", - "camelcase": "^4.0.0", - "chalk": "^2.0.1", - "cli-boxes": "^1.0.0", - "string-width": "^2.0.0", + "ansi-align": "^3.0.0", + "camelcase": "^5.3.1", + "chalk": "^2.4.2", + "cli-boxes": "^2.2.0", + "string-width": "^3.0.0", "term-size": "^1.2.0", + "type-fest": "^0.3.0", "widest-line": "^2.0.0" + }, + "dependencies": { + "camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true + }, + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + }, + "type-fest": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.3.1.tgz", + "integrity": "sha512-cUGJnCdr4STbePCgqNFbpVNCepa+kAVohJs1sLhxzdH+gnEoOd8VhbYa7pD3zZYGiURWM2xzEII3fQcRizDkYQ==", + "dev": true + } } }, "brace-expansion": { @@ -736,32 +734,12 @@ } }, "braces": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz", - "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", "dev": true, "requires": { - "arr-flatten": "^1.1.0", - "array-unique": "^0.3.2", - "extend-shallow": "^2.0.1", - "fill-range": "^4.0.0", - "isobject": "^3.0.1", - "repeat-element": "^1.1.2", - "snapdragon": "^0.8.1", - "snapdragon-node": "^2.0.1", - "split-string": "^3.0.2", - "to-regex": "^3.0.1" - }, - "dependencies": { - "extend-shallow": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", - "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", - "dev": true, - "requires": { - "is-extendable": "^0.1.0" - } - } + "fill-range": "^7.0.1" } }, "buffer-from": { @@ -770,27 +748,36 @@ "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", "dev": true }, - "builtin-modules": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz", - "integrity": "sha1-Jw8HbFpywC9bZaR9+Uxf46J4iS8=", - "dev": true - }, - "cache-base": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz", - "integrity": "sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ==", + "cacheable-request": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.1.0.tgz", + "integrity": "sha512-Oj3cAGPCqOZX7Rz64Uny2GYAZNliQSqfbePrgAQ1wKAihYmCUnraBtJtKcGR4xz7wF+LoJC+ssFZvv5BgF9Igg==", "dev": true, "requires": { - "collection-visit": "^1.0.0", - "component-emitter": "^1.2.1", - "get-value": "^2.0.6", - "has-value": "^1.0.0", - "isobject": "^3.0.1", - "set-value": "^2.0.0", - "to-object-path": "^0.3.0", - "union-value": "^1.0.0", - "unset-value": "^1.0.0" + "clone-response": "^1.0.2", + "get-stream": "^5.1.0", + "http-cache-semantics": "^4.0.0", + "keyv": "^3.0.0", + "lowercase-keys": "^2.0.0", + "normalize-url": "^4.1.0", + "responselike": "^1.0.2" + }, + "dependencies": { + "get-stream": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.1.0.tgz", + "integrity": "sha512-EXr1FOzrzTfGeL0gQdeFEvOMm2mzMOglyiOXSTpPC+iAjAKftbr3jpCMWynogwYnM+eSj9sHGc6wjIcDvYiygw==", + "dev": true, + "requires": { + "pump": "^3.0.0" + } + }, + "lowercase-keys": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz", + "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==", + "dev": true + } } }, "call-matcher": { @@ -828,48 +815,69 @@ "quick-lru": "^1.0.0" } }, - "capture-stack-trace": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/capture-stack-trace/-/capture-stack-trace-1.0.1.tgz", - "integrity": "sha512-mYQLZnx5Qt1JgB1WEiMCf2647plpGeQ2NMR/5L0HNZzGQo4fuSPnK+wjfPnKZV0aiJDgzmWqqkV/g7JD+DW0qw==", - "dev": true - }, "chalk": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.1.tgz", - "integrity": "sha512-ObN6h1v2fTJSmUXoS3nMQ92LbDK9be4TV+6G+omQlGJFdcUX5heKi1LZ1YnRMIgwTLEj3E24bT6tYni50rlCfQ==", + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", "dev": true, "requires": { "ansi-styles": "^3.2.1", "escape-string-regexp": "^1.0.5", "supports-color": "^5.3.0" + }, + "dependencies": { + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "requires": { + "color-convert": "^1.9.0" + } + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "dev": true + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "requires": { + "has-flag": "^3.0.0" + } + } } }, "chokidar": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-2.0.4.tgz", - "integrity": "sha512-z9n7yt9rOvIJrMhvDtDictKrkFHeihkNl6uWMmZlmL6tJtX9Cs+87oK+teBx+JIgzvbX3yZHT3eF8vpbDxHJXQ==", - "dev": true, - "requires": { - "anymatch": "^2.0.0", - "async-each": "^1.0.0", - "braces": "^2.3.0", - "fsevents": "^1.2.2", - "glob-parent": "^3.1.0", - "inherits": "^2.0.1", - "is-binary-path": "^1.0.0", - "is-glob": "^4.0.0", - "lodash.debounce": "^4.0.8", - "normalize-path": "^2.1.1", - "path-is-absolute": "^1.0.0", - "readdirp": "^2.0.0", - "upath": "^1.0.5" + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.0.2.tgz", + "integrity": "sha512-c4PR2egjNjI1um6bamCQ6bUNPDiyofNQruHvKgHQ4gDUP/ITSVSzNsiI5OWtHOsX323i5ha/kk4YmOZ1Ktg7KA==", + "dev": true, + "requires": { + "anymatch": "^3.0.1", + "braces": "^3.0.2", + "fsevents": "^2.0.6", + "glob-parent": "^5.0.0", + "is-binary-path": "^2.1.0", + "is-glob": "^4.0.1", + "normalize-path": "^3.0.0", + "readdirp": "^3.1.1" } }, + "chunkd": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/chunkd/-/chunkd-1.0.0.tgz", + "integrity": "sha512-xx3Pb5VF9QaqCotolyZ1ywFBgyuJmu6+9dLiqBxgelEse9Xsr3yUlpoX3O4Oh11M00GT2kYMsRByTKIMJW2Lkg==", + "dev": true + }, "ci-info": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-1.6.0.tgz", - "integrity": "sha512-vsGdkwSCDpWmP80ncATX7iea5DWQemg1UgCW5J8tqjU3lYw4FBYuj89J0CTVomA7BEfvSZd84GmHko+MxFQU2A==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz", + "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==", "dev": true }, "ci-parallel-vars": { @@ -878,33 +886,10 @@ "integrity": "sha512-u6dx20FBXm+apMi+5x7UVm6EH7BL1gc4XrcnQewjcB7HWRcor/V5qWc3RG2HwpgDJ26gIi2DSEu3B7sXynAw/g==", "dev": true }, - "class-utils": { - "version": "0.3.6", - "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz", - "integrity": "sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==", - "dev": true, - "requires": { - "arr-union": "^3.1.0", - "define-property": "^0.2.5", - "isobject": "^3.0.0", - "static-extend": "^0.1.1" - }, - "dependencies": { - "define-property": { - "version": "0.2.5", - "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", - "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=", - "dev": true, - "requires": { - "is-descriptor": "^0.1.0" - } - } - } - }, "clean-stack": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-1.3.0.tgz", - "integrity": "sha1-noIVAa6XmYbEax1m0tQy2y/UrjE=", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", + "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", "dev": true }, "clean-yaml-object": { @@ -914,34 +899,34 @@ "dev": true }, "cli-boxes": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-1.0.0.tgz", - "integrity": "sha1-T6kXw+WclKAEzWH47lCdplFocUM=", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-2.2.0.tgz", + "integrity": "sha512-gpaBrMAizVEANOpfZp/EEUixTXDyGt7DFzdK5hU+UbWt/J0lB0w20ncZj59Z9a93xHb9u12zF5BS6i9RKbtg4w==", "dev": true }, "cli-cursor": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", - "integrity": "sha1-s12sN2R5+sw+lHR9QdDQ9SOP/LU=", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", + "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", "dev": true, "requires": { - "restore-cursor": "^2.0.0" + "restore-cursor": "^3.1.0" } }, "cli-spinners": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-1.3.1.tgz", - "integrity": "sha512-1QL4544moEsDVH9T/l6Cemov/37iv1RtoKf7NJ04A60+4MREXNfx/QvavbH6QoGdsD4N4Mwy49cmaINR/o2mdg==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.2.0.tgz", + "integrity": "sha512-tgU3fKwzYjiLEQgPMD9Jt+JjHVL9kW93FiIMX/l7rivvOD4/LL0Mf7gda3+4U2KJBloybwgj5KEoQgGRioMiKQ==", "dev": true }, "cli-truncate": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/cli-truncate/-/cli-truncate-1.1.0.tgz", - "integrity": "sha512-bAtZo0u82gCfaAGfSNxUdTI9mNyza7D8w4CVCcaOsy7sgwDzvx6ekr6cuWJqY3UGzgnQ1+4wgENup5eIhgxEYA==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/cli-truncate/-/cli-truncate-2.0.0.tgz", + "integrity": "sha512-C4hp+8GCIFVsUUiXcw+ce+7wexVWImw8rQrgMBFsqerx9LvvcGlwm6sMjQYAEmV/Xb87xc1b5Ttx505MSpZVqg==", "dev": true, "requires": { - "slice-ansi": "^1.0.0", - "string-width": "^2.0.0" + "slice-ansi": "^2.1.0", + "string-width": "^4.1.0" } }, "clone": { @@ -950,6 +935,15 @@ "integrity": "sha1-2jCcwmPfFZlMaIypAheco8fNfH4=", "dev": true }, + "clone-response": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.2.tgz", + "integrity": "sha1-0dyXOSAxTfZ/vrlCI7TuNQI56Ws=", + "dev": true, + "requires": { + "mimic-response": "^1.0.0" + } + }, "code-excerpt": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/code-excerpt/-/code-excerpt-2.1.1.tgz", @@ -959,16 +953,6 @@ "convert-to-spaces": "^1.0.1" } }, - "collection-visit": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz", - "integrity": "sha1-S8A3PBZLwykbTTaMgpzxqApZ3KA=", - "dev": true, - "requires": { - "map-visit": "^1.0.0", - "object-visit": "^1.0.0" - } - }, "color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", @@ -990,12 +974,6 @@ "integrity": "sha1-zVL28HEuC6q5fW+XModPIvR3UsA=", "dev": true }, - "component-emitter": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.1.tgz", - "integrity": "sha1-E3kY1teCg/ffemt8WmPhQOaUJeY=", - "dev": true - }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -1003,28 +981,39 @@ "dev": true }, "concordance": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/concordance/-/concordance-3.0.0.tgz", - "integrity": "sha512-CZBzJ3/l5QJjlZM20WY7+5GP5pMTw+1UEbThcpMw8/rojsi5sBCiD8ZbBLtD+jYpRGAkwuKuqk108c154V9eyQ==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/concordance/-/concordance-4.0.0.tgz", + "integrity": "sha512-l0RFuB8RLfCS0Pt2Id39/oCPykE01pyxgAFypWTlaGRgvLkZrtczZ8atEHpTeEIW+zYWXTBuA9cCSeEOScxReQ==", "dev": true, "requires": { "date-time": "^2.1.0", "esutils": "^2.0.2", - "fast-diff": "^1.1.1", - "function-name-support": "^0.2.0", + "fast-diff": "^1.1.2", "js-string-escape": "^1.0.1", "lodash.clonedeep": "^4.5.0", "lodash.flattendeep": "^4.4.0", - "lodash.merge": "^4.6.0", + "lodash.islength": "^4.0.1", + "lodash.merge": "^4.6.1", "md5-hex": "^2.0.0", - "semver": "^5.3.0", - "well-known-symbols": "^1.0.0" + "semver": "^5.5.1", + "well-known-symbols": "^2.0.0" + }, + "dependencies": { + "md5-hex": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/md5-hex/-/md5-hex-2.0.0.tgz", + "integrity": "sha1-0FiOnxx0lUSS7NJKwKxs6ZfZLjM=", + "dev": true, + "requires": { + "md5-o-matic": "^0.1.1" + } + } } }, "configstore": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/configstore/-/configstore-3.1.2.tgz", - "integrity": "sha512-vtv5HtGjcYUgFrXc6Kx747B83MRRVS5R1VTEQoXvuP+kMI+if6uywV0nDGoiydJRy4yk7h9od5Og0kxx4zUXmw==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/configstore/-/configstore-4.0.0.tgz", + "integrity": "sha512-CmquAXFBocrzaSM8mtGPMM/HiWmyIpr4CcJl/rgY2uCObZ/S7cKU0silxslqJejl+t/T9HS8E0PUNQD81JGUEQ==", "dev": true, "requires": { "dot-prop": "^4.1.0", @@ -1033,6 +1022,49 @@ "unique-string": "^1.0.0", "write-file-atomic": "^2.0.0", "xdg-basedir": "^3.0.0" + }, + "dependencies": { + "dot-prop": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-4.2.0.tgz", + "integrity": "sha512-tUMXrxlExSW6U2EXiiKGSBVdYgtV8qlHL+C10TsW4PURY/ic+eaysnSkwB4kA/mBlCyy/IKDJ+Lc3wbWeaXtuQ==", + "dev": true, + "requires": { + "is-obj": "^1.0.0" + } + }, + "is-obj": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", + "integrity": "sha1-PkcprB9f3gJc19g6iW2rn09n2w8=", + "dev": true + }, + "make-dir": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz", + "integrity": "sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==", + "dev": true, + "requires": { + "pify": "^3.0.0" + } + }, + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "dev": true + }, + "write-file-atomic": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-2.4.3.tgz", + "integrity": "sha512-GaETH5wwsX+GcnzhPgKcKjJ6M2Cq3/iZp1WyY/X1CSqrW+jVNM9Y7D8EC2sM4ZG/V8wZlSniJnCKWPmBYAucRQ==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.11", + "imurmurhash": "^0.1.4", + "signal-exit": "^3.0.2" + } + } } }, "convert-source-map": { @@ -1050,16 +1082,10 @@ "integrity": "sha1-fj5Iu+bZl7FBfdyihoIEtNPYVxU=", "dev": true }, - "copy-descriptor": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz", - "integrity": "sha1-Z29us8OZl8LuGsOpJP1hJHSPV40=", - "dev": true - }, "core-js": { - "version": "2.5.7", - "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.5.7.tgz", - "integrity": "sha512-RszJCAxg/PP6uzXVXL6BsxSXx/B05oJAQ2vkJRjyjrEcNVycaqOmNb5OTxZPE3xa5gwZduqza6L9JOCenh/Ecw==", + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz", + "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==", "dev": true }, "core-util-is": { @@ -1067,15 +1093,6 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, - "create-error-class": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/create-error-class/-/create-error-class-3.0.2.tgz", - "integrity": "sha1-Br56vvlHo/FKMP1hBnHUAbyot7Y=", - "dev": true, - "requires": { - "capture-stack-trace": "^1.0.0" - } - }, "cross-spawn": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz", @@ -1112,9 +1129,9 @@ } }, "debug": { - "version": "3.2.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.5.tgz", - "integrity": "sha512-D61LaDQPQkxJ5AUM2mbSJRbPkNs/TmdmOeLAi1hgDkpDfIfetSrjmWhccwtuResSwMbACjx/xXQofvM9CE/aeg==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", "dev": true, "requires": { "ms": "^2.1.1" @@ -1144,17 +1161,28 @@ } } }, - "decode-uri-component": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", - "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=", - "dev": true + "decompress-response": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", + "integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=", + "dev": true, + "requires": { + "mimic-response": "^1.0.0" + } }, "deep-equal": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.0.1.tgz", - "integrity": "sha1-9dJgKStmDghO/0zbyfCK0yR0SLU=", - "dev": true + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.1.0.tgz", + "integrity": "sha512-ZbfWJq/wN1Z273o7mUSjILYqehAktR2NVoSrOukDkU9kg2v/Uv89yU4Cvz8seJeAmtN5oqiefKq8FPuXOboqLw==", + "dev": true, + "requires": { + "is-arguments": "^1.0.4", + "is-date-object": "^1.0.1", + "is-regex": "^1.0.4", + "object-is": "^1.0.1", + "object-keys": "^1.1.1", + "regexp.prototype.flags": "^1.2.0" + } }, "deep-extend": { "version": "0.6.0", @@ -1171,61 +1199,51 @@ "clone": "^1.0.2" } }, - "define-property": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz", - "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==", + "defer-to-connect": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-1.0.2.tgz", + "integrity": "sha512-k09hcQcTDY+cwgiwa6PYKLm3jlagNzQ+RSvhjzESOGOx+MNOuXkxTfEvPrO1IOQ81tArCFYQgi631clB70RpQw==", + "dev": true + }, + "define-properties": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", + "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", "dev": true, "requires": { - "is-descriptor": "^1.0.2", - "isobject": "^3.0.1" - }, - "dependencies": { - "is-accessor-descriptor": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", - "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", - "dev": true, - "requires": { - "kind-of": "^6.0.0" - } - }, - "is-data-descriptor": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", - "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", - "dev": true, - "requires": { - "kind-of": "^6.0.0" - } - }, - "is-descriptor": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", - "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", - "dev": true, - "requires": { - "is-accessor-descriptor": "^1.0.0", - "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.2" - } - } + "object-keys": "^1.0.12" } }, "del": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/del/-/del-3.0.0.tgz", - "integrity": "sha1-U+z2mf/LyzljdpGrE7rxYIGXZuU=", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/del/-/del-4.1.1.tgz", + "integrity": "sha512-QwGuEUouP2kVwQenAsOof5Fv8K9t3D8Ca8NxcXKrIpEHjTXK5J2nXLdP+ALI1cgv8wj7KuwBhTwBkOZSJKM5XQ==", "dev": true, "requires": { + "@types/glob": "^7.1.1", "globby": "^6.1.0", - "is-path-cwd": "^1.0.0", - "is-path-in-cwd": "^1.0.0", - "p-map": "^1.1.1", - "pify": "^3.0.0", - "rimraf": "^2.2.8" + "is-path-cwd": "^2.0.0", + "is-path-in-cwd": "^2.0.0", + "p-map": "^2.0.0", + "pify": "^4.0.1", + "rimraf": "^2.6.3" }, "dependencies": { + "array-union": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-1.0.2.tgz", + "integrity": "sha1-mjRBDk9OPaI96jdb5b5w8kd47Dk=", + "dev": true, + "requires": { + "array-uniq": "^1.0.1" + } + }, + "array-uniq": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.3.tgz", + "integrity": "sha1-r2rId6Jcx/dOBYiUdThY39sk/bY=", + "dev": true + }, "globby": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/globby/-/globby-6.1.0.tgz", @@ -1250,22 +1268,21 @@ } }, "dir-glob": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-2.0.0.tgz", - "integrity": "sha512-37qirFDz8cA5fimp9feo43fSuRo2gHwaIn6dXL8Ber1dGwUosDrGZeCCXq57WnIqE4aQ+u3eQZzsk1yOzhdwag==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", + "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", "dev": true, "requires": { - "arrify": "^1.0.1", - "path-type": "^3.0.0" + "path-type": "^4.0.0" } }, "dot-prop": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-4.2.0.tgz", - "integrity": "sha512-tUMXrxlExSW6U2EXiiKGSBVdYgtV8qlHL+C10TsW4PURY/ic+eaysnSkwB4kA/mBlCyy/IKDJ+Lc3wbWeaXtuQ==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.1.0.tgz", + "integrity": "sha512-n1oC6NBF+KM9oVXtjmen4Yo7HyAVWV2UUl50dCYJdw2924K6dX9bf9TTTWaKtYlRn0FEtxG27KS80ayVLixxJA==", "dev": true, "requires": { - "is-obj": "^1.0.0" + "is-obj": "^2.0.0" } }, "duplexer3": { @@ -1280,6 +1297,12 @@ "integrity": "sha512-r4eRSeStEGf6M5SKdrQhhLK5bOwOBxQhIE3YSTnZE3GpKiLfnnhE+tPtrJE79+eDJgm39BM6LSoI8SCx4HbwlQ==", "dev": true }, + "emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, "empower-core": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/empower-core/-/empower-core-1.2.0.tgz", @@ -1290,6 +1313,15 @@ "core-js": "^2.0.0" } }, + "end-of-stream": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz", + "integrity": "sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q==", + "dev": true, + "requires": { + "once": "^1.4.0" + } + }, "equal-length": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/equal-length/-/equal-length-1.0.1.tgz", @@ -1312,15 +1344,15 @@ "dev": true }, "escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", + "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", "dev": true }, "esm": { - "version": "3.0.84", - "resolved": "https://registry.npmjs.org/esm/-/esm-3.0.84.tgz", - "integrity": "sha512-SzSGoZc17S7P+12R9cg21Bdb7eybX25RnIeRZ80xZs+VZ3kdQKzqTp2k4hZJjR7p9l0186TTXSgrxzlMDBktlw==", + "version": "3.2.25", + "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz", + "integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==", "dev": true }, "espower-location-detector": { @@ -1351,15 +1383,15 @@ } }, "estraverse": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.2.0.tgz", - "integrity": "sha1-De4/7TH81GlhjOc0IJn8GvoL2xM=", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", "dev": true }, "esutils": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz", - "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", "dev": true }, "execa": { @@ -1377,767 +1409,117 @@ "strip-eof": "^1.0.0" } }, - "expand-brackets": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-2.1.4.tgz", - "integrity": "sha1-t3c14xXOMPa27/D4OwQVGiJEliI=", - "dev": true, - "requires": { - "debug": "^2.3.3", - "define-property": "^0.2.5", - "extend-shallow": "^2.0.1", - "posix-character-classes": "^0.1.0", - "regex-not": "^1.0.0", - "snapdragon": "^0.8.1", - "to-regex": "^3.0.1" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "dev": true, - "requires": { - "ms": "2.0.0" - } - }, - "define-property": { - "version": "0.2.5", - "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", - "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=", - "dev": true, - "requires": { - "is-descriptor": "^0.1.0" - } - }, - "extend-shallow": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", - "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", - "dev": true, - "requires": { - "is-extendable": "^0.1.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", - "dev": true - } - } - }, - "extend-shallow": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz", - "integrity": "sha1-Jqcarwc7OfshJxcnRhMcJwQCjbg=", - "dev": true, - "requires": { - "assign-symbols": "^1.0.0", - "is-extendable": "^1.0.1" - }, - "dependencies": { - "is-extendable": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", - "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", - "dev": true, - "requires": { - "is-plain-object": "^2.0.4" - } - } - } - }, - "extglob": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/extglob/-/extglob-2.0.4.tgz", - "integrity": "sha512-Nmb6QXkELsuBr24CJSkilo6UHHgbekK5UiZgfE6UHD3Eb27YC6oD+bhcT+tJ6cl8dmsgdQxnWlcry8ksBIBLpw==", - "dev": true, - "requires": { - "array-unique": "^0.3.2", - "define-property": "^1.0.0", - "expand-brackets": "^2.1.4", - "extend-shallow": "^2.0.1", - "fragment-cache": "^0.2.1", - "regex-not": "^1.0.0", - "snapdragon": "^0.8.1", - "to-regex": "^3.0.1" - }, - "dependencies": { - "define-property": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", - "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=", - "dev": true, - "requires": { - "is-descriptor": "^1.0.0" - } - }, - "extend-shallow": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", - "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", - "dev": true, - "requires": { - "is-extendable": "^0.1.0" - } - }, - "is-accessor-descriptor": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", - "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", - "dev": true, - "requires": { - "kind-of": "^6.0.0" - } - }, - "is-data-descriptor": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", - "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", - "dev": true, - "requires": { - "kind-of": "^6.0.0" - } - }, - "is-descriptor": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", - "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", - "dev": true, - "requires": { - "is-accessor-descriptor": "^1.0.0", - "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.2" - } - } - } - }, "fast-diff": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/fast-diff/-/fast-diff-1.1.2.tgz", - "integrity": "sha512-KaJUt+M9t1qaIteSvjc6P3RbMdXsNhK61GRftR6SNxqmhthcd9MGIi4T+o0jD8LUSpSnSKXE20nLtJ3fOHxQig==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/fast-diff/-/fast-diff-1.2.0.tgz", + "integrity": "sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w==", "dev": true }, - "figures": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", - "integrity": "sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI=", - "dev": true, - "requires": { - "escape-string-regexp": "^1.0.5" - } - }, - "fill-range": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz", - "integrity": "sha1-1USBHUKPmOsGpj3EAtJAPDKMOPc=", - "dev": true, - "requires": { - "extend-shallow": "^2.0.1", - "is-number": "^3.0.0", - "repeat-string": "^1.6.1", - "to-regex-range": "^2.1.0" - }, - "dependencies": { - "extend-shallow": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", - "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", - "dev": true, - "requires": { - "is-extendable": "^0.1.0" - } - } - } - }, - "find-up": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", - "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", + "fast-glob": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.0.4.tgz", + "integrity": "sha512-wkIbV6qg37xTJwqSsdnIphL1e+LaGz4AIQqr00mIubMaEhv1/HEmJ0uuCGZRNRUkZZmOB5mJKO0ZUTVq+SxMQg==", "dev": true, "requires": { - "locate-path": "^2.0.0" + "@nodelib/fs.stat": "^2.0.1", + "@nodelib/fs.walk": "^1.2.1", + "glob-parent": "^5.0.0", + "is-glob": "^4.0.1", + "merge2": "^1.2.3", + "micromatch": "^4.0.2" } }, - "for-in": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", - "integrity": "sha1-gQaNKVqBQuwKxybG4iAMMPttXoA=", - "dev": true - }, - "fragment-cache": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz", - "integrity": "sha1-QpD60n8T6Jvn8zeZxrxaCr//DRk=", + "fastq": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.6.0.tgz", + "integrity": "sha512-jmxqQ3Z/nXoeyDmWAzF9kH1aGZSis6e/SbfPmJpUnyZ0ogr6iscHQaml4wsEepEWSdtmpy+eVXmCRIMpxaXqOA==", "dev": true, "requires": { - "map-cache": "^0.2.2" + "reusify": "^1.0.0" } }, - "fs.realpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", - "dev": true - }, - "fsevents": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.4.tgz", - "integrity": "sha512-z8H8/diyk76B7q5wg+Ud0+CqzcAF3mBBI/bA5ne5zrRUUIvNkJY//D3BqyH571KuAC4Nr7Rw7CjWX4r0y9DvNg==", - "dev": true, - "optional": true, - "requires": { - "nan": "^2.9.2", - "node-pre-gyp": "^0.10.0" - }, - "dependencies": { - "abbrev": { - "version": "1.1.1", - "bundled": true, - "dev": true, - "optional": true - }, - "ansi-regex": { - "version": "2.1.1", - "bundled": true, - "dev": true - }, - "aproba": { - "version": "1.2.0", - "bundled": true, - "dev": true, - "optional": true - }, - "are-we-there-yet": { - "version": "1.1.4", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "delegates": "^1.0.0", - "readable-stream": "^2.0.6" - } - }, - "balanced-match": { - "version": "1.0.0", - "bundled": true, - "dev": true - }, - "brace-expansion": { - "version": "1.1.11", - "bundled": true, - "dev": true, - "requires": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "chownr": { - "version": "1.0.1", - "bundled": true, - "dev": true, - "optional": true - }, - "code-point-at": { - "version": "1.1.0", - "bundled": true, - "dev": true - }, - "concat-map": { - "version": "0.0.1", - "bundled": true, - "dev": true - }, - "console-control-strings": { - "version": "1.1.0", - "bundled": true, - "dev": true - }, - "core-util-is": { - "version": "1.0.2", - "bundled": true, - "dev": true, - "optional": true - }, - "debug": { - "version": "2.6.9", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "ms": "2.0.0" - } - }, - "deep-extend": { - "version": "0.5.1", - "bundled": true, - "dev": true, - "optional": true - }, - "delegates": { - "version": "1.0.0", - "bundled": true, - "dev": true, - "optional": true - }, - "detect-libc": { - "version": "1.0.3", - "bundled": true, - "dev": true, - "optional": true - }, - "fs-minipass": { - "version": "1.2.5", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "minipass": "^2.2.1" - } - }, - "fs.realpath": { - "version": "1.0.0", - "bundled": true, - "dev": true, - "optional": true - }, - "gauge": { - "version": "2.7.4", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "aproba": "^1.0.3", - "console-control-strings": "^1.0.0", - "has-unicode": "^2.0.0", - "object-assign": "^4.1.0", - "signal-exit": "^3.0.0", - "string-width": "^1.0.1", - "strip-ansi": "^3.0.1", - "wide-align": "^1.1.0" - } - }, - "glob": { - "version": "7.1.2", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, - "has-unicode": { - "version": "2.0.1", - "bundled": true, - "dev": true, - "optional": true - }, - "iconv-lite": { - "version": "0.4.21", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "safer-buffer": "^2.1.0" - } - }, - "ignore-walk": { - "version": "3.0.1", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "minimatch": "^3.0.4" - } - }, - "inflight": { - "version": "1.0.6", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "once": "^1.3.0", - "wrappy": "1" - } - }, - "inherits": { - "version": "2.0.3", - "bundled": true, - "dev": true - }, - "ini": { - "version": "1.3.5", - "bundled": true, - "dev": true, - "optional": true - }, - "is-fullwidth-code-point": { - "version": "1.0.0", - "bundled": true, - "dev": true, - "requires": { - "number-is-nan": "^1.0.0" - } - }, - "isarray": { - "version": "1.0.0", - "bundled": true, - "dev": true, - "optional": true - }, - "minimatch": { - "version": "3.0.4", - "bundled": true, - "dev": true, - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "minimist": { - "version": "0.0.8", - "bundled": true, - "dev": true - }, - "minipass": { - "version": "2.2.4", - "bundled": true, - "dev": true, - "requires": { - "safe-buffer": "^5.1.1", - "yallist": "^3.0.0" - } - }, - "minizlib": { - "version": "1.1.0", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "minipass": "^2.2.1" - } - }, - "mkdirp": { - "version": "0.5.1", - "bundled": true, - "dev": true, - "requires": { - "minimist": "0.0.8" - } - }, - "ms": { - "version": "2.0.0", - "bundled": true, - "dev": true, - "optional": true - }, - "needle": { - "version": "2.2.0", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "debug": "^2.1.2", - "iconv-lite": "^0.4.4", - "sax": "^1.2.4" - } - }, - "node-pre-gyp": { - "version": "0.10.0", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "detect-libc": "^1.0.2", - "mkdirp": "^0.5.1", - "needle": "^2.2.0", - "nopt": "^4.0.1", - "npm-packlist": "^1.1.6", - "npmlog": "^4.0.2", - "rc": "^1.1.7", - "rimraf": "^2.6.1", - "semver": "^5.3.0", - "tar": "^4" - } - }, - "nopt": { - "version": "4.0.1", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "abbrev": "1", - "osenv": "^0.1.4" - } - }, - "npm-bundled": { - "version": "1.0.3", - "bundled": true, - "dev": true, - "optional": true - }, - "npm-packlist": { - "version": "1.1.10", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "ignore-walk": "^3.0.1", - "npm-bundled": "^1.0.1" - } - }, - "npmlog": { - "version": "4.1.2", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "are-we-there-yet": "~1.1.2", - "console-control-strings": "~1.1.0", - "gauge": "~2.7.3", - "set-blocking": "~2.0.0" - } - }, - "number-is-nan": { - "version": "1.0.1", - "bundled": true, - "dev": true - }, - "object-assign": { - "version": "4.1.1", - "bundled": true, - "dev": true, - "optional": true - }, - "once": { - "version": "1.4.0", - "bundled": true, - "dev": true, - "requires": { - "wrappy": "1" - } - }, - "os-homedir": { - "version": "1.0.2", - "bundled": true, - "dev": true, - "optional": true - }, - "os-tmpdir": { - "version": "1.0.2", - "bundled": true, - "dev": true, - "optional": true - }, - "osenv": { - "version": "0.1.5", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "os-homedir": "^1.0.0", - "os-tmpdir": "^1.0.0" - } - }, - "path-is-absolute": { - "version": "1.0.1", - "bundled": true, - "dev": true, - "optional": true - }, - "process-nextick-args": { - "version": "2.0.0", - "bundled": true, - "dev": true, - "optional": true - }, - "rc": { - "version": "1.2.7", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "deep-extend": "^0.5.1", - "ini": "~1.3.0", - "minimist": "^1.2.0", - "strip-json-comments": "~2.0.1" - }, - "dependencies": { - "minimist": { - "version": "1.2.0", - "bundled": true, - "dev": true, - "optional": true - } - } - }, - "readable-stream": { - "version": "2.3.6", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "rimraf": { - "version": "2.6.2", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "glob": "^7.0.5" - } - }, - "safe-buffer": { - "version": "5.1.1", - "bundled": true, - "dev": true - }, - "safer-buffer": { - "version": "2.1.2", - "bundled": true, - "dev": true, - "optional": true - }, - "sax": { - "version": "1.2.4", - "bundled": true, - "dev": true, - "optional": true - }, - "semver": { - "version": "5.5.0", - "bundled": true, - "dev": true, - "optional": true - }, - "set-blocking": { - "version": "2.0.0", - "bundled": true, - "dev": true, - "optional": true - }, - "signal-exit": { - "version": "3.0.2", - "bundled": true, - "dev": true, - "optional": true - }, - "string-width": { - "version": "1.0.2", - "bundled": true, - "dev": true, - "requires": { - "code-point-at": "^1.0.0", - "is-fullwidth-code-point": "^1.0.0", - "strip-ansi": "^3.0.0" - } - }, - "string_decoder": { - "version": "1.1.1", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "safe-buffer": "~5.1.0" - } - }, - "strip-ansi": { - "version": "3.0.1", - "bundled": true, - "dev": true, - "requires": { - "ansi-regex": "^2.0.0" - } - }, - "strip-json-comments": { - "version": "2.0.1", - "bundled": true, - "dev": true, - "optional": true - }, - "tar": { - "version": "4.4.1", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "chownr": "^1.0.1", - "fs-minipass": "^1.2.5", - "minipass": "^2.2.4", - "minizlib": "^1.1.0", - "mkdirp": "^0.5.0", - "safe-buffer": "^5.1.1", - "yallist": "^3.0.2" - } - }, - "util-deprecate": { - "version": "1.0.2", - "bundled": true, - "dev": true, - "optional": true - }, - "wide-align": { - "version": "1.1.2", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "string-width": "^1.0.2" - } - }, - "wrappy": { - "version": "1.0.2", - "bundled": true, - "dev": true - }, - "yallist": { - "version": "3.0.2", - "bundled": true, + "figures": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-3.0.0.tgz", + "integrity": "sha512-HKri+WoWoUgr83pehn/SIgLOMZ9nAWC6dcGj26RY2R4F50u4+RTUz0RCrUlOV3nKRAICW1UGzyb+kcX2qK1S/g==", + "dev": true, + "requires": { + "escape-string-regexp": "^1.0.5" + }, + "dependencies": { + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", "dev": true } } }, - "function-name-support": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/function-name-support/-/function-name-support-0.2.0.tgz", - "integrity": "sha1-VdO/qm6v1QWlD5vIH99XVkoLsHE=", + "fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "dev": true, + "requires": { + "to-regex-range": "^5.0.1" + } + }, + "find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "requires": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + } + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", "dev": true }, - "get-port": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/get-port/-/get-port-4.0.0.tgz", - "integrity": "sha512-Yy3yNI2oShgbaWg4cmPhWjkZfktEvpKI09aDX4PZzNtlU9obuYrX7x2mumQsrNxlF+Ls7OtMQW/u+X4s896bOQ==", + "fsevents": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.0.7.tgz", + "integrity": "sha512-a7YT0SV3RB+DjYcppwVDLtn13UQnmg0SWZS7ezZD0UjnLwXmy8Zm21GMVGLaFGimIqcvyMQaOJBrop8MyOp1kQ==", + "dev": true, + "optional": true + }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", "dev": true }, + "get-port": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.0.0.tgz", + "integrity": "sha512-imzMU0FjsZqNa6BqOjbbW6w5BivHIuQKopjpPqcnx0AVHJQKCxK1O+Ab3OrVXhrekqfVMjwA9ZYu062R+KcIsQ==", + "dev": true, + "requires": { + "type-fest": "^0.3.0" + }, + "dependencies": { + "type-fest": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.3.1.tgz", + "integrity": "sha512-cUGJnCdr4STbePCgqNFbpVNCepa+kAVohJs1sLhxzdH+gnEoOd8VhbYa7pD3zZYGiURWM2xzEII3fQcRizDkYQ==", + "dev": true + } + } + }, "get-stream": { "version": "3.0.0", - "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=", "dev": true }, - "get-value": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz", - "integrity": "sha1-3BXKHGcjh8p2vTesCjlbogQqLCg=", - "dev": true - }, "glob": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", - "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz", + "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==", "dev": true, "requires": { "fs.realpath": "^1.0.0", @@ -2149,24 +1531,12 @@ } }, "glob-parent": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-3.1.0.tgz", - "integrity": "sha1-nmr2KZ2NO9K9QEMIMr0RPfkGxa4=", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.0.0.tgz", + "integrity": "sha512-Z2RwiujPRGluePM6j699ktJYxmPpJKCfpGA13jz2hmFZC7gKetzrWvg5KN3+OsIFmydGyZ1AVwERCq1w/ZZwRg==", "dev": true, "requires": { - "is-glob": "^3.1.0", - "path-dirname": "^1.0.0" - }, - "dependencies": { - "is-glob": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-3.1.0.tgz", - "integrity": "sha1-e6WuJCF4BKxwcHuWkiVnSGzD6Eo=", - "dev": true, - "requires": { - "is-extglob": "^2.1.0" - } - } + "is-glob": "^4.0.1" } }, "global-dirs": { @@ -2179,106 +1549,124 @@ } }, "globals": { - "version": "11.8.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-11.8.0.tgz", - "integrity": "sha512-io6LkyPVuzCHBSQV9fmOwxZkUk6nIaGmxheLDgmuFv89j0fm2aqDbIXKAGfzCMHqz3HLF2Zf8WSG6VqMh2qFmA==", + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", "dev": true }, "globby": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/globby/-/globby-7.1.1.tgz", - "integrity": "sha1-+yzP+UAfhgCUXfral0QMypcrhoA=", + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/globby/-/globby-10.0.1.tgz", + "integrity": "sha512-sSs4inE1FB2YQiymcmTv6NWENryABjUNPeWhOvmn4SjtKybglsyPZxFB3U1/+L1bYi0rNZDqCLlHyLYDl1Pq5A==", "dev": true, "requires": { - "array-union": "^1.0.1", - "dir-glob": "^2.0.0", - "glob": "^7.1.2", - "ignore": "^3.3.5", - "pify": "^3.0.0", - "slash": "^1.0.0" - }, - "dependencies": { - "slash": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-1.0.0.tgz", - "integrity": "sha1-xB8vbDn8FtHNF61LXYlhFK5HDVU=", - "dev": true - } + "@types/glob": "^7.1.1", + "array-union": "^2.1.0", + "dir-glob": "^3.0.1", + "fast-glob": "^3.0.3", + "glob": "^7.1.3", + "ignore": "^5.1.1", + "merge2": "^1.2.3", + "slash": "^3.0.0" } }, "got": { - "version": "6.7.1", - "resolved": "http://registry.npmjs.org/got/-/got-6.7.1.tgz", - "integrity": "sha1-JAzQV4WpoY5WHcG0S0HHY+8ejbA=", + "version": "9.6.0", + "resolved": "https://registry.npmjs.org/got/-/got-9.6.0.tgz", + "integrity": "sha512-R7eWptXuGYxwijs0eV+v3o6+XH1IqVK8dJOEecQfTmkncw9AV4dcw/Dhxi8MdlqPthxxpZyizMzyg8RTmEsG+Q==", "dev": true, "requires": { - "create-error-class": "^3.0.0", + "@sindresorhus/is": "^0.14.0", + "@szmarczak/http-timer": "^1.1.2", + "cacheable-request": "^6.0.0", + "decompress-response": "^3.3.0", "duplexer3": "^0.1.4", - "get-stream": "^3.0.0", - "is-redirect": "^1.0.0", - "is-retry-allowed": "^1.0.0", - "is-stream": "^1.0.0", - "lowercase-keys": "^1.0.0", - "safe-buffer": "^5.0.1", - "timed-out": "^4.0.0", - "unzip-response": "^2.0.1", - "url-parse-lax": "^1.0.0" + "get-stream": "^4.1.0", + "lowercase-keys": "^1.0.1", + "mimic-response": "^1.0.1", + "p-cancelable": "^1.0.0", + "to-readable-stream": "^1.0.0", + "url-parse-lax": "^3.0.0" + }, + "dependencies": { + "get-stream": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", + "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", + "dev": true, + "requires": { + "pump": "^3.0.0" + } + } } }, "graceful-fs": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz", - "integrity": "sha1-Dovf5NHduIVNZOBOp8AOKgJuVlg=", + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.2.tgz", + "integrity": "sha512-IItsdsea19BoLC7ELy13q1iJFNmd7ofZH5+X/pJr90/nRoPEX0DJo1dHDbgtYWOhJhcCgMDTOw84RZ72q6lB+Q==", "dev": true }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dev": true, + "requires": { + "function-bind": "^1.1.1" + } + }, "has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", "dev": true }, - "has-value": { + "has-symbols": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-value/-/has-value-1.0.0.tgz", - "integrity": "sha1-GLKB2lhbHFxR3vJMkw7SmgvmsXc=", - "dev": true, - "requires": { - "get-value": "^2.0.6", - "has-values": "^1.0.0", - "isobject": "^3.0.0" - } + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.0.tgz", + "integrity": "sha1-uhqPGvKg/DllD1yFA2dwQSIGO0Q=", + "dev": true }, - "has-values": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-values/-/has-values-1.0.0.tgz", - "integrity": "sha1-lbC2P+whRmGab+V/51Yo1aOe/k8=", + "has-yarn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/has-yarn/-/has-yarn-2.1.0.tgz", + "integrity": "sha512-UqBRqi4ju7T+TqGNdqAO0PaSVGsDGJUBQvk9eUWNGRY1CFGDzYhLWoM7JQEemnlvVcv/YEmc2wNW8BC24EnUsw==", + "dev": true + }, + "hasha": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/hasha/-/hasha-5.0.0.tgz", + "integrity": "sha512-PqWdhnQhq6tqD32hZv+l1e5mJHNSudjnaAzgAHfkGiU0ABN6lmbZF8abJIulQHbZ7oiHhP8yL6O910ICMc+5pw==", "dev": true, "requires": { - "is-number": "^3.0.0", - "kind-of": "^4.0.0" + "is-stream": "^1.1.0", + "type-fest": "^0.3.0" }, "dependencies": { - "kind-of": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-4.0.0.tgz", - "integrity": "sha1-IIE989cSkosgc3hpGkUGb65y3Vc=", - "dev": true, - "requires": { - "is-buffer": "^1.1.5" - } + "type-fest": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.3.1.tgz", + "integrity": "sha512-cUGJnCdr4STbePCgqNFbpVNCepa+kAVohJs1sLhxzdH+gnEoOd8VhbYa7pD3zZYGiURWM2xzEII3fQcRizDkYQ==", + "dev": true } } }, "hosted-git-info": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.7.1.tgz", - "integrity": "sha512-7T/BxH19zbcCTa8XkMlbK5lTo1WtgkFi3GvdWEyNuc4Vex7/9Dqbnpsf4JMydcfj9HCg4zUWFTL3Za6lapg5/w==", + "version": "2.8.4", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.4.tgz", + "integrity": "sha512-pzXIvANXEFrc5oFFXRMkbLPQ2rXRoDERwDLyrcUxGhaZhgP54BBSl9Oheh7Vv0T090cszWBxPjkQQ5Sq1PbBRQ==", + "dev": true + }, + "http-cache-semantics": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.0.3.tgz", + "integrity": "sha512-TcIMG3qeVLgDr1TEd2XvHaTnMPwYQUQMIBLy+5pLSDKYFc7UIqj39w8EGzZkaxoLv/l2K8HaI0t5AVA+YYgUew==", "dev": true }, "ignore": { - "version": "3.3.10", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-3.3.10.tgz", - "integrity": "sha512-Pgs951kaMm5GXP7MOvxERINe3gsaVjUWFm+UZPSq9xYriQAksyhg0csnS0KXSNRD5NmNdapXEpjxG49+AKh/ug==", + "version": "5.1.4", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.4.tgz", + "integrity": "sha512-MzbUSahkTW1u7JpKKjY7LCARd1fU5W2rLdxlM4kdkayuCwZImjkpluF9CM1aLewYJguPDqewLam18Y6AU69A8A==", "dev": true }, "ignore-by-default": { @@ -2294,13 +1682,13 @@ "dev": true }, "import-local": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/import-local/-/import-local-1.0.0.tgz", - "integrity": "sha512-vAaZHieK9qjGo58agRBg+bhHX3hoTZU/Oa3GESWLz7t1U62fk63aHuDJJEteXoDeTCcPmUT+z38gkHPZkkmpmQ==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.0.2.tgz", + "integrity": "sha512-vjL3+w0oulAVZ0hBHnxa/Nm5TAurf9YLQJDhqRZyqb+VKGOB6LU8t9H1Nr5CIo16vh9XfJTOoHwU0B71S557gA==", "dev": true, "requires": { - "pkg-dir": "^2.0.0", - "resolve-cwd": "^2.0.0" + "pkg-dir": "^4.2.0", + "resolve-cwd": "^3.0.0" } }, "imurmurhash": { @@ -2310,9 +1698,9 @@ "dev": true }, "indent-string": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-3.2.0.tgz", - "integrity": "sha1-Sl/W0nzDMvN+VBmlBNu4NxBckok=", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", + "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", "dev": true }, "inflight": { @@ -2342,25 +1730,11 @@ "integrity": "sha512-Y75zBYLkh0lJ9qxeHlMjQ7bSbyiSqNW/UOPWDmzC7cXskL1hekSITh1Oc6JV0XCWWZ9DE8VYSB71xocLk3gmGw==", "dev": true }, - "is-accessor-descriptor": { - "version": "0.1.6", - "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", - "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=", - "dev": true, - "requires": { - "kind-of": "^3.0.2" - }, - "dependencies": { - "kind-of": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", - "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", - "dev": true, - "requires": { - "is-buffer": "^1.1.5" - } - } - } + "is-arguments": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.0.4.tgz", + "integrity": "sha512-xPh0Rmt8NE65sNzvyUmWgI1tz3mKq74lGA0mL8LYZcoIzKOzDh6HmrYm3d18k60nHerC8A9Km8kYu87zfSFnLA==", + "dev": true }, "is-arrayish": { "version": "0.2.1", @@ -2369,87 +1743,33 @@ "dev": true }, "is-binary-path": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-1.0.1.tgz", - "integrity": "sha1-dfFmQrSA8YenEcgUFh/TpKdlWJg=", - "dev": true, - "requires": { - "binary-extensions": "^1.0.0" - } - }, - "is-buffer": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", - "dev": true - }, - "is-builtin-module": { - "version": "1.0.0", - "resolved": "http://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz", - "integrity": "sha1-VAVy0096wxGfj3bDDLwbHgN6/74=", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", "dev": true, "requires": { - "builtin-modules": "^1.0.0" + "binary-extensions": "^2.0.0" } }, "is-ci": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-1.2.1.tgz", - "integrity": "sha512-s6tfsaQaQi3JNciBH6shVqEDvhGut0SUXr31ag8Pd8BBbVVlcGfWhpPmEOoM6RJ5TFhbypvf5yyRw/VXW1IiWg==", - "dev": true, - "requires": { - "ci-info": "^1.5.0" - } - }, - "is-data-descriptor": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", - "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=", - "dev": true, - "requires": { - "kind-of": "^3.0.2" - }, - "dependencies": { - "kind-of": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", - "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", - "dev": true, - "requires": { - "is-buffer": "^1.1.5" - } - } - } - }, - "is-descriptor": { - "version": "0.1.6", - "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", - "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz", + "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==", "dev": true, "requires": { - "is-accessor-descriptor": "^0.1.6", - "is-data-descriptor": "^0.1.4", - "kind-of": "^5.0.0" - }, - "dependencies": { - "kind-of": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", - "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", - "dev": true - } + "ci-info": "^2.0.0" } }, - "is-error": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/is-error/-/is-error-2.2.1.tgz", - "integrity": "sha1-aEqW2EB2V3yY9M20DG0mpRI78Zw=", + "is-date-object": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz", + "integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY=", "dev": true }, - "is-extendable": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", - "integrity": "sha1-YrEQ4omkcUGOPsNqYX1HLjAd/Ik=", + "is-error": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/is-error/-/is-error-2.2.2.tgz", + "integrity": "sha512-IOQqts/aHWbiisY5DuPJQ0gcbvaLFCa7fBa9xoLfxBZvQ+ZI/Zh9xoI7Gk+G64N0FdK4AbibytHht2tWgpJWLg==", "dev": true }, "is-extglob": { @@ -2465,9 +1785,9 @@ "dev": true }, "is-glob": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.0.tgz", - "integrity": "sha1-lSHHaEXMJhCoUgPd8ICpWML/q8A=", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz", + "integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==", "dev": true, "requires": { "is-extglob": "^2.1.1" @@ -2481,71 +1801,65 @@ "requires": { "global-dirs": "^0.1.0", "is-path-inside": "^1.0.0" - } - }, - "is-npm": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-npm/-/is-npm-1.0.0.tgz", - "integrity": "sha1-8vtjpl5JBbQGyGBydloaTceTufQ=", - "dev": true - }, - "is-number": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz", - "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", - "dev": true, - "requires": { - "kind-of": "^3.0.2" }, "dependencies": { - "kind-of": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", - "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", + "is-path-inside": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-1.0.1.tgz", + "integrity": "sha1-jvW33lBDej/cprToZe96pVy0gDY=", "dev": true, "requires": { - "is-buffer": "^1.1.5" + "path-is-inside": "^1.0.1" } } } }, + "is-npm": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-npm/-/is-npm-3.0.0.tgz", + "integrity": "sha512-wsigDr1Kkschp2opC4G3yA6r9EgVA6NjRpWzIi9axXqeIaAATPRJc4uLujXe3Nd9uO8KoDyA4MD6aZSeXTADhA==", + "dev": true + }, + "is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true + }, "is-obj": { - "version": "1.0.1", - "resolved": "http://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", - "integrity": "sha1-PkcprB9f3gJc19g6iW2rn09n2w8=", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", + "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", "dev": true }, "is-observable": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-observable/-/is-observable-1.1.0.tgz", - "integrity": "sha512-NqCa4Sa2d+u7BWc6CukaObG3Fh+CU9bvixbpcXYhy2VvYS7vVGIdAgnIS5Ks3A/cqk4rebLJ9s8zBstT2aKnIA==", - "dev": true, - "requires": { - "symbol-observable": "^1.1.0" - } + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-observable/-/is-observable-2.0.0.tgz", + "integrity": "sha512-fhBZv3eFKUbyHXZ1oHujdo2tZ+CNbdpdzzlENgCGZUC8keoGxUew2jYFLYcUB4qo7LDD03o4KK11m/QYD7kEjg==", + "dev": true }, "is-path-cwd": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-1.0.0.tgz", - "integrity": "sha1-0iXsIxMuie3Tj9p2dHLmLmXxEG0=", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-2.2.0.tgz", + "integrity": "sha512-w942bTcih8fdJPJmQHFzkS76NEP8Kzzvmw92cXsazb8intwLqPibPPdXf4ANdKV3rYMuuQYGIWtvz9JilB3NFQ==", "dev": true }, "is-path-in-cwd": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-path-in-cwd/-/is-path-in-cwd-1.0.1.tgz", - "integrity": "sha512-FjV1RTW48E7CWM7eE/J2NJvAEEVektecDBVBE5Hh3nM1Jd0kvhHtX68Pr3xsDf857xt3Y4AkwVULK1Vku62aaQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-path-in-cwd/-/is-path-in-cwd-2.1.0.tgz", + "integrity": "sha512-rNocXHgipO+rvnP6dk3zI20RpOtrAM/kzbB258Uw5BWr3TpXi861yzjo16Dn4hUox07iw5AyeMLHWsujkjzvRQ==", "dev": true, "requires": { - "is-path-inside": "^1.0.0" + "is-path-inside": "^2.1.0" } }, "is-path-inside": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-1.0.1.tgz", - "integrity": "sha1-jvW33lBDej/cprToZe96pVy0gDY=", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-2.1.0.tgz", + "integrity": "sha512-wiyhTzfDWsvwAW53OBWF5zuvaOGlZ6PwYxAbPVDhpm+gM09xKQGjBq/8uYN12aDvMxnAnq3dxTyoSoRNmg5YFg==", "dev": true, "requires": { - "path-is-inside": "^1.0.1" + "path-is-inside": "^1.0.2" } }, "is-plain-obj": { @@ -2555,12 +1869,12 @@ "dev": true }, "is-plain-object": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", - "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-3.0.0.tgz", + "integrity": "sha512-tZIpofR+P05k8Aocp7UI/2UTa9lTJSebCXpFFoR9aibpokDj/uXBsJ8luUu0tTVYKkMU6URDUuOfJZ7koewXvg==", "dev": true, "requires": { - "isobject": "^3.0.1" + "isobject": "^4.0.0" } }, "is-promise": { @@ -2569,17 +1883,14 @@ "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=", "dev": true }, - "is-redirect": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-redirect/-/is-redirect-1.0.0.tgz", - "integrity": "sha1-HQPd7VO9jbDzDCbk+V02/HyH3CQ=", - "dev": true - }, - "is-retry-allowed": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-1.1.0.tgz", - "integrity": "sha1-EaBgVotnM5REAz0BJaYaINVk+zQ=", - "dev": true + "is-regex": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz", + "integrity": "sha1-VRdIm1RwkbCTDglWVM7SXul+lJE=", + "dev": true, + "requires": { + "has": "^1.0.1" + } }, "is-stream": { "version": "1.1.0", @@ -2587,6 +1898,12 @@ "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=", "dev": true }, + "is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=", + "dev": true + }, "is-url": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz", @@ -2599,10 +1916,10 @@ "integrity": "sha1-Sw2hRCEE0bM2NA6AeX6GXPOffXI=", "dev": true }, - "is-windows": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz", - "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==", + "is-yarn-global": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/is-yarn-global/-/is-yarn-global-0.3.0.tgz", + "integrity": "sha512-VjSeb/lHmkoyd8ryPVIKvOCn4D1koMqY+vqyjjUfc3xyKtP4dYOxM44sZrnqQSzSds3xyOrUTLTC9LVCVgLngw==", "dev": true }, "isarray": { @@ -2617,9 +1934,9 @@ "dev": true }, "isobject": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", - "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-4.0.0.tgz", + "integrity": "sha512-S/2fF5wH8SJA/kmwr6HYhK/RI/OkhD84k8ntalo0iJjZikgq1XFvR5M8NPT1x5F7fBwCG3qHfnzeP/Vh/ZxCUA==", "dev": true }, "js-string-escape": { @@ -2635,9 +1952,9 @@ "dev": true }, "js-yaml": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.12.0.tgz", - "integrity": "sha512-PIt2cnwmPfL4hKNwqeiuz4bKfnzHTBv6HyVgjahA6mPLwPDzjDWrplJBMjHUFxku/N3FlmrbyPclad+I+4mJ3A==", + "version": "3.13.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz", + "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==", "dev": true, "requires": { "argparse": "^1.0.7", @@ -2645,9 +1962,15 @@ } }, "jsesc": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.1.tgz", - "integrity": "sha1-5CGiqOINawgZ3yiQj3glJrlt0f4=", + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", + "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", + "dev": true + }, + "json-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.0.tgz", + "integrity": "sha1-Wx85evx11ne96Lz8Dkfh+aPZqJg=", "dev": true }, "json-parse-better-errors": { @@ -2657,24 +1980,30 @@ "dev": true }, "json5": { - "version": "0.5.1", - "resolved": "http://registry.npmjs.org/json5/-/json5-0.5.1.tgz", - "integrity": "sha1-Hq3nrMASA0rYTiOWdn6tn6VJWCE=", - "dev": true + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.1.0.tgz", + "integrity": "sha512-8Mh9h6xViijj36g7Dxi+Y4S6hNGV96vcJZr/SrlHh1LR/pEn/8j/+qIBbs44YKl69Lrfctp4QD+AdWLTMqEZAQ==", + "dev": true, + "requires": { + "minimist": "^1.2.0" + } }, - "kind-of": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz", - "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==", - "dev": true + "keyv": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-3.1.0.tgz", + "integrity": "sha512-9ykJ/46SN/9KPM/sichzQ7OvXyGDYKGTaDlKMGCAlg2UK8KRy4jb0d8sFc+0Tt0YYnThq8X2RZgCg74RPxgcVA==", + "dev": true, + "requires": { + "json-buffer": "3.0.0" + } }, "latest-version": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-3.1.0.tgz", - "integrity": "sha1-ogU4P+oyKzO1rjsYq+4NwvNW7hU=", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-5.1.0.tgz", + "integrity": "sha512-weT+r0kTkRQdCdYCNtkMwWXQTMEswKrFBkm4ckQOMVhhqhIMI1UT2hMj+1iigIhgSZm5gTmrRXBNoGUgaTY1xA==", "dev": true, "requires": { - "package-json": "^4.0.0" + "package-json": "^6.3.0" } }, "load-json-file": { @@ -2687,28 +2016,29 @@ "parse-json": "^4.0.0", "pify": "^3.0.0", "strip-bom": "^3.0.0" + }, + "dependencies": { + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "dev": true + } } }, "locate-path": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", - "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", "dev": true, "requires": { - "p-locate": "^2.0.0", - "path-exists": "^3.0.0" + "p-locate": "^4.1.0" } }, "lodash": { - "version": "4.17.11", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", - "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==", - "dev": true - }, - "lodash.clone": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/lodash.clone/-/lodash.clone-4.5.0.tgz", - "integrity": "sha1-GVhwRQ9aExkkeN9Lw9I9LeoZB7Y=", + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", "dev": true }, "lodash.clonedeep": { @@ -2717,40 +2047,22 @@ "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=", "dev": true }, - "lodash.clonedeepwith": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/lodash.clonedeepwith/-/lodash.clonedeepwith-4.5.0.tgz", - "integrity": "sha1-buMFc6A6GmDWcKYu8zwQzxr9vdQ=", - "dev": true - }, - "lodash.debounce": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", - "integrity": "sha1-gteb/zCmfEAF/9XiUVMArZyk168=", - "dev": true - }, - "lodash.difference": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", - "integrity": "sha1-nMtOUF1Ia5FlE0V3KIWi3yf9AXw=", - "dev": true - }, - "lodash.flatten": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", - "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=", - "dev": true - }, "lodash.flattendeep": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz", "integrity": "sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=", "dev": true }, + "lodash.islength": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/lodash.islength/-/lodash.islength-4.0.1.tgz", + "integrity": "sha1-Tpho1FJXXXUK/9NYyXlUPcIO1Xc=", + "dev": true + }, "lodash.merge": { - "version": "4.6.1", - "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.1.tgz", - "integrity": "sha512-AOYza4+Hf5z1/0Hztxpm2/xiPZgi/cjMqdnKTUWTBSKchJlxXXuUSxCCl8rJlf4g6yww/j6mA8nC8Hw/EZWxKQ==", + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", "dev": true }, "log-symbols": { @@ -2763,13 +2075,13 @@ } }, "loud-rejection": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/loud-rejection/-/loud-rejection-1.6.0.tgz", - "integrity": "sha1-W0b4AUft7leIcPCG0Eghz5mOVR8=", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/loud-rejection/-/loud-rejection-2.1.0.tgz", + "integrity": "sha512-g/6MQxUXYHeVqZ4PGpPL1fS1fOvlXoi7bay0pizmjAd/3JhyXwxzwrnr74yzdmhuerlslbRJ3x7IOXzFz0cE5w==", "dev": true, "requires": { "currently-unhandled": "^0.4.1", - "signal-exit": "^3.0.0" + "signal-exit": "^3.0.2" } }, "lowercase-keys": { @@ -2779,9 +2091,9 @@ "dev": true }, "lru-cache": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.3.tgz", - "integrity": "sha512-fFEhvcgzuIoJVUF8fYr5KR0YqxD238zgObTps31YdADwPPAp82a4M8TrckkWyx7ekNlf9aBcVn81cFwwXngrJA==", + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", + "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", "dev": true, "requires": { "pseudomap": "^1.0.2", @@ -2789,51 +2101,44 @@ } }, "make-dir": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz", - "integrity": "sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.0.0.tgz", + "integrity": "sha512-grNJDhb8b1Jm1qeqW5R/O63wUo4UXo2v2HMic6YT9i/HBlF93S8jkMgH7yugvY9ABDShH4VZMn8I+U8+fCNegw==", "dev": true, "requires": { - "pify": "^3.0.0" + "semver": "^6.0.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + } } }, - "map-cache": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/map-cache/-/map-cache-0.2.2.tgz", - "integrity": "sha1-wyq9C9ZSXZsFFkW7TyasXcmKDb8=", - "dev": true - }, "map-obj": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-2.0.0.tgz", "integrity": "sha1-plzSkIepJZi4eRJXpSPgISIqwfk=", "dev": true }, - "map-visit": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/map-visit/-/map-visit-1.0.0.tgz", - "integrity": "sha1-7Nyo8TFE5mDxtb1B8S80edmN+48=", - "dev": true, - "requires": { - "object-visit": "^1.0.0" - } - }, "matcher": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/matcher/-/matcher-1.1.1.tgz", - "integrity": "sha512-+BmqxWIubKTRKNWx/ahnCkk3mG8m7OturVlqq6HiojGJTd5hVYbgZm6WzcYPCoB+KBT4Vd6R7WSRG2OADNaCjg==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-2.0.0.tgz", + "integrity": "sha512-nlmfSlgHBFx36j/Pl/KQPbIaqE8Zf0TqmSMjsuddHDg6PMSVgmyW9HpkLs0o0M1n2GIZ/S2BZBLIww/xjhiGng==", "dev": true, "requires": { - "escape-string-regexp": "^1.0.4" + "escape-string-regexp": "^2.0.0" } }, "md5-hex": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/md5-hex/-/md5-hex-2.0.0.tgz", - "integrity": "sha1-0FiOnxx0lUSS7NJKwKxs6ZfZLjM=", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/md5-hex/-/md5-hex-3.0.1.tgz", + "integrity": "sha512-BUiRtTtV39LIJwinWBjqVsU9xhdnz7/i889V859IBFpuqGAj6LuOvHv5XLbgZ2R7ptJoJaEcxkv88/h25T7Ciw==", "dev": true, "requires": { - "md5-o-matic": "^0.1.1" + "blueimp-md5": "^2.10.0" } }, "md5-o-matic": { @@ -2857,33 +2162,46 @@ "redent": "^2.0.0", "trim-newlines": "^2.0.0", "yargs-parser": "^10.0.0" + }, + "dependencies": { + "loud-rejection": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/loud-rejection/-/loud-rejection-1.6.0.tgz", + "integrity": "sha1-W0b4AUft7leIcPCG0Eghz5mOVR8=", + "dev": true, + "requires": { + "currently-unhandled": "^0.4.1", + "signal-exit": "^3.0.0" + } + } } }, + "merge2": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.2.4.tgz", + "integrity": "sha512-FYE8xI+6pjFOhokZu0We3S5NKCirLbCzSh2Usf3qEyr4X8U+0jNg9P8RZ4qz+V2UoECLVwSyzU3LxXBaLGtD3A==", + "dev": true + }, "micromatch": { - "version": "3.1.10", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz", - "integrity": "sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==", - "dev": true, - "requires": { - "arr-diff": "^4.0.0", - "array-unique": "^0.3.2", - "braces": "^2.3.1", - "define-property": "^2.0.2", - "extend-shallow": "^3.0.2", - "extglob": "^2.0.4", - "fragment-cache": "^0.2.1", - "kind-of": "^6.0.2", - "nanomatch": "^1.2.9", - "object.pick": "^1.3.0", - "regex-not": "^1.0.0", - "snapdragon": "^0.8.1", - "to-regex": "^3.0.2" + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.2.tgz", + "integrity": "sha512-y7FpHSbMUMoyPbYUSzO6PaZ6FyRnQOpHuKwbo1G+Knck95XVU4QAiKdGEnj5wwoS7PlOgthX/09u5iFJ+aYf5Q==", + "dev": true, + "requires": { + "braces": "^3.0.1", + "picomatch": "^2.0.5" } }, "mimic-fn": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", - "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true + }, + "mimic-response": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", + "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==", "dev": true }, "minimatch": { @@ -2896,9 +2214,9 @@ } }, "minimist": { - "version": "0.0.8", - "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", "dev": true }, "minimist-options": { @@ -2909,112 +2227,80 @@ "requires": { "arrify": "^1.0.1", "is-plain-obj": "^1.1.0" - } - }, - "mixin-deep": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.1.tgz", - "integrity": "sha512-8ZItLHeEgaqEvd5lYBXfm4EZSFCX29Jb9K+lAHhDKzReKBQKj3R+7NOF6tjqYi9t4oI8VUfaWITJQm86wnXGNQ==", - "dev": true, - "requires": { - "for-in": "^1.0.2", - "is-extendable": "^1.0.1" }, "dependencies": { - "is-extendable": { + "arrify": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", - "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", - "dev": true, - "requires": { - "is-plain-object": "^2.0.4" - } + "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", + "integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=", + "dev": true } } }, "mkdirp": { "version": "0.5.1", - "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", "dev": true, "requires": { "minimist": "0.0.8" + }, + "dependencies": { + "minimist": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", + "dev": true + } } }, "ms": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", - "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "dev": true }, - "multimatch": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/multimatch/-/multimatch-2.1.0.tgz", - "integrity": "sha1-nHkGoi+0wCkZ4vX3UWG0zb1LKis=", - "dev": true, - "requires": { - "array-differ": "^1.0.0", - "array-union": "^1.0.1", - "arrify": "^1.0.0", - "minimatch": "^3.0.0" - } - }, "mysql": { - "version": "github:mysqljs/mysql#cf5d1e396a343ffa0fba23b3791d2dd5da20f30e", - "from": "github:mysqljs/mysql", + "version": "2.17.1", + "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.17.1.tgz", + "integrity": "sha512-7vMqHQ673SAk5C8fOzTG2LpPcf3bNt0oL3sFpxPEEFp1mdlDcrLK0On7z8ZYKaaHrHwNcQ/MTUz7/oobZ2OyyA==", "requires": { - "bignumber.js": "4.1.0", + "bignumber.js": "7.2.1", "readable-stream": "2.3.6", "safe-buffer": "5.1.2", "sqlstring": "2.3.1" - } - }, - "nan": { - "version": "2.11.1", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.11.1.tgz", - "integrity": "sha512-iji6k87OSXa0CcrLl9z+ZiYSuR2o+c0bGuNmXdrhTQTakxytAFsC56SArGYoiHlJlFoHSnvmhpceZJaXkVuOtA==", - "dev": true, - "optional": true - }, - "nanomatch": { - "version": "1.2.13", - "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", - "integrity": "sha512-fpoe2T0RbHwBTBUOftAfBPaDEi06ufaUai0mE6Yn1kacc3SnTErfb/h+X94VXzI64rKFHYImXSvdwGGCmwOqCA==", - "dev": true, - "requires": { - "arr-diff": "^4.0.0", - "array-unique": "^0.3.2", - "define-property": "^2.0.2", - "extend-shallow": "^3.0.2", - "fragment-cache": "^0.2.1", - "is-windows": "^1.0.2", - "kind-of": "^6.0.2", - "object.pick": "^1.3.0", - "regex-not": "^1.0.0", - "snapdragon": "^0.8.1", - "to-regex": "^3.0.1" + }, + "dependencies": { + "bignumber.js": { + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz", + "integrity": "sha512-S4XzBk5sMB+Rcb/LNcpzXr57VRTxgAvaAEDAl1AwRx27j00hT84O6OkteE7u8UB3NuaaygCRrEpqox4uDOrbdQ==" + } } }, "normalize-package-data": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.4.0.tgz", - "integrity": "sha512-9jjUFbTPfEy3R/ad/2oNbKtW9Hgovl5O1FvFWKkKblNXoN/Oou6+9+KKohPK13Yc3/TyunyWhJp6gvRNR/PPAw==", + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", + "integrity": "sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==", "dev": true, "requires": { "hosted-git-info": "^2.1.4", - "is-builtin-module": "^1.0.0", + "resolve": "^1.10.0", "semver": "2 || 3 || 4 || 5", "validate-npm-package-license": "^3.0.1" } }, "normalize-path": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-2.1.1.tgz", - "integrity": "sha1-GrKLVW4Zg2Oowab35vogE3/mrtk=", - "dev": true, - "requires": { - "remove-trailing-separator": "^1.0.1" - } + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true + }, + "normalize-url": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.3.0.tgz", + "integrity": "sha512-0NLtR71o4k6GLP+mr6Ty34c5GA6CMoEsncKJxvQd8NzPxaHRJNnb5gZE8R1XF4CPIS7QPHLJ74IFszwtNVAHVQ==", + "dev": true }, "npm-run-path": { "version": "2.0.2", @@ -3031,82 +2317,38 @@ "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", "dev": true }, - "object-copy": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/object-copy/-/object-copy-0.1.0.tgz", - "integrity": "sha1-fn2Fi3gb18mRpBupde04EnVOmYw=", - "dev": true, - "requires": { - "copy-descriptor": "^0.1.0", - "define-property": "^0.2.5", - "kind-of": "^3.0.3" - }, - "dependencies": { - "define-property": { - "version": "0.2.5", - "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", - "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=", - "dev": true, - "requires": { - "is-descriptor": "^0.1.0" - } - }, - "kind-of": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", - "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", - "dev": true, - "requires": { - "is-buffer": "^1.1.5" - } - } - } - }, - "object-visit": { + "object-is": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/object-visit/-/object-visit-1.0.1.tgz", - "integrity": "sha1-95xEk68MU3e1n+OdOV5BBC3QRbs=", - "dev": true, - "requires": { - "isobject": "^3.0.0" - } + "resolved": "https://registry.npmjs.org/object-is/-/object-is-1.0.1.tgz", + "integrity": "sha1-CqYOyZiaCz7Xlc9NBvYs8a1lObY=", + "dev": true + }, + "object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "dev": true }, - "object.pick": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/object.pick/-/object.pick-1.3.0.tgz", - "integrity": "sha1-h6EKxMFpS9Lhy/U1kaZhQftd10c=", + "object.assign": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.0.tgz", + "integrity": "sha512-exHJeq6kBKj58mqGyTQ9DFvrZC/eR6OwxzoM9YRoGBqrXYonaFyGiFMuc9VZrXf7DarreEwMpurG3dd+CNyW5w==", "dev": true, "requires": { - "isobject": "^3.0.1" + "define-properties": "^1.1.2", + "function-bind": "^1.1.1", + "has-symbols": "^1.0.0", + "object-keys": "^1.0.11" } }, "observable-to-promise": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/observable-to-promise/-/observable-to-promise-0.5.0.tgz", - "integrity": "sha1-yCjw8NxH6fhq+KSXfF1VB2znqR8=", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/observable-to-promise/-/observable-to-promise-1.0.0.tgz", + "integrity": "sha512-cqnGUrNsE6vdVDTPAX9/WeVzwy/z37vdxupdQXU8vgTXRFH72KCZiZga8aca2ulRPIeem8W3vW9rQHBwfIl2WA==", "dev": true, "requires": { - "is-observable": "^0.2.0", + "is-observable": "^2.0.0", "symbol-observable": "^1.0.4" - }, - "dependencies": { - "is-observable": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/is-observable/-/is-observable-0.2.0.tgz", - "integrity": "sha1-s2ExHYPG5dcmyr9eJQsCNxBvWuI=", - "dev": true, - "requires": { - "symbol-observable": "^0.2.2" - }, - "dependencies": { - "symbol-observable": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-0.2.4.tgz", - "integrity": "sha1-lag9smGG1q9+ehjb2XYKL4bQj0A=", - "dev": true - } - } - } } }, "once": { @@ -3119,26 +2361,62 @@ } }, "onetime": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", - "integrity": "sha1-BnQoIw/WdEOyeUsiu6UotoZ5YtQ=", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.0.tgz", + "integrity": "sha512-5NcSkPHhwTVFIQN+TUqXoS5+dlElHXdpAWu9I0HP20YOtIi+aZ0Ct82jdlILDxjLEAWwvm+qj1m6aEtsDVmm6Q==", "dev": true, "requires": { - "mimic-fn": "^1.0.0" + "mimic-fn": "^2.1.0" } }, "ora": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/ora/-/ora-3.0.0.tgz", - "integrity": "sha512-LBS97LFe2RV6GJmXBi6OKcETKyklHNMV0xw7BtsVn2MlsgsydyZetSCbCANr+PFLmDyv4KV88nn0eCKza665Mg==", + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/ora/-/ora-3.4.0.tgz", + "integrity": "sha512-eNwHudNbO1folBP3JsZ19v9azXWtQZjICdr3Q0TDPIaeBQ3mXLrh54wM+er0+hSp+dWKf+Z8KM58CYzEyIYxYg==", "dev": true, "requires": { - "chalk": "^2.3.1", + "chalk": "^2.4.2", "cli-cursor": "^2.1.0", - "cli-spinners": "^1.1.0", + "cli-spinners": "^2.0.0", "log-symbols": "^2.2.0", - "strip-ansi": "^4.0.0", + "strip-ansi": "^5.2.0", "wcwidth": "^1.0.1" + }, + "dependencies": { + "cli-cursor": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", + "integrity": "sha1-s12sN2R5+sw+lHR9QdDQ9SOP/LU=", + "dev": true, + "requires": { + "restore-cursor": "^2.0.0" + } + }, + "mimic-fn": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", + "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", + "dev": true + }, + "onetime": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", + "integrity": "sha1-BnQoIw/WdEOyeUsiu6UotoZ5YtQ=", + "dev": true, + "requires": { + "mimic-fn": "^1.0.0" + } + }, + "restore-cursor": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", + "integrity": "sha1-n37ih/gv0ybU/RYpI9YhKe7g368=", + "dev": true, + "requires": { + "onetime": "^2.0.0", + "signal-exit": "^3.0.2" + } + } } }, "os-tmpdir": { @@ -3147,6 +2425,12 @@ "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", "dev": true }, + "p-cancelable": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-1.1.0.tgz", + "integrity": "sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw==", + "dev": true + }, "p-finally": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", @@ -3154,57 +2438,65 @@ "dev": true }, "p-limit": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", - "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.1.tgz", + "integrity": "sha512-85Tk+90UCVWvbDavCLKPOLC9vvY8OwEX/RtKF+/1OADJMVlFfEHOiMTPVyxg7mk/dKa+ipdHm0OUkTvCpMTuwg==", "dev": true, "requires": { - "p-try": "^1.0.0" + "p-try": "^2.0.0" } }, "p-locate": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", - "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", "dev": true, "requires": { - "p-limit": "^1.1.0" + "p-limit": "^2.2.0" } }, "p-map": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-1.2.0.tgz", - "integrity": "sha512-r6zKACMNhjPJMTl8KcFH4li//gkrXWfbD6feV8l6doRHlzljFWGJ2AP6iKaCJXyZmAUMOPtvbW7EXkbWO/pLEA==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", + "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==", "dev": true }, "p-try": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", - "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", "dev": true }, "package-hash": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/package-hash/-/package-hash-2.0.0.tgz", - "integrity": "sha1-eK4ybIngWk2BO2hgGXevBcANKg0=", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/package-hash/-/package-hash-4.0.0.tgz", + "integrity": "sha512-whdkPIooSu/bASggZ96BWVvZTRMOFxnyUG5PnTSGKoJE2gd5mbVNmR2Nj20QFzxYYgAXpoqC+AiXzl+UMRh7zQ==", "dev": true, "requires": { - "graceful-fs": "^4.1.11", + "graceful-fs": "^4.1.15", + "hasha": "^5.0.0", "lodash.flattendeep": "^4.4.0", - "md5-hex": "^2.0.0", "release-zalgo": "^1.0.0" } }, "package-json": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/package-json/-/package-json-4.0.1.tgz", - "integrity": "sha1-iGmgQBJTZhxMTKPabCEh7VVfXu0=", + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/package-json/-/package-json-6.5.0.tgz", + "integrity": "sha512-k3bdm2n25tkyxcjSKzB5x8kfVxlMdgsbPr0GkZcwHsLpba6cBjqCt1KlcChKEvxHIcTB1FVMuwoijZ26xex5MQ==", "dev": true, "requires": { - "got": "^6.7.1", - "registry-auth-token": "^3.0.1", - "registry-url": "^3.0.3", - "semver": "^5.1.0" + "got": "^9.6.0", + "registry-auth-token": "^4.0.0", + "registry-url": "^5.0.0", + "semver": "^6.2.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + } } }, "parse-json": { @@ -3218,27 +2510,15 @@ } }, "parse-ms": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-1.0.1.tgz", - "integrity": "sha1-VjRtR0nXjyNDDKDHE4UK75GqNh0=", - "dev": true - }, - "pascalcase": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/pascalcase/-/pascalcase-0.1.1.tgz", - "integrity": "sha1-s2PlXoAGym/iF4TS2yK9FdeRfxQ=", - "dev": true - }, - "path-dirname": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/path-dirname/-/path-dirname-1.0.2.tgz", - "integrity": "sha1-zDPSTVJeCZpTiMAzbG4yuRYGCeA=", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz", + "integrity": "sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA==", "dev": true }, "path-exists": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", - "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", "dev": true }, "path-is-absolute": { @@ -3266,18 +2546,21 @@ "dev": true }, "path-type": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz", - "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==", - "dev": true, - "requires": { - "pify": "^3.0.0" - } + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "dev": true + }, + "picomatch": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.0.7.tgz", + "integrity": "sha512-oLHIdio3tZ0qH76NybpeneBhYVj0QFTfXEFTc/B3zKQspYfYYkWYgFsmzo+4kvId/bQRcNkVeguI3y+CD22BtA==", + "dev": true }, "pify": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", - "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", "dev": true }, "pinkie": { @@ -3296,58 +2579,107 @@ } }, "pkg-conf": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/pkg-conf/-/pkg-conf-2.1.0.tgz", - "integrity": "sha1-ISZRTKbyq/69FoWW3xi6V4Z/AFg=", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/pkg-conf/-/pkg-conf-3.1.0.tgz", + "integrity": "sha512-m0OTbR/5VPNPqO1ph6Fqbj7Hv6QU7gR/tQW40ZqrL1rjgCU85W6C1bJn0BItuJqnR98PWzw7Z8hHeChD1WrgdQ==", "dev": true, "requires": { - "find-up": "^2.0.0", - "load-json-file": "^4.0.0" + "find-up": "^3.0.0", + "load-json-file": "^5.2.0" + }, + "dependencies": { + "find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, + "requires": { + "locate-path": "^3.0.0" + } + }, + "load-json-file": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-5.3.0.tgz", + "integrity": "sha512-cJGP40Jc/VXUsp8/OrnyKyTZ1y6v/dphm3bioS+RrKXjK2BB6wHUd6JptZEFDGgGahMT+InnZO5i1Ei9mpC8Bw==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.15", + "parse-json": "^4.0.0", + "pify": "^4.0.1", + "strip-bom": "^3.0.0", + "type-fest": "^0.3.0" + } + }, + "locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dev": true, + "requires": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + } + }, + "p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, + "requires": { + "p-limit": "^2.0.0" + } + }, + "path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "dev": true + }, + "type-fest": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.3.1.tgz", + "integrity": "sha512-cUGJnCdr4STbePCgqNFbpVNCepa+kAVohJs1sLhxzdH+gnEoOd8VhbYa7pD3zZYGiURWM2xzEII3fQcRizDkYQ==", + "dev": true + } } }, "pkg-dir": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz", - "integrity": "sha1-9tXREJ4Z1j7fQo4L1X4Sd3YVM0s=", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", "dev": true, "requires": { - "find-up": "^2.1.0" + "find-up": "^4.0.0" } }, "plur": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/plur/-/plur-3.0.1.tgz", - "integrity": "sha512-lJl0ojUynAM1BZn58Pas2WT/TXeC1+bS+UqShl0x9+49AtOn7DixRXVzaC8qrDOIxNDmepKnLuMTH7NQmkX0PA==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/plur/-/plur-3.1.1.tgz", + "integrity": "sha512-t1Ax8KUvV3FFII8ltczPn2tJdjqbd1sIzu6t4JL7nQ3EyeL/lTrj5PWKb06ic5/6XYDr65rQ4uzQEGN70/6X5w==", "dev": true, "requires": { "irregular-plurals": "^2.0.0" } }, - "posix-character-classes": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/posix-character-classes/-/posix-character-classes-0.1.1.tgz", - "integrity": "sha1-AerA/jta9xoqbAL+q7jB/vfgDqs=", - "dev": true - }, "prepend-http": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-1.0.4.tgz", - "integrity": "sha1-1PRWKwzjaW5BrFLQ4ALlemNdxtw=", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz", + "integrity": "sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc=", "dev": true }, "pretty-ms": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-3.2.0.tgz", - "integrity": "sha512-ZypexbfVUGTFxb0v+m1bUyy92DHe5SyYlnyY0msyms5zd3RwyvNgyxZZsXXgoyzlxjx5MiqtXUdhUfvQbe0A2Q==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-5.0.0.tgz", + "integrity": "sha512-94VRYjL9k33RzfKiGokPBPpsmloBYSf5Ri+Pq19zlsEcUKFob+admeXr5eFDRuPjFmEOcjJvPGdillYOJyvZ7Q==", "dev": true, "requires": { - "parse-ms": "^1.0.0" + "parse-ms": "^2.1.0" } }, "process-nextick-args": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", - "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, "pseudomap": { "version": "1.0.2", @@ -3355,6 +2687,16 @@ "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=", "dev": true }, + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dev": true, + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "quick-lru": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-1.1.0.tgz", @@ -3371,14 +2713,6 @@ "ini": "~1.3.0", "minimist": "^1.2.0", "strip-json-comments": "~2.0.1" - }, - "dependencies": { - "minimist": { - "version": "1.2.0", - "resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", - "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", - "dev": true - } } }, "read-pkg": { @@ -3390,6 +2724,23 @@ "load-json-file": "^4.0.0", "normalize-package-data": "^2.3.2", "path-type": "^3.0.0" + }, + "dependencies": { + "path-type": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz", + "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==", + "dev": true, + "requires": { + "pify": "^3.0.0" + } + }, + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "dev": true + } } }, "read-pkg-up": { @@ -3400,11 +2751,62 @@ "requires": { "find-up": "^2.0.0", "read-pkg": "^3.0.0" + }, + "dependencies": { + "find-up": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", + "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", + "dev": true, + "requires": { + "locate-path": "^2.0.0" + } + }, + "locate-path": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", + "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", + "dev": true, + "requires": { + "p-locate": "^2.0.0", + "path-exists": "^3.0.0" + } + }, + "p-limit": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", + "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "dev": true, + "requires": { + "p-try": "^1.0.0" + } + }, + "p-locate": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", + "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", + "dev": true, + "requires": { + "p-limit": "^1.1.0" + } + }, + "p-try": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", + "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", + "dev": true + }, + "path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "dev": true + } } }, "readable-stream": { "version": "2.3.6", - "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "requires": { "core-util-is": "~1.0.0", @@ -3417,14 +2819,12 @@ } }, "readdirp": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-2.2.1.tgz", - "integrity": "sha512-1JU/8q+VgFZyxwrJ+SVIOsh+KywWGpds3NTqikiKpDMZWScmAYyKIgqkO+ARvNWJfXeXR1zxz7aHF4u4CyH6vQ==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.1.2.tgz", + "integrity": "sha512-8rhl0xs2cxfVsqzreYCvs8EwBfn/DhVdqtoLmw19uI3SC5avYX9teCurlErfpPXGmYtMHReGaP2RsLnFvz/lnw==", "dev": true, "requires": { - "graceful-fs": "^4.1.11", - "micromatch": "^3.1.10", - "readable-stream": "^2.0.2" + "picomatch": "^2.0.4" } }, "redent": { @@ -3435,6 +2835,14 @@ "requires": { "indent-string": "^3.0.0", "strip-indent": "^2.0.0" + }, + "dependencies": { + "indent-string": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-3.2.0.tgz", + "integrity": "sha1-Sl/W0nzDMvN+VBmlBNu4NxBckok=", + "dev": true + } } }, "regenerate": { @@ -3444,67 +2852,66 @@ "dev": true }, "regenerate-unicode-properties": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/regenerate-unicode-properties/-/regenerate-unicode-properties-7.0.0.tgz", - "integrity": "sha512-s5NGghCE4itSlUS+0WUj88G6cfMVMmH8boTPNvABf8od+2dhT9WDlWu8n01raQAJZMOK8Ch6jSexaRO7swd6aw==", + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/regenerate-unicode-properties/-/regenerate-unicode-properties-8.1.0.tgz", + "integrity": "sha512-LGZzkgtLY79GeXLm8Dp0BVLdQlWICzBnJz/ipWUgo59qBaZ+BHtq51P2q1uVZlppMuUAT37SDk39qUbjTWB7bA==", "dev": true, "requires": { "regenerate": "^1.4.0" } }, - "regex-not": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz", - "integrity": "sha512-J6SDjUgDxQj5NusnOtdFxDwN/+HWykR8GELwctJ7mdqhcyy1xEc4SRFHUXvxTp661YaVKAjfRLZ9cCqS6tn32A==", + "regexp.prototype.flags": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.2.0.tgz", + "integrity": "sha512-ztaw4M1VqgMwl9HlPpOuiYgItcHlunW0He2fE6eNfT6E/CF2FtYi9ofOYe4mKntstYk0Fyh/rDRBdS3AnxjlrA==", "dev": true, "requires": { - "extend-shallow": "^3.0.2", - "safe-regex": "^1.1.0" + "define-properties": "^1.1.2" } }, "regexpu-core": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-4.2.0.tgz", - "integrity": "sha512-Z835VSnJJ46CNBttalHD/dB+Sj2ezmY6Xp38npwU87peK6mqOzOpV8eYktdkLTEkzzD+JsTcxd84ozd8I14+rw==", + "version": "4.5.5", + "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-4.5.5.tgz", + "integrity": "sha512-FpI67+ky9J+cDizQUJlIlNZFKual/lUkFr1AG6zOCpwZ9cLrg8UUVakyUQJD7fCDIe9Z2nwTQJNPyonatNmDFQ==", "dev": true, "requires": { "regenerate": "^1.4.0", - "regenerate-unicode-properties": "^7.0.0", - "regjsgen": "^0.4.0", - "regjsparser": "^0.3.0", + "regenerate-unicode-properties": "^8.1.0", + "regjsgen": "^0.5.0", + "regjsparser": "^0.6.0", "unicode-match-property-ecmascript": "^1.0.4", - "unicode-match-property-value-ecmascript": "^1.0.2" + "unicode-match-property-value-ecmascript": "^1.1.0" } }, "registry-auth-token": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-3.3.2.tgz", - "integrity": "sha512-JL39c60XlzCVgNrO+qq68FoNb56w/m7JYvGR2jT5iR1xBrUA3Mfx5Twk5rqTThPmQKMWydGmq8oFtDlxfrmxnQ==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-4.0.0.tgz", + "integrity": "sha512-lpQkHxd9UL6tb3k/aHAVfnVtn+Bcs9ob5InuFLLEDqSqeq+AljB8GZW9xY0x7F+xYwEcjKe07nyoxzEYz6yvkw==", "dev": true, "requires": { - "rc": "^1.1.6", + "rc": "^1.2.8", "safe-buffer": "^5.0.1" } }, "registry-url": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/registry-url/-/registry-url-3.1.0.tgz", - "integrity": "sha1-PU74cPc93h138M+aOBQyRE4XSUI=", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/registry-url/-/registry-url-5.1.0.tgz", + "integrity": "sha512-8acYXXTI0AkQv6RAOjE3vOaIXZkT9wo4LOFbBKYQEEnnMNBpKqdUrI6S4NT0KPIo/WVvJ5tE/X5LF/TQUf0ekw==", "dev": true, "requires": { - "rc": "^1.0.1" + "rc": "^1.2.8" } }, "regjsgen": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/regjsgen/-/regjsgen-0.4.0.tgz", - "integrity": "sha512-X51Lte1gCYUdlwhF28+2YMO0U6WeN0GLpgpA7LK7mbdDnkQYiwvEpmpe0F/cv5L14EbxgrdayAG3JETBv0dbXA==", + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/regjsgen/-/regjsgen-0.5.0.tgz", + "integrity": "sha512-RnIrLhrXCX5ow/E5/Mh2O4e/oa1/jW0eaBKTSy3LaCj+M3Bqvm97GWDp2yUtzIs4LEn65zR2yiYGFqb2ApnzDA==", "dev": true }, "regjsparser": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/regjsparser/-/regjsparser-0.3.0.tgz", - "integrity": "sha512-zza72oZBBHzt64G7DxdqrOo/30bhHkwMUoT0WqfGu98XLd7N+1tsy5MJ96Bk4MD0y74n629RhmrGW6XlnLLwCA==", + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/regjsparser/-/regjsparser-0.6.0.tgz", + "integrity": "sha512-RQ7YyokLiQBomUJuUG8iGVvkgOLxwyZM8k6d3q5SAXpg4r5TZJZigKFvC6PpD+qQ98bCDC5YelPeA3EucDoNeQ==", "dev": true, "requires": { "jsesc": "~0.5.0" @@ -3527,24 +2934,6 @@ "es6-error": "^4.0.1" } }, - "remove-trailing-separator": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz", - "integrity": "sha1-wkvOKig62tW8P1jg1IJJuSN52O8=", - "dev": true - }, - "repeat-element": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/repeat-element/-/repeat-element-1.1.3.tgz", - "integrity": "sha512-ahGq0ZnV5m5XtZLMb+vP76kcAM5nkLqk0lpqAuojSKGgQtn4eRi4ZZGm2olo2zKFH+sMsWaqOCW1dqAnOru72g==", - "dev": true - }, - "repeat-string": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", - "integrity": "sha1-jcrkcOHIirwtYA//Sndihtp15jc=", - "dev": true - }, "require-precompiled": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/require-precompiled/-/require-precompiled-0.1.0.tgz", @@ -3552,78 +2941,78 @@ "dev": true }, "resolve": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.8.1.tgz", - "integrity": "sha512-AicPrAC7Qu1JxPCZ9ZgCZlY35QgFnNqc+0LtbRNxnVw4TXvjQ72wnuL9JQcEBgXkI9JM8MsT9kaQoHcpCRJOYA==", + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.12.0.tgz", + "integrity": "sha512-B/dOmuoAik5bKcD6s6nXDCjzUKnaDvdkRyAk6rsmsKLipWj4797iothd7jmmUhWTfinVMU+wc56rYKsit2Qy4w==", "dev": true, "requires": { - "path-parse": "^1.0.5" + "path-parse": "^1.0.6" } }, "resolve-cwd": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-2.0.0.tgz", - "integrity": "sha1-AKn3OHVW4nA46uIyyqNypqWbZlo=", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", + "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==", "dev": true, "requires": { - "resolve-from": "^3.0.0" + "resolve-from": "^5.0.0" } }, "resolve-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-3.0.0.tgz", - "integrity": "sha1-six699nWiBvItuZTM17rywoYh0g=", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", "dev": true }, - "resolve-url": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz", - "integrity": "sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo=", - "dev": true + "responselike": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/responselike/-/responselike-1.0.2.tgz", + "integrity": "sha1-kYcg7ztjHFZCvgaPFa3lpG9Loec=", + "dev": true, + "requires": { + "lowercase-keys": "^1.0.0" + } }, "restore-cursor": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", - "integrity": "sha1-n37ih/gv0ybU/RYpI9YhKe7g368=", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", + "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", "dev": true, "requires": { - "onetime": "^2.0.0", + "onetime": "^5.1.0", "signal-exit": "^3.0.2" } }, - "ret": { - "version": "0.1.15", - "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz", - "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", + "reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", "dev": true }, "rimraf": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.2.tgz", - "integrity": "sha512-lreewLK/BlghmxtfH36YYVg1i8IAce4TI7oao75I1g245+6BctqTVQiBP3YUJ9C6DQOXJmkYR9X9fCLtCOJc5w==", + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", "dev": true, "requires": { - "glob": "^7.0.5" + "glob": "^7.1.3" } }, + "run-parallel": { + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.1.9.tgz", + "integrity": "sha512-DEqnSRTDw/Tc3FXf49zedI638Z9onwUotBMiUFKmrO2sdFKIbXamXGQ3Axd4qgphxKB4kw/qP1w5kTxnfU1B9Q==", + "dev": true + }, "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" }, - "safe-regex": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", - "integrity": "sha1-QKNmnzsHfR6UPURinhV91IAjvy4=", - "dev": true, - "requires": { - "ret": "~0.1.10" - } - }, "semver": { - "version": "5.5.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.5.1.tgz", - "integrity": "sha512-PqpAxfrEhlSUWge8dwIp4tZnQ25DIOthpiaHNIthsjEFQD6EvqUKUDM7L8O2rShkFccYo1VjJR0coWfNkCubRw==", + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", "dev": true }, "semver-diff": { @@ -3641,29 +3030,6 @@ "integrity": "sha1-ULZ51WNc34Rme9yOWa9OW4HV9go=", "dev": true }, - "set-value": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.0.tgz", - "integrity": "sha512-hw0yxk9GT/Hr5yJEYnHNKYXkIA8mVJgd9ditYZCe16ZczcaELYYcfvaXesNACk2O8O0nTiPQcQhGUQj8JLzeeg==", - "dev": true, - "requires": { - "extend-shallow": "^2.0.1", - "is-extendable": "^0.1.1", - "is-plain-object": "^2.0.3", - "split-string": "^3.0.1" - }, - "dependencies": { - "extend-shallow": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", - "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", - "dev": true, - "requires": { - "is-extendable": "^0.1.0" - } - } - } - }, "shebang-command": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", @@ -3686,144 +3052,29 @@ "dev": true }, "slash": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz", - "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", "dev": true }, "slice-ansi": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-1.0.0.tgz", - "integrity": "sha512-POqxBK6Lb3q6s047D/XsDVNPnF9Dl8JSaqe9h9lURl0OdNqy/ujDrOiIHtsqXMGbWWTIomRzAMaTyawAU//Reg==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-2.1.0.tgz", + "integrity": "sha512-Qu+VC3EwYLldKa1fCxuuvULvSJOKEgk9pi8dZeCVK7TqBfUNTH4sFkk4joj8afVSfAYgJoSOetjx9QWOJ5mYoQ==", "dev": true, "requires": { + "ansi-styles": "^3.2.0", + "astral-regex": "^1.0.0", "is-fullwidth-code-point": "^2.0.0" - } - }, - "slide": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/slide/-/slide-1.1.6.tgz", - "integrity": "sha1-VusCfWW00tzmyy4tMsTUr8nh1wc=", - "dev": true - }, - "snapdragon": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz", - "integrity": "sha512-FtyOnWN/wCHTVXOMwvSv26d+ko5vWlIDD6zoUJ7LW8vh+ZBC8QdljveRP+crNrtBwioEUWy/4dMtbBjA4ioNlg==", - "dev": true, - "requires": { - "base": "^0.11.1", - "debug": "^2.2.0", - "define-property": "^0.2.5", - "extend-shallow": "^2.0.1", - "map-cache": "^0.2.2", - "source-map": "^0.5.6", - "source-map-resolve": "^0.5.0", - "use": "^3.1.0" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "dev": true, - "requires": { - "ms": "2.0.0" - } - }, - "define-property": { - "version": "0.2.5", - "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", - "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=", - "dev": true, - "requires": { - "is-descriptor": "^0.1.0" - } - }, - "extend-shallow": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", - "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", - "dev": true, - "requires": { - "is-extendable": "^0.1.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", - "dev": true - } - } - }, - "snapdragon-node": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/snapdragon-node/-/snapdragon-node-2.1.1.tgz", - "integrity": "sha512-O27l4xaMYt/RSQ5TR3vpWCAB5Kb/czIcqUFOM/C4fYcLnbZUc1PkjTAMjof2pBWaSTwOUd6qUHcFGVGj7aIwnw==", - "dev": true, - "requires": { - "define-property": "^1.0.0", - "isobject": "^3.0.0", - "snapdragon-util": "^3.0.1" - }, - "dependencies": { - "define-property": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", - "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=", - "dev": true, - "requires": { - "is-descriptor": "^1.0.0" - } - }, - "is-accessor-descriptor": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", - "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", - "dev": true, - "requires": { - "kind-of": "^6.0.0" - } - }, - "is-data-descriptor": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", - "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", - "dev": true, - "requires": { - "kind-of": "^6.0.0" - } - }, - "is-descriptor": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", - "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", - "dev": true, - "requires": { - "is-accessor-descriptor": "^1.0.0", - "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.2" - } - } - } - }, - "snapdragon-util": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/snapdragon-util/-/snapdragon-util-3.0.1.tgz", - "integrity": "sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ==", - "dev": true, - "requires": { - "kind-of": "^3.2.0" }, "dependencies": { - "kind-of": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", - "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", "dev": true, "requires": { - "is-buffer": "^1.1.5" + "color-convert": "^1.9.0" } } } @@ -3834,23 +3085,10 @@ "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=", "dev": true }, - "source-map-resolve": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/source-map-resolve/-/source-map-resolve-0.5.2.tgz", - "integrity": "sha512-MjqsvNwyz1s0k81Goz/9vRBe9SZdB09Bdw+/zYyO+3CuPk6fouTaxscHkgtE8jKvf01kVfl8riHzERQ/kefaSA==", - "dev": true, - "requires": { - "atob": "^2.1.1", - "decode-uri-component": "^0.2.0", - "resolve-url": "^0.2.1", - "source-map-url": "^0.4.0", - "urix": "^0.1.0" - } - }, "source-map-support": { - "version": "0.5.9", - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.9.tgz", - "integrity": "sha512-gR6Rw4MvUlYy83vP0vxoVNzM6t8MUXqNuRsuBmBHQDu1Fh6X015FrLdgoDKcNdkwGubozq0P4N0Q37UyFVr1EA==", + "version": "0.5.13", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz", + "integrity": "sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w==", "dev": true, "requires": { "buffer-from": "^1.0.0", @@ -3865,16 +3103,10 @@ } } }, - "source-map-url": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/source-map-url/-/source-map-url-0.4.0.tgz", - "integrity": "sha1-PpNdfd1zYxuXZZlW1VEo6HtQhKM=", - "dev": true - }, "spdx-correct": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.0.1.tgz", - "integrity": "sha512-hxSPZbRZvSDuOvADntOElzJpenIR7wXJkuoUcUtS0erbgt2fgeaoPIYretfKpslMhfFDY4k0MZ2F5CUzhBsSvQ==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.0.tgz", + "integrity": "sha512-lr2EZCctC2BNR7j7WzJ2FpDznxky1sjfxvvYEyzxNyb6lZXHODmEoJeFu4JupYlkfha1KZpJyoqiJ7pgA1qq8Q==", "dev": true, "requires": { "spdx-expression-parse": "^3.0.0", @@ -3898,20 +3130,11 @@ } }, "spdx-license-ids": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.1.tgz", - "integrity": "sha512-TfOfPcYGBB5sDuPn3deByxPhmfegAhpDYKSOXZQN81Oyrrif8ZCodOLzK3AesELnCx03kikhyDwh0pfvvQvF8w==", + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.5.tgz", + "integrity": "sha512-J+FWzZoynJEXGphVIS+XEh3kFSjZX/1i9gFBaWQcB+/tmpe2qUsSBABpcxqxnAxFdiUFEgAX1bjYGQvIZmoz9Q==", "dev": true }, - "split-string": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz", - "integrity": "sha512-NzNVhJDYpwceVVii8/Hu6DKfD2G+NrQHlS/V/qgv763EYudVwEcMQNxd2lh+0VrUByXN/oJkl5grOhYWvQUYiw==", - "dev": true, - "requires": { - "extend-shallow": "^3.0.0" - } - }, "sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -3924,42 +3147,30 @@ "integrity": "sha1-R1OT/56RR5rqYtyvDKPRSYOn+0A=" }, "stack-utils": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-1.0.1.tgz", - "integrity": "sha1-1PM6tU6OOHeLDKXP07OvsS22hiA=", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-1.0.2.tgz", + "integrity": "sha512-MTX+MeG5U994cazkjd/9KNAapsHnibjMLnfXodlkXw76JEea0UiNzrqidzo1emMwk7w5Qhc9jd4Bn9TBb1MFwA==", "dev": true }, - "static-extend": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/static-extend/-/static-extend-0.1.2.tgz", - "integrity": "sha1-YICcOcv/VTNyJv1eC1IPNB8ftcY=", + "string-width": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.1.0.tgz", + "integrity": "sha512-NrX+1dVVh+6Y9dnQ19pR0pP4FiEIlUvdTGn8pw6CKTNq5sgib2nIhmUNT5TAmhWmvKr3WcxBcP3E8nWezuipuQ==", "dev": true, "requires": { - "define-property": "^0.2.5", - "object-copy": "^0.1.0" + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^5.2.0" }, "dependencies": { - "define-property": { - "version": "0.2.5", - "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", - "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=", - "dev": true, - "requires": { - "is-descriptor": "^0.1.0" - } + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true } } }, - "string-width": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", - "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", - "dev": true, - "requires": { - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^4.0.0" - } - }, "string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", @@ -3969,12 +3180,12 @@ } }, "strip-ansi": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", - "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", "dev": true, "requires": { - "ansi-regex": "^3.0.0" + "ansi-regex": "^4.1.0" } }, "strip-bom": { @@ -3984,9 +3195,9 @@ "dev": true }, "strip-bom-buf": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/strip-bom-buf/-/strip-bom-buf-1.0.0.tgz", - "integrity": "sha1-HLRar1dTD0yvhsf3UXnSyaUd1XI=", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-bom-buf/-/strip-bom-buf-2.0.0.tgz", + "integrity": "sha512-gLFNHucd6gzb8jMsl5QmZ3QgnUJmp7qn4uUSHNwEXumAp7YizoGYw19ZUVfuq4aBOQUtyn2k8X/CwzWB73W2lQ==", "dev": true, "requires": { "is-utf8": "^0.2.1" @@ -4021,15 +3232,52 @@ "js-yaml": "^3.10.0", "serialize-error": "^2.1.0", "strip-ansi": "^4.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "dev": true + }, + "arrify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", + "integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=", + "dev": true + }, + "indent-string": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-3.2.0.tgz", + "integrity": "sha1-Sl/W0nzDMvN+VBmlBNu4NxBckok=", + "dev": true + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "dev": true, + "requires": { + "ansi-regex": "^3.0.0" + } + } } }, "supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.0.0.tgz", + "integrity": "sha512-WRt32iTpYEZWYOpcetGm0NPeSvaebccx7hhS/5M6sAiqnhedtFCHFxkjzZlJvFNCPowiKSFGiZk5USQDFy83vQ==", "dev": true, "requires": { - "has-flag": "^3.0.0" + "has-flag": "^4.0.0" + }, + "dependencies": { + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + } } }, "symbol-observable": { @@ -4053,58 +3301,25 @@ "integrity": "sha1-mcW/VZWJZq9tBtg73zgA3IL67F0=", "dev": true }, - "timed-out": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-4.0.1.tgz", - "integrity": "sha1-8y6srFoXW+ol1/q1Zas+2HQe9W8=", - "dev": true - }, "to-fast-properties": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", "integrity": "sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4=", "dev": true }, - "to-object-path": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/to-object-path/-/to-object-path-0.3.0.tgz", - "integrity": "sha1-KXWIt7Dn4KwI4E5nL4XB9JmeF68=", - "dev": true, - "requires": { - "kind-of": "^3.0.2" - }, - "dependencies": { - "kind-of": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", - "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", - "dev": true, - "requires": { - "is-buffer": "^1.1.5" - } - } - } - }, - "to-regex": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/to-regex/-/to-regex-3.0.2.tgz", - "integrity": "sha512-FWtleNAtZ/Ki2qtqej2CXTOayOH9bHDQF+Q48VpWyDXjbYxA4Yz8iDB31zXOBUlOHHKidDbqGVrTUvQMPmBGBw==", - "dev": true, - "requires": { - "define-property": "^2.0.2", - "extend-shallow": "^3.0.2", - "regex-not": "^1.0.2", - "safe-regex": "^1.1.0" - } + "to-readable-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/to-readable-stream/-/to-readable-stream-1.0.0.tgz", + "integrity": "sha512-Iq25XBt6zD5npPhlLVXGFN3/gyR2/qODcKNNyTMd4vbm39HUaOiAM4PMq0eMVC/Tkxz+Zjdsc55g9yyz+Yq00Q==", + "dev": true }, "to-regex-range": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-2.1.1.tgz", - "integrity": "sha1-fIDBe53+vlmeJzZ+DU3VWQFB2zg=", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", "dev": true, "requires": { - "is-number": "^3.0.0", - "repeat-string": "^1.6.1" + "is-number": "^7.0.0" } }, "trim-newlines": { @@ -4125,6 +3340,21 @@ "integrity": "sha1-yy4SAwZ+DI3h9hQJS5/kVwTqYAM=", "dev": true }, + "type-fest": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.5.2.tgz", + "integrity": "sha512-DWkS49EQKVX//Tbupb9TFa19c7+MK1XmzkrZUR8TAktmE/DizXoaoJV6TZ/tSIPXipqNiRI6CyAe7x69Jb6RSw==", + "dev": true + }, + "typedarray-to-buffer": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", + "integrity": "sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==", + "dev": true, + "requires": { + "is-typedarray": "^1.0.0" + } + }, "uid2": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/uid2/-/uid2-0.0.3.tgz", @@ -4148,52 +3378,17 @@ } }, "unicode-match-property-value-ecmascript": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/unicode-match-property-value-ecmascript/-/unicode-match-property-value-ecmascript-1.0.2.tgz", - "integrity": "sha512-Rx7yODZC1L/T8XKo/2kNzVAQaRE88AaMvI1EF/Xnj3GW2wzN6fop9DDWuFAKUVFH7vozkz26DzP0qyWLKLIVPQ==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/unicode-match-property-value-ecmascript/-/unicode-match-property-value-ecmascript-1.1.0.tgz", + "integrity": "sha512-hDTHvaBk3RmFzvSl0UVrUmC3PuW9wKVnpoUDYH0JDkSIovzw+J5viQmeYHxVSBptubnr7PbH2e0fnpDRQnQl5g==", "dev": true }, "unicode-property-aliases-ecmascript": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-1.0.4.tgz", - "integrity": "sha512-2WSLa6OdYd2ng8oqiGIWnJqyFArvhn+5vgx5GTxMbUYjCYKUcuKS62YLFF0R/BDGlB1yzXjQOLtPAfHsgirEpg==", + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-1.0.5.tgz", + "integrity": "sha512-L5RAqCfXqAwR3RriF8pM0lU0w4Ryf/GgzONwi6KnL1taJQa7x1TCxdJnILX59WIGOwR57IVxn7Nej0fz1Ny6fw==", "dev": true }, - "union-value": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.0.tgz", - "integrity": "sha1-XHHDTLW61dzr4+oM0IIHulqhrqQ=", - "dev": true, - "requires": { - "arr-union": "^3.1.0", - "get-value": "^2.0.6", - "is-extendable": "^0.1.1", - "set-value": "^0.4.3" - }, - "dependencies": { - "extend-shallow": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", - "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", - "dev": true, - "requires": { - "is-extendable": "^0.1.0" - } - }, - "set-value": { - "version": "0.4.3", - "resolved": "https://registry.npmjs.org/set-value/-/set-value-0.4.3.tgz", - "integrity": "sha1-fbCPnT0i3H945Trzw79GZuzfzPE=", - "dev": true, - "requires": { - "extend-shallow": "^2.0.1", - "is-extendable": "^0.1.1", - "is-plain-object": "^2.0.1", - "to-object-path": "^0.3.0" - } - } - } - }, "unique-string": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-1.0.0.tgz", @@ -4214,97 +3409,35 @@ "uid2": "0.0.3" } }, - "unset-value": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/unset-value/-/unset-value-1.0.0.tgz", - "integrity": "sha1-g3aHP30jNRef+x5vw6jtDfyKtVk=", - "dev": true, - "requires": { - "has-value": "^0.3.1", - "isobject": "^3.0.0" - }, - "dependencies": { - "has-value": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/has-value/-/has-value-0.3.1.tgz", - "integrity": "sha1-ex9YutpiyoJ+wKIHgCVlSEWZXh8=", - "dev": true, - "requires": { - "get-value": "^2.0.3", - "has-values": "^0.1.4", - "isobject": "^2.0.0" - }, - "dependencies": { - "isobject": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/isobject/-/isobject-2.1.0.tgz", - "integrity": "sha1-8GVWEJaj8dou9GJy+BXIQNh+DIk=", - "dev": true, - "requires": { - "isarray": "1.0.0" - } - } - } - }, - "has-values": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/has-values/-/has-values-0.1.4.tgz", - "integrity": "sha1-bWHeldkd/Km5oCCJrThL/49it3E=", - "dev": true - } - } - }, - "unzip-response": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/unzip-response/-/unzip-response-2.0.1.tgz", - "integrity": "sha1-0vD3N9FrBhXnKmk17QQhRXLVb5c=", - "dev": true - }, - "upath": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/upath/-/upath-1.1.0.tgz", - "integrity": "sha512-bzpH/oBhoS/QI/YtbkqCg6VEiPYjSZtrHQM6/QnJS6OL9pKUFLqb3aFh4Scvwm45+7iAgiMkLhSbaZxUqmrprw==", - "dev": true - }, "update-notifier": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/update-notifier/-/update-notifier-2.5.0.tgz", - "integrity": "sha512-gwMdhgJHGuj/+wHJJs9e6PcCszpxR1b236igrOkUofGhqJuG+amlIKwApH1IW1WWl7ovZxsX49lMBWLxSdm5Dw==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/update-notifier/-/update-notifier-3.0.1.tgz", + "integrity": "sha512-grrmrB6Zb8DUiyDIaeRTBCkgISYUgETNe7NglEbVsrLWXeESnlCSP50WfRSj/GmzMPl6Uchj24S/p80nP/ZQrQ==", "dev": true, "requires": { - "boxen": "^1.2.1", + "boxen": "^3.0.0", "chalk": "^2.0.1", - "configstore": "^3.0.0", + "configstore": "^4.0.0", + "has-yarn": "^2.1.0", "import-lazy": "^2.1.0", - "is-ci": "^1.0.10", + "is-ci": "^2.0.0", "is-installed-globally": "^0.1.0", - "is-npm": "^1.0.0", - "latest-version": "^3.0.0", + "is-npm": "^3.0.0", + "is-yarn-global": "^0.3.0", + "latest-version": "^5.0.0", "semver-diff": "^2.0.0", "xdg-basedir": "^3.0.0" } }, - "urix": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz", - "integrity": "sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI=", - "dev": true - }, "url-parse-lax": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-1.0.0.tgz", - "integrity": "sha1-evjzA2Rem9eaJy56FKxovAYJ2nM=", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-3.0.0.tgz", + "integrity": "sha1-FrXK/Afb42dsGxmZF3gj1lA6yww=", "dev": true, "requires": { - "prepend-http": "^1.0.1" + "prepend-http": "^2.0.0" } }, - "use": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz", - "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==", - "dev": true - }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -4330,9 +3463,9 @@ } }, "well-known-symbols": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/well-known-symbols/-/well-known-symbols-1.0.0.tgz", - "integrity": "sha1-c8eK6Bp3Jqj6WY4ogIAcixYiVRg=", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/well-known-symbols/-/well-known-symbols-2.0.0.tgz", + "integrity": "sha512-ZMjC3ho+KXo0BfJb7JgtQ5IBuvnShdlACNkKkdsqBmYw3bPAaJfPeYUo6tLUaT5tG/Gkh7xkpBhKRQ9e7pyg9Q==", "dev": true }, "which": { @@ -4345,12 +3478,39 @@ } }, "widest-line": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-2.0.0.tgz", - "integrity": "sha1-AUKk6KJD+IgsAjOqDgKBqnYVInM=", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-2.0.1.tgz", + "integrity": "sha512-Ba5m9/Fa4Xt9eb2ELXt77JxVDV8w7qQrH0zS/TWSJdLyAwQjWoOzpzj5lwVftDz6n/EOu3tNACS84v509qwnJA==", "dev": true, "requires": { "string-width": "^2.1.1" + }, + "dependencies": { + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "dev": true + }, + "string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "dev": true, + "requires": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + } + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "dev": true, + "requires": { + "ansi-regex": "^3.0.0" + } + } } }, "wrappy": { @@ -4360,14 +3520,15 @@ "dev": true }, "write-file-atomic": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-2.3.0.tgz", - "integrity": "sha512-xuPeK4OdjWqtfi59ylvVL0Yn35SF3zgcAcv7rBPFHVaEapaDr4GdGgm3j7ckTwH9wHL7fGmgfAnb0+THrHb8tA==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-3.0.0.tgz", + "integrity": "sha512-EIgkf60l2oWsffja2Sf2AL384dx328c0B+cIYPTQq5q2rOYuDV00/iPFBOUiDKKwKMOhkymH8AidPaRvzfxY+Q==", "dev": true, "requires": { - "graceful-fs": "^4.1.11", "imurmurhash": "^0.1.4", - "signal-exit": "^3.0.2" + "is-typedarray": "^1.0.0", + "signal-exit": "^3.0.2", + "typedarray-to-buffer": "^3.1.5" } }, "xdg-basedir": { @@ -4377,9 +3538,9 @@ "dev": true }, "xtend": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", - "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=", + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", "dev": true }, "yallist": { diff --git a/_integration/javascript/package.json b/_integration/javascript/package.json index b49f1a6c3..da6670c88 100644 --- a/_integration/javascript/package.json +++ b/_integration/javascript/package.json @@ -7,9 +7,9 @@ "test": "./node_modules/.bin/ava" }, "dependencies": { - "mysql": "github:mysqljs/mysql" + "mysql": "2.17.1" }, "devDependencies": { - "ava": "1.0.0-beta.8" + "ava": "2.3.0" } } diff --git a/_integration/python-sqlalchemy/requirements.txt b/_integration/python-sqlalchemy/requirements.txt index a0e94bb24..4a2392f06 100644 --- a/_integration/python-sqlalchemy/requirements.txt +++ b/_integration/python-sqlalchemy/requirements.txt @@ -1,2 +1,3 @@ pandas -sqlalchemy \ No newline at end of file +sqlalchemy +mysqlclient diff --git a/_integration/python-sqlalchemy/test.py b/_integration/python-sqlalchemy/test.py index 6c8767d7e..e1baa052d 100644 --- a/_integration/python-sqlalchemy/test.py +++ b/_integration/python-sqlalchemy/test.py @@ -6,14 +6,13 @@ class TestMySQL(unittest.TestCase): def test_connect(self): - engine = sqlalchemy.create_engine('mysql+pymysql://root:@127.0.0.1:3306/mydb') + engine = sqlalchemy.create_engine('mysql+mysqldb://root:@127.0.0.1:3306/mydb') with engine.connect() as conn: expected = { "name": {0: 'John Doe', 1: 'John Doe', 2: 'Jane Doe', 3: 'Evil Bob'}, "email": {0: 'john@doe.com', 1: 'johnalt@doe.com', 2: 'jane@doe.com', 3: 'evilbob@gmail.com'}, - "phone_numbers": {0: '["555-555-555"]', 1: '[]', 2: '[]', 3: '["555-666-555","666-666-666"]'}, + "phone_numbers": {0: ['555-555-555'], 1: [], 2: [], 3: ['555-666-555', '666-666-666']}, } - repo_df = pd.read_sql_table("mytable", con=conn) d = repo_df.to_dict() del d["created_at"] diff --git a/_integration/ruby/Makefile b/_integration/ruby/Makefile index cd1fbe1d2..5a52ad20b 100644 --- a/_integration/ruby/Makefile +++ b/_integration/ruby/Makefile @@ -1,4 +1,5 @@ vendor/bundle: + gem install bundler --version=1.16.5 bundler install --path vendor/bundle dependencies: vendor/bundle @@ -6,4 +7,4 @@ dependencies: vendor/bundle test: dependencies bundler exec ruby mysql_test.rb -.PHONY: test \ No newline at end of file +.PHONY: test diff --git a/_scripts/go-vitess/LICENSE b/_scripts/go-vitess/LICENSE deleted file mode 100644 index 261eeb9e9..000000000 --- a/_scripts/go-vitess/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/_scripts/go-vitess/Makefile b/_scripts/go-vitess/Makefile deleted file mode 100644 index f4f86e25e..000000000 --- a/_scripts/go-vitess/Makefile +++ /dev/null @@ -1,42 +0,0 @@ -# Tooling to create the package `gopkg.in/src-d/go-vitess.v1`. - -# config -PACKAGE := gopkg.in/src-d/go-vitess.v1 -REMOTE := git@github.com:src-d/go-vitess.git -VITESS_GIT := git@github.com:vitessio/vitess.git -VITESS_PKG := vitess.io/vitess/go -VITESS_SRC := ${GOPATH}/src/${PACKAGE} - -CWD := $(PWD) - -all: prepare-package -prepare-package: | filter-branch rename-packages prepare-git commit - -clone: - git clone --single-branch --depth 1 --no-tags ${VITESS_GIT} ${VITESS_SRC} - -filter-branch: clone - cd ${VITESS_SRC} && \ - git filter-branch --subdirectory-filter go && \ - rm -fr README.md cacheservice cmd exit ioutil2 memcache race ratelimiter stats/influxdbbackend stats/opentsdb stats/prometheusbackend testfiles vtbench zk vt/mysqlctl/cephbackupstorage && \ - cp -f "${CWD}/doc.go" "${CWD}/README.md" "${CWD}/LICENSE" ${VITESS_SRC} && \ - cp -f "${CWD}/vt/log/log.go" "${VITESS_SRC}/vt/log/log.go" - -rename-packages: - cd ${VITESS_SRC} && \ - gorep -from=${VITESS_PKG} -to=${PACKAGE} - -prepare-git: - cd ${VITESS_SRC} && \ - git remote rm origin && \ - git remote add origin $(REMOTE) - -commit: - cd ${VITESS_SRC} && \ - git add . && \ - git commit -m "update from upstream ${VITESS_GIT}" -a -s - -clean: - rm -rf ${VITESS_SRC} - -.PHONY: clean diff --git a/_scripts/go-vitess/README.md b/_scripts/go-vitess/README.md deleted file mode 100644 index a71d1b499..000000000 --- a/_scripts/go-vitess/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# go-vitess [![GoDoc](https://godoc.org/gopkg.in/src-d/go-vitess.v1?status.svg)](https://godoc.org/gopkg.in/src-d/go-vitess.v1) - -`go-vitess` is an automatic filter-branch done by an [script](https://github.com/src-d/go-mysql-server/blob/master/_scripts/go-vitess/Makefile), of the great [Vitess](github.com/vitessio/vitess) project. - -The goal is keeping the `vitess.io/vitess/go` package and all the dependent packages as a standalone versioned golang library, to be used by other projects. - -It holds all the packages to create your own MySQL server and a full SQL parser. - -## Installation - -```sh -go get -v -u gopkg.in/src-d/go-vitess.v1/... -``` - -## Contributions - -Since the code belongs to the upstream of [Vitess](github.com/vitessio/vitess), -the issue neither pull requests aren't accepted to this repository. - -## License - -Apache License 2.0, see [LICENSE](LICENSE). diff --git a/_scripts/go-vitess/doc.go b/_scripts/go-vitess/doc.go deleted file mode 100644 index c6ed7951d..000000000 --- a/_scripts/go-vitess/doc.go +++ /dev/null @@ -1,9 +0,0 @@ -// Package vitess is an automatic filter-branch, of the great Vitess project. -// -// The goal is keeping the `vitess.io/vitess/go` package and all -// the dependent packages as a standalone versioned golang library, to be used -// by other projects. -// -// It holds all the packages to create your own MySQL server and a full SQL -// parser. -package vitess // import "gopkg.in/src-d/go-vitess.v1" diff --git a/auth/audit.go b/auth/audit.go index e28942dad..1bcd9eec5 100644 --- a/auth/audit.go +++ b/auth/audit.go @@ -4,8 +4,8 @@ import ( "net" "time" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-vitess.v1/mysql" + "github.com/src-d/go-mysql-server/sql" + "vitess.io/vitess/go/mysql" "github.com/sirupsen/logrus" ) diff --git a/auth/audit_test.go b/auth/audit_test.go index a41ee1d60..82e777be2 100644 --- a/auth/audit_test.go +++ b/auth/audit_test.go @@ -1,3 +1,5 @@ +// +build !windows + package auth_test import ( @@ -5,8 +7,8 @@ import ( "testing" "time" - "gopkg.in/src-d/go-mysql-server.v0/auth" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/auth" + "github.com/src-d/go-mysql-server/sql" "github.com/sanity-io/litter" "github.com/sirupsen/logrus" diff --git a/auth/auth.go b/auth/auth.go index 40042cceb..d2ea68d5e 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -3,9 +3,9 @@ package auth import ( "strings" + "github.com/src-d/go-mysql-server/sql" "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-vitess.v1/mysql" + "vitess.io/vitess/go/mysql" ) // Permission holds permissions required by a query or grated to a user. diff --git a/auth/common_test.go b/auth/common_test.go index 07b8e3462..492b6881b 100644 --- a/auth/common_test.go +++ b/auth/common_test.go @@ -1,3 +1,5 @@ +// +build !windows + package auth_test import ( @@ -8,26 +10,26 @@ import ( "os" "testing" + sqle "github.com/src-d/go-mysql-server" + "github.com/src-d/go-mysql-server/auth" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/server" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/analyzer" + "github.com/src-d/go-mysql-server/sql/index/pilosa" "github.com/stretchr/testify/require" - sqle "gopkg.in/src-d/go-mysql-server.v0" - "gopkg.in/src-d/go-mysql-server.v0/auth" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/server" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/analyzer" - "gopkg.in/src-d/go-mysql-server.v0/sql/index/pilosa" ) const port = 3336 func authEngine(au auth.Auth) (string, *sqle.Engine, error) { - db := mem.NewDatabase("test") + db := memory.NewDatabase("test") catalog := sql.NewCatalog() catalog.AddDatabase(db) tblName := "test" - table := mem.NewTable(tblName, sql.Schema{ + table := memory.NewTable(tblName, sql.Schema{ {Name: "id", Type: sql.Text, Nullable: false, Source: tblName}, {Name: "name", Type: sql.Text, Nullable: false, Source: tblName}, }) @@ -101,21 +103,22 @@ func testAuthentication( for _, c := range tests { t.Run(fmt.Sprintf("%s-%s", c.user, c.password), func(t *testing.T) { - req := require.New(t) + r := require.New(t) - db, err := dsql.Open("mysql", connString(c.user, c.password)) - req.NoError(err) + var db *dsql.DB + db, err = dsql.Open("mysql", connString(c.user, c.password)) + r.NoError(err) _, err = db.Query("SELECT 1") if c.success { - req.NoError(err) + r.NoError(err) } else { - req.Error(err) - req.Contains(err.Error(), "Access denied") + r.Error(err) + r.Contains(err.Error(), "Access denied") } err = db.Close() - req.NoError(err) + r.NoError(err) if extra != nil { extra(t, c) @@ -196,20 +199,21 @@ func testAudit( for _, c := range tests { t.Run(c.user, func(t *testing.T) { - req := require.New(t) + r := require.New(t) - db, err := dsql.Open("mysql", connString(c.user, "")) - req.NoError(err) + var db *dsql.DB + db, err = dsql.Open("mysql", connString(c.user, "")) + r.NoError(err) _, err = db.Query(c.query) if c.success { - req.NoError(err) + r.NoError(err) } else { - req.Error(err) + r.Error(err) } err = db.Close() - req.NoError(err) + r.NoError(err) if extra != nil { extra(t, c) diff --git a/auth/native.go b/auth/native.go index 780168162..6d0744e12 100644 --- a/auth/native.go +++ b/auth/native.go @@ -9,10 +9,10 @@ import ( "regexp" "strings" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-vitess.v1/mysql" + "vitess.io/vitess/go/mysql" ) var ( diff --git a/auth/native_test.go b/auth/native_test.go index 9b16f4066..10cb43255 100644 --- a/auth/native_test.go +++ b/auth/native_test.go @@ -1,3 +1,5 @@ +// +build !windows + package auth_test import ( @@ -5,7 +7,7 @@ import ( "os" "testing" - "gopkg.in/src-d/go-mysql-server.v0/auth" + "github.com/src-d/go-mysql-server/auth" _ "github.com/go-sql-driver/mysql" "github.com/stretchr/testify/require" diff --git a/auth/none.go b/auth/none.go index 100f482da..8da092936 100644 --- a/auth/none.go +++ b/auth/none.go @@ -1,9 +1,9 @@ package auth import ( - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" - "gopkg.in/src-d/go-vitess.v1/mysql" + "vitess.io/vitess/go/mysql" ) // None is an Auth method that always succeeds. diff --git a/auth/none_test.go b/auth/none_test.go index abeb4a5b1..36d2495dc 100644 --- a/auth/none_test.go +++ b/auth/none_test.go @@ -1,9 +1,11 @@ +// +build !windows + package auth_test import ( "testing" - "gopkg.in/src-d/go-mysql-server.v0/auth" + "github.com/src-d/go-mysql-server/auth" ) func TestNoneAuthentication(t *testing.T) { diff --git a/benchmark/metadata.go b/benchmark/metadata.go index b8867cf8f..16545eff1 100644 --- a/benchmark/metadata.go +++ b/benchmark/metadata.go @@ -1,6 +1,6 @@ package benchmark -import "gopkg.in/src-d/go-mysql-server.v0/sql" +import "github.com/src-d/go-mysql-server/sql" type tableMetadata struct { schema sql.Schema diff --git a/benchmark/tpc_h_test.go b/benchmark/tpc_h_test.go index 63d3cb709..5524069fa 100644 --- a/benchmark/tpc_h_test.go +++ b/benchmark/tpc_h_test.go @@ -10,10 +10,10 @@ import ( "path/filepath" "testing" - "gopkg.in/src-d/go-mysql-server.v0" + sqle "github.com/src-d/go-mysql-server" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" ) var scriptsPath = "../_scripts/tpc-h/" @@ -83,11 +83,11 @@ func executeQueries(b *testing.B, e *sqle.Engine) error { } func genDB(b *testing.B) (sql.Database, error) { - db := mem.NewDatabase("tpch") + db := memory.NewDatabase("tpch") for _, m := range tpchTableMetadata { b.Log("generating table", m.name) - t := mem.NewTable(m.name, m.schema) + t := memory.NewTable(m.name, m.schema) if err := insertDataToTable(m.name, t, len(m.schema)); err != nil { return nil, err } @@ -98,7 +98,7 @@ func genDB(b *testing.B) (sql.Database, error) { return db, nil } -func insertDataToTable(name string, t *mem.Table, columnCount int) error { +func insertDataToTable(name string, t *memory.Table, columnCount int) error { f, err := os.Open(name + ".tbl") if err != nil { return err diff --git a/engine.go b/engine.go index c8140105a..f09c167c5 100644 --- a/engine.go +++ b/engine.go @@ -1,14 +1,17 @@ -package sqle // import "gopkg.in/src-d/go-mysql-server.v0" +package sqle import ( + "time" + + "github.com/go-kit/kit/metrics/discard" opentracing "github.com/opentracing/opentracing-go" "github.com/sirupsen/logrus" - "gopkg.in/src-d/go-mysql-server.v0/auth" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/analyzer" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression/function" - "gopkg.in/src-d/go-mysql-server.v0/sql/parse" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" + "github.com/src-d/go-mysql-server/auth" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/analyzer" + "github.com/src-d/go-mysql-server/sql/expression/function" + "github.com/src-d/go-mysql-server/sql/parse" + "github.com/src-d/go-mysql-server/sql/plan" ) // Config for the Engine. @@ -26,6 +29,34 @@ type Engine struct { Auth auth.Auth } +var ( + // QueryCounter describes a metric that accumulates number of queries monotonically. + QueryCounter = discard.NewCounter() + + // QueryErrorCounter describes a metric that accumulates number of failed queries monotonically. + QueryErrorCounter = discard.NewCounter() + + // QueryHistogram describes a queries latency. + QueryHistogram = discard.NewHistogram() +) + +func observeQuery(ctx *sql.Context, query string) func(err error) { + logrus.WithField("query", query).Debug("executing query") + span, _ := ctx.Span("query", opentracing.Tag{Key: "query", Value: query}) + + t := time.Now() + return func(err error) { + if err != nil { + QueryErrorCounter.With("query", query, "error", err.Error()).Add(1) + } else { + QueryCounter.With("query", query).Add(1) + QueryHistogram.With("query", query, "duration", "seconds").Observe(time.Since(t).Seconds()) + } + + span.Finish() + } +} + // New creates a new Engine with custom configuration. To create an Engine with // the default settings use `NewDefault`. func New(c *sql.Catalog, a *analyzer.Analyzer, cfg *Config) *Engine { @@ -34,9 +65,16 @@ func New(c *sql.Catalog, a *analyzer.Analyzer, cfg *Config) *Engine { versionPostfix = cfg.VersionPostfix } - c.RegisterFunctions(function.Defaults) - c.RegisterFunction("version", sql.FunctionN(function.NewVersion(versionPostfix))) - c.RegisterFunction("database", sql.Function0(function.NewDatabase(c))) + c.MustRegister( + sql.FunctionN{ + Name: "version", + Fn: function.NewVersion(versionPostfix), + }, + sql.Function0{ + Name: "database", + Fn: function.NewDatabase(c), + }) + c.MustRegister(function.Defaults...) // use auth.None if auth is not specified var au auth.Auth @@ -62,12 +100,16 @@ func (e *Engine) Query( ctx *sql.Context, query string, ) (sql.Schema, sql.RowIter, error) { - span, ctx := ctx.Span("query", opentracing.Tag{Key: "query", Value: query}) - defer span.Finish() + var ( + parsed, analyzed sql.Node + iter sql.RowIter + err error + ) - logrus.WithField("query", query).Debug("executing query") + finish := observeQuery(ctx, query) + defer finish(err) - parsed, err := parse.Parse(ctx, query) + parsed, err = parse.Parse(ctx, query) if err != nil { return nil, nil, err } @@ -78,7 +120,7 @@ func (e *Engine) Query( case *plan.CreateIndex: typ = sql.CreateIndexProcess perm = auth.ReadPerm | auth.WritePerm - case *plan.InsertInto, *plan.DropIndex, *plan.UnlockTables, *plan.LockTables: + case *plan.InsertInto, *plan.DeleteFrom, *plan.Update, *plan.DropIndex, *plan.UnlockTables, *plan.LockTables: perm = auth.ReadPerm | auth.WritePerm } @@ -98,12 +140,12 @@ func (e *Engine) Query( return nil, nil, err } - analyzed, err := e.Analyzer.Analyze(ctx, parsed) + analyzed, err = e.Analyzer.Analyze(ctx, parsed) if err != nil { return nil, nil, err } - iter, err := analyzed.RowIter(ctx) + iter, err = analyzed.RowIter(ctx) if err != nil { return nil, nil, err } @@ -111,6 +153,18 @@ func (e *Engine) Query( return analyzed.Schema(), iter, nil } +// Async returns true if the query is async. If there are any errors with the +// query it returns false +func (e *Engine) Async(ctx *sql.Context, query string) bool { + parsed, err := parse.Parse(ctx, query) + if err != nil { + return false + } + + asyncNode, ok := parsed.(sql.AsyncNode) + return ok && asyncNode.IsAsync() +} + // AddDatabase adds the given database to the catalog. func (e *Engine) AddDatabase(db sql.Database) { e.Catalog.AddDatabase(db) diff --git a/engine_pilosa_test.go b/engine_pilosa_test.go new file mode 100644 index 000000000..a6da4c5a9 --- /dev/null +++ b/engine_pilosa_test.go @@ -0,0 +1,209 @@ +// +build !windows + +package sqle_test + +import ( + "context" + "io/ioutil" + "os" + "testing" + "time" + + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/index/pilosa" + "github.com/src-d/go-mysql-server/test" + + "github.com/stretchr/testify/require" +) + +func TestIndexes(t *testing.T) { + e := newEngine(t) + + tmpDir, err := ioutil.TempDir(os.TempDir(), "pilosa-test") + require.NoError(t, err) + + require.NoError(t, os.MkdirAll(tmpDir, 0644)) + e.Catalog.RegisterIndexDriver(pilosa.NewDriver(tmpDir)) + + _, _, err = e.Query( + newCtx(), + "CREATE INDEX idx_i ON mytable USING pilosa (i) WITH (async = false)", + ) + require.NoError(t, err) + + _, _, err = e.Query( + newCtx(), + "CREATE INDEX idx_s ON mytable USING pilosa (s) WITH (async = false)", + ) + require.NoError(t, err) + + _, _, err = e.Query( + newCtx(), + "CREATE INDEX idx_is ON mytable USING pilosa (i, s) WITH (async = false)", + ) + require.NoError(t, err) + + defer func() { + done, err := e.Catalog.DeleteIndex("mydb", "idx_i", true) + require.NoError(t, err) + <-done + + done, err = e.Catalog.DeleteIndex("mydb", "idx_s", true) + require.NoError(t, err) + <-done + + done, err = e.Catalog.DeleteIndex("foo", "idx_is", true) + require.NoError(t, err) + <-done + }() + + testCases := []struct { + query string + expected []sql.Row + }{ + { + "SELECT * FROM mytable WHERE i = 2", + []sql.Row{ + {int64(2), "second row"}, + }, + }, + { + "SELECT * FROM mytable WHERE i > 1", + []sql.Row{ + {int64(3), "third row"}, + {int64(2), "second row"}, + }, + }, + { + "SELECT * FROM mytable WHERE i < 3", + []sql.Row{ + {int64(1), "first row"}, + {int64(2), "second row"}, + }, + }, + { + "SELECT * FROM mytable WHERE i <= 2", + []sql.Row{ + {int64(2), "second row"}, + {int64(1), "first row"}, + }, + }, + { + "SELECT * FROM mytable WHERE i >= 2", + []sql.Row{ + {int64(2), "second row"}, + {int64(3), "third row"}, + }, + }, + { + "SELECT * FROM mytable WHERE i = 2 AND s = 'second row'", + []sql.Row{ + {int64(2), "second row"}, + }, + }, + { + "SELECT * FROM mytable WHERE i = 2 AND s = 'third row'", + ([]sql.Row)(nil), + }, + { + "SELECT * FROM mytable WHERE i BETWEEN 1 AND 2", + []sql.Row{ + {int64(1), "first row"}, + {int64(2), "second row"}, + }, + }, + { + "SELECT * FROM mytable WHERE i = 1 OR i = 2", + []sql.Row{ + {int64(1), "first row"}, + {int64(2), "second row"}, + }, + }, + { + "SELECT * FROM mytable WHERE i = 1 AND i = 2", + ([]sql.Row)(nil), + }, + { + "SELECT i as mytable_i FROM mytable WHERE mytable_i = 2", + []sql.Row{ + {int64(2)}, + }, + }, + { + "SELECT i as mytable_i FROM mytable WHERE mytable_i > 1", + []sql.Row{ + {int64(3)}, + {int64(2)}, + }, + }, + { + "SELECT i as mytable_i, s as mytable_s FROM mytable WHERE mytable_i = 2 AND mytable_s = 'second row'", + []sql.Row{ + {int64(2), "second row"}, + }, + }, + { + "SELECT s, SUBSTRING(s, 1, 1) AS sub_s FROM mytable WHERE sub_s = 's'", + []sql.Row{ + {"second row", "s"}, + }, + }, + { + "SELECT count(i) AS mytable_i, SUBSTR(s, -3) AS mytable_s FROM mytable WHERE i > 0 AND mytable_s='row' GROUP BY mytable_s", + []sql.Row{ + {int64(3), "row"}, + }, + }, + { + "SELECT mytable_i FROM (SELECT i AS mytable_i FROM mytable) as t WHERE mytable_i > 1", + []sql.Row{ + {int64(2)}, + {int64(3)}, + }, + }, + } + + for _, tt := range testCases { + t.Run(tt.query, func(t *testing.T) { + require := require.New(t) + + tracer := new(test.MemTracer) + ctx := sql.NewContext(context.TODO(), sql.WithTracer(tracer)) + + _, it, err := e.Query(ctx, tt.query) + require.NoError(err) + + rows, err := sql.RowIterToRows(it) + require.NoError(err) + + require.ElementsMatch(tt.expected, rows) + require.Equal("plan.ResolvedTable", tracer.Spans[len(tracer.Spans)-1]) + }) + } +} + +func TestCreateIndex(t *testing.T) { + require := require.New(t) + e := newEngine(t) + + tmpDir, err := ioutil.TempDir(os.TempDir(), "pilosa-test") + require.NoError(err) + + require.NoError(os.MkdirAll(tmpDir, 0644)) + e.Catalog.RegisterIndexDriver(pilosa.NewDriver(tmpDir)) + + _, iter, err := e.Query(newCtx(), "CREATE INDEX myidx ON mytable USING pilosa (i)") + require.NoError(err) + rows, err := sql.RowIterToRows(iter) + require.NoError(err) + require.Len(rows, 0) + + defer func() { + time.Sleep(1 * time.Second) + done, err := e.Catalog.DeleteIndex("foo", "myidx", true) + require.NoError(err) + <-done + + require.NoError(os.RemoveAll(tmpDir)) + }() +} diff --git a/engine_test.go b/engine_test.go index 6119d38f2..b022a3e6b 100644 --- a/engine_test.go +++ b/engine_test.go @@ -3,22 +3,22 @@ package sqle_test import ( "context" "io" - "io/ioutil" "math" - "os" "strings" "sync/atomic" "testing" "time" - "gopkg.in/src-d/go-mysql-server.v0" - "gopkg.in/src-d/go-mysql-server.v0/auth" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/analyzer" - "gopkg.in/src-d/go-mysql-server.v0/sql/index/pilosa" - "gopkg.in/src-d/go-mysql-server.v0/sql/parse" - "gopkg.in/src-d/go-mysql-server.v0/test" + "github.com/opentracing/opentracing-go" + + sqle "github.com/src-d/go-mysql-server" + "github.com/src-d/go-mysql-server/auth" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/analyzer" + "github.com/src-d/go-mysql-server/sql/parse" + "github.com/src-d/go-mysql-server/sql/plan" + "github.com/src-d/go-mysql-server/test" "github.com/stretchr/testify/require" ) @@ -31,14 +31,90 @@ var queries = []struct { "SELECT i FROM mytable;", []sql.Row{{int64(1)}, {int64(2)}, {int64(3)}}, }, + { + "SELECT i + 1 FROM mytable;", + []sql.Row{{int64(2)}, {int64(3)}, {int64(4)}}, + }, + { + "SELECT -i FROM mytable;", + []sql.Row{{int64(-1)}, {int64(-2)}, {int64(-3)}}, + }, + { + "SELECT i FROM mytable where -i = -2;", + []sql.Row{{int64(2)}}, + }, { "SELECT i FROM mytable WHERE i = 2;", []sql.Row{{int64(2)}}, }, + { + "SELECT i FROM mytable WHERE i > 2;", + []sql.Row{{int64(3)}}, + }, + { + "SELECT i FROM mytable WHERE i < 2;", + []sql.Row{{int64(1)}}, + }, + { + "SELECT i FROM mytable WHERE i <> 2;", + []sql.Row{{int64(1)}, {int64(3)}}, + }, + { + "SELECT f32 FROM floattable WHERE f64 = 2.0;", + []sql.Row{{float32(2.0)}}, + }, + { + "SELECT f32 FROM floattable WHERE f64 < 2.0;", + []sql.Row{{float32(-1.0)}, {float32(-1.5)}, {float32(1.0)}, {float32(1.5)}}, + }, + { + "SELECT f32 FROM floattable WHERE f64 > 2.0;", + []sql.Row{{float32(2.5)}}, + }, + { + "SELECT f32 FROM floattable WHERE f64 <> 2.0;", + []sql.Row{{float32(-1.0)}, {float32(-1.5)}, {float32(1.0)}, {float32(1.5)}, {float32(2.5)}}, + }, + { + "SELECT f64 FROM floattable WHERE f32 = 2.0;", + []sql.Row{{float64(2.0)}}, + }, + { + "SELECT f64 FROM floattable WHERE f32 = -1.5;", + []sql.Row{{float64(-1.5)}}, + }, + { + "SELECT f64 FROM floattable WHERE -f32 = -2.0;", + []sql.Row{{float64(2.0)}}, + }, + { + "SELECT f64 FROM floattable WHERE f32 < 2.0;", + []sql.Row{{float64(-1.0)}, {float64(-1.5)}, {float64(1.0)}, {float64(1.5)}}, + }, + { + "SELECT f64 FROM floattable WHERE f32 > 2.0;", + []sql.Row{{float64(2.5)}}, + }, + { + "SELECT f64 FROM floattable WHERE f32 <> 2.0;", + []sql.Row{{float64(-1.0)}, {float64(-1.5)}, {float64(1.0)}, {float64(1.5)}, {float64(2.5)}}, + }, + { + "SELECT f32 FROM floattable ORDER BY f64;", + []sql.Row{{float32(-1.5)}, {float32(-1.0)}, {float32(1.0)}, {float32(1.5)}, {float32(2.0)}, {float32(2.5)}}, + }, { "SELECT i FROM mytable ORDER BY i DESC;", []sql.Row{{int64(3)}, {int64(2)}, {int64(1)}}, }, + { + "SELECT i FROM mytable WHERE 'hello';", + []sql.Row{}, + }, + { + "SELECT i FROM mytable WHERE not 'hello';", + []sql.Row{{int64(1)}, {int64(2)}, {int64(3)}}, + }, { "SELECT i FROM mytable WHERE s = 'first row' ORDER BY i DESC;", []sql.Row{{int64(1)}}, @@ -47,17 +123,65 @@ var queries = []struct { "SELECT i FROM mytable WHERE s = 'first row' ORDER BY i DESC LIMIT 1;", []sql.Row{{int64(1)}}, }, + { + "SELECT i FROM mytable ORDER BY i LIMIT 1 OFFSET 1;", + []sql.Row{{int64(2)}}, + }, + { + "SELECT i FROM mytable ORDER BY i LIMIT 1,1;", + []sql.Row{{int64(2)}}, + }, + { + "SELECT i FROM mytable ORDER BY i LIMIT 3,1;", + nil, + }, + { + "SELECT i FROM mytable ORDER BY i LIMIT 2,100;", + []sql.Row{{int64(3)}}, + }, + { + "SELECT i FROM niltable WHERE b IS NULL", + []sql.Row{{int64(2)}, {nil}}, + }, + { + "SELECT i FROM niltable WHERE b IS NOT NULL", + []sql.Row{{int64(1)}, {nil}, {int64(4)}}, + }, + { + "SELECT i FROM niltable WHERE b", + []sql.Row{{int64(1)}, {int64(4)}}, + }, + { + "SELECT i FROM niltable WHERE NOT b", + []sql.Row{{nil}}, + }, + { + "SELECT i FROM niltable WHERE b IS TRUE", + []sql.Row{{int64(1)}, {int64(4)}}, + }, + { + "SELECT i FROM niltable WHERE b IS NOT TRUE", + []sql.Row{{int64(2)}, {nil}, {nil}}, + }, + { + "SELECT f FROM niltable WHERE b IS FALSE", + []sql.Row{{3.0}}, + }, + { + "SELECT i FROM niltable WHERE b IS NOT FALSE", + []sql.Row{{int64(1)}, {int64(2)}, {int64(4)}, {nil}}, + }, { "SELECT COUNT(*) FROM mytable;", - []sql.Row{{int32(3)}}, + []sql.Row{{int64(3)}}, }, { "SELECT COUNT(*) FROM mytable LIMIT 1;", - []sql.Row{{int32(3)}}, + []sql.Row{{int64(3)}}, }, { "SELECT COUNT(*) AS c FROM mytable;", - []sql.Row{{int32(3)}}, + []sql.Row{{int64(3)}}, }, { "SELECT substring(s, 2, 3) FROM mytable", @@ -67,6 +191,28 @@ var queries = []struct { `SELECT substring("foo", 2, 2)`, []sql.Row{{"oo"}}, }, + { + `SELECT SUBSTRING_INDEX('a.b.c.d.e.f', '.', 2)`, + []sql.Row{ + {"a.b"}, + }, + }, + { + `SELECT SUBSTRING_INDEX('a.b.c.d.e.f', '.', -2)`, + []sql.Row{ + {"e.f"}, + }, + }, + { + `SELECT SUBSTRING_INDEX(SUBSTRING_INDEX('source{d}', '{d}', 1), 'r', -1)`, + []sql.Row{ + {"ce"}, + }, + }, + { + `SELECT SUBSTRING_INDEX(mytable.s, "d", 1) as s FROM mytable INNER JOIN othertable ON (SUBSTRING_INDEX(mytable.s, "d", 1) = SUBSTRING_INDEX(othertable.s2, "d", 1)) GROUP BY 1 HAVING s = 'secon'`, + []sql.Row{{"secon"}}, + }, { "SELECT YEAR('2007-12-11') FROM mytable", []sql.Row{{int32(2007)}, {int32(2007)}, {int32(2007)}}, @@ -107,6 +253,26 @@ var queries = []struct { "SELECT DAYOFYEAR('20071211') FROM mytable", []sql.Row{{int32(345)}, {int32(345)}, {int32(345)}}, }, + { + "SELECT YEARWEEK('0000-01-01')", + []sql.Row{{int32(1)}}, + }, + { + "SELECT YEARWEEK('9999-12-31')", + []sql.Row{{int32(999952)}}, + }, + { + "SELECT YEARWEEK('2008-02-20', 1)", + []sql.Row{{int32(200808)}}, + }, + { + "SELECT YEARWEEK('1987-01-01')", + []sql.Row{{int32(198652)}}, + }, + { + "SELECT YEARWEEK('1987-01-01', 20), YEARWEEK('1987-01-01', 1), YEARWEEK('1987-01-01', 2), YEARWEEK('1987-01-01', 3), YEARWEEK('1987-01-01', 4), YEARWEEK('1987-01-01', 5), YEARWEEK('1987-01-01', 6), YEARWEEK('1987-01-01', 7)", + []sql.Row{{int32(198653), int32(198701), int32(198652), int32(198701), int32(198653), int32(198652), int32(198653), int32(198652)}}, + }, { "SELECT i FROM mytable WHERE i BETWEEN 1 AND 2", []sql.Row{{int64(1)}, {int64(2)}}, @@ -175,9 +341,35 @@ var queries = []struct { ) t GROUP BY fi`, []sql.Row{ - {int32(1), "first row"}, - {int32(1), "second row"}, - {int32(1), "third row"}, + {int64(1), "first row"}, + {int64(1), "second row"}, + {int64(1), "third row"}, + }, + }, + { + `SELECT fi, COUNT(*) FROM ( + SELECT tbl.s AS fi + FROM mytable tbl + ) t + GROUP BY fi + ORDER BY COUNT(*) ASC`, + []sql.Row{ + {"first row", int64(1)}, + {"second row", int64(1)}, + {"third row", int64(1)}, + }, + }, + { + `SELECT COUNT(*), fi FROM ( + SELECT tbl.s AS fi + FROM mytable tbl + ) t + GROUP BY fi + ORDER BY COUNT(*) ASC`, + []sql.Row{ + {int64(1), "first row"}, + {int64(1), "second row"}, + {int64(1), "third row"}, }, }, { @@ -187,25 +379,25 @@ var queries = []struct { ) t GROUP BY 2`, []sql.Row{ - {int32(1), "first row"}, - {int32(1), "second row"}, - {int32(1), "third row"}, + {int64(1), "first row"}, + {int64(1), "second row"}, + {int64(1), "third row"}, }, }, { `SELECT COUNT(*) as cnt, s as fi FROM mytable GROUP BY fi`, []sql.Row{ - {int32(1), "first row"}, - {int32(1), "second row"}, - {int32(1), "third row"}, + {int64(1), "first row"}, + {int64(1), "second row"}, + {int64(1), "third row"}, }, }, { `SELECT COUNT(*) as cnt, s as fi FROM mytable GROUP BY 2`, []sql.Row{ - {int32(1), "first row"}, - {int32(1), "second row"}, - {int32(1), "third row"}, + {int64(1), "first row"}, + {int64(1), "second row"}, + {int64(1), "third row"}, }, }, { @@ -332,41 +524,41 @@ var queries = []struct { { `SELECT COUNT(*) c, i as foo FROM mytable GROUP BY i ORDER BY i DESC`, []sql.Row{ - {int32(1), int64(3)}, - {int32(1), int64(2)}, - {int32(1), int64(1)}, + {int64(1), int64(3)}, + {int64(1), int64(2)}, + {int64(1), int64(1)}, }, }, { `SELECT COUNT(*) c, i as foo FROM mytable GROUP BY 2 ORDER BY 2 DESC`, []sql.Row{ - {int32(1), int64(3)}, - {int32(1), int64(2)}, - {int32(1), int64(1)}, + {int64(1), int64(3)}, + {int64(1), int64(2)}, + {int64(1), int64(1)}, }, }, { - `SELECT COUNT(*) c, i as foo FROM mytable GROUP BY i ORDER BY foo, i DESC`, + `SELECT COUNT(*) c, i as foo FROM mytable GROUP BY i ORDER BY foo DESC`, []sql.Row{ - {int32(1), int64(3)}, - {int32(1), int64(2)}, - {int32(1), int64(1)}, + {int64(1), int64(3)}, + {int64(1), int64(2)}, + {int64(1), int64(1)}, }, }, { - `SELECT COUNT(*) c, i as foo FROM mytable GROUP BY 2 ORDER BY foo, i DESC`, + `SELECT COUNT(*) c, i as foo FROM mytable GROUP BY 2 ORDER BY foo DESC`, []sql.Row{ - {int32(1), int64(3)}, - {int32(1), int64(2)}, - {int32(1), int64(1)}, + {int64(1), int64(3)}, + {int64(1), int64(2)}, + {int64(1), int64(1)}, }, }, { `SELECT COUNT(*) c, i as i FROM mytable GROUP BY 2`, []sql.Row{ - {int32(1), int64(3)}, - {int32(1), int64(2)}, - {int32(1), int64(1)}, + {int64(1), int64(3)}, + {int64(1), int64(2)}, + {int64(1), int64(1)}, }, }, { @@ -392,7 +584,7 @@ var queries = []struct { { `SELECT COALESCE(NULL, NULL, NULL, COALESCE(NULL, 1234567890))`, []sql.Row{ - {int64(1234567890)}, + {int32(1234567890)}, }, }, { @@ -421,6 +613,22 @@ var queries = []struct { {float64(4), int64(3)}, }, }, + { + "SELECT SUM(i), i FROM mytable GROUP BY i ORDER BY 1+SUM(i) ASC", + []sql.Row{ + {float64(1), int64(1)}, + {float64(2), int64(2)}, + {float64(3), int64(3)}, + }, + }, + { + "SELECT i, SUM(i) FROM mytable GROUP BY i ORDER BY SUM(i) DESC", + []sql.Row{ + {int64(3), float64(3)}, + {int64(2), float64(2)}, + {int64(1), float64(1)}, + }, + }, { `/*!40101 SET NAMES utf8 */`, []sql.Row{}, @@ -429,6 +637,10 @@ var queries = []struct { `SHOW DATABASES`, []sql.Row{{"mydb"}, {"foo"}}, }, + { + `SHOW SCHEMAS`, + []sql.Row{{"mydb"}, {"foo"}}, + }, { `SELECT SCHEMA_NAME, DEFAULT_CHARACTER_SET_NAME, DEFAULT_COLLATION_NAME FROM information_schema.SCHEMATA`, []sql.Row{ @@ -487,6 +699,11 @@ var queries = []struct { {"mytable", "InnoDB", "10", "Fixed", int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), nil, nil, nil, "utf8_bin", nil, nil}, {"othertable", "InnoDB", "10", "Fixed", int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), nil, nil, nil, "utf8_bin", nil, nil}, {"tabletest", "InnoDB", "10", "Fixed", int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), nil, nil, nil, "utf8_bin", nil, nil}, + {"bigtable", "InnoDB", "10", "Fixed", int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), nil, nil, nil, "utf8_bin", nil, nil}, + {"floattable", "InnoDB", "10", "Fixed", int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), nil, nil, nil, "utf8_bin", nil, nil}, + {"niltable", "InnoDB", "10", "Fixed", int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), nil, nil, nil, "utf8_bin", nil, nil}, + {"newlinetable", "InnoDB", "10", "Fixed", int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), nil, nil, nil, "utf8_bin", nil, nil}, + {"typestable", "InnoDB", "10", "Fixed", int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), nil, nil, nil, "utf8_bin", nil, nil}, }, }, { @@ -494,6 +711,11 @@ var queries = []struct { []sql.Row{ {"mytable", "InnoDB", "10", "Fixed", int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), nil, nil, nil, "utf8_bin", nil, nil}, {"othertable", "InnoDB", "10", "Fixed", int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), nil, nil, nil, "utf8_bin", nil, nil}, + {"bigtable", "InnoDB", "10", "Fixed", int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), nil, nil, nil, "utf8_bin", nil, nil}, + {"floattable", "InnoDB", "10", "Fixed", int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), nil, nil, nil, "utf8_bin", nil, nil}, + {"niltable", "InnoDB", "10", "Fixed", int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), nil, nil, nil, "utf8_bin", nil, nil}, + {"newlinetable", "InnoDB", "10", "Fixed", int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), nil, nil, nil, "utf8_bin", nil, nil}, + {"typestable", "InnoDB", "10", "Fixed", int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), nil, nil, nil, "utf8_bin", nil, nil}, }, }, { @@ -543,6 +765,8 @@ var queries = []struct { {"ndbinfo_version", ""}, {"sql_select_limit", math.MaxInt32}, {"transaction_isolation", "READ UNCOMMITTED"}, + {"version", ""}, + {"version_comment", ""}, }, }, { @@ -568,6 +792,22 @@ var queries = []struct { `SELECT JSON_EXTRACT("foo", "$")`, []sql.Row{{"foo"}}, }, + { + `SELECT JSON_UNQUOTE('"foo"')`, + []sql.Row{{"foo"}}, + }, + { + `SELECT JSON_UNQUOTE('[1, 2, 3]')`, + []sql.Row{{"[1, 2, 3]"}}, + }, + { + `SELECT JSON_UNQUOTE('"\\t\\u0032"')`, + []sql.Row{{"\t2"}}, + }, + { + `SELECT JSON_UNQUOTE('"\t\\u0032"')`, + []sql.Row{{"\t2"}}, + }, { `SELECT CONNECTION_ID()`, []sql.Row{{uint32(1)}}, @@ -615,6 +855,11 @@ var queries = []struct { {"mytable"}, {"othertable"}, {"tabletest"}, + {"bigtable"}, + {"floattable"}, + {"niltable"}, + {"newlinetable"}, + {"typestable"}, }, }, { @@ -623,8 +868,8 @@ var queries = []struct { WHERE TABLE_SCHEMA='mydb' AND TABLE_NAME='mytable' `, []sql.Row{ - {"s", "TEXT"}, - {"i", "INT64"}, + {"s", "text"}, + {"i", "bigint"}, }, }, { @@ -638,6 +883,27 @@ var queries = []struct { {"i"}, {"s2"}, {"i2"}, + {"t"}, + {"n"}, + {"f32"}, + {"f64"}, + {"b"}, + {"f"}, + {"id"}, + {"i8"}, + {"i16"}, + {"i32"}, + {"i64"}, + {"u8"}, + {"u16"}, + {"u32"}, + {"u64"}, + {"ti"}, + {"da"}, + {"te"}, + {"bo"}, + {"js"}, + {"bl"}, }, }, { @@ -651,6 +917,27 @@ var queries = []struct { {"i"}, {"s2"}, {"i2"}, + {"t"}, + {"n"}, + {"f32"}, + {"f64"}, + {"b"}, + {"f"}, + {"id"}, + {"i8"}, + {"i16"}, + {"i32"}, + {"i64"}, + {"u8"}, + {"u16"}, + {"u32"}, + {"u64"}, + {"ti"}, + {"da"}, + {"te"}, + {"bo"}, + {"js"}, + {"bl"}, }, }, { @@ -664,6 +951,27 @@ var queries = []struct { {"i"}, {"s2"}, {"i2"}, + {"t"}, + {"n"}, + {"f32"}, + {"f64"}, + {"b"}, + {"f"}, + {"id"}, + {"i8"}, + {"i16"}, + {"i32"}, + {"i64"}, + {"u8"}, + {"u16"}, + {"u32"}, + {"u64"}, + {"ti"}, + {"da"}, + {"te"}, + {"bo"}, + {"js"}, + {"bl"}, }, }, { @@ -675,7 +983,7 @@ var queries = []struct { }, { `SELECT -1`, - []sql.Row{{int64(-1)}}, + []sql.Row{{int8(-1)}}, }, { ` @@ -697,6 +1005,11 @@ var queries = []struct { {"mytable", "InnoDB", "10", "Fixed", int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), nil, nil, nil, "utf8_bin", nil, nil}, {"othertable", "InnoDB", "10", "Fixed", int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), nil, nil, nil, "utf8_bin", nil, nil}, {"tabletest", "InnoDB", "10", "Fixed", int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), nil, nil, nil, "utf8_bin", nil, nil}, + {"bigtable", "InnoDB", "10", "Fixed", int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), nil, nil, nil, "utf8_bin", nil, nil}, + {"floattable", "InnoDB", "10", "Fixed", int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), nil, nil, nil, "utf8_bin", nil, nil}, + {"niltable", "InnoDB", "10", "Fixed", int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), nil, nil, nil, "utf8_bin", nil, nil}, + {"newlinetable", "InnoDB", "10", "Fixed", int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), nil, nil, nil, "utf8_bin", nil, nil}, + {"typestable", "InnoDB", "10", "Fixed", int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), nil, nil, nil, "utf8_bin", nil, nil}, }, }, { @@ -732,13 +1045,13 @@ var queries = []struct { { `SELECT nullif(123, 321)`, []sql.Row{ - {int64(123)}, + {int8(123)}, }, }, { `SELECT ifnull(123, NULL)`, []sql.Row{ - {int64(123)}, + {int8(123)}, }, }, { @@ -750,19 +1063,19 @@ var queries = []struct { { `SELECT ifnull(NULL, 123)`, []sql.Row{ - {int64(123)}, + {int8(123)}, }, }, { `SELECT ifnull(123, 123)`, []sql.Row{ - {int64(123)}, + {int8(123)}, }, }, { `SELECT ifnull(123, 321)`, []sql.Row{ - {int64(123)}, + {int8(123)}, }, }, { @@ -774,7 +1087,7 @@ var queries = []struct { { `SELECT round(15, 1)`, []sql.Row{ - {int64(15)}, + {int8(15)}, }, }, { @@ -807,6 +1120,11 @@ var queries = []struct { {"mytable"}, {"othertable"}, {"tabletest"}, + {"bigtable"}, + {"floattable"}, + {"niltable"}, + {"newlinetable"}, + {"typestable"}, }, }, { @@ -815,14 +1133,11 @@ var queries = []struct { {"mytable", "BASE TABLE"}, {"othertable", "BASE TABLE"}, {"tabletest", "BASE TABLE"}, - }, - }, - { - "SHOW FULL TABLES", - []sql.Row{ - {"mytable", "BASE TABLE"}, - {"othertable", "BASE TABLE"}, - {"tabletest", "BASE TABLE"}, + {"bigtable", "BASE TABLE"}, + {"floattable", "BASE TABLE"}, + {"niltable", "BASE TABLE"}, + {"newlinetable", "BASE TABLE"}, + {"typestable", "BASE TABLE"}, }, }, { @@ -836,6 +1151,11 @@ var queries = []struct { []sql.Row{ {"mytable"}, {"othertable"}, + {"bigtable"}, + {"floattable"}, + {"niltable"}, + {"newlinetable"}, + {"typestable"}, }, }, { @@ -868,70 +1188,466 @@ var queries = []struct { "ROLLBACK", []sql.Row{}, }, -} - -func TestQueries(t *testing.T) { - e := newEngine(t) - - ep := newEngineWithParallelism(t, 2) - - t.Run("sequential", func(t *testing.T) { - for _, tt := range queries { - testQuery(t, e, tt.query, tt.expected) - } - }) - - t.Run("parallel", func(t *testing.T) { - for _, tt := range queries { - testQuery(t, ep, tt.query, tt.expected) - } - }) -} - -func TestSessionSelectLimit(t *testing.T) { - ctx := newCtx() - ctx.Session.Set("sql_select_limit", sql.Int64, int64(1)) - - q := []struct { - query string - expected []sql.Row - }{ - { - "SELECT * FROM mytable ORDER BY i", - []sql.Row{{int64(1), "first row"}}, - }, - { - "SELECT * FROM mytable ORDER BY i LIMIT 2", - []sql.Row{ - {int64(1), "first row"}, - {int64(2), "second row"}, - }, + { + "SELECT substring(s, 1, 1) FROM mytable ORDER BY substring(s, 1, 1)", + []sql.Row{{"f"}, {"s"}, {"t"}}, + }, + { + "SELECT substring(s, 1, 1), count(*) FROM mytable GROUP BY substring(s, 1, 1)", + []sql.Row{{"f", int64(1)}, {"s", int64(1)}, {"t", int64(1)}}, + }, + { + "SELECT SLEEP(0.5)", + []sql.Row{{int(0)}}, + }, + { + "SELECT TO_BASE64('foo')", + []sql.Row{{string("Zm9v")}}, + }, + { + "SELECT FROM_BASE64('YmFy')", + []sql.Row{{string("bar")}}, + }, + { + "SELECT DATE_ADD('2018-05-02', INTERVAL 1 DAY)", + []sql.Row{{time.Date(2018, time.May, 3, 0, 0, 0, 0, time.UTC)}}, + }, + { + "SELECT DATE_SUB('2018-05-02', INTERVAL 1 DAY)", + []sql.Row{{time.Date(2018, time.May, 1, 0, 0, 0, 0, time.UTC)}}, + }, + { + "SELECT '2018-05-02' + INTERVAL 1 DAY", + []sql.Row{{time.Date(2018, time.May, 3, 0, 0, 0, 0, time.UTC)}}, + }, + { + "SELECT '2018-05-02' - INTERVAL 1 DAY", + []sql.Row{{time.Date(2018, time.May, 1, 0, 0, 0, 0, time.UTC)}}, + }, + { + `SELECT i AS i FROM mytable ORDER BY i`, + []sql.Row{{int64(1)}, {int64(2)}, {int64(3)}}, + }, + { + ` + SELECT + i, + foo + FROM ( + SELECT + i, + COUNT(s) AS foo + FROM mytable + GROUP BY i + ) AS q + ORDER BY foo DESC + `, + []sql.Row{ + {int64(1), int64(1)}, + {int64(2), int64(1)}, + {int64(3), int64(1)}, }, - { - "SELECT i FROM (SELECT i FROM mytable LIMIT 2) t ORDER BY i", - []sql.Row{{int64(1)}}, + }, + { + "SELECT n, COUNT(n) FROM bigtable GROUP BY n HAVING COUNT(n) > 2", + []sql.Row{{int64(1), int64(3)}, {int64(2), int64(3)}}, + }, + { + "SELECT n, MAX(n) FROM bigtable GROUP BY n HAVING COUNT(n) > 2", + []sql.Row{{int64(1), int64(1)}, {int64(2), int64(2)}}, + }, + { + "SELECT substring(mytable.s, 1, 5) as s FROM mytable INNER JOIN othertable ON (substring(mytable.s, 1, 5) = SUBSTRING(othertable.s2, 1, 5)) GROUP BY 1 HAVING s = \"secon\"", + []sql.Row{{"secon"}}, + }, + { + ` + SELECT COLUMN_NAME as COLUMN_NAME FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME LIKE '%table' + GROUP BY 1 HAVING SUBSTRING(COLUMN_NAME, 1, 1) = "s" + `, + []sql.Row{{"s"}, {"s2"}}, + }, + { + "SELECT s, i FROM mytable GROUP BY i ORDER BY SUBSTRING(s, 1, 1) DESC", + []sql.Row{ + {string("third row"), int64(3)}, + {string("second row"), int64(2)}, + {string("first row"), int64(1)}, }, - { - "SELECT i FROM (SELECT i FROM mytable) t ORDER BY i LIMIT 2", - []sql.Row{{int64(1)}}, + }, + { + "SELECT s, i FROM mytable GROUP BY i HAVING count(*) > 0 ORDER BY SUBSTRING(s, 1, 1) DESC", + []sql.Row{ + {string("third row"), int64(3)}, + {string("second row"), int64(2)}, + {string("first row"), int64(1)}, }, - } - e := newEngine(t) - t.Run("sql_select_limit", func(t *testing.T) { - for _, tt := range q { - testQueryWithContext(ctx, t, e, tt.query, tt.expected) - } - }) -} - -func TestSessionDefaults(t *testing.T) { - ctx := newCtx() - ctx.Session.Set("auto_increment_increment", sql.Int64, 0) - ctx.Session.Set("max_allowed_packet", sql.Int64, 0) - ctx.Session.Set("sql_select_limit", sql.Int64, 0) - ctx.Session.Set("ndbinfo_version", sql.Text, "non default value") - - q := `SET @@auto_increment_increment=DEFAULT, + }, + { + "SELECT CONVERT('9999-12-31 23:59:59', DATETIME)", + []sql.Row{{time.Date(9999, time.December, 31, 23, 59, 59, 0, time.UTC)}}, + }, + { + "SELECT CONVERT('10000-12-31 23:59:59', DATETIME)", + []sql.Row{{nil}}, + }, + { + "SELECT '9999-12-31 23:59:59' + INTERVAL 1 DAY", + []sql.Row{{nil}}, + }, + { + "SELECT DATE_ADD('9999-12-31 23:59:59', INTERVAL 1 DAY)", + []sql.Row{{nil}}, + }, + { + `SELECT t.date_col FROM (SELECT CONVERT('2019-06-06 00:00:00', DATETIME) as date_col) t WHERE t.date_col > '0000-01-01 00:00:00'`, + []sql.Row{{time.Date(2019, time.June, 6, 0, 0, 0, 0, time.UTC)}}, + }, + { + `SELECT t.date_col FROM (SELECT CONVERT('2019-06-06 00:00:00', DATETIME) as date_col) t GROUP BY t.date_col`, + []sql.Row{{time.Date(2019, time.June, 6, 0, 0, 0, 0, time.UTC)}}, + }, + { + `SELECT i AS foo FROM mytable ORDER BY mytable.i`, + []sql.Row{{int64(1)}, {int64(2)}, {int64(3)}}, + }, + { + `SELECT JSON_EXTRACT('[1, 2, 3]', '$.[0]')`, + []sql.Row{{float64(1)}}, + }, + { + `SELECT ARRAY_LENGTH(JSON_EXTRACT('[1, 2, 3]', '$'))`, + []sql.Row{{int32(3)}}, + }, + { + `SELECT ARRAY_LENGTH(JSON_EXTRACT('[{"i":0}, {"i":1, "y":"yyy"}, {"i":2, "x":"xxx"}]', '$.i'))`, + []sql.Row{{int32(3)}}, + }, + { + `SELECT GREATEST(1, 2, 3, 4)`, + []sql.Row{{int64(4)}}, + }, + { + `SELECT GREATEST(1, 2, "3", 4)`, + []sql.Row{{float64(4)}}, + }, + { + `SELECT GREATEST(1, 2, "9", "foo999")`, + []sql.Row{{float64(9)}}, + }, + { + `SELECT GREATEST("aaa", "bbb", "ccc")`, + []sql.Row{{"ccc"}}, + }, + { + `SELECT GREATEST(i, s) FROM mytable`, + []sql.Row{{float64(1)}, {float64(2)}, {float64(3)}}, + }, + { + `SELECT LEAST(1, 2, 3, 4)`, + []sql.Row{{int64(1)}}, + }, + { + `SELECT LEAST(1, 2, "3", 4)`, + []sql.Row{{float64(1)}}, + }, + { + `SELECT LEAST(1, 2, "9", "foo999")`, + []sql.Row{{float64(1)}}, + }, + { + `SELECT LEAST("aaa", "bbb", "ccc")`, + []sql.Row{{"aaa"}}, + }, + { + `SELECT LEAST(i, s) FROM mytable`, + []sql.Row{{float64(1)}, {float64(2)}, {float64(3)}}, + }, + { + "SELECT i, i2, s2 FROM mytable LEFT JOIN othertable ON i = i2 - 1", + []sql.Row{ + {int64(1), int64(2), "second"}, + {int64(2), int64(3), "first"}, + {int64(3), nil, nil}, + }, + }, + { + "SELECT i, i2, s2 FROM mytable RIGHT JOIN othertable ON i = i2 - 1", + []sql.Row{ + {nil, int64(1), "third"}, + {int64(1), int64(2), "second"}, + {int64(2), int64(3), "first"}, + }, + }, + { + "SELECT i, i2, s2 FROM mytable LEFT OUTER JOIN othertable ON i = i2 - 1", + []sql.Row{ + {int64(1), int64(2), "second"}, + {int64(2), int64(3), "first"}, + {int64(3), nil, nil}, + }, + }, + { + "SELECT i, i2, s2 FROM mytable RIGHT OUTER JOIN othertable ON i = i2 - 1", + []sql.Row{ + {nil, int64(1), "third"}, + {int64(1), int64(2), "second"}, + {int64(2), int64(3), "first"}, + }, + }, + { + `SELECT CHAR_LENGTH('áé'), LENGTH('àè')`, + []sql.Row{{int32(2), int32(4)}}, + }, + { + "SELECT i, COUNT(i) AS `COUNT(i)` FROM (SELECT i FROM mytable) t GROUP BY i ORDER BY i, `COUNT(i)` DESC", + []sql.Row{{int64(1), int64(1)}, {int64(2), int64(1)}, {int64(3), int64(1)}}, + }, + { + "SELECT i FROM mytable WHERE NOT s ORDER BY 1 DESC", + []sql.Row{ + {int64(3)}, + {int64(2)}, + {int64(1)}, + }, + }, + { + "SELECT i FROM mytable WHERE NOT(NOT i) ORDER BY 1 DESC", + []sql.Row{ + {int64(3)}, + {int64(2)}, + {int64(1)}, + }, + }, + { + `SELECT NOW() - NOW()`, + []sql.Row{{int64(0)}}, + }, + { + `SELECT NOW() - (NOW() - INTERVAL 1 SECOND)`, + []sql.Row{{int64(1)}}, + }, + { + `SELECT SUBSTR(SUBSTRING('0123456789ABCDEF', 1, 10), -4)`, + []sql.Row{{"6789"}}, + }, + { + `SELECT CASE i WHEN 1 THEN i ELSE NULL END FROM mytable`, + []sql.Row{{int64(1)}, {nil}, {nil}}, + }, + { + `SELECT (NULL+1)`, + []sql.Row{{nil}}, + }, + { + `SELECT ARRAY_LENGTH(null)`, + []sql.Row{{nil}}, + }, + { + `SELECT ARRAY_LENGTH("foo")`, + []sql.Row{{nil}}, + }, + { + `SELECT * FROM mytable WHERE NULL AND i = 3`, + []sql.Row{}, + }, + { + `SELECT 1 FROM mytable GROUP BY i HAVING i > 1`, + []sql.Row{{int8(1)}, {int8(1)}}, + }, + { + `SELECT avg(i) FROM mytable GROUP BY i HAVING avg(i) > 1`, + []sql.Row{{float64(2)}, {float64(3)}}, + }, + { + `SELECT s AS s, COUNT(*) AS count, AVG(i) AS ` + "`AVG(i)`" + ` + FROM ( + SELECT * FROM mytable + ) AS expr_qry + GROUP BY s + HAVING ((AVG(i) > 0)) + ORDER BY count DESC + LIMIT 10000`, + []sql.Row{ + {"first row", int64(1), float64(1)}, + {"second row", int64(1), float64(2)}, + {"third row", int64(1), float64(3)}, + }, + }, + { + `SELECT FIRST(i) FROM (SELECT i FROM mytable ORDER BY i) t`, + []sql.Row{{int64(1)}}, + }, + { + `SELECT LAST(i) FROM (SELECT i FROM mytable ORDER BY i) t`, + []sql.Row{{int64(3)}}, + }, + { + `SELECT COUNT(DISTINCT t.i) FROM tabletest t, mytable t2`, + []sql.Row{{int64(3)}}, + }, + { + `SELECT CASE WHEN NULL THEN "yes" ELSE "no" END AS test`, + []sql.Row{{"no"}}, + }, + { + `SELECT + table_schema, + table_name, + CASE + WHEN table_type = 'BASE TABLE' THEN + CASE + WHEN table_schema = 'mysql' + OR table_schema = 'performance_schema' THEN 'SYSTEM TABLE' + ELSE 'TABLE' + END + WHEN table_type = 'TEMPORARY' THEN 'LOCAL_TEMPORARY' + ELSE table_type + END AS TABLE_TYPE + FROM information_schema.tables + WHERE table_schema = 'mydb' + AND table_name = 'mytable' + HAVING table_type IN ('TABLE', 'VIEW') + ORDER BY table_type, table_schema, table_name`, + []sql.Row{{"mydb", "mytable", "TABLE"}}, + }, + { + `SELECT REGEXP_MATCHES("bopbeepbop", "bop")`, + []sql.Row{{[]interface{}{"bop", "bop"}}}, + }, + { + `SELECT EXPLODE(REGEXP_MATCHES("bopbeepbop", "bop"))`, + []sql.Row{{"bop"}, {"bop"}}, + }, + { + `SELECT EXPLODE(REGEXP_MATCHES("helloworld", "bop"))`, + []sql.Row{}, + }, + { + `SELECT EXPLODE(REGEXP_MATCHES("", ""))`, + []sql.Row{{""}}, + }, + { + `SELECT REGEXP_MATCHES(NULL, "")`, + []sql.Row{{nil}}, + }, + { + `SELECT REGEXP_MATCHES("", NULL)`, + []sql.Row{{nil}}, + }, + { + `SELECT REGEXP_MATCHES("", "", NULL)`, + []sql.Row{{nil}}, + }, + { + "SELECT * FROM newlinetable WHERE s LIKE '%text%'", + []sql.Row{ + {int64(1), "\nthere is some text in here"}, + {int64(2), "there is some\ntext in here"}, + {int64(3), "there is some text\nin here"}, + {int64(4), "there is some text in here\n"}, + {int64(5), "there is some text in here"}, + }, + }, + { + `SELECT i FROM mytable WHERE i = (SELECT 1)`, + []sql.Row{{int64(1)}}, + }, + { + `SELECT i FROM mytable WHERE i IN (SELECT i FROM mytable)`, + []sql.Row{ + {int64(1)}, + {int64(2)}, + {int64(3)}, + }, + }, + { + `SELECT i FROM mytable WHERE i NOT IN (SELECT i FROM mytable ORDER BY i ASC LIMIT 2)`, + []sql.Row{ + {int64(3)}, + }, + }, + { + `SELECT (SELECT i FROM mytable ORDER BY i ASC LIMIT 1) AS x`, + []sql.Row{{int64(1)}}, + }, + { + `SELECT DISTINCT n FROM bigtable ORDER BY t`, + []sql.Row{ + {int64(1)}, + {int64(9)}, + {int64(7)}, + {int64(3)}, + {int64(2)}, + {int64(8)}, + {int64(6)}, + {int64(5)}, + {int64(4)}, + }, + }, +} + +func TestQueries(t *testing.T) { + e := newEngine(t) + t.Run("sequential", func(t *testing.T) { + for _, tt := range queries { + testQuery(t, e, tt.query, tt.expected) + } + }) + + ep := newEngineWithParallelism(t, 2) + t.Run("parallel", func(t *testing.T) { + for _, tt := range queries { + testQuery(t, ep, tt.query, tt.expected) + } + }) +} + +func TestSessionSelectLimit(t *testing.T) { + ctx := newCtx() + ctx.Session.Set("sql_select_limit", sql.Int64, int64(1)) + + q := []struct { + query string + expected []sql.Row + }{ + { + "SELECT * FROM mytable ORDER BY i", + []sql.Row{{int64(1), "first row"}}, + }, + { + "SELECT * FROM mytable ORDER BY i LIMIT 2", + []sql.Row{ + {int64(1), "first row"}, + {int64(2), "second row"}, + }, + }, + { + "SELECT i FROM (SELECT i FROM mytable LIMIT 2) t ORDER BY i", + []sql.Row{{int64(1)}}, + }, + { + "SELECT i FROM (SELECT i FROM mytable) t ORDER BY i LIMIT 2", + []sql.Row{{int64(1)}}, + }, + } + e := newEngine(t) + t.Run("sql_select_limit", func(t *testing.T) { + for _, tt := range q { + testQueryWithContext(ctx, t, e, tt.query, tt.expected) + } + }) +} + +func TestSessionDefaults(t *testing.T) { + ctx := newCtx() + ctx.Session.Set("auto_increment_increment", sql.Int64, 0) + ctx.Session.Set("max_allowed_packet", sql.Int64, 0) + ctx.Session.Set("sql_select_limit", sql.Int64, 0) + ctx.Session.Set("ndbinfo_version", sql.Text, "non default value") + + q := `SET @@auto_increment_increment=DEFAULT, @@max_allowed_packet=DEFAULT, @@sql_select_limit=DEFAULT, @@ndbinfo_version=DEFAULT` @@ -1104,14 +1820,14 @@ func TestDescribe(t *testing.T) { query := `DESCRIBE FORMAT=TREE SELECT * FROM mytable` expectedSeq := []sql.Row{ - sql.NewRow("Table(mytable)"), + sql.NewRow("Table(mytable): Projected "), sql.NewRow(" ├─ Column(i, INT64, nullable=false)"), sql.NewRow(" └─ Column(s, TEXT, nullable=false)"), } expectedParallel := []sql.Row{ {"Exchange(parallelism=2)"}, - {" └─ Table(mytable)"}, + {" └─ Table(mytable): Projected "}, {" ├─ Column(i, INT64, nullable=false)"}, {" └─ Column(s, TEXT, nullable=false)"}, } @@ -1146,16 +1862,525 @@ func TestOrderByColumns(t *testing.T) { } func TestInsertInto(t *testing.T) { - e := newEngine(t) - testQuery(t, e, - "INSERT INTO mytable (s, i) VALUES ('x', 999);", - []sql.Row{{int64(1)}}, - ) + var insertions = []struct { + insertQuery string + expectedInsert []sql.Row + selectQuery string + expectedSelect []sql.Row + }{ + { + "INSERT INTO mytable (s, i) VALUES ('x', 999);", + []sql.Row{{int64(1)}}, + "SELECT i FROM mytable WHERE s = 'x';", + []sql.Row{{int64(999)}}, + }, + { + "INSERT INTO mytable SET s = 'x', i = 999;", + []sql.Row{{int64(1)}}, + "SELECT i FROM mytable WHERE s = 'x';", + []sql.Row{{int64(999)}}, + }, + { + "INSERT INTO mytable VALUES (999, 'x');", + []sql.Row{{int64(1)}}, + "SELECT i FROM mytable WHERE s = 'x';", + []sql.Row{{int64(999)}}, + }, + { + "INSERT INTO mytable SET i = 999, s = 'x';", + []sql.Row{{int64(1)}}, + "SELECT i FROM mytable WHERE s = 'x';", + []sql.Row{{int64(999)}}, + }, + { + `INSERT INTO typestable VALUES ( + 999, 127, 32767, 2147483647, 9223372036854775807, + 255, 65535, 4294967295, 18446744073709551615, + 3.40282346638528859811704183484516925440e+38, 1.797693134862315708145274237317043567981e+308, + '2132-04-05 12:51:36', '2231-11-07', + 'random text', true, '{"key":"value"}', 'blobdata' + );`, + []sql.Row{{int64(1)}}, + "SELECT * FROM typestable WHERE id = 999;", + []sql.Row{{ + int64(999), int8(math.MaxInt8), int16(math.MaxInt16), int32(math.MaxInt32), int64(math.MaxInt64), + uint8(math.MaxUint8), uint16(math.MaxUint16), uint32(math.MaxUint32), uint64(math.MaxUint64), + float64(math.MaxFloat32), float64(math.MaxFloat64), + timeParse(sql.TimestampLayout, "2132-04-05 12:51:36"), timeParse(sql.DateLayout, "2231-11-07"), + "random text", true, `{"key":"value"}`, "blobdata", + }}, + }, + { + `INSERT INTO typestable SET + id = 999, i8 = 127, i16 = 32767, i32 = 2147483647, i64 = 9223372036854775807, + u8 = 255, u16 = 65535, u32 = 4294967295, u64 = 18446744073709551615, + f32 = 3.40282346638528859811704183484516925440e+38, f64 = 1.797693134862315708145274237317043567981e+308, + ti = '2132-04-05 12:51:36', da = '2231-11-07', + te = 'random text', bo = true, js = '{"key":"value"}', bl = 'blobdata' + ;`, + []sql.Row{{int64(1)}}, + "SELECT * FROM typestable WHERE id = 999;", + []sql.Row{{ + int64(999), int8(math.MaxInt8), int16(math.MaxInt16), int32(math.MaxInt32), int64(math.MaxInt64), + uint8(math.MaxUint8), uint16(math.MaxUint16), uint32(math.MaxUint32), uint64(math.MaxUint64), + float64(math.MaxFloat32), float64(math.MaxFloat64), + timeParse(sql.TimestampLayout, "2132-04-05 12:51:36"), timeParse(sql.DateLayout, "2231-11-07"), + "random text", true, `{"key":"value"}`, "blobdata", + }}, + }, + { + `INSERT INTO typestable VALUES ( + 999, -128, -32768, -2147483648, -9223372036854775808, + 0, 0, 0, 0, + 1.401298464324817070923729583289916131280e-45, 4.940656458412465441765687928682213723651e-324, + '0010-04-05 12:51:36', '0101-11-07', + '', false, '', '' + );`, + []sql.Row{{int64(1)}}, + "SELECT * FROM typestable WHERE id = 999;", + []sql.Row{{ + int64(999), int8(-math.MaxInt8 - 1), int16(-math.MaxInt16 - 1), int32(-math.MaxInt32 - 1), int64(-math.MaxInt64 - 1), + uint8(0), uint16(0), uint32(0), uint64(0), + float64(math.SmallestNonzeroFloat32), float64(math.SmallestNonzeroFloat64), + timeParse(sql.TimestampLayout, "0010-04-05 12:51:36"), timeParse(sql.DateLayout, "0101-11-07"), + "", false, ``, "", + }}, + }, + { + `INSERT INTO typestable SET + id = 999, i8 = -128, i16 = -32768, i32 = -2147483648, i64 = -9223372036854775808, + u8 = 0, u16 = 0, u32 = 0, u64 = 0, + f32 = 1.401298464324817070923729583289916131280e-45, f64 = 4.940656458412465441765687928682213723651e-324, + ti = '0010-04-05 12:51:36', da = '0101-11-07', + te = '', bo = false, js = '', bl = '' + ;`, + []sql.Row{{int64(1)}}, + "SELECT * FROM typestable WHERE id = 999;", + []sql.Row{{ + int64(999), int8(-math.MaxInt8 - 1), int16(-math.MaxInt16 - 1), int32(-math.MaxInt32 - 1), int64(-math.MaxInt64 - 1), + uint8(0), uint16(0), uint32(0), uint64(0), + float64(math.SmallestNonzeroFloat32), float64(math.SmallestNonzeroFloat64), + timeParse(sql.TimestampLayout, "0010-04-05 12:51:36"), timeParse(sql.DateLayout, "0101-11-07"), + "", false, ``, "", + }}, + }, + { + `INSERT INTO typestable VALUES (999, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null);`, + []sql.Row{{int64(1)}}, + "SELECT * FROM typestable WHERE id = 999;", + []sql.Row{{int64(999), nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil}}, + }, + { + `INSERT INTO typestable SET id=999, i8=null, i16=null, i32=null, i64=null, u8=null, u16=null, u32=null, u64=null, + f32=null, f64=null, ti=null, da=null, te=null, bo=null, js=null, bl=null;`, + []sql.Row{{int64(1)}}, + "SELECT * FROM typestable WHERE id = 999;", + []sql.Row{{int64(999), nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil}}, + }, + } - testQuery(t, e, - "SELECT i FROM mytable WHERE s = 'x';", - []sql.Row{{int64(999)}}, - ) + for _, insertion := range insertions { + e := newEngine(t) + ctx := newCtx() + testQueryWithContext(ctx, t, e, insertion.insertQuery, insertion.expectedInsert) + testQueryWithContext(ctx, t, e, insertion.selectQuery, insertion.expectedSelect) + } +} + +func TestInsertIntoErrors(t *testing.T) { + var expectedFailures = []struct { + name string + query string + }{ + { + "too few values", + "INSERT INTO mytable (s, i) VALUES ('x');", + }, + { + "too many values one column", + "INSERT INTO mytable (s) VALUES ('x', 999);", + }, + { + "too many values two columns", + "INSERT INTO mytable (i, s) VALUES (999, 'x', 'y');", + }, + { + "too few values no columns specified", + "INSERT INTO mytable VALUES (999);", + }, + { + "too many values no columns specified", + "INSERT INTO mytable VALUES (999, 'x', 'y');", + }, + { + "non-existent column values", + "INSERT INTO mytable (i, s, z) VALUES (999, 'x', 999);", + }, + { + "non-existent column set", + "INSERT INTO mytable SET i = 999, s = 'x', z = 999;", + }, + { + "duplicate column", + "INSERT INTO mytable (i, s, s) VALUES (999, 'x', 'x');", + }, + { + "duplicate column set", + "INSERT INTO mytable SET i = 999, s = 'y', s = 'y';", + }, + { + "null given to non-nullable", + "INSERT INTO mytable (i, s) VALUES (null, 'y');", + }, + } + + for _, expectedFailure := range expectedFailures { + t.Run(expectedFailure.name, func(t *testing.T) { + _, _, err := newEngine(t).Query(newCtx(), expectedFailure.query) + require.Error(t, err) + }) + } +} + +func TestReplaceInto(t *testing.T) { + var insertions = []struct { + replaceQuery string + expectedReplace []sql.Row + selectQuery string + expectedSelect []sql.Row + }{ + { + "REPLACE INTO mytable VALUES (1, 'first row');", + []sql.Row{{int64(2)}}, + "SELECT s FROM mytable WHERE i = 1;", + []sql.Row{{"first row"}}, + }, + { + "REPLACE INTO mytable SET i = 1, s = 'first row';", + []sql.Row{{int64(2)}}, + "SELECT s FROM mytable WHERE i = 1;", + []sql.Row{{"first row"}}, + }, + { + "REPLACE INTO mytable VALUES (1, 'new row same i');", + []sql.Row{{int64(1)}}, + "SELECT s FROM mytable WHERE i = 1;", + []sql.Row{{"first row"}, {"new row same i"}}, + }, + { + "REPLACE INTO mytable (s, i) VALUES ('x', 999);", + []sql.Row{{int64(1)}}, + "SELECT i FROM mytable WHERE s = 'x';", + []sql.Row{{int64(999)}}, + }, + { + "REPLACE INTO mytable SET s = 'x', i = 999;", + []sql.Row{{int64(1)}}, + "SELECT i FROM mytable WHERE s = 'x';", + []sql.Row{{int64(999)}}, + }, + { + "REPLACE INTO mytable VALUES (999, 'x');", + []sql.Row{{int64(1)}}, + "SELECT i FROM mytable WHERE s = 'x';", + []sql.Row{{int64(999)}}, + }, + { + "REPLACE INTO mytable SET i = 999, s = 'x';", + []sql.Row{{int64(1)}}, + "SELECT i FROM mytable WHERE s = 'x';", + []sql.Row{{int64(999)}}, + }, + { + `REPLACE INTO typestable VALUES ( + 999, 127, 32767, 2147483647, 9223372036854775807, + 255, 65535, 4294967295, 18446744073709551615, + 3.40282346638528859811704183484516925440e+38, 1.797693134862315708145274237317043567981e+308, + '2132-04-05 12:51:36', '2231-11-07', + 'random text', true, '{"key":"value"}', 'blobdata' + );`, + []sql.Row{{int64(1)}}, + "SELECT * FROM typestable WHERE id = 999;", + []sql.Row{{ + int64(999), int8(math.MaxInt8), int16(math.MaxInt16), int32(math.MaxInt32), int64(math.MaxInt64), + uint8(math.MaxUint8), uint16(math.MaxUint16), uint32(math.MaxUint32), uint64(math.MaxUint64), + float64(math.MaxFloat32), float64(math.MaxFloat64), + timeParse(sql.TimestampLayout, "2132-04-05 12:51:36"), timeParse(sql.DateLayout, "2231-11-07"), + "random text", true, `{"key":"value"}`, "blobdata", + }}, + }, + { + `REPLACE INTO typestable SET + id = 999, i8 = 127, i16 = 32767, i32 = 2147483647, i64 = 9223372036854775807, + u8 = 255, u16 = 65535, u32 = 4294967295, u64 = 18446744073709551615, + f32 = 3.40282346638528859811704183484516925440e+38, f64 = 1.797693134862315708145274237317043567981e+308, + ti = '2132-04-05 12:51:36', da = '2231-11-07', + te = 'random text', bo = true, js = '{"key":"value"}', bl = 'blobdata' + ;`, + []sql.Row{{int64(1)}}, + "SELECT * FROM typestable WHERE id = 999;", + []sql.Row{{ + int64(999), int8(math.MaxInt8), int16(math.MaxInt16), int32(math.MaxInt32), int64(math.MaxInt64), + uint8(math.MaxUint8), uint16(math.MaxUint16), uint32(math.MaxUint32), uint64(math.MaxUint64), + float64(math.MaxFloat32), float64(math.MaxFloat64), + timeParse(sql.TimestampLayout, "2132-04-05 12:51:36"), timeParse(sql.DateLayout, "2231-11-07"), + "random text", true, `{"key":"value"}`, "blobdata", + }}, + }, + { + `REPLACE INTO typestable VALUES ( + 999, -128, -32768, -2147483648, -9223372036854775808, + 0, 0, 0, 0, + 1.401298464324817070923729583289916131280e-45, 4.940656458412465441765687928682213723651e-324, + '0010-04-05 12:51:36', '0101-11-07', + '', false, '', '' + );`, + []sql.Row{{int64(1)}}, + "SELECT * FROM typestable WHERE id = 999;", + []sql.Row{{ + int64(999), int8(-math.MaxInt8 - 1), int16(-math.MaxInt16 - 1), int32(-math.MaxInt32 - 1), int64(-math.MaxInt64 - 1), + uint8(0), uint16(0), uint32(0), uint64(0), + float64(math.SmallestNonzeroFloat32), float64(math.SmallestNonzeroFloat64), + timeParse(sql.TimestampLayout, "0010-04-05 12:51:36"), timeParse(sql.DateLayout, "0101-11-07"), + "", false, ``, "", + }}, + }, + { + `REPLACE INTO typestable SET + id = 999, i8 = -128, i16 = -32768, i32 = -2147483648, i64 = -9223372036854775808, + u8 = 0, u16 = 0, u32 = 0, u64 = 0, + f32 = 1.401298464324817070923729583289916131280e-45, f64 = 4.940656458412465441765687928682213723651e-324, + ti = '0010-04-05 12:51:36', da = '0101-11-07', + te = '', bo = false, js = '', bl = '' + ;`, + []sql.Row{{int64(1)}}, + "SELECT * FROM typestable WHERE id = 999;", + []sql.Row{{ + int64(999), int8(-math.MaxInt8 - 1), int16(-math.MaxInt16 - 1), int32(-math.MaxInt32 - 1), int64(-math.MaxInt64 - 1), + uint8(0), uint16(0), uint32(0), uint64(0), + float64(math.SmallestNonzeroFloat32), float64(math.SmallestNonzeroFloat64), + timeParse(sql.TimestampLayout, "0010-04-05 12:51:36"), timeParse(sql.DateLayout, "0101-11-07"), + "", false, ``, "", + }}, + }, + { + `REPLACE INTO typestable VALUES (999, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null);`, + []sql.Row{{int64(1)}}, + "SELECT * FROM typestable WHERE id = 999;", + []sql.Row{{int64(999), nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil}}, + }, + { + `REPLACE INTO typestable SET id=999, i8=null, i16=null, i32=null, i64=null, u8=null, u16=null, u32=null, u64=null, + f32=null, f64=null, ti=null, da=null, te=null, bo=null, js=null, bl=null;`, + []sql.Row{{int64(1)}}, + "SELECT * FROM typestable WHERE id = 999;", + []sql.Row{{int64(999), nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil}}, + }, + } + + for _, insertion := range insertions { + e := newEngine(t) + ctx := newCtx() + testQueryWithContext(ctx, t, e, insertion.replaceQuery, insertion.expectedReplace) + testQueryWithContext(ctx, t, e, insertion.selectQuery, insertion.expectedSelect) + } +} + +func TestReplaceIntoErrors(t *testing.T) { + var expectedFailures = []struct { + name string + query string + }{ + { + "too few values", + "REPLACE INTO mytable (s, i) VALUES ('x');", + }, + { + "too many values one column", + "REPLACE INTO mytable (s) VALUES ('x', 999);", + }, + { + "too many values two columns", + "REPLACE INTO mytable (i, s) VALUES (999, 'x', 'y');", + }, + { + "too few values no columns specified", + "REPLACE INTO mytable VALUES (999);", + }, + { + "too many values no columns specified", + "REPLACE INTO mytable VALUES (999, 'x', 'y');", + }, + { + "non-existent column values", + "REPLACE INTO mytable (i, s, z) VALUES (999, 'x', 999);", + }, + { + "non-existent column set", + "REPLACE INTO mytable SET i = 999, s = 'x', z = 999;", + }, + { + "duplicate column values", + "REPLACE INTO mytable (i, s, s) VALUES (999, 'x', 'x');", + }, + { + "duplicate column set", + "REPLACE INTO mytable SET i = 999, s = 'y', s = 'y';", + }, + { + "null given to non-nullable values", + "INSERT INTO mytable (i, s) VALUES (null, 'y');", + }, + { + "null given to non-nullable set", + "INSERT INTO mytable SET i = null, s = 'y';", + }, + } + + for _, expectedFailure := range expectedFailures { + t.Run(expectedFailure.name, func(t *testing.T) { + _, _, err := newEngine(t).Query(newCtx(), expectedFailure.query) + require.Error(t, err) + }) + } +} + +func TestUpdate(t *testing.T) { + var updates = []struct { + updateQuery string + expectedUpdate []sql.Row + selectQuery string + expectedSelect []sql.Row + }{ + { + "UPDATE mytable SET s = 'updated';", + []sql.Row{{int64(3), int64(3)}}, + "SELECT * FROM mytable;", + []sql.Row{{int64(1), "updated"}, {int64(2), "updated"}, {int64(3), "updated"}}, + }, + { + "UPDATE mytable SET s = 'updated' WHERE i > 9999;", + []sql.Row{{int64(0), int64(0)}}, + "SELECT * FROM mytable;", + []sql.Row{{int64(1), "first row"}, {int64(2), "second row"}, {int64(3), "third row"}}, + }, + { + "UPDATE mytable SET s = 'updated' WHERE i = 1;", + []sql.Row{{int64(1), int64(1)}}, + "SELECT * FROM mytable;", + []sql.Row{{int64(1), "updated"}, {int64(2), "second row"}, {int64(3), "third row"}}, + }, + { + "UPDATE mytable SET s = 'updated' WHERE i <> 9999;", + []sql.Row{{int64(3), int64(3)}}, + "SELECT * FROM mytable;", + []sql.Row{{int64(1), "updated"}, {int64(2), "updated"}, {int64(3), "updated"}}, + }, + { + "UPDATE floattable SET f32 = f32 + f32, f64 = f32 * f64 WHERE i = 2;", + []sql.Row{{int64(1), int64(1)}}, + "SELECT * FROM floattable WHERE i = 2;", + []sql.Row{{int64(2), float32(3.0), float64(4.5)}}, + }, + { + "UPDATE floattable SET f32 = 5, f32 = 4 WHERE i = 1;", + []sql.Row{{int64(1), int64(1)}}, + "SELECT f32 FROM floattable WHERE i = 1;", + []sql.Row{{float32(4.0)}}, + }, + { + "UPDATE mytable SET s = 'first row' WHERE i = 1;", + []sql.Row{{int64(1), int64(0)}}, + "SELECT * FROM mytable;", + []sql.Row{{int64(1), "first row"}, {int64(2), "second row"}, {int64(3), "third row"}}, + }, + { + "UPDATE niltable SET b = NULL WHERE f IS NULL;", + []sql.Row{{int64(2), int64(1)}}, + "SELECT * FROM niltable WHERE f IS NULL;", + []sql.Row{{int64(4), nil, nil}, {nil, nil, nil}}, + }, + { + "UPDATE mytable SET s = 'updated' ORDER BY i ASC LIMIT 2;", + []sql.Row{{int64(2), int64(2)}}, + "SELECT * FROM mytable;", + []sql.Row{{int64(1), "updated"}, {int64(2), "updated"}, {int64(3), "third row"}}, + }, + { + "UPDATE mytable SET s = 'updated' ORDER BY i DESC LIMIT 2;", + []sql.Row{{int64(2), int64(2)}}, + "SELECT * FROM mytable;", + []sql.Row{{int64(1), "first row"}, {int64(2), "updated"}, {int64(3), "updated"}}, + }, + { + "UPDATE mytable SET s = 'updated' ORDER BY i LIMIT 1 OFFSET 1;", + []sql.Row{{int64(1), int64(1)}}, + "SELECT * FROM mytable;", + []sql.Row{{int64(1), "first row"}, {int64(2), "updated"}, {int64(3), "third row"}}, + }, + { + "UPDATE mytable SET s = 'updated';", + []sql.Row{{int64(3), int64(3)}}, + "SELECT * FROM mytable;", + []sql.Row{{int64(1), "updated"}, {int64(2), "updated"}, {int64(3), "updated"}}, + }, + } + + for _, update := range updates { + e := newEngine(t) + ctx := newCtx() + testQueryWithContext(ctx, t, e, update.updateQuery, update.expectedUpdate) + testQueryWithContext(ctx, t, e, update.selectQuery, update.expectedSelect) + } +} + +func TestUpdateErrors(t *testing.T) { + var expectedFailures = []struct { + name string + query string + }{ + { + "invalid table", + "UPDATE doesnotexist SET i = 0;", + }, + { + "invalid column set", + "UPDATE mytable SET z = 0;", + }, + { + "invalid column set value", + "UPDATE mytable SET i = z;", + }, + { + "invalid column where", + "UPDATE mytable SET s = 'hi' WHERE z = 1;", + }, + { + "invalid column order by", + "UPDATE mytable SET s = 'hi' ORDER BY z;", + }, + { + "negative limit", + "UPDATE mytable SET s = 'hi' LIMIT -1;", + }, + { + "negative offset", + "UPDATE mytable SET s = 'hi' LIMIT 1 OFFSET -1;", + }, + { + "set null on non-nullable", + "UPDATE mytable SET s = NULL;", + }, + } + + for _, expectedFailure := range expectedFailures { + t.Run(expectedFailure.name, func(t *testing.T) { + _, _, err := newEngine(t).Query(newCtx(), expectedFailure.query) + require.Error(t, err) + }) + } } const testNumPartitions = 5 @@ -1163,7 +2388,7 @@ const testNumPartitions = 5 func TestAmbiguousColumnResolution(t *testing.T) { require := require.New(t) - table := mem.NewPartitionedTable("foo", sql.Schema{ + table := memory.NewPartitionedTable("foo", sql.Schema{ {Name: "a", Type: sql.Int64, Source: "foo"}, {Name: "b", Type: sql.Text, Source: "foo"}, }, testNumPartitions) @@ -1175,7 +2400,7 @@ func TestAmbiguousColumnResolution(t *testing.T) { sql.NewRow(int64(3), "baz"), ) - table2 := mem.NewPartitionedTable("bar", sql.Schema{ + table2 := memory.NewPartitionedTable("bar", sql.Schema{ {Name: "b", Type: sql.Text, Source: "bar"}, {Name: "c", Type: sql.Int64, Source: "bar"}, }, testNumPartitions) @@ -1186,7 +2411,7 @@ func TestAmbiguousColumnResolution(t *testing.T) { sql.NewRow("pux", int64(1)), ) - db := mem.NewDatabase("mydb") + db := memory.NewDatabase("mydb") db.AddTable("foo", table) db.AddTable("bar", table2) @@ -1219,13 +2444,14 @@ func TestAmbiguousColumnResolution(t *testing.T) { require.Equal(expected, rs) } -func TestDDL(t *testing.T) { +func TestCreateTable(t *testing.T) { require := require.New(t) e := newEngine(t) testQuery(t, e, - "CREATE TABLE t1(a INTEGER, b TEXT, c DATE,"+ - "d TIMESTAMP, e VARCHAR(20), f BLOB NOT NULL)", + "CREATE TABLE t1(a INTEGER, b TEXT, c DATE, "+ + "d TIMESTAMP, e VARCHAR(20), f BLOB NOT NULL, "+ + "b1 BOOL, b2 BOOLEAN NOT NULL, g DATETIME, h CHAR(40))", []sql.Row(nil), ) @@ -1242,15 +2468,96 @@ func TestDDL(t *testing.T) { {Name: "d", Type: sql.Timestamp, Nullable: true, Source: "t1"}, {Name: "e", Type: sql.Text, Nullable: true, Source: "t1"}, {Name: "f", Type: sql.Blob, Source: "t1"}, + {Name: "b1", Type: sql.Uint8, Nullable: true, Source: "t1"}, + {Name: "b2", Type: sql.Uint8, Source: "t1"}, + {Name: "g", Type: sql.Datetime, Nullable: true, Source: "t1"}, + {Name: "h", Type: sql.Text, Nullable: true, Source: "t1"}, } - require.Equal(s, testTable.Schema()) + require.Equal(s, testTable.Schema()) + + testQuery(t, e, + "CREATE TABLE t2 (a INTEGER NOT NULL PRIMARY KEY, "+ + "b VARCHAR(10) NOT NULL)", + []sql.Row(nil), + ) + + db, err = e.Catalog.Database("mydb") + require.NoError(err) + + testTable, ok = db.Tables()["t2"] + require.True(ok) + + s = sql.Schema{ + {Name: "a", Type: sql.Int32, Nullable: false, PrimaryKey: true, Source: "t2"}, + {Name: "b", Type: sql.Text, Nullable: false, Source: "t2"}, + } + + require.Equal(s, testTable.Schema()) + + testQuery(t, e, + "CREATE TABLE t3(a INTEGER NOT NULL,"+ + "b TEXT NOT NULL,"+ + "c bool, primary key (a,b))", + []sql.Row(nil), + ) + + db, err = e.Catalog.Database("mydb") + require.NoError(err) + + testTable, ok = db.Tables()["t3"] + require.True(ok) + + s = sql.Schema{ + {Name: "a", Type: sql.Int32, Nullable: false, PrimaryKey: true, Source: "t3"}, + {Name: "b", Type: sql.Text, Nullable: false, PrimaryKey: true, Source: "t3"}, + {Name: "c", Type: sql.Uint8, Nullable: true, Source: "t3"}, + } + + require.Equal(s, testTable.Schema()) +} + +func TestDropTable(t *testing.T) { + require := require.New(t) + + e := newEngine(t) + db, err := e.Catalog.Database("mydb") + require.NoError(err) + + _, ok := db.Tables()["mytable"] + require.True(ok) + + testQuery(t, e, + "DROP TABLE IF EXISTS mytable, not_exist", + []sql.Row(nil), + ) + + _, ok = db.Tables()["mytable"] + require.False(ok) + + _, ok = db.Tables()["othertable"] + require.True(ok) + _, ok = db.Tables()["tabletest"] + require.True(ok) + + testQuery(t, e, + "DROP TABLE IF EXISTS othertable, tabletest", + []sql.Row(nil), + ) + + _, ok = db.Tables()["othertable"] + require.False(ok) + _, ok = db.Tables()["tabletest"] + require.False(ok) + + _, _, err = e.Query(newCtx(), "DROP TABLE not_exist") + require.Error(err) } func TestNaturalJoin(t *testing.T) { require := require.New(t) - t1 := mem.NewPartitionedTable("t1", sql.Schema{ + t1 := memory.NewPartitionedTable("t1", sql.Schema{ {Name: "a", Type: sql.Text, Source: "t1"}, {Name: "b", Type: sql.Text, Source: "t1"}, {Name: "c", Type: sql.Text, Source: "t1"}, @@ -1263,7 +2570,7 @@ func TestNaturalJoin(t *testing.T) { sql.NewRow("a_3", "b_3", "c_3"), ) - t2 := mem.NewPartitionedTable("t2", sql.Schema{ + t2 := memory.NewPartitionedTable("t2", sql.Schema{ {Name: "a", Type: sql.Text, Source: "t2"}, {Name: "b", Type: sql.Text, Source: "t2"}, {Name: "d", Type: sql.Text, Source: "t2"}, @@ -1276,7 +2583,7 @@ func TestNaturalJoin(t *testing.T) { sql.NewRow("a_3", "b_3", "d_3"), ) - db := mem.NewDatabase("mydb") + db := memory.NewDatabase("mydb") db.AddTable("t1", t1) db.AddTable("t2", t2) @@ -1302,7 +2609,7 @@ func TestNaturalJoin(t *testing.T) { func TestNaturalJoinEqual(t *testing.T) { require := require.New(t) - t1 := mem.NewPartitionedTable("t1", sql.Schema{ + t1 := memory.NewPartitionedTable("t1", sql.Schema{ {Name: "a", Type: sql.Text, Source: "t1"}, {Name: "b", Type: sql.Text, Source: "t1"}, {Name: "c", Type: sql.Text, Source: "t1"}, @@ -1315,7 +2622,7 @@ func TestNaturalJoinEqual(t *testing.T) { sql.NewRow("a_3", "b_3", "c_3"), ) - t2 := mem.NewPartitionedTable("t2", sql.Schema{ + t2 := memory.NewPartitionedTable("t2", sql.Schema{ {Name: "a", Type: sql.Text, Source: "t2"}, {Name: "b", Type: sql.Text, Source: "t2"}, {Name: "c", Type: sql.Text, Source: "t2"}, @@ -1328,7 +2635,7 @@ func TestNaturalJoinEqual(t *testing.T) { sql.NewRow("a_3", "b_3", "c_3"), ) - db := mem.NewDatabase("mydb") + db := memory.NewDatabase("mydb") db.AddTable("t1", t1) db.AddTable("t2", t2) @@ -1354,7 +2661,7 @@ func TestNaturalJoinEqual(t *testing.T) { func TestNaturalJoinDisjoint(t *testing.T) { require := require.New(t) - t1 := mem.NewPartitionedTable("t1", sql.Schema{ + t1 := memory.NewPartitionedTable("t1", sql.Schema{ {Name: "a", Type: sql.Text, Source: "t1"}, }, testNumPartitions) @@ -1365,7 +2672,7 @@ func TestNaturalJoinDisjoint(t *testing.T) { sql.NewRow("a3"), ) - t2 := mem.NewPartitionedTable("t2", sql.Schema{ + t2 := memory.NewPartitionedTable("t2", sql.Schema{ {Name: "b", Type: sql.Text, Source: "t2"}, }, testNumPartitions) insertRows( @@ -1375,7 +2682,7 @@ func TestNaturalJoinDisjoint(t *testing.T) { sql.NewRow("b3"), ) - db := mem.NewDatabase("mydb") + db := memory.NewDatabase("mydb") db.AddTable("t1", t1) db.AddTable("t2", t2) @@ -1407,7 +2714,7 @@ func TestNaturalJoinDisjoint(t *testing.T) { func TestInnerNestedInNaturalJoins(t *testing.T) { require := require.New(t) - table1 := mem.NewPartitionedTable("table1", sql.Schema{ + table1 := memory.NewPartitionedTable("table1", sql.Schema{ {Name: "i", Type: sql.Int32, Source: "table1"}, {Name: "f", Type: sql.Float64, Source: "table1"}, {Name: "t", Type: sql.Text, Source: "table1"}, @@ -1420,7 +2727,7 @@ func TestInnerNestedInNaturalJoins(t *testing.T) { sql.NewRow(int32(10), float64(2.1), "table1"), ) - table2 := mem.NewPartitionedTable("table2", sql.Schema{ + table2 := memory.NewPartitionedTable("table2", sql.Schema{ {Name: "i2", Type: sql.Int32, Source: "table2"}, {Name: "f2", Type: sql.Float64, Source: "table2"}, {Name: "t2", Type: sql.Text, Source: "table2"}, @@ -1433,7 +2740,7 @@ func TestInnerNestedInNaturalJoins(t *testing.T) { sql.NewRow(int32(20), float64(2.2), "table2"), ) - table3 := mem.NewPartitionedTable("table3", sql.Schema{ + table3 := memory.NewPartitionedTable("table3", sql.Schema{ {Name: "i", Type: sql.Int32, Source: "table3"}, {Name: "f2", Type: sql.Float64, Source: "table3"}, {Name: "t3", Type: sql.Text, Source: "table3"}, @@ -1441,12 +2748,12 @@ func TestInnerNestedInNaturalJoins(t *testing.T) { insertRows( t, table3, - sql.NewRow(int32(1), float64(2.3), "table3"), - sql.NewRow(int32(2), float64(2.3), "table3"), - sql.NewRow(int32(30), float64(2.3), "table3"), + sql.NewRow(int32(1), float64(2.2), "table3"), + sql.NewRow(int32(2), float64(2.2), "table3"), + sql.NewRow(int32(30), float64(2.2), "table3"), ) - db := mem.NewDatabase("mydb") + db := memory.NewDatabase("mydb") db.AddTable("table1", table1) db.AddTable("table2", table2) db.AddTable("table3", table3) @@ -1476,6 +2783,8 @@ func testQuery(t *testing.T, e *sqle.Engine, q string, expected []sql.Row) { } func testQueryWithContext(ctx *sql.Context, t *testing.T, e *sqle.Engine, q string, expected []sql.Row) { + orderBy := strings.Contains(strings.ToUpper(q), " ORDER BY ") + t.Run(q, func(t *testing.T) { require := require.New(t) _, iter, err := e.Query(ctx, q) @@ -1484,7 +2793,11 @@ func testQueryWithContext(ctx *sql.Context, t *testing.T, e *sqle.Engine, q stri rows, err := sql.RowIterToRows(iter) require.NoError(err) - require.ElementsMatch(expected, rows) + if orderBy { + require.Equal(expected, rows) + } else { + require.ElementsMatch(expected, rows) + } }) } @@ -1493,7 +2806,7 @@ func newEngine(t *testing.T) *sqle.Engine { } func newEngineWithParallelism(t *testing.T, parallelism int) *sqle.Engine { - table := mem.NewPartitionedTable("mytable", sql.Schema{ + table := memory.NewPartitionedTable("mytable", sql.Schema{ {Name: "i", Type: sql.Int64, Source: "mytable"}, {Name: "s", Type: sql.Text, Source: "mytable"}, }, testNumPartitions) @@ -1505,7 +2818,7 @@ func newEngineWithParallelism(t *testing.T, parallelism int) *sqle.Engine { sql.NewRow(int64(3), "third row"), ) - table2 := mem.NewPartitionedTable("othertable", sql.Schema{ + table2 := memory.NewPartitionedTable("othertable", sql.Schema{ {Name: "s2", Type: sql.Text, Source: "othertable"}, {Name: "i2", Type: sql.Int64, Source: "othertable"}, }, testNumPartitions) @@ -1517,7 +2830,7 @@ func newEngineWithParallelism(t *testing.T, parallelism int) *sqle.Engine { sql.NewRow("third", int64(1)), ) - table3 := mem.NewPartitionedTable("tabletest", sql.Schema{ + table3 := memory.NewPartitionedTable("tabletest", sql.Schema{ {Name: "i", Type: sql.Int32, Source: "tabletest"}, {Name: "s", Type: sql.Text, Source: "tabletest"}, }, testNumPartitions) @@ -1529,7 +2842,7 @@ func newEngineWithParallelism(t *testing.T, parallelism int) *sqle.Engine { sql.NewRow(int64(3), "third row"), ) - table4 := mem.NewPartitionedTable("other_table", sql.Schema{ + table4 := memory.NewPartitionedTable("other_table", sql.Schema{ {Name: "text", Type: sql.Text, Source: "tabletest"}, {Name: "number", Type: sql.Int32, Source: "tabletest"}, }, testNumPartitions) @@ -1541,12 +2854,105 @@ func newEngineWithParallelism(t *testing.T, parallelism int) *sqle.Engine { sql.NewRow("c", int32(0)), ) - db := mem.NewDatabase("mydb") + bigtable := memory.NewPartitionedTable("bigtable", sql.Schema{ + {Name: "t", Type: sql.Text, Source: "bigtable"}, + {Name: "n", Type: sql.Int64, Source: "bigtable"}, + }, testNumPartitions) + + insertRows( + t, bigtable, + sql.NewRow("a", int64(1)), + sql.NewRow("s", int64(2)), + sql.NewRow("f", int64(3)), + sql.NewRow("g", int64(1)), + sql.NewRow("h", int64(2)), + sql.NewRow("j", int64(3)), + sql.NewRow("k", int64(1)), + sql.NewRow("l", int64(2)), + sql.NewRow("ñ", int64(4)), + sql.NewRow("z", int64(5)), + sql.NewRow("x", int64(6)), + sql.NewRow("c", int64(7)), + sql.NewRow("v", int64(8)), + sql.NewRow("b", int64(9)), + ) + + floatTable := memory.NewPartitionedTable("floattable", sql.Schema{ + {Name: "i", Type: sql.Int64, Source: "floattable"}, + {Name: "f32", Type: sql.Float32, Source: "floattable"}, + {Name: "f64", Type: sql.Float64, Source: "floattable"}, + }, testNumPartitions) + + insertRows( + t, floatTable, + sql.NewRow(int64(1), float32(1.0), float64(1.0)), + sql.NewRow(int64(2), float32(1.5), float64(1.5)), + sql.NewRow(int64(3), float32(2.0), float64(2.0)), + sql.NewRow(int64(4), float32(2.5), float64(2.5)), + sql.NewRow(int64(-1), float32(-1.0), float64(-1.0)), + sql.NewRow(int64(-2), float32(-1.5), float64(-1.5)), + ) + + nilTable := memory.NewPartitionedTable("niltable", sql.Schema{ + {Name: "i", Type: sql.Int64, Source: "niltable", Nullable: true}, + {Name: "b", Type: sql.Boolean, Source: "niltable", Nullable: true}, + {Name: "f", Type: sql.Float64, Source: "niltable", Nullable: true}, + }, testNumPartitions) + + insertRows( + t, nilTable, + sql.NewRow(int64(1), true, float64(1.0)), + sql.NewRow(int64(2), nil, float64(2.0)), + sql.NewRow(nil, false, float64(3.0)), + sql.NewRow(int64(4), true, nil), + sql.NewRow(nil, nil, nil), + ) + + newlineTable := memory.NewPartitionedTable("newlinetable", sql.Schema{ + {Name: "i", Type: sql.Int64, Source: "newlinetable"}, + {Name: "s", Type: sql.Text, Source: "newlinetable"}, + }, testNumPartitions) + + insertRows( + t, newlineTable, + sql.NewRow(int64(1), "\nthere is some text in here"), + sql.NewRow(int64(2), "there is some\ntext in here"), + sql.NewRow(int64(3), "there is some text\nin here"), + sql.NewRow(int64(4), "there is some text in here\n"), + sql.NewRow(int64(5), "there is some text in here"), + ) + + typestable := memory.NewPartitionedTable("typestable", sql.Schema{ + {Name: "id", Type: sql.Int64, Source: "typestable"}, + {Name: "i8", Type: sql.Int8, Source: "typestable", Nullable: true}, + {Name: "i16", Type: sql.Int16, Source: "typestable", Nullable: true}, + {Name: "i32", Type: sql.Int32, Source: "typestable", Nullable: true}, + {Name: "i64", Type: sql.Int64, Source: "typestable", Nullable: true}, + {Name: "u8", Type: sql.Uint8, Source: "typestable", Nullable: true}, + {Name: "u16", Type: sql.Uint16, Source: "typestable", Nullable: true}, + {Name: "u32", Type: sql.Uint32, Source: "typestable", Nullable: true}, + {Name: "u64", Type: sql.Uint64, Source: "typestable", Nullable: true}, + {Name: "f32", Type: sql.Float32, Source: "typestable", Nullable: true}, + {Name: "f64", Type: sql.Float64, Source: "typestable", Nullable: true}, + {Name: "ti", Type: sql.Timestamp, Source: "typestable", Nullable: true}, + {Name: "da", Type: sql.Date, Source: "typestable", Nullable: true}, + {Name: "te", Type: sql.Text, Source: "typestable", Nullable: true}, + {Name: "bo", Type: sql.Boolean, Source: "typestable", Nullable: true}, + {Name: "js", Type: sql.JSON, Source: "typestable", Nullable: true}, + {Name: "bl", Type: sql.Blob, Source: "typestable", Nullable: true}, + }, testNumPartitions) + + db := memory.NewDatabase("mydb") db.AddTable("mytable", table) db.AddTable("othertable", table2) db.AddTable("tabletest", table3) + db.AddTable("bigtable", bigtable) + db.AddTable("floattable", floatTable) + db.AddTable("niltable", nilTable) + db.AddTable("newlinetable", newlineTable) + db.AddTable("typestable", typestable) - db2 := mem.NewDatabase("foo") + db2 := memory.NewDatabase("foo") db2.AddTable("other_table", table4) catalog := sql.NewCatalog() @@ -1564,8 +2970,8 @@ func newEngineWithParallelism(t *testing.T, parallelism int) *sqle.Engine { return sqle.New(catalog, a, new(sqle.Config)) } -const expectedTree = `Offset(2) - └─ Limit(5) +const expectedTree = `Limit(5) + └─ Offset(2) └─ Project(t.foo, bar.baz) └─ Filter(foo > qux) └─ InnerJoin(foo = baz) @@ -1615,154 +3021,10 @@ func TestInvalidRegexp(t *testing.T) { require.Error(err) } -func TestIndexes(t *testing.T) { - e := newEngine(t) - - tmpDir, err := ioutil.TempDir(os.TempDir(), "pilosa-test") - require.NoError(t, err) - - require.NoError(t, os.MkdirAll(tmpDir, 0644)) - e.Catalog.RegisterIndexDriver(pilosa.NewDriver(tmpDir)) - - _, _, err = e.Query( - newCtx(), - "CREATE INDEX myidx ON mytable USING pilosa (i) WITH (async = false)", - ) - require.NoError(t, err) - - _, _, err = e.Query( - newCtx(), - "CREATE INDEX myidx_multi ON mytable USING pilosa (i, s) WITH (async = false)", - ) - require.NoError(t, err) - - defer func() { - done, err := e.Catalog.DeleteIndex("mydb", "myidx", true) - require.NoError(t, err) - <-done - - done, err = e.Catalog.DeleteIndex("foo", "myidx_multi", true) - require.NoError(t, err) - <-done - }() - - testCases := []struct { - query string - expected []sql.Row - }{ - { - "SELECT * FROM mytable WHERE i = 2", - []sql.Row{ - {int64(2), "second row"}, - }, - }, - { - "SELECT * FROM mytable WHERE i > 1", - []sql.Row{ - {int64(3), "third row"}, - {int64(2), "second row"}, - }, - }, - { - "SELECT * FROM mytable WHERE i < 3", - []sql.Row{ - {int64(1), "first row"}, - {int64(2), "second row"}, - }, - }, - { - "SELECT * FROM mytable WHERE i <= 2", - []sql.Row{ - {int64(2), "second row"}, - {int64(1), "first row"}, - }, - }, - { - "SELECT * FROM mytable WHERE i >= 2", - []sql.Row{ - {int64(2), "second row"}, - {int64(3), "third row"}, - }, - }, - { - "SELECT * FROM mytable WHERE i = 2 AND s = 'second row'", - []sql.Row{ - {int64(2), "second row"}, - }, - }, - { - "SELECT * FROM mytable WHERE i = 2 AND s = 'third row'", - ([]sql.Row)(nil), - }, - { - "SELECT * FROM mytable WHERE i BETWEEN 1 AND 2", - []sql.Row{ - {int64(1), "first row"}, - {int64(2), "second row"}, - }, - }, - { - "SELECT * FROM mytable WHERE i = 1 OR i = 2", - []sql.Row{ - {int64(1), "first row"}, - {int64(2), "second row"}, - }, - }, - { - "SELECT * FROM mytable WHERE i = 1 AND i = 2", - ([]sql.Row)(nil), - }, - } - - for _, tt := range testCases { - t.Run(tt.query, func(t *testing.T) { - require := require.New(t) - - tracer := new(test.MemTracer) - ctx := sql.NewContext(context.TODO(), sql.WithTracer(tracer)) - - _, it, err := e.Query(ctx, tt.query) - require.NoError(err) - - rows, err := sql.RowIterToRows(it) - require.NoError(err) - - require.ElementsMatch(tt.expected, rows) - require.Equal("plan.ResolvedTable", tracer.Spans[len(tracer.Spans)-1]) - }) - } -} - -func TestCreateIndex(t *testing.T) { - require := require.New(t) - e := newEngine(t) - - tmpDir, err := ioutil.TempDir(os.TempDir(), "pilosa-test") - require.NoError(err) - - require.NoError(os.MkdirAll(tmpDir, 0644)) - e.Catalog.RegisterIndexDriver(pilosa.NewDriver(tmpDir)) - - _, iter, err := e.Query(newCtx(), "CREATE INDEX myidx ON mytable USING pilosa (i)") - require.NoError(err) - rows, err := sql.RowIterToRows(iter) - require.NoError(err) - require.Len(rows, 0) - - defer func() { - time.Sleep(1 * time.Second) - done, err := e.Catalog.DeleteIndex("foo", "myidx", true) - require.NoError(err) - <-done - - require.NoError(os.RemoveAll(tmpDir)) - }() -} - func TestOrderByGroupBy(t *testing.T) { require := require.New(t) - table := mem.NewPartitionedTable("members", sql.Schema{ + table := memory.NewPartitionedTable("members", sql.Schema{ {Name: "id", Type: sql.Int64, Source: "members"}, {Name: "team", Type: sql.Text, Source: "members"}, }, testNumPartitions) @@ -1777,7 +3039,7 @@ func TestOrderByGroupBy(t *testing.T) { sql.NewRow(int64(8), "purple"), ) - db := mem.NewDatabase("db") + db := memory.NewDatabase("db") db.AddTable("members", table) e := sqle.NewDefault() @@ -1793,9 +3055,9 @@ func TestOrderByGroupBy(t *testing.T) { require.NoError(err) expected := []sql.Row{ - {"purple", int32(1)}, - {"red", int32(2)}, - {"orange", int32(3)}, + {"purple", int64(1)}, + {"red", int64(2)}, + {"orange", int64(3)}, } require.Equal(expected, rows) @@ -1810,6 +3072,12 @@ func TestOrderByGroupBy(t *testing.T) { require.NoError(err) require.Equal(expected, rows) + + _, _, err = e.Query( + newCtx(), + "SELECT team, COUNT(*) FROM members GROUP BY team ORDER BY columndoesnotexist", + ) + require.Error(err) } func TestTracing(t *testing.T) { @@ -1857,12 +3125,12 @@ func TestTracing(t *testing.T) { func TestReadOnly(t *testing.T) { require := require.New(t) - table := mem.NewPartitionedTable("mytable", sql.Schema{ + table := memory.NewPartitionedTable("mytable", sql.Schema{ {Name: "i", Type: sql.Int64, Source: "mytable"}, {Name: "s", Type: sql.Text, Source: "mytable"}, }, testNumPartitions) - db := mem.NewDatabase("mydb") + db := memory.NewDatabase("mydb") db.AddTable("mytable", table) catalog := sql.NewCatalog() @@ -1908,7 +3176,7 @@ func TestSessionVariables(t *testing.T) { rows, err := sql.RowIterToRows(iter) require.NoError(err) - require.Equal([]sql.Row{{int64(1), ",STRICT_TRANS_TABLES"}}, rows) + require.Equal([]sql.Row{{int8(1), ",STRICT_TRANS_TABLES"}}, rows) } func TestSessionVariablesONOFF(t *testing.T) { @@ -1963,11 +3231,11 @@ func TestUse(t *testing.T) { func TestLocks(t *testing.T) { require := require.New(t) - t1 := newLockableTable(mem.NewTable("t1", nil)) - t2 := newLockableTable(mem.NewTable("t2", nil)) - t3 := mem.NewTable("t3", nil) + t1 := newLockableTable(memory.NewTable("t1", nil)) + t2 := newLockableTable(memory.NewTable("t2", nil)) + t3 := memory.NewTable("t3", nil) catalog := sql.NewCatalog() - db := mem.NewDatabase("db") + db := memory.NewDatabase("db") db.AddTable("t1", t1) db.AddTable("t2", t2) db.AddTable("t3", t3) @@ -1996,6 +3264,260 @@ func TestLocks(t *testing.T) { require.Equal(1, t2.unlocks) } +func TestDescribeNoPruneColumns(t *testing.T) { + require := require.New(t) + ctx := newCtx() + e := newEngine(t) + query := `DESCRIBE FORMAT=TREE SELECT SUBSTRING(s, 1, 1) as foo, s, i FROM mytable WHERE foo = 'f'` + parsed, err := parse.Parse(ctx, query) + require.NoError(err) + result, err := e.Analyzer.Analyze(ctx, parsed) + require.NoError(err) + + qp, ok := result.(*plan.QueryProcess) + require.True(ok) + + d, ok := qp.Child.(*plan.DescribeQuery) + require.True(ok) + + p, ok := d.Child.(*plan.Project) + require.True(ok) + + require.Len(p.Schema(), 3) +} + +func TestDeleteFrom(t *testing.T) { + var deletions = []struct { + deleteQuery string + expectedDelete []sql.Row + selectQuery string + expectedSelect []sql.Row + }{ + { + "DELETE FROM mytable;", + []sql.Row{{int64(3)}}, + "SELECT * FROM mytable;", + []sql.Row{}, + }, + { + "DELETE FROM mytable WHERE i = 2;", + []sql.Row{{int64(1)}}, + "SELECT * FROM mytable;", + []sql.Row{{int64(1), "first row"}, {int64(3), "third row"}}, + }, + { + "DELETE FROM mytable WHERE i < 3;", + []sql.Row{{int64(2)}}, + "SELECT * FROM mytable;", + []sql.Row{{int64(3), "third row"}}, + }, + { + "DELETE FROM mytable WHERE i > 1;", + []sql.Row{{int64(2)}}, + "SELECT * FROM mytable;", + []sql.Row{{int64(1), "first row"}}, + }, + { + "DELETE FROM mytable WHERE i <= 2;", + []sql.Row{{int64(2)}}, + "SELECT * FROM mytable;", + []sql.Row{{int64(3), "third row"}}, + }, + { + "DELETE FROM mytable WHERE i >= 2;", + []sql.Row{{int64(2)}}, + "SELECT * FROM mytable;", + []sql.Row{{int64(1), "first row"}}, + }, + { + "DELETE FROM mytable WHERE s = 'first row';", + []sql.Row{{int64(1)}}, + "SELECT * FROM mytable;", + []sql.Row{{int64(2), "second row"}, {int64(3), "third row"}}, + }, + { + "DELETE FROM mytable WHERE s <> 'dne';", + []sql.Row{{int64(3)}}, + "SELECT * FROM mytable;", + []sql.Row{}, + }, + { + "DELETE FROM mytable WHERE s LIKE '%row';", + []sql.Row{{int64(3)}}, + "SELECT * FROM mytable;", + []sql.Row{}, + }, + { + "DELETE FROM mytable WHERE s = 'dne';", + []sql.Row{{int64(0)}}, + "SELECT * FROM mytable;", + []sql.Row{{int64(1), "first row"}, {int64(2), "second row"}, {int64(3), "third row"}}, + }, + { + "DELETE FROM mytable WHERE i = 'invalid';", + []sql.Row{{int64(0)}}, + "SELECT * FROM mytable;", + []sql.Row{{int64(1), "first row"}, {int64(2), "second row"}, {int64(3), "third row"}}, + }, + { + "DELETE FROM mytable ORDER BY i ASC LIMIT 2;", + []sql.Row{{int64(2)}}, + "SELECT * FROM mytable;", + []sql.Row{{int64(3), "third row"}}, + }, + { + "DELETE FROM mytable ORDER BY i DESC LIMIT 1;", + []sql.Row{{int64(1)}}, + "SELECT * FROM mytable;", + []sql.Row{{int64(1), "first row"}, {int64(2), "second row"}}, + }, + { + "DELETE FROM mytable ORDER BY i DESC LIMIT 1 OFFSET 1;", + []sql.Row{{int64(1)}}, + "SELECT * FROM mytable;", + []sql.Row{{int64(1), "first row"}, {int64(3), "third row"}}, + }, + } + + for _, deletion := range deletions { + e := newEngine(t) + ctx := newCtx() + testQueryWithContext(ctx, t, e, deletion.deleteQuery, deletion.expectedDelete) + testQueryWithContext(ctx, t, e, deletion.selectQuery, deletion.expectedSelect) + } +} + +func TestDeleteFromErrors(t *testing.T) { + var expectedFailures = []struct { + name string + query string + }{ + { + "invalid table", + "DELETE FROM invalidtable WHERE x < 1;", + }, + { + "invalid column", + "DELETE FROM mytable WHERE z = 'dne';", + }, + { + "negative limit", + "DELETE FROM mytable LIMIT -1;", + }, + { + "negative offset", + "DELETE FROM mytable LIMIT 1 OFFSET -1;", + }, + { + "missing keyword from", + "DELETE mytable WHERE id = 1;", + }, + } + + for _, expectedFailure := range expectedFailures { + t.Run(expectedFailure.name, func(t *testing.T) { + _, _, err := newEngine(t).Query(newCtx(), expectedFailure.query) + require.Error(t, err) + }) + } +} + +type mockSpan struct { + opentracing.Span + finished bool +} + +func (m *mockSpan) Finish() { + m.finished = true +} + +func TestRootSpanFinish(t *testing.T) { + e := newEngine(t) + fakeSpan := &mockSpan{Span: opentracing.NoopTracer{}.StartSpan("")} + ctx := sql.NewContext( + context.Background(), + sql.WithRootSpan(fakeSpan), + ) + + _, iter, err := e.Query(ctx, "SELECT 1") + require.NoError(t, err) + + _, err = sql.RowIterToRows(iter) + require.NoError(t, err) + + require.True(t, fakeSpan.finished) +} + +var generatorQueries = []struct { + query string + expected []sql.Row +}{ + { + `SELECT a, EXPLODE(b), c FROM t`, + []sql.Row{ + {int64(1), "a", "first"}, + {int64(1), "b", "first"}, + {int64(2), "c", "second"}, + {int64(2), "d", "second"}, + {int64(3), "e", "third"}, + {int64(3), "f", "third"}, + }, + }, + { + `SELECT a, EXPLODE(b) AS x, c FROM t`, + []sql.Row{ + {int64(1), "a", "first"}, + {int64(1), "b", "first"}, + {int64(2), "c", "second"}, + {int64(2), "d", "second"}, + {int64(3), "e", "third"}, + {int64(3), "f", "third"}, + }, + }, + { + `SELECT EXPLODE(SPLIT(c, "")) FROM t LIMIT 5`, + []sql.Row{ + {"f"}, + {"i"}, + {"r"}, + {"s"}, + {"t"}, + }, + }, + { + `SELECT a, EXPLODE(b) AS x, c FROM t WHERE x = 'e'`, + []sql.Row{ + {int64(3), "e", "third"}, + }, + }, +} + +func TestGenerators(t *testing.T) { + table := memory.NewPartitionedTable("t", sql.Schema{ + {Name: "a", Type: sql.Int64, Source: "t"}, + {Name: "b", Type: sql.Array(sql.Text), Source: "t"}, + {Name: "c", Type: sql.Text, Source: "t"}, + }, testNumPartitions) + + insertRows( + t, table, + sql.NewRow(int64(1), []interface{}{"a", "b"}, "first"), + sql.NewRow(int64(2), []interface{}{"c", "d"}, "second"), + sql.NewRow(int64(3), []interface{}{"e", "f"}, "third"), + ) + + db := memory.NewDatabase("db") + db.AddTable("t", table) + + catalog := sql.NewCatalog() + catalog.AddDatabase(db) + e := sqle.New(catalog, analyzer.NewDefault(catalog), new(sqle.Config)) + + for _, q := range generatorQueries { + testQuery(t, e, q.query, q.expected) + } +} + func insertRows(t *testing.T, table sql.Inserter, rows ...sql.Row) { t.Helper() @@ -2026,6 +3548,14 @@ func newLockableTable(t sql.Table) *lockableTable { return &lockableTable{Table: t} } +func timeParse(layout string, value string) time.Time { + t, err := time.Parse(layout, value) + if err != nil { + panic(err) + } + return t +} + var _ sql.Lockable = (*lockableTable)(nil) func (l *lockableTable) Lock(ctx *sql.Context, write bool) error { diff --git a/example_test.go b/example_test.go index ec7653511..eddcae535 100644 --- a/example_test.go +++ b/example_test.go @@ -4,9 +4,9 @@ import ( "fmt" "io" - "gopkg.in/src-d/go-mysql-server.v0" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" ) func Example() { @@ -45,8 +45,8 @@ func checkIfError(err error) { } func createTestDatabase() sql.Database { - db := mem.NewDatabase("test") - table := mem.NewTable("mytable", sql.Schema{ + db := memory.NewDatabase("test") + table := memory.NewTable("mytable", sql.Schema{ {Name: "name", Type: sql.Text, Source: "mytable"}, {Name: "email", Type: sql.Text, Source: "mytable"}, }) diff --git a/go.mod b/go.mod index e20938986..e8aab26b7 100644 --- a/go.mod +++ b/go.mod @@ -1,55 +1,28 @@ -module gopkg.in/src-d/go-mysql-server.v0 +module github.com/src-d/go-mysql-server require ( - github.com/BurntSushi/toml v0.3.1 // indirect - github.com/CAFxX/gcnotifier v0.0.0-20170518020117-39b0596a2da3 // indirect - github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 // indirect - github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 // indirect - github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect - github.com/boltdb/bolt v1.3.1 - github.com/cespare/xxhash v1.1.0 // indirect - github.com/circonus-labs/circonus-gometrics v2.2.5+incompatible // indirect - github.com/circonus-labs/circonusllhist v0.1.3 // indirect - github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect - github.com/go-ole/go-ole v1.2.2 // indirect + github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect + github.com/VividCortex/gohistogram v1.0.0 // indirect + github.com/go-kit/kit v0.8.0 + github.com/go-ole/go-ole v1.2.4 // indirect github.com/go-sql-driver/mysql v1.4.1 - github.com/gogo/protobuf v1.2.0 // indirect - github.com/google/go-cmp v0.2.0 // indirect - github.com/gorilla/context v1.1.1 // indirect - github.com/gorilla/handlers v1.4.0 // indirect - github.com/gorilla/mux v1.6.2 // indirect - github.com/hashicorp/consul v1.4.0 // indirect - github.com/hashicorp/go-immutable-radix v1.0.0 // indirect - github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c // indirect - github.com/hashicorp/go-multierror v1.0.0 // indirect - github.com/hashicorp/go-retryablehttp v0.5.0 // indirect - github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86 // indirect - github.com/hashicorp/memberlist v0.1.0 // indirect - github.com/hashicorp/serf v0.8.1 // indirect - github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d // indirect - github.com/miekg/dns v1.1.1 // indirect + github.com/gogo/protobuf v1.2.1 // indirect + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b + github.com/golang/protobuf v1.3.0 // indirect + github.com/hashicorp/golang-lru v0.5.3 github.com/mitchellh/hashstructure v1.0.0 github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852 github.com/opentracing/opentracing-go v1.0.2 - github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c // indirect - github.com/pelletier/go-toml v1.2.0 // indirect - github.com/pilosa/pilosa v1.2.0 - github.com/pkg/errors v0.8.0 // indirect - github.com/prometheus/client_golang v0.9.2 // indirect + github.com/pilosa/pilosa v1.3.0 github.com/sanity-io/litter v1.1.0 - github.com/satori/go.uuid v1.2.0 // indirect - github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect - github.com/shirou/gopsutil v2.18.11+incompatible // indirect github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect - github.com/sirupsen/logrus v1.1.1 + github.com/sirupsen/logrus v1.3.0 github.com/spf13/cast v1.3.0 - github.com/stretchr/testify v1.2.2 - github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926 // indirect - github.com/uber/jaeger-client-go v2.15.0+incompatible // indirect - github.com/uber/jaeger-lib v2.0.0+incompatible // indirect - google.golang.org/grpc v1.16.0 // indirect + github.com/src-d/go-oniguruma v1.0.0 + github.com/stretchr/testify v1.3.0 + go.etcd.io/bbolt v1.3.2 + google.golang.org/grpc v1.19.0 // indirect gopkg.in/src-d/go-errors.v1 v1.0.0 - gopkg.in/src-d/go-vitess.v1 v1.4.0 - gopkg.in/vmihailenco/msgpack.v2 v2.9.1 // indirect gopkg.in/yaml.v2 v2.2.2 + vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible ) diff --git a/go.sum b/go.sum new file mode 100644 index 000000000..c2918275d --- /dev/null +++ b/go.sum @@ -0,0 +1,179 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d h1:n0G4ckjMEj7bWuGYUX0i8YlBeBBJuZ+HEHvHfyBDZtI= +github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d/go.mod h1:Rn2zM2MnHze07LwkneP48TWt6UiZhzQTwCvw6djVGfE= +github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5VkQn65PNbbyACDC8xw8Hpp/NEurdPmQDQ= +github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE= +github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= +github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= +github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.0 h1:kbxbvI4Un1LUWKxufD+BiE6AEExYYgkQLQmLFqA1LFk= +github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/gorilla/handlers v1.3.0 h1:tsg9qP3mjt1h4Roxp+M1paRjrVBfPSOpBuVclh6YluI= +github.com/gorilla/handlers v1.3.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U= +github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= +github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/memberlist v0.1.3 h1:EmmoJme1matNzb+hMpDuR/0sbJSUisxyqBGG676r31M= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/hashstructure v1.0.0 h1:ZkRJX1CyOoTkar7p/mLS5TZU4nJ1Rn/F8u9dGS02Q3Y= +github.com/mitchellh/hashstructure v1.0.0/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852 h1:Yl0tPBa8QPjGmesFh1D0rDy+q1Twx6FyU7VWHi8wZbI= +github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852/go.mod h1:eqOVx5Vwu4gd2mmMZvVZsgIqNSaW3xxRThUJ0k/TPk4= +github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pilosa/pilosa v1.3.0 h1:P27JB4tIqAN4Yc2Fw7wS5neD7JNkFKRUmwfyV87JMwQ= +github.com/pilosa/pilosa v1.3.0/go.mod h1:97yLL9mpUqOj9naKu5XA/b/U6JLe3JGGUlc2HOTDw+A= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001 h1:YDeskXpkNDhPdWN3REluVa46HQOVuVkjkd2sWnrABNQ= +github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/sanity-io/litter v1.1.0 h1:BllcKWa3VbZmOZbDCoszYLk7zCsKHz5Beossi8SUcTc= +github.com/sanity-io/litter v1.1.0/go.mod h1:CJ0VCw2q4qKU7LaQr3n7UOSHzgEMgcGco7N/SkZQPjw= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= +github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= +github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= +github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/src-d/go-oniguruma v1.0.0 h1:JDk5PUAjreGsGAKLsoDLNmrsaryjJ5RqT3h+Si6aw/E= +github.com/src-d/go-oniguruma v1.0.0/go.mod h1:chVbff8kcVtmrhxtZ3yBVLLquXbzCS6DrxQaAK/CeqM= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk= +github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo= +github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3 h1:KYQXGkl6vs02hK7pK4eIbw0NpNPedieTSTEiJ//bwGs= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519 h1:x6rhz8Y9CjbgQkccRGmELH6K+LJj7tOoh3XWeC1yaQM= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5 h1:x6r4Jo0KNzOOzYd8lbcRsqjuqEASK6ob3auvWYM4/8U= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a h1:1n5lsVfiQW3yfsRGu98756EH1YthsFqr/5mxHduZW2A= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180831171423-11092d34479b h1:lohp5blsw53GBXtLyLNaTXPXS9pJ1tiTw61ZHUoE9Qw= +google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/grpc v1.19.0 h1:cfg4PD8YEdSFnm7qLV4++93WcmhH2nIUhMjhdCvl3j8= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/src-d/go-errors.v1 v1.0.0 h1:cooGdZnCjYbeS1zb1s6pVAAimTdKceRrpn7aKOnNIfc= +gopkg.in/src-d/go-errors.v1 v1.0.0/go.mod h1:q1cBlomlw2FnDBDNGlnh6X0jPihy+QxZfMMNxPCbdYg= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I= +modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= +modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE= +modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible h1:GWnLrAdetgJM0Co5bwwczO49iFZBSInpyGAT77BP9Y0= +vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible/go.mod h1:h4qvkyNYTOC0xI+vcidSWoka0gQAZc9ZPHbkHo48gP0= diff --git a/internal/regex/regex.go b/internal/regex/regex.go index e630731e6..33b92e1ff 100644 --- a/internal/regex/regex.go +++ b/internal/regex/regex.go @@ -1,6 +1,9 @@ package regex -import errors "gopkg.in/src-d/go-errors.v1" +import ( + "github.com/go-kit/kit/metrics/discard" + errors "gopkg.in/src-d/go-errors.v1" +) var ( // ErrRegexAlreadyRegistered is returned when there is a previously @@ -21,8 +24,22 @@ type Matcher interface { Match(text string) bool } +// Disposer interface is used to release resources. +// The interface should be implemented by all go binding for native C libraries +type Disposer interface { + Dispose() +} + // Constructor creates a new Matcher. -type Constructor func(re string) (Matcher, error) +type Constructor func(re string) (Matcher, Disposer, error) + +var ( + // CompileHistogram describes a regexp compile time. + CompileHistogram = discard.NewHistogram() + + // MatchHistogram describes a regexp match time. + MatchHistogram = discard.NewHistogram() +) // Register add a new regex engine to the registry. func Register(name string, c Constructor) error { @@ -56,10 +73,10 @@ func Engines() []string { } // New creates a new Matcher with the specified regex engine. -func New(name, re string) (Matcher, error) { +func New(name, re string) (Matcher, Disposer, error) { n, ok := registry[name] if !ok { - return nil, ErrRegexNotFound.New(name) + return nil, nil, ErrRegexNotFound.New(name) } return n(re) @@ -70,13 +87,11 @@ func Default() string { if defaultEngine != "" { return defaultEngine } - - _, ok := registry["oniguruma"] - if ok { - return "oniguruma" + if _, ok := registry["go"]; ok { + return "go" } - return "go" + return "oniguruma" } // SetDefault sets the regex engine returned by Default. diff --git a/internal/regex/regex_go.go b/internal/regex/regex_go.go index 6113f518e..e4c1be7c5 100644 --- a/internal/regex/regex_go.go +++ b/internal/regex/regex_go.go @@ -1,6 +1,9 @@ package regex -import "regexp" +import ( + "regexp" + "time" +) // Go holds go regex engine Matcher. type Go struct { @@ -9,21 +12,28 @@ type Go struct { // Match implements Matcher interface. func (r *Go) Match(s string) bool { + t := time.Now() + defer MatchHistogram.With("string", s, "duration", "seconds").Observe(time.Since(t).Seconds()) + return r.reg.MatchString(s) } +// Dispose implements Disposer interface. +func (*Go) Dispose() {} + // NewGo creates a new Matcher using go regex engine. -func NewGo(re string) (Matcher, error) { +func NewGo(re string) (Matcher, Disposer, error) { + t := time.Now() reg, err := regexp.Compile(re) if err != nil { - return nil, err + return nil, nil, err } + CompileHistogram.With("regex", re, "duration", "seconds").Observe(time.Since(t).Seconds()) r := Go{ reg: reg, } - - return &r, nil + return &r, &r, nil } func init() { diff --git a/internal/regex/regex_oniguruma.go b/internal/regex/regex_oniguruma.go index d02ea72d3..39330d417 100644 --- a/internal/regex/regex_oniguruma.go +++ b/internal/regex/regex_oniguruma.go @@ -2,7 +2,11 @@ package regex -import "github.com/moovweb/rubex" +import ( + "time" + + rubex "github.com/src-d/go-oniguruma" +) // Oniguruma holds a rubex regular expression Matcher. type Oniguruma struct { @@ -11,21 +15,31 @@ type Oniguruma struct { // Match implements Matcher interface. func (r *Oniguruma) Match(s string) bool { + t := time.Now() + defer MatchHistogram.With("string", s, "duration", "seconds").Observe(time.Since(t).Seconds()) + return r.reg.MatchString(s) } +// Dispose implements Disposer interface. +// The function releases resources for oniguruma's precompiled regex +func (r *Oniguruma) Dispose() { + r.reg.Free() +} + // NewOniguruma creates a new Matcher using oniguruma engine. -func NewOniguruma(re string) (Matcher, error) { +func NewOniguruma(re string) (Matcher, Disposer, error) { + t := time.Now() reg, err := rubex.Compile(re) if err != nil { - return nil, err + return nil, nil, err } + CompileHistogram.With("regex", re, "duration", "seconds").Observe(time.Since(t).Seconds()) r := Oniguruma{ reg: reg, } - - return &r, nil + return &r, &r, nil } func init() { diff --git a/internal/regex/regex_test.go b/internal/regex/regex_test.go index 4ede84adc..118a19c85 100644 --- a/internal/regex/regex_test.go +++ b/internal/regex/regex_test.go @@ -6,12 +6,12 @@ import ( "github.com/stretchr/testify/require" ) -func dummy(s string) (Matcher, error) { return nil, nil } +func dummy(s string) (Matcher, Disposer, error) { return nil, nil, nil } func getDefault() string { for _, n := range Engines() { if n == "oniguruma" { - return "oniguruma" + return n } } @@ -33,16 +33,14 @@ func TestRegistration(t *testing.T) { engines = Engines() require.Len(engines, number) - err = Register("go", dummy) - require.Equal(true, ErrRegexAlreadyRegistered.Is(err)) - err = Register("nil", dummy) require.NoError(err) require.Len(Engines(), number+1) - matcher, err := New("nil", "") + matcher, disposer, err := New("nil", "") require.NoError(err) require.Nil(matcher) + require.Nil(disposer) } func TestDefault(t *testing.T) { @@ -65,11 +63,46 @@ func TestMatcher(t *testing.T) { } t.Run(name, func(t *testing.T) { - m, err := New(name, "a{3}") + m, d, err := New(name, "a{3}") require.NoError(t, err) require.Equal(t, true, m.Match("ooaaaoo")) require.Equal(t, false, m.Match("ooaaoo")) + + d.Dispose() + }) + } +} + +func TestMatcherMultiPatterns(t *testing.T) { + const ( + email = `[\w\.+-]+@[\w\.-]+\.[\w\.-]+` + url = `[\w]+://[^/\s?#]+[^\s?#]+(?:\?[^\s#]*)?(?:#[^\s]*)?` + ip = `(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9])\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9])` + + data = `mysql://root@255.255.255.255:3306` + ) + + for _, name := range Engines() { + if name == "nil" { + continue + } + + t.Run(name, func(t *testing.T) { + m, d, err := New(name, email) + require.NoError(t, err) + require.Equal(t, true, m.Match(data)) + d.Dispose() + + m, d, err = New(name, url) + require.NoError(t, err) + require.Equal(t, true, m.Match(data)) + d.Dispose() + + m, d, err = New(name, ip) + require.NoError(t, err) + require.Equal(t, true, m.Match(data)) + d.Dispose() }) } } diff --git a/internal/similartext/similartext.go b/internal/similartext/similartext.go new file mode 100644 index 000000000..a7beeb09d --- /dev/null +++ b/internal/similartext/similartext.go @@ -0,0 +1,107 @@ +package similartext + +import ( + "fmt" + "reflect" + "strings" +) + +func min(a, b int) int { + if a < b { + return a + } + return b +} + +// DistanceForStrings returns the edit distance between source and target. +// It has a runtime proportional to len(source) * len(target) and memory use +// proportional to len(target). +// Taken (simplified, for strings and with default options) from: +// https://github.com/texttheater/golang-levenshtein +func distanceForStrings(source, target string) int { + height := len(source) + 1 + width := len(target) + 1 + matrix := make([][]int, 2) + + for i := 0; i < 2; i++ { + matrix[i] = make([]int, width) + matrix[i][0] = i + } + for j := 1; j < width; j++ { + matrix[0][j] = j + } + + for i := 1; i < height; i++ { + cur := matrix[i%2] + prev := matrix[(i-1)%2] + cur[0] = i + for j := 1; j < width; j++ { + delCost := prev[j] + 1 + matchSubCost := prev[j-1] + if source[i-1] != target[j-1] { + matchSubCost += 2 + } + insCost := cur[j-1] + 1 + cur[j] = min(delCost, min(matchSubCost, insCost)) + } + } + return matrix[(height-1)%2][width-1] +} + +// MaxDistanceIgnored is the maximum Levenshtein distance from which +// we won't consider a string similar at all and thus will be ignored. +var DistanceSkipped = 3 + +// Find returns a string with suggestions for name(s) in `names` +// similar to the string `src` until a max distance of `DistanceSkipped`. +func Find(names []string, src string) string { + if len(src) == 0 { + return "" + } + + minDistance := -1 + matchMap := make(map[int][]string) + + for _, name := range names { + dist := distanceForStrings(name, src) + if dist >= DistanceSkipped { + continue + } + + if minDistance == -1 || dist < minDistance { + minDistance = dist + } + + matchMap[dist] = append(matchMap[dist], name) + } + + if len(matchMap) == 0 { + return "" + } + + return fmt.Sprintf(", maybe you mean %s?", + strings.Join(matchMap[minDistance], " or ")) +} + +// FindFromMap does the same as Find but taking a map instead +// of a string array as first argument. +func FindFromMap(names interface{}, src string) string { + rnames := reflect.ValueOf(names) + if rnames.Kind() != reflect.Map { + panic("Implementation error: non map used as first argument " + + "to FindFromMap") + } + + t := rnames.Type() + if t.Key().Kind() != reflect.String { + panic("Implementation error: non string key for map used as " + + "first argument to FindFromMap") + } + + var namesList []string + for _, kv := range rnames.MapKeys() { + namesList = append(namesList, kv.String()) + } + + return Find(namesList, src) +} diff --git a/internal/similartext/similartext_test.go b/internal/similartext/similartext_test.go new file mode 100644 index 000000000..bfacc6802 --- /dev/null +++ b/internal/similartext/similartext_test.go @@ -0,0 +1,52 @@ +package similartext + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestFind(t *testing.T) { + require := require.New(t) + + var names []string + res := Find(names, "") + require.Empty(res) + + names = []string{"foo", "bar", "aka", "ake"} + res = Find(names, "baz") + require.Equal(", maybe you mean bar?", res) + + res = Find(names, "") + require.Empty(res) + + res = Find(names, "foo") + require.Equal(", maybe you mean foo?", res) + + res = Find(names, "willBeTooDifferent") + require.Empty(res) + + res = Find(names, "aki") + require.Equal(", maybe you mean aka or ake?", res) +} + +func TestFindFromMap(t *testing.T) { + require := require.New(t) + + var names map[string]int + res := FindFromMap(names, "") + require.Empty(res) + + names = map[string]int{ + "foo": 1, + "bar": 2, + } + res = FindFromMap(names, "baz") + require.Equal(", maybe you mean bar?", res) + + res = FindFromMap(names, "") + require.Empty(res) + + res = FindFromMap(names, "foo") + require.Equal(", maybe you mean foo?", res) +} diff --git a/internal/sockstate/netstat.go b/internal/sockstate/netstat.go new file mode 100644 index 000000000..500aa3879 --- /dev/null +++ b/internal/sockstate/netstat.go @@ -0,0 +1,86 @@ +package sockstate + +import ( + "fmt" + "net" + + "gopkg.in/src-d/go-errors.v1" +) + +// OS independent part of the netstat_[OS].go modules +// Taken (simplified, privatized and with utility functions added) from: +// https://github.com/cakturk/go-netstat + +// skState type represents socket connection state +type skState uint8 + +func (s skState) String() string { + return skStates[s] +} + +// ErrSocketCheckNotImplemented will be returned for OS where the socket checks is not implemented yet +var ErrSocketCheckNotImplemented = errors.NewKind("socket checking not implemented for this OS") + +// Socket states +const ( + Established skState = 0x01 + SynSent skState = 0x02 + SynRecv skState = 0x03 + FinWait1 skState = 0x04 + FinWait2 skState = 0x05 + TimeWait skState = 0x06 + Close skState = 0x07 + CloseWait skState = 0x08 + LastAck skState = 0x09 + Listen skState = 0x0a + Closing skState = 0x0b +) + +var skStates = [...]string{ + "UNKNOWN", + "ESTABLISHED", + "SYN_SENT", + "SYN_RECV", + "FIN_WAIT1", + "FIN_WAIT2", + "TIME_WAIT", + "", // CLOSE + "CLOSE_WAIT", + "LAST_ACK", + "LISTEN", + "CLOSING", +} + +// sockAddr represents an ip:port pair +type sockAddr struct { + IP net.IP + Port uint16 +} + +func (s *sockAddr) String() string { + return fmt.Sprintf("%v:%d", s.IP, s.Port) +} + +// sockTabEntry type represents each line of the /proc/net/tcp +type sockTabEntry struct { + Ino string + LocalAddr *sockAddr + RemoteAddr *sockAddr + State skState + UID uint32 + Process *process +} + +// process holds the PID and process name to which each socket belongs +type process struct { + pid int + name string +} + +func (p *process) String() string { + return fmt.Sprintf("%d/%s", p.pid, p.name) +} + +// AcceptFn is used to filter socket entries. The value returned indicates +// whether the element is to be appended to the socket list. +type AcceptFn func(*sockTabEntry) bool diff --git a/internal/sockstate/netstat_darwin.go b/internal/sockstate/netstat_darwin.go new file mode 100644 index 000000000..f68ffd989 --- /dev/null +++ b/internal/sockstate/netstat_darwin.go @@ -0,0 +1,21 @@ +// +build darwin + +package sockstate + +import ( + "net" + + "github.com/sirupsen/logrus" +) + +// tcpSocks returns a slice of active TCP sockets containing only those +// elements that satisfy the accept function +func tcpSocks(accept AcceptFn) ([]sockTabEntry, error) { + // (juanjux) TODO: not implemented + logrus.Warn("Connection checking not implemented for Darwin") + return nil, ErrSocketCheckNotImplemented.New() +} + +func GetConnInode(c *net.TCPConn) (n uint64, err error) { + return 0, ErrSocketCheckNotImplemented.New() +} diff --git a/internal/sockstate/netstat_linux.go b/internal/sockstate/netstat_linux.go new file mode 100644 index 000000000..a7eb6ff62 --- /dev/null +++ b/internal/sockstate/netstat_linux.go @@ -0,0 +1,260 @@ +// +build linux + +package sockstate + +// Taken (simplified and with utility functions added) from https://github.com/cakturk/go-netstat + +import ( + "bufio" + "bytes" + "encoding/binary" + "fmt" + "io" + "io/ioutil" + "log" + "net" + "os" + "path" + "strconv" + "strings" +) + +const ( + pathTCP4Tab = "/proc/net/tcp" + pathTCP6Tab = "/proc/net/tcp6" + ipv4StrLen = 8 + ipv6StrLen = 32 +) + +type procFd struct { + base string + pid int + sktab []sockTabEntry + p *process +} + +const sockPrefix = "socket:[" + +func getProcName(s []byte) string { + i := bytes.Index(s, []byte("(")) + if i < 0 { + return "" + } + j := bytes.LastIndex(s, []byte(")")) + if i < 0 { + return "" + } + if i > j { + return "" + } + return string(s[i+1 : j]) +} + +func (p *procFd) iterFdDir() { + // link name is of the form socket:[5860846] + fddir := path.Join(p.base, "/fd") + fi, err := ioutil.ReadDir(fddir) + if err != nil { + return + } + var buf [128]byte + + for _, file := range fi { + fd := path.Join(fddir, file.Name()) + lname, err := os.Readlink(fd) + if err != nil { + continue + } + + for i := range p.sktab { + sk := &p.sktab[i] + ss := sockPrefix + sk.Ino + "]" + if ss != lname { + continue + } + if p.p == nil { + stat, err := os.Open(path.Join(p.base, "stat")) + if err != nil { + return + } + n, err := stat.Read(buf[:]) + _ = stat.Close() + if err != nil { + return + } + z := bytes.SplitN(buf[:n], []byte(" "), 3) + name := getProcName(z[1]) + p.p = &process{p.pid, name} + } + sk.Process = p.p + } + } +} + +func extractProcInfo(sktab []sockTabEntry) { + const basedir = "/proc" + fi, err := ioutil.ReadDir(basedir) + if err != nil { + return + } + + for _, file := range fi { + if !file.IsDir() { + continue + } + pid, err := strconv.Atoi(file.Name()) + if err != nil { + continue + } + base := path.Join(basedir, file.Name()) + proc := procFd{base: base, pid: pid, sktab: sktab} + proc.iterFdDir() + } +} + +func parseIPv4(s string) (net.IP, error) { + v, err := strconv.ParseUint(s, 16, 32) + if err != nil { + return nil, err + } + ip := make(net.IP, net.IPv4len) + binary.LittleEndian.PutUint32(ip, uint32(v)) + return ip, nil +} + +func parseIPv6(s string) (net.IP, error) { + ip := make(net.IP, net.IPv6len) + const grpLen = 4 + i, j := 0, 4 + for len(s) != 0 { + grp := s[0:8] + u, err := strconv.ParseUint(grp, 16, 32) + binary.LittleEndian.PutUint32(ip[i:j], uint32(u)) + if err != nil { + return nil, err + } + i, j = i+grpLen, j+grpLen + s = s[8:] + } + return ip, nil +} + +func parseAddr(s string) (*sockAddr, error) { + fields := strings.Split(s, ":") + if len(fields) < 2 { + return nil, fmt.Errorf("sockstate: not enough fields: %v", s) + } + var ip net.IP + var err error + switch len(fields[0]) { + case ipv4StrLen: + ip, err = parseIPv4(fields[0]) + case ipv6StrLen: + ip, err = parseIPv6(fields[0]) + default: + log.Fatal("Badly formatted connection address:", s) + } + if err != nil { + return nil, err + } + v, err := strconv.ParseUint(fields[1], 16, 16) + if err != nil { + return nil, err + } + return &sockAddr{IP: ip, Port: uint16(v)}, nil +} + +func parseSocktab(r io.Reader, accept AcceptFn) ([]sockTabEntry, error) { + br := bufio.NewScanner(r) + tab := make([]sockTabEntry, 0, 4) + + // Discard title + br.Scan() + + for br.Scan() { + var e sockTabEntry + line := br.Text() + // Skip comments + if i := strings.Index(line, "#"); i >= 0 { + line = line[:i] + } + fields := strings.Fields(line) + if len(fields) < 12 { + return nil, fmt.Errorf("sockstate: not enough fields: %v, %v", len(fields), fields) + } + addr, err := parseAddr(fields[1]) + if err != nil { + return nil, err + } + e.LocalAddr = addr + addr, err = parseAddr(fields[2]) + if err != nil { + return nil, err + } + e.RemoteAddr = addr + u, err := strconv.ParseUint(fields[3], 16, 8) + if err != nil { + return nil, err + } + e.State = skState(u) + u, err = strconv.ParseUint(fields[7], 10, 32) + if err != nil { + return nil, err + } + e.UID = uint32(u) + e.Ino = fields[9] + if accept(&e) { + tab = append(tab, e) + } + } + return tab, br.Err() +} + +// tcpSocks returns a slice of active TCP sockets containing only those +// elements that satisfy the accept function +func tcpSocks(accept AcceptFn) ([]sockTabEntry, error) { + paths := [2]string{pathTCP4Tab, pathTCP6Tab} + var allTabs []sockTabEntry + for _, p := range paths { + f, err := os.Open(p) + defer func() { + _ = f.Close() + }() + if err != nil { + return nil, err + } + + t, err := parseSocktab(f, accept) + if err != nil { + return nil, err + } + allTabs = append(allTabs, t...) + + } + extractProcInfo(allTabs) + return allTabs, nil +} + +// GetConnInode returns the Linux inode number of a TCP connection +func GetConnInode(c *net.TCPConn) (n uint64, err error) { + f, err := c.File() + if err != nil { + return + } + + socketStr := fmt.Sprintf("/proc/%d/fd/%d", os.Getpid(), f.Fd()) + socketLnk, err := os.Readlink(socketStr) + if err != nil { + return + } + + if strings.HasPrefix(socketLnk, sockPrefix) { + _, err = fmt.Sscanf(socketLnk, sockPrefix+"%d]", &n) + if err != nil { + return + } + } else { + err = ErrNoSocketLink.New() + } + return +} diff --git a/internal/sockstate/netstat_windows.go b/internal/sockstate/netstat_windows.go new file mode 100644 index 000000000..1f8d98fc9 --- /dev/null +++ b/internal/sockstate/netstat_windows.go @@ -0,0 +1,21 @@ +// +build windows + +package sockstate + +import ( + "net" + + "github.com/sirupsen/logrus" +) + +// tcpSocks returns a slice of active TCP sockets containing only those +// elements that satisfy the accept function +func tcpSocks(accept AcceptFn) ([]sockTabEntry, error) { + // (juanjux) TODO: not implemented + logrus.Warn("Connection checking not implemented for Windows") + return nil, ErrSocketCheckNotImplemented.New() +} + +func GetConnInode(c *net.TCPConn) (n uint64, err error) { + return 0, ErrSocketCheckNotImplemented.New() +} diff --git a/internal/sockstate/sockstate.go b/internal/sockstate/sockstate.go new file mode 100644 index 000000000..f47b05a34 --- /dev/null +++ b/internal/sockstate/sockstate.go @@ -0,0 +1,60 @@ +package sockstate + +import ( + "gopkg.in/src-d/go-errors.v1" + "strconv" +) + +type SockState uint8 + +const ( + Broken = iota + Other + Error +) + +var ErrNoSocketLink = errors.NewKind("couldn't resolve file descriptor link to socket") + +// ErrMultipleSocketsForInode is returned when more than one socket is found for an inode +var ErrMultipleSocketsForInode = errors.NewKind("more than one socket found for inode") + +func GetInodeSockState(port int, inode uint64) (SockState, error) { + socks, err := tcpSocks(func(s *sockTabEntry) bool { + if s.LocalAddr.Port != uint16(port) { + return false + } + + si, err := strconv.ParseUint(s.Ino, 10, 64) + if err != nil { + return false + } + return inode == si + }) + if err != nil { + return Error, err + } + + switch len(socks) { + case 0: + return Broken, nil + case 1: + switch socks[0].State { + case CloseWait: + fallthrough + case TimeWait: + fallthrough + case FinWait1: + fallthrough + case FinWait2: + fallthrough + case Close: + fallthrough + case Closing: + return Broken, nil + default: + return Other, nil + } + default: // more than one sock for inode, impossible? + return Error, ErrMultipleSocketsForInode.New() + } +} diff --git a/_scripts/go-vitess/vt/log/log.go b/log.go similarity index 52% rename from _scripts/go-vitess/vt/log/log.go rename to log.go index 2d27ade29..346b545f7 100644 --- a/_scripts/go-vitess/vt/log/log.go +++ b/log.go @@ -1,78 +1,74 @@ -// You can modify this file to hook up a different logging library instead of logrus. -// If you adapt to a different logging framework, you may need to use that -// framework's equivalent of *Depth() functions so the file and line number printed -// point to the real caller instead of your adapter function. +package sqle -package log - -import "github.com/sirupsen/logrus" - -// Level is used with V() to test log verbosity. -type Level = logrus.Level +import ( + "github.com/golang/glog" + "github.com/sirupsen/logrus" + vtlog "vitess.io/vitess/go/vt/log" +) -var ( +func init() { // V quickly checks if the logging verbosity meets a threshold. - V = func(level int) bool { + vtlog.V = func(level glog.Level) glog.Verbose { lvl := logrus.GetLevel() - switch level { + switch int32(level) { case 0: - return lvl == logrus.InfoLevel + return glog.Verbose(lvl == logrus.InfoLevel) case 1: - return lvl == logrus.WarnLevel + return glog.Verbose(lvl == logrus.WarnLevel) case 2: - return lvl == logrus.ErrorLevel + return glog.Verbose(lvl == logrus.ErrorLevel) case 3: - return lvl == logrus.FatalLevel + return glog.Verbose(lvl == logrus.FatalLevel) default: - return false + return glog.Verbose(false) } } // Flush ensures any pending I/O is written. - Flush = func() {} + vtlog.Flush = func() {} // Info formats arguments like fmt.Print. - Info = logrus.Info + vtlog.Info = logrus.Info // Infof formats arguments like fmt.Printf. - Infof = logrus.Infof + vtlog.Infof = logrus.Infof // InfoDepth formats arguments like fmt.Print and uses depth to choose which call frame to log. - InfoDepth = func(_ int, args ...interface{}) { + vtlog.InfoDepth = func(_ int, args ...interface{}) { logrus.Info(args...) } // Warning formats arguments like fmt.Print. - Warning = logrus.Warning + vtlog.Warning = logrus.Warning // Warningf formats arguments like fmt.Printf. - Warningf = logrus.Warningf + vtlog.Warningf = logrus.Warningf // WarningDepth formats arguments like fmt.Print and uses depth to choose which call frame to log. - WarningDepth = func(depth int, args ...interface{}) { + vtlog.WarningDepth = func(depth int, args ...interface{}) { logrus.Warning(args...) } // Error formats arguments like fmt.Print. - Error = logrus.Error + vtlog.Error = logrus.Error // Errorf formats arguments like fmt.Printf. - Errorf = logrus.Errorf + vtlog.Errorf = logrus.Errorf // ErrorDepth formats arguments like fmt.Print and uses depth to choose which call frame to log. - ErrorDepth = func(_ int, args ...interface{}) { + vtlog.ErrorDepth = func(_ int, args ...interface{}) { logrus.Error(args...) } // Exit formats arguments like fmt.Print. - Exit = logrus.Panic + vtlog.Exit = logrus.Panic // Exitf formats arguments like fmt.Printf. - Exitf = logrus.Panicf + vtlog.Exitf = logrus.Panicf // ExitDepth formats arguments like fmt.Print and uses depth to choose which call frame to log. - ExitDepth = func(_ int, args ...interface{}) { + vtlog.ExitDepth = func(_ int, args ...interface{}) { logrus.Panic(args...) } // Fatal formats arguments like fmt.Print. - Fatal = logrus.Fatal + vtlog.Fatal = logrus.Fatal // Fatalf formats arguments like fmt.Printf - Fatalf = logrus.Fatalf + vtlog.Fatalf = logrus.Fatalf // FatalDepth formats arguments like fmt.Print and uses depth to choose which call frame to log. - FatalDepth = func(_ int, args ...interface{}) { + vtlog.FatalDepth = func(_ int, args ...interface{}) { logrus.Fatal(args...) } -) +} diff --git a/mem/database.go b/memory/database.go similarity index 63% rename from mem/database.go rename to memory/database.go index 0cd30a179..133c4f5de 100644 --- a/mem/database.go +++ b/memory/database.go @@ -1,7 +1,7 @@ -package mem // import "gopkg.in/src-d/go-mysql-server.v0/mem" +package memory import ( - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // Database is an in-memory database. @@ -33,8 +33,8 @@ func (d *Database) AddTable(name string, t sql.Table) { d.tables[name] = t } -// Create creates a table with the given name and schema -func (d *Database) Create(name string, schema sql.Schema) error { +// CreateTable creates a table with the given name and schema +func (d *Database) CreateTable(ctx *sql.Context, name string, schema sql.Schema) error { _, ok := d.tables[name] if ok { return sql.ErrTableAlreadyExists.New(name) @@ -43,3 +43,15 @@ func (d *Database) Create(name string, schema sql.Schema) error { d.tables[name] = NewTable(name, schema) return nil } + +// DropTable drops the table with the given name +func (d *Database) DropTable(ctx *sql.Context, name string) error { + _, ok := d.tables[name] + if !ok { + return sql.ErrTableNotFound.New(name) + } + + delete(d.tables, name) + return nil +} + diff --git a/mem/database_test.go b/memory/database_test.go similarity index 73% rename from mem/database_test.go rename to memory/database_test.go index dcaf0cc25..c6a2f9182 100644 --- a/mem/database_test.go +++ b/memory/database_test.go @@ -1,10 +1,10 @@ -package mem +package memory import ( "testing" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestDatabase_Name(t *testing.T) { @@ -19,8 +19,7 @@ func TestDatabase_AddTable(t *testing.T) { tables := db.Tables() require.Equal(0, len(tables)) - var altDb sql.Alterable = db - err := altDb.Create("test_table", nil) + err := db.CreateTable(sql.NewEmptyContext(), "test_table", nil) require.NoError(err) tables = db.Tables() @@ -29,6 +28,6 @@ func TestDatabase_AddTable(t *testing.T) { require.True(ok) require.NotNil(tt) - err = altDb.Create("test_table", nil) + err = db.CreateTable(sql.NewEmptyContext(), "test_table", nil) require.Error(err) } diff --git a/mem/table.go b/memory/table.go similarity index 86% rename from mem/table.go rename to memory/table.go index bb9217732..088956316 100644 --- a/mem/table.go +++ b/memory/table.go @@ -1,4 +1,4 @@ -package mem +package memory import ( "bytes" @@ -7,9 +7,9 @@ import ( "io" "strconv" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) // Table represents an in-memory database table. @@ -257,6 +257,70 @@ func (t *Table) Insert(ctx *sql.Context, row sql.Row) error { return nil } +// Delete the given row from the table. +func (t *Table) Delete(ctx *sql.Context, row sql.Row) error { + if err := checkRow(t.schema, row); err != nil { + return err + } + + matches := false + for partitionIndex, partition := range t.partitions { + for partitionRowIndex, partitionRow := range partition { + matches = true + for rIndex, val := range row { + if val != partitionRow[rIndex] { + matches = false + break + } + } + if matches { + t.partitions[partitionIndex] = append(partition[:partitionRowIndex], partition[partitionRowIndex+1:]...) + break + } + } + if matches { + break + } + } + + if !matches { + return sql.ErrDeleteRowNotFound + } + + return nil +} + +func (t *Table) Update(ctx *sql.Context, oldRow sql.Row, newRow sql.Row) error { + if err := checkRow(t.schema, oldRow); err != nil { + return err + } + if err := checkRow(t.schema, newRow); err != nil { + return err + } + + matches := false + for partitionIndex, partition := range t.partitions { + for partitionRowIndex, partitionRow := range partition { + matches = true + for rIndex, val := range oldRow { + if val != partitionRow[rIndex] { + matches = false + break + } + } + if matches { + t.partitions[partitionIndex][partitionRowIndex] = newRow + break + } + } + if matches { + break + } + } + + return nil +} + func checkRow(schema sql.Schema, row sql.Row) error { if len(row) != len(schema) { return sql.ErrUnexpectedRowLength.New(len(schema), len(row)) @@ -312,14 +376,14 @@ func (t *Table) HandledFilters(filters []sql.Expression) []sql.Expression { var handled []sql.Expression for _, f := range filters { var hasOtherFields bool - f.TransformUp(func(e sql.Expression) (sql.Expression, error) { + expression.Inspect(f, func(e sql.Expression) bool { if e, ok := e.(*expression.GetField); ok { if e.Table() != t.name || !t.schema.Contains(e.Name(), t.name) { hasOtherFields = true + return false } } - - return e, nil + return true }) if !hasOtherFields { diff --git a/mem/table_test.go b/memory/table_test.go similarity index 90% rename from mem/table_test.go rename to memory/table_test.go index 98ae27ee8..224ea5645 100644 --- a/mem/table_test.go +++ b/memory/table_test.go @@ -1,13 +1,13 @@ -package mem +package memory import ( "fmt" "io" "testing" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestTablePartitionsCount(t *testing.T) { @@ -151,24 +151,24 @@ var tests = []struct { }, indexColumns: []string{"col1", "col3"}, expectedKeyValues: []*indexKeyValue{ - &indexKeyValue{sql.NewRow("a", int64(100)), &indexValue{Key: "0", Pos: 0}}, - &indexKeyValue{sql.NewRow("c", int64(100)), &indexValue{Key: "0", Pos: 1}}, - &indexKeyValue{sql.NewRow("e", int64(200)), &indexValue{Key: "0", Pos: 2}}, - &indexKeyValue{sql.NewRow("b", int64(100)), &indexValue{Key: "1", Pos: 0}}, - &indexKeyValue{sql.NewRow("d", int64(200)), &indexValue{Key: "1", Pos: 1}}, - &indexKeyValue{sql.NewRow("f", int64(200)), &indexValue{Key: "1", Pos: 2}}, + {sql.NewRow("a", int64(100)), &indexValue{Key: "0", Pos: 0}}, + {sql.NewRow("c", int64(100)), &indexValue{Key: "0", Pos: 1}}, + {sql.NewRow("e", int64(200)), &indexValue{Key: "0", Pos: 2}}, + {sql.NewRow("b", int64(100)), &indexValue{Key: "1", Pos: 0}}, + {sql.NewRow("d", int64(200)), &indexValue{Key: "1", Pos: 1}}, + {sql.NewRow("f", int64(200)), &indexValue{Key: "1", Pos: 2}}, }, lookup: &dummyLookup{ values: map[string][]*indexValue{ - "0": []*indexValue{ - &indexValue{Key: "0", Pos: 0}, - &indexValue{Key: "0", Pos: 1}, - &indexValue{Key: "0", Pos: 2}, + "0": { + {Key: "0", Pos: 0}, + {Key: "0", Pos: 1}, + {Key: "0", Pos: 2}, }, - "1": []*indexValue{ - &indexValue{Key: "1", Pos: 0}, - &indexValue{Key: "1", Pos: 1}, - &indexValue{Key: "1", Pos: 2}, + "1": { + {Key: "1", Pos: 0}, + {Key: "1", Pos: 1}, + {Key: "1", Pos: 2}, }, }, }, @@ -194,13 +194,16 @@ func TestTable(t *testing.T) { require.NoError(err) for i := 0; i < test.numPartitions; i++ { - p, err := pIter.Next() + var p sql.Partition + p, err = pIter.Next() require.NoError(err) - iter, err := table.PartitionRows(sql.NewEmptyContext(), p) + var iter sql.RowIter + iter, err = table.PartitionRows(sql.NewEmptyContext(), p) require.NoError(err) - rows, err := sql.RowIterToRows(iter) + var rows []sql.Row + rows, err = sql.RowIterToRows(iter) require.NoError(err) expected := table.partitions[string(p.Key())] diff --git a/server/context.go b/server/context.go index 04bef3aa5..6ee2ee508 100644 --- a/server/context.go +++ b/server/context.go @@ -4,9 +4,9 @@ import ( "context" "sync" - opentracing "github.com/opentracing/opentracing-go" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-vitess.v1/mysql" + "github.com/opentracing/opentracing-go" + "github.com/src-d/go-mysql-server/sql" + "vitess.io/vitess/go/mysql" ) // SessionBuilder creates sessions given a MySQL connection and a server address. @@ -24,10 +24,11 @@ func DefaultSessionBuilder(c *mysql.Conn, addr string) sql.Session { // SessionManager is in charge of creating new sessions for the given // connections and keep track of which sessions are in each connection, so -// they can be cancelled is the connection is closed. +// they can be cancelled if the connection is closed. type SessionManager struct { addr string tracer opentracing.Tracer + memory *sql.MemoryManager mu *sync.Mutex builder SessionBuilder sessions map[uint32]sql.Session @@ -38,11 +39,13 @@ type SessionManager struct { func NewSessionManager( builder SessionBuilder, tracer opentracing.Tracer, + memory *sql.MemoryManager, addr string, ) *SessionManager { return &SessionManager{ addr: addr, tracer: tracer, + memory: memory, mu: new(sync.Mutex), builder: builder, sessions: make(map[uint32]sql.Session), @@ -64,6 +67,12 @@ func (s *SessionManager) NewSession(conn *mysql.Conn) { s.mu.Unlock() } +func (s *SessionManager) session(conn *mysql.Conn) sql.Session { + s.mu.Lock() + defer s.mu.Unlock() + return s.sessions[conn.ConnectionID] +} + // NewContext creates a new context for the session at the given conn. func (s *SessionManager) NewContext(conn *mysql.Conn) *sql.Context { return s.NewContextWithQuery(conn, "") @@ -88,6 +97,8 @@ func (s *SessionManager) NewContextWithQuery( sql.WithTracer(s.tracer), sql.WithPid(s.nextPid()), sql.WithQuery(query), + sql.WithMemoryManager(s.memory), + sql.WithRootSpan(s.tracer.StartSpan("query")), ) return context diff --git a/server/handler.go b/server/handler.go index 422a2ac3e..ed71a5f18 100644 --- a/server/handler.go +++ b/server/handler.go @@ -1,54 +1,88 @@ package server import ( + "context" "io" + "net" "regexp" "strconv" "strings" "sync" "time" - errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0" - "gopkg.in/src-d/go-mysql-server.v0/auth" - "gopkg.in/src-d/go-mysql-server.v0/sql" + sqle "github.com/src-d/go-mysql-server" + "github.com/src-d/go-mysql-server/auth" + "github.com/src-d/go-mysql-server/internal/sockstate" + "github.com/src-d/go-mysql-server/sql" + "gopkg.in/src-d/go-errors.v1" "github.com/sirupsen/logrus" - "gopkg.in/src-d/go-vitess.v1/mysql" - "gopkg.in/src-d/go-vitess.v1/sqltypes" - "gopkg.in/src-d/go-vitess.v1/vt/proto/query" + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/proto/query" ) var regKillCmd = regexp.MustCompile(`^kill (?:(query|connection) )?(\d+)$`) -var errConnectionNotFound = errors.NewKind("Connection not found: %c") +var errConnectionNotFound = errors.NewKind("connection not found: %c") + +// ErrRowTimeout will be returned if the wait for the row is longer than the connection timeout +var ErrRowTimeout = errors.NewKind("row read wait bigger than connection timeout") + +// ErrConnectionWasClosed will be returned if we try to use a previously closed connection +var ErrConnectionWasClosed = errors.NewKind("connection was closed") // TODO parametrize const rowsBatch = 100 +const tcpCheckerSleepTime = 1 + +type conntainer struct { + MysqlConn *mysql.Conn + NetConn net.Conn +} // Handler is a connection handler for a SQLe engine. type Handler struct { - mu sync.Mutex - e *sqle.Engine - sm *SessionManager - c map[uint32]*mysql.Conn + mu sync.Mutex + e *sqle.Engine + sm *SessionManager + c map[uint32]conntainer + readTimeout time.Duration + lc []*net.Conn } // NewHandler creates a new Handler given a SQLe engine. -func NewHandler(e *sqle.Engine, sm *SessionManager) *Handler { +func NewHandler(e *sqle.Engine, sm *SessionManager, rt time.Duration) *Handler { return &Handler{ - e: e, - sm: sm, - c: make(map[uint32]*mysql.Conn), + e: e, + sm: sm, + c: make(map[uint32]conntainer), + readTimeout: rt, } } +// AddNetConnection is used to add the net.Conn to the Handler when available (usually on the +// Listener.Accept() method) +func (h *Handler) AddNetConnection(c *net.Conn) { + h.lc = append(h.lc, c) +} + // NewConnection reports that a new connection has been established. func (h *Handler) NewConnection(c *mysql.Conn) { h.mu.Lock() if _, ok := h.c[c.ConnectionID]; !ok { - h.c[c.ConnectionID] = c + // Retrieve the latest net.Conn stored by Listener.Accept(), if called, and remove it + var netConn net.Conn + if len(h.lc) > 0 { + netConn = *h.lc[len(h.lc)-1] + h.lc = h.lc[:len(h.lc)-1] + } else { + logrus.Debug("Could not find TCP socket connection after Accept(), " + + "connection checker won't run") + } + h.c[c.ConnectionID] = conntainer{c, netConn} } + h.mu.Unlock() logrus.Infof("NewConnection: client %v", c.ConnectionID) @@ -62,6 +96,9 @@ func (h *Handler) ConnectionClosed(c *mysql.Conn) { delete(h.c, c.ConnectionID) h.mu.Unlock() + // If connection was closed, kill only its associated queries. + h.e.Catalog.ProcessList.KillOnlyQueries(c.ConnectionID) + if err := h.e.Catalog.UnlockTables(nil, c.ConnectionID); err != nil { logrus.Errorf("unable to unlock tables on session close: %s", err) } @@ -77,6 +114,13 @@ func (h *Handler) ComQuery( ) (err error) { ctx := h.sm.NewContextWithQuery(c, query) + if !h.e.Async(ctx, query) { + newCtx, cancel := context.WithCancel(ctx) + ctx = ctx.WithContext(newCtx) + + defer cancel() + } + handled, err := h.handleKill(c, query) if err != nil { return err @@ -93,13 +137,105 @@ func (h *Handler) ComQuery( q.Query(ctx, time.Since(start), err) } }() - if err != nil { return err } + nc, ok := h.c[c.ConnectionID] + if !ok { + return ErrConnectionWasClosed.New() + } + var r *sqltypes.Result var proccesedAtLeastOneBatch bool + + // Reads rows from the row reading goroutine + rowChan := make(chan sql.Row) + // To send errors from the two goroutines to the main one + errChan := make(chan error) + // To close the goroutines + quit := make(chan struct{}) + + // Default waitTime is one minute if there is not timeout configured, in which case + // it will loop to iterate again unless the socket died by the OS timeout or other problems. + // If there is a timeout, it will be enforced to ensure that Vitess has a chance to + // call Handler.CloseConnection() + waitTime := 1 * time.Minute + + if h.readTimeout > 0 { + waitTime = h.readTimeout + } + timer := time.NewTimer(waitTime) + defer timer.Stop() + + // This goroutine will be select{}ed giving a chance to Vitess to call the + // handler.CloseConnection callback and enforcing the timeout if configured + go func() { + for { + select { + case <-quit: + return + default: + row, err := rows.Next() + if err != nil { + errChan <- err + return + } + rowChan <- row + } + } + }() + + // This second goroutine will check the socket + // and try to determine if the socket is in CLOSE_WAIT state + // (because the remote client closed the connection). + go func() { + tcpConn, ok := nc.NetConn.(*net.TCPConn) + if !ok { + logrus.Debug("Connection checker exiting, connection isn't TCP") + return + } + + inode, err := sockstate.GetConnInode(tcpConn) + if err != nil || inode == 0 { + if sockstate.ErrSocketCheckNotImplemented.Is(err) { + logrus.Warn("Connection checker exiting, not supported in this OS") + } else { + errChan <- err + } + return + } + + t, ok := nc.NetConn.LocalAddr().(*net.TCPAddr) + if !ok { + logrus.Warn("Connection checker exiting, could not get local port") + return + } + + for { + select { + case <-quit: + return + default: + } + + st, err := sockstate.GetInodeSockState(t.Port, inode) + switch st { + case sockstate.Broken: + errChan <- ErrConnectionWasClosed.New() + return + case sockstate.Error: + errChan <- err + return + default: // Established + // (juanjux) this check is not free, each iteration takes about 9 milliseconds to run on my machine + // thus the small wait between checks + time.Sleep(tcpCheckerSleepTime * time.Second) + } + } + }() + +rowLoop: for { if r == nil { r = &sqltypes.Result{Fields: schemaToFields(schema)} @@ -107,27 +243,41 @@ func (h *Handler) ComQuery( if r.RowsAffected == rowsBatch { if err := callback(r); err != nil { + close(quit) return err } r = nil proccesedAtLeastOneBatch = true - continue } - row, err := rows.Next() - if err != nil { + select { + case err = <-errChan: if err == io.EOF { - break + break rowLoop } - + close(quit) return err - } + case row := <-rowChan: + outputRow, err := rowToSQL(schema, row) + if err != nil { + close(quit) + return err + } - r.Rows = append(r.Rows, rowToSQL(schema, row)) - r.RowsAffected++ + r.Rows = append(r.Rows, outputRow) + r.RowsAffected++ + case <-timer.C: + if h.readTimeout != 0 { + // Cancel and return so Vitess can call the CloseConnection callback + close(quit) + return ErrRowTimeout.New() + } + } + timer.Reset(waitTime) } + close(quit) if err := rows.Close(); err != nil { return err @@ -150,12 +300,11 @@ func (h *Handler) ComQuery( // ComQuery callback if the result does not contain any fields, // or after the last ComQuery call completes. func (h *Handler) WarningCount(c *mysql.Conn) uint16 { - sess, ok := h.sm.sessions[c.ConnectionID] - if !ok { - return 0 + if sess := h.sm.session(c); sess != nil { + return sess.WarningCount() } - return sess.WarningCount() + return 0 } func (h *Handler) handleKill(conn *mysql.Conn, query string) (bool, error) { @@ -165,7 +314,7 @@ func (h *Handler) handleKill(conn *mysql.Conn, query string) (bool, error) { return false, nil } - id, err := strconv.ParseUint(s[2], 10, 64) + id, err := strconv.ParseUint(s[2], 10, 32) if err != nil { return false, err } @@ -173,45 +322,62 @@ func (h *Handler) handleKill(conn *mysql.Conn, query string) (bool, error) { // KILL CONNECTION and KILL should close the connection. KILL QUERY only // cancels the query. // - // https://dev.mysql.com/doc/refman/5.7/en/kill.html + // https://dev.mysql.com/doc/refman/8.0/en/kill.html + // + // KILL [CONNECTION | QUERY] processlist_id + // - KILL QUERY terminates the statement the connection is currently executing, + // but leaves the connection itself intact. + + // - KILL CONNECTION is the same as KILL with no modifier: + // It terminates the connection associated with the given processlist_id, + // after terminating any statement the connection is executing. + connID := uint32(id) + h.e.Catalog.Kill(connID) + if s[1] != "query" { + logrus.Infof("kill connection: id %d", connID) - if s[1] == "query" { - logrus.Infof("kill query: id %v", id) - h.e.Catalog.KillConnection(uint32(id)) - } else { - logrus.Infof("kill connection: id %v, pid: %v", conn.ConnectionID, id) h.mu.Lock() - c, ok := h.c[conn.ConnectionID] - delete(h.c, conn.ConnectionID) + c, ok := h.c[connID] + if ok { + delete(h.c, connID) + } h.mu.Unlock() - if !ok { - return false, errConnectionNotFound.New(conn.ConnectionID) + return false, errConnectionNotFound.New(connID) } - h.e.Catalog.KillConnection(uint32(id)) - h.sm.CloseConn(c) - c.Close() + h.sm.CloseConn(c.MysqlConn) + c.MysqlConn.Close() } return true, nil } -func rowToSQL(s sql.Schema, row sql.Row) []sqltypes.Value { +func rowToSQL(s sql.Schema, row sql.Row) ([]sqltypes.Value, error) { o := make([]sqltypes.Value, len(row)) + var err error for i, v := range row { - o[i] = s[i].Type.SQL(v) + o[i], err = s[i].Type.SQL(v) + if err != nil { + return nil, err + } } - return o + return o, nil } func schemaToFields(s sql.Schema) []*query.Field { fields := make([]*query.Field, len(s)) for i, c := range s { + var charset uint32 = mysql.CharacterSetUtf8 + if c.Type == sql.Blob { + charset = mysql.CharacterSetBinary + } + fields[i] = &query.Field{ - Name: c.Name, - Type: c.Type.Type(), + Name: c.Name, + Type: c.Type.Type(), + Charset: charset, } } diff --git a/server/handler_linux_test.go b/server/handler_linux_test.go new file mode 100644 index 000000000..9dfc6eb31 --- /dev/null +++ b/server/handler_linux_test.go @@ -0,0 +1,52 @@ +package server + +import ( + "fmt" + "net" + "testing" + + "github.com/opentracing/opentracing-go" + "github.com/src-d/go-mysql-server/sql" + "github.com/stretchr/testify/require" + "vitess.io/vitess/go/sqltypes" +) + +func TestBrokenConnection(t *testing.T) { + require := require.New(t) + e := setupMemDB(require) + + port, err := getFreePort() + require.NoError(err) + + ready := make(chan struct{}) + go brokenTestServer(t, ready, port) + <-ready + conn, err := net.Dial("tcp", "localhost:"+port) + require.NoError(err) + + h := NewHandler( + e, + NewSessionManager( + testSessionBuilder, + opentracing.NoopTracer{}, + sql.NewMemoryManager(nil), + "foo", + ), + 0, + ) + h.AddNetConnection(&conn) + c := newConn(1) + h.NewConnection(c) + + // (juanjux) Note that this is a little fuzzy because sometimes sockets take one or two seconds + // to go into TIME_WAIT but 4 seconds hopefully is enough + wait := tcpCheckerSleepTime * 2 + if wait < 4 { + wait = 4 + } + q := fmt.Sprintf("SELECT SLEEP(%d)", wait) + err = h.ComQuery(c, q, func(res *sqltypes.Result) error { + return nil + }) + require.EqualError(err, "connection was closed") +} diff --git a/server/handler_test.go b/server/handler_test.go index d3ef99d5c..fb074cafa 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -1,56 +1,41 @@ package server import ( + "fmt" "net" - "reflect" "testing" - "unsafe" + "time" - "gopkg.in/src-d/go-mysql-server.v0" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-vitess.v1/mysql" - "gopkg.in/src-d/go-vitess.v1/sqltypes" + sqle "github.com/src-d/go-mysql-server" + "github.com/src-d/go-mysql-server/sql" - opentracing "github.com/opentracing/opentracing-go" + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/proto/query" + + "github.com/opentracing/opentracing-go" "github.com/stretchr/testify/require" ) -func setupMemDB(require *require.Assertions) *sqle.Engine { - e := sqle.NewDefault() - db := mem.NewDatabase("test") - e.AddDatabase(db) - - tableTest := mem.NewTable("test", sql.Schema{{Name: "c1", Type: sql.Int32, Source: "test"}}) - - for i := 0; i < 1010; i++ { - require.NoError(tableTest.Insert( - sql.NewEmptyContext(), - sql.NewRow(int32(i)), - )) - } - - db.AddTable("test", tableTest) - - return e -} - func TestHandlerOutput(t *testing.T) { - // This session builder is used as dummy mysql Conn is not complete and - // causes panic when accessing remote address. - testSessionBuilder := func(c *mysql.Conn, addr string) sql.Session { - client := "127.0.0.1:34567" - return sql.NewSession(addr, client, c.User, c.ConnectionID) - } e := setupMemDB(require.New(t)) dummyConn := &mysql.Conn{ConnectionID: 1} - handler := NewHandler(e, NewSessionManager(testSessionBuilder, opentracing.NoopTracer{}, "foo")) + handler := NewHandler( + e, + NewSessionManager( + testSessionBuilder, + opentracing.NoopTracer{}, + sql.NewMemoryManager(nil), + "foo", + ), + 0, + ) handler.NewConnection(dummyConn) - type exptectedValues struct { + type expectedValues struct { callsToCallback int - lenLastBacth int + lenLastBatch int lastRowsAffected uint64 } @@ -59,16 +44,16 @@ func TestHandlerOutput(t *testing.T) { handler *Handler conn *mysql.Conn query string - expected exptectedValues + expected expectedValues }{ { name: "select all without limit", handler: handler, conn: dummyConn, query: "SELECT * FROM test", - expected: exptectedValues{ + expected: expectedValues{ callsToCallback: 11, - lenLastBacth: 10, + lenLastBatch: 10, lastRowsAffected: uint64(10), }, }, @@ -77,9 +62,9 @@ func TestHandlerOutput(t *testing.T) { handler: handler, conn: dummyConn, query: "SELECT * FROM test limit 100", - expected: exptectedValues{ + expected: expectedValues{ callsToCallback: 1, - lenLastBacth: 100, + lenLastBatch: 100, lastRowsAffected: uint64(100), }, }, @@ -88,9 +73,9 @@ func TestHandlerOutput(t *testing.T) { handler: handler, conn: dummyConn, query: "SELECT * FROM test limit 60", - expected: exptectedValues{ + expected: expectedValues{ callsToCallback: 1, - lenLastBacth: 60, + lenLastBatch: 60, lastRowsAffected: uint64(60), }, }, @@ -99,9 +84,9 @@ func TestHandlerOutput(t *testing.T) { handler: handler, conn: dummyConn, query: "SELECT * FROM test limit 200", - expected: exptectedValues{ + expected: expectedValues{ callsToCallback: 2, - lenLastBacth: 100, + lenLastBatch: 100, lastRowsAffected: uint64(100), }, }, @@ -110,9 +95,9 @@ func TestHandlerOutput(t *testing.T) { handler: handler, conn: dummyConn, query: "SELECT * FROM test limit 530", - expected: exptectedValues{ + expected: expectedValues{ callsToCallback: 6, - lenLastBacth: 30, + lenLastBatch: 30, lastRowsAffected: uint64(30), }, }, @@ -121,44 +106,24 @@ func TestHandlerOutput(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { var callsToCallback int - var lenLastBacth int + var lenLastBatch int var lastRowsAffected uint64 err := handler.ComQuery(test.conn, test.query, func(res *sqltypes.Result) error { callsToCallback++ - lenLastBacth = len(res.Rows) + lenLastBatch = len(res.Rows) lastRowsAffected = res.RowsAffected return nil }) require.NoError(t, err) require.Equal(t, test.expected.callsToCallback, callsToCallback) - require.Equal(t, test.expected.lenLastBacth, lenLastBacth) + require.Equal(t, test.expected.lenLastBatch, lenLastBatch) require.Equal(t, test.expected.lastRowsAffected, lastRowsAffected) }) } } -func newConn(id uint32) *mysql.Conn { - conn := &mysql.Conn{ - ConnectionID: id, - } - - // Set conn so it does not panic when we close it - val := reflect.ValueOf(conn).Elem() - field := val.FieldByName("conn") - field = reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem() - field.Set(reflect.ValueOf(new(mockConn))) - - return conn -} - -type mockConn struct { - net.Conn -} - -func (c *mockConn) Close() error { return nil } - func TestHandlerKill(t *testing.T) { require := require.New(t) e := setupMemDB(require) @@ -167,18 +132,24 @@ func TestHandlerKill(t *testing.T) { e, NewSessionManager( func(conn *mysql.Conn, addr string) sql.Session { - return sql.NewBaseSession() + return sql.NewSession(addr, "", "", conn.ConnectionID) }, opentracing.NoopTracer{}, + sql.NewMemoryManager(nil), "foo", ), + 0, ) require.Len(handler.c, 0) + var dummyNetConn net.Conn conn1 := newConn(1) + conntainer1 := conntainer{conn1, dummyNetConn} handler.NewConnection(conn1) + conn2 := newConn(2) + conntainer2 := conntainer{conn2, dummyNetConn} handler.NewConnection(conn2) require.Len(handler.sm.sessions, 0) @@ -192,20 +163,24 @@ func TestHandlerKill(t *testing.T) { require.Len(handler.sm.sessions, 1) require.Len(handler.c, 2) - require.Equal(conn1, handler.c[1]) - require.Equal(conn2, handler.c[2]) - + require.Equal(conntainer1, handler.c[1]) + require.Equal(conntainer2, handler.c[2]) assertNoConnProcesses(t, e, conn2.ConnectionID) - err = handler.ComQuery(conn2, "KILL 1", func(res *sqltypes.Result) error { + ctx1 := handler.sm.NewContextWithQuery(conn1, "SELECT 1") + ctx1, err = handler.e.Catalog.AddProcess(ctx1, sql.QueryProcess, "SELECT 1") + require.NoError(err) + + err = handler.ComQuery(conn2, "KILL "+fmt.Sprint(ctx1.ID()), func(res *sqltypes.Result) error { return nil }) require.NoError(err) - require.Len(handler.sm.sessions, 0) + require.Len(handler.sm.sessions, 1) require.Len(handler.c, 1) - require.Equal(conn1, handler.c[1]) - assertNoConnProcesses(t, e, conn2.ConnectionID) + _, ok := handler.c[1] + require.False(ok) + assertNoConnProcesses(t, e, conn1.ConnectionID) } func assertNoConnProcesses(t *testing.T, e *sqle.Engine, conn uint32) { @@ -217,3 +192,102 @@ func assertNoConnProcesses(t *testing.T, e *sqle.Engine, conn uint32) { } } } + +func TestSchemaToFields(t *testing.T) { + require := require.New(t) + + schema := sql.Schema{ + {Name: "foo", Type: sql.Blob}, + {Name: "bar", Type: sql.Text}, + {Name: "baz", Type: sql.Int64}, + } + + expected := []*query.Field{ + {Name: "foo", Type: query.Type_BLOB, Charset: mysql.CharacterSetBinary}, + {Name: "bar", Type: query.Type_TEXT, Charset: mysql.CharacterSetUtf8}, + {Name: "baz", Type: query.Type_INT64, Charset: mysql.CharacterSetUtf8}, + } + + fields := schemaToFields(schema) + require.Equal(expected, fields) +} + +func TestHandlerTimeout(t *testing.T) { + require := require.New(t) + + e := setupMemDB(require) + e2 := setupMemDB(require) + + timeOutHandler := NewHandler( + e, NewSessionManager(testSessionBuilder, + opentracing.NoopTracer{}, + sql.NewMemoryManager(nil), + "foo"), + 1*time.Second) + + noTimeOutHandler := NewHandler( + e2, NewSessionManager(testSessionBuilder, + opentracing.NoopTracer{}, + sql.NewMemoryManager(nil), + "foo"), + 0) + require.Equal(1*time.Second, timeOutHandler.readTimeout) + require.Equal(0*time.Second, noTimeOutHandler.readTimeout) + + connTimeout := newConn(1) + timeOutHandler.NewConnection(connTimeout) + + connNoTimeout := newConn(2) + noTimeOutHandler.NewConnection(connNoTimeout) + + err := timeOutHandler.ComQuery(connTimeout, "SELECT SLEEP(2)", func(res *sqltypes.Result) error { + return nil + }) + require.EqualError(err, "row read wait bigger than connection timeout") + + err = timeOutHandler.ComQuery(connTimeout, "SELECT SLEEP(0.5)", func(res *sqltypes.Result) error { + return nil + }) + require.NoError(err) + + err = noTimeOutHandler.ComQuery(connNoTimeout, "SELECT SLEEP(2)", func(res *sqltypes.Result) error { + return nil + }) + require.NoError(err) +} + +func TestOkClosedConnection(t *testing.T) { + require := require.New(t) + e := setupMemDB(require) + port, err := getFreePort() + require.NoError(err) + + ready := make(chan struct{}) + go okTestServer(t, ready, port) + <-ready + conn, err := net.Dial("tcp", "localhost:"+port) + require.NoError(err) + defer func() { + _ = conn.Close() + }() + + h := NewHandler( + e, + NewSessionManager( + testSessionBuilder, + opentracing.NoopTracer{}, + sql.NewMemoryManager(nil), + "foo", + ), + 0, + ) + h.AddNetConnection(&conn) + c := newConn(1) + h.NewConnection(c) + + q := fmt.Sprintf("SELECT SLEEP(%d)", tcpCheckerSleepTime*4) + err = h.ComQuery(c, q, func(res *sqltypes.Result) error { + return nil + }) + require.NoError(err) +} diff --git a/server/handler_test_common.go b/server/handler_test_common.go new file mode 100644 index 000000000..2abecff30 --- /dev/null +++ b/server/handler_test_common.go @@ -0,0 +1,108 @@ +package server + +import ( + "io/ioutil" + "net" + "reflect" + "strconv" + "testing" + "unsafe" + + sqle "github.com/src-d/go-mysql-server" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql" +) + +func setupMemDB(require *require.Assertions) *sqle.Engine { + e := sqle.NewDefault() + db := memory.NewDatabase("test") + e.AddDatabase(db) + + tableTest := memory.NewTable("test", sql.Schema{{Name: "c1", Type: sql.Int32, Source: "test"}}) + + for i := 0; i < 1010; i++ { + require.NoError(tableTest.Insert( + sql.NewEmptyContext(), + sql.NewRow(int32(i)), + )) + } + + db.AddTable("test", tableTest) + + return e +} + +func getFreePort() (string, error) { + addr, err := net.ResolveTCPAddr("tcp", "localhost:0") + if err != nil { + return "", err + } + + l, err := net.ListenTCP("tcp", addr) + if err != nil { + return "", err + } + defer l.Close() + return strconv.Itoa(l.Addr().(*net.TCPAddr).Port), nil +} + +func testServer(t *testing.T, ready chan struct{}, port string, breakConn bool) { + l, err := net.Listen("tcp", ":"+port) + defer func() { + _ = l.Close() + }() + if err != nil { + t.Fatal(err) + } + close(ready) + conn, err := l.Accept() + if err != nil { + return + } + + if !breakConn { + defer func() { + _ = conn.Close() + }() + + _, err = ioutil.ReadAll(conn) + if err != nil { + t.Fatal(err) + } + } // else: dirty return without closing or reading to force the socket into TIME_WAIT +} +func okTestServer(t *testing.T, ready chan struct{}, port string) { + testServer(t, ready, port, false) +} +func brokenTestServer(t *testing.T, ready chan struct{}, port string) { + testServer(t, ready, port, true) +} + +// This session builder is used as dummy mysql Conn is not complete and +// causes panic when accessing remote address. +func testSessionBuilder(c *mysql.Conn, addr string) sql.Session { + const client = "127.0.0.1:34567" + return sql.NewSession(addr, client, c.User, c.ConnectionID) +} + +type mockConn struct { + net.Conn +} + +func (c *mockConn) Close() error { return nil } + +func newConn(id uint32) *mysql.Conn { + conn := &mysql.Conn{ + ConnectionID: id, + } + + // Set conn so it does not panic when we close it + val := reflect.ValueOf(conn).Elem() + field := val.FieldByName("conn") + field = reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem() + field.Set(reflect.ValueOf(new(mockConn))) + + return conn +} diff --git a/server/listener.go b/server/listener.go new file mode 100644 index 000000000..f1bc018f1 --- /dev/null +++ b/server/listener.go @@ -0,0 +1,29 @@ +package server + +import ( + "net" +) + +type Listener struct { + net.Listener + h *Handler +} + +// NewListener creates a new Listener. +func NewListener(protocol, address string, handler *Handler) (*Listener, error) { + l, err := net.Listen(protocol, address) + if err != nil { + return nil, err + } + return &Listener{l, handler}, nil +} + +func (l *Listener) Accept() (net.Conn, error) { + conn, err := l.Listener.Accept() + if err != nil { + return nil, err + } + + l.h.AddNetConnection(&conn) + return conn, err +} diff --git a/server/server.go b/server/server.go index 48ac7fb0b..15fdb45c1 100644 --- a/server/server.go +++ b/server/server.go @@ -1,18 +1,19 @@ -package server // import "gopkg.in/src-d/go-mysql-server.v0/server" +package server import ( "time" - opentracing "github.com/opentracing/opentracing-go" - "gopkg.in/src-d/go-mysql-server.v0" - "gopkg.in/src-d/go-mysql-server.v0/auth" + "github.com/opentracing/opentracing-go" + sqle "github.com/src-d/go-mysql-server" + "github.com/src-d/go-mysql-server/auth" - "gopkg.in/src-d/go-vitess.v1/mysql" + "vitess.io/vitess/go/mysql" ) // Server is a MySQL server for SQLe engines. type Server struct { Listener *mysql.Listener + h *Handler } // Config for the mysql server. @@ -54,14 +55,23 @@ func NewServer(cfg Config, e *sqle.Engine, sb SessionBuilder) (*Server, error) { cfg.ConnWriteTimeout = 0 } - handler := NewHandler(e, NewSessionManager(sb, tracer, cfg.Address)) + handler := NewHandler(e, + NewSessionManager( + sb, tracer, + e.Catalog.MemoryManager, + cfg.Address), + cfg.ConnReadTimeout) a := cfg.Auth.Mysql() - l, err := mysql.NewListener(cfg.Protocol, cfg.Address, a, handler, cfg.ConnReadTimeout, cfg.ConnWriteTimeout) + l, err := NewListener(cfg.Protocol, cfg.Address, handler) + if err != nil { + return nil, err + } + vtListnr, err := mysql.NewFromListener(l, a, handler, cfg.ConnReadTimeout, cfg.ConnWriteTimeout) if err != nil { return nil, err } - return &Server{Listener: l}, nil + return &Server{Listener: vtListnr, h: handler}, nil } // Start starts accepting connections on the server. diff --git a/sql/analyzer/aggregations.go b/sql/analyzer/aggregations.go index 0f67e9442..9afa5b49f 100644 --- a/sql/analyzer/aggregations.go +++ b/sql/analyzer/aggregations.go @@ -1,9 +1,9 @@ package analyzer import ( - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" ) func reorderAggregations(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { @@ -16,7 +16,7 @@ func reorderAggregations(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, e a.Log("reorder aggregations, node of type: %T", n) - return n.TransformUp(func(n sql.Node) (sql.Node, error) { + return plan.TransformUp(n, func(n sql.Node) (sql.Node, error) { switch n := n.(type) { case *plan.GroupBy: if !hasHiddenAggregations(n.Aggregate...) { @@ -38,7 +38,7 @@ func fixAggregations(projection, grouping []sql.Expression, child sql.Node) (sql for i, p := range projection { var transformed bool - e, err := p.TransformUp(func(e sql.Expression) (sql.Expression, error) { + e, err := expression.TransformUp(p, func(e sql.Expression) (sql.Expression, error) { agg, ok := e.(sql.Aggregation) if !ok { return e, nil diff --git a/sql/analyzer/aggregations_test.go b/sql/analyzer/aggregations_test.go index 590f18212..b4333becd 100644 --- a/sql/analyzer/aggregations_test.go +++ b/sql/analyzer/aggregations_test.go @@ -3,18 +3,18 @@ package analyzer import ( "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/expression/function/aggregation" + "github.com/src-d/go-mysql-server/sql/plan" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/aggregation" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" ) func TestReorderAggregations(t *testing.T) { require := require.New(t) - table := mem.NewTable("foo", sql.Schema{ + table := memory.NewTable("foo", sql.Schema{ {Name: "a", Type: sql.Int64, Source: "foo"}, {Name: "b", Type: sql.Int64, Source: "foo"}, {Name: "c", Type: sql.Int64, Source: "foo"}, @@ -67,7 +67,7 @@ func TestReorderAggregations(t *testing.T) { func TestReorderAggregationsMultiple(t *testing.T) { require := require.New(t) - table := mem.NewTable("foo", sql.Schema{ + table := memory.NewTable("foo", sql.Schema{ {Name: "a", Type: sql.Int64, Source: "foo"}, {Name: "b", Type: sql.Int64, Source: "foo"}, {Name: "c", Type: sql.Int64, Source: "foo"}, @@ -98,7 +98,7 @@ func TestReorderAggregationsMultiple(t *testing.T) { []sql.Expression{ expression.NewArithmetic( expression.NewGetField(0, sql.Float64, "SUM(foo.a)", false), - expression.NewGetField(1, sql.Int32, "COUNT(foo.a)", false), + expression.NewGetField(1, sql.Int64, "COUNT(foo.a)", false), "/", ), expression.NewGetFieldWithTable(2, sql.Int64, "foo", "b", false), diff --git a/sql/analyzer/analyzer.go b/sql/analyzer/analyzer.go index 8097ca995..f7bc456d5 100644 --- a/sql/analyzer/analyzer.go +++ b/sql/analyzer/analyzer.go @@ -1,12 +1,12 @@ -package analyzer // import "gopkg.in/src-d/go-mysql-server.v0/sql/analyzer" +package analyzer import ( "os" opentracing "github.com/opentracing/opentracing-go" "github.com/sirupsen/logrus" + "github.com/src-d/go-mysql-server/sql" "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) const debugAnalyzerKey = "DEBUG_ANALYZER" diff --git a/sql/analyzer/analyzer_test.go b/sql/analyzer/analyzer_test.go index 335ce19d6..53f85fcd5 100644 --- a/sql/analyzer/analyzer_test.go +++ b/sql/analyzer/analyzer_test.go @@ -4,26 +4,26 @@ import ( "fmt" "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" ) func TestAnalyzer_Analyze(t *testing.T) { require := require.New(t) - table := mem.NewTable("mytable", sql.Schema{ + table := memory.NewTable("mytable", sql.Schema{ {Name: "i", Type: sql.Int32, Source: "mytable"}, {Name: "t", Type: sql.Text, Source: "mytable"}, }) - table2 := mem.NewTable("mytable2", sql.Schema{ + table2 := memory.NewTable("mytable2", sql.Schema{ {Name: "i2", Type: sql.Int32, Source: "mytable2"}, }) - db := mem.NewDatabase("mydb") + db := memory.NewDatabase("mydb") db.AddTable("mytable", table) db.AddTable("mytable2", table2) @@ -143,7 +143,7 @@ func TestAnalyzer_Analyze(t *testing.T) { expression.NewGetFieldWithTable(0, sql.Int32, "mytable", "i", false), expression.NewLiteral(int32(1), sql.Int32), ), - }).(*mem.Table).WithProjection([]string{"i"}), + }).(*memory.Table).WithProjection([]string{"i"}), ) require.NoError(err) require.Equal(expected, analyzed) @@ -187,11 +187,11 @@ func TestAnalyzer_Analyze(t *testing.T) { func TestMaxIterations(t *testing.T) { require := require.New(t) tName := "my-table" - table := mem.NewTable(tName, sql.Schema{ + table := memory.NewTable(tName, sql.Schema{ {Name: "i", Type: sql.Int32, Source: tName}, {Name: "t", Type: sql.Text, Source: tName}, }) - db := mem.NewDatabase("mydb") + db := memory.NewDatabase("mydb") db.AddTable(tName, table) catalog := sql.NewCatalog() @@ -205,7 +205,7 @@ func TestMaxIterations(t *testing.T) { case *plan.ResolvedTable: count++ name := fmt.Sprintf("mytable-%v", count) - table := mem.NewTable(name, sql.Schema{ + table := memory.NewTable(name, sql.Schema{ {Name: "i", Type: sql.Int32, Source: name}, {Name: "t", Type: sql.Text, Source: name}, }) @@ -220,7 +220,7 @@ func TestMaxIterations(t *testing.T) { require.NoError(err) require.Equal( plan.NewResolvedTable( - mem.NewTable("mytable-1000", sql.Schema{ + memory.NewTable("mytable-1000", sql.Schema{ {Name: "i", Type: sql.Int32, Source: "mytable-1000"}, {Name: "t", Type: sql.Text, Source: "mytable-1000"}, }), @@ -272,25 +272,25 @@ func countRules(batches []*Batch) int { func TestMixInnerAndNaturalJoins(t *testing.T) { var require = require.New(t) - table := mem.NewTable("mytable", sql.Schema{ + table := memory.NewTable("mytable", sql.Schema{ {Name: "i", Type: sql.Int32, Source: "mytable"}, {Name: "f", Type: sql.Float64, Source: "mytable"}, {Name: "t", Type: sql.Text, Source: "mytable"}, }) - table2 := mem.NewTable("mytable2", sql.Schema{ + table2 := memory.NewTable("mytable2", sql.Schema{ {Name: "i2", Type: sql.Int32, Source: "mytable2"}, {Name: "f2", Type: sql.Float64, Source: "mytable2"}, {Name: "t2", Type: sql.Text, Source: "mytable2"}, }) - table3 := mem.NewTable("mytable3", sql.Schema{ + table3 := memory.NewTable("mytable3", sql.Schema{ {Name: "i", Type: sql.Int32, Source: "mytable3"}, {Name: "f2", Type: sql.Float64, Source: "mytable3"}, {Name: "t3", Type: sql.Text, Source: "mytable3"}, }) - db := mem.NewDatabase("mydb") + db := memory.NewDatabase("mydb") db.AddTable("mytable", table) db.AddTable("mytable2", table2) db.AddTable("mytable3", table3) @@ -400,25 +400,25 @@ func TestReorderProjectionUnresolvedChild(t *testing.T) { ), ) - commits := mem.NewTable("commits", sql.Schema{ + commits := memory.NewTable("commits", sql.Schema{ {Name: "repository_id", Source: "commits", Type: sql.Text}, {Name: "commit_hash", Source: "commits", Type: sql.Text}, {Name: "commit_author_when", Source: "commits", Type: sql.Text}, }) - refs := mem.NewTable("refs", sql.Schema{ + refs := memory.NewTable("refs", sql.Schema{ {Name: "repository_id", Source: "refs", Type: sql.Text}, {Name: "ref_name", Source: "refs", Type: sql.Text}, }) - refCommits := mem.NewTable("ref_commits", sql.Schema{ + refCommits := memory.NewTable("ref_commits", sql.Schema{ {Name: "repository_id", Source: "ref_commits", Type: sql.Text}, {Name: "ref_name", Source: "ref_commits", Type: sql.Text}, {Name: "commit_hash", Source: "ref_commits", Type: sql.Text}, {Name: "history_index", Source: "ref_commits", Type: sql.Int64}, }) - db := mem.NewDatabase("") + db := memory.NewDatabase("") db.AddTable("refs", refs) db.AddTable("ref_commits", refCommits) db.AddTable("commits", commits) diff --git a/sql/analyzer/assign_catalog.go b/sql/analyzer/assign_catalog.go index 8d07247a7..0a8f76ee8 100644 --- a/sql/analyzer/assign_catalog.go +++ b/sql/analyzer/assign_catalog.go @@ -1,8 +1,8 @@ package analyzer import ( - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/plan" ) // assignCatalog sets the catalog in the required nodes. @@ -10,7 +10,7 @@ func assignCatalog(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) span, _ := ctx.Span("assign_catalog") defer span.Finish() - return n.TransformUp(func(n sql.Node) (sql.Node, error) { + return plan.TransformUp(n, func(n sql.Node) (sql.Node, error) { if !n.Resolved() { return n, nil } diff --git a/sql/analyzer/assign_catalog_test.go b/sql/analyzer/assign_catalog_test.go index 67377bf3a..39aed1a4f 100644 --- a/sql/analyzer/assign_catalog_test.go +++ b/sql/analyzer/assign_catalog_test.go @@ -3,24 +3,24 @@ package analyzer import ( "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/plan" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" ) func TestAssignCatalog(t *testing.T) { require := require.New(t) f := getRule("assign_catalog") - db := mem.NewDatabase("foo") + db := memory.NewDatabase("foo") c := sql.NewCatalog() c.AddDatabase(db) a := NewDefault(c) a.Catalog.IndexRegistry = sql.NewIndexRegistry() - tbl := mem.NewTable("foo", nil) + tbl := memory.NewTable("foo", nil) node, err := f.Apply(sql.NewEmptyContext(), a, plan.NewCreateIndex("", plan.NewResolvedTable(tbl), nil, "", make(map[string]string))) diff --git a/sql/analyzer/assign_indexes.go b/sql/analyzer/assign_indexes.go index a0431c79f..b6bcb1327 100644 --- a/sql/analyzer/assign_indexes.go +++ b/sql/analyzer/assign_indexes.go @@ -3,10 +3,10 @@ package analyzer import ( "reflect" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" ) var errInvalidInRightEvaluation = errors.NewKind("expecting evaluation of IN expression right hand side to be a tuple, but it is %T") @@ -35,15 +35,42 @@ func assignIndexes(a *Analyzer, node sql.Node) (map[string]*indexLookup, error) } }() - var err error + aliases := make(map[string]sql.Expression) + var ( + err error + fn func(node sql.Node) bool + ) + fn = func(n sql.Node) bool { + if n == nil { + return true + } + + if prj, ok := n.(*plan.Project); ok { + for _, ex := range prj.Expressions() { + if alias, ok := ex.(*expression.Alias); ok { + if _, ok := aliases[alias.Name()]; !ok { + aliases[alias.Name()] = alias.Child + } + } + } + } else { + for _, ch := range n.Children() { + plan.Inspect(ch, fn) + } + } + + return true + } + plan.Inspect(node, func(node sql.Node) bool { filter, ok := node.(*plan.Filter) if !ok { return true } + fn(filter.Child) var result map[string]*indexLookup - result, err = getIndexes(filter.Expression, a) + result, err = getIndexes(filter.Expression, aliases, a) if err != nil { return false } @@ -60,16 +87,16 @@ func assignIndexes(a *Analyzer, node sql.Node) (map[string]*indexLookup, error) return indexes, err } -func getIndexes(e sql.Expression, a *Analyzer) (map[string]*indexLookup, error) { +func getIndexes(e sql.Expression, aliases map[string]sql.Expression, a *Analyzer) (map[string]*indexLookup, error) { var result = make(map[string]*indexLookup) switch e := e.(type) { case *expression.Or: - leftIndexes, err := getIndexes(e.Left, a) + leftIndexes, err := getIndexes(e.Left, aliases, a) if err != nil { return nil, err } - rightIndexes, err := getIndexes(e.Right, a) + rightIndexes, err := getIndexes(e.Right, aliases, a) if err != nil { return nil, err } @@ -101,7 +128,7 @@ func getIndexes(e sql.Expression, a *Analyzer) (map[string]*indexLookup, error) // the right branch is evaluable and the indexlookup supports set // operations. if !isEvaluable(c.Left()) && isEvaluable(c.Right()) { - idx := a.Catalog.IndexByExpression(a.Catalog.CurrentDatabase(), c.Left()) + idx := a.Catalog.IndexByExpression(a.Catalog.CurrentDatabase(), unifyExpressions(aliases, c.Left())...) if idx != nil { var nidx sql.NegateIndex if negate { @@ -178,7 +205,7 @@ func getIndexes(e sql.Expression, a *Analyzer) (map[string]*indexLookup, error) *expression.GreaterThan, *expression.LessThanOrEqual, *expression.GreaterThanOrEqual: - idx, lookup, err := getComparisonIndex(a, e.(expression.Comparer)) + idx, lookup, err := getComparisonIndex(a, e.(expression.Comparer), aliases) if err != nil || lookup == nil { return result, err } @@ -188,7 +215,7 @@ func getIndexes(e sql.Expression, a *Analyzer) (map[string]*indexLookup, error) lookup: lookup, } case *expression.Not: - r, err := getNegatedIndexes(a, e) + r, err := getNegatedIndexes(a, e, aliases) if err != nil { return nil, err } @@ -198,7 +225,7 @@ func getIndexes(e sql.Expression, a *Analyzer) (map[string]*indexLookup, error) } case *expression.Between: if !isEvaluable(e.Val) && isEvaluable(e.Upper) && isEvaluable(e.Lower) { - idx := a.Catalog.IndexByExpression(a.Catalog.CurrentDatabase(), e.Val) + idx := a.Catalog.IndexByExpression(a.Catalog.CurrentDatabase(), unifyExpressions(aliases, e.Val)...) if idx != nil { // release the index if it was not used defer func() { @@ -238,7 +265,7 @@ func getIndexes(e sql.Expression, a *Analyzer) (map[string]*indexLookup, error) exprs := splitExpression(e) used := make(map[sql.Expression]struct{}) - result, err := getMultiColumnIndexes(exprs, a, used) + result, err := getMultiColumnIndexes(exprs, a, used, aliases) if err != nil { return nil, err } @@ -248,7 +275,7 @@ func getIndexes(e sql.Expression, a *Analyzer) (map[string]*indexLookup, error) continue } - indexes, err := getIndexes(e, a) + indexes, err := getIndexes(e, aliases, a) if err != nil { return nil, err } @@ -262,6 +289,28 @@ func getIndexes(e sql.Expression, a *Analyzer) (map[string]*indexLookup, error) return result, nil } +func unifyExpressions(aliases map[string]sql.Expression, expr ...sql.Expression) []sql.Expression { + expressions := make([]sql.Expression, len(expr)) + + for i, e := range expr { + uex := e + name := e.String() + if n, ok := e.(sql.Nameable); ok { + name = n.Name() + } + + if aliases != nil && len(aliases) > 0 { + if alias, ok := aliases[name]; ok { + uex = alias + } + } + + expressions[i] = uex + } + + return expressions +} + func betweenIndexLookup(index sql.Index, upper, lower []interface{}) (sql.IndexLookup, error) { ai, isAscend := index.(sql.AscendIndex) di, isDescend := index.(sql.DescendIndex) @@ -293,6 +342,7 @@ func betweenIndexLookup(index sql.Index, upper, lower []interface{}) (sql.IndexL func getComparisonIndex( a *Analyzer, e expression.Comparer, + aliases map[string]sql.Expression, ) (sql.Index, sql.IndexLookup, error) { left, right := e.Left(), e.Right() // if the form is SOMETHING OP {INDEXABLE EXPR}, swap it, so it's {INDEXABLE EXPR} OP SOMETHING @@ -301,7 +351,7 @@ func getComparisonIndex( } if !isEvaluable(left) && isEvaluable(right) { - idx := a.Catalog.IndexByExpression(a.Catalog.CurrentDatabase(), left) + idx := a.Catalog.IndexByExpression(a.Catalog.CurrentDatabase(), unifyExpressions(aliases, left)...) if idx != nil { value, err := right.Eval(sql.NewEmptyContext(), nil) if err != nil { @@ -363,10 +413,10 @@ func comparisonIndexLookup( return nil, nil } -func getNegatedIndexes(a *Analyzer, not *expression.Not) (map[string]*indexLookup, error) { +func getNegatedIndexes(a *Analyzer, not *expression.Not, aliases map[string]sql.Expression) (map[string]*indexLookup, error) { switch e := not.Child.(type) { case *expression.Not: - return getIndexes(e.Child, a) + return getIndexes(e.Child, aliases, a) case *expression.Equals: left, right := e.Left(), e.Right() // if the form is SOMETHING OP {INDEXABLE EXPR}, swap it, so it's {INDEXABLE EXPR} OP SOMETHING @@ -378,7 +428,7 @@ func getNegatedIndexes(a *Analyzer, not *expression.Not) (map[string]*indexLooku return nil, nil } - idx := a.Catalog.IndexByExpression(a.Catalog.CurrentDatabase(), left) + idx := a.Catalog.IndexByExpression(a.Catalog.CurrentDatabase(), unifyExpressions(aliases, left)...) if idx == nil { return nil, nil } @@ -410,37 +460,37 @@ func getNegatedIndexes(a *Analyzer, not *expression.Not) (map[string]*indexLooku return result, nil case *expression.GreaterThan: lte := expression.NewLessThanOrEqual(e.Left(), e.Right()) - return getIndexes(lte, a) + return getIndexes(lte, aliases, a) case *expression.GreaterThanOrEqual: lt := expression.NewLessThan(e.Left(), e.Right()) - return getIndexes(lt, a) + return getIndexes(lt, aliases, a) case *expression.LessThan: gte := expression.NewGreaterThanOrEqual(e.Left(), e.Right()) - return getIndexes(gte, a) + return getIndexes(gte, aliases, a) case *expression.LessThanOrEqual: gt := expression.NewGreaterThan(e.Left(), e.Right()) - return getIndexes(gt, a) + return getIndexes(gt, aliases, a) case *expression.Between: or := expression.NewOr( expression.NewLessThan(e.Val, e.Lower), expression.NewGreaterThan(e.Val, e.Upper), ) - return getIndexes(or, a) + return getIndexes(or, aliases, a) case *expression.Or: and := expression.NewAnd( expression.NewNot(e.Left), expression.NewNot(e.Right), ) - return getIndexes(and, a) + return getIndexes(and, aliases, a) case *expression.And: or := expression.NewOr( expression.NewNot(e.Left), expression.NewNot(e.Right), ) - return getIndexes(or, a) + return getIndexes(or, aliases, a) default: return nil, nil @@ -481,6 +531,7 @@ func getMultiColumnIndexes( exprs []sql.Expression, a *Analyzer, used map[sql.Expression]struct{}, + aliases map[string]sql.Expression, ) (map[string]*indexLookup, error) { result := make(map[string]*indexLookup) columnExprs := columnExprsByTable(exprs) @@ -502,7 +553,7 @@ func getMultiColumnIndexes( } if len(selected) > 0 { - index, lookup, err := getMultiColumnIndexForExpressions(a, selected, exps, used) + index, lookup, err := getMultiColumnIndexForExpressions(a, selected, exps, used, aliases) if err != nil || lookup == nil { if index != nil { a.Catalog.ReleaseIndex(index) @@ -534,8 +585,9 @@ func getMultiColumnIndexForExpressions( selected []sql.Expression, exprs []columnExpr, used map[sql.Expression]struct{}, + aliases map[string]sql.Expression, ) (index sql.Index, lookup sql.IndexLookup, err error) { - index = a.Catalog.IndexByExpression(a.Catalog.CurrentDatabase(), selected...) + index = a.Catalog.IndexByExpression(a.Catalog.CurrentDatabase(), unifyExpressions(aliases, selected...)...) if index != nil { var first sql.Expression for _, e := range exprs { @@ -707,8 +759,20 @@ func containsColumns(e sql.Expression) bool { return result } +func containsSubquery(e sql.Expression) bool { + var result bool + expression.Inspect(e, func(e sql.Expression) bool { + if _, ok := e.(*expression.Subquery); ok { + result = true + return false + } + return true + }) + return result +} + func isEvaluable(e sql.Expression) bool { - return !containsColumns(e) + return !containsColumns(e) && !containsSubquery(e) } func canMergeIndexes(a, b sql.IndexLookup) bool { diff --git a/sql/analyzer/assign_indexes_test.go b/sql/analyzer/assign_indexes_test.go index c7145f2c9..d8e61f05a 100644 --- a/sql/analyzer/assign_indexes_test.go +++ b/sql/analyzer/assign_indexes_test.go @@ -5,11 +5,11 @@ import ( "strings" "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" ) func TestNegateIndex(t *testing.T) { @@ -29,7 +29,7 @@ func TestNegateIndex(t *testing.T) { a := NewDefault(catalog) - t1 := mem.NewTable("t1", sql.Schema{ + t1 := memory.NewTable("t1", sql.Schema{ {Name: "foo", Type: sql.Int64, Source: "t1"}, }) @@ -86,11 +86,11 @@ func TestAssignIndexes(t *testing.T) { a := NewDefault(catalog) - t1 := mem.NewTable("t1", sql.Schema{ + t1 := memory.NewTable("t1", sql.Schema{ {Name: "foo", Type: sql.Int64, Source: "t1"}, }) - t2 := mem.NewTable("t2", sql.Schema{ + t2 := memory.NewTable("t2", sql.Schema{ {Name: "bar", Type: sql.Int64, Source: "t2"}, {Name: "baz", Type: sql.Int64, Source: "t2"}, }) @@ -682,7 +682,7 @@ func TestGetIndexes(t *testing.T) { t.Run(tt.expr.String(), func(t *testing.T) { require := require.New(t) - result, err := getIndexes(tt.expr, a) + result, err := getIndexes(tt.expr, nil, a) if tt.ok { require.NoError(err) require.Equal(tt.expected, result) @@ -779,7 +779,7 @@ func TestGetMultiColumnIndexes(t *testing.T) { lit(6), ), } - result, err := getMultiColumnIndexes(exprs, a, used) + result, err := getMultiColumnIndexes(exprs, a, used, nil) require.NoError(err) expected := map[string]*indexLookup{ @@ -861,7 +861,7 @@ func TestContainsSources(t *testing.T) { func TestNodeSources(t *testing.T) { sources := nodeSources( plan.NewResolvedTable( - mem.NewTable("foo", sql.Schema{ + memory.NewTable("foo", sql.Schema{ {Source: "foo"}, {Source: "foo"}, {Source: "bar"}, diff --git a/sql/analyzer/batch.go b/sql/analyzer/batch.go index 9d6484a41..8153e32c7 100644 --- a/sql/analyzer/batch.go +++ b/sql/analyzer/batch.go @@ -3,7 +3,7 @@ package analyzer import ( "reflect" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // RuleFunc is the function to be applied in a rule. diff --git a/sql/analyzer/common_test.go b/sql/analyzer/common_test.go index d21c97254..a5beb26c9 100644 --- a/sql/analyzer/common_test.go +++ b/sql/analyzer/common_test.go @@ -1,8 +1,8 @@ package analyzer import ( - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" ) func not(e sql.Expression) sql.Expression { diff --git a/sql/analyzer/convert_dates.go b/sql/analyzer/convert_dates.go new file mode 100644 index 000000000..c7af55431 --- /dev/null +++ b/sql/analyzer/convert_dates.go @@ -0,0 +1,186 @@ +package analyzer + +import ( + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/expression/function" + "github.com/src-d/go-mysql-server/sql/expression/function/aggregation" + "github.com/src-d/go-mysql-server/sql/plan" +) + +// convertDates wraps all expressions of date and datetime type with converts +// to ensure the date range is validated. +func convertDates(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { + if !n.Resolved() { + return n, nil + } + + // Replacements contains a mapping from columns to the alias they will be + // replaced by. + var replacements = make(map[tableCol]string) + + return plan.TransformUp(n, func(n sql.Node) (sql.Node, error) { + exp, ok := n.(sql.Expressioner) + if !ok { + return n, nil + } + + // nodeReplacements are all the replacements found in the current node. + // These replacements are not applied to the current node, only to + // parent nodes. + var nodeReplacements = make(map[tableCol]string) + + var expressions = make(map[string]bool) + switch exp := exp.(type) { + case *plan.Project: + for _, e := range exp.Projections { + expressions[e.String()] = true + } + case *plan.GroupBy: + for _, e := range exp.Aggregate { + expressions[e.String()] = true + } + } + + var result sql.Node + var err error + switch exp := exp.(type) { + case *plan.GroupBy: + var aggregate = make([]sql.Expression, len(exp.Aggregate)) + for i, a := range exp.Aggregate { + agg, err := expression.TransformUp(a, func(e sql.Expression) (sql.Expression, error) { + return addDateConvert(e, exp, replacements, nodeReplacements, expressions, true) + }) + if err != nil { + return nil, err + } + + aggregate[i] = agg + + if _, ok := agg.(*expression.Alias); !ok && agg.String() != a.String() { + nodeReplacements[tableCol{"", a.String()}] = agg.String() + } + } + + var grouping = make([]sql.Expression, len(exp.Grouping)) + for i, g := range exp.Grouping { + gr, err := expression.TransformUp(g, func(e sql.Expression) (sql.Expression, error) { + return addDateConvert(e, exp, replacements, nodeReplacements, expressions, false) + }) + if err != nil { + return nil, err + } + grouping[i] = gr + } + + result = plan.NewGroupBy(aggregate, grouping, exp.Child) + case *plan.Project: + var projections = make([]sql.Expression, len(exp.Projections)) + for i, e := range exp.Projections { + expr, err := expression.TransformUp(e, func(e sql.Expression) (sql.Expression, error) { + return addDateConvert(e, exp, replacements, nodeReplacements, expressions, true) + }) + if err != nil { + return nil, err + } + + projections[i] = expr + + if _, ok := expr.(*expression.Alias); !ok && expr.String() != e.String() { + nodeReplacements[tableCol{"", e.String()}] = expr.String() + } + } + + result = plan.NewProject(projections, exp.Child) + default: + result, err = plan.TransformExpressions(n, func(e sql.Expression) (sql.Expression, error) { + return addDateConvert(e, n, replacements, nodeReplacements, expressions, false) + }) + } + + if err != nil { + return nil, err + } + + // We're done with this node, so copy all the replacements found in + // this node to the global replacements in order to make the necesssary + // changes in parent nodes. + for tc, n := range nodeReplacements { + replacements[tc] = n + } + + return result, err + }) +} + +func addDateConvert( + e sql.Expression, + node sql.Node, + replacements, nodeReplacements map[tableCol]string, + expressions map[string]bool, + aliasRootProjections bool, +) (sql.Expression, error) { + var result sql.Expression + + // No need to wrap expressions that already validate times, such as + // convert, date_add, etc and those expressions whose Type method + // cannot be called because they are placeholders. + switch e := e.(type) { + case *aggregation.Max: + child, err := addDateConvert(e.Child, node, replacements, nodeReplacements, expressions, false) + if err != nil { + return nil, err + } + + return aggregation.NewMax(child), nil + case *aggregation.Min: + child, err := addDateConvert(e.Child, node, replacements, nodeReplacements, expressions, false) + if err != nil { + return nil, err + } + + return aggregation.NewMin(child), nil + case *expression.Convert, + *expression.Arithmetic, + *function.DateAdd, + *function.DateSub, + *expression.Star, + *expression.DefaultColumn, + *expression.Alias: + return e, nil + default: + // If it's a replacement, just replace it with the correct GetField + // because we know that it's already converted to a correct date + // and there is no point to do so again. + if gf, ok := e.(*expression.GetField); ok { + if name, ok := replacements[tableCol{gf.Table(), gf.Name()}]; ok { + return expression.NewGetField(gf.Index(), gf.Type(), name, gf.IsNullable()), nil + } + } + + switch e.Type() { + case sql.Date: + result = expression.NewConvert(e, expression.ConvertToDate) + case sql.Timestamp: + result = expression.NewConvert(e, expression.ConvertToDatetime) + default: + result = e + } + } + + // Only do this if it's a root expression in a project or group by. + switch node.(type) { + case *plan.Project, *plan.GroupBy: + // If it was originally a GetField, and it's not anymore it's + // because we wrapped it in a convert. We need to make it an alias + // and propagate the changes up the chain. + if gf, ok := e.(*expression.GetField); ok && expressions[e.String()] && aliasRootProjections { + if _, ok := result.(*expression.GetField); !ok { + result = expression.NewAlias(result, gf.Name()) + nodeReplacements[tableCol{gf.Table(), gf.Name()}] = gf.Name() + } + } + } + + return result, nil +} diff --git a/sql/analyzer/convert_dates_test.go b/sql/analyzer/convert_dates_test.go new file mode 100644 index 000000000..72ba53868 --- /dev/null +++ b/sql/analyzer/convert_dates_test.go @@ -0,0 +1,293 @@ +package analyzer + +import ( + "testing" + + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/expression/function" + "github.com/src-d/go-mysql-server/sql/expression/function/aggregation" + "github.com/src-d/go-mysql-server/sql/plan" + "github.com/stretchr/testify/require" +) + +func TestConvertDates(t *testing.T) { + testCases := []struct { + name string + in sql.Expression + out sql.Expression + }{ + { + "arithmetic with dates", + expression.NewPlus(expression.NewLiteral("", sql.Timestamp), expression.NewLiteral("", sql.Timestamp)), + expression.NewPlus( + expression.NewConvert( + expression.NewLiteral("", sql.Timestamp), + expression.ConvertToDatetime, + ), + expression.NewConvert( + expression.NewLiteral("", sql.Timestamp), + expression.ConvertToDatetime, + ), + ), + }, + { + "star", + expression.NewStar(), + expression.NewStar(), + }, + { + "default column", + expression.NewDefaultColumn("foo"), + expression.NewDefaultColumn("foo"), + }, + { + "convert to date", + expression.NewConvert( + expression.NewPlus( + expression.NewLiteral("", sql.Timestamp), + expression.NewLiteral("", sql.Timestamp), + ), + expression.ConvertToDatetime, + ), + expression.NewConvert( + expression.NewPlus( + expression.NewConvert( + expression.NewLiteral("", sql.Timestamp), + expression.ConvertToDatetime, + ), + expression.NewConvert( + expression.NewLiteral("", sql.Timestamp), + expression.ConvertToDatetime, + ), + ), + expression.ConvertToDatetime, + ), + }, + { + "min aggregation", + aggregation.NewMin( + expression.NewGetField(0, sql.Timestamp, "foo", false), + ), + aggregation.NewMin( + expression.NewConvert( + expression.NewGetField(0, sql.Timestamp, "foo", false), + expression.ConvertToDatetime, + ), + ), + }, + { + "max aggregation", + aggregation.NewMax( + expression.NewGetField(0, sql.Timestamp, "foo", false), + ), + aggregation.NewMax( + expression.NewConvert( + expression.NewGetField(0, sql.Timestamp, "foo", false), + expression.ConvertToDatetime, + ), + ), + }, + { + "convert to other type", + expression.NewConvert( + expression.NewLiteral("", sql.Text), + expression.ConvertToBinary, + ), + expression.NewConvert( + expression.NewLiteral("", sql.Text), + expression.ConvertToBinary, + ), + }, + { + "datetime col in alias", + expression.NewAlias( + expression.NewLiteral("", sql.Timestamp), + "foo", + ), + expression.NewAlias( + expression.NewConvert( + expression.NewLiteral("", sql.Timestamp), + expression.ConvertToDatetime, + ), + "foo", + ), + }, + { + "date col in alias", + expression.NewAlias( + expression.NewLiteral("", sql.Date), + "foo", + ), + expression.NewAlias( + expression.NewConvert( + expression.NewLiteral("", sql.Date), + expression.ConvertToDate, + ), + "foo", + ), + }, + { + "date add", + newDateAdd( + expression.NewLiteral("", sql.Timestamp), + expression.NewInterval(expression.NewLiteral(int64(1), sql.Int64), "DAY"), + ), + newDateAdd( + expression.NewConvert( + expression.NewLiteral("", sql.Timestamp), + expression.ConvertToDatetime, + ), + expression.NewInterval(expression.NewLiteral(int64(1), sql.Int64), "DAY"), + ), + }, + { + "date sub", + newDateSub( + expression.NewLiteral("", sql.Timestamp), + expression.NewInterval(expression.NewLiteral(int64(1), sql.Int64), "DAY"), + ), + newDateSub( + expression.NewConvert( + expression.NewLiteral("", sql.Timestamp), + expression.ConvertToDatetime, + ), + expression.NewInterval(expression.NewLiteral(int64(1), sql.Int64), "DAY"), + ), + }, + } + + table := plan.NewResolvedTable(memory.NewTable("t", nil)) + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + input := plan.NewProject([]sql.Expression{tt.in}, table) + expected := plan.NewProject([]sql.Expression{tt.out}, table) + result, err := convertDates(sql.NewEmptyContext(), nil, input) + require.NoError(t, err) + require.Equal(t, expected, result) + }) + } +} + +func TestConvertDatesProject(t *testing.T) { + table := plan.NewResolvedTable(memory.NewTable("t", nil)) + input := plan.NewFilter( + expression.NewEquals( + expression.NewGetField(0, sql.Int64, "foo", false), + expression.NewLiteral("2019-06-06 00:00:00", sql.Text), + ), + plan.NewProject([]sql.Expression{ + expression.NewGetField(0, sql.Timestamp, "foo", false), + }, table), + ) + expected := plan.NewFilter( + expression.NewEquals( + expression.NewGetField(0, sql.Int64, "foo", false), + expression.NewLiteral("2019-06-06 00:00:00", sql.Text), + ), + plan.NewProject([]sql.Expression{ + expression.NewAlias( + expression.NewConvert( + expression.NewGetField(0, sql.Timestamp, "foo", false), + expression.ConvertToDatetime, + ), + "foo", + ), + }, table), + ) + + result, err := convertDates(sql.NewEmptyContext(), nil, input) + require.NoError(t, err) + require.Equal(t, expected, result) +} + +func TestConvertDatesGroupBy(t *testing.T) { + table := plan.NewResolvedTable(memory.NewTable("t", nil)) + input := plan.NewFilter( + expression.NewEquals( + expression.NewGetField(0, sql.Int64, "foo", false), + expression.NewLiteral("2019-06-06 00:00:00", sql.Text), + ), + plan.NewGroupBy( + []sql.Expression{ + expression.NewGetField(0, sql.Timestamp, "foo", false), + }, + []sql.Expression{ + expression.NewGetField(0, sql.Timestamp, "foo", false), + }, table, + ), + ) + expected := plan.NewFilter( + expression.NewEquals( + expression.NewGetField(0, sql.Int64, "foo", false), + expression.NewLiteral("2019-06-06 00:00:00", sql.Text), + ), + plan.NewGroupBy( + []sql.Expression{ + expression.NewAlias( + expression.NewConvert( + expression.NewGetField(0, sql.Timestamp, "foo", false), + expression.ConvertToDatetime, + ), + "foo", + ), + }, + []sql.Expression{ + expression.NewConvert( + expression.NewGetField(0, sql.Timestamp, "foo", false), + expression.ConvertToDatetime, + ), + }, + table, + ), + ) + + result, err := convertDates(sql.NewEmptyContext(), nil, input) + require.NoError(t, err) + require.Equal(t, expected, result) +} + +func TestConvertDatesFieldReference(t *testing.T) { + table := plan.NewResolvedTable(memory.NewTable("t", nil)) + input := plan.NewFilter( + expression.NewEquals( + expression.NewGetField(0, sql.Int64, "DAYOFWEEK(foo)", false), + expression.NewLiteral("2019-06-06 00:00:00", sql.Text), + ), + plan.NewProject([]sql.Expression{ + function.NewDayOfWeek( + expression.NewGetField(0, sql.Timestamp, "foo", false), + ), + }, table), + ) + expected := plan.NewFilter( + expression.NewEquals( + expression.NewGetField(0, sql.Int64, "DAYOFWEEK(convert(foo, datetime))", false), + expression.NewLiteral("2019-06-06 00:00:00", sql.Text), + ), + plan.NewProject([]sql.Expression{ + function.NewDayOfWeek( + expression.NewConvert( + expression.NewGetField(0, sql.Timestamp, "foo", false), + expression.ConvertToDatetime, + ), + ), + }, table), + ) + + result, err := convertDates(sql.NewEmptyContext(), nil, input) + require.NoError(t, err) + require.Equal(t, expected, result) +} + +func newDateAdd(l, r sql.Expression) sql.Expression { + e, _ := function.NewDateAdd(l, r) + return e +} + +func newDateSub(l, r sql.Expression) sql.Expression { + e, _ := function.NewDateSub(l, r) + return e +} diff --git a/sql/analyzer/filters.go b/sql/analyzer/filters.go index 2932e7118..bbe54adc7 100644 --- a/sql/analyzer/filters.go +++ b/sql/analyzer/filters.go @@ -3,8 +3,8 @@ package analyzer import ( "reflect" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" ) type filters map[string][]sql.Expression @@ -20,7 +20,7 @@ func exprToTableFilters(expr sql.Expression) filters { for _, expr := range splitExpression(expr) { var seenTables = make(map[string]struct{}) var lastTable string - _, _ = expr.TransformUp(func(e sql.Expression) (sql.Expression, error) { + expression.Inspect(expr, func(e sql.Expression) bool { f, ok := e.(*expression.GetField) if ok { if _, ok := seenTables[f.Table()]; !ok { @@ -29,7 +29,7 @@ func exprToTableFilters(expr sql.Expression) filters { } } - return e, nil + return true }) if len(seenTables) == 1 { diff --git a/sql/analyzer/filters_test.go b/sql/analyzer/filters_test.go index 0e171ad95..999b95a6a 100644 --- a/sql/analyzer/filters_test.go +++ b/sql/analyzer/filters_test.go @@ -3,9 +3,9 @@ package analyzer import ( "testing" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestFiltersMerge(t *testing.T) { diff --git a/sql/analyzer/optimization_rules.go b/sql/analyzer/optimization_rules.go index f8795348b..88283cf9f 100644 --- a/sql/analyzer/optimization_rules.go +++ b/sql/analyzer/optimization_rules.go @@ -1,10 +1,10 @@ package analyzer import ( - errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" + "gopkg.in/src-d/go-errors.v1" ) func eraseProjection(ctx *sql.Context, a *Analyzer, node sql.Node) (sql.Node, error) { @@ -17,7 +17,7 @@ func eraseProjection(ctx *sql.Context, a *Analyzer, node sql.Node) (sql.Node, er a.Log("erase projection, node of type: %T", node) - return node.TransformUp(func(node sql.Node) (sql.Node, error) { + return plan.TransformUp(node, func(node sql.Node) (sql.Node, error) { project, ok := node.(*plan.Project) if ok && project.Schema().Equals(project.Child.Schema()) { a.Log("project erased") @@ -33,19 +33,22 @@ func optimizeDistinct(ctx *sql.Context, a *Analyzer, node sql.Node) (sql.Node, e defer span.Finish() a.Log("optimize distinct, node of type: %T", node) - if node, ok := node.(*plan.Distinct); ok { - var isSorted bool - _, _ = node.TransformUp(func(node sql.Node) (sql.Node, error) { + if n, ok := node.(*plan.Distinct); ok { + var sortField *expression.GetField + plan.Inspect(n, func(node sql.Node) bool { a.Log("checking for optimization in node of type: %T", node) - if _, ok := node.(*plan.Sort); ok { - isSorted = true + if sort, ok := node.(*plan.Sort); ok && sortField == nil { + if col, ok := sort.SortFields[0].Column.(*expression.GetField); ok { + sortField = col + } + return false } - return node, nil + return true }) - if isSorted { + if sortField != nil && n.Schema().Contains(sortField.Name(), sortField.Table()) { a.Log("distinct optimized for ordered output") - return plan.NewOrderedDistinct(node.Child), nil + return plan.NewOrderedDistinct(n.Child), nil } } @@ -65,7 +68,7 @@ func reorderProjection(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, err a.Log("reorder projection, node of type: %T", n) // Then we transform the projection - return n.TransformUp(func(node sql.Node) (sql.Node, error) { + return plan.TransformUp(n, func(node sql.Node) (sql.Node, error) { project, ok := node.(*plan.Project) // When we transform the projection, the children will always be // unresolved in the case we want to fix, as the reorder happens just @@ -92,7 +95,7 @@ func reorderProjection(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, err // And add projection nodes where needed in the child tree. var didNeedReorder bool - child, err := project.Child.TransformUp(func(node sql.Node) (sql.Node, error) { + child, err := plan.TransformUp(project.Child, func(node sql.Node) (sql.Node, error) { var requiredColumns []string switch node := node.(type) { case *plan.Sort, *plan.Filter: @@ -200,7 +203,7 @@ func moveJoinConditionsToFilter(ctx *sql.Context, a *Analyzer, n sql.Node) (sql. a.Log("moving join conditions to filter, node of type: %T", n) - return n.TransformUp(func(n sql.Node) (sql.Node, error) { + return plan.TransformUp(n, func(n sql.Node) (sql.Node, error) { join, ok := n.(*plan.InnerJoin) if !ok { return n, nil @@ -268,7 +271,7 @@ func removeUnnecessaryConverts(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.N a.Log("removing unnecessary converts, node of type: %T", n) - return n.TransformExpressionsUp(func(e sql.Expression) (sql.Expression, error) { + return plan.TransformExpressionsUp(n, func(e sql.Expression) (sql.Expression, error) { if c, ok := e.(*expression.Convert); ok && c.Child.Type() == c.Type() { return c.Child, nil } @@ -336,13 +339,13 @@ func evalFilter(ctx *sql.Context, a *Analyzer, node sql.Node) (sql.Node, error) a.Log("evaluating filters, node of type: %T", node) - return node.TransformUp(func(node sql.Node) (sql.Node, error) { + return plan.TransformUp(node, func(node sql.Node) (sql.Node, error) { filter, ok := node.(*plan.Filter) if !ok { return node, nil } - e, err := filter.Expression.TransformUp(func(e sql.Expression) (sql.Expression, error) { + e, err := expression.TransformUp(filter.Expression, func(e sql.Expression) (sql.Expression, error) { switch e := e.(type) { case *expression.Or: if isTrue(e.Left) { @@ -379,29 +382,20 @@ func evalFilter(ctx *sql.Context, a *Analyzer, node sql.Node) (sql.Node, error) return e.Left, nil } + return e, nil + case *expression.Literal, expression.Tuple: return e, nil default: if !isEvaluable(e) { return e, nil } - if _, ok := e.(*expression.Literal); ok { - return e, nil - } - + // All other expressions types can be evaluated once and turned into literals for the rest of query execution val, err := e.Eval(ctx, nil) if err != nil { - return nil, err - } - - val, err = sql.Boolean.Convert(val) - if err != nil { - // don't make it fail because of this, just return the - // original expression return e, nil } - - return expression.NewLiteral(val.(bool), sql.Boolean), nil + return expression.NewLiteral(val, e.Type()), nil } }) if err != nil { diff --git a/sql/analyzer/optimization_rules_test.go b/sql/analyzer/optimization_rules_test.go index 6e0d5c11e..60dee3b0f 100644 --- a/sql/analyzer/optimization_rules_test.go +++ b/sql/analyzer/optimization_rules_test.go @@ -3,17 +3,17 @@ package analyzer import ( "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" ) func TestReorderProjection(t *testing.T) { f := getRule("reorder_projection") - table := mem.NewTable("mytable", sql.Schema{{ + table := memory.NewTable("mytable", sql.Schema{{ Name: "i", Source: "mytable", Type: sql.Int64, }}) @@ -137,7 +137,7 @@ func TestEraseProjection(t *testing.T) { require := require.New(t) f := getRule("erase_projection") - table := mem.NewTable("mytable", sql.Schema{{ + table := memory.NewTable("mytable", sql.Schema{{ Name: "i", Source: "mytable", Type: sql.Int64, }}) @@ -186,38 +186,68 @@ func TestEraseProjection(t *testing.T) { } func TestOptimizeDistinct(t *testing.T) { - require := require.New(t) - - t1 := mem.NewTable("foo", nil) - t2 := mem.NewTable("foo", nil) + t1 := memory.NewTable("foo", sql.Schema{ + {Name: "a", Source: "foo"}, + {Name: "b", Source: "foo"}, + }) - notSorted := plan.NewDistinct(plan.NewResolvedTable(t1)) - sorted := plan.NewDistinct(plan.NewSort(nil, plan.NewResolvedTable(t2))) + testCases := []struct { + name string + child sql.Node + optimized bool + }{ + { + "without sort", + plan.NewResolvedTable(t1), + false, + }, + { + "sort but column not projected", + plan.NewSort( + []plan.SortField{ + {Column: gf(0, "foo", "c")}, + }, + plan.NewResolvedTable(t1), + ), + false, + }, + { + "sort and column projected", + plan.NewSort( + []plan.SortField{ + {Column: gf(0, "foo", "a")}, + }, + plan.NewResolvedTable(t1), + ), + true, + }, + } rule := getRule("optimize_distinct") - analyzedNotSorted, err := rule.Apply(sql.NewEmptyContext(), nil, notSorted) - require.NoError(err) - - analyzedSorted, err := rule.Apply(sql.NewEmptyContext(), nil, sorted) - require.NoError(err) + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + node, err := rule.Apply(sql.NewEmptyContext(), nil, plan.NewDistinct(tt.child)) + require.NoError(t, err) - require.Equal(notSorted, analyzedNotSorted) - require.Equal(plan.NewOrderedDistinct(sorted.Child), analyzedSorted) + _, ok := node.(*plan.OrderedDistinct) + require.Equal(t, tt.optimized, ok) + }) + } } func TestMoveJoinConditionsToFilter(t *testing.T) { - t1 := mem.NewTable("t1", sql.Schema{ + t1 := memory.NewTable("t1", sql.Schema{ {Name: "a", Source: "t1", Type: sql.Int64}, {Name: "b", Source: "t1", Type: sql.Int64}, }) - t2 := mem.NewTable("t2", sql.Schema{ + t2 := memory.NewTable("t2", sql.Schema{ {Name: "c", Source: "t2", Type: sql.Int64}, {Name: "d", Source: "t2", Type: sql.Int64}, }) - t3 := mem.NewTable("t3", sql.Schema{ + t3 := memory.NewTable("t3", sql.Schema{ {Name: "e", Source: "t3", Type: sql.Int64}, {Name: "f", Source: "t3", Type: sql.Int64}, }) @@ -295,7 +325,7 @@ func TestMoveJoinConditionsToFilter(t *testing.T) { } func TestEvalFilter(t *testing.T) { - inner := mem.NewTable("foo", nil) + inner := memory.NewTable("foo", nil) rule := getRule("eval_filter") testCases := []struct { @@ -428,7 +458,7 @@ func TestRemoveUnnecessaryConverts(t *testing.T) { node := plan.NewProject([]sql.Expression{ expression.NewConvert(tt.childExpr, tt.castType), }, - plan.NewResolvedTable(mem.NewTable("foo", nil)), + plan.NewResolvedTable(memory.NewTable("foo", nil)), ) result, err := removeUnnecessaryConverts( diff --git a/sql/analyzer/parallelize.go b/sql/analyzer/parallelize.go index 126ab6d1e..a56b9479e 100644 --- a/sql/analyzer/parallelize.go +++ b/sql/analyzer/parallelize.go @@ -1,8 +1,17 @@ package analyzer import ( - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" + "strconv" + + "github.com/go-kit/kit/metrics/discard" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/plan" +) + +var ( + // ParallelQueryCounter describes a metric that accumulates + // number of parallel queries monotonically. + ParallelQueryCounter = discard.NewCounter() ) func shouldParallelize(node sql.Node) bool { @@ -25,10 +34,11 @@ func parallelize(ctx *sql.Context, a *Analyzer, node sql.Node) (sql.Node, error) return node, nil } - node, err := node.TransformUp(func(node sql.Node) (sql.Node, error) { + node, err := plan.TransformUp(node, func(node sql.Node) (sql.Node, error) { if !isParallelizable(node) { return node, nil } + ParallelQueryCounter.With("parallelism", strconv.Itoa(a.Parallelism)).Add(1) return plan.NewExchange(a.Parallelism, node), nil }) @@ -37,7 +47,7 @@ func parallelize(ctx *sql.Context, a *Analyzer, node sql.Node) (sql.Node, error) return nil, err } - return node.TransformUp(removeRedundantExchanges) + return plan.TransformUp(node, removeRedundantExchanges) } // removeRedundantExchanges removes all the exchanges except for the topmost @@ -48,13 +58,17 @@ func removeRedundantExchanges(node sql.Node) (sql.Node, error) { return node, nil } - e := &protectedExchange{exchange} - return e.TransformUp(func(node sql.Node) (sql.Node, error) { + child, err := plan.TransformUp(exchange.Child, func(node sql.Node) (sql.Node, error) { if exchange, ok := node.(*plan.Exchange); ok { return exchange.Child, nil } return node, nil }) + if err != nil { + return nil, err + } + + return exchange.WithChildren(child) } func isParallelizable(node sql.Node) bool { @@ -93,21 +107,3 @@ func isParallelizable(node sql.Node) bool { return ok && tableSeen && lastWasTable } - -// protectedExchange is a placeholder node that protects a certain exchange -// node from being removed during transformations. -type protectedExchange struct { - *plan.Exchange -} - -// TransformUp transforms the child with the given transform function but it -// will not call the transform function with the new instance. Instead of -// another protectedExchange, it will return an Exchange. -func (e *protectedExchange) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - child, err := e.Child.TransformUp(f) - if err != nil { - return nil, err - } - - return plan.NewExchange(e.Parallelism, child), nil -} diff --git a/sql/analyzer/parallelize_test.go b/sql/analyzer/parallelize_test.go index 8cbeb1d1f..5f554a442 100644 --- a/sql/analyzer/parallelize_test.go +++ b/sql/analyzer/parallelize_test.go @@ -3,16 +3,16 @@ package analyzer import ( "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" ) func TestParallelize(t *testing.T) { require := require.New(t) - table := mem.NewTable("t", nil) + table := memory.NewTable("t", nil) rule := getRuleFrom(OnceAfterAll, "parallelize") node := plan.NewProject( nil, @@ -57,7 +57,7 @@ func TestParallelize(t *testing.T) { func TestParallelizeCreateIndex(t *testing.T) { require := require.New(t) - table := mem.NewTable("t", nil) + table := memory.NewTable("t", nil) rule := getRuleFrom(OnceAfterAll, "parallelize") node := plan.NewCreateIndex( "", @@ -73,7 +73,7 @@ func TestParallelizeCreateIndex(t *testing.T) { } func TestIsParallelizable(t *testing.T) { - table := mem.NewTable("t", nil) + table := memory.NewTable("t", nil) testCases := []struct { name string @@ -172,7 +172,7 @@ func TestIsParallelizable(t *testing.T) { func TestRemoveRedundantExchanges(t *testing.T) { require := require.New(t) - table := mem.NewTable("t", nil) + table := memory.NewTable("t", nil) node := plan.NewProject( nil, @@ -222,7 +222,7 @@ func TestRemoveRedundantExchanges(t *testing.T) { ), ) - result, err := node.TransformUp(removeRedundantExchanges) + result, err := plan.TransformUp(node, removeRedundantExchanges) require.NoError(err) require.Equal(expected, result) } diff --git a/sql/analyzer/process.go b/sql/analyzer/process.go index 5d486acc2..aa0d9d0c7 100644 --- a/sql/analyzer/process.go +++ b/sql/analyzer/process.go @@ -1,8 +1,8 @@ package analyzer import ( - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/plan" ) // trackProcess will wrap the query in a process node and add progress items @@ -19,7 +19,7 @@ func trackProcess(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { processList := a.Catalog.ProcessList var seen = make(map[string]struct{}) - n, err := n.TransformUp(func(n sql.Node) (sql.Node, error) { + n, err := plan.TransformUp(n, func(n sql.Node) (sql.Node, error) { switch n := n.(type) { case *plan.ResolvedTable: switch n.Table.(type) { @@ -40,20 +40,29 @@ func trackProcess(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { } total = count } - processList.AddProgressItem(ctx.Pid(), name, total) + processList.AddTableProgress(ctx.Pid(), name, total) seen[name] = struct{}{} - notify := func() { - processList.UpdateProgress(ctx.Pid(), name, 1) + onPartitionDone := func(partitionName string) { + processList.UpdateTableProgress(ctx.Pid(), name, 1) + processList.RemovePartitionProgress(ctx.Pid(), name, partitionName) + } + + onPartitionStart := func(partitionName string) { + processList.AddPartitionProgress(ctx.Pid(), name, partitionName, -1) + } + + onRowNext := func(partitionName string) { + processList.UpdatePartitionProgress(ctx.Pid(), name, partitionName, 1) } var t sql.Table switch table := n.Table.(type) { case sql.IndexableTable: - t = plan.NewProcessIndexableTable(table, notify) + t = plan.NewProcessIndexableTable(table, onPartitionDone, onPartitionStart, onRowNext) default: - t = plan.NewProcessTable(table, notify) + t = plan.NewProcessTable(table, onPartitionDone, onPartitionStart, onRowNext) } return plan.NewResolvedTable(t), nil @@ -73,7 +82,7 @@ func trackProcess(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { // Remove QueryProcess nodes from the subqueries. Otherwise, the process // will be marked as done as soon as a subquery finishes. - node, err := n.TransformUp(func(n sql.Node) (sql.Node, error) { + node, err := plan.TransformUp(n, func(n sql.Node) (sql.Node, error) { if sq, ok := n.(*plan.SubqueryAlias); ok { if qp, ok := sq.Child.(*plan.QueryProcess); ok { return plan.NewSubqueryAlias(sq.Name(), qp.Child), nil @@ -85,5 +94,10 @@ func trackProcess(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { return nil, err } - return plan.NewQueryProcess(node, func() { processList.Done(ctx.Pid()) }), nil + return plan.NewQueryProcess(node, func() { + processList.Done(ctx.Pid()) + if span := ctx.RootSpan(); span != nil { + span.Finish() + } + }), nil } diff --git a/sql/analyzer/process_test.go b/sql/analyzer/process_test.go index 0c2007481..91271af4c 100644 --- a/sql/analyzer/process_test.go +++ b/sql/analyzer/process_test.go @@ -5,11 +5,11 @@ import ( "testing" "time" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" ) func TestTrackProcess(t *testing.T) { @@ -19,8 +19,8 @@ func TestTrackProcess(t *testing.T) { a := NewDefault(catalog) node := plan.NewInnerJoin( - plan.NewResolvedTable(&table{mem.NewPartitionedTable("foo", nil, 2)}), - plan.NewResolvedTable(mem.NewPartitionedTable("bar", nil, 4)), + plan.NewResolvedTable(&table{memory.NewPartitionedTable("foo", nil, 2)}), + plan.NewResolvedTable(memory.NewPartitionedTable("bar", nil, 4)), expression.NewLiteral(int64(1), sql.Int64), ) @@ -35,10 +35,16 @@ func TestTrackProcess(t *testing.T) { require.Len(processes, 1) require.Equal("SELECT foo", processes[0].Query) require.Equal(sql.QueryProcess, processes[0].Type) - require.Equal(map[string]sql.Progress{ - "foo": sql.Progress{Total: 2}, - "bar": sql.Progress{Total: 4}, - }, processes[0].Progress) + require.Equal( + map[string]sql.TableProgress{ + "foo": sql.TableProgress{ + Progress: sql.Progress{Name: "foo", Done: 0, Total: 2}, + PartitionsProgress: map[string]sql.PartitionProgress{}}, + "bar": sql.TableProgress{ + Progress: sql.Progress{Name: "bar", Done: 0, Total: 4}, + PartitionsProgress: map[string]sql.PartitionProgress{}}, + }, + processes[0].Progress) proc, ok := result.(*plan.QueryProcess) require.True(ok) @@ -80,7 +86,7 @@ func TestTrackProcessSubquery(t *testing.T) { nil, plan.NewSubqueryAlias("f", plan.NewQueryProcess( - plan.NewResolvedTable(mem.NewTable("foo", nil)), + plan.NewResolvedTable(memory.NewTable("foo", nil)), nil, ), ), @@ -92,7 +98,7 @@ func TestTrackProcessSubquery(t *testing.T) { expectedChild := plan.NewProject( nil, plan.NewSubqueryAlias("f", - plan.NewResolvedTable(mem.NewTable("foo", nil)), + plan.NewResolvedTable(memory.NewTable("foo", nil)), ), ) diff --git a/sql/analyzer/prune_columns.go b/sql/analyzer/prune_columns.go index 9274541f5..0b4121ebe 100644 --- a/sql/analyzer/prune_columns.go +++ b/sql/analyzer/prune_columns.go @@ -3,9 +3,9 @@ package analyzer import ( "fmt" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" ) type usedColumns map[string]map[string]struct{} @@ -16,6 +16,15 @@ func pruneColumns(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { return n, nil } + if describe, ok := n.(*plan.DescribeQuery); ok { + pruned, err := pruneColumns(ctx, a, describe.Child) + if err != nil { + return nil, err + } + + return plan.NewDescribeQuery(describe.Format, pruned), nil + } + columns := make(usedColumns) // All the columns required for the output of the query must be mark as @@ -122,7 +131,7 @@ func pruneSubqueries( n sql.Node, parentColumns usedColumns, ) (sql.Node, error) { - return n.TransformUp(func(n sql.Node) (sql.Node, error) { + return plan.TransformUp(n, func(n sql.Node) (sql.Node, error) { subq, ok := n.(*plan.SubqueryAlias) if !ok { return n, nil @@ -133,7 +142,7 @@ func pruneSubqueries( } func pruneUnusedColumns(n sql.Node, columns usedColumns) (sql.Node, error) { - return n.TransformUp(func(n sql.Node) (sql.Node, error) { + return plan.TransformUp(n, func(n sql.Node) (sql.Node, error) { switch n := n.(type) { case *plan.Project: return pruneProject(n, columns), nil @@ -145,13 +154,8 @@ func pruneUnusedColumns(n sql.Node, columns usedColumns) (sql.Node, error) { }) } -type tableColumnPair struct { - table string - column string -} - func fixRemainingFieldsIndexes(n sql.Node) (sql.Node, error) { - return n.TransformUp(func(n sql.Node) (sql.Node, error) { + return plan.TransformUp(n, func(n sql.Node) (sql.Node, error) { switch n := n.(type) { case *plan.SubqueryAlias: child, err := fixRemainingFieldsIndexes(n.Child) @@ -161,8 +165,7 @@ func fixRemainingFieldsIndexes(n sql.Node) (sql.Node, error) { return plan.NewSubqueryAlias(n.Name(), child), nil default: - exp, ok := n.(sql.Expressioner) - if !ok { + if _, ok := n.(sql.Expressioner); !ok { return n, nil } @@ -175,20 +178,20 @@ func fixRemainingFieldsIndexes(n sql.Node) (sql.Node, error) { return n, nil } - indexes := make(map[tableColumnPair]int) + indexes := make(map[tableCol]int) for i, col := range schema { - indexes[tableColumnPair{col.Source, col.Name}] = i + indexes[tableCol{col.Source, col.Name}] = i } - return exp.TransformExpressions(func(e sql.Expression) (sql.Expression, error) { + return plan.TransformExpressions(n, func(e sql.Expression) (sql.Expression, error) { gf, ok := e.(*expression.GetField) if !ok { return e, nil } - idx, ok := indexes[tableColumnPair{gf.Table(), gf.Name()}] + idx, ok := indexes[tableCol{gf.Table(), gf.Name()}] if !ok { - return nil, fmt.Errorf("unable to find column %q of table %q", gf.Name(), gf.Table()) + return nil, ErrColumnTableNotFound.New(gf.Table(), gf.Name()) } if idx == gf.Index() { diff --git a/sql/analyzer/prune_columns_test.go b/sql/analyzer/prune_columns_test.go index fb98bb2bd..e2a1c2abd 100644 --- a/sql/analyzer/prune_columns_test.go +++ b/sql/analyzer/prune_columns_test.go @@ -3,24 +3,24 @@ package analyzer import ( "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" ) func TestPruneColumns(t *testing.T) { rule := getRuleFrom(OnceAfterDefault, "prune_columns") a := NewDefault(nil) - t1 := plan.NewResolvedTable(mem.NewTable("t1", sql.Schema{ + t1 := plan.NewResolvedTable(memory.NewTable("t1", sql.Schema{ {Name: "foo", Type: sql.Int64, Source: "t1"}, {Name: "bar", Type: sql.Int64, Source: "t1"}, {Name: "bax", Type: sql.Int64, Source: "t1"}, })) - t2 := plan.NewResolvedTable(mem.NewTable("t2", sql.Schema{ + t2 := plan.NewResolvedTable(memory.NewTable("t2", sql.Schema{ {Name: "foo", Type: sql.Int64, Source: "t2"}, {Name: "baz", Type: sql.Int64, Source: "t2"}, {Name: "bux", Type: sql.Int64, Source: "t2"}, diff --git a/sql/analyzer/pushdown.go b/sql/analyzer/pushdown.go index 0f9c47cc7..07049c413 100644 --- a/sql/analyzer/pushdown.go +++ b/sql/analyzer/pushdown.go @@ -4,9 +4,9 @@ import ( "reflect" "sync" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" ) func pushdown(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { @@ -20,242 +20,294 @@ func pushdown(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { // don't do pushdown on certain queries switch n.(type) { - case *plan.InsertInto, *plan.CreateIndex: + case *plan.InsertInto, *plan.DeleteFrom, *plan.Update, *plan.CreateIndex: return n, nil } - var fieldsByTable = make(map[string][]string) - var exprsByTable = make(map[string][]sql.Expression) - type tableField struct { - table string - field string - } - var tableFields = make(map[tableField]struct{}) - a.Log("finding used columns in node") colSpan, _ := ctx.Span("find_pushdown_columns") // First step is to find all col exprs and group them by the table they mention. // Even if they appear multiple times, only the first one will be used. + fieldsByTable := findFieldsByTable(n) + + colSpan.Finish() + + a.Log("finding filters in node") + filters := findFilters(ctx, n) + + indexSpan, _ := ctx.Span("assign_indexes") + indexes, err := assignIndexes(a, n) + if err != nil { + return nil, err + } + indexSpan.Finish() + + a.Log("transforming nodes with pushdown of filters, projections and indexes") + + return transformPushdown(a, n, filters, indexes, fieldsByTable) +} + +// fixFieldIndexesOnExpressions executes fixFieldIndexes on a list of exprs. +func fixFieldIndexesOnExpressions(schema sql.Schema, expressions ...sql.Expression) ([]sql.Expression, error) { + var result = make([]sql.Expression, len(expressions)) + for i, e := range expressions { + var err error + result[i], err = fixFieldIndexes(schema, e) + if err != nil { + return nil, err + } + } + return result, nil +} + +// fixFieldIndexes transforms the given expression setting correct indexes +// for GetField expressions according to the schema of the row in the table +// and not the one where the filter came from. +func fixFieldIndexes(schema sql.Schema, exp sql.Expression) (sql.Expression, error) { + return expression.TransformUp(exp, func(e sql.Expression) (sql.Expression, error) { + switch e := e.(type) { + case *expression.GetField: + // we need to rewrite the indexes for the table row + for i, col := range schema { + if e.Name() == col.Name && e.Table() == col.Source { + return expression.NewGetFieldWithTable( + i, + e.Type(), + e.Table(), + e.Name(), + e.IsNullable(), + ), nil + } + } + + return nil, ErrFieldMissing.New(e.Name()) + } + + return e, nil + }) +} + +func findFieldsByTable(n sql.Node) map[string][]string { + var fieldsByTable = make(map[string][]string) plan.InspectExpressions(n, func(e sql.Expression) bool { - if e, ok := e.(*expression.GetField); ok { - tf := tableField{e.Table(), e.Name()} - if _, ok := tableFields[tf]; !ok { - a.Log("found used column %s.%s", e.Table(), e.Name()) - tableFields[tf] = struct{}{} - fieldsByTable[e.Table()] = append(fieldsByTable[e.Table()], e.Name()) - exprsByTable[e.Table()] = append(exprsByTable[e.Table()], e) + if gf, ok := e.(*expression.GetField); ok { + if !stringContains(fieldsByTable[gf.Table()], gf.Name()) { + fieldsByTable[gf.Table()] = append(fieldsByTable[gf.Table()], gf.Name()) } } return true }) + return fieldsByTable +} - colSpan.Finish() - - a.Log("finding filters in node") - - filterSpan, _ := ctx.Span("find_pushdown_filters") +func findFilters(ctx *sql.Context, n sql.Node) filters { + span, _ := ctx.Span("find_pushdown_filters") + defer span.Finish() - // then find all filters, also by table. Note that filters that mention + // Find all filters, also by table. Note that filters that mention // more than one table will not be passed to neither. filters := make(filters) plan.Inspect(n, func(node sql.Node) bool { - a.Log("inspecting node of type: %T", node) switch node := node.(type) { case *plan.Filter: fs := exprToTableFilters(node.Expression) - a.Log("found filters for %d tables %s", len(fs), node.Expression) filters.merge(fs) } return true }) - filterSpan.Finish() - - indexSpan, _ := ctx.Span("assign_indexes") - indexes, err := assignIndexes(a, n) - if err != nil { - return nil, err - } - indexSpan.Finish() - - a.Log("transforming nodes with pushdown of filters, projections and indexes") + return filters +} +func transformPushdown( + a *Analyzer, + n sql.Node, + filters filters, + indexes map[string]*indexLookup, + fieldsByTable map[string][]string, +) (sql.Node, error) { // Now all nodes can be transformed. Since traversal of the tree is done // from inner to outer the filters have to be processed first so they get // to the tables. var handledFilters []sql.Expression var queryIndexes []sql.Index - node, err := n.TransformUp(func(node sql.Node) (sql.Node, error) { + node, err := plan.TransformUp(n, func(node sql.Node) (sql.Node, error) { a.Log("transforming node of type: %T", node) switch node := node.(type) { case *plan.Filter: - if len(handledFilters) == 0 { - a.Log("no handled filters, leaving filter untouched") - return node, nil - } - - unhandled := getUnhandledFilters( - splitExpression(node.Expression), - handledFilters, + return pushdownFilter(a, node, handledFilters) + case *plan.ResolvedTable: + return pushdownTable( + a, + node, + filters, + &handledFilters, + &queryIndexes, + fieldsByTable, + indexes, ) + default: + return transformExpressioners(node) + } + }) - if len(unhandled) == 0 { - a.Log("filter node has no unhandled filters, so it will be removed") - return node.Child, nil - } + release := func() { + for _, idx := range queryIndexes { + a.Catalog.ReleaseIndex(idx) + } + } - a.Log( - "%d handled filters removed from filter node, filter has now %d filters", - len(handledFilters), - len(unhandled), - ) + if err != nil { + release() + return nil, err + } - return plan.NewFilter(expression.JoinAnd(unhandled...), node.Child), nil - case *plan.ResolvedTable: - var table = node.Table - - if ft, ok := table.(sql.FilteredTable); ok { - tableFilters := filters[node.Name()] - handled := ft.HandledFilters(tableFilters) - handledFilters = append(handledFilters, handled...) - schema := node.Schema() - handled, err = fixFieldIndexesOnExpressions(schema, handled...) - if err != nil { - return nil, err - } + if len(queryIndexes) > 0 { + return &releaser{node, release}, nil + } - table = ft.WithFilters(handled) - a.Log( - "table %q transformed with pushdown of filters, %d filters handled of %d", - node.Name(), - len(handled), - len(tableFilters), - ) - } + return node, nil +} - if pt, ok := table.(sql.ProjectedTable); ok { - table = pt.WithProjection(fieldsByTable[node.Name()]) - a.Log("table %q transformed with pushdown of projection", node.Name()) - } +func transformExpressioners(node sql.Node) (sql.Node, error) { + if _, ok := node.(sql.Expressioner); !ok { + return node, nil + } - if it, ok := table.(sql.IndexableTable); ok { - indexLookup, ok := indexes[node.Name()] - if ok { - queryIndexes = append(queryIndexes, indexLookup.indexes...) - table = it.WithIndexLookup(indexLookup.lookup) - a.Log("table %q transformed with pushdown of index", node.Name()) - } - } + var schemas []sql.Schema + for _, child := range node.Children() { + schemas = append(schemas, child.Schema()) + } - return plan.NewResolvedTable(table), nil - default: - expressioner, ok := node.(sql.Expressioner) - if !ok { - return node, nil - } + if len(schemas) < 1 { + return node, nil + } - var schemas []sql.Schema - for _, child := range node.Children() { - schemas = append(schemas, child.Schema()) + n, err := plan.TransformExpressions(node, func(e sql.Expression) (sql.Expression, error) { + for _, schema := range schemas { + fixed, err := fixFieldIndexes(schema, e) + if err == nil { + return fixed, nil } - if len(schemas) < 1 { - return node, nil + if ErrFieldMissing.Is(err) { + continue } - n, err := expressioner.TransformExpressions(func(e sql.Expression) (sql.Expression, error) { - for _, schema := range schemas { - fixed, err := fixFieldIndexes(schema, e) - if err == nil { - return fixed, nil - } + return nil, err + } - if ErrFieldMissing.Is(err) { - continue - } + return e, nil + }) - return nil, err - } + if err != nil { + return nil, err + } - return e, nil - }) + switch j := n.(type) { + case *plan.InnerJoin: + cond, err := fixFieldIndexes(j.Schema(), j.Cond) + if err != nil { + return nil, err + } - if err != nil { - return nil, err - } + n = plan.NewInnerJoin(j.Left, j.Right, cond) + case *plan.RightJoin: + cond, err := fixFieldIndexes(j.Schema(), j.Cond) + if err != nil { + return nil, err + } - if ij, ok := n.(*plan.InnerJoin); ok { - cond, err := fixFieldIndexes(ij.Schema(), ij.Cond) - if err != nil { - return nil, err - } + n = plan.NewRightJoin(j.Left, j.Right, cond) + case *plan.LeftJoin: + cond, err := fixFieldIndexes(j.Schema(), j.Cond) + if err != nil { + return nil, err + } - n = plan.NewInnerJoin(ij.Left, ij.Right, cond) - } + n = plan.NewLeftJoin(j.Left, j.Right, cond) + } - return n, nil - } - }) + return n, nil +} - release := func() { - for _, idx := range queryIndexes { - a.Catalog.ReleaseIndex(idx) +func pushdownTable( + a *Analyzer, + node *plan.ResolvedTable, + filters filters, + handledFilters *[]sql.Expression, + queryIndexes *[]sql.Index, + fieldsByTable map[string][]string, + indexes map[string]*indexLookup, +) (sql.Node, error) { + var table = node.Table + + if ft, ok := table.(sql.FilteredTable); ok { + tableFilters := filters[node.Name()] + handled := ft.HandledFilters(tableFilters) + *handledFilters = append(*handledFilters, handled...) + schema := node.Schema() + handled, err := fixFieldIndexesOnExpressions(schema, handled...) + if err != nil { + return nil, err } + + table = ft.WithFilters(handled) + a.Log( + "table %q transformed with pushdown of filters, %d filters handled of %d", + node.Name(), + len(handled), + len(tableFilters), + ) } - if err != nil { - release() - return nil, err + if pt, ok := table.(sql.ProjectedTable); ok { + table = pt.WithProjection(fieldsByTable[node.Name()]) + a.Log("table %q transformed with pushdown of projection", node.Name()) } - if len(queryIndexes) > 0 { - return &releaser{node, release}, nil + if it, ok := table.(sql.IndexableTable); ok { + indexLookup, ok := indexes[node.Name()] + if ok { + *queryIndexes = append(*queryIndexes, indexLookup.indexes...) + table = it.WithIndexLookup(indexLookup.lookup) + a.Log("table %q transformed with pushdown of index", node.Name()) + } } - return node, nil + return plan.NewResolvedTable(table), nil } -// fixFieldIndexesOnExpressions executes fixFieldIndexes on a list of exprs. -func fixFieldIndexesOnExpressions(schema sql.Schema, expressions ...sql.Expression) ([]sql.Expression, error) { - var result = make([]sql.Expression, len(expressions)) - for i, e := range expressions { - var err error - result[i], err = fixFieldIndexes(schema, e) - if err != nil { - return nil, err - } +func pushdownFilter( + a *Analyzer, + node *plan.Filter, + handledFilters []sql.Expression, +) (sql.Node, error) { + if len(handledFilters) == 0 { + a.Log("no handled filters, leaving filter untouched") + return node, nil } - return result, nil -} -// fixFieldIndexes transforms the given expression setting correct indexes -// for GetField expressions according to the schema of the row in the table -// and not the one where the filter came from. -func fixFieldIndexes(schema sql.Schema, exp sql.Expression) (sql.Expression, error) { - return exp.TransformUp(func(e sql.Expression) (sql.Expression, error) { - switch e := e.(type) { - case *expression.GetField: - // we need to rewrite the indexes for the table row - for i, col := range schema { - if e.Name() == col.Name && e.Table() == col.Source { - return expression.NewGetFieldWithTable( - i, - e.Type(), - e.Table(), - e.Name(), - e.IsNullable(), - ), nil - } - } + unhandled := getUnhandledFilters( + splitExpression(node.Expression), + handledFilters, + ) - return nil, ErrFieldMissing.New(e.Name()) - } + if len(unhandled) == 0 { + a.Log("filter node has no unhandled filters, so it will be removed") + return node.Child, nil + } - return e, nil - }) + a.Log( + "%d handled filters removed from filter node, filter has now %d filters", + len(handledFilters), + len(unhandled), + ) + + return plan.NewFilter(expression.JoinAnd(unhandled...), node.Child), nil } type releaser struct { @@ -263,8 +315,6 @@ type releaser struct { Release func() } -var _ sql.Node = (*releaser)(nil) - func (r *releaser) Resolved() bool { return r.Child.Resolved() } @@ -287,20 +337,11 @@ func (r *releaser) Schema() sql.Schema { return r.Child.Schema() } -func (r *releaser) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - child, err := r.Child.TransformUp(f) - if err != nil { - return nil, err +func (r *releaser) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(r, len(children), 1) } - return f(&releaser{child, r.Release}) -} - -func (r *releaser) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { - child, err := r.Child.TransformExpressionsUp(f) - if err != nil { - return nil, err - } - return &releaser{child, r.Release}, nil + return &releaser{children[0], r.Release}, nil } func (r *releaser) String() string { @@ -329,7 +370,10 @@ func (i *releaseIter) Next() (sql.Row, error) { return row, nil } -func (i *releaseIter) Close() error { +func (i *releaseIter) Close() (err error) { i.once.Do(i.release) - return nil + if i.child != nil { + err = i.child.Close() + } + return err } diff --git a/sql/analyzer/pushdown_test.go b/sql/analyzer/pushdown_test.go index ba2ffa85a..818eb40a0 100644 --- a/sql/analyzer/pushdown_test.go +++ b/sql/analyzer/pushdown_test.go @@ -3,30 +3,30 @@ package analyzer import ( "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" ) func TestPushdownProjectionAndFilters(t *testing.T) { require := require.New(t) f := getRule("pushdown") - table := mem.NewTable("mytable", sql.Schema{ + table := memory.NewTable("mytable", sql.Schema{ {Name: "i", Type: sql.Int32, Source: "mytable"}, {Name: "f", Type: sql.Float64, Source: "mytable"}, {Name: "t", Type: sql.Text, Source: "mytable"}, }) - table2 := mem.NewTable("mytable2", sql.Schema{ + table2 := memory.NewTable("mytable2", sql.Schema{ {Name: "i2", Type: sql.Int32, Source: "mytable2"}, {Name: "f2", Type: sql.Float64, Source: "mytable2"}, {Name: "t2", Type: sql.Text, Source: "mytable2"}, }) - db := mem.NewDatabase("mydb") + db := memory.NewDatabase("mydb") db.AddTable("mytable", table) db.AddTable("mytable2", table2) @@ -66,14 +66,14 @@ func TestPushdownProjectionAndFilters(t *testing.T) { expression.NewGetFieldWithTable(1, sql.Float64, "mytable", "f", false), expression.NewLiteral(3.14, sql.Float64), ), - }).(*mem.Table).WithProjection([]string{"i", "f"}), + }).(*memory.Table).WithProjection([]string{"i", "f"}), ), plan.NewResolvedTable( table2.WithFilters([]sql.Expression{ expression.NewIsNull( expression.NewGetFieldWithTable(0, sql.Int32, "mytable2", "i2", false), ), - }).(*mem.Table).WithProjection([]string{"i2"}), + }).(*memory.Table).WithProjection([]string{"i2"}), ), ), ) @@ -86,19 +86,19 @@ func TestPushdownProjectionAndFilters(t *testing.T) { func TestPushdownIndexable(t *testing.T) { require := require.New(t) - table := mem.NewTable("mytable", sql.Schema{ + table := memory.NewTable("mytable", sql.Schema{ {Name: "i", Type: sql.Int32, Source: "mytable"}, {Name: "f", Type: sql.Float64, Source: "mytable"}, {Name: "t", Type: sql.Text, Source: "mytable"}, }) - table2 := mem.NewTable("mytable2", sql.Schema{ + table2 := memory.NewTable("mytable2", sql.Schema{ {Name: "i2", Type: sql.Int32, Source: "mytable2"}, {Name: "f2", Type: sql.Float64, Source: "mytable2"}, {Name: "t2", Type: sql.Text, Source: "mytable2"}, }) - db := mem.NewDatabase("") + db := memory.NewDatabase("") db.AddTable("mytable", table) db.AddTable("mytable2", table2) @@ -187,8 +187,8 @@ func TestPushdownIndexable(t *testing.T) { expression.NewGetFieldWithTable(0, sql.Int32, "mytable", "i", false), expression.NewLiteral(1, sql.Int32), ), - }).(*mem.Table). - WithProjection([]string{"i", "f"}).(*mem.Table). + }).(*memory.Table). + WithProjection([]string{"i", "f"}).(*memory.Table). WithIndexLookup(&mergeableIndexLookup{id: "3.14"}), ), plan.NewResolvedTable( @@ -199,8 +199,8 @@ func TestPushdownIndexable(t *testing.T) { expression.NewLiteral(2, sql.Int32), ), ), - }).(*mem.Table). - WithProjection([]string{"i2"}).(*mem.Table). + }).(*memory.Table). + WithProjection([]string{"i2"}).(*memory.Table). WithIndexLookup(&negateIndexLookup{value: "2"}), ), ), @@ -212,7 +212,7 @@ func TestPushdownIndexable(t *testing.T) { require.NoError(err) // we need to remove the release function to compare, otherwise it will fail - result, err = result.TransformUp(func(node sql.Node) (sql.Node, error) { + result, err = plan.TransformUp(result, func(node sql.Node) (sql.Node, error) { switch node := node.(type) { case *releaser: return &releaser{Child: node.Child}, nil diff --git a/sql/analyzer/resolve_columns.go b/sql/analyzer/resolve_columns.go index 39be0aa19..79e976739 100644 --- a/sql/analyzer/resolve_columns.go +++ b/sql/analyzer/resolve_columns.go @@ -5,11 +5,12 @@ import ( "sort" "strings" - errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" - "gopkg.in/src-d/go-vitess.v1/vt/sqlparser" + "github.com/src-d/go-mysql-server/internal/similartext" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" + "gopkg.in/src-d/go-errors.v1" + "vitess.io/vitess/go/vt/sqlparser" ) func checkAliases(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { @@ -97,8 +98,25 @@ func (deferredColumn) IsNullable() bool { return true } -func (e deferredColumn) TransformUp(fn sql.TransformExprFunc) (sql.Expression, error) { - return fn(e) +// Children implements the Expression interface. +func (deferredColumn) Children() []sql.Expression { return nil } + +// WithChildren implements the Expression interface. +func (e deferredColumn) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(e, len(children), 0) + } + return e, nil +} + +type tableCol struct { + table string + col string +} + +type indexedCol struct { + *sql.Column + index int } // column is the common interface that groups UnresolvedColumn and deferredColumn. @@ -109,299 +127,267 @@ type column interface { } func qualifyColumns(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { - span, _ := ctx.Span("qualify_columns") - defer span.Finish() - - a.Log("qualify columns") - tables := make(map[string]sql.Node) - tableAliases := make(map[string]string) - colIndex := make(map[string][]string) - - indexCols := func(table string, schema sql.Schema) { - for _, col := range schema { - name := strings.ToLower(col.Name) - colIndex[name] = append(colIndex[name], strings.ToLower(table)) + return plan.TransformUp(n, func(n sql.Node) (sql.Node, error) { + if _, ok := n.(sql.Expressioner); !ok || n.Resolved() { + return n, nil } - } - var projects, seenProjects int - plan.Inspect(n, func(n sql.Node) bool { - if _, ok := n.(*plan.Project); ok { - projects++ - } - return true + columns := getNodeAvailableColumns(n) + tables := getNodeAvailableTables(n) + + return plan.TransformExpressions(n, func(e sql.Expression) (sql.Expression, error) { + return qualifyExpression(e, columns, tables) + }) }) +} - return n.TransformUp(func(n sql.Node) (sql.Node, error) { - a.Log("transforming node of type: %T", n) - switch n := n.(type) { - case *plan.TableAlias: - switch t := n.Child.(type) { - case *plan.ResolvedTable, *plan.UnresolvedTable: - name := strings.ToLower(t.(sql.Nameable).Name()) - tableAliases[strings.ToLower(n.Name())] = name - default: - tables[strings.ToLower(n.Name())] = n.Child - indexCols(n.Name(), n.Schema()) - } - case *plan.ResolvedTable, *plan.SubqueryAlias: - name := strings.ToLower(n.(sql.Nameable).Name()) - tables[name] = n - indexCols(name, n.Schema()) +func qualifyExpression( + e sql.Expression, + columns map[string][]string, + tables map[string]string, +) (sql.Expression, error) { + switch col := e.(type) { + case column: + // Skip this step for global and session variables + if isGlobalOrSessionColumn(col) { + return col, nil } - result, err := n.TransformExpressionsUp(func(e sql.Expression) (sql.Expression, error) { - a.Log("transforming expression of type: %T", e) - switch col := e.(type) { - case *expression.UnresolvedColumn: - // Skip this step for global and session variables - if isGlobalOrSessionColumn(col) { - return col, nil + name, table := strings.ToLower(col.Name()), strings.ToLower(col.Table()) + availableTables := dedupStrings(columns[name]) + if table != "" { + table, ok := tables[table] + if !ok { + if len(tables) == 0 { + return nil, sql.ErrTableNotFound.New(col.Table()) } - col = expression.NewUnresolvedQualifiedColumn(col.Table(), col.Name()) - - name := strings.ToLower(col.Name()) - table := strings.ToLower(col.Table()) - if table == "" { - tables := dedupStrings(colIndex[name]) - switch len(tables) { - case 0: - // If there are no tables that have any column with the column - // name let's just return it as it is. This may be an alias, so - // we'll wait for the reorder of the - return col, nil - case 1: - col = expression.NewUnresolvedQualifiedColumn( - tables[0], - col.Name(), - ) - default: - if _, ok := n.(*plan.GroupBy); ok { - return expression.NewUnresolvedColumn(col.Name()), nil - } - return nil, ErrAmbiguousColumnName.New(col.Name(), strings.Join(tables, ", ")) - } - } else { - if real, ok := tableAliases[table]; ok { - col = expression.NewUnresolvedQualifiedColumn( - real, - col.Name(), - ) - } - - if _, ok := tables[col.Table()]; !ok { - return nil, sql.ErrTableNotFound.New(col.Table()) - } - } + similar := similartext.FindFromMap(tables, col.Table()) + return nil, sql.ErrTableNotFound.New(col.Table() + similar) + } - a.Log("column %q was qualified with table %q", col.Name(), col.Table()) + // If the table exists but it's not available for this node it + // means some work is still needed, so just return the column + // and let it be resolved in the next pass. + if !stringContains(availableTables, table) { return col, nil - case *expression.Star: - if col.Table != "" { - if real, ok := tableAliases[strings.ToLower(col.Table)]; ok { - col = expression.NewQualifiedStar(real) - } + } - if _, ok := tables[strings.ToLower(col.Table)]; !ok { - return nil, sql.ErrTableNotFound.New(col.Table) - } + return expression.NewUnresolvedQualifiedColumn(table, col.Name()), nil + } - return col, nil - } - default: - // If any other kind of expression has a star, just replace it - // with an unqualified star because it cannot be expanded. - return e.TransformUp(func(e sql.Expression) (sql.Expression, error) { - if _, ok := e.(*expression.Star); ok { - return expression.NewStar(), nil - } - return e, nil - }) + switch len(availableTables) { + case 0: + // If there are no tables that have any column with the column + // name let's just return it as it is. This may be an alias, so + // we'll wait for the reorder of the projection. + return col, nil + case 1: + return expression.NewUnresolvedQualifiedColumn( + availableTables[0], + col.Name(), + ), nil + default: + return nil, ErrAmbiguousColumnName.New(col.Name(), strings.Join(availableTables, ", ")) + } + case *expression.Star: + if col.Table != "" { + if real, ok := tables[strings.ToLower(col.Table)]; ok { + col = expression.NewQualifiedStar(real) } + if _, ok := tables[strings.ToLower(col.Table)]; !ok { + return nil, sql.ErrTableNotFound.New(col.Table) + } + } + return col, nil + default: + // If any other kind of expression has a star, just replace it + // with an unqualified star because it cannot be expanded. + return expression.TransformUp(e, func(e sql.Expression) (sql.Expression, error) { + if _, ok := e.(*expression.Star); ok { + return expression.NewStar(), nil + } return e, nil }) + } +} - if err != nil { - return nil, err - } +func getNodeAvailableColumns(n sql.Node) map[string][]string { + var columns = make(map[string][]string) + getColumnsInNodes(n.Children(), columns) + return columns +} - // We should ignore the topmost project, because some nodes are - // reordered, such as Sort, and they would not be resolved well. - if n, ok := result.(*plan.Project); ok && projects-seenProjects > 1 { - seenProjects++ - - // We need to modify the indexed columns to only contain what is - // projected in this project. If the column is not qualified by any - // table, just keep the ones that are currently in the index. - // If it is, then just make those tables available for the column. - // If we don't do this, columns that are not projected will be - // available in this step and may cause false errors or unintended - // results. - var projected = make(map[string][]string) - for _, p := range n.Projections { - var table, col string - switch p := p.(type) { - case column: - table = p.Table() - col = p.Name() - default: - continue - } +func getColumnsInNodes(nodes []sql.Node, columns map[string][]string) { + indexCol := func(table, col string) { + col = strings.ToLower(col) + columns[col] = append(columns[col], strings.ToLower(table)) + } - col = strings.ToLower(col) - table = strings.ToLower(table) - if table != "" { - projected[col] = append(projected[col], table) - } else { - projected[col] = append(projected[col], colIndex[col]...) - } + indexExpressions := func(exprs []sql.Expression) { + for _, e := range exprs { + switch e := e.(type) { + case *expression.Alias: + indexCol("", e.Name()) + case *expression.GetField: + indexCol(e.Table(), e.Name()) + case *expression.UnresolvedColumn: + indexCol(e.Table(), e.Name()) } + } + } - colIndex = make(map[string][]string) - for col, tables := range projected { - colIndex[col] = dedupStrings(tables) + for _, node := range nodes { + switch n := node.(type) { + case *plan.ResolvedTable, *plan.SubqueryAlias: + for _, col := range n.Schema() { + indexCol(col.Source, col.Name) } + case *plan.Project: + indexExpressions(n.Projections) + case *plan.GroupBy: + indexExpressions(n.Aggregate) + default: + getColumnsInNodes(n.Children(), columns) } + } +} - return result, nil - }) +func getNodeAvailableTables(n sql.Node) map[string]string { + tables := make(map[string]string) + getNodesAvailableTables(tables, n.Children()...) + return tables +} + +func getNodesAvailableTables(tables map[string]string, nodes ...sql.Node) { + for _, n := range nodes { + switch n := n.(type) { + case *plan.SubqueryAlias, *plan.ResolvedTable: + name := strings.ToLower(n.(sql.Nameable).Name()) + tables[name] = name + case *plan.TableAlias: + switch t := n.Child.(type) { + case *plan.ResolvedTable, *plan.UnresolvedTable: + name := strings.ToLower(t.(sql.Nameable).Name()) + alias := strings.ToLower(n.Name()) + tables[alias] = name + // Also add the name of the table because you can refer to a + // table with either the alias or the name. + tables[name] = name + } + default: + getNodesAvailableTables(tables, n.Children()...) + } + } } var errGlobalVariablesNotSupported = errors.NewKind("can't resolve global variable, %s was requested") +const ( + sessionTable = "@@" + sqlparser.SessionStr + sessionPrefix = sqlparser.SessionStr + "." + globalPrefix = sqlparser.GlobalStr + "." +) + func resolveColumns(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { span, ctx := ctx.Span("resolve_columns") defer span.Finish() a.Log("resolve columns, node of type: %T", n) - return n.TransformUp(func(n sql.Node) (sql.Node, error) { + return plan.TransformUp(n, func(n sql.Node) (sql.Node, error) { a.Log("transforming node of type: %T", n) if n.Resolved() { return n, nil } - colMap := make(map[string][]*sql.Column) - for _, child := range n.Children() { - if !child.Resolved() { - return n, nil - } - - for _, col := range child.Schema() { - name := strings.ToLower(col.Name) - colMap[name] = append(colMap[name], col) - } - } - - expressioner, ok := n.(sql.Expressioner) - if !ok { + if _, ok := n.(sql.Expressioner); !ok { return n, nil } - // make sure all children are resolved before resolving a node + // We need to use the schema, so all children must be resolved. for _, c := range n.Children() { if !c.Resolved() { - a.Log("a children with type %T of node %T were not resolved, skipping", c, n) return n, nil } } - return expressioner.TransformExpressions(func(e sql.Expression) (sql.Expression, error) { + columns := findChildIndexedColumns(n) + return plan.TransformExpressions(n, func(e sql.Expression) (sql.Expression, error) { a.Log("transforming expression of type: %T", e) - if e.Resolved() { - return e, nil - } uc, ok := e.(column) - if !ok { + if !ok || e.Resolved() { return e, nil } - const ( - sessionTable = "@@" + sqlparser.SessionStr - sessionPrefix = sqlparser.SessionStr + "." - globalPrefix = sqlparser.GlobalStr + "." - ) - name := strings.ToLower(uc.Name()) - table := strings.ToLower(uc.Table()) - columns, ok := colMap[name] - if !ok { - switch uc := uc.(type) { - case *expression.UnresolvedColumn: - if isGlobalOrSessionColumn(uc) { - if table != "" && table != sessionTable { - return nil, errGlobalVariablesNotSupported.New(uc) - } - - name := strings.TrimLeft(uc.Name(), "@") - name = strings.TrimPrefix(strings.TrimPrefix(name, globalPrefix), sessionPrefix) - typ, value := ctx.Get(name) - return expression.NewGetSessionField(name, typ, value), nil - } - - a.Log("evaluation of column %q was deferred", uc.Name()) - return &deferredColumn{uc}, nil - - default: - if table != "" { - return nil, ErrColumnTableNotFound.New(uc.Table(), uc.Name()) - } - - return nil, ErrColumnNotFound.New(uc.Name()) - } + if isGlobalOrSessionColumn(uc) { + return resolveGlobalOrSessionColumn(ctx, uc) } - var col *sql.Column - var found bool - for _, c := range columns { - _, ok := n.(*plan.GroupBy) - if ok || (strings.ToLower(c.Source) == table) { - col = c - found = true - break - } - } + return resolveColumnExpression(ctx, uc, columns) + }) + }) +} - if !found { - if table != "" { - return nil, ErrColumnTableNotFound.New(uc.Table(), uc.Name()) - } +func findChildIndexedColumns(n sql.Node) map[tableCol]indexedCol { + var idx int + var columns = make(map[tableCol]indexedCol) + + for _, child := range n.Children() { + for _, col := range child.Schema() { + columns[tableCol{ + table: strings.ToLower(col.Source), + col: strings.ToLower(col.Name), + }] = indexedCol{col, idx} + idx++ + } + } - switch uc := uc.(type) { - case *expression.UnresolvedColumn: - return &deferredColumn{uc}, nil - default: - return nil, ErrColumnNotFound.New(uc.Name()) - } - } + return columns +} - var schema sql.Schema - // If expressioner and unary node we must take the - // child's schema to correctly select the indexes - // in the row is going to be evaluated in this node - if plan.IsUnary(n) { - schema = n.Children()[0].Schema() - } else { - schema = n.Schema() - } +func resolveGlobalOrSessionColumn(ctx *sql.Context, col column) (sql.Expression, error) { + if col.Table() != "" && strings.ToLower(col.Table()) != sessionTable { + return nil, errGlobalVariablesNotSupported.New(col) + } - idx := schema.IndexOf(col.Name, col.Source) - if idx < 0 { - return nil, ErrColumnNotFound.New(col.Name) + name := strings.TrimLeft(col.Name(), "@") + name = strings.TrimPrefix(strings.TrimPrefix(name, globalPrefix), sessionPrefix) + typ, value := ctx.Get(name) + return expression.NewGetSessionField(name, typ, value), nil +} + +func resolveColumnExpression( + ctx *sql.Context, + e column, + columns map[tableCol]indexedCol, +) (sql.Expression, error) { + name := strings.ToLower(e.Name()) + table := strings.ToLower(e.Table()) + col, ok := columns[tableCol{table, name}] + if !ok { + switch uc := e.(type) { + case *expression.UnresolvedColumn: + // Defer the resolution of the column to give the analyzer more + // time to resolve other parts so this can be resolved. + return &deferredColumn{uc}, nil + default: + if table != "" { + return nil, ErrColumnTableNotFound.New(e.Table(), e.Name()) } - a.Log("column resolved to %q.%q", col.Source, col.Name) + return nil, ErrColumnNotFound.New(e.Name()) + } + } - return expression.NewGetFieldWithTable( - idx, - col.Type, - col.Source, - col.Name, - col.Nullable, - ), nil - }) - }) + return expression.NewGetFieldWithTable( + col.index, + col.Type, + col.Source, + col.Name, + col.Nullable, + ), nil } // resolveGroupingColumns reorders the aggregation in a groupby so aliases @@ -413,7 +399,7 @@ func resolveGroupingColumns(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node return n, nil } - return n.TransformUp(func(n sql.Node) (sql.Node, error) { + return plan.TransformUp(n, func(n sql.Node) (sql.Node, error) { g, ok := n.(*plan.GroupBy) if n.Resolved() || !ok || len(g.Grouping) == 0 { return n, nil @@ -529,7 +515,7 @@ func resolveGroupingColumns(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node if len(renames) > 0 { for i, expr := range newAggregate { var err error - newAggregate[i], err = expr.TransformUp(func(e sql.Expression) (sql.Expression, error) { + newAggregate[i], err = expression.TransformUp(expr, func(e sql.Expression) (sql.Expression, error) { col, ok := e.(*expression.UnresolvedColumn) if ok { // We need to make sure we don't rename the reference to the @@ -578,6 +564,6 @@ func dedupStrings(in []string) []string { return result } -func isGlobalOrSessionColumn(col *expression.UnresolvedColumn) bool { +func isGlobalOrSessionColumn(col column) bool { return strings.HasPrefix(col.Name(), "@@") || strings.HasPrefix(col.Table(), "@@") } diff --git a/sql/analyzer/resolve_columns_test.go b/sql/analyzer/resolve_columns_test.go index afd180fd4..4b111e943 100644 --- a/sql/analyzer/resolve_columns_test.go +++ b/sql/analyzer/resolve_columns_test.go @@ -4,17 +4,17 @@ import ( "context" "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" ) func TestQualifyColumnsProject(t *testing.T) { require := require.New(t) - table := mem.NewTable("foo", sql.Schema{ + table := memory.NewTable("foo", sql.Schema{ {Name: "a", Type: sql.Text, Source: "foo"}, {Name: "b", Type: sql.Text, Source: "foo"}, }) @@ -56,7 +56,7 @@ func TestMisusedAlias(t *testing.T) { require := require.New(t) f := getRule("check_aliases") - table := mem.NewTable("mytable", sql.Schema{ + table := memory.NewTable("mytable", sql.Schema{ {Name: "i", Type: sql.Int32}, }) @@ -79,10 +79,10 @@ func TestQualifyColumns(t *testing.T) { require := require.New(t) f := getRule("qualify_columns") - table := mem.NewTable("mytable", sql.Schema{{Name: "i", Type: sql.Int32}}) - table2 := mem.NewTable("mytable2", sql.Schema{{Name: "i", Type: sql.Int32}}) - sessionTable := mem.NewTable("@@session", sql.Schema{{Name: "autocommit", Type: sql.Int64}}) - globalTable := mem.NewTable("@@global", sql.Schema{{Name: "max_allowed_packet", Type: sql.Int64}}) + table := memory.NewTable("mytable", sql.Schema{{Name: "i", Type: sql.Int32, Source: "mytable"}}) + table2 := memory.NewTable("mytable2", sql.Schema{{Name: "i", Type: sql.Int32, Source: "mytable2"}}) + sessionTable := memory.NewTable("@@session", sql.Schema{{Name: "autocommit", Type: sql.Int64, Source: "@@session"}}) + globalTable := memory.NewTable("@@global", sql.Schema{{Name: "max_allowed_packet", Type: sql.Int64, Source: "@@global"}}) node := plan.NewProject( []sql.Expression{ @@ -250,7 +250,7 @@ func TestQualifyColumnsQualifiedStar(t *testing.T) { require := require.New(t) f := getRule("qualify_columns") - table := mem.NewTable("mytable", sql.Schema{{Name: "i", Type: sql.Int32}}) + table := memory.NewTable("mytable", sql.Schema{{Name: "i", Type: sql.Int32}}) node := plan.NewProject( []sql.Expression{ @@ -334,7 +334,7 @@ func TestResolveGroupingColumns(t *testing.T) { expression.NewUnresolvedColumn("a"), expression.NewUnresolvedColumn("b"), }, - plan.NewResolvedTable(mem.NewTable("table", nil)), + plan.NewResolvedTable(memory.NewTable("table", nil)), ) expected := plan.NewGroupBy( @@ -367,7 +367,7 @@ func TestResolveGroupingColumns(t *testing.T) { ), expression.NewUnresolvedColumn("c"), }, - plan.NewResolvedTable(mem.NewTable("table", nil)), + plan.NewResolvedTable(memory.NewTable("table", nil)), ), ) diff --git a/sql/analyzer/resolve_database.go b/sql/analyzer/resolve_database.go index c5037597b..2c5d0f628 100644 --- a/sql/analyzer/resolve_database.go +++ b/sql/analyzer/resolve_database.go @@ -1,7 +1,8 @@ package analyzer import ( - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/plan" ) func resolveDatabase(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { @@ -10,7 +11,7 @@ func resolveDatabase(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error a.Log("resolve database, node of type: %T", n) - return n.TransformUp(func(n sql.Node) (sql.Node, error) { + return plan.TransformUp(n, func(n sql.Node) (sql.Node, error) { d, ok := n.(sql.Databaser) if !ok { return n, nil diff --git a/sql/analyzer/resolve_functions.go b/sql/analyzer/resolve_functions.go index ec034a76d..34bac11b0 100644 --- a/sql/analyzer/resolve_functions.go +++ b/sql/analyzer/resolve_functions.go @@ -1,8 +1,9 @@ package analyzer import ( - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" ) func resolveFunctions(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { @@ -10,13 +11,13 @@ func resolveFunctions(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, erro defer span.Finish() a.Log("resolve functions, node of type %T", n) - return n.TransformUp(func(n sql.Node) (sql.Node, error) { + return plan.TransformUp(n, func(n sql.Node) (sql.Node, error) { a.Log("transforming node of type: %T", n) if n.Resolved() { return n, nil } - return n.TransformExpressionsUp(func(e sql.Expression) (sql.Expression, error) { + return plan.TransformExpressionsUp(n, func(e sql.Expression) (sql.Expression, error) { a.Log("transforming expression of type: %T", e) if e.Resolved() { return e, nil diff --git a/sql/analyzer/resolve_generators.go b/sql/analyzer/resolve_generators.go new file mode 100644 index 000000000..4635e24d6 --- /dev/null +++ b/sql/analyzer/resolve_generators.go @@ -0,0 +1,97 @@ +package analyzer + +import ( + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/expression/function" + "github.com/src-d/go-mysql-server/sql/plan" + "gopkg.in/src-d/go-errors.v1" +) + +var ( + errMultipleGenerators = errors.NewKind("there can't be more than 1 instance of EXPLODE in a SELECT") + errExplodeNotArray = errors.NewKind("argument of type %q given to EXPLODE, expecting array") +) + +func resolveGenerators(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { + return plan.TransformUp(n, func(n sql.Node) (sql.Node, error) { + p, ok := n.(*plan.Project) + if !ok { + return n, nil + } + + projection := p.Projections + + g, err := findGenerator(projection) + if err != nil { + return nil, err + } + + // There might be no generator in the project, in that case we don't + // have to do anything. + if g == nil { + return n, nil + } + + projection[g.idx] = g.expr + + var name string + if n, ok := g.expr.(sql.Nameable); ok { + name = n.Name() + } else { + name = g.expr.String() + } + + return plan.NewGenerate( + plan.NewProject(projection, p.Child), + expression.NewGetField(g.idx, g.expr.Type(), name, g.expr.IsNullable()), + ), nil + }) +} + +type generator struct { + idx int + expr sql.Expression +} + +// findGenerator will find in the given projection a generator column. If there +// is no generator, it will return nil. +// If there are is than one generator or the argument to explode is not an +// array it will fail. +// All occurrences of Explode will be replaced with Generate. +func findGenerator(exprs []sql.Expression) (*generator, error) { + var g = &generator{idx: -1} + for i, e := range exprs { + var found bool + switch e := e.(type) { + case *function.Explode: + found = true + g.expr = function.NewGenerate(e.Child) + case *expression.Alias: + if exp, ok := e.Child.(*function.Explode); ok { + found = true + g.expr = expression.NewAlias( + function.NewGenerate(exp.Child), + e.Name(), + ) + } + } + + if found { + if g.idx >= 0 { + return nil, errMultipleGenerators.New() + } + g.idx = i + + if !sql.IsArray(g.expr.Type()) { + return nil, errExplodeNotArray.New(g.expr.Type()) + } + } + } + + if g.expr == nil { + return nil, nil + } + + return g, nil +} diff --git a/sql/analyzer/resolve_generators_test.go b/sql/analyzer/resolve_generators_test.go new file mode 100644 index 000000000..8a7106df8 --- /dev/null +++ b/sql/analyzer/resolve_generators_test.go @@ -0,0 +1,117 @@ +package analyzer + +import ( + "testing" + + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/expression/function" + "github.com/src-d/go-mysql-server/sql/plan" + "github.com/stretchr/testify/require" + "gopkg.in/src-d/go-errors.v1" +) + +func TestResolveGenerators(t *testing.T) { + testCases := []struct { + name string + node sql.Node + expected sql.Node + err *errors.Kind + }{ + { + name: "regular explode", + node: plan.NewProject( + []sql.Expression{ + expression.NewGetField(0, sql.Int64, "a", false), + function.NewExplode(expression.NewGetField(1, sql.Array(sql.Int64), "b", false)), + expression.NewGetField(2, sql.Int64, "c", false), + }, + plan.NewUnresolvedTable("foo", ""), + ), + expected: plan.NewGenerate( + plan.NewProject( + []sql.Expression{ + expression.NewGetField(0, sql.Int64, "a", false), + function.NewGenerate(expression.NewGetField(1, sql.Array(sql.Int64), "b", false)), + expression.NewGetField(2, sql.Int64, "c", false), + }, + plan.NewUnresolvedTable("foo", ""), + ), + expression.NewGetField(1, sql.Array(sql.Int64), "EXPLODE(b)", false), + ), + err: nil, + }, + { + name: "explode with alias", + node: plan.NewProject( + []sql.Expression{ + expression.NewGetField(0, sql.Int64, "a", false), + expression.NewAlias( + function.NewExplode( + expression.NewGetField(1, sql.Array(sql.Int64), "b", false), + ), + "x", + ), + expression.NewGetField(2, sql.Int64, "c", false), + }, + plan.NewUnresolvedTable("foo", ""), + ), + expected: plan.NewGenerate( + plan.NewProject( + []sql.Expression{ + expression.NewGetField(0, sql.Int64, "a", false), + expression.NewAlias( + function.NewGenerate( + expression.NewGetField(1, sql.Array(sql.Int64), "b", false), + ), + "x", + ), + expression.NewGetField(2, sql.Int64, "c", false), + }, + plan.NewUnresolvedTable("foo", ""), + ), + expression.NewGetField(1, sql.Array(sql.Int64), "x", false), + ), + err: nil, + }, + { + name: "non array type on explode", + node: plan.NewProject( + []sql.Expression{ + expression.NewGetField(0, sql.Int64, "a", false), + function.NewExplode(expression.NewGetField(1, sql.Int64, "b", false)), + }, + plan.NewUnresolvedTable("foo", ""), + ), + expected: nil, + err: errExplodeNotArray, + }, + { + name: "more than one generator", + node: plan.NewProject( + []sql.Expression{ + expression.NewGetField(0, sql.Int64, "a", false), + function.NewExplode(expression.NewGetField(1, sql.Array(sql.Int64), "b", false)), + function.NewExplode(expression.NewGetField(2, sql.Array(sql.Int64), "c", false)), + }, + plan.NewUnresolvedTable("foo", ""), + ), + expected: nil, + err: errMultipleGenerators, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + require := require.New(t) + result, err := resolveGenerators(sql.NewEmptyContext(), nil, tt.node) + if tt.err != nil { + require.Error(err) + require.True(tt.err.Is(err)) + } else { + require.NoError(err) + require.Equal(tt.expected, result) + } + }) + } +} diff --git a/sql/analyzer/resolve_having.go b/sql/analyzer/resolve_having.go new file mode 100644 index 000000000..fdac4f67c --- /dev/null +++ b/sql/analyzer/resolve_having.go @@ -0,0 +1,502 @@ +package analyzer + +import ( + "reflect" + "strings" + + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/expression/function/aggregation" + "github.com/src-d/go-mysql-server/sql/plan" + "gopkg.in/src-d/go-errors.v1" +) + +func resolveHaving(ctx *sql.Context, a *Analyzer, node sql.Node) (sql.Node, error) { + return plan.TransformUp(node, func(node sql.Node) (sql.Node, error) { + having, ok := node.(*plan.Having) + if !ok { + return node, nil + } + + if !having.Child.Resolved() { + return node, nil + } + + originalSchema := having.Schema() + + var requiresProjection bool + if containsAggregation(having.Cond) { + var err error + having, requiresProjection, err = replaceAggregations(having) + if err != nil { + return nil, err + } + } + + missingCols := findMissingColumns(having, having.Cond) + // If all the columns required by the having are available, do nothing about it. + if len(missingCols) > 0 { + var err error + having, err = pushMissingColumnsUp(having, missingCols) + if err != nil { + return nil, err + } + requiresProjection = true + } + + if !requiresProjection { + return having, nil + } + + return projectOriginalAggregation(having, originalSchema), nil + }) +} + +func findMissingColumns(node sql.Node, expr sql.Expression) []string { + var schemaCols []string + for _, col := range node.Schema() { + schemaCols = append(schemaCols, strings.ToLower(col.Name)) + } + + var missingCols []string + for _, n := range findExprNameables(expr) { + name := strings.ToLower(n.Name()) + if !stringContains(schemaCols, name) { + missingCols = append(missingCols, n.Name()) + } + } + + return missingCols +} + +func projectOriginalAggregation(having *plan.Having, schema sql.Schema) *plan.Project { + var projection []sql.Expression + for i, col := range schema { + projection = append( + projection, + expression.NewGetFieldWithTable(i, col.Type, col.Source, col.Name, col.Nullable), + ) + } + + return plan.NewProject(projection, having) +} + +var errHavingChildMissingRef = errors.NewKind("cannot find column %s referenced in HAVING clause in either GROUP BY or its child") + +func pushMissingColumnsUp( + having *plan.Having, + missingCols []string, +) (*plan.Having, error) { + groupBy, err := findGroupBy(having) + if err != nil { + return nil, err + } + + schema := groupBy.Child.Schema() + var newAggregate []sql.Expression + for _, c := range missingCols { + idx := -1 + for i, col := range schema { + if strings.ToLower(c) == strings.ToLower(col.Name) { + idx = i + break + } + } + if idx < 0 { + return nil, errHavingChildMissingRef.New(c) + } + col := schema[idx] + newAggregate = append( + newAggregate, + expression.NewGetFieldWithTable(idx, col.Type, col.Source, col.Name, col.Nullable), + ) + } + + node, err := addColumnsToGroupBy(having, newAggregate) + if err != nil { + return nil, err + } + + return node.(*plan.Having), nil +} + +func findGroupBy(n sql.Node) (*plan.GroupBy, error) { + children := n.Children() + if len(children) != 1 { + return nil, errHavingNeedsGroupBy.New() + } + + if g, ok := children[0].(*plan.GroupBy); ok { + return g, nil + } + + return findGroupBy(children[0]) +} + +func addColumnsToGroupBy(node sql.Node, columns []sql.Expression) (sql.Node, error) { + switch node := node.(type) { + case *plan.Project: + child, err := addColumnsToGroupBy(node.Child, columns) + if err != nil { + return nil, err + } + + var newProjections = make([]sql.Expression, len(columns)) + for i, col := range columns { + var name = col.String() + var table string + if n, ok := col.(sql.Nameable); ok { + name = n.Name() + } + + if t, ok := col.(sql.Tableable); ok { + table = t.Table() + } + + newProjections[i] = expression.NewGetFieldWithTable( + len(child.Schema())-len(columns)+i, + col.Type(), + table, + name, + col.IsNullable(), + ) + } + + return plan.NewProject(append(node.Projections, newProjections...), child), nil + case *plan.Filter, + *plan.Sort, + *plan.Limit, + *plan.Offset, + *plan.Distinct, + *plan.Having: + child, err := addColumnsToGroupBy(node.Children()[0], columns) + if err != nil { + return nil, err + } + return node.WithChildren(child) + case *plan.GroupBy: + return plan.NewGroupBy(append(node.Aggregate, columns...), node.Grouping, node.Child), nil + default: + return nil, errHavingNeedsGroupBy.New() + } +} + +// pushColumnsUp pushes up the group by columns with the given indexes. +// It returns the resultant node, the indexes of those pushed up columns in the +// resultant node and an error, if any. +func pushColumnsUp(node sql.Node, columns []int) (sql.Node, []int, error) { + switch node := node.(type) { + case *plan.Project: + child, columns, err := pushColumnsUp(node.Child, columns) + if err != nil { + return nil, nil, err + } + + var seen = make(map[int]int) + for i, col := range node.Projections { + switch col := col.(type) { + case *expression.Alias: + if f, ok := col.Child.(*expression.GetField); ok { + seen[f.Index()] = i + } + case *expression.GetField: + seen[col.Index()] = i + } + } + + var newProjections = make([]sql.Expression, len(node.Projections)) + copy(newProjections, node.Projections) + schema := child.Schema() + var newColumns []int + + for _, idx := range columns { + if newIdx, ok := seen[idx]; ok { + newColumns = append(newColumns, newIdx) + continue + } + + col := schema[idx] + newIdx := len(newProjections) + newProjections = append(newProjections, expression.NewGetFieldWithTable( + newIdx, + col.Type, + col.Source, + col.Name, + col.Nullable, + )) + newColumns = append(newColumns, newIdx) + } + + return plan.NewProject(newProjections, child), newColumns, nil + case *plan.Filter: + child, columns, err := pushColumnsUp(node.Child, columns) + if err != nil { + return nil, nil, err + } + return plan.NewFilter(node.Expression, child), columns, nil + case *plan.Sort: + child, columns, err := pushColumnsUp(node.Child, columns) + if err != nil { + return nil, nil, err + } + return plan.NewSort(node.SortFields, child), columns, nil + case *plan.Limit: + child, columns, err := pushColumnsUp(node.Child, columns) + if err != nil { + return nil, nil, err + } + return plan.NewLimit(node.Limit, child), columns, nil + case *plan.Offset: + child, columns, err := pushColumnsUp(node.Child, columns) + if err != nil { + return nil, nil, err + } + return plan.NewOffset(node.Offset, child), columns, nil + case *plan.Distinct: + child, columns, err := pushColumnsUp(node.Child, columns) + if err != nil { + return nil, nil, err + } + return plan.NewDistinct(child), columns, nil + case *plan.GroupBy: + return node, columns, nil + case *plan.Having: + child, columns, err := pushColumnsUp(node.Child, columns) + if err != nil { + return nil, nil, err + } + return plan.NewHaving(node.Cond, child), columns, nil + default: + return nil, nil, errHavingNeedsGroupBy.New() + } +} + +func replaceAggregations(having *plan.Having) (*plan.Having, bool, error) { + groupBy, err := findGroupBy(having) + if err != nil { + return nil, false, err + } + + var newAggregate []sql.Expression + + var pushUp []int + var tokenToIdx = make(map[int]int) + var pushUpToken = -1 + + // We need to find all aggregations inside the having condition. The ones + // that are already present in the group by will be pushed up and the ones + // that are not, will be added to the group by and pushed up. + // + // To push up already existing aggregations we need to change all possible + // projections between the having and the group by, so we will need to + // assign some fake token indexes to replace later with the actual column + // indexes after they have been pushed up. This is because some of these + // may have already been projected in some projection and we cannot ensure + // from here what the final index will be. + cond, err := expression.TransformUp(having.Cond, func(e sql.Expression) (sql.Expression, error) { + agg, ok := e.(sql.Aggregation) + if !ok { + return e, nil + } + + for i, expr := range groupBy.Aggregate { + if aggregationEquals(agg, expr) { + token := pushUpToken + pushUpToken-- + pushUp = append(pushUp, i) + tokenToIdx[token] = len(pushUp) - 1 + return expression.NewGetField( + token, + expr.Type(), + expr.String(), + expr.IsNullable(), + ), nil + } + } + + newAggregate = append(newAggregate, agg) + return expression.NewGetField( + len(having.Child.Schema())+len(newAggregate)-1, + agg.Type(), + agg.String(), + agg.IsNullable(), + ), nil + }) + if err != nil { + return nil, false, err + } + + // The new aggregations will be added to the group by and pushed up until + // the topmost node. + having = plan.NewHaving(cond, having.Child) + node, err := addColumnsToGroupBy(having, newAggregate) + if err != nil { + return nil, false, err + } + + // Then, the ones that already existed are pushed up and we get the final + // indexes at the topmost node (the having) in the same order. + node, pushedUpColumns, err := pushColumnsUp(node, pushUp) + if err != nil { + return nil, false, err + } + + newSchema := node.Schema() + requiresProjection := len(newSchema) != len(having.Schema()) + having = node.(*plan.Having) + + // Now, the tokens are replaced with the actual columns, now that we know + // what the indexes are. + cond, err = expression.TransformUp(having.Cond, func(e sql.Expression) (sql.Expression, error) { + f, ok := e.(*expression.GetField) + if !ok { + return e, nil + } + + idx, ok := tokenToIdx[f.Index()] + if !ok { + return e, nil + } + + idx = pushedUpColumns[idx] + col := newSchema[idx] + return expression.NewGetFieldWithTable(idx, col.Type, col.Source, col.Name, col.Nullable), nil + }) + if err != nil { + return nil, false, err + } + + return plan.NewHaving(cond, having.Child), requiresProjection, nil +} + +func aggregationEquals(a, b sql.Expression) bool { + // First unwrap aliases + if alias, ok := b.(*expression.Alias); ok { + b = alias.Child + } else if alias, ok := a.(*expression.Alias); ok { + a = alias.Child + } + + switch a := a.(type) { + case *aggregation.Count: + // it doesn't matter what's inside a Count, the result will be + // the same. + _, ok := b.(*aggregation.Count) + return ok + case *aggregation.CountDistinct: + // it doesn't matter what's inside a Count, the result will be + // the same. + _, ok := b.(*aggregation.CountDistinct) + return ok + case *aggregation.Sum: + b, ok := b.(*aggregation.Sum) + if !ok { + return false + } + + return aggregationChildEquals(a.Child, b.Child) + case *aggregation.Avg: + b, ok := b.(*aggregation.Avg) + if !ok { + return false + } + + return aggregationChildEquals(a.Child, b.Child) + case *aggregation.Min: + b, ok := b.(*aggregation.Min) + if !ok { + return false + } + + return aggregationChildEquals(a.Child, b.Child) + case *aggregation.Max: + b, ok := b.(*aggregation.Max) + if !ok { + return false + } + + return aggregationChildEquals(a.Child, b.Child) + case *aggregation.First: + b, ok := b.(*aggregation.First) + if !ok { + return false + } + + return aggregationChildEquals(a.Child, b.Child) + case *aggregation.Last: + b, ok := b.(*aggregation.Last) + if !ok { + return false + } + + return aggregationChildEquals(a.Child, b.Child) + default: + return false + } +} + +// aggregationChildEquals checks if expression a coming from the having +// matches expression b coming from the group by. To do that, columns in +// a need to be replaced to match the ones in b if their name or table and +// name match. +func aggregationChildEquals(a, b sql.Expression) bool { + var fieldsByName = make(map[string]sql.Expression) + var fieldsByTableCol = make(map[tableCol]sql.Expression) + expression.Inspect(b, func(e sql.Expression) bool { + gf, ok := e.(*expression.GetField) + if ok { + fieldsByTableCol[tableCol{ + strings.ToLower(gf.Table()), + strings.ToLower(gf.Name()), + }] = e + fieldsByName[strings.ToLower(gf.Name())] = e + } + return true + }) + + a, err := expression.TransformUp(a, func(e sql.Expression) (sql.Expression, error) { + var table, name string + switch e := e.(type) { + case *expression.UnresolvedColumn: + table = strings.ToLower(e.Table()) + name = strings.ToLower(e.Name()) + case *expression.GetField: + table = strings.ToLower(e.Table()) + name = strings.ToLower(e.Name()) + } + + if table == "" { + f, ok := fieldsByName[name] + if !ok { + return e, nil + } + return f, nil + } + + f, ok := fieldsByTableCol[tableCol{table, name}] + if !ok { + return e, nil + } + return f, nil + }) + if err != nil { + return false + } + + return reflect.DeepEqual(a, b) +} + +var errHavingNeedsGroupBy = errors.NewKind("found HAVING clause with no GROUP BY") + +func hasAggregations(expr sql.Expression) bool { + var has bool + expression.Inspect(expr, func(e sql.Expression) bool { + _, ok := e.(sql.Aggregation) + if ok { + has = true + return false + } + return true + }) + return has +} diff --git a/sql/analyzer/resolve_having_test.go b/sql/analyzer/resolve_having_test.go new file mode 100644 index 000000000..cc3707671 --- /dev/null +++ b/sql/analyzer/resolve_having_test.go @@ -0,0 +1,311 @@ +package analyzer + +import ( + "testing" + + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/expression/function/aggregation" + "github.com/src-d/go-mysql-server/sql/plan" + "github.com/stretchr/testify/require" + "gopkg.in/src-d/go-errors.v1" +) + +func TestResolveHaving(t *testing.T) { + testCases := []struct { + name string + input sql.Node + expected sql.Node + err *errors.Kind + }{ + { + "replace existing aggregation in group by", + plan.NewHaving( + expression.NewGreaterThan( + aggregation.NewAvg(expression.NewUnresolvedColumn("foo")), + expression.NewLiteral(int64(5), sql.Int64), + ), + plan.NewGroupBy( + []sql.Expression{ + expression.NewAlias(aggregation.NewAvg(expression.NewGetFieldWithTable(0, sql.Int64, "t", "foo", false)), "x"), + expression.NewGetField(0, sql.Int64, "foo", false), + }, + []sql.Expression{expression.NewGetField(0, sql.Int64, "foo", false)}, + plan.NewResolvedTable(memory.NewTable("t", nil)), + ), + ), + plan.NewHaving( + expression.NewGreaterThan( + expression.NewGetField(0, sql.Float64, "x", true), + expression.NewLiteral(int64(5), sql.Int64), + ), + plan.NewGroupBy( + []sql.Expression{ + expression.NewAlias(aggregation.NewAvg(expression.NewGetFieldWithTable(0, sql.Int64, "t", "foo", false)), "x"), + expression.NewGetField(0, sql.Int64, "foo", false), + }, + []sql.Expression{expression.NewGetField(0, sql.Int64, "foo", false)}, + plan.NewResolvedTable(memory.NewTable("t", nil)), + ), + ), + nil, + }, + { + "push down aggregation to group by", + plan.NewHaving( + expression.NewGreaterThan( + aggregation.NewCount(expression.NewStar()), + expression.NewLiteral(int64(5), sql.Int64), + ), + plan.NewGroupBy( + []sql.Expression{ + expression.NewAlias(aggregation.NewAvg(expression.NewGetField(0, sql.Int64, "foo", false)), "x"), + expression.NewGetFieldWithTable(0, sql.Int64, "t", "foo", false), + }, + []sql.Expression{expression.NewGetField(0, sql.Int64, "foo", false)}, + plan.NewResolvedTable(memory.NewTable("t", nil)), + ), + ), + plan.NewProject( + []sql.Expression{ + expression.NewGetField(0, sql.Float64, "x", true), + expression.NewGetFieldWithTable(1, sql.Int64, "t", "foo", false), + }, + plan.NewHaving( + expression.NewGreaterThan( + expression.NewGetField(2, sql.Int64, "COUNT(*)", false), + expression.NewLiteral(int64(5), sql.Int64), + ), + plan.NewGroupBy( + []sql.Expression{ + expression.NewAlias(aggregation.NewAvg(expression.NewGetField(0, sql.Int64, "foo", false)), "x"), + expression.NewGetFieldWithTable(0, sql.Int64, "t", "foo", false), + aggregation.NewCount(expression.NewStar()), + }, + []sql.Expression{expression.NewGetField(0, sql.Int64, "foo", false)}, + plan.NewResolvedTable(memory.NewTable("t", nil)), + ), + ), + ), + nil, + }, + { + "push up missing column", + plan.NewHaving( + expression.NewGreaterThan( + expression.NewUnresolvedColumn("i"), + expression.NewLiteral(int64(5), sql.Int64), + ), + plan.NewGroupBy( + []sql.Expression{ + expression.NewGetFieldWithTable(1, sql.Int64, "t", "foo", false), + }, + []sql.Expression{expression.NewGetFieldWithTable(1, sql.Int64, "t", "foo", false)}, + plan.NewResolvedTable(memory.NewTable("t", sql.Schema{ + {Type: sql.Int64, Name: "i", Source: "t"}, + {Type: sql.Int64, Name: "i", Source: "foo"}, + })), + ), + ), + plan.NewProject( + []sql.Expression{ + expression.NewGetFieldWithTable(0, sql.Int64, "t", "foo", false), + }, + plan.NewHaving( + expression.NewGreaterThan( + expression.NewUnresolvedColumn("i"), + expression.NewLiteral(int64(5), sql.Int64), + ), + plan.NewGroupBy( + []sql.Expression{ + expression.NewGetFieldWithTable(1, sql.Int64, "t", "foo", false), + expression.NewGetFieldWithTable(0, sql.Int64, "t", "i", false), + }, + []sql.Expression{expression.NewGetFieldWithTable(1, sql.Int64, "t", "foo", false)}, + plan.NewResolvedTable(memory.NewTable("t", sql.Schema{ + {Type: sql.Int64, Name: "i", Source: "t"}, + {Type: sql.Int64, Name: "i", Source: "foo"}, + })), + ), + ), + ), + nil, + }, + { + "push up missing column with nodes in between", + plan.NewHaving( + expression.NewGreaterThan( + expression.NewUnresolvedColumn("i"), + expression.NewLiteral(int64(5), sql.Int64), + ), + plan.NewProject( + []sql.Expression{ + expression.NewGetFieldWithTable(0, sql.Int64, "t", "foo", false), + }, + plan.NewGroupBy( + []sql.Expression{ + expression.NewGetFieldWithTable(1, sql.Int64, "t", "foo", false), + }, + []sql.Expression{expression.NewGetFieldWithTable(1, sql.Int64, "t", "foo", false)}, + plan.NewResolvedTable(memory.NewTable("t", sql.Schema{ + {Type: sql.Int64, Name: "i", Source: "t"}, + {Type: sql.Int64, Name: "i", Source: "foo"}, + })), + ), + ), + ), + plan.NewProject( + []sql.Expression{ + expression.NewGetFieldWithTable(0, sql.Int64, "t", "foo", false), + }, + plan.NewHaving( + expression.NewGreaterThan( + expression.NewUnresolvedColumn("i"), + expression.NewLiteral(int64(5), sql.Int64), + ), + plan.NewProject( + []sql.Expression{ + expression.NewGetFieldWithTable(0, sql.Int64, "t", "foo", false), + expression.NewGetFieldWithTable(1, sql.Int64, "t", "i", false), + }, + plan.NewGroupBy( + []sql.Expression{ + expression.NewGetFieldWithTable(1, sql.Int64, "t", "foo", false), + expression.NewGetFieldWithTable(0, sql.Int64, "t", "i", false), + }, + []sql.Expression{expression.NewGetFieldWithTable(1, sql.Int64, "t", "foo", false)}, + plan.NewResolvedTable(memory.NewTable("t", sql.Schema{ + {Type: sql.Int64, Name: "i", Source: "t"}, + {Type: sql.Int64, Name: "i", Source: "foo"}, + })), + ), + ), + ), + ), + nil, + }, + { + "push down aggregations with nodes in between", + plan.NewHaving( + expression.NewGreaterThan( + aggregation.NewCount(expression.NewStar()), + expression.NewLiteral(int64(5), sql.Int64), + ), + plan.NewProject( + []sql.Expression{ + expression.NewAlias(expression.NewGetField(0, sql.Float64, "avg(foo)", false), "x"), + expression.NewGetFieldWithTable(1, sql.Int64, "t", "foo", false), + }, + plan.NewGroupBy( + []sql.Expression{ + aggregation.NewAvg(expression.NewGetField(0, sql.Int64, "foo", false)), + expression.NewGetFieldWithTable(0, sql.Int64, "t", "foo", false), + }, + []sql.Expression{expression.NewGetField(0, sql.Int64, "foo", false)}, + plan.NewResolvedTable(memory.NewTable("t", nil)), + ), + ), + ), + plan.NewProject( + []sql.Expression{ + expression.NewGetField(0, sql.Float64, "x", false), + expression.NewGetFieldWithTable(1, sql.Int64, "t", "foo", false), + }, + plan.NewHaving( + expression.NewGreaterThan( + expression.NewGetField(2, sql.Int64, "COUNT(*)", false), + expression.NewLiteral(int64(5), sql.Int64), + ), + plan.NewProject( + []sql.Expression{ + expression.NewAlias(expression.NewGetField(0, sql.Float64, "avg(foo)", false), "x"), + expression.NewGetFieldWithTable(1, sql.Int64, "t", "foo", false), + expression.NewGetField(2, sql.Int64, "COUNT(*)", false), + }, + plan.NewGroupBy( + []sql.Expression{ + aggregation.NewAvg(expression.NewGetField(0, sql.Int64, "foo", false)), + expression.NewGetFieldWithTable(0, sql.Int64, "t", "foo", false), + aggregation.NewCount(expression.NewStar()), + }, + []sql.Expression{expression.NewGetField(0, sql.Int64, "foo", false)}, + plan.NewResolvedTable(memory.NewTable("t", nil)), + ), + ), + ), + ), + nil, + }, + { + "replace existing aggregation in group by with nodes in between", + plan.NewHaving( + expression.NewGreaterThan( + aggregation.NewAvg(expression.NewUnresolvedColumn("foo")), + expression.NewLiteral(int64(5), sql.Int64), + ), + plan.NewProject( + []sql.Expression{ + expression.NewGetField(0, sql.Float64, "x", false), + expression.NewGetField(1, sql.Int64, "foo", false), + }, + plan.NewGroupBy( + []sql.Expression{ + expression.NewAlias(aggregation.NewAvg(expression.NewGetFieldWithTable(0, sql.Int64, "t", "foo", false)), "x"), + expression.NewGetField(0, sql.Int64, "foo", false), + }, + []sql.Expression{expression.NewGetField(0, sql.Int64, "foo", false)}, + plan.NewResolvedTable(memory.NewTable("t", nil)), + ), + ), + ), + plan.NewHaving( + expression.NewGreaterThan( + expression.NewGetField(0, sql.Float64, "x", false), + expression.NewLiteral(int64(5), sql.Int64), + ), + plan.NewProject( + []sql.Expression{ + expression.NewGetField(0, sql.Float64, "x", false), + expression.NewGetField(1, sql.Int64, "foo", false), + }, + plan.NewGroupBy( + []sql.Expression{ + expression.NewAlias(aggregation.NewAvg(expression.NewGetFieldWithTable(0, sql.Int64, "t", "foo", false)), "x"), + expression.NewGetField(0, sql.Int64, "foo", false), + }, + []sql.Expression{expression.NewGetField(0, sql.Int64, "foo", false)}, + plan.NewResolvedTable(memory.NewTable("t", nil)), + ), + ), + ), + nil, + }, + { + "missing groupby", + plan.NewHaving( + expression.NewGreaterThan( + aggregation.NewCount(expression.NewStar()), + expression.NewLiteral(int64(5), sql.Int64), + ), + plan.NewResolvedTable(memory.NewTable("t", nil)), + ), + nil, + errHavingNeedsGroupBy, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + require := require.New(t) + result, err := resolveHaving(sql.NewEmptyContext(), nil, tt.input) + if tt.err != nil { + require.Error(err) + require.True(tt.err.Is(err)) + } else { + require.NoError(err) + require.Equal(tt.expected, result) + } + }) + } +} diff --git a/sql/analyzer/resolve_natural_joins.go b/sql/analyzer/resolve_natural_joins.go index 2de60b6e9..a6cf1fdb9 100644 --- a/sql/analyzer/resolve_natural_joins.go +++ b/sql/analyzer/resolve_natural_joins.go @@ -1,266 +1,137 @@ package analyzer import ( - "reflect" + "strings" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" ) -type transformedJoin struct { - node sql.Node - condCols map[string]*transformedSource -} - -type transformedSource struct { - correct string - wrong []string -} - func resolveNaturalJoins(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { span, _ := ctx.Span("resolve_natural_joins") defer span.Finish() - if n.Resolved() { - return n, nil - } - - var transformed []*transformedJoin - var aliasTables = map[string][]string{} - var colsToUnresolve = map[string]*transformedSource{} - a.Log("resolving natural joins, node of type %T", n) - node, err := n.TransformUp(func(n sql.Node) (sql.Node, error) { - a.Log("transforming node of type: %T", n) + var replacements = make(map[tableCol]tableCol) + var tableAliases = make(map[string]string) - if alias, ok := n.(*plan.TableAlias); ok { - table := alias.Child.(*plan.ResolvedTable).Name() - aliasTables[alias.Name()] = append(aliasTables[alias.Name()], table) + return plan.TransformUp(n, func(node sql.Node) (sql.Node, error) { + switch n := node.(type) { + case *plan.TableAlias: + alias := n.Name() + table := n.Child.(*plan.ResolvedTable).Name() + tableAliases[strings.ToLower(alias)] = table return n, nil - } - - if n.Resolved() { + case *plan.NaturalJoin: + return resolveNaturalJoin(n, replacements) + case sql.Expressioner: + return replaceExpressions(node, replacements, tableAliases) + default: return n, nil } + }) +} - join, ok := n.(*plan.NaturalJoin) - if !ok { - return n, nil - } - - // we need both leaves resolved before resolving this one - if !join.Left.Resolved() || !join.Right.Resolved() { - return n, nil - } - - leftSchema, rightSchema := join.Left.Schema(), join.Right.Schema() - - var conditions, common, left, right []sql.Expression - var seen = make(map[string]struct{}) - - for i, lcol := range leftSchema { - var found bool - leftCol := expression.NewGetFieldWithTable( - i, - lcol.Type, - lcol.Source, - lcol.Name, - lcol.Nullable, - ) - - for j, rcol := range rightSchema { - if lcol.Name == rcol.Name { - common = append(common, leftCol) - - conditions = append( - conditions, - expression.NewEquals( - leftCol, - expression.NewGetFieldWithTable( - len(leftSchema)+j, - rcol.Type, - rcol.Source, - rcol.Name, - rcol.Nullable, - ), - ), - ) - - found = true - seen[lcol.Name] = struct{}{} - if source, ok := colsToUnresolve[lcol.Name]; ok { - source.correct = lcol.Source - source.wrong = append(source.wrong, rcol.Source) - } else { - colsToUnresolve[lcol.Name] = &transformedSource{ - correct: lcol.Source, - wrong: []string{rcol.Source}, - } - } - - break - } - } +func resolveNaturalJoin( + n *plan.NaturalJoin, + replacements map[tableCol]tableCol, +) (sql.Node, error) { + // Both sides of the natural join need to be resolved in order to resolve + // the natural join itself. + if !n.Left.Resolved() || !n.Right.Resolved() { + return n, nil + } - if !found { - left = append(left, leftCol) + leftSchema := n.Left.Schema() + rightSchema := n.Right.Schema() + + var conditions, common, left, right []sql.Expression + for i, lcol := range leftSchema { + leftCol := expression.NewGetFieldWithTable( + i, + lcol.Type, + lcol.Source, + lcol.Name, + lcol.Nullable, + ) + if idx, rcol := findCol(rightSchema, lcol.Name); rcol != nil { + common = append(common, leftCol) + replacements[tableCol{strings.ToLower(rcol.Source), strings.ToLower(rcol.Name)}] = tableCol{ + strings.ToLower(lcol.Source), strings.ToLower(lcol.Name), } - } - if len(conditions) == 0 { - return plan.NewCrossJoin(join.Left, join.Right), nil - } - - for i, col := range rightSchema { - if _, ok := seen[col.Name]; !ok { - right = append( - right, + conditions = append( + conditions, + expression.NewEquals( + leftCol, expression.NewGetFieldWithTable( - len(leftSchema)+i, - col.Type, - col.Source, - col.Name, - col.Nullable, + len(leftSchema)+idx, + rcol.Type, + rcol.Source, + rcol.Name, + rcol.Nullable, ), - ) - } - } - - projections := append(append(common, left...), right...) - - tj := &transformedJoin{ - node: plan.NewProject( - projections, - plan.NewInnerJoin( - join.Left, - join.Right, - expression.JoinAnd(conditions...), ), - ), - condCols: colsToUnresolve, + ) + } else { + left = append(left, leftCol) } - - transformed = append(transformed, tj) - - return tj.node, nil - }) - - if err != nil || len(transformed) == 0 { - return node, err } - var transformedSeen bool - return node.TransformUp(func(node sql.Node) (sql.Node, error) { - if ok, _ := isTransformedNode(node, transformed); ok { - transformedSeen = true - return node, nil - } - - if !transformedSeen { - return node, nil - } - - expressioner, ok := node.(sql.Expressioner) - if !ok { - return node, nil - } - - return expressioner.TransformExpressions(func(e sql.Expression) (sql.Expression, error) { - var col, table string - switch e := e.(type) { - case *expression.GetField: - col, table = e.Name(), e.Table() - case *expression.UnresolvedColumn: - col, table = e.Name(), e.Table() - default: - return e, nil - } - - sources, ok := colsToUnresolve[col] - if !ok { - return e, nil - } - - if !mustUnresolve(aliasTables, table, sources.wrong) { - return e, nil - } - - return expression.NewUnresolvedQualifiedColumn( - sources.correct, - col, - ), nil - }) - }) -} - -func isTransformedNode(node sql.Node, transformed []*transformedJoin) (is bool, colsToUnresolve map[string]*transformedSource) { - var project *plan.Project - var join *plan.InnerJoin - switch n := node.(type) { - case *plan.Project: - var ok bool - join, ok = n.Child.(*plan.InnerJoin) - if !ok { - return - } - - project = n - case *plan.InnerJoin: - join = n - - default: - return + if len(conditions) == 0 { + return plan.NewCrossJoin(n.Left, n.Right), nil } - for _, t := range transformed { - tproject, ok := t.node.(*plan.Project) - if !ok { - return - } - - tjoin, ok := tproject.Child.(*plan.InnerJoin) - if !ok { - return - } - - if project != nil && !reflect.DeepEqual(project.Projections, tproject.Projections) { - continue - } - - if reflect.DeepEqual(join.Cond, tjoin.Cond) { - is = true - colsToUnresolve = t.condCols + for i, col := range rightSchema { + source := strings.ToLower(col.Source) + name := strings.ToLower(col.Name) + if _, ok := replacements[tableCol{source, name}]; !ok { + right = append( + right, + expression.NewGetFieldWithTable( + len(leftSchema)+i, + col.Type, + col.Source, + col.Name, + col.Nullable, + ), + ) } } - return + return plan.NewProject( + append(append(common, left...), right...), + plan.NewInnerJoin(n.Left, n.Right, expression.JoinAnd(conditions...)), + ), nil } -func mustUnresolve(aliasTable map[string][]string, table string, wrongSources []string) bool { - return isIn(table, wrongSources) || isAliasFor(aliasTable, table, wrongSources) -} - -func isIn(s string, l []string) bool { - for _, e := range l { - if s == e { - return true +func findCol(s sql.Schema, name string) (int, *sql.Column) { + for i, c := range s { + if strings.ToLower(c.Name) == strings.ToLower(name) { + return i, c } } - - return false + return -1, nil } -func isAliasFor(aliasTable map[string][]string, table string, wrongSources []string) bool { - tables, ok := aliasTable[table] - if !ok { - return false - } +func replaceExpressions( + n sql.Node, + replacements map[tableCol]tableCol, + tableAliases map[string]string, +) (sql.Node, error) { + return plan.TransformExpressions(n, func(e sql.Expression) (sql.Expression, error) { + switch e := e.(type) { + case *expression.GetField, *expression.UnresolvedColumn: + var tableName = e.(sql.Tableable).Table() + if t, ok := tableAliases[strings.ToLower(tableName)]; ok { + tableName = t + } - for _, t := range tables { - if isIn(t, wrongSources) { - return true + name := e.(sql.Nameable).Name() + if col, ok := replacements[tableCol{strings.ToLower(tableName), strings.ToLower(name)}]; ok { + return expression.NewUnresolvedQualifiedColumn(col.table, col.col), nil + } } - } - - return false + return e, nil + }) } diff --git a/sql/analyzer/resolve_natural_joins_test.go b/sql/analyzer/resolve_natural_joins_test.go index 1698565a2..4030f8f93 100644 --- a/sql/analyzer/resolve_natural_joins_test.go +++ b/sql/analyzer/resolve_natural_joins_test.go @@ -3,23 +3,23 @@ package analyzer import ( "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" ) func TestResolveNaturalJoins(t *testing.T) { require := require.New(t) - left := mem.NewTable("t1", sql.Schema{ + left := memory.NewTable("t1", sql.Schema{ {Name: "a", Type: sql.Int64, Source: "t1"}, {Name: "b", Type: sql.Int64, Source: "t1"}, {Name: "c", Type: sql.Int64, Source: "t1"}, }) - right := mem.NewTable("t2", sql.Schema{ + right := memory.NewTable("t2", sql.Schema{ {Name: "d", Type: sql.Int64, Source: "t2"}, {Name: "c", Type: sql.Int64, Source: "t2"}, {Name: "b", Type: sql.Int64, Source: "t2"}, @@ -66,13 +66,13 @@ func TestResolveNaturalJoinsColumns(t *testing.T) { rule := getRule("resolve_natural_joins") require := require.New(t) - left := mem.NewTable("t1", sql.Schema{ + left := memory.NewTable("t1", sql.Schema{ {Name: "a", Type: sql.Int64, Source: "t1"}, {Name: "b", Type: sql.Int64, Source: "t1"}, {Name: "c", Type: sql.Int64, Source: "t1"}, }) - right := mem.NewTable("t2", sql.Schema{ + right := memory.NewTable("t2", sql.Schema{ {Name: "d", Type: sql.Int64, Source: "t2"}, {Name: "c", Type: sql.Int64, Source: "t2"}, {Name: "b", Type: sql.Int64, Source: "t2"}, @@ -128,13 +128,13 @@ func TestResolveNaturalJoinsTableAlias(t *testing.T) { rule := getRule("resolve_natural_joins") require := require.New(t) - left := mem.NewTable("t1", sql.Schema{ + left := memory.NewTable("t1", sql.Schema{ {Name: "a", Type: sql.Int64, Source: "t1"}, {Name: "b", Type: sql.Int64, Source: "t1"}, {Name: "c", Type: sql.Int64, Source: "t1"}, }) - right := mem.NewTable("t2", sql.Schema{ + right := memory.NewTable("t2", sql.Schema{ {Name: "d", Type: sql.Int64, Source: "t2"}, {Name: "c", Type: sql.Int64, Source: "t2"}, {Name: "b", Type: sql.Int64, Source: "t2"}, @@ -192,21 +192,21 @@ func TestResolveNaturalJoinsChained(t *testing.T) { rule := getRule("resolve_natural_joins") require := require.New(t) - left := mem.NewTable("t1", sql.Schema{ + left := memory.NewTable("t1", sql.Schema{ {Name: "a", Type: sql.Int64, Source: "t1"}, {Name: "b", Type: sql.Int64, Source: "t1"}, {Name: "c", Type: sql.Int64, Source: "t1"}, {Name: "f", Type: sql.Int64, Source: "t1"}, }) - right := mem.NewTable("t2", sql.Schema{ + right := memory.NewTable("t2", sql.Schema{ {Name: "d", Type: sql.Int64, Source: "t2"}, {Name: "c", Type: sql.Int64, Source: "t2"}, {Name: "b", Type: sql.Int64, Source: "t2"}, {Name: "e", Type: sql.Int64, Source: "t2"}, }) - upperRight := mem.NewTable("t3", sql.Schema{ + upperRight := memory.NewTable("t3", sql.Schema{ {Name: "a", Type: sql.Int64, Source: "t3"}, {Name: "b", Type: sql.Int64, Source: "t3"}, {Name: "f", Type: sql.Int64, Source: "t3"}, @@ -297,13 +297,13 @@ func TestResolveNaturalJoinsChained(t *testing.T) { func TestResolveNaturalJoinsEqual(t *testing.T) { require := require.New(t) - left := mem.NewTable("t1", sql.Schema{ + left := memory.NewTable("t1", sql.Schema{ {Name: "a", Type: sql.Int64, Source: "t1"}, {Name: "b", Type: sql.Int64, Source: "t1"}, {Name: "c", Type: sql.Int64, Source: "t1"}, }) - right := mem.NewTable("t2", sql.Schema{ + right := memory.NewTable("t2", sql.Schema{ {Name: "a", Type: sql.Int64, Source: "t2"}, {Name: "b", Type: sql.Int64, Source: "t2"}, {Name: "c", Type: sql.Int64, Source: "t2"}, @@ -350,13 +350,13 @@ func TestResolveNaturalJoinsEqual(t *testing.T) { func TestResolveNaturalJoinsDisjoint(t *testing.T) { require := require.New(t) - left := mem.NewTable("t1", sql.Schema{ + left := memory.NewTable("t1", sql.Schema{ {Name: "a", Type: sql.Int64, Source: "t1"}, {Name: "b", Type: sql.Int64, Source: "t1"}, {Name: "c", Type: sql.Int64, Source: "t1"}, }) - right := mem.NewTable("t2", sql.Schema{ + right := memory.NewTable("t2", sql.Schema{ {Name: "d", Type: sql.Int64, Source: "t2"}, {Name: "e", Type: sql.Int64, Source: "t2"}, }) diff --git a/sql/analyzer/resolve_orderby.go b/sql/analyzer/resolve_orderby.go index a678c3594..b7e7dfc65 100644 --- a/sql/analyzer/resolve_orderby.go +++ b/sql/analyzer/resolve_orderby.go @@ -3,10 +3,10 @@ package analyzer import ( "strings" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" ) func resolveOrderBy(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { @@ -14,7 +14,7 @@ func resolveOrderBy(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) defer span.Finish() a.Log("resolving order bys, node of type: %T", n) - return n.TransformUp(func(n sql.Node) (sql.Node, error) { + return plan.TransformUp(n, func(n sql.Node) (sql.Node, error) { a.Log("transforming node of type: %T", n) sort, ok := n.(*plan.Sort) if !ok { @@ -29,21 +29,21 @@ func resolveOrderBy(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) childNewCols := columnsDefinedInNode(sort.Child) var schemaCols []string for _, col := range sort.Child.Schema() { - schemaCols = append(schemaCols, col.Name) + schemaCols = append(schemaCols, strings.ToLower(col.Name)) } var colsFromChild []string var missingCols []string for _, f := range sort.SortFields { - n, ok := f.Column.(sql.Nameable) - if !ok { - continue - } - - if stringContains(childNewCols, n.Name()) { - colsFromChild = append(colsFromChild, n.Name()) - } else if !stringContains(schemaCols, n.Name()) { - missingCols = append(missingCols, n.Name()) + ns := findExprNameables(f.Column) + + for _, n := range ns { + name := strings.ToLower(n.Name()) + if stringContains(childNewCols, name) { + colsFromChild = append(colsFromChild, n.Name()) + } else if !stringContains(schemaCols, name) { + missingCols = append(missingCols, n.Name()) + } } } @@ -141,7 +141,7 @@ func columnsDefinedInNode(n sql.Node) []string { for _, e := range exprs { alias, ok := e.(*expression.Alias) if ok { - cols = append(cols, alias.Name()) + cols = append(cols, strings.ToLower(alias.Name())) } } @@ -163,9 +163,21 @@ func pushSortDown(sort *plan.Sort) (sql.Node, error) { child.Grouping, plan.NewSort(sort.SortFields, child.Child), ), nil + case *plan.ResolvedTable: + return sort, nil default: - // Can't do anything here, there should be either a project or a groupby - // below an order by. + children := child.Children() + if len(children) == 1 { + newChild, err := pushSortDown(plan.NewSort(sort.SortFields, children[0])) + if err != nil { + return nil, err + } + + return child.WithChildren(newChild) + } + + // If the child has more than one children we don't know to which side + // the sort must be pushed down. return nil, errSortPushdown.New(child) } } @@ -173,7 +185,7 @@ func pushSortDown(sort *plan.Sort) (sql.Node, error) { func resolveOrderByLiterals(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { a.Log("resolve order by literals") - return n.TransformUp(func(n sql.Node) (sql.Node, error) { + return plan.TransformUp(n, func(n sql.Node) (sql.Node, error) { sort, ok := n.(*plan.Sort) if !ok { return n, nil @@ -184,7 +196,14 @@ func resolveOrderByLiterals(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node return n, nil } - var fields = make([]plan.SortField, len(sort.SortFields)) + schema := sort.Child.Schema() + var ( + fields = make([]plan.SortField, len(sort.SortFields)) + schemaCols = make([]string, len(schema)) + ) + for i, col := range sort.Child.Schema() { + schemaCols[i] = col.Name + } for i, f := range sort.SortFields { if lit, ok := f.Column.(*expression.Literal); ok && sql.IsNumber(f.Column.Type()) { // it is safe to eval literals with no context and/or row @@ -200,24 +219,48 @@ func resolveOrderByLiterals(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node // column access is 1-indexed idx := int(v.(int64)) - 1 - - schema := sort.Child.Schema() if idx >= len(schema) || idx < 0 { return nil, ErrOrderByColumnIndex.New(idx + 1) } fields[i] = plan.SortField{ - Column: expression.NewUnresolvedColumn(schema[idx].Name), + Column: expression.NewUnresolvedColumn(schemaCols[idx]), Order: f.Order, NullOrdering: f.NullOrdering, } - a.Log("replaced order by column %d with %s", idx+1, schema[idx].Name) + a.Log("replaced order by column %d with %s", idx+1, schemaCols[idx]) } else { - fields[i] = f + if agg, ok := f.Column.(sql.Aggregation); ok { + name := agg.String() + if nameable, ok := f.Column.(sql.Nameable); ok { + name = nameable.Name() + } + + fields[i] = plan.SortField{ + Column: expression.NewUnresolvedColumn(name), + Order: f.Order, + NullOrdering: f.NullOrdering, + } + } else { + fields[i] = f + } } } return plan.NewSort(fields, sort.Child), nil }) } + +func findExprNameables(e sql.Expression) []sql.Nameable { + var result []sql.Nameable + expression.Inspect(e, func(e sql.Expression) bool { + n, ok := e.(sql.Nameable) + if ok { + result = append(result, n) + return false + } + return true + }) + return result +} diff --git a/sql/analyzer/resolve_orderby_test.go b/sql/analyzer/resolve_orderby_test.go index 29186c4a5..119c197a4 100644 --- a/sql/analyzer/resolve_orderby_test.go +++ b/sql/analyzer/resolve_orderby_test.go @@ -3,11 +3,11 @@ package analyzer import ( "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" ) func TestResolveOrderBy(t *testing.T) { @@ -15,7 +15,7 @@ func TestResolveOrderBy(t *testing.T) { a := NewDefault(nil) ctx := sql.NewEmptyContext() - table := mem.NewTable("foo", sql.Schema{ + table := memory.NewTable("foo", sql.Schema{ {Name: "a", Type: sql.Int64, Source: "foo"}, {Name: "b", Type: sql.Int64, Source: "foo"}, }) @@ -242,7 +242,7 @@ func TestResolveOrderByLiterals(t *testing.T) { require := require.New(t) f := getRule("resolve_orderby_literals") - table := mem.NewTable("t", sql.Schema{ + table := memory.NewTable("t", sql.Schema{ {Name: "a", Type: sql.Int64, Source: "t"}, {Name: "b", Type: sql.Int64, Source: "t"}, }) diff --git a/sql/analyzer/resolve_stars.go b/sql/analyzer/resolve_stars.go index c33432945..2261752f3 100644 --- a/sql/analyzer/resolve_stars.go +++ b/sql/analyzer/resolve_stars.go @@ -1,9 +1,9 @@ package analyzer import ( - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" ) func resolveStar(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { @@ -11,7 +11,7 @@ func resolveStar(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { defer span.Finish() a.Log("resolving star, node of type: %T", n) - return n.TransformUp(func(n sql.Node) (sql.Node, error) { + return plan.TransformUp(n, func(n sql.Node) (sql.Node, error) { a.Log("transforming node of type: %T", n) if n.Resolved() { return n, nil diff --git a/sql/analyzer/resolve_stars_test.go b/sql/analyzer/resolve_stars_test.go index 691f544ae..b6006271f 100644 --- a/sql/analyzer/resolve_stars_test.go +++ b/sql/analyzer/resolve_stars_test.go @@ -3,22 +3,22 @@ package analyzer import ( "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" ) func TestResolveStar(t *testing.T) { f := getRule("resolve_star") - table := mem.NewTable("mytable", sql.Schema{ + table := memory.NewTable("mytable", sql.Schema{ {Name: "a", Type: sql.Int32, Source: "mytable"}, {Name: "b", Type: sql.Int32, Source: "mytable"}, }) - table2 := mem.NewTable("mytable2", sql.Schema{ + table2 := memory.NewTable("mytable2", sql.Schema{ {Name: "c", Type: sql.Int32, Source: "mytable2"}, {Name: "d", Type: sql.Int32, Source: "mytable2"}, }) diff --git a/sql/analyzer/resolve_subqueries.go b/sql/analyzer/resolve_subqueries.go index 89fdb463f..20b97df3f 100644 --- a/sql/analyzer/resolve_subqueries.go +++ b/sql/analyzer/resolve_subqueries.go @@ -1,8 +1,9 @@ package analyzer import ( - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" ) func resolveSubqueries(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { @@ -10,7 +11,7 @@ func resolveSubqueries(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, err defer span.Finish() a.Log("resolving subqueries") - return n.TransformUp(func(n sql.Node) (sql.Node, error) { + n, err := plan.TransformUp(n, func(n sql.Node) (sql.Node, error) { switch n := n.(type) { case *plan.SubqueryAlias: a.Log("found subquery %q with child of type %T", n.Name(), n.Child) @@ -24,4 +25,25 @@ func resolveSubqueries(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, err return n, nil } }) + if err != nil { + return nil, err + } + + return plan.TransformExpressionsUp(n, func(e sql.Expression) (sql.Expression, error) { + s, ok := e.(*expression.Subquery) + if !ok || s.Resolved() { + return e, nil + } + + q, err := a.Analyze(ctx, s.Query) + if err != nil { + return nil, err + } + + if qp, ok := q.(*plan.QueryProcess); ok { + q = qp.Child + } + + return s.WithQuery(q), nil + }) } diff --git a/sql/analyzer/resolve_subqueries_test.go b/sql/analyzer/resolve_subqueries_test.go index ec05cc8f4..04e97832a 100644 --- a/sql/analyzer/resolve_subqueries_test.go +++ b/sql/analyzer/resolve_subqueries_test.go @@ -3,23 +3,23 @@ package analyzer import ( "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" ) func TestResolveSubqueries(t *testing.T) { require := require.New(t) - table1 := mem.NewTable("foo", sql.Schema{{Name: "a", Type: sql.Int64, Source: "foo"}}) - table2 := mem.NewTable("bar", sql.Schema{ + table1 := memory.NewTable("foo", sql.Schema{{Name: "a", Type: sql.Int64, Source: "foo"}}) + table2 := memory.NewTable("bar", sql.Schema{ {Name: "b", Type: sql.Int64, Source: "bar"}, {Name: "k", Type: sql.Int64, Source: "bar"}, }) - table3 := mem.NewTable("baz", sql.Schema{{Name: "c", Type: sql.Int64, Source: "baz"}}) - db := mem.NewDatabase("mydb") + table3 := memory.NewTable("baz", sql.Schema{{Name: "c", Type: sql.Int64, Source: "baz"}}) + db := memory.NewDatabase("mydb") db.AddTable("foo", table1) db.AddTable("bar", table2) db.AddTable("baz", table3) diff --git a/sql/analyzer/resolve_tables.go b/sql/analyzer/resolve_tables.go index d9b46190c..ec495cd36 100644 --- a/sql/analyzer/resolve_tables.go +++ b/sql/analyzer/resolve_tables.go @@ -1,15 +1,15 @@ package analyzer import ( - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/plan" ) const dualTableName = "dual" var dualTable = func() sql.Table { - t := mem.NewTable(dualTableName, sql.Schema{ + t := memory.NewTable(dualTableName, sql.Schema{ {Name: "dummy", Source: dualTableName, Type: sql.Text, Nullable: false}, }) _ = t.Insert(sql.NewEmptyContext(), sql.NewRow("x")) @@ -21,7 +21,7 @@ func resolveTables(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) defer span.Finish() a.Log("resolve table, node of type: %T", n) - return n.TransformUp(func(n sql.Node) (sql.Node, error) { + return plan.TransformUp(n, func(n sql.Node) (sql.Node, error) { a.Log("transforming node of type: %T", n) if n.Resolved() { return n, nil diff --git a/sql/analyzer/resolve_tables_test.go b/sql/analyzer/resolve_tables_test.go index 86bc68d74..fbc4fcca0 100644 --- a/sql/analyzer/resolve_tables_test.go +++ b/sql/analyzer/resolve_tables_test.go @@ -3,11 +3,11 @@ package analyzer import ( "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" ) func TestResolveTables(t *testing.T) { @@ -15,8 +15,8 @@ func TestResolveTables(t *testing.T) { f := getRule("resolve_tables") - table := mem.NewTable("mytable", sql.Schema{{Name: "i", Type: sql.Int32}}) - db := mem.NewDatabase("mydb") + table := memory.NewTable("mytable", sql.Schema{{Name: "i", Type: sql.Int32}}) + db := memory.NewDatabase("mydb") db.AddTable("mytable", table) catalog := sql.NewCatalog() @@ -54,14 +54,14 @@ func TestResolveTablesNested(t *testing.T) { f := getRule("resolve_tables") - table := mem.NewTable("mytable", sql.Schema{{Name: "i", Type: sql.Int32}}) - table2 := mem.NewTable("my_other_table", sql.Schema{{Name: "i", Type: sql.Int32}}) - db := mem.NewDatabase("mydb") + table := memory.NewTable("mytable", sql.Schema{{Name: "i", Type: sql.Int32}}) + table2 := memory.NewTable("my_other_table", sql.Schema{{Name: "i", Type: sql.Int32}}) + db := memory.NewDatabase("mydb") db.AddTable("mytable", table) catalog := sql.NewCatalog() catalog.AddDatabase(db) - db2 := mem.NewDatabase("my_other_db") + db2 := memory.NewDatabase("my_other_db") db2.AddTable("my_other_table", table2) catalog.AddDatabase(db2) diff --git a/sql/analyzer/rules.go b/sql/analyzer/rules.go index 0386fe57f..c2b8daf00 100644 --- a/sql/analyzer/rules.go +++ b/sql/analyzer/rules.go @@ -15,6 +15,7 @@ var DefaultRules = []Rule{ {"resolve_database", resolveDatabase}, {"resolve_star", resolveStar}, {"resolve_functions", resolveFunctions}, + {"resolve_having", resolveHaving}, {"reorder_aggregations", reorderAggregations}, {"reorder_projection", reorderProjection}, {"move_join_conds_to_filter", moveJoinConditionsToFilter}, @@ -33,9 +34,11 @@ var OnceBeforeDefault = []Rule{ // OnceAfterDefault contains the rules to be applied just once after the // DefaultRules. var OnceAfterDefault = []Rule{ + {"resolve_generators", resolveGenerators}, {"remove_unnecessary_converts", removeUnnecessaryConverts}, {"assign_catalog", assignCatalog}, {"prune_columns", pruneColumns}, + {"convert_dates", convertDates}, {"pushdown", pushdown}, {"erase_projection", eraseProjection}, } diff --git a/sql/analyzer/validation_rules.go b/sql/analyzer/validation_rules.go index 500b3608c..7d35ee6be 100644 --- a/sql/analyzer/validation_rules.go +++ b/sql/analyzer/validation_rules.go @@ -3,10 +3,11 @@ package analyzer import ( "strings" - errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/expression/function" + "github.com/src-d/go-mysql-server/sql/plan" + "gopkg.in/src-d/go-errors.v1" ) const ( @@ -17,6 +18,9 @@ const ( validateProjectTuplesRule = "validate_project_tuples" validateIndexCreationRule = "validate_index_creation" validateCaseResultTypesRule = "validate_case_result_types" + validateIntervalUsageRule = "validate_interval_usage" + validateExplodeUsageRule = "validate_explode_usage" + validateSubqueryColumnsRule = "validate_subquery_columns" ) var ( @@ -43,6 +47,23 @@ var ( "expecting all case branches to return values of type %s, " + "but found value %q of type %s on %s", ) + // ErrIntervalInvalidUse is returned when an interval expression is not + // correctly used. + ErrIntervalInvalidUse = errors.NewKind( + "invalid use of an interval, which can only be used with DATE_ADD, " + + "DATE_SUB and +/- operators to subtract from or add to a date", + ) + // ErrExplodeInvalidUse is returned when an EXPLODE function is used + // outside a Project node. + ErrExplodeInvalidUse = errors.NewKind( + "using EXPLODE is not supported outside a Project node", + ) + + // ErrSubqueryColumns is returned when an expression subquery returns + // more than a single column. + ErrSubqueryColumns = errors.NewKind( + "subquery expressions can only return a single column", + ) ) // DefaultValidationRules to apply while analyzing nodes. @@ -54,6 +75,9 @@ var DefaultValidationRules = []Rule{ {validateProjectTuplesRule, validateProjectTuples}, {validateIndexCreationRule, validateIndexCreation}, {validateCaseResultTypesRule, validateCaseResultTypes}, + {validateIntervalUsageRule, validateIntervalUsage}, + {validateExplodeUsageRule, validateExplodeUsage}, + {validateSubqueryColumnsRule, validateSubqueryColumns}, } func validateIsResolved(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { @@ -186,27 +210,36 @@ func validateSchema(t *plan.ResolvedTable) error { return nil } -func validateProjectTuples(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { - span, _ := ctx.Span("validate_project_tuples") - defer span.Finish() +func findProjectTuples(n sql.Node) (sql.Node, error) { + if n == nil { + return n, nil + } switch n := n.(type) { - case *plan.Project: - for i, e := range n.Projections { + case *plan.Project, *plan.GroupBy: + for i, e := range n.(sql.Expressioner).Expressions() { if sql.IsTuple(e.Type()) { return nil, ErrProjectTuple.New(i+1, sql.NumColumns(e.Type())) } } - case *plan.GroupBy: - for i, e := range n.Aggregate { - if sql.IsTuple(e.Type()) { - return nil, ErrProjectTuple.New(i+1, sql.NumColumns(e.Type())) + default: + for _, ch := range n.Children() { + _, err := findProjectTuples(ch) + if err != nil { + return nil, err } } } + return n, nil } +func validateProjectTuples(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { + span, _ := ctx.Span("validate_project_tuples") + defer span.Finish() + return findProjectTuples(n) +} + func validateCaseResultTypes(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { span, ctx := ctx.Span("validate_case_result_types") defer span.Finish() @@ -217,14 +250,14 @@ func validateCaseResultTypes(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Nod case *expression.Case: typ := e.Type() for _, b := range e.Branches { - if b.Value.Type() != typ { + if b.Value.Type() != typ && b.Value.Type() != sql.Null { err = ErrCaseResultType.New(typ, b.Value, b.Value.Type(), e) return false } } if e.Else != nil { - if e.Else.Type() != typ { + if e.Else.Type() != typ && e.Else.Type() != sql.Null { err = ErrCaseResultType.New(typ, e.Else, e.Else.Type(), e) return false } @@ -243,6 +276,79 @@ func validateCaseResultTypes(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Nod return n, nil } +func validateIntervalUsage(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { + var invalid bool + plan.InspectExpressions(n, func(e sql.Expression) bool { + // If it's already invalid just skip everything else. + if invalid { + return false + } + + switch e := e.(type) { + case *function.DateAdd, *function.DateSub: + return false + case *expression.Arithmetic: + if e.Op == "+" || e.Op == "-" { + return false + } + case *expression.Interval: + invalid = true + } + + return true + }) + + if invalid { + return nil, ErrIntervalInvalidUse.New() + } + + return n, nil +} + +func validateExplodeUsage(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { + var invalid bool + plan.InspectExpressions(n, func(e sql.Expression) bool { + // If it's already invalid just skip everything else. + if invalid { + return false + } + + // All usage of Explode will be incorrect because the ones in projects + // would have already been converted to Generate, so we only have to + // look for those. + if _, ok := e.(*function.Explode); ok { + invalid = true + } + + return true + }) + + if invalid { + return nil, ErrExplodeInvalidUse.New() + } + + return n, nil +} + +func validateSubqueryColumns(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) { + valid := true + plan.InspectExpressions(n, func(e sql.Expression) bool { + s, ok := e.(*expression.Subquery) + if ok && len(s.Query.Schema()) != 1 { + valid = false + return false + } + + return true + }) + + if !valid { + return nil, ErrSubqueryColumns.New() + } + + return n, nil +} + func stringContains(strs []string, target string) bool { for _, s := range strs { if s == target { diff --git a/sql/analyzer/validation_rules_test.go b/sql/analyzer/validation_rules_test.go index 8715e36f9..543fbc688 100644 --- a/sql/analyzer/validation_rules_test.go +++ b/sql/analyzer/validation_rules_test.go @@ -3,11 +3,12 @@ package analyzer import ( "testing" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/aggregation" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/expression/function" + "github.com/src-d/go-mysql-server/sql/expression/function/aggregation" + "github.com/src-d/go-mysql-server/sql/plan" "github.com/stretchr/testify/require" ) @@ -56,7 +57,7 @@ func TestValidateGroupBy(t *testing.T) { {Name: "col2", Type: sql.Int64}, } - child := mem.NewTable("test", childSchema) + child := memory.NewTable("test", childSchema) rows := []sql.Row{ sql.NewRow("col1_1", int64(1111)), @@ -101,7 +102,7 @@ func TestValidateGroupByErr(t *testing.T) { {Name: "col2", Type: sql.Int64}, } - child := mem.NewTable("test", childSchema) + child := memory.NewTable("test", childSchema) rows := []sql.Row{ sql.NewRow("col1_1", int64(1111)), @@ -144,7 +145,7 @@ func TestValidateSchemaSource(t *testing.T) { { "table with valid schema", plan.NewResolvedTable( - mem.NewTable( + memory.NewTable( "mytable", sql.Schema{ {Name: "foo", Source: "mytable"}, @@ -157,7 +158,7 @@ func TestValidateSchemaSource(t *testing.T) { { "table with invalid schema", plan.NewResolvedTable( - mem.NewTable( + memory.NewTable( "mytable", sql.Schema{ {Name: "foo", Source: ""}, @@ -170,7 +171,7 @@ func TestValidateSchemaSource(t *testing.T) { { "table alias with table", plan.NewTableAlias("foo", plan.NewResolvedTable( - mem.NewTable("mytable", sql.Schema{ + memory.NewTable("mytable", sql.Schema{ {Name: "foo", Source: "mytable"}, }), )), @@ -234,11 +235,38 @@ func TestValidateProjectTuples(t *testing.T) { plan.NewProject([]sql.Expression{ expression.NewTuple( expression.NewLiteral(1, sql.Int64), - expression.NewLiteral(1, sql.Int64), + expression.NewLiteral(2, sql.Int64), ), }, nil), false, }, + { + "distinct with a 2 elem tuple inside the project", + plan.NewDistinct( + plan.NewProject([]sql.Expression{ + expression.NewTuple( + expression.NewLiteral(1, sql.Int64), + expression.NewLiteral(2, sql.Int64), + ), + }, nil)), + false, + }, + { + "alias with a tuple", + plan.NewProject( + []sql.Expression{ + expression.NewAlias( + expression.NewTuple( + expression.NewLiteral(1, sql.Int64), + expression.NewLiteral(2, sql.Int64), + ), + "foo", + ), + }, + plan.NewUnresolvedTable("dual", ""), + ), + false, + }, { "groupby with no tuple", plan.NewGroupBy([]sql.Expression{ @@ -283,7 +311,7 @@ func TestValidateProjectTuples(t *testing.T) { } func TestValidateIndexCreation(t *testing.T) { - table := mem.NewTable("foo", sql.Schema{ + table := memory.NewTable("foo", sql.Schema{ {Name: "a", Source: "foo"}, {Name: "b", Source: "foo"}, }) @@ -431,20 +459,261 @@ func TestValidateCaseResultTypes(t *testing.T) { } } -type dummyNode struct{ resolved bool } +func mustFunc(e sql.Expression, err error) sql.Expression { + if err != nil { + panic(err) + } + return e +} + +func TestValidateIntervalUsage(t *testing.T) { + testCases := []struct { + name string + node sql.Node + ok bool + }{ + { + "date add", + plan.NewProject( + []sql.Expression{ + mustFunc(function.NewDateAdd( + expression.NewLiteral("2018-05-01", sql.Text), + expression.NewInterval( + expression.NewLiteral(int64(1), sql.Int64), + "DAY", + ), + )), + }, + plan.NewUnresolvedTable("dual", ""), + ), + true, + }, + { + "date sub", + plan.NewProject( + []sql.Expression{ + mustFunc(function.NewDateSub( + expression.NewLiteral("2018-05-01", sql.Text), + expression.NewInterval( + expression.NewLiteral(int64(1), sql.Int64), + "DAY", + ), + )), + }, + plan.NewUnresolvedTable("dual", ""), + ), + true, + }, + { + "+ op", + plan.NewProject( + []sql.Expression{ + expression.NewPlus( + expression.NewLiteral("2018-05-01", sql.Text), + expression.NewInterval( + expression.NewLiteral(int64(1), sql.Int64), + "DAY", + ), + ), + }, + plan.NewUnresolvedTable("dual", ""), + ), + true, + }, + { + "- op", + plan.NewProject( + []sql.Expression{ + expression.NewMinus( + expression.NewLiteral("2018-05-01", sql.Text), + expression.NewInterval( + expression.NewLiteral(int64(1), sql.Int64), + "DAY", + ), + ), + }, + plan.NewUnresolvedTable("dual", ""), + ), + true, + }, + { + "invalid", + plan.NewProject( + []sql.Expression{ + expression.NewInterval( + expression.NewLiteral(int64(1), sql.Int64), + "DAY", + ), + }, + plan.NewUnresolvedTable("dual", ""), + ), + false, + }, + { + "alias", + plan.NewProject( + []sql.Expression{ + expression.NewAlias( + expression.NewInterval( + expression.NewLiteral(int64(1), sql.Int64), + "DAY", + ), + "foo", + ), + }, + plan.NewUnresolvedTable("dual", ""), + ), + false, + }, + } -func (n dummyNode) String() string { return "dummynode" } -func (n dummyNode) Resolved() bool { return n.resolved } -func (dummyNode) Schema() sql.Schema { return nil } -func (dummyNode) Children() []sql.Node { return nil } -func (dummyNode) RowIter(*sql.Context) (sql.RowIter, error) { return nil, nil } -func (dummyNode) TransformUp(sql.TransformNodeFunc) (sql.Node, error) { return nil, nil } -func (dummyNode) TransformExpressionsUp( - sql.TransformExprFunc, -) (sql.Node, error) { - return nil, nil + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + require := require.New(t) + + _, err := validateIntervalUsage(sql.NewEmptyContext(), nil, tt.node) + if tt.ok { + require.NoError(err) + } else { + require.Error(err) + require.True(ErrIntervalInvalidUse.Is(err)) + } + }) + } } +func TestValidateExplodeUsage(t *testing.T) { + testCases := []struct { + name string + node sql.Node + ok bool + }{ + { + "valid", + plan.NewGenerate( + plan.NewProject( + []sql.Expression{ + expression.NewAlias( + function.NewGenerate( + expression.NewGetField(0, sql.Array(sql.Int64), "f", false), + ), + "foo", + ), + }, + plan.NewUnresolvedTable("dual", ""), + ), + expression.NewGetField(0, sql.Array(sql.Int64), "foo", false), + ), + true, + }, + { + "where", + plan.NewFilter( + function.NewArrayLength( + function.NewExplode( + expression.NewGetField(0, sql.Array(sql.Int64), "foo", false), + ), + ), + plan.NewGenerate( + plan.NewProject( + []sql.Expression{ + expression.NewAlias( + function.NewGenerate( + expression.NewGetField(0, sql.Array(sql.Int64), "f", false), + ), + "foo", + ), + }, + plan.NewUnresolvedTable("dual", ""), + ), + expression.NewGetField(0, sql.Array(sql.Int64), "foo", false), + ), + ), + false, + }, + { + "group by", + plan.NewGenerate( + plan.NewGroupBy( + []sql.Expression{ + expression.NewAlias( + function.NewExplode( + expression.NewGetField(0, sql.Array(sql.Int64), "f", false), + ), + "foo", + ), + }, + []sql.Expression{ + expression.NewAlias( + function.NewExplode( + expression.NewGetField(0, sql.Array(sql.Int64), "f", false), + ), + "foo", + ), + }, + plan.NewUnresolvedTable("dual", ""), + ), + expression.NewGetField(0, sql.Array(sql.Int64), "foo", false), + ), + false, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + require := require.New(t) + + _, err := validateExplodeUsage(sql.NewEmptyContext(), nil, tt.node) + if tt.ok { + require.NoError(err) + } else { + require.Error(err) + require.True(ErrExplodeInvalidUse.Is(err)) + } + }) + } +} + +func TestValidateSubqueryColumns(t *testing.T) { + require := require.New(t) + ctx := sql.NewEmptyContext() + + node := plan.NewProject([]sql.Expression{ + expression.NewSubquery(plan.NewProject( + []sql.Expression{ + lit(1), + lit(2), + }, + dummyNode{true}, + )), + }, dummyNode{true}) + + _, err := validateSubqueryColumns(ctx, nil, node) + require.Error(err) + require.True(ErrSubqueryColumns.Is(err)) + + node = plan.NewProject([]sql.Expression{ + expression.NewSubquery(plan.NewProject( + []sql.Expression{ + lit(1), + }, + dummyNode{true}, + )), + }, dummyNode{true}) + + _, err = validateSubqueryColumns(ctx, nil, node) + require.NoError(err) +} + +type dummyNode struct{ resolved bool } + +func (n dummyNode) String() string { return "dummynode" } +func (n dummyNode) Resolved() bool { return n.resolved } +func (dummyNode) Schema() sql.Schema { return nil } +func (dummyNode) Children() []sql.Node { return nil } +func (dummyNode) RowIter(*sql.Context) (sql.RowIter, error) { return nil, nil } +func (dummyNode) WithChildren(...sql.Node) (sql.Node, error) { return nil, nil } + func getValidationRule(name string) Rule { for _, rule := range DefaultValidationRules { if rule.Name == name { diff --git a/sql/analyzer/warnings.go b/sql/analyzer/warnings.go index 642ab9211..a2b7dfbd7 100644 --- a/sql/analyzer/warnings.go +++ b/sql/analyzer/warnings.go @@ -1,8 +1,8 @@ package analyzer import ( - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/plan" ) func clearWarnings(ctx *sql.Context, a *Analyzer, node sql.Node) (sql.Node, error) { diff --git a/sql/cache.go b/sql/cache.go new file mode 100644 index 000000000..01e28b858 --- /dev/null +++ b/sql/cache.go @@ -0,0 +1,131 @@ +package sql + +import ( + "fmt" + "hash/crc64" + "runtime" + + lru "github.com/hashicorp/golang-lru" + errors "gopkg.in/src-d/go-errors.v1" +) + +var table = crc64.MakeTable(crc64.ISO) + +// CacheKey returns a hash of the given value to be used as key in +// a cache. +func CacheKey(v interface{}) uint64 { + return crc64.Checksum([]byte(fmt.Sprintf("%#v", v)), table) +} + +// ErrKeyNotFound is returned when the key could not be found in the cache. +var ErrKeyNotFound = errors.NewKind("memory: key %d not found in cache") + +type lruCache struct { + memory Freeable + reporter Reporter + size int + cache *lru.Cache +} + +func newLRUCache(memory Freeable, r Reporter, size uint) *lruCache { + lru, _ := lru.New(int(size)) + return &lruCache{memory, r, int(size), lru} +} + +func (l *lruCache) Put(k uint64, v interface{}) error { + if releaseMemoryIfNeeded(l.reporter, l.Free, l.memory.Free) { + l.cache.Add(k, v) + } + return nil +} + +func (l *lruCache) Get(k uint64) (interface{}, error) { + v, ok := l.cache.Get(k) + if !ok { + return nil, ErrKeyNotFound.New(k) + } + + return v, nil +} + +func (l *lruCache) Free() { + l.cache, _ = lru.New(l.size) +} + +func (l *lruCache) Dispose() { + l.memory = nil + l.cache = nil +} + +type rowsCache struct { + memory Freeable + reporter Reporter + rows []Row +} + +func newRowsCache(memory Freeable, r Reporter) *rowsCache { + return &rowsCache{memory, r, nil} +} + +func (c *rowsCache) Add(row Row) error { + if !releaseMemoryIfNeeded(c.reporter, c.memory.Free) { + return ErrNoMemoryAvailable.New() + } + + c.rows = append(c.rows, row) + return nil +} + +func (c *rowsCache) Get() []Row { return c.rows } + +func (c *rowsCache) Dispose() { + c.memory = nil + c.rows = nil +} + +type historyCache struct { + memory Freeable + reporter Reporter + cache map[uint64]interface{} +} + +func newHistoryCache(memory Freeable, r Reporter) *historyCache { + return &historyCache{memory, r, make(map[uint64]interface{})} +} + +func (h *historyCache) Put(k uint64, v interface{}) error { + if !releaseMemoryIfNeeded(h.reporter, h.memory.Free) { + return ErrNoMemoryAvailable.New() + } + h.cache[k] = v + return nil +} + +func (h *historyCache) Get(k uint64) (interface{}, error) { + v, ok := h.cache[k] + if !ok { + return nil, ErrKeyNotFound.New(k) + } + return v, nil +} + +func (h *historyCache) Dispose() { + h.memory = nil + h.cache = nil +} + +// releasesMemoryIfNeeded releases memory if needed using the following steps +// until there is available memory. It returns whether or not there was +// available memory after all the steps. +func releaseMemoryIfNeeded(r Reporter, steps ...func()) bool { + for _, s := range steps { + if HasAvailableMemory(r) { + return true + } + + s() + runtime.GC() + } + + return HasAvailableMemory(r) +} diff --git a/sql/cache_test.go b/sql/cache_test.go new file mode 100644 index 000000000..7984f7986 --- /dev/null +++ b/sql/cache_test.go @@ -0,0 +1,169 @@ +package sql + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCacheKey(t *testing.T) { + k := CacheKey(1) + require.Equal(t, uint64(0x4320000000000000), k) +} + +func TestLRUCache(t *testing.T) { + t.Run("basic methods", func(t *testing.T) { + require := require.New(t) + + cache := newLRUCache(mockMemory{}, fixedReporter(5, 50), 10) + + require.NoError(cache.Put(1, "foo")) + v, err := cache.Get(1) + require.NoError(err) + require.Equal("foo", v) + + _, err = cache.Get(2) + require.Error(err) + require.True(ErrKeyNotFound.Is(err)) + + // Free the cache and check previous entry disappeared. + cache.Free() + + _, err = cache.Get(1) + require.Error(err) + require.True(ErrKeyNotFound.Is(err)) + + cache.Dispose() + require.Panics(func() { + _, _ = cache.Get(1) + }) + }) + + t.Run("no memory available", func(t *testing.T) { + require := require.New(t) + cache := newLRUCache(mockMemory{}, fixedReporter(51, 50), 5) + + require.NoError(cache.Put(1, "foo")) + _, err := cache.Get(1) + require.Error(err) + require.True(ErrKeyNotFound.Is(err)) + }) + + t.Run("free required to add entry", func(t *testing.T) { + require := require.New(t) + var freed bool + cache := newLRUCache( + mockMemory{func() { + freed = true + }}, + mockReporter{func() uint64 { + if freed { + return 0 + } + return 51 + }, 50}, + 5, + ) + require.NoError(cache.Put(1, "foo")) + v, err := cache.Get(1) + require.NoError(err) + require.Equal("foo", v) + require.True(freed) + }) +} + +func TestHistoryCache(t *testing.T) { + t.Run("basic methods", func(t *testing.T) { + require := require.New(t) + + cache := newHistoryCache(mockMemory{}, fixedReporter(5, 50)) + + require.NoError(cache.Put(1, "foo")) + v, err := cache.Get(1) + require.NoError(err) + require.Equal("foo", v) + + _, err = cache.Get(2) + require.Error(err) + require.True(ErrKeyNotFound.Is(err)) + + cache.Dispose() + require.Panics(func() { + _ = cache.Put(2, "foo") + }) + }) + + t.Run("no memory available", func(t *testing.T) { + require := require.New(t) + cache := newHistoryCache(mockMemory{}, fixedReporter(51, 50)) + + err := cache.Put(1, "foo") + require.Error(err) + require.True(ErrNoMemoryAvailable.Is(err)) + }) + + t.Run("free required to add entry", func(t *testing.T) { + require := require.New(t) + var freed bool + cache := newHistoryCache( + mockMemory{func() { + freed = true + }}, + mockReporter{func() uint64 { + if freed { + return 0 + } + return 51 + }, 50}, + ) + require.NoError(cache.Put(1, "foo")) + v, err := cache.Get(1) + require.NoError(err) + require.Equal("foo", v) + require.True(freed) + }) +} + +func TestRowsCache(t *testing.T) { + t.Run("basic methods", func(t *testing.T) { + require := require.New(t) + + cache := newRowsCache(mockMemory{}, fixedReporter(5, 50)) + + require.NoError(cache.Add(Row{1})) + require.Len(cache.Get(), 1) + + cache.Dispose() + require.Panics(func() { + _ = cache.Add(Row{2}) + }) + }) + + t.Run("no memory available", func(t *testing.T) { + require := require.New(t) + cache := newRowsCache(mockMemory{}, fixedReporter(51, 50)) + + err := cache.Add(Row{1, "foo"}) + require.Error(err) + require.True(ErrNoMemoryAvailable.Is(err)) + }) + + t.Run("free required to add entry", func(t *testing.T) { + require := require.New(t) + var freed bool + cache := newRowsCache( + mockMemory{func() { + freed = true + }}, + mockReporter{func() uint64 { + if freed { + return 0 + } + return 51 + }, 50}, + ) + require.NoError(cache.Add(Row{1, "foo"})) + require.Len(cache.Get(), 1) + require.True(freed) + }) +} diff --git a/sql/catalog.go b/sql/catalog.go index 12683343b..ee3a48ade 100644 --- a/sql/catalog.go +++ b/sql/catalog.go @@ -5,6 +5,8 @@ import ( "strings" "sync" + "github.com/src-d/go-mysql-server/internal/similartext" + "gopkg.in/src-d/go-errors.v1" ) @@ -16,6 +18,7 @@ type Catalog struct { FunctionRegistry *IndexRegistry *ProcessList + *MemoryManager mu sync.RWMutex currentDatabase string @@ -34,6 +37,7 @@ func NewCatalog() *Catalog { return &Catalog{ FunctionRegistry: NewFunctionRegistry(), IndexRegistry: NewIndexRegistry(), + MemoryManager: NewMemoryManager(ProcessMemory), ProcessList: NewProcessList(), locks: make(sessionLocks), } @@ -93,14 +97,21 @@ type Databases []Database // Database returns the Database with the given name if it exists. func (d Databases) Database(name string) (Database, error) { + + if len(d) == 0 { + return nil, ErrDatabaseNotFound.New(name) + } + name = strings.ToLower(name) + var dbNames []string for _, db := range d { if strings.ToLower(db.Name()) == name { return db, nil } + dbNames = append(dbNames, db.Name()) } - - return nil, ErrDatabaseNotFound.New(name) + similar := similartext.Find(dbNames, name) + return nil, ErrDatabaseNotFound.New(name + similar) } // Add adds a new database. @@ -118,6 +129,10 @@ func (d Databases) Table(dbName string, tableName string) (Table, error) { tableName = strings.ToLower(tableName) tables := db.Tables() + if len(tables) == 0 { + return nil, ErrTableNotFound.New(tableName) + } + // Try to get the table by key, but if the name is not the same, // then use the slow path and iterate over all tables comparing // the name. @@ -129,7 +144,8 @@ func (d Databases) Table(dbName string, tableName string) (Table, error) { } } - return nil, ErrTableNotFound.New(tableName) + similar := similartext.FindFromMap(tables, tableName) + return nil, ErrTableNotFound.New(tableName + similar) } return table, nil @@ -164,8 +180,8 @@ func (c *Catalog) UnlockTables(ctx *Context, id uint32) error { table, err := c.dbs.Table(db, t) if err == nil { if lockable, ok := table.(Lockable); ok { - if err := lockable.Unlock(ctx, id); err != nil { - errors = append(errors, err.Error()) + if e := lockable.Unlock(ctx, id); e != nil { + errors = append(errors, e.Error()) } } } else { diff --git a/sql/catalog_test.go b/sql/catalog_test.go index 6e5e9b9fa..3aef282e4 100644 --- a/sql/catalog_test.go +++ b/sql/catalog_test.go @@ -3,9 +3,9 @@ package sql_test import ( "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestCatalogCurrentDatabase(t *testing.T) { @@ -14,7 +14,7 @@ func TestCatalogCurrentDatabase(t *testing.T) { c := sql.NewCatalog() require.Equal("", c.CurrentDatabase()) - c.AddDatabase(mem.NewDatabase("foo")) + c.AddDatabase(memory.NewDatabase("foo")) require.Equal("foo", c.CurrentDatabase()) c.SetCurrentDatabase("bar") @@ -25,9 +25,9 @@ func TestAllDatabases(t *testing.T) { require := require.New(t) var dbs = sql.Databases{ - mem.NewDatabase("a"), - mem.NewDatabase("b"), - mem.NewDatabase("c"), + memory.NewDatabase("a"), + memory.NewDatabase("b"), + memory.NewDatabase("c"), } c := sql.NewCatalog() @@ -46,9 +46,13 @@ func TestCatalogDatabase(t *testing.T) { require.EqualError(err, "database not found: foo") require.Nil(db) - mydb := mem.NewDatabase("foo") + mydb := memory.NewDatabase("foo") c.AddDatabase(mydb) + db, err = c.Database("flo") + require.EqualError(err, "database not found: flo, maybe you mean foo?") + require.Nil(db) + db, err = c.Database("foo") require.NoError(err) require.Equal(mydb, db) @@ -63,16 +67,20 @@ func TestCatalogTable(t *testing.T) { require.EqualError(err, "database not found: foo") require.Nil(table) - db := mem.NewDatabase("foo") + db := memory.NewDatabase("foo") c.AddDatabase(db) table, err = c.Table("foo", "bar") require.EqualError(err, "table not found: bar") require.Nil(table) - mytable := mem.NewTable("bar", nil) + mytable := memory.NewTable("bar", nil) db.AddTable("bar", mytable) + table, err = c.Table("foo", "baz") + require.EqualError(err, "table not found: baz, maybe you mean bar?") + require.Nil(table) + table, err = c.Table("foo", "bar") require.NoError(err) require.Equal(mytable, table) @@ -85,9 +93,9 @@ func TestCatalogTable(t *testing.T) { func TestCatalogUnlockTables(t *testing.T) { require := require.New(t) - db := mem.NewDatabase("db") - t1 := newLockableTable(mem.NewTable("t1", nil)) - t2 := newLockableTable(mem.NewTable("t2", nil)) + db := memory.NewDatabase("db") + t1 := newLockableTable(memory.NewTable("t1", nil)) + t2 := newLockableTable(memory.NewTable("t2", nil)) db.AddTable("t1", t1) db.AddTable("t2", t2) diff --git a/sql/core.go b/sql/core.go index ed8bf612c..16ef6fa8d 100644 --- a/sql/core.go +++ b/sql/core.go @@ -1,8 +1,11 @@ -package sql // import "gopkg.in/src-d/go-mysql-server.v0/sql" +package sql import ( "fmt" "io" + "math" + "strconv" + "time" "gopkg.in/src-d/go-errors.v1" ) @@ -22,6 +25,13 @@ var ( //ErrUnexpectedRowLength is thrown when the obtained row has more columns than the schema ErrUnexpectedRowLength = errors.NewKind("expected %d values, got %d") + + // ErrInvalidChildrenNumber is returned when the WithChildren method of a + // node or expression is called with an invalid number of arguments. + ErrInvalidChildrenNumber = errors.NewKind("%T: invalid children number, got %d, expected %d") + + // ErrDeleteRowNotFound + ErrDeleteRowNotFound = errors.NewKind("row was not found when attempting to delete").New() ) // Nameable is something that has a name. @@ -42,17 +52,6 @@ type Resolvable interface { Resolved() bool } -// Transformable is a node which can be transformed. -type Transformable interface { - // TransformUp transforms all nodes and returns the result of this transformation. - // Transformation is not propagated to subqueries. - TransformUp(TransformNodeFunc) (Node, error) - // TransformExpressionsUp transforms all expressions inside the node and all its - // children and returns a node with the result of the transformations. - // Transformation is not propagated to subqueries. - TransformExpressionsUp(TransformExprFunc) (Node, error) -} - // TransformNodeFunc is a function that given a node will return that node // as is or transformed along with an error, if any. type TransformNodeFunc func(Node) (Node, error) @@ -71,11 +70,13 @@ type Expression interface { IsNullable() bool // Eval evaluates the given row and returns a result. Eval(*Context, Row) (interface{}, error) - // TransformUp transforms the expression and all its children with the - // given transform function. - TransformUp(TransformExprFunc) (Expression, error) // Children returns the children expressions of this expression. Children() []Expression + // WithChildren returns a copy of the expression with children replaced. + // It will return an error if the number of children is different than + // the current number of children. They must be given in the same order + // as they are returned by Children. + WithChildren(...Expression) (Expression, error) } // Aggregation implements an aggregation expression, where an @@ -97,7 +98,6 @@ type Aggregation interface { // Node is a node in the execution plan tree. type Node interface { Resolvable - Transformable fmt.Stringer // Schema of the node. Schema() Schema @@ -105,16 +105,36 @@ type Node interface { Children() []Node // RowIter produces a row iterator from this node. RowIter(*Context) (RowIter, error) + // WithChildren returns a copy of the node with children replaced. + // It will return an error if the number of children is different than + // the current number of children. They must be given in the same order + // as they are returned by Children. + WithChildren(...Node) (Node, error) +} + +// OpaqueNode is a node that doesn't allow transformations to its children and +// acts a a black box. +type OpaqueNode interface { + Node + // Opaque reports whether the node is opaque or not. + Opaque() bool +} + +// AsyncNode is a node that can be executed asynchronously. +type AsyncNode interface { + // IsAsync reports whether the node is async or not. + IsAsync() bool } // Expressioner is a node that contains expressions. type Expressioner interface { // Expressions returns the list of expressions contained by the node. Expressions() []Expression - // TransformExpressions applies for each expression in this node - // the expression's TransformUp method with the given function, and - // return a new node with the transformed expressions. - TransformExpressions(TransformExprFunc) (Node, error) + // WithExpressions returns a copy of the node with expressions replaced. + // It will return an error if the number of expressions is different than + // the current number of expressions. They must be given in the same order + // as they are returned by Expressions. + WithExpressions(...Expression) (Node, error) } // Databaser is a node that contains a reference to a database. @@ -191,6 +211,24 @@ type Inserter interface { Insert(*Context, Row) error } +// Deleter allow rows to be deleted from tables. +type Deleter interface { + // Delete the given row. Returns ErrDeleteRowNotFound if the row was not found. + Delete(*Context, Row) error +} + +// Replacer allows rows to be replaced through a Delete (if applicable) then Insert. +type Replacer interface { + Deleter + Inserter +} + +// Updater allows rows to be updated. +type Updater interface { + // Update the given row. Provides both the old and new rows. + Update(ctx *Context, old Row, new Row) error +} + // Database represents the database. type Database interface { Nameable @@ -198,9 +236,14 @@ type Database interface { Tables() map[string]Table } -// Alterable should be implemented by databases that can handle DDL statements -type Alterable interface { - Create(name string, schema Schema) error +// TableCreator should be implemented by databases that can create new tables. +type TableCreator interface { + CreateTable(ctx *Context, name string, schema Schema) error +} + +// TableDropper should be implemented by databases that can drop tables. +type TableDropper interface { + DropTable(ctx *Context, name string) error } // Lockable should be implemented by tables that can be locked and unlocked. @@ -218,3 +261,50 @@ type Lockable interface { // available. Unlock(ctx *Context, id uint32) error } + +// EvaluateCondition evaluates a condition, which is an expression whose value +// will be coerced to boolean. +func EvaluateCondition(ctx *Context, cond Expression, row Row) (bool, error) { + v, err := cond.Eval(ctx, row) + if err != nil { + return false, err + } + + switch b := v.(type) { + case bool: + return b, nil + case int: + return b != int(0), nil + case int64: + return b != int64(0), nil + case int32: + return b != int32(0), nil + case int16: + return b != int16(0), nil + case int8: + return b != int8(0), nil + case uint: + return b != uint(0), nil + case uint64: + return b != uint64(0), nil + case uint32: + return b != uint32(0), nil + case uint16: + return b != uint16(0), nil + case uint8: + return b != uint8(0), nil + case time.Duration: + return int64(b) != 0, nil + case time.Time: + return b.UnixNano() != 0, nil + case float64: + return int(math.Round(v.(float64))) != 0, nil + case float32: + return int(math.Round(float64(v.(float32)))) != 0, nil + case string: + parsed, err := strconv.ParseFloat(v.(string), 64) + return err == nil && int(parsed) != 0, nil + default: + return false, nil + } +} diff --git a/sql/core_test.go b/sql/core_test.go new file mode 100644 index 000000000..cf3e23acd --- /dev/null +++ b/sql/core_test.go @@ -0,0 +1,49 @@ +package sql_test + +import ( + "fmt" + "testing" + "time" + + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/stretchr/testify/require" +) + +var conditions = []struct { + evaluated bool + value interface{} + t sql.Type +}{ + {true, int16(1), sql.Int16}, + {false, int16(0), sql.Int16}, + {true, int32(1), sql.Int32}, + {false, int32(0), sql.Int32}, + {true, int(1), sql.Int64}, + {false, int(0), sql.Int64}, + {true, float32(1), sql.Float32}, + {true, float64(1), sql.Float64}, + {false, float32(0), sql.Float32}, + {false, float64(0), sql.Float64}, + {true, float32(0.5), sql.Float32}, + {true, float64(0.5), sql.Float64}, + {true, "1", sql.Text}, + {false, "0", sql.Text}, + {false, "foo", sql.Text}, + {false, "0.5", sql.Text}, + {false, time.Duration(0), sql.Timestamp}, + {true, time.Duration(1), sql.Timestamp}, + {false, false, sql.Boolean}, + {true, true, sql.Boolean}, +} + +func TestEvaluateCondition(t *testing.T) { + for _, v := range conditions { + t.Run(fmt.Sprint(v.value, " evaluated to ", v.evaluated, " type ", v.t), func(t *testing.T) { + require := require.New(t) + b, err := sql.EvaluateCondition(sql.NewEmptyContext(), expression.NewLiteral(v.value, v.t), sql.NewRow()) + require.NoError(err) + require.Equal(v.evaluated, b) + }) + } +} diff --git a/sql/expression/alias.go b/sql/expression/alias.go index ee382b6d3..c7485dfd9 100644 --- a/sql/expression/alias.go +++ b/sql/expression/alias.go @@ -3,7 +3,7 @@ package expression import ( "fmt" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // Alias is a node that gives a name to an expression. @@ -31,13 +31,12 @@ func (e *Alias) String() string { return fmt.Sprintf("%s as %s", e.Child, e.name) } -// TransformUp implements the Expression interface. -func (e *Alias) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - child, err := e.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (e *Alias) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(e, len(children), 1) } - return f(NewAlias(child, e.name)) + return NewAlias(children[0], e.name), nil } // Name implements the Nameable interface. diff --git a/sql/expression/arithmetic.go b/sql/expression/arithmetic.go index 005bd6260..d7044e06e 100644 --- a/sql/expression/arithmetic.go +++ b/sql/expression/arithmetic.go @@ -3,11 +3,12 @@ package expression import ( "fmt" "reflect" + "time" errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-vitess.v1/vt/sqlparser" + "vitess.io/vitess/go/vt/sqlparser" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) var ( @@ -21,7 +22,7 @@ var ( // Arithmetic expressions (+, -, *, /, ...) type Arithmetic struct { BinaryExpression - op string + Op string } // NewArithmetic creates a new Arithmetic sql.Expression. @@ -85,13 +86,30 @@ func NewMod(left, right sql.Expression) *Arithmetic { } func (a *Arithmetic) String() string { - return fmt.Sprintf("%s %s %s", a.Left, a.op, a.Right) + return fmt.Sprintf("%s %s %s", a.Left, a.Op, a.Right) +} + +// IsNullable implements the sql.Expression interface. +func (a *Arithmetic) IsNullable() bool { + if a.Type() == sql.Timestamp { + return true + } + + return a.BinaryExpression.IsNullable() } // Type returns the greatest type for given operation. func (a *Arithmetic) Type() sql.Type { - switch a.op { + switch a.Op { case sqlparser.PlusStr, sqlparser.MinusStr, sqlparser.MultStr, sqlparser.DivStr: + if isInterval(a.Left) || isInterval(a.Right) { + return sql.Timestamp + } + + if sql.IsTime(a.Left.Type()) && sql.IsTime(a.Right.Type()) { + return sql.Int64 + } + if sql.IsInteger(a.Left.Type()) && sql.IsInteger(a.Right.Type()) { if sql.IsUnsigned(a.Left.Type()) && sql.IsUnsigned(a.Right.Type()) { return sql.Uint64 @@ -114,19 +132,17 @@ func (a *Arithmetic) Type() sql.Type { return sql.Float64 } -// TransformUp implements the Expression interface. -func (a *Arithmetic) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - l, err := a.Left.TransformUp(f) - if err != nil { - return nil, err - } +func isInterval(expr sql.Expression) bool { + _, ok := expr.(*Interval) + return ok +} - r, err := a.Right.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (a *Arithmetic) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 2 { + return nil, sql.ErrInvalidChildrenNumber.New(a, len(children), 2) } - - return f(NewArithmetic(l, r, a.op)) + return NewArithmetic(children[0], children[1], a.Op), nil } // Eval implements the Expression interface. @@ -136,12 +152,16 @@ func (a *Arithmetic) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return nil, err } + if lval == nil || rval == nil { + return nil, nil + } + lval, rval, err = a.convertLeftRight(lval, rval) if err != nil { return nil, err } - switch a.op { + switch a.Op { case sqlparser.PlusStr: return plus(lval, rval) case sqlparser.MinusStr: @@ -166,37 +186,63 @@ func (a *Arithmetic) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return mod(lval, rval) } - return nil, errUnableToEval.New(lval, a.op, rval) + return nil, errUnableToEval.New(lval, a.Op, rval) } func (a *Arithmetic) evalLeftRight(ctx *sql.Context, row sql.Row) (interface{}, interface{}, error) { - lval, err := a.Left.Eval(ctx, row) - if err != nil { - return nil, nil, err + var lval, rval interface{} + var err error + + if i, ok := a.Left.(*Interval); ok { + lval, err = i.EvalDelta(ctx, row) + if err != nil { + return nil, nil, err + } + } else { + lval, err = a.Left.Eval(ctx, row) + if err != nil { + return nil, nil, err + } } - rval, err := a.Right.Eval(ctx, row) - if err != nil { - return nil, nil, err + if i, ok := a.Right.(*Interval); ok { + rval, err = i.EvalDelta(ctx, row) + if err != nil { + return nil, nil, err + } + } else { + rval, err = a.Right.Eval(ctx, row) + if err != nil { + return nil, nil, err + } } return lval, rval, nil } -func (a *Arithmetic) convertLeftRight(lval interface{}, rval interface{}) (interface{}, interface{}, error) { +func (a *Arithmetic) convertLeftRight(left interface{}, right interface{}) (interface{}, interface{}, error) { + var err error typ := a.Type() - lval64, err := typ.Convert(lval) - if err != nil { - return nil, nil, err + if i, ok := left.(*TimeDelta); ok { + left = i + } else { + left, err = typ.Convert(left) + if err != nil { + return nil, nil, err + } } - rval64, err := typ.Convert(rval) - if err != nil { - return nil, nil, err + if i, ok := right.(*TimeDelta); ok { + right = i + } else { + right, err = typ.Convert(right) + if err != nil { + return nil, nil, err + } } - return lval64, rval64, nil + return left, right, nil } func plus(lval, rval interface{}) (interface{}, error) { @@ -218,6 +264,18 @@ func plus(lval, rval interface{}) (interface{}, error) { case float64: return l + r, nil } + case time.Time: + switch r := rval.(type) { + case *TimeDelta: + return sql.ValidateTime(r.Add(l)), nil + case time.Time: + return l.Unix() + r.Unix(), nil + } + case *TimeDelta: + switch r := rval.(type) { + case time.Time: + return sql.ValidateTime(l.Add(r)), nil + } } return nil, errUnableToCast.New(lval, rval) @@ -242,6 +300,13 @@ func minus(lval, rval interface{}) (interface{}, error) { case float64: return l - r, nil } + case time.Time: + switch r := rval.(type) { + case *TimeDelta: + return sql.ValidateTime(r.Sub(l)), nil + case time.Time: + return l.Unix() - r.Unix(), nil + } } return nil, errUnableToCast.New(lval, rval) @@ -477,12 +542,10 @@ func (e *UnaryMinus) String() string { return fmt.Sprintf("-%s", e.Child) } -// TransformUp implements the sql.Expression interface. -func (e *UnaryMinus) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - c, err := e.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (e *UnaryMinus) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(e, len(children), 1) } - - return f(NewUnaryMinus(c)) + return NewUnaryMinus(children[0]), nil } diff --git a/sql/expression/arithmetic_test.go b/sql/expression/arithmetic_test.go index d56047504..811039734 100644 --- a/sql/expression/arithmetic_test.go +++ b/sql/expression/arithmetic_test.go @@ -2,9 +2,10 @@ package expression import ( "testing" + "time" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestPlus(t *testing.T) { @@ -38,6 +39,29 @@ func TestPlus(t *testing.T) { require.Equal(float64(5), result) } +func TestPlusInterval(t *testing.T) { + require := require.New(t) + + expected := time.Date(2018, time.May, 2, 0, 0, 0, 0, time.UTC) + op := NewPlus( + NewLiteral("2018-05-01", sql.Text), + NewInterval(NewLiteral(int64(1), sql.Int64), "DAY"), + ) + + result, err := op.Eval(sql.NewEmptyContext(), nil) + require.NoError(err) + require.Equal(expected, result) + + op = NewPlus( + NewInterval(NewLiteral(int64(1), sql.Int64), "DAY"), + NewLiteral("2018-05-01", sql.Text), + ) + + result, err = op.Eval(sql.NewEmptyContext(), nil) + require.NoError(err) + require.Equal(expected, result) +} + func TestMinus(t *testing.T) { var testCases = []struct { name string @@ -69,6 +93,20 @@ func TestMinus(t *testing.T) { require.Equal(float64(0), result) } +func TestMinusInterval(t *testing.T) { + require := require.New(t) + + expected := time.Date(2018, time.May, 1, 0, 0, 0, 0, time.UTC) + op := NewMinus( + NewLiteral("2018-05-02", sql.Text), + NewInterval(NewLiteral(int64(1), sql.Int64), "DAY"), + ) + + result, err := op.Eval(sql.NewEmptyContext(), nil) + require.NoError(err) + require.Equal(expected, result) +} + func TestMult(t *testing.T) { var testCases = []struct { name string diff --git a/sql/expression/between.go b/sql/expression/between.go index 15ad0027e..15114890b 100644 --- a/sql/expression/between.go +++ b/sql/expression/between.go @@ -3,7 +3,7 @@ package expression import ( "fmt" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // Between checks a value is between two given values. @@ -98,22 +98,10 @@ func (b *Between) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return cmpLower >= 0 && cmpUpper <= 0, nil } -// TransformUp implements the Expression interface. -func (b *Between) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - val, err := b.Val.TransformUp(f) - if err != nil { - return nil, err - } - - lower, err := b.Lower.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (b *Between) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 3 { + return nil, sql.ErrInvalidChildrenNumber.New(b, len(children), 3) } - - upper, err := b.Upper.TransformUp(f) - if err != nil { - return nil, err - } - - return f(NewBetween(val, lower, upper)) + return NewBetween(children[0], children[1], children[2]), nil } diff --git a/sql/expression/between_test.go b/sql/expression/between_test.go index eb0ced088..58fd7bb1a 100644 --- a/sql/expression/between_test.go +++ b/sql/expression/between_test.go @@ -3,8 +3,8 @@ package expression import ( "testing" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestBetween(t *testing.T) { diff --git a/sql/expression/boolean.go b/sql/expression/boolean.go index 5030f8ac4..73815fb7d 100644 --- a/sql/expression/boolean.go +++ b/sql/expression/boolean.go @@ -2,8 +2,9 @@ package expression import ( "fmt" + "reflect" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // Not is a node that negates an expression. @@ -32,18 +33,29 @@ func (e *Not) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return nil, nil } - return !v.(bool), nil + b, ok := v.(bool) + if !ok { + v, _ = e.Type().Convert(v) + if v == nil { + return nil, nil + } + + if b, ok = v.(bool); !ok { + return nil, sql.ErrInvalidType.New(reflect.TypeOf(v).String()) + } + } + + return !b, nil } func (e *Not) String() string { return fmt.Sprintf("NOT(%s)", e.Child) } -// TransformUp implements the Expression interface. -func (e *Not) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - child, err := e.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (e *Not) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(e, len(children), 1) } - return f(NewNot(child)) + return NewNot(children[0]), nil } diff --git a/sql/expression/boolean_test.go b/sql/expression/boolean_test.go index cb69b0d2c..aff618485 100644 --- a/sql/expression/boolean_test.go +++ b/sql/expression/boolean_test.go @@ -2,9 +2,10 @@ package expression import ( "testing" + "time" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestNot(t *testing.T) { @@ -14,4 +15,9 @@ func TestNot(t *testing.T) { require.False(eval(t, e, sql.NewRow(true)).(bool)) require.True(eval(t, e, sql.NewRow(false)).(bool)) require.Nil(eval(t, e, sql.NewRow(nil))) + require.False(eval(t, e, sql.NewRow(1)).(bool)) + require.True(eval(t, e, sql.NewRow(0)).(bool)) + require.False(eval(t, e, sql.NewRow(time.Now())).(bool)) + require.False(eval(t, e, sql.NewRow(time.Second)).(bool)) + require.True(eval(t, e, sql.NewRow("any string always false")).(bool)) } diff --git a/sql/expression/case.go b/sql/expression/case.go index 77d751a64..feb5f15f3 100644 --- a/sql/expression/case.go +++ b/sql/expression/case.go @@ -3,7 +3,7 @@ package expression import ( "bytes" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // CaseBranch is a single branch of a case expression. @@ -28,9 +28,16 @@ func NewCase(expr sql.Expression, branches []CaseBranch, elseExpr sql.Expression // Type implements the sql.Expression interface. func (c *Case) Type() sql.Type { for _, b := range c.Branches { - return b.Value.Type() + if b.Value.Type() != sql.Null { + return b.Value.Type() + } + } + + if c.Else.Type() != sql.Null { + return c.Else.Type() } - return c.Else.Type() + + return sql.Null } // IsNullable implements the sql.Expression interface. @@ -101,17 +108,12 @@ func (c *Case) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { cond = b.Cond } - v, err := cond.Eval(ctx, row) - if err != nil { - return nil, err - } - - v, err = sql.Boolean.Convert(v) + ok, err := sql.EvaluateCondition(ctx, cond, row) if err != nil { return nil, err } - if v == true { + if ok { return b.Value.Eval(ctx, row) } } @@ -123,44 +125,41 @@ func (c *Case) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return nil, nil } -// TransformUp implements the sql.Expression interface. -func (c *Case) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - var expr sql.Expression - var err error - +// WithChildren implements the Expression interface. +func (c *Case) WithChildren(children ...sql.Expression) (sql.Expression, error) { + var expected = len(c.Branches) * 2 if c.Expr != nil { - expr, err = c.Expr.TransformUp(f) - if err != nil { - return nil, err - } + expected++ } - var branches []CaseBranch - for _, b := range c.Branches { - var nb CaseBranch - - nb.Cond, err = b.Cond.TransformUp(f) - if err != nil { - return nil, err - } + if c.Else != nil { + expected++ + } - nb.Value, err = b.Value.TransformUp(f) - if err != nil { - return nil, err - } + if len(children) != expected { + return nil, sql.ErrInvalidChildrenNumber.New(c, len(children), expected) + } - branches = append(branches, nb) + var expr, elseExpr sql.Expression + if c.Expr != nil { + expr = children[0] + children = children[1:] } - var elseExpr sql.Expression if c.Else != nil { - elseExpr, err = c.Else.TransformUp(f) - if err != nil { - return nil, err - } + elseExpr = children[len(children)-1] + children = children[:len(children)-1] + } + + var branches []CaseBranch + for i := 0; i < len(children); i += 2 { + branches = append(branches, CaseBranch{ + Cond: children[i], + Value: children[i+1], + }) } - return f(NewCase(expr, branches, elseExpr)) + return NewCase(expr, branches, elseExpr), nil } func (c *Case) String() string { diff --git a/sql/expression/case_test.go b/sql/expression/case_test.go index 3168bd830..80a24aa47 100644 --- a/sql/expression/case_test.go +++ b/sql/expression/case_test.go @@ -3,8 +3,8 @@ package expression import ( "testing" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestCase(t *testing.T) { @@ -127,3 +127,20 @@ func TestCase(t *testing.T) { }) } } + +func TestCaseNullBranch(t *testing.T) { + require := require.New(t) + f := NewCase( + NewGetField(0, sql.Int64, "x", false), + []CaseBranch{ + { + Cond: NewLiteral(int64(1), sql.Int64), + Value: NewLiteral(nil, sql.Null), + }, + }, + nil, + ) + result, err := f.Eval(sql.NewEmptyContext(), sql.Row{int64(1)}) + require.NoError(err) + require.Nil(result) +} diff --git a/sql/expression/common.go b/sql/expression/common.go index 8d1a5ef45..b124f269e 100644 --- a/sql/expression/common.go +++ b/sql/expression/common.go @@ -1,7 +1,7 @@ package expression import ( - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // IsUnary returns whether the expression is unary or not. diff --git a/sql/expression/common_test.go b/sql/expression/common_test.go index 571008c69..e00d3270f 100644 --- a/sql/expression/common_test.go +++ b/sql/expression/common_test.go @@ -3,8 +3,8 @@ package expression import ( "testing" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func eval(t *testing.T, e sql.Expression, row sql.Row) interface{} { diff --git a/sql/expression/comparison.go b/sql/expression/comparison.go index 2c52b78a8..87601579b 100644 --- a/sql/expression/comparison.go +++ b/sql/expression/comparison.go @@ -4,9 +4,9 @@ import ( "fmt" "sync" + "github.com/src-d/go-mysql-server/internal/regex" + "github.com/src-d/go-mysql-server/sql" errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/internal/regex" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) // Comparer implements a comparison expression. @@ -71,32 +71,32 @@ func (c *comparison) evalLeftAndRight(ctx *sql.Context, row sql.Row) (interface{ func (c *comparison) castLeftAndRight(left, right interface{}) (interface{}, interface{}, error) { if sql.IsNumber(c.Left().Type()) || sql.IsNumber(c.Right().Type()) { if sql.IsDecimal(c.Left().Type()) || sql.IsDecimal(c.Right().Type()) { - left, right, err := convertLeftAndRight(left, right, ConvertToDecimal) + l, r, err := convertLeftAndRight(left, right, ConvertToDecimal) if err != nil { return nil, nil, err } c.compareType = sql.Float64 - return left, right, nil + return l, r, nil } if sql.IsSigned(c.Left().Type()) || sql.IsSigned(c.Right().Type()) { - left, right, err := convertLeftAndRight(left, right, ConvertToSigned) + l, r, err := convertLeftAndRight(left, right, ConvertToSigned) if err != nil { return nil, nil, err } c.compareType = sql.Int64 - return left, right, nil + return l, r, nil } - left, right, err := convertLeftAndRight(left, right, ConvertToUnsigned) + l, r, err := convertLeftAndRight(left, right, ConvertToUnsigned) if err != nil { return nil, nil, err } c.compareType = sql.Uint64 - return left, right, nil + return l, r, nil } left, right, err := convertLeftAndRight(left, right, ConvertToChar) @@ -157,19 +157,12 @@ func (e *Equals) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return result == 0, nil } -// TransformUp implements the Expression interface. -func (e *Equals) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - left, err := e.Left().TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (e *Equals) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 2 { + return nil, sql.ErrInvalidChildrenNumber.New(e, len(children), 2) } - - right, err := e.Right().TransformUp(f) - if err != nil { - return nil, err - } - - return f(NewEquals(left, right)) + return NewEquals(children[0], children[1]), nil } func (e *Equals) String() string { @@ -229,8 +222,9 @@ func (re *Regexp) compareRegexp(ctx *sql.Context, row sql.Row) (interface{}, err } var ( - matcher regex.Matcher - right interface{} + matcher regex.Matcher + disposer regex.Disposer + right interface{} ) // eval right and convert to text if !re.cached || re.pool == nil { @@ -245,12 +239,12 @@ func (re *Regexp) compareRegexp(ctx *sql.Context, row sql.Row) (interface{}, err } // for non-cached regex every time create a new matcher if !re.cached { - matcher, err = regex.New(regex.Default(), right.(string)) + matcher, disposer, err = regex.New(regex.Default(), right.(string)) } else { if re.pool == nil { re.pool = &sync.Pool{ New: func() interface{} { - r, e := regex.New(regex.Default(), right.(string)) + r, _, e := regex.New(regex.Default(), right.(string)) if e != nil { err = e return nil @@ -268,25 +262,21 @@ func (re *Regexp) compareRegexp(ctx *sql.Context, row sql.Row) (interface{}, err } ok := matcher.Match(left.(string)) - if re.pool != nil && re.cached { + + if !re.cached { + disposer.Dispose() + } else if re.pool != nil { re.pool.Put(matcher) } return ok, nil } -// TransformUp implements the Expression interface. -func (re *Regexp) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - left, err := re.Left().TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (re *Regexp) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 2 { + return nil, sql.ErrInvalidChildrenNumber.New(re, len(children), 2) } - - right, err := re.Right().TransformUp(f) - if err != nil { - return nil, err - } - - return f(NewRegexp(left, right)) + return NewRegexp(children[0], children[1]), nil } func (re *Regexp) String() string { @@ -317,19 +307,12 @@ func (gt *GreaterThan) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) return result == 1, nil } -// TransformUp implements the Expression interface. -func (gt *GreaterThan) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - left, err := gt.Left().TransformUp(f) - if err != nil { - return nil, err - } - - right, err := gt.Right().TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (gt *GreaterThan) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 2 { + return nil, sql.ErrInvalidChildrenNumber.New(gt, len(children), 2) } - - return f(NewGreaterThan(left, right)) + return NewGreaterThan(children[0], children[1]), nil } func (gt *GreaterThan) String() string { @@ -360,19 +343,12 @@ func (lt *LessThan) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return result == -1, nil } -// TransformUp implements the Expression interface. -func (lt *LessThan) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - left, err := lt.Left().TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (lt *LessThan) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 2 { + return nil, sql.ErrInvalidChildrenNumber.New(lt, len(children), 2) } - - right, err := lt.Right().TransformUp(f) - if err != nil { - return nil, err - } - - return f(NewLessThan(left, right)) + return NewLessThan(children[0], children[1]), nil } func (lt *LessThan) String() string { @@ -404,19 +380,12 @@ func (gte *GreaterThanOrEqual) Eval(ctx *sql.Context, row sql.Row) (interface{}, return result > -1, nil } -// TransformUp implements the Expression interface. -func (gte *GreaterThanOrEqual) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - left, err := gte.Left().TransformUp(f) - if err != nil { - return nil, err - } - - right, err := gte.Right().TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (gte *GreaterThanOrEqual) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 2 { + return nil, sql.ErrInvalidChildrenNumber.New(gte, len(children), 2) } - - return f(NewGreaterThanOrEqual(left, right)) + return NewGreaterThanOrEqual(children[0], children[1]), nil } func (gte *GreaterThanOrEqual) String() string { @@ -448,19 +417,12 @@ func (lte *LessThanOrEqual) Eval(ctx *sql.Context, row sql.Row) (interface{}, er return result < 1, nil } -// TransformUp implements the Expression interface. -func (lte *LessThanOrEqual) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - left, err := lte.Left().TransformUp(f) - if err != nil { - return nil, err - } - - right, err := lte.Right().TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (lte *LessThanOrEqual) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 2 { + return nil, sql.ErrInvalidChildrenNumber.New(lte, len(children), 2) } - - return f(NewLessThanOrEqual(left, right)) + return NewLessThanOrEqual(children[0], children[1]), nil } func (lte *LessThanOrEqual) String() string { @@ -504,7 +466,6 @@ func (in *In) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return nil, err } - // TODO: support subqueries switch right := in.Right().(type) { case Tuple: for _, el := range right { @@ -534,25 +495,46 @@ func (in *In) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { } } + return false, nil + case *Subquery: + if leftElems > 1 { + return nil, ErrInvalidOperandColumns.New(leftElems, 1) + } + + typ := right.Type() + values, err := right.EvalMultiple(ctx) + if err != nil { + return nil, err + } + + for _, val := range values { + val, err = typ.Convert(val) + if err != nil { + return nil, err + } + + cmp, err := typ.Compare(left, val) + if err != nil { + return nil, err + } + + if cmp == 0 { + return true, nil + } + } + return false, nil default: return nil, ErrUnsupportedInOperand.New(right) } } -// TransformUp implements the Expression interface. -func (in *In) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - left, err := in.Left().TransformUp(f) - if err != nil { - return nil, err - } - - right, err := in.Right().TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (in *In) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 2 { + return nil, sql.ErrInvalidChildrenNumber.New(in, len(children), 2) } - - return f(NewIn(left, right)) + return NewIn(children[0], children[1]), nil } func (in *In) String() string { @@ -592,7 +574,6 @@ func (in *NotIn) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return nil, err } - // TODO: support subqueries switch right := in.Right().(type) { case Tuple: for _, el := range right { @@ -622,25 +603,46 @@ func (in *NotIn) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { } } + return true, nil + case *Subquery: + if leftElems > 1 { + return nil, ErrInvalidOperandColumns.New(leftElems, 1) + } + + typ := right.Type() + values, err := right.EvalMultiple(ctx) + if err != nil { + return nil, err + } + + for _, val := range values { + val, err = typ.Convert(val) + if err != nil { + return nil, err + } + + cmp, err := typ.Compare(left, val) + if err != nil { + return nil, err + } + + if cmp == 0 { + return false, nil + } + } + return true, nil default: return nil, ErrUnsupportedInOperand.New(right) } } -// TransformUp implements the Expression interface. -func (in *NotIn) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - left, err := in.Left().TransformUp(f) - if err != nil { - return nil, err - } - - right, err := in.Right().TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (in *NotIn) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 2 { + return nil, sql.ErrInvalidChildrenNumber.New(in, len(children), 2) } - - return f(NewNotIn(left, right)) + return NewNotIn(children[0], children[1]), nil } func (in *NotIn) String() string { diff --git a/sql/expression/comparison_test.go b/sql/expression/comparison_test.go index 20129e8b5..802c3676a 100644 --- a/sql/expression/comparison_test.go +++ b/sql/expression/comparison_test.go @@ -1,11 +1,14 @@ -package expression +package expression_test import ( "testing" + "github.com/src-d/go-mysql-server/internal/regex" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/internal/regex" - "gopkg.in/src-d/go-mysql-server.v0/sql" "github.com/stretchr/testify/require" ) @@ -95,11 +98,11 @@ var likeComparisonCases = map[sql.Type]map[int][][]interface{}{ func TestEquals(t *testing.T) { require := require.New(t) for resultType, cmpCase := range comparisonCases { - get0 := NewGetField(0, resultType, "col1", true) + get0 := expression.NewGetField(0, resultType, "col1", true) require.NotNil(get0) - get1 := NewGetField(1, resultType, "col2", true) + get1 := expression.NewGetField(1, resultType, "col2", true) require.NotNil(get1) - eq := NewEquals(get0, get1) + eq := expression.NewEquals(get0, get1) require.NotNil(eq) require.Equal(sql.Boolean, eq.Type()) for cmpResult, cases := range cmpCase { @@ -122,11 +125,11 @@ func TestEquals(t *testing.T) { func TestLessThan(t *testing.T) { require := require.New(t) for resultType, cmpCase := range comparisonCases { - get0 := NewGetField(0, resultType, "col1", true) + get0 := expression.NewGetField(0, resultType, "col1", true) require.NotNil(get0) - get1 := NewGetField(1, resultType, "col2", true) + get1 := expression.NewGetField(1, resultType, "col2", true) require.NotNil(get1) - eq := NewLessThan(get0, get1) + eq := expression.NewLessThan(get0, get1) require.NotNil(eq) require.Equal(sql.Boolean, eq.Type()) for cmpResult, cases := range cmpCase { @@ -149,11 +152,11 @@ func TestLessThan(t *testing.T) { func TestGreaterThan(t *testing.T) { require := require.New(t) for resultType, cmpCase := range comparisonCases { - get0 := NewGetField(0, resultType, "col1", true) + get0 := expression.NewGetField(0, resultType, "col1", true) require.NotNil(get0) - get1 := NewGetField(1, resultType, "col2", true) + get1 := expression.NewGetField(1, resultType, "col2", true) require.NotNil(get1) - eq := NewGreaterThan(get0, get1) + eq := expression.NewGreaterThan(get0, get1) require.NotNil(eq) require.Equal(sql.Boolean, eq.Type()) for cmpResult, cases := range cmpCase { @@ -185,13 +188,13 @@ func testRegexpCases(t *testing.T) { require := require.New(t) for resultType, cmpCase := range likeComparisonCases { - get0 := NewGetField(0, resultType, "col1", true) + get0 := expression.NewGetField(0, resultType, "col1", true) require.NotNil(get0) - get1 := NewGetField(1, resultType, "col2", true) + get1 := expression.NewGetField(1, resultType, "col2", true) require.NotNil(get1) for cmpResult, cases := range cmpCase { for _, pair := range cases { - eq := NewRegexp(get0, get1) + eq := expression.NewRegexp(get0, get1) require.NotNil(eq) require.Equal(sql.Boolean, eq.Type()) @@ -214,9 +217,9 @@ func TestInvalidRegexp(t *testing.T) { t.Helper() require := require.New(t) - col1 := NewGetField(0, sql.Text, "col1", true) - invalid := NewLiteral("*col1", sql.Text) - r := NewRegexp(col1, invalid) + col1 := expression.NewGetField(0, sql.Text, "col1", true) + invalid := expression.NewLiteral("*col1", sql.Text) + r := expression.NewRegexp(col1, invalid) row := sql.NewRow("col1") _, err := r.Eval(sql.NewEmptyContext(), row) @@ -234,10 +237,10 @@ func TestIn(t *testing.T) { }{ { "left is nil", - NewLiteral(nil, sql.Null), - NewTuple( - NewLiteral(int64(1), sql.Int64), - NewLiteral(int64(2), sql.Int64), + expression.NewLiteral(nil, sql.Null), + expression.NewTuple( + expression.NewLiteral(int64(1), sql.Int64), + expression.NewLiteral(int64(2), sql.Int64), ), nil, nil, @@ -245,32 +248,32 @@ func TestIn(t *testing.T) { }, { "left and right don't have the same cols", - NewLiteral(1, sql.Int64), - NewTuple( - NewTuple( - NewLiteral(int64(1), sql.Int64), - NewLiteral(int64(1), sql.Int64), + expression.NewLiteral(1, sql.Int64), + expression.NewTuple( + expression.NewTuple( + expression.NewLiteral(int64(1), sql.Int64), + expression.NewLiteral(int64(1), sql.Int64), ), - NewLiteral(int64(2), sql.Int64), + expression.NewLiteral(int64(2), sql.Int64), ), nil, nil, - ErrInvalidOperandColumns, + expression.ErrInvalidOperandColumns, }, { "right is an unsupported operand", - NewLiteral(1, sql.Int64), - NewLiteral(int64(2), sql.Int64), + expression.NewLiteral(1, sql.Int64), + expression.NewLiteral(int64(2), sql.Int64), nil, nil, - ErrUnsupportedInOperand, + expression.ErrUnsupportedInOperand, }, { "left is in right", - NewGetField(0, sql.Int64, "foo", false), - NewTuple( - NewGetField(0, sql.Int64, "foo", false), - NewLiteral(int64(2), sql.Int64), + expression.NewGetField(0, sql.Int64, "foo", false), + expression.NewTuple( + expression.NewGetField(0, sql.Int64, "foo", false), + expression.NewLiteral(int64(2), sql.Int64), ), sql.NewRow(int64(1)), true, @@ -278,10 +281,10 @@ func TestIn(t *testing.T) { }, { "left is not in right", - NewGetField(0, sql.Int64, "foo", false), - NewTuple( - NewGetField(1, sql.Int64, "bar", false), - NewLiteral(int64(2), sql.Int64), + expression.NewGetField(0, sql.Int64, "foo", false), + expression.NewTuple( + expression.NewGetField(1, sql.Int64, "bar", false), + expression.NewLiteral(int64(2), sql.Int64), ), sql.NewRow(int64(1), int64(3)), false, @@ -293,7 +296,96 @@ func TestIn(t *testing.T) { t.Run(tt.name, func(t *testing.T) { require := require.New(t) - result, err := NewIn(tt.left, tt.right).Eval(sql.NewEmptyContext(), tt.row) + result, err := expression.NewIn(tt.left, tt.right). + Eval(sql.NewEmptyContext(), tt.row) + if tt.err != nil { + require.Error(err) + require.True(tt.err.Is(err)) + } else { + require.NoError(err) + require.Equal(tt.result, result) + } + }) + } +} + +func TestInSubquery(t *testing.T) { + ctx := sql.NewEmptyContext() + table := memory.NewTable("foo", sql.Schema{ + {Name: "t", Source: "foo", Type: sql.Text}, + }) + + require.NoError(t, table.Insert(ctx, sql.Row{"one"})) + require.NoError(t, table.Insert(ctx, sql.Row{"two"})) + require.NoError(t, table.Insert(ctx, sql.Row{"three"})) + + project := func(expr sql.Expression) sql.Node { + return plan.NewProject([]sql.Expression{ + expr, + }, plan.NewResolvedTable(table)) + } + + testCases := []struct { + name string + left sql.Expression + right sql.Node + row sql.Row + result interface{} + err *errors.Kind + }{ + { + "left is nil", + expression.NewLiteral(nil, sql.Null), + project( + expression.NewLiteral(int64(1), sql.Int64), + ), + nil, + nil, + nil, + }, + { + "left and right don't have the same cols", + expression.NewTuple( + expression.NewLiteral(int64(1), sql.Int64), + expression.NewLiteral(int64(1), sql.Int64), + ), + project( + expression.NewLiteral(int64(2), sql.Int64), + ), + nil, + nil, + expression.ErrInvalidOperandColumns, + }, + { + "left is in right", + expression.NewGetField(0, sql.Text, "foo", false), + project( + expression.NewGetField(0, sql.Text, "foo", false), + ), + sql.NewRow("two"), + true, + nil, + }, + { + "left is not in right", + expression.NewGetField(0, sql.Text, "foo", false), + project( + expression.NewGetField(0, sql.Text, "foo", false), + ), + sql.NewRow("four"), + false, + nil, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + require := require.New(t) + + result, err := expression.NewIn( + tt.left, + expression.NewSubquery(tt.right), + ).Eval(sql.NewEmptyContext(), tt.row) if tt.err != nil { require.Error(err) require.True(tt.err.Is(err)) @@ -316,10 +408,10 @@ func TestNotIn(t *testing.T) { }{ { "left is nil", - NewLiteral(nil, sql.Null), - NewTuple( - NewLiteral(int64(1), sql.Int64), - NewLiteral(int64(2), sql.Int64), + expression.NewLiteral(nil, sql.Null), + expression.NewTuple( + expression.NewLiteral(int64(1), sql.Int64), + expression.NewLiteral(int64(2), sql.Int64), ), nil, nil, @@ -327,32 +419,32 @@ func TestNotIn(t *testing.T) { }, { "left and right don't have the same cols", - NewLiteral(1, sql.Int64), - NewTuple( - NewTuple( - NewLiteral(int64(1), sql.Int64), - NewLiteral(int64(1), sql.Int64), + expression.NewLiteral(1, sql.Int64), + expression.NewTuple( + expression.NewTuple( + expression.NewLiteral(int64(1), sql.Int64), + expression.NewLiteral(int64(1), sql.Int64), ), - NewLiteral(int64(2), sql.Int64), + expression.NewLiteral(int64(2), sql.Int64), ), nil, nil, - ErrInvalidOperandColumns, + expression.ErrInvalidOperandColumns, }, { "right is an unsupported operand", - NewLiteral(1, sql.Int64), - NewLiteral(int64(2), sql.Int64), + expression.NewLiteral(1, sql.Int64), + expression.NewLiteral(int64(2), sql.Int64), nil, nil, - ErrUnsupportedInOperand, + expression.ErrUnsupportedInOperand, }, { "left is in right", - NewGetField(0, sql.Int64, "foo", false), - NewTuple( - NewGetField(0, sql.Int64, "foo", false), - NewLiteral(int64(2), sql.Int64), + expression.NewGetField(0, sql.Int64, "foo", false), + expression.NewTuple( + expression.NewGetField(0, sql.Int64, "foo", false), + expression.NewLiteral(int64(2), sql.Int64), ), sql.NewRow(int64(1)), false, @@ -360,10 +452,10 @@ func TestNotIn(t *testing.T) { }, { "left is not in right", - NewGetField(0, sql.Int64, "foo", false), - NewTuple( - NewGetField(1, sql.Int64, "bar", false), - NewLiteral(int64(2), sql.Int64), + expression.NewGetField(0, sql.Int64, "foo", false), + expression.NewTuple( + expression.NewGetField(1, sql.Int64, "bar", false), + expression.NewLiteral(int64(2), sql.Int64), ), sql.NewRow(int64(1), int64(3)), true, @@ -375,7 +467,8 @@ func TestNotIn(t *testing.T) { t.Run(tt.name, func(t *testing.T) { require := require.New(t) - result, err := NewNotIn(tt.left, tt.right).Eval(sql.NewEmptyContext(), tt.row) + result, err := expression.NewNotIn(tt.left, tt.right). + Eval(sql.NewEmptyContext(), tt.row) if tt.err != nil { require.Error(err) require.True(tt.err.Is(err)) @@ -386,3 +479,98 @@ func TestNotIn(t *testing.T) { }) } } + +func TestNotInSubquery(t *testing.T) { + ctx := sql.NewEmptyContext() + table := memory.NewTable("foo", sql.Schema{ + {Name: "t", Source: "foo", Type: sql.Text}, + }) + + require.NoError(t, table.Insert(ctx, sql.Row{"one"})) + require.NoError(t, table.Insert(ctx, sql.Row{"two"})) + require.NoError(t, table.Insert(ctx, sql.Row{"three"})) + + project := func(expr sql.Expression) sql.Node { + return plan.NewProject([]sql.Expression{ + expr, + }, plan.NewResolvedTable(table)) + } + + testCases := []struct { + name string + left sql.Expression + right sql.Node + row sql.Row + result interface{} + err *errors.Kind + }{ + { + "left is nil", + expression.NewLiteral(nil, sql.Null), + project( + expression.NewLiteral(int64(1), sql.Int64), + ), + nil, + nil, + nil, + }, + { + "left and right don't have the same cols", + expression.NewTuple( + expression.NewLiteral(int64(1), sql.Int64), + expression.NewLiteral(int64(1), sql.Int64), + ), + project( + expression.NewLiteral(int64(2), sql.Int64), + ), + nil, + nil, + expression.ErrInvalidOperandColumns, + }, + { + "left is in right", + expression.NewGetField(0, sql.Text, "foo", false), + project( + expression.NewGetField(0, sql.Text, "foo", false), + ), + sql.NewRow("two"), + false, + nil, + }, + { + "left is not in right", + expression.NewGetField(0, sql.Text, "foo", false), + project( + expression.NewGetField(0, sql.Text, "foo", false), + ), + sql.NewRow("four"), + true, + nil, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + require := require.New(t) + + result, err := expression.NewNotIn( + tt.left, + expression.NewSubquery(tt.right), + ).Eval(sql.NewEmptyContext(), tt.row) + if tt.err != nil { + require.Error(err) + require.True(tt.err.Is(err)) + } else { + require.NoError(err) + require.Equal(tt.result, result) + } + }) + } +} + +func eval(t *testing.T, e sql.Expression, row sql.Row) interface{} { + t.Helper() + v, err := e.Eval(sql.NewEmptyContext(), row) + require.NoError(t, err) + return v +} diff --git a/sql/expression/convert.go b/sql/expression/convert.go index 28012d5b4..bcfe778e0 100644 --- a/sql/expression/convert.go +++ b/sql/expression/convert.go @@ -8,8 +8,8 @@ import ( "time" "github.com/spf13/cast" + "github.com/src-d/go-mysql-server/sql" errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) // ErrConvertExpression is returned when a conversion is not possible. @@ -51,6 +51,16 @@ func NewConvert(expr sql.Expression, castToType string) *Convert { } } +// IsNullable implements the Expression interface. +func (c *Convert) IsNullable() bool { + switch c.castToType { + case ConvertToDate, ConvertToDatetime: + return true + default: + return c.Child.IsNullable() + } +} + // Type implements the Expression interface. func (c *Convert) Type() sql.Type { switch c.castToType { @@ -58,8 +68,10 @@ func (c *Convert) Type() sql.Type { return sql.Blob case ConvertToChar, ConvertToNChar: return sql.Text - case ConvertToDate, ConvertToDatetime: + case ConvertToDate: return sql.Date + case ConvertToDatetime: + return sql.Timestamp case ConvertToDecimal: return sql.Float64 case ConvertToJSON: @@ -78,14 +90,12 @@ func (c *Convert) String() string { return fmt.Sprintf("convert(%v, %v)", c.Child, c.castToType) } -// TransformUp implements the Expression interface. -func (c *Convert) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - child, err := c.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (c *Convert) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(c, len(children), 1) } - - return f(NewConvert(child, c.castToType)) + return NewConvert(children[0], c.castToType), nil } // Eval implements the Expression interface. @@ -143,7 +153,7 @@ func convertValue(val interface{}, castTo string) (interface{}, error) { } } - return d, nil + return sql.ValidateTime(d.(time.Time)), nil case ConvertToDecimal: d, err := cast.ToFloat64E(val) if err != nil { diff --git a/sql/expression/convert_test.go b/sql/expression/convert_test.go index 45f12d708..726c68f8e 100644 --- a/sql/expression/convert_test.go +++ b/sql/expression/convert_test.go @@ -4,8 +4,8 @@ import ( "testing" "time" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestConvert(t *testing.T) { diff --git a/sql/expression/default.go b/sql/expression/default.go index 7d1153beb..82c5cb9e7 100644 --- a/sql/expression/default.go +++ b/sql/expression/default.go @@ -1,7 +1,7 @@ package expression import ( - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // DefaultColumn is an default expression of a column that is not yet resolved. @@ -53,8 +53,10 @@ func (*DefaultColumn) Eval(ctx *sql.Context, r sql.Row) (interface{}, error) { panic("default column is a placeholder node, but Eval was called") } -// TransformUp implements the sql.Expression interface. -func (c *DefaultColumn) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - n := *c - return f(&n) +// WithChildren implements the Expression interface. +func (c *DefaultColumn) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(c, len(children), 0) + } + return c, nil } diff --git a/sql/expression/doc.go b/sql/expression/doc.go deleted file mode 100644 index 4e82f949c..000000000 --- a/sql/expression/doc.go +++ /dev/null @@ -1 +0,0 @@ -package expression // import "gopkg.in/src-d/go-mysql-server.v0/sql/expression" diff --git a/sql/expression/function/aggregation/avg.go b/sql/expression/function/aggregation/avg.go index a13ea4141..eae1c8c0e 100644 --- a/sql/expression/function/aggregation/avg.go +++ b/sql/expression/function/aggregation/avg.go @@ -1,10 +1,10 @@ -package aggregation // import "gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/aggregation" +package aggregation import ( "fmt" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" ) // Avg node to calculate the average from numeric column @@ -53,13 +53,12 @@ func (a *Avg) Eval(ctx *sql.Context, buffer sql.Row) (interface{}, error) { return sum / float64(rows), nil } -// TransformUp implements AggregationExpression interface. -func (a *Avg) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - child, err := a.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (a *Avg) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(a, len(children), 1) } - return f(NewAvg(child)) + return NewAvg(children[0]), nil } // NewBuffer implements AggregationExpression interface. (AggregationExpression) diff --git a/sql/expression/function/aggregation/avg_test.go b/sql/expression/function/aggregation/avg_test.go index b77cb6a27..b63114185 100644 --- a/sql/expression/function/aggregation/avg_test.go +++ b/sql/expression/function/aggregation/avg_test.go @@ -3,9 +3,9 @@ package aggregation import ( "testing" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestAvg_String(t *testing.T) { diff --git a/sql/expression/function/aggregation/common_test.go b/sql/expression/function/aggregation/common_test.go index 5e22bc3b8..759a81601 100644 --- a/sql/expression/function/aggregation/common_test.go +++ b/sql/expression/function/aggregation/common_test.go @@ -3,8 +3,8 @@ package aggregation import ( "testing" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func eval(t *testing.T, e sql.Expression, row sql.Row) interface{} { @@ -15,3 +15,17 @@ func eval(t *testing.T, e sql.Expression, row sql.Row) interface{} { require.NoError(t, err) return v } + +func aggregate(t *testing.T, agg sql.Aggregation, rows ...sql.Row) interface{} { + t.Helper() + + ctx := sql.NewEmptyContext() + buf := agg.NewBuffer() + for _, row := range rows { + require.NoError(t, agg.Update(ctx, buf, row)) + } + + v, err := agg.Eval(ctx, buf) + require.NoError(t, err) + return v +} diff --git a/sql/expression/function/aggregation/count.go b/sql/expression/function/aggregation/count.go index f28e30142..0c6a8294b 100644 --- a/sql/expression/function/aggregation/count.go +++ b/sql/expression/function/aggregation/count.go @@ -3,8 +3,9 @@ package aggregation import ( "fmt" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" + "github.com/mitchellh/hashstructure" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" ) // Count node to count how many rows are in the result set. @@ -19,12 +20,12 @@ func NewCount(e sql.Expression) *Count { // NewBuffer creates a new buffer for the aggregation. func (c *Count) NewBuffer() sql.Row { - return sql.NewRow(int32(0)) + return sql.NewRow(int64(0)) } // Type returns the type of the result. func (c *Count) Type() sql.Type { - return sql.Int32 + return sql.Int64 } // IsNullable returns whether the return value can be null. @@ -45,13 +46,12 @@ func (c *Count) String() string { return fmt.Sprintf("COUNT(%s)", c.Child) } -// TransformUp implements the Expression interface. -func (c *Count) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - child, err := c.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (c *Count) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(c, len(children), 1) } - return f(NewCount(child)) + return NewCount(children[0]), nil } // Update implements the Aggregation interface. @@ -71,7 +71,7 @@ func (c *Count) Update(ctx *sql.Context, buffer, row sql.Row) error { } if inc { - buffer[0] = buffer[0].(int32) + int32(1) + buffer[0] = buffer[0].(int64) + int64(1) } return nil @@ -79,7 +79,7 @@ func (c *Count) Update(ctx *sql.Context, buffer, row sql.Row) error { // Merge implements the Aggregation interface. func (c *Count) Merge(ctx *sql.Context, buffer, partial sql.Row) error { - buffer[0] = buffer[0].(int32) + partial[0].(int32) + buffer[0] = buffer[0].(int64) + partial[0].(int64) return nil } @@ -88,3 +88,93 @@ func (c *Count) Eval(ctx *sql.Context, buffer sql.Row) (interface{}, error) { count := buffer[0] return count, nil } + +// CountDistinct node to count how many rows are in the result set. +type CountDistinct struct { + expression.UnaryExpression +} + +// NewCountDistinct creates a new CountDistinct node. +func NewCountDistinct(e sql.Expression) *CountDistinct { + return &CountDistinct{expression.UnaryExpression{Child: e}} +} + +// NewBuffer creates a new buffer for the aggregation. +func (c *CountDistinct) NewBuffer() sql.Row { + return sql.NewRow(make(map[uint64]struct{})) +} + +// Type returns the type of the result. +func (c *CountDistinct) Type() sql.Type { + return sql.Int64 +} + +// IsNullable returns whether the return value can be null. +func (c *CountDistinct) IsNullable() bool { + return false +} + +// Resolved implements the Expression interface. +func (c *CountDistinct) Resolved() bool { + if _, ok := c.Child.(*expression.Star); ok { + return true + } + + return c.Child.Resolved() +} + +func (c *CountDistinct) String() string { + return fmt.Sprintf("COUNT(DISTINCT %s)", c.Child) +} + +// WithChildren implements the Expression interface. +func (c *CountDistinct) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(c, len(children), 1) + } + return NewCountDistinct(children[0]), nil +} + +// Update implements the Aggregation interface. +func (c *CountDistinct) Update(ctx *sql.Context, buffer, row sql.Row) error { + seen := buffer[0].(map[uint64]struct{}) + var value interface{} + if _, ok := c.Child.(*expression.Star); ok { + value = row + } else { + v, err := c.Child.Eval(ctx, row) + if v == nil { + return nil + } + + if err != nil { + return err + } + + value = v + } + + hash, err := hashstructure.Hash(value, nil) + if err != nil { + return fmt.Errorf("count distinct unable to hash value: %s", err) + } + + seen[hash] = struct{}{} + + return nil +} + +// Merge implements the Aggregation interface. +func (c *CountDistinct) Merge(ctx *sql.Context, buffer, partial sql.Row) error { + seen := buffer[0].(map[uint64]struct{}) + for k := range partial[0].(map[uint64]struct{}) { + seen[k] = struct{}{} + } + return nil +} + +// Eval implements the Aggregation interface. +func (c *CountDistinct) Eval(ctx *sql.Context, buffer sql.Row) (interface{}, error) { + seen := buffer[0].(map[uint64]struct{}) + return int64(len(seen)), nil +} diff --git a/sql/expression/function/aggregation/count_test.go b/sql/expression/function/aggregation/count_test.go index 1459b2fcc..ea27b0504 100644 --- a/sql/expression/function/aggregation/count_test.go +++ b/sql/expression/function/aggregation/count_test.go @@ -3,73 +3,123 @@ package aggregation import ( "testing" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) -func TestCount_String(t *testing.T) { - require := require.New(t) - - c := NewCount(expression.NewLiteral("foo", sql.Text)) - require.Equal(`COUNT("foo")`, c.String()) -} - -func TestCount_Eval_1(t *testing.T) { +func TestCountEval1(t *testing.T) { require := require.New(t) ctx := sql.NewEmptyContext() c := NewCount(expression.NewLiteral(1, sql.Int32)) b := c.NewBuffer() - require.Equal(int32(0), eval(t, c, b)) + require.Equal(int64(0), eval(t, c, b)) require.NoError(c.Update(ctx, b, nil)) require.NoError(c.Update(ctx, b, sql.NewRow("foo"))) require.NoError(c.Update(ctx, b, sql.NewRow(1))) require.NoError(c.Update(ctx, b, sql.NewRow(nil))) require.NoError(c.Update(ctx, b, sql.NewRow(1, 2, 3))) - require.Equal(int32(5), eval(t, c, b)) + require.Equal(int64(5), eval(t, c, b)) b2 := c.NewBuffer() require.NoError(c.Update(ctx, b2, nil)) require.NoError(c.Update(ctx, b2, sql.NewRow("foo"))) require.NoError(c.Merge(ctx, b, b2)) - require.Equal(int32(7), eval(t, c, b)) + require.Equal(int64(7), eval(t, c, b)) } -func TestCount_Eval_Star(t *testing.T) { +func TestCountEvalStar(t *testing.T) { require := require.New(t) ctx := sql.NewEmptyContext() c := NewCount(expression.NewStar()) b := c.NewBuffer() - require.Equal(int32(0), eval(t, c, b)) + require.Equal(int64(0), eval(t, c, b)) - c.Update(ctx, b, nil) - c.Update(ctx, b, sql.NewRow("foo")) - c.Update(ctx, b, sql.NewRow(1)) - c.Update(ctx, b, sql.NewRow(nil)) - c.Update(ctx, b, sql.NewRow(1, 2, 3)) - require.Equal(int32(5), eval(t, c, b)) + require.NoError(c.Update(ctx, b, nil)) + require.NoError(c.Update(ctx, b, sql.NewRow("foo"))) + require.NoError(c.Update(ctx, b, sql.NewRow(1))) + require.NoError(c.Update(ctx, b, sql.NewRow(nil))) + require.NoError(c.Update(ctx, b, sql.NewRow(1, 2, 3))) + require.Equal(int64(5), eval(t, c, b)) b2 := c.NewBuffer() - c.Update(ctx, b2, sql.NewRow()) - c.Update(ctx, b2, sql.NewRow("foo")) - c.Merge(ctx, b, b2) - require.Equal(int32(7), eval(t, c, b)) + require.NoError(c.Update(ctx, b2, sql.NewRow())) + require.NoError(c.Update(ctx, b2, sql.NewRow("foo"))) + require.NoError(c.Merge(ctx, b, b2)) + require.Equal(int64(7), eval(t, c, b)) } -func TestCount_Eval_String(t *testing.T) { +func TestCountEvalString(t *testing.T) { require := require.New(t) ctx := sql.NewEmptyContext() c := NewCount(expression.NewGetField(0, sql.Text, "", true)) b := c.NewBuffer() - require.Equal(int32(0), eval(t, c, b)) + require.Equal(int64(0), eval(t, c, b)) - c.Update(ctx, b, sql.NewRow("foo")) - require.Equal(int32(1), eval(t, c, b)) + require.NoError(c.Update(ctx, b, sql.NewRow("foo"))) + require.Equal(int64(1), eval(t, c, b)) + + require.NoError(c.Update(ctx, b, sql.NewRow(nil))) + require.Equal(int64(1), eval(t, c, b)) +} + +func TestCountDistinctEval1(t *testing.T) { + require := require.New(t) + ctx := sql.NewEmptyContext() + + c := NewCountDistinct(expression.NewLiteral(1, sql.Int32)) + b := c.NewBuffer() + require.Equal(int64(0), eval(t, c, b)) + + require.NoError(c.Update(ctx, b, nil)) + require.NoError(c.Update(ctx, b, sql.NewRow("foo"))) + require.NoError(c.Update(ctx, b, sql.NewRow(1))) + require.NoError(c.Update(ctx, b, sql.NewRow(nil))) + require.NoError(c.Update(ctx, b, sql.NewRow(1, 2, 3))) + require.Equal(int64(1), eval(t, c, b)) +} - c.Update(ctx, b, sql.NewRow(nil)) - require.Equal(int32(1), eval(t, c, b)) +func TestCountDistinctEvalStar(t *testing.T) { + require := require.New(t) + ctx := sql.NewEmptyContext() + + c := NewCountDistinct(expression.NewStar()) + b := c.NewBuffer() + require.Equal(int64(0), eval(t, c, b)) + + require.NoError(c.Update(ctx, b, nil)) + require.NoError(c.Update(ctx, b, sql.NewRow("foo"))) + require.NoError(c.Update(ctx, b, sql.NewRow(1))) + require.NoError(c.Update(ctx, b, sql.NewRow(nil))) + require.NoError(c.Update(ctx, b, sql.NewRow(1, 2, 3))) + require.Equal(int64(5), eval(t, c, b)) + + b2 := c.NewBuffer() + require.NoError(c.Update(ctx, b2, sql.NewRow(1))) + require.NoError(c.Update(ctx, b2, sql.NewRow("foo"))) + require.NoError(c.Update(ctx, b2, sql.NewRow(5))) + require.NoError(c.Merge(ctx, b, b2)) + + require.Equal(int64(6), eval(t, c, b)) +} + +func TestCountDistinctEvalString(t *testing.T) { + require := require.New(t) + ctx := sql.NewEmptyContext() + + c := NewCountDistinct(expression.NewGetField(0, sql.Text, "", true)) + b := c.NewBuffer() + require.Equal(int64(0), eval(t, c, b)) + + require.NoError(c.Update(ctx, b, sql.NewRow("foo"))) + require.Equal(int64(1), eval(t, c, b)) + + require.NoError(c.Update(ctx, b, sql.NewRow(nil))) + require.NoError(c.Update(ctx, b, sql.NewRow("foo"))) + require.NoError(c.Update(ctx, b, sql.NewRow("bar"))) + require.Equal(int64(2), eval(t, c, b)) } diff --git a/sql/expression/function/aggregation/first.go b/sql/expression/function/aggregation/first.go new file mode 100644 index 000000000..20ef8af6c --- /dev/null +++ b/sql/expression/function/aggregation/first.go @@ -0,0 +1,71 @@ +package aggregation + +import ( + "fmt" + + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" +) + +// First agregation returns the first of all values in the selected column. +// It implements the Aggregation interface. +type First struct { + expression.UnaryExpression +} + +// NewFirst returns a new First node. +func NewFirst(e sql.Expression) *First { + return &First{expression.UnaryExpression{Child: e}} +} + +// Type returns the resultant type of the aggregation. +func (f *First) Type() sql.Type { + return f.Child.Type() +} + +func (f *First) String() string { + return fmt.Sprintf("FIRST(%s)", f.Child) +} + +// WithChildren implements the sql.Expression interface. +func (f *First) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(f, len(children), 1) + } + return NewFirst(children[0]), nil +} + +// NewBuffer creates a new buffer to compute the result. +func (f *First) NewBuffer() sql.Row { + return sql.NewRow(nil) +} + +// Update implements the Aggregation interface. +func (f *First) Update(ctx *sql.Context, buffer, row sql.Row) error { + if buffer[0] != nil { + return nil + } + + v, err := f.Child.Eval(ctx, row) + if err != nil { + return err + } + + if v == nil { + return nil + } + + buffer[0] = v + + return nil +} + +// Merge implements the Aggregation interface. +func (f *First) Merge(ctx *sql.Context, buffer, partial sql.Row) error { + return nil +} + +// Eval implements the Aggregation interface. +func (f *First) Eval(ctx *sql.Context, buffer sql.Row) (interface{}, error) { + return buffer[0], nil +} diff --git a/sql/expression/function/aggregation/first_test.go b/sql/expression/function/aggregation/first_test.go new file mode 100644 index 000000000..0ec3bdba6 --- /dev/null +++ b/sql/expression/function/aggregation/first_test.go @@ -0,0 +1,29 @@ +package aggregation + +import ( + "testing" + + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/stretchr/testify/require" +) + +func TestFirst(t *testing.T) { + testCases := []struct { + name string + rows []sql.Row + expected interface{} + }{ + {"no rows", nil, nil}, + {"one row", []sql.Row{{"first"}}, "first"}, + {"three rows", []sql.Row{{"first"}, {"second"}, {"last"}}, "first"}, + } + + agg := NewFirst(expression.NewGetField(0, sql.Text, "", false)) + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + result := aggregate(t, agg, tt.rows...) + require.Equal(t, tt.expected, result) + }) + } +} diff --git a/sql/expression/function/aggregation/last.go b/sql/expression/function/aggregation/last.go new file mode 100644 index 000000000..55457a5e5 --- /dev/null +++ b/sql/expression/function/aggregation/last.go @@ -0,0 +1,68 @@ +package aggregation + +import ( + "fmt" + + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" +) + +// Last agregation returns the last of all values in the selected column. +// It implements the Aggregation interface. +type Last struct { + expression.UnaryExpression +} + +// NewLast returns a new Last node. +func NewLast(e sql.Expression) *Last { + return &Last{expression.UnaryExpression{Child: e}} +} + +// Type returns the resultant type of the aggregation. +func (l *Last) Type() sql.Type { + return l.Child.Type() +} + +func (l *Last) String() string { + return fmt.Sprintf("LAST(%s)", l.Child) +} + +// WithChildren implements the sql.Expression interface. +func (l *Last) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(l, len(children), 1) + } + return NewLast(children[0]), nil +} + +// NewBuffer creates a new buffer to compute the result. +func (l *Last) NewBuffer() sql.Row { + return sql.NewRow(nil) +} + +// Update implements the Aggregation interface. +func (l *Last) Update(ctx *sql.Context, buffer, row sql.Row) error { + v, err := l.Child.Eval(ctx, row) + if err != nil { + return err + } + + if v == nil { + return nil + } + + buffer[0] = v + + return nil +} + +// Merge implements the Aggregation interface. +func (l *Last) Merge(ctx *sql.Context, buffer, partial sql.Row) error { + buffer[0] = partial[0] + return nil +} + +// Eval implements the Aggregation interface. +func (l *Last) Eval(ctx *sql.Context, buffer sql.Row) (interface{}, error) { + return buffer[0], nil +} diff --git a/sql/expression/function/aggregation/last_test.go b/sql/expression/function/aggregation/last_test.go new file mode 100644 index 000000000..4271eebf4 --- /dev/null +++ b/sql/expression/function/aggregation/last_test.go @@ -0,0 +1,29 @@ +package aggregation + +import ( + "testing" + + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/stretchr/testify/require" +) + +func TestLast(t *testing.T) { + testCases := []struct { + name string + rows []sql.Row + expected interface{} + }{ + {"no rows", nil, nil}, + {"one row", []sql.Row{{"first"}}, "first"}, + {"three rows", []sql.Row{{"first"}, {"second"}, {"last"}}, "last"}, + } + + agg := NewLast(expression.NewGetField(0, sql.Text, "", false)) + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + result := aggregate(t, agg, tt.rows...) + require.Equal(t, tt.expected, result) + }) + } +} diff --git a/sql/expression/function/aggregation/max.go b/sql/expression/function/aggregation/max.go index 1c793a8e1..e47211f2c 100644 --- a/sql/expression/function/aggregation/max.go +++ b/sql/expression/function/aggregation/max.go @@ -4,8 +4,8 @@ import ( "fmt" "reflect" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" ) // Max agregation returns the greatest value of the selected column. @@ -38,13 +38,12 @@ func (m *Max) IsNullable() bool { return false } -// TransformUp implements the Transformable interface. -func (m *Max) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - child, err := m.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (m *Max) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(m, len(children), 1) } - return f(NewMax(child)) + return NewMax(children[0]), nil } // NewBuffer creates a new buffer to compute the result. diff --git a/sql/expression/function/aggregation/max_test.go b/sql/expression/function/aggregation/max_test.go index 8f467bc2d..f8da09942 100644 --- a/sql/expression/function/aggregation/max_test.go +++ b/sql/expression/function/aggregation/max_test.go @@ -4,9 +4,9 @@ import ( "testing" "time" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestMax_String(t *testing.T) { diff --git a/sql/expression/function/aggregation/min.go b/sql/expression/function/aggregation/min.go index 3fef114ea..8e73e0812 100644 --- a/sql/expression/function/aggregation/min.go +++ b/sql/expression/function/aggregation/min.go @@ -4,8 +4,8 @@ import ( "fmt" "reflect" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" ) // Min aggregation returns the smallest value of the selected column. @@ -38,13 +38,12 @@ func (m *Min) IsNullable() bool { return true } -// TransformUp implements the Transformable interface. -func (m *Min) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - child, err := m.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (m *Min) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(m, len(children), 1) } - return f(NewMin(child)) + return NewMin(children[0]), nil } // NewBuffer creates a new buffer to compute the result. diff --git a/sql/expression/function/aggregation/min_test.go b/sql/expression/function/aggregation/min_test.go index c5090aa34..6039219d6 100644 --- a/sql/expression/function/aggregation/min_test.go +++ b/sql/expression/function/aggregation/min_test.go @@ -4,9 +4,9 @@ import ( "testing" "time" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestMin_Name(t *testing.T) { diff --git a/sql/expression/function/aggregation/sum.go b/sql/expression/function/aggregation/sum.go index 104de369b..09df362be 100644 --- a/sql/expression/function/aggregation/sum.go +++ b/sql/expression/function/aggregation/sum.go @@ -3,8 +3,8 @@ package aggregation import ( "fmt" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" ) // Sum agregation returns the sum of all values in the selected column. @@ -27,13 +27,12 @@ func (m *Sum) String() string { return fmt.Sprintf("SUM(%s)", m.Child) } -// TransformUp implements the Transformable interface. -func (m *Sum) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - child, err := m.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (m *Sum) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(m, len(children), 1) } - return f(NewSum(child)) + return NewSum(children[0]), nil } // NewBuffer creates a new buffer to compute the result. diff --git a/sql/expression/function/aggregation/sum_test.go b/sql/expression/function/aggregation/sum_test.go index f5d68970b..37b29324e 100644 --- a/sql/expression/function/aggregation/sum_test.go +++ b/sql/expression/function/aggregation/sum_test.go @@ -3,9 +3,9 @@ package aggregation import ( "testing" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestSum(t *testing.T) { diff --git a/sql/expression/function/arraylength.go b/sql/expression/function/arraylength.go index cfbc1cacc..61a902cd9 100644 --- a/sql/expression/function/arraylength.go +++ b/sql/expression/function/arraylength.go @@ -1,11 +1,10 @@ -package function // import "gopkg.in/src-d/go-mysql-server.v0/sql/expression/function" +package function import ( "fmt" - "reflect" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" ) // ArrayLength returns the length of an array. @@ -27,19 +26,18 @@ func (f *ArrayLength) String() string { return fmt.Sprintf("array_length(%s)", f.Child) } -// TransformUp implements the Expression interface. -func (f *ArrayLength) TransformUp(fn sql.TransformExprFunc) (sql.Expression, error) { - child, err := f.Child.TransformUp(fn) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (f *ArrayLength) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(f, len(children), 1) } - return fn(NewArrayLength(child)) + return NewArrayLength(children[0]), nil } // Eval implements the Expression interface. func (f *ArrayLength) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { - if !sql.IsArray(f.Child.Type()) { - return nil, sql.ErrInvalidType.New(f.Child.Type().Type().String()) + if t := f.Child.Type(); !sql.IsArray(t) && t != sql.JSON { + return nil, nil } child, err := f.Child.Eval(ctx, row) @@ -53,7 +51,7 @@ func (f *ArrayLength) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { array, ok := child.([]interface{}) if !ok { - return nil, sql.ErrInvalidType.New(reflect.TypeOf(child)) + return nil, nil } return int32(len(array)), nil diff --git a/sql/expression/function/arraylength_test.go b/sql/expression/function/arraylength_test.go index 034b6afa7..7e9437144 100644 --- a/sql/expression/function/arraylength_test.go +++ b/sql/expression/function/arraylength_test.go @@ -3,10 +3,10 @@ package function import ( "testing" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestArrayLength(t *testing.T) { @@ -19,7 +19,7 @@ func TestArrayLength(t *testing.T) { err *errors.Kind }{ {"array is nil", sql.NewRow(nil), nil, nil}, - {"array is not of right type", sql.NewRow(5), nil, sql.ErrInvalidType}, + {"array is not of right type", sql.NewRow(5), nil, nil}, {"array is ok", sql.NewRow([]interface{}{1, 2, 3}), int32(3), nil}, } @@ -40,7 +40,7 @@ func TestArrayLength(t *testing.T) { f = NewArrayLength(expression.NewGetField(0, sql.Tuple(sql.Int64, sql.Int64), "", false)) require := require.New(t) - _, err := f.Eval(sql.NewEmptyContext(), []interface{}{int64(1), int64(2)}) - require.Error(err) - require.True(sql.ErrInvalidType.Is(err)) + v, err := f.Eval(sql.NewEmptyContext(), []interface{}{int64(1), int64(2)}) + require.NoError(err) + require.Nil(v) } diff --git a/sql/expression/function/ceil_round_floor.go b/sql/expression/function/ceil_round_floor.go index 9e7ddc5ee..c056f82da 100644 --- a/sql/expression/function/ceil_round_floor.go +++ b/sql/expression/function/ceil_round_floor.go @@ -5,8 +5,8 @@ import ( "math" "reflect" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" ) // Ceil returns the smallest integer value not less than X. @@ -32,13 +32,12 @@ func (c *Ceil) String() string { return fmt.Sprintf("CEIL(%s)", c.Child) } -// TransformUp implements the Expression interface. -func (c *Ceil) TransformUp(fn sql.TransformExprFunc) (sql.Expression, error) { - child, err := c.Child.TransformUp(fn) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (c *Ceil) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(c, len(children), 1) } - return fn(NewCeil(child)) + return NewCeil(children[0]), nil } // Eval implements the Expression interface. @@ -99,13 +98,12 @@ func (f *Floor) String() string { return fmt.Sprintf("FLOOR(%s)", f.Child) } -// TransformUp implements the Expression interface. -func (f *Floor) TransformUp(fn sql.TransformExprFunc) (sql.Expression, error) { - child, err := f.Child.TransformUp(fn) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (f *Floor) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(f, len(children), 1) } - return fn(NewFloor(child)) + return NewFloor(children[0]), nil } // Eval implements the Expression interface. @@ -189,7 +187,8 @@ func (r *Round) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { dVal := float64(0) if r.Right != nil { - dTemp, err := r.Right.Eval(ctx, row) + var dTemp interface{} + dTemp, err = r.Right.Eval(ctx, row) if err != nil { return nil, err } @@ -204,6 +203,18 @@ func (r *Round) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { dVal = float64(dNum) case int32: dVal = float64(dNum) + case int16: + dVal = float64(dNum) + case int8: + dVal = float64(dNum) + case uint64: + dVal = float64(dNum) + case uint32: + dVal = float64(dNum) + case uint16: + dVal = float64(dNum) + case uint8: + dVal = float64(dNum) case int: dVal = float64(dNum) default: @@ -234,6 +245,18 @@ func (r *Round) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return int64(math.Round(float64(xNum)*math.Pow(10.0, dVal)) / math.Pow(10.0, dVal)), nil case int32: return int32(math.Round(float64(xNum)*math.Pow(10.0, dVal)) / math.Pow(10.0, dVal)), nil + case int16: + return int16(math.Round(float64(xNum)*math.Pow(10.0, dVal)) / math.Pow(10.0, dVal)), nil + case int8: + return int8(math.Round(float64(xNum)*math.Pow(10.0, dVal)) / math.Pow(10.0, dVal)), nil + case uint64: + return uint64(math.Round(float64(xNum)*math.Pow(10.0, dVal)) / math.Pow(10.0, dVal)), nil + case uint32: + return uint32(math.Round(float64(xNum)*math.Pow(10.0, dVal)) / math.Pow(10.0, dVal)), nil + case uint16: + return uint16(math.Round(float64(xNum)*math.Pow(10.0, dVal)) / math.Pow(10.0, dVal)), nil + case uint8: + return uint8(math.Round(float64(xNum)*math.Pow(10.0, dVal)) / math.Pow(10.0, dVal)), nil case int: return int(math.Round(float64(xNum)*math.Pow(10.0, dVal)) / math.Pow(10.0, dVal)), nil default: @@ -268,29 +291,7 @@ func (r *Round) Type() sql.Type { return sql.Int32 } -// TransformUp implements the Expression interface. -func (r *Round) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - var args = make([]sql.Expression, 2) - - arg, err := r.Left.TransformUp(f) - if err != nil { - return nil, err - } - args[0] = arg - - args[1] = nil - if r.Right != nil { - arg, err := r.Right.TransformUp(f) - if err != nil { - return nil, err - } - args[1] = arg - } - - expr, err := NewRound(args...) - if err != nil { - return nil, err - } - - return f(expr) +// WithChildren implements the Expression interface. +func (r *Round) WithChildren(children ...sql.Expression) (sql.Expression, error) { + return NewRound(children...) } diff --git a/sql/expression/function/ceil_round_floor_test.go b/sql/expression/function/ceil_round_floor_test.go index d221b5957..4af2456ef 100644 --- a/sql/expression/function/ceil_round_floor_test.go +++ b/sql/expression/function/ceil_round_floor_test.go @@ -3,10 +3,10 @@ package function import ( "testing" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestCeil(t *testing.T) { @@ -156,6 +156,50 @@ func TestRound(t *testing.T) { {"int32 with float d", sql.Int32, sql.Float64, sql.NewRow(int32(5), float32(2.123)), int32(5), nil}, {"int32 with float negative d", sql.Int32, sql.Float64, sql.NewRow(int32(52), float32(-1)), int32(50), nil}, {"int32 with blob d", sql.Int32, sql.Blob, sql.NewRow(int32(5), []byte{1, 2, 3}), int32(5), nil}, + {"int16 is nil", sql.Int16, sql.Int16, sql.NewRow(nil, nil), nil, nil}, + {"int16 without d", sql.Int16, sql.Int16, sql.NewRow(int16(5), nil), int16(5), nil}, + {"int16 with d", sql.Int16, sql.Int16, sql.NewRow(int16(5), 2), int16(5), nil}, + {"int16 with negative d", sql.Int16, sql.Int16, sql.NewRow(int16(52), -1), int16(50), nil}, + {"int16 with float d", sql.Int16, sql.Float64, sql.NewRow(int16(5), float32(2.123)), int16(5), nil}, + {"int16 with float negative d", sql.Int16, sql.Float64, sql.NewRow(int16(52), float32(-1)), int16(50), nil}, + {"int16 with blob d", sql.Int16, sql.Blob, sql.NewRow(int16(5), []byte{1, 2, 3}), int16(5), nil}, + {"int8 is nil", sql.Int8, sql.Int8, sql.NewRow(nil, nil), nil, nil}, + {"int8 without d", sql.Int8, sql.Int8, sql.NewRow(int8(5), nil), int8(5), nil}, + {"int8 with d", sql.Int8, sql.Int8, sql.NewRow(int8(5), 2), int8(5), nil}, + {"int8 with negative d", sql.Int8, sql.Int8, sql.NewRow(int8(52), -1), int8(50), nil}, + {"int8 with float d", sql.Int8, sql.Float64, sql.NewRow(int8(5), float32(2.123)), int8(5), nil}, + {"int8 with float negative d", sql.Int8, sql.Float64, sql.NewRow(int8(52), float32(-1)), int8(50), nil}, + {"int8 with blob d", sql.Int8, sql.Blob, sql.NewRow(int8(5), []byte{1, 2, 3}), int8(5), nil}, + {"uint64 is nil", sql.Uint64, sql.Int32, sql.NewRow(nil, nil), nil, nil}, + {"uint64 without d", sql.Uint64, sql.Int32, sql.NewRow(uint64(5), nil), uint64(5), nil}, + {"uint64 with d", sql.Uint64, sql.Int32, sql.NewRow(uint64(5), 2), uint64(5), nil}, + {"uint64 with negative d", sql.Uint64, sql.Int32, sql.NewRow(uint64(52), -1), uint64(50), nil}, + {"uint64 with float d", sql.Uint64, sql.Float64, sql.NewRow(uint64(5), float32(2.123)), uint64(5), nil}, + {"uint64 with float negative d", sql.Uint64, sql.Float64, sql.NewRow(uint64(52), float32(-1)), uint64(50), nil}, + {"uint32 with blob d", sql.Uint32, sql.Blob, sql.NewRow(uint32(5), []byte{1, 2, 3}), uint32(5), nil}, + {"uint32 is nil", sql.Uint32, sql.Int32, sql.NewRow(nil, nil), nil, nil}, + {"uint32 without d", sql.Uint32, sql.Int32, sql.NewRow(uint32(5), nil), uint32(5), nil}, + {"uint32 with d", sql.Uint32, sql.Int32, sql.NewRow(uint32(5), 2), uint32(5), nil}, + {"uint32 with negative d", sql.Uint32, sql.Int32, sql.NewRow(uint32(52), -1), uint32(50), nil}, + {"uint32 with float d", sql.Uint32, sql.Float64, sql.NewRow(uint32(5), float32(2.123)), uint32(5), nil}, + {"uint32 with float negative d", sql.Uint32, sql.Float64, sql.NewRow(uint32(52), float32(-1)), uint32(50), nil}, + {"uint32 with blob d", sql.Uint32, sql.Blob, sql.NewRow(uint32(5), []byte{1, 2, 3}), uint32(5), nil}, + {"uint16 with blob d", sql.Uint16, sql.Blob, sql.NewRow(uint16(5), []byte{1, 2, 3}), uint16(5), nil}, + {"uint16 is nil", sql.Uint16, sql.Int16, sql.NewRow(nil, nil), nil, nil}, + {"uint16 without d", sql.Uint16, sql.Int16, sql.NewRow(uint16(5), nil), uint16(5), nil}, + {"uint16 with d", sql.Uint16, sql.Int16, sql.NewRow(uint16(5), 2), uint16(5), nil}, + {"uint16 with negative d", sql.Uint16, sql.Int16, sql.NewRow(uint16(52), -1), uint16(50), nil}, + {"uint16 with float d", sql.Uint16, sql.Float64, sql.NewRow(uint16(5), float32(2.123)), uint16(5), nil}, + {"uint16 with float negative d", sql.Uint16, sql.Float64, sql.NewRow(uint16(52), float32(-1)), uint16(50), nil}, + {"uint16 with blob d", sql.Uint16, sql.Blob, sql.NewRow(uint16(5), []byte{1, 2, 3}), uint16(5), nil}, + {"uint8 with blob d", sql.Uint8, sql.Blob, sql.NewRow(uint8(5), []byte{1, 2, 3}), uint8(5), nil}, + {"uint8 is nil", sql.Uint8, sql.Int8, sql.NewRow(nil, nil), nil, nil}, + {"uint8 without d", sql.Uint8, sql.Int8, sql.NewRow(uint8(5), nil), uint8(5), nil}, + {"uint8 with d", sql.Uint8, sql.Int8, sql.NewRow(uint8(5), 2), uint8(5), nil}, + {"uint8 with negative d", sql.Uint8, sql.Int8, sql.NewRow(uint8(52), -1), uint8(50), nil}, + {"uint8 with float d", sql.Uint8, sql.Float64, sql.NewRow(uint8(5), float32(2.123)), uint8(5), nil}, + {"uint8 with float negative d", sql.Uint8, sql.Float64, sql.NewRow(uint8(52), float32(-1)), uint8(50), nil}, + {"uint8 with blob d", sql.Uint8, sql.Blob, sql.NewRow(uint8(5), []byte{1, 2, 3}), uint8(5), nil}, {"blob is nil", sql.Blob, sql.Int32, sql.NewRow(nil, nil), nil, nil}, {"blob is ok", sql.Blob, sql.Int32, sql.NewRow([]byte{1, 2, 3}, nil), int32(0), nil}, {"text int without d", sql.Text, sql.Int32, sql.NewRow("5", nil), int32(5), nil}, diff --git a/sql/expression/function/coalesce.go b/sql/expression/function/coalesce.go index e7a0b760b..07f7f64e6 100644 --- a/sql/expression/function/coalesce.go +++ b/sql/expression/function/coalesce.go @@ -4,7 +4,7 @@ import ( "fmt" "strings" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // Coalesce returns the first non-NULL value in the list, or NULL if there are no non-NULL values. @@ -59,29 +59,9 @@ func (c *Coalesce) String() string { return fmt.Sprintf("coalesce(%s)", strings.Join(args, ", ")) } -// TransformUp implements the sql.Expression interface. -func (c *Coalesce) TransformUp(fn sql.TransformExprFunc) (sql.Expression, error) { - var ( - args = make([]sql.Expression, len(c.args)) - err error - ) - - for i, arg := range c.args { - if arg != nil { - arg, err = arg.TransformUp(fn) - if err != nil { - return nil, err - } - } - args[i] = arg - } - - expr, err := NewCoalesce(args...) - if err != nil { - return nil, err - } - - return fn(expr) +// WithChildren implements the Expression interface. +func (*Coalesce) WithChildren(children ...sql.Expression) (sql.Expression, error) { + return NewCoalesce(children...) } // Resolved implements the sql.Expression interface. diff --git a/sql/expression/function/coalesce_test.go b/sql/expression/function/coalesce_test.go index b2c81233f..a1fd6a89d 100644 --- a/sql/expression/function/coalesce_test.go +++ b/sql/expression/function/coalesce_test.go @@ -3,9 +3,9 @@ package function import ( "testing" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestEmptyCoalesce(t *testing.T) { diff --git a/sql/expression/function/common_test.go b/sql/expression/function/common_test.go index 2d86f8c86..3fbc45989 100644 --- a/sql/expression/function/common_test.go +++ b/sql/expression/function/common_test.go @@ -3,8 +3,8 @@ package function import ( "testing" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func eval(t *testing.T, e sql.Expression, row sql.Row) interface{} { diff --git a/sql/expression/function/concat.go b/sql/expression/function/concat.go index 5777a030a..56e7bcbab 100644 --- a/sql/expression/function/concat.go +++ b/sql/expression/function/concat.go @@ -4,8 +4,8 @@ import ( "fmt" "strings" + "github.com/src-d/go-mysql-server/sql" errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) // Concat joins several strings together. @@ -63,23 +63,9 @@ func (f *Concat) String() string { return fmt.Sprintf("concat(%s)", strings.Join(args, ", ")) } -// TransformUp implements the Expression interface. -func (f *Concat) TransformUp(fn sql.TransformExprFunc) (sql.Expression, error) { - var args = make([]sql.Expression, len(f.args)) - for i, arg := range f.args { - arg, err := arg.TransformUp(fn) - if err != nil { - return nil, err - } - args[i] = arg - } - - expr, err := NewConcat(args...) - if err != nil { - return nil, err - } - - return fn(expr) +// WithChildren implements the Expression interface. +func (*Concat) WithChildren(children ...sql.Expression) (sql.Expression, error) { + return NewConcat(children...) } // Resolved implements the Expression interface. diff --git a/sql/expression/function/concat_test.go b/sql/expression/function/concat_test.go index dccadab0b..450c0c442 100644 --- a/sql/expression/function/concat_test.go +++ b/sql/expression/function/concat_test.go @@ -3,9 +3,9 @@ package function import ( "testing" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestConcat(t *testing.T) { diff --git a/sql/expression/function/concat_ws.go b/sql/expression/function/concat_ws.go index 4c856da1a..c1e2dacc1 100644 --- a/sql/expression/function/concat_ws.go +++ b/sql/expression/function/concat_ws.go @@ -4,7 +4,7 @@ import ( "fmt" "strings" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // ConcatWithSeparator joins several strings together. The first argument is @@ -61,23 +61,9 @@ func (f *ConcatWithSeparator) String() string { return fmt.Sprintf("concat_ws(%s)", strings.Join(args, ", ")) } -// TransformUp implements the Expression interface. -func (f *ConcatWithSeparator) TransformUp(fn sql.TransformExprFunc) (sql.Expression, error) { - var args = make([]sql.Expression, len(f.args)) - for i, arg := range f.args { - arg, err := arg.TransformUp(fn) - if err != nil { - return nil, err - } - args[i] = arg - } - - expr, err := NewConcatWithSeparator(args...) - if err != nil { - return nil, err - } - - return fn(expr) +// WithChildren implements the Expression interface. +func (*ConcatWithSeparator) WithChildren(children ...sql.Expression) (sql.Expression, error) { + return NewConcatWithSeparator(children...) } // Resolved implements the Expression interface. diff --git a/sql/expression/function/concat_ws_test.go b/sql/expression/function/concat_ws_test.go index abeb5b752..7ec7bdc0c 100644 --- a/sql/expression/function/concat_ws_test.go +++ b/sql/expression/function/concat_ws_test.go @@ -3,9 +3,9 @@ package function import ( "testing" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestConcatWithSeparator(t *testing.T) { @@ -72,7 +72,7 @@ func TestConcatWithSeparator(t *testing.T) { t.Run("concat_ws array", func(t *testing.T) { require := require.New(t) f, err := NewConcatWithSeparator( - expression.NewLiteral([]interface{}{",",5, "bar", true}, sql.Array(sql.Text)), + expression.NewLiteral([]interface{}{",", 5, "bar", true}, sql.Array(sql.Text)), ) require.NoError(err) diff --git a/sql/expression/function/connection_id.go b/sql/expression/function/connection_id.go index 8fc6a651c..7ee96ef58 100644 --- a/sql/expression/function/connection_id.go +++ b/sql/expression/function/connection_id.go @@ -1,6 +1,6 @@ package function -import "gopkg.in/src-d/go-mysql-server.v0/sql" +import "github.com/src-d/go-mysql-server/sql" // ConnectionID returns the current connection id. type ConnectionID struct{} @@ -19,9 +19,12 @@ func (ConnectionID) Type() sql.Type { return sql.Uint32 } // Resolved implements the sql.Expression interface. func (ConnectionID) Resolved() bool { return true } -// TransformUp implements the sql.Expression interface. -func (ConnectionID) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - return f(ConnectionID{}) +// WithChildren implements the Expression interface. +func (c ConnectionID) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(c, len(children), 0) + } + return c, nil } // IsNullable implements the sql.Expression interface. diff --git a/sql/expression/function/connection_id_test.go b/sql/expression/function/connection_id_test.go index 29cb8a2b1..5a9699899 100644 --- a/sql/expression/function/connection_id_test.go +++ b/sql/expression/function/connection_id_test.go @@ -4,8 +4,8 @@ import ( "context" "testing" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestConnectionID(t *testing.T) { diff --git a/sql/expression/function/database.go b/sql/expression/function/database.go index 8e1d98624..1246e488c 100644 --- a/sql/expression/function/database.go +++ b/sql/expression/function/database.go @@ -1,7 +1,7 @@ package function import ( - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // Database stands for DATABASE() function @@ -29,9 +29,12 @@ func (*Database) String() string { return "DATABASE()" } -// TransformUp implements the sql.Expression interface. -func (db *Database) TransformUp(fn sql.TransformExprFunc) (sql.Expression, error) { - return fn(db) +// WithChildren implements the Expression interface. +func (d *Database) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(d, len(children), 0) + } + return NewDatabase(d.catalog)(), nil } // Resolved implements the sql.Expression interface. diff --git a/sql/expression/function/date.go b/sql/expression/function/date.go new file mode 100644 index 000000000..919775a42 --- /dev/null +++ b/sql/expression/function/date.go @@ -0,0 +1,159 @@ +package function + +import ( + "fmt" + "time" + + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" +) + +// DateAdd adds an interval to a date. +type DateAdd struct { + Date sql.Expression + Interval *expression.Interval +} + +// NewDateAdd creates a new date add function. +func NewDateAdd(args ...sql.Expression) (sql.Expression, error) { + if len(args) != 2 { + return nil, sql.ErrInvalidArgumentNumber.New("DATE_ADD", 2, len(args)) + } + + i, ok := args[1].(*expression.Interval) + if !ok { + return nil, fmt.Errorf("DATE_ADD expects an interval as second parameter") + } + + return &DateAdd{args[0], i}, nil +} + +// Children implements the sql.Expression interface. +func (d *DateAdd) Children() []sql.Expression { + return []sql.Expression{d.Date, d.Interval} +} + +// Resolved implements the sql.Expression interface. +func (d *DateAdd) Resolved() bool { + return d.Date.Resolved() && d.Interval.Resolved() +} + +// IsNullable implements the sql.Expression interface. +func (d *DateAdd) IsNullable() bool { + return true +} + +// Type implements the sql.Expression interface. +func (d *DateAdd) Type() sql.Type { return sql.Date } + +// WithChildren implements the Expression interface. +func (d *DateAdd) WithChildren(children ...sql.Expression) (sql.Expression, error) { + return NewDateAdd(children...) +} + +// Eval implements the sql.Expression interface. +func (d *DateAdd) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { + date, err := d.Date.Eval(ctx, row) + if err != nil { + return nil, err + } + + if date == nil { + return nil, nil + } + + date, err = sql.Timestamp.Convert(date) + if err != nil { + return nil, err + } + + delta, err := d.Interval.EvalDelta(ctx, row) + if err != nil { + return nil, err + } + + if delta == nil { + return nil, nil + } + + return sql.ValidateTime(delta.Add(date.(time.Time))), nil +} + +func (d *DateAdd) String() string { + return fmt.Sprintf("DATE_ADD(%s, %s)", d.Date, d.Interval) +} + +// DateSub subtracts an interval from a date. +type DateSub struct { + Date sql.Expression + Interval *expression.Interval +} + +// NewDateSub creates a new date add function. +func NewDateSub(args ...sql.Expression) (sql.Expression, error) { + if len(args) != 2 { + return nil, sql.ErrInvalidArgumentNumber.New("DATE_SUB", 2, len(args)) + } + + i, ok := args[1].(*expression.Interval) + if !ok { + return nil, fmt.Errorf("DATE_SUB expects an interval as second parameter") + } + + return &DateSub{args[0], i}, nil +} + +// Children implements the sql.Expression interface. +func (d *DateSub) Children() []sql.Expression { + return []sql.Expression{d.Date, d.Interval} +} + +// Resolved implements the sql.Expression interface. +func (d *DateSub) Resolved() bool { + return d.Date.Resolved() && d.Interval.Resolved() +} + +// IsNullable implements the sql.Expression interface. +func (d *DateSub) IsNullable() bool { + return true +} + +// Type implements the sql.Expression interface. +func (d *DateSub) Type() sql.Type { return sql.Date } + +// WithChildren implements the Expression interface. +func (d *DateSub) WithChildren(children ...sql.Expression) (sql.Expression, error) { + return NewDateSub(children...) +} + +// Eval implements the sql.Expression interface. +func (d *DateSub) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { + date, err := d.Date.Eval(ctx, row) + if err != nil { + return nil, err + } + + if date == nil { + return nil, nil + } + + date, err = sql.Timestamp.Convert(date) + if err != nil { + return nil, err + } + + delta, err := d.Interval.EvalDelta(ctx, row) + if err != nil { + return nil, err + } + + if delta == nil { + return nil, nil + } + + return sql.ValidateTime(delta.Sub(date.(time.Time))), nil +} + +func (d *DateSub) String() string { + return fmt.Sprintf("DATE_SUB(%s, %s)", d.Date, d.Interval) +} diff --git a/sql/expression/function/date_test.go b/sql/expression/function/date_test.go new file mode 100644 index 000000000..18f436ed9 --- /dev/null +++ b/sql/expression/function/date_test.go @@ -0,0 +1,87 @@ +package function + +import ( + "testing" + "time" + + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/stretchr/testify/require" +) + +func TestDateAdd(t *testing.T) { + require := require.New(t) + + _, err := NewDateAdd() + require.Error(err) + + _, err = NewDateAdd(expression.NewLiteral("2018-05-02", sql.Text)) + require.Error(err) + + _, err = NewDateAdd( + expression.NewLiteral("2018-05-02", sql.Text), + expression.NewLiteral(int64(1), sql.Int64), + ) + require.Error(err) + + f, err := NewDateAdd( + expression.NewGetField(0, sql.Text, "foo", false), + expression.NewInterval( + expression.NewLiteral(int64(1), sql.Int64), + "DAY", + ), + ) + require.NoError(err) + + ctx := sql.NewEmptyContext() + expected := time.Date(2018, time.May, 3, 0, 0, 0, 0, time.UTC) + + result, err := f.Eval(ctx, sql.Row{"2018-05-02"}) + require.NoError(err) + require.Equal(expected, result) + + result, err = f.Eval(ctx, sql.Row{nil}) + require.NoError(err) + require.Nil(result) + + _, err = f.Eval(ctx, sql.Row{"asdasdasd"}) + require.Error(err) +} +func TestDateSub(t *testing.T) { + require := require.New(t) + + _, err := NewDateSub() + require.Error(err) + + _, err = NewDateSub(expression.NewLiteral("2018-05-02", sql.Text)) + require.Error(err) + + _, err = NewDateSub( + expression.NewLiteral("2018-05-02", sql.Text), + expression.NewLiteral(int64(1), sql.Int64), + ) + require.Error(err) + + f, err := NewDateSub( + expression.NewGetField(0, sql.Text, "foo", false), + expression.NewInterval( + expression.NewLiteral(int64(1), sql.Int64), + "DAY", + ), + ) + require.NoError(err) + + ctx := sql.NewEmptyContext() + expected := time.Date(2018, time.May, 1, 0, 0, 0, 0, time.UTC) + + result, err := f.Eval(ctx, sql.Row{"2018-05-02"}) + require.NoError(err) + require.Equal(expected, result) + + result, err = f.Eval(ctx, sql.Row{nil}) + require.NoError(err) + require.Nil(result) + + _, err = f.Eval(ctx, sql.Row{"asdasdasd"}) + require.Error(err) +} diff --git a/sql/expression/function/explode.go b/sql/expression/function/explode.go new file mode 100644 index 000000000..51cd2b66b --- /dev/null +++ b/sql/expression/function/explode.go @@ -0,0 +1,91 @@ +package function + +import ( + "fmt" + + "github.com/src-d/go-mysql-server/sql" +) + +// Explode is a function that generates a row for each value of its child. +// It is a placeholder expression node. +type Explode struct { + Child sql.Expression +} + +// NewExplode creates a new Explode function. +func NewExplode(child sql.Expression) sql.Expression { + return &Explode{child} +} + +// Resolved implements the sql.Expression interface. +func (e *Explode) Resolved() bool { return e.Child.Resolved() } + +// Children implements the sql.Expression interface. +func (e *Explode) Children() []sql.Expression { return []sql.Expression{e.Child} } + +// IsNullable implements the sql.Expression interface. +func (e *Explode) IsNullable() bool { return e.Child.IsNullable() } + +// Type implements the sql.Expression interface. +func (e *Explode) Type() sql.Type { + return sql.UnderlyingType(e.Child.Type()) +} + +// Eval implements the sql.Expression interface. +func (e *Explode) Eval(*sql.Context, sql.Row) (interface{}, error) { + panic("eval method of Explode is only a placeholder") +} + +func (e *Explode) String() string { + return fmt.Sprintf("EXPLODE(%s)", e.Child) +} + +// WithChildren implements the Expression interface. +func (e *Explode) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(e, len(children), 1) + } + return NewExplode(children[0]), nil +} + +// Generate is a function that generates a row for each value of its child. +// This is the non-placeholder counterpart of Explode. +type Generate struct { + Child sql.Expression +} + +// NewGenerate creates a new Generate function. +func NewGenerate(child sql.Expression) sql.Expression { + return &Generate{child} +} + +// Resolved implements the sql.Expression interface. +func (e *Generate) Resolved() bool { return e.Child.Resolved() } + +// Children implements the sql.Expression interface. +func (e *Generate) Children() []sql.Expression { return []sql.Expression{e.Child} } + +// IsNullable implements the sql.Expression interface. +func (e *Generate) IsNullable() bool { return e.Child.IsNullable() } + +// Type implements the sql.Expression interface. +func (e *Generate) Type() sql.Type { + return e.Child.Type() +} + +// Eval implements the sql.Expression interface. +func (e *Generate) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { + return e.Child.Eval(ctx, row) +} + +func (e *Generate) String() string { + return fmt.Sprintf("EXPLODE(%s)", e.Child) +} + +// WithChildren implements the Expression interface. +func (e *Generate) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(e, len(children), 1) + } + return NewGenerate(children[0]), nil +} diff --git a/sql/expression/function/greatest_least.go b/sql/expression/function/greatest_least.go new file mode 100644 index 000000000..9e822d8ed --- /dev/null +++ b/sql/expression/function/greatest_least.go @@ -0,0 +1,308 @@ +package function + +import ( + "fmt" + "strconv" + "strings" + + "github.com/src-d/go-mysql-server/sql" + "gopkg.in/src-d/go-errors.v1" +) + +var ErrUintOverflow = errors.NewKind( + "Unsigned integer too big to fit on signed integer") + +// compEval is used to implement Greatest/Least Eval() using a comparison function +func compEval( + returnType sql.Type, + args []sql.Expression, + ctx *sql.Context, + row sql.Row, + cmp compareFn, +) (interface{}, error) { + + if returnType == sql.Null { + return nil, nil + } + + var selectedNum float64 + var selectedString string + + for i, arg := range args { + val, err := arg.Eval(ctx, row) + if err != nil { + return nil, err + } + + switch t := val.(type) { + case int, int8, int16, int32, int64, uint, + uint8, uint16, uint32, uint64: + switch x := t.(type) { + case int: + t = int64(x) + case int8: + t = int64(x) + case int16: + t = int64(x) + case int32: + t = int64(x) + case uint: + i := int64(x) + if i < 0 { + return nil, ErrUintOverflow.New() + } + t = i + case uint64: + i := int64(x) + if i < 0 { + return nil, ErrUintOverflow.New() + } + t = i + case uint8: + t = int64(x) + case uint16: + t = int64(x) + case uint32: + t = int64(x) + } + ival := t.(int64) + if i == 0 || cmp(ival, int64(selectedNum)) { + selectedNum = float64(ival) + } + case float32, float64: + if x, ok := t.(float32); ok { + t = float64(x) + } + + fval := t.(float64) + if i == 0 || cmp(fval, float64(selectedNum)) { + selectedNum = fval + } + + case string: + if returnType == sql.Text && (i == 0 || cmp(t, selectedString)) { + selectedString = t + } + + fval, err := strconv.ParseFloat(t, 64) + if err != nil { + // MySQL just ignores non numerically convertible string arguments + // when mixed with numeric ones + continue + } + + if i == 0 || cmp(fval, selectedNum) { + selectedNum = fval + } + default: + return nil, ErrUnsupportedType.New(t) + } + + } + + switch returnType { + case sql.Int64: + return int64(selectedNum), nil + case sql.Text: + return selectedString, nil + } + + // sql.Float64 + return float64(selectedNum), nil +} + +// compRetType is used to determine the type from args based on the rules described for +// Greatest/Least +func compRetType(args ...sql.Expression) (sql.Type, error) { + if len(args) == 0 { + return nil, sql.ErrInvalidArgumentNumber.New("LEAST", "1 or more", 0) + } + + allString := true + allInt := true + + for _, arg := range args { + argType := arg.Type() + if sql.IsTuple(argType) { + return nil, sql.ErrInvalidType.New("tuple") + } else if sql.IsNumber(argType) { + allString = false + if sql.IsDecimal(argType) { + allString = false + allInt = false + } + } else if sql.IsText(argType) { + allInt = false + } else if argType == sql.Null { + // When a Null is present the return will always de Null + return sql.Null, nil + } else { + return nil, ErrUnsupportedType.New(argType) + } + } + + if allString { + return sql.Text, nil + } else if allInt { + return sql.Int64, nil + } + + return sql.Float64, nil +} + +// Greatest returns the argument with the greatest numerical or string value. It allows for +// numeric (ints anf floats) and string arguments and will return the used type +// when all arguments are of the same type or floats if there are numerically +// convertible strings or integers mixed with floats. When ints or floats +// are mixed with non numerically convertible strings, those are ignored. +type Greatest struct { + Args []sql.Expression + returnType sql.Type +} + +// ErrUnsupportedType is returned when an argument to Greatest or Latest is not numeric or string +var ErrUnsupportedType = errors.NewKind("unsupported type for greatest/latest argument: %T") + +// NewGreatest creates a new Greatest UDF +func NewGreatest(args ...sql.Expression) (sql.Expression, error) { + retType, err := compRetType(args...) + if err != nil { + return nil, err + } + + return &Greatest{args, retType}, nil +} + +// Type implements the Expression interface. +func (f *Greatest) Type() sql.Type { return f.returnType } + +// IsNullable implements the Expression interface. +func (f *Greatest) IsNullable() bool { + for _, arg := range f.Args { + if arg.IsNullable() { + return true + } + } + return false +} + +func (f *Greatest) String() string { + var args = make([]string, len(f.Args)) + for i, arg := range f.Args { + args[i] = arg.String() + } + return fmt.Sprintf("greatest(%s)", strings.Join(args, ", ")) +} + +// WithChildren implements the Expression interface. +func (f *Greatest) WithChildren(children ...sql.Expression) (sql.Expression, error) { + return NewGreatest(children...) +} + +// Resolved implements the Expression interface. +func (f *Greatest) Resolved() bool { + for _, arg := range f.Args { + if !arg.Resolved() { + return false + } + } + return true +} + +// Children implements the Expression interface. +func (f *Greatest) Children() []sql.Expression { return f.Args } + +type compareFn func(interface{}, interface{}) bool + +func greaterThan(a, b interface{}) bool { + switch i := a.(type) { + case int64: + return i > b.(int64) + case float64: + return i > b.(float64) + case string: + return i > b.(string) + } + panic("Implementation error on greaterThan") +} + +func lessThan(a, b interface{}) bool { + switch i := a.(type) { + case int64: + return i < b.(int64) + case float64: + return i < b.(float64) + case string: + return i < b.(string) + } + panic("Implementation error on lessThan") +} + +// Eval implements the Expression interface. +func (f *Greatest) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { + return compEval(f.returnType, f.Args, ctx, row, greaterThan) +} + +// Least returns the argument with the least numerical or string value. It allows for +// numeric (ints anf floats) and string arguments and will return the used type +// when all arguments are of the same type or floats if there are numerically +// convertible strings or integers mixed with floats. When ints or floats +// are mixed with non numerically convertible strings, those are ignored. +type Least struct { + Args []sql.Expression + returnType sql.Type +} + +// NewLeast creates a new Least UDF +func NewLeast(args ...sql.Expression) (sql.Expression, error) { + retType, err := compRetType(args...) + if err != nil { + return nil, err + } + + return &Least{args, retType}, nil +} + +// Type implements the Expression interface. +func (f *Least) Type() sql.Type { return f.returnType } + +// IsNullable implements the Expression interface. +func (f *Least) IsNullable() bool { + for _, arg := range f.Args { + if arg.IsNullable() { + return true + } + } + return false +} + +func (f *Least) String() string { + var args = make([]string, len(f.Args)) + for i, arg := range f.Args { + args[i] = arg.String() + } + return fmt.Sprintf("least(%s)", strings.Join(args, ", ")) +} + +// WithChildren implements the Expression interface. +func (f *Least) WithChildren(children ...sql.Expression) (sql.Expression, error) { + return NewLeast(children...) +} + +// Resolved implements the Expression interface. +func (f *Least) Resolved() bool { + for _, arg := range f.Args { + if !arg.Resolved() { + return false + } + } + return true +} + +// Children implements the Expression interface. +func (f *Least) Children() []sql.Expression { return f.Args } + +// Eval implements the Expression interface. +func (f *Least) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { + return compEval(f.returnType, f.Args, ctx, row, lessThan) +} diff --git a/sql/expression/function/greatest_least_test.go b/sql/expression/function/greatest_least_test.go new file mode 100644 index 000000000..b75ea0bba --- /dev/null +++ b/sql/expression/function/greatest_least_test.go @@ -0,0 +1,214 @@ +package function + +import ( + "testing" + "unsafe" + + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/stretchr/testify/require" +) + +func TestGreatest(t *testing.T) { + testCases := []struct { + name string + args []sql.Expression + expected interface{} + }{ + { + "null", + []sql.Expression{ + expression.NewLiteral(nil, sql.Null), + expression.NewLiteral(5, sql.Int64), + expression.NewLiteral(1, sql.Int64), + }, + nil, + }, + { + "negative and all ints", + []sql.Expression{ + expression.NewLiteral(int64(-1), sql.Int64), + expression.NewLiteral(int64(5), sql.Int64), + expression.NewLiteral(int64(1), sql.Int64), + }, + int64(5), + }, + { + "string mixed", + []sql.Expression{ + expression.NewLiteral(string("9"), sql.Text), + expression.NewLiteral(int64(5), sql.Int64), + expression.NewLiteral(int64(1), sql.Int64), + }, + float64(9), + }, + { + "unconvertible string mixed ignored", + []sql.Expression{ + expression.NewLiteral(string("10.5"), sql.Text), + expression.NewLiteral(string("foobar"), sql.Int64), + expression.NewLiteral(int64(5), sql.Int64), + expression.NewLiteral(int64(1), sql.Int64), + }, + float64(10.5), + }, + { + "float mixed", + []sql.Expression{ + expression.NewLiteral(float64(10.0), sql.Float64), + expression.NewLiteral(int(5), sql.Int64), + expression.NewLiteral(int(1), sql.Int64), + }, + float64(10.0), + }, + { + "all strings", + []sql.Expression{ + expression.NewLiteral("aaa", sql.Text), + expression.NewLiteral("bbb", sql.Text), + expression.NewLiteral("9999", sql.Text), + expression.NewLiteral("", sql.Text), + }, + "bbb", + }, + { + "all strings and empty", + []sql.Expression{ + expression.NewLiteral("aaa", sql.Text), + expression.NewLiteral("bbb", sql.Text), + expression.NewLiteral("9999", sql.Text), + expression.NewLiteral("", sql.Text), + }, + "bbb", + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + require := require.New(t) + + f, err := NewGreatest(tt.args...) + require.NoError(err) + + output, err := f.Eval(sql.NewEmptyContext(), nil) + require.NoError(err) + require.Equal(tt.expected, output) + }) + } +} + +func TestGreatestUnsignedOverflow(t *testing.T) { + require := require.New(t) + + var x int + var gr sql.Expression + var err error + + switch unsafe.Sizeof(x) { + case 4: + gr, err = NewGreatest( + expression.NewLiteral(int32(1), sql.Int32), + expression.NewLiteral(uint32(4294967295), sql.Uint32), + ) + require.NoError(err) + case 8: + gr, err = NewGreatest( + expression.NewLiteral(int64(1), sql.Int64), + expression.NewLiteral(uint64(18446744073709551615), sql.Uint64), + ) + require.NoError(err) + default: + // non 32/64 bits?? + return + } + + _, err = gr.Eval(sql.NewEmptyContext(), nil) + require.EqualError(err, "Unsigned integer too big to fit on signed integer") +} + +func TestLeast(t *testing.T) { + testCases := []struct { + name string + args []sql.Expression + expected interface{} + }{ + { + "null", + []sql.Expression{ + expression.NewLiteral(nil, sql.Null), + expression.NewLiteral(5, sql.Int64), + expression.NewLiteral(1, sql.Int64), + }, + nil, + }, + { + "negative and all ints", + []sql.Expression{ + expression.NewLiteral(int64(-1), sql.Int64), + expression.NewLiteral(int64(5), sql.Int64), + expression.NewLiteral(int64(1), sql.Int64), + }, + int64(-1), + }, + { + "string mixed", + []sql.Expression{ + expression.NewLiteral(string("10"), sql.Text), + expression.NewLiteral(int64(5), sql.Int64), + expression.NewLiteral(int64(1), sql.Int64), + }, + float64(1), + }, + { + "unconvertible string mixed ignored", + []sql.Expression{ + expression.NewLiteral(string("10.5"), sql.Text), + expression.NewLiteral(string("foobar"), sql.Int64), + expression.NewLiteral(int64(5), sql.Int64), + expression.NewLiteral(int64(1), sql.Int64), + }, + float64(1), + }, + { + "float mixed", + []sql.Expression{ + expression.NewLiteral(float64(10.0), sql.Float64), + expression.NewLiteral(int(5), sql.Int64), + expression.NewLiteral(int(1), sql.Int64), + }, + float64(1.0), + }, + { + "all strings", + []sql.Expression{ + expression.NewLiteral("aaa", sql.Text), + expression.NewLiteral("bbb", sql.Text), + expression.NewLiteral("9999", sql.Text), + }, + "9999", + }, + { + "all strings and empty", + []sql.Expression{ + expression.NewLiteral("aaa", sql.Text), + expression.NewLiteral("bbb", sql.Text), + expression.NewLiteral("9999", sql.Text), + expression.NewLiteral("", sql.Text), + }, + "", + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + require := require.New(t) + + f, err := NewLeast(tt.args...) + require.NoError(err) + + output, err := f.Eval(sql.NewEmptyContext(), nil) + require.NoError(err) + require.Equal(tt.expected, output) + }) + } +} diff --git a/sql/expression/function/ifnull.go b/sql/expression/function/ifnull.go index 95c10e355..566d62ea6 100644 --- a/sql/expression/function/ifnull.go +++ b/sql/expression/function/ifnull.go @@ -3,8 +3,8 @@ package function import ( "fmt" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" ) // IfNull function returns the specified value IF the expression is NULL, otherwise return the expression. @@ -65,17 +65,10 @@ func (f *IfNull) String() string { return fmt.Sprintf("ifnull(%s, %s)", f.Left, f.Right) } -// TransformUp implements the Expression interface. -func (f *IfNull) TransformUp(fn sql.TransformExprFunc) (sql.Expression, error) { - left, err := f.Left.TransformUp(fn) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (f *IfNull) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 2 { + return nil, sql.ErrInvalidChildrenNumber.New(f, len(children), 2) } - - right, err := f.Right.TransformUp(fn) - if err != nil { - return nil, err - } - - return fn(NewIfNull(left, right)) + return NewIfNull(children[0], children[1]), nil } diff --git a/sql/expression/function/ifnull_test.go b/sql/expression/function/ifnull_test.go index 481975b12..1751fe215 100644 --- a/sql/expression/function/ifnull_test.go +++ b/sql/expression/function/ifnull_test.go @@ -3,9 +3,9 @@ package function import ( "testing" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestIfNull(t *testing.T) { diff --git a/sql/expression/function/isbinary.go b/sql/expression/function/isbinary.go index 09f6dd3d0..e3edf74d6 100644 --- a/sql/expression/function/isbinary.go +++ b/sql/expression/function/isbinary.go @@ -4,8 +4,8 @@ import ( "bytes" "fmt" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" ) // IsBinary is a function that returns whether a blob is binary or not. @@ -45,13 +45,12 @@ func (ib *IsBinary) String() string { return fmt.Sprintf("IS_BINARY(%s)", ib.Child) } -// TransformUp implements the Expression interface. -func (ib *IsBinary) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - child, err := ib.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (ib *IsBinary) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(ib, len(children), 1) } - return f(NewIsBinary(child)) + return NewIsBinary(children[0]), nil } // Type implements the Expression interface. diff --git a/sql/expression/function/isbinary_test.go b/sql/expression/function/isbinary_test.go index d8cc4ae2a..375ac03e5 100644 --- a/sql/expression/function/isbinary_test.go +++ b/sql/expression/function/isbinary_test.go @@ -3,9 +3,9 @@ package function import ( "testing" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestIsBinary(t *testing.T) { diff --git a/sql/expression/function/json_extract.go b/sql/expression/function/json_extract.go index b49796c5c..c3f8f65eb 100644 --- a/sql/expression/function/json_extract.go +++ b/sql/expression/function/json_extract.go @@ -6,7 +6,7 @@ import ( "strings" "github.com/oliveagle/jsonpath" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // JSONExtract extracts data from a json document using json paths. @@ -47,16 +47,11 @@ func (j *JSONExtract) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return nil, err } - js, err = sql.JSON.Convert(js) + doc, err := unmarshalVal(js) if err != nil { return nil, err } - var doc interface{} - if err := json.Unmarshal(js.([]byte), &doc); err != nil { - return nil, err - } - var result = make([]interface{}, len(j.Paths)) for i, p := range j.Paths { path, err := p.Eval(ctx, row) @@ -69,10 +64,12 @@ func (j *JSONExtract) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return nil, err } - result[i], err = jsonpath.JsonPathLookup(doc, path.(string)) + c, err := jsonpath.Compile(path.(string)) if err != nil { return nil, err } + + result[i], _ = c.Lookup(doc) // err ignored } if len(result) == 1 { @@ -82,6 +79,20 @@ func (j *JSONExtract) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return result, nil } +func unmarshalVal(v interface{}) (interface{}, error) { + v, err := sql.JSON.Convert(v) + if err != nil { + return nil, err + } + + var doc interface{} + if err := json.Unmarshal(v.([]byte), &doc); err != nil { + return nil, err + } + + return doc, nil +} + // IsNullable implements the sql.Expression interface. func (j *JSONExtract) IsNullable() bool { for _, p := range j.Paths { @@ -97,22 +108,9 @@ func (j *JSONExtract) Children() []sql.Expression { return append([]sql.Expression{j.JSON}, j.Paths...) } -// TransformUp implements the sql.Expression interface. -func (j *JSONExtract) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - json, err := j.JSON.TransformUp(f) - if err != nil { - return nil, err - } - - paths := make([]sql.Expression, len(j.Paths)) - for i, p := range j.Paths { - paths[i], err = p.TransformUp(f) - if err != nil { - return nil, err - } - } - - return f(&JSONExtract{json, paths}) +// WithChildren implements the Expression interface. +func (j *JSONExtract) WithChildren(children ...sql.Expression) (sql.Expression, error) { + return NewJSONExtract(children...) } func (j *JSONExtract) String() string { diff --git a/sql/expression/function/json_extract_test.go b/sql/expression/function/json_extract_test.go index 648f99e1a..06f94cbfc 100644 --- a/sql/expression/function/json_extract_test.go +++ b/sql/expression/function/json_extract_test.go @@ -1,11 +1,12 @@ package function import ( + "errors" "testing" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestJSONExtract(t *testing.T) { @@ -46,22 +47,30 @@ func TestJSONExtract(t *testing.T) { f sql.Expression row sql.Row expected interface{} + err error }{ - {f2, sql.Row{json, "$.b.c"}, "foo"}, - {f3, sql.Row{json, "$.b.c", "$.b.d"}, []interface{}{"foo", true}}, + {f2, sql.Row{json, "FOO"}, nil, errors.New("should start with '$'")}, + {f2, sql.Row{nil, "$.b.c"}, nil, nil}, + {f2, sql.Row{json, "$.foo"}, nil, nil}, + {f2, sql.Row{json, "$.b.c"}, "foo", nil}, + {f3, sql.Row{json, "$.b.c", "$.b.d"}, []interface{}{"foo", true}, nil}, {f4, sql.Row{json, "$.b.c", "$.b.d", "$.e[0][*]"}, []interface{}{ "foo", true, []interface{}{1., 2.}, - }}, + }, nil}, } for _, tt := range testCases { t.Run(tt.f.String(), func(t *testing.T) { require := require.New(t) - result, err := tt.f.Eval(sql.NewEmptyContext(), tt.row) - require.NoError(err) + if tt.err == nil { + require.NoError(err) + } else { + require.Equal(err.Error(), tt.err.Error()) + } + require.Equal(tt.expected, result) }) } diff --git a/sql/expression/function/json_unquote.go b/sql/expression/function/json_unquote.go new file mode 100644 index 000000000..8a0c42de3 --- /dev/null +++ b/sql/expression/function/json_unquote.go @@ -0,0 +1,138 @@ +package function + +import ( + "bytes" + "encoding/binary" + "encoding/hex" + "fmt" + "reflect" + "unicode/utf8" + + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" +) + +// JSONUnquote unquotes JSON value and returns the result as a utf8mb4 string. +// Returns NULL if the argument is NULL. +// An error occurs if the value starts and ends with double quotes but is not a valid JSON string literal. +type JSONUnquote struct { + expression.UnaryExpression +} + +// NewJSONUnquote creates a new JSONUnquote UDF. +func NewJSONUnquote(json sql.Expression) sql.Expression { + return &JSONUnquote{expression.UnaryExpression{Child: json}} +} + +func (js *JSONUnquote) String() string { + return fmt.Sprintf("JSON_UNQUOTE(%s)", js.Child) +} + +// Type implements the Expression interface. +func (*JSONUnquote) Type() sql.Type { + return sql.Text +} + +// WithChildren implements the Expression interface. +func (js *JSONUnquote) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(js, len(children), 1) + } + return NewJSONUnquote(children[0]), nil +} + +// Eval implements the Expression interface. +func (js *JSONUnquote) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { + json, err := js.Child.Eval(ctx, row) + if json == nil || err != nil { + return json, err + } + + ex, err := sql.Text.Convert(json) + if err != nil { + return nil, err + } + str, ok := ex.(string) + if !ok { + return nil, sql.ErrInvalidType.New(reflect.TypeOf(ex).String()) + } + + return unquote(str) +} + +// The implementation is taken from TiDB +// https://github.com/pingcap/tidb/blob/a594287e9f402037b06930026906547000006bb6/types/json/binary_functions.go#L89 +func unquote(s string) (string, error) { + ret := new(bytes.Buffer) + for i := 0; i < len(s); i++ { + if s[i] == '\\' { + i++ + if i == len(s) { + return "", fmt.Errorf("Missing a closing quotation mark in string") + } + switch s[i] { + case '"': + ret.WriteByte('"') + case 'b': + ret.WriteByte('\b') + case 'f': + ret.WriteByte('\f') + case 'n': + ret.WriteByte('\n') + case 'r': + ret.WriteByte('\r') + case 't': + ret.WriteByte('\t') + case '\\': + ret.WriteByte('\\') + case 'u': + if i+4 > len(s) { + return "", fmt.Errorf("Invalid unicode: %s", s[i+1:]) + } + char, size, err := decodeEscapedUnicode([]byte(s[i+1 : i+5])) + if err != nil { + return "", err + } + ret.Write(char[0:size]) + i += 4 + default: + // For all other escape sequences, backslash is ignored. + ret.WriteByte(s[i]) + } + } else { + ret.WriteByte(s[i]) + } + } + + str := ret.String() + strlen := len(str) + // Remove prefix and suffix '"'. + if strlen > 1 { + head, tail := str[0], str[strlen-1] + if head == '"' && tail == '"' { + return str[1 : strlen-1], nil + } + } + return str, nil +} + +// decodeEscapedUnicode decodes unicode into utf8 bytes specified in RFC 3629. +// According RFC 3629, the max length of utf8 characters is 4 bytes. +// And MySQL use 4 bytes to represent the unicode which must be in [0, 65536). +// The implementation is taken from TiDB: +// https://github.com/pingcap/tidb/blob/a594287e9f402037b06930026906547000006bb6/types/json/binary_functions.go#L136 +func decodeEscapedUnicode(s []byte) (char [4]byte, size int, err error) { + size, err = hex.Decode(char[0:2], s) + if err != nil || size != 2 { + // The unicode must can be represented in 2 bytes. + return char, 0, err + } + var unicode uint16 + err = binary.Read(bytes.NewReader(char[0:2]), binary.BigEndian, &unicode) + if err != nil { + return char, 0, err + } + size = utf8.RuneLen(rune(unicode)) + utf8.EncodeRune(char[0:size], rune(unicode)) + return +} diff --git a/sql/expression/function/json_unquote_test.go b/sql/expression/function/json_unquote_test.go new file mode 100644 index 000000000..d5d054f10 --- /dev/null +++ b/sql/expression/function/json_unquote_test.go @@ -0,0 +1,37 @@ +package function + +import ( + "testing" + + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/stretchr/testify/require" +) + +func TestJSONUnquote(t *testing.T) { + require := require.New(t) + js := NewJSONUnquote(expression.NewGetField(0, sql.Text, "json", false)) + + testCases := []struct { + row sql.Row + expected interface{} + err bool + }{ + {sql.Row{nil}, nil, false}, + {sql.Row{"\"abc\""}, `abc`, false}, + {sql.Row{"[1, 2, 3]"}, `[1, 2, 3]`, false}, + {sql.Row{"\"\t\u0032\""}, "\t2", false}, + {sql.Row{"\\"}, nil, true}, + } + + for _, tt := range testCases { + result, err := js.Eval(sql.NewEmptyContext(), tt.row) + + if !tt.err { + require.NoError(err) + require.Equal(tt.expected, result) + } else { + require.NotNil(err) + } + } +} diff --git a/sql/expression/function/length.go b/sql/expression/function/length.go new file mode 100644 index 000000000..49d46aaf8 --- /dev/null +++ b/sql/expression/function/length.go @@ -0,0 +1,91 @@ +package function + +import ( + "fmt" + "unicode/utf8" + + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" +) + +// Length returns the length of a string or binary content, either in bytes +// or characters. +type Length struct { + expression.UnaryExpression + CountType CountType +} + +// CountType is the kind of length count. +type CountType bool + +const ( + // NumBytes counts the number of bytes in a string or binary content. + NumBytes = CountType(false) + // NumChars counts the number of characters in a string or binary content. + NumChars = CountType(true) +) + +// NewLength returns a new LENGTH function. +func NewLength(e sql.Expression) sql.Expression { + return &Length{expression.UnaryExpression{Child: e}, NumBytes} +} + +// NewCharLength returns a new CHAR_LENGTH function. +func NewCharLength(e sql.Expression) sql.Expression { + return &Length{expression.UnaryExpression{Child: e}, NumChars} +} + +// WithChildren implements the Expression interface. +func (l *Length) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(l, len(children), 1) + } + + return &Length{expression.UnaryExpression{Child: children[0]}, l.CountType}, nil +} + +// Type implements the sql.Expression interface. +func (l *Length) Type() sql.Type { return sql.Int32 } + +func (l *Length) String() string { + if l.CountType == NumBytes { + return fmt.Sprintf("LENGTH(%s)", l.Child) + } + return fmt.Sprintf("CHAR_LENGTH(%s)", l.Child) +} + +// Eval implements the sql.Expression interface. +func (l *Length) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { + val, err := l.Child.Eval(ctx, row) + if err != nil { + return nil, err + } + + if val == nil { + return nil, nil + } + + var content string + switch l.Child.Type() { + case sql.Blob: + val, err = sql.Blob.Convert(val) + if err != nil { + return nil, err + } + + content = string(val.([]byte)) + default: + val, err = sql.Text.Convert(val) + if err != nil { + return nil, err + } + + content = string(val.(string)) + } + + if l.CountType == NumBytes { + return int32(len(content)), nil + } + + return int32(utf8.RuneCountInString(content)), nil +} diff --git a/sql/expression/function/length_test.go b/sql/expression/function/length_test.go new file mode 100644 index 000000000..59c6f95d9 --- /dev/null +++ b/sql/expression/function/length_test.go @@ -0,0 +1,104 @@ +package function + +import ( + "testing" + + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/stretchr/testify/require" +) + +func TestLength(t *testing.T) { + testCases := []struct { + name string + input interface{} + inputType sql.Type + fn func(sql.Expression) sql.Expression + expected interface{} + }{ + { + "length string", + "fóo", + sql.Text, + NewLength, + int32(4), + }, + { + "length binary", + []byte("fóo"), + sql.Blob, + NewLength, + int32(4), + }, + { + "length empty", + "", + sql.Blob, + NewLength, + int32(0), + }, + { + "length empty binary", + []byte{}, + sql.Blob, + NewLength, + int32(0), + }, + { + "length nil", + nil, + sql.Blob, + NewLength, + nil, + }, + { + "char_length string", + "fóo", + sql.Text, + NewCharLength, + int32(3), + }, + { + "char_length binary", + []byte("fóo"), + sql.Blob, + NewCharLength, + int32(3), + }, + { + "char_length empty", + "", + sql.Blob, + NewCharLength, + int32(0), + }, + { + "char_length empty binary", + []byte{}, + sql.Blob, + NewCharLength, + int32(0), + }, + { + "char_length nil", + nil, + sql.Blob, + NewCharLength, + nil, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + require := require.New(t) + + result, err := tt.fn(expression.NewGetField(0, tt.inputType, "foo", false)).Eval( + sql.NewEmptyContext(), + sql.Row{tt.input}, + ) + + require.NoError(err) + require.Equal(tt.expected, result) + }) + } +} diff --git a/sql/expression/function/logarithm.go b/sql/expression/function/logarithm.go index 18f74b1be..eb6355a83 100644 --- a/sql/expression/function/logarithm.go +++ b/sql/expression/function/logarithm.go @@ -5,9 +5,9 @@ import ( "math" "reflect" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) // ErrInvalidArgumentForLogarithm is returned when an invalid argument value is passed to a @@ -45,13 +45,12 @@ func (l *LogBase) String() string { } } -// TransformUp implements the Expression interface. -func (l *LogBase) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - child, err := l.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (l *LogBase) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(l, len(children), 1) } - return f(NewLogBase(l.base, child)) + return NewLogBase(l.base, children[0]), nil } // Type returns the resultant type of the function. @@ -108,26 +107,9 @@ func (l *Log) String() string { return fmt.Sprintf("log(%s, %s)", l.Left, l.Right) } -// TransformUp implements the Expression interface. -func (l *Log) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - var args = make([]sql.Expression, 2) - arg, err := l.Left.TransformUp(f) - if err != nil { - return nil, err - } - args[0] = arg - - arg, err = l.Right.TransformUp(f) - if err != nil { - return nil, err - } - args[1] = arg - expr, err := NewLog(args...) - if err != nil { - return nil, err - } - - return f(expr) +// WithChildren implements the Expression interface. +func (l *Log) WithChildren(children ...sql.Expression) (sql.Expression, error) { + return NewLog(children...) } // Children implements the Expression interface. diff --git a/sql/expression/function/logarithm_test.go b/sql/expression/function/logarithm_test.go index 1ab718c07..7c5c7d59d 100644 --- a/sql/expression/function/logarithm_test.go +++ b/sql/expression/function/logarithm_test.go @@ -1,14 +1,14 @@ package function import ( - "testing" - "math" "fmt" + "math" + "testing" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-errors.v1" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" + "gopkg.in/src-d/go-errors.v1" ) var epsilon = math.Nextafter(1, 2) - 1 @@ -42,7 +42,7 @@ func TestLn(t *testing.T) { require.True(tt.err.Is(err)) } else { require.NoError(err) - require.InEpsilonf(tt.expected, result, epsilon, fmt.Sprintf("Actual is: %v", result), ) + require.InEpsilonf(tt.expected, result, epsilon, fmt.Sprintf("Actual is: %v", result)) } }) } @@ -85,7 +85,7 @@ func TestLog2(t *testing.T) { require.True(tt.err.Is(err)) } else { require.NoError(err) - require.InEpsilonf(tt.expected, result, epsilon, fmt.Sprintf("Actual is: %v", result), ) + require.InEpsilonf(tt.expected, result, epsilon, fmt.Sprintf("Actual is: %v", result)) } }) } @@ -128,7 +128,7 @@ func TestLog10(t *testing.T) { require.True(tt.err.Is(err)) } else { require.NoError(err) - require.InEpsilonf(tt.expected, result, epsilon, fmt.Sprintf("Actual is: %v", result), ) + require.InEpsilonf(tt.expected, result, epsilon, fmt.Sprintf("Actual is: %v", result)) } }) } @@ -194,7 +194,7 @@ func TestLog(t *testing.T) { require.True(tt.err.Is(err)) } else { require.NoError(err) - require.InEpsilonf(tt.expected, result, epsilon, fmt.Sprintf("Actual is: %v", result), ) + require.InEpsilonf(tt.expected, result, epsilon, fmt.Sprintf("Actual is: %v", result)) } }) } diff --git a/sql/expression/function/lower_upper.go b/sql/expression/function/lower_upper.go index 519509c70..f0d6cdb9d 100644 --- a/sql/expression/function/lower_upper.go +++ b/sql/expression/function/lower_upper.go @@ -2,9 +2,10 @@ package function import ( "fmt" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" "strings" + + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" ) // Lower is a function that returns the lowercase of the text provided. @@ -43,13 +44,12 @@ func (l *Lower) String() string { return fmt.Sprintf("LOWER(%s)", l.Child) } -// TransformUp implements the Expression interface. -func (l *Lower) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - child, err := l.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (l *Lower) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(l, len(children), 1) } - return f(NewLower(child)) + return NewLower(children[0]), nil } // Type implements the Expression interface. @@ -93,13 +93,12 @@ func (u *Upper) String() string { return fmt.Sprintf("UPPER(%s)", u.Child) } -// TransformUp implements the Expression interface. -func (u *Upper) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - child, err := u.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (u *Upper) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(u, len(children), 1) } - return f(NewUpper(child)) + return NewUpper(children[0]), nil } // Type implements the Expression interface. diff --git a/sql/expression/function/lower_upper_test.go b/sql/expression/function/lower_upper_test.go index f2099daa5..e06dca8a9 100644 --- a/sql/expression/function/lower_upper_test.go +++ b/sql/expression/function/lower_upper_test.go @@ -3,9 +3,9 @@ package function import ( "testing" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestLower(t *testing.T) { diff --git a/sql/expression/function/nullif.go b/sql/expression/function/nullif.go index 93074700f..49b5a5d9d 100644 --- a/sql/expression/function/nullif.go +++ b/sql/expression/function/nullif.go @@ -3,8 +3,8 @@ package function import ( "fmt" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" ) // NullIf function compares two expressions and returns NULL if they are equal. Otherwise, the first expression is returned. @@ -57,17 +57,10 @@ func (f *NullIf) String() string { return fmt.Sprintf("nullif(%s, %s)", f.Left, f.Right) } -// TransformUp implements the Expression interface. -func (f *NullIf) TransformUp(fn sql.TransformExprFunc) (sql.Expression, error) { - left, err := f.Left.TransformUp(fn) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (f *NullIf) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 2 { + return nil, sql.ErrInvalidChildrenNumber.New(f, len(children), 2) } - - right, err := f.Right.TransformUp(fn) - if err != nil { - return nil, err - } - - return fn(NewNullIf(left, right)) + return NewNullIf(children[0], children[1]), nil } diff --git a/sql/expression/function/nullif_test.go b/sql/expression/function/nullif_test.go index 24dd83989..c62284cbe 100644 --- a/sql/expression/function/nullif_test.go +++ b/sql/expression/function/nullif_test.go @@ -3,9 +3,9 @@ package function import ( "testing" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestNullIf(t *testing.T) { @@ -28,6 +28,13 @@ func TestNullIf(t *testing.T) { ) require.Equal(t, sql.Text, f.Type()) + var3 := sql.VarChar(3) + f = NewNullIf( + expression.NewGetField(0, var3, "ex1", true), + expression.NewGetField(1, var3, "ex2", true), + ) + require.Equal(t, var3, f.Type()) + for _, tc := range testCases { v, err := f.Eval(sql.NewEmptyContext(), sql.NewRow(tc.ex1, tc.ex2)) require.NoError(t, err) diff --git a/sql/expression/function/regexp_matches.go b/sql/expression/function/regexp_matches.go new file mode 100644 index 000000000..417e91f5e --- /dev/null +++ b/sql/expression/function/regexp_matches.go @@ -0,0 +1,204 @@ +package function + +import ( + "fmt" + "regexp" + "strings" + + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + errors "gopkg.in/src-d/go-errors.v1" +) + +// RegexpMatches returns the matches of a regular expression. +type RegexpMatches struct { + Text sql.Expression + Pattern sql.Expression + Flags sql.Expression + + cacheable bool + re *regexp.Regexp +} + +// NewRegexpMatches creates a new RegexpMatches expression. +func NewRegexpMatches(args ...sql.Expression) (sql.Expression, error) { + var r RegexpMatches + switch len(args) { + case 3: + r.Flags = args[2] + fallthrough + case 2: + r.Text = args[0] + r.Pattern = args[1] + default: + return nil, sql.ErrInvalidArgumentNumber.New("regexp_matches", "2 or 3", len(args)) + } + + if canBeCached(r.Pattern) && (r.Flags == nil || canBeCached(r.Flags)) { + r.cacheable = true + } + + return &r, nil +} + +// Type implements the sql.Expression interface. +func (r *RegexpMatches) Type() sql.Type { return sql.Array(sql.Text) } + +// IsNullable implements the sql.Expression interface. +func (r *RegexpMatches) IsNullable() bool { return true } + +// Children implements the sql.Expression interface. +func (r *RegexpMatches) Children() []sql.Expression { + var result = []sql.Expression{r.Text, r.Pattern} + if r.Flags != nil { + result = append(result, r.Flags) + } + return result +} + +// Resolved implements the sql.Expression interface. +func (r *RegexpMatches) Resolved() bool { + return r.Text.Resolved() && r.Pattern.Resolved() && (r.Flags == nil || r.Flags.Resolved()) +} + +// WithChildren implements the sql.Expression interface. +func (r *RegexpMatches) WithChildren(children ...sql.Expression) (sql.Expression, error) { + required := 2 + if r.Flags != nil { + required = 3 + } + + if len(children) != required { + return nil, sql.ErrInvalidChildrenNumber.New(r, len(children), required) + } + + return NewRegexpMatches(children...) +} + +func (r *RegexpMatches) String() string { + var args []string + for _, e := range r.Children() { + args = append(args, e.String()) + } + return fmt.Sprintf("regexp_matches(%s)", strings.Join(args, ", ")) +} + +// Eval implements the sql.Expression interface. +func (r *RegexpMatches) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { + span, ctx := ctx.Span("function.RegexpMatches") + defer span.Finish() + + var re *regexp.Regexp + var err error + if r.cacheable { + if r.re == nil { + r.re, err = r.compileRegex(ctx, nil) + if err != nil { + return nil, err + } + + if r.re == nil { + return nil, nil + } + } + re = r.re + } else { + re, err = r.compileRegex(ctx, row) + if err != nil { + return nil, err + } + + if re == nil { + return nil, nil + } + } + + text, err := r.Text.Eval(ctx, row) + if err != nil { + return nil, err + } + + if text == nil { + return nil, nil + } + + text, err = sql.Text.Convert(text) + if err != nil { + return nil, err + } + + matches := re.FindAllStringSubmatch(text.(string), -1) + if len(matches) == 0 { + return nil, nil + } + + var result []interface{} + for _, m := range matches { + for _, sm := range m { + result = append(result, sm) + } + } + + return result, nil +} + +func (r *RegexpMatches) compileRegex(ctx *sql.Context, row sql.Row) (*regexp.Regexp, error) { + pattern, err := r.Pattern.Eval(ctx, row) + if err != nil { + return nil, err + } + + if pattern == nil { + return nil, nil + } + + pattern, err = sql.Text.Convert(pattern) + if err != nil { + return nil, err + } + + var flags string + if r.Flags != nil { + f, err := r.Flags.Eval(ctx, row) + if err != nil { + return nil, err + } + + if f == nil { + return nil, nil + } + + f, err = sql.Text.Convert(f) + if err != nil { + return nil, err + } + + flags = f.(string) + for _, f := range flags { + if !validRegexpFlags[f] { + return nil, errInvalidRegexpFlag.New(f) + } + } + + flags = fmt.Sprintf("(?%s)", flags) + } + + return regexp.Compile(flags + pattern.(string)) +} + +var errInvalidRegexpFlag = errors.NewKind("invalid regexp flag: %v") + +var validRegexpFlags = map[rune]bool{ + 'i': true, +} + +func canBeCached(e sql.Expression) bool { + var hasCols bool + expression.Inspect(e, func(e sql.Expression) bool { + if _, ok := e.(*expression.GetField); ok { + hasCols = true + } + return true + }) + return !hasCols +} diff --git a/sql/expression/function/regexp_matches_test.go b/sql/expression/function/regexp_matches_test.go new file mode 100644 index 000000000..4a7fc35c5 --- /dev/null +++ b/sql/expression/function/regexp_matches_test.go @@ -0,0 +1,146 @@ +package function + +import ( + "testing" + + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/stretchr/testify/require" + + errors "gopkg.in/src-d/go-errors.v1" +) + +func TestRegexpMatches(t *testing.T) { + testCases := []struct { + pattern interface{} + text interface{} + flags interface{} + expected interface{} + err *errors.Kind + }{ + { + `^foobar(.*)bye$`, + "foobarhellobye", + "", + []interface{}{"foobarhellobye", "hello"}, + nil, + }, + { + "bop", + "bopbeepbop", + "", + []interface{}{"bop", "bop"}, + nil, + }, + { + "bop", + "bopbeepBop", + "i", + []interface{}{"bop", "Bop"}, + nil, + }, + { + "bop", + "helloworld", + "", + nil, + nil, + }, + { + "foo", + "", + "", + nil, + nil, + }, + { + "", + "", + "", + []interface{}{""}, + nil, + }, + { + "bop", + nil, + "", + nil, + nil, + }, + { + "bop", + "beep", + nil, + nil, + nil, + }, + { + nil, + "bop", + "", + nil, + nil, + }, + { + "bop", + "bopbeepBop", + "ix", + nil, + errInvalidRegexpFlag, + }, + } + + t.Run("cacheable", func(t *testing.T) { + for _, tt := range testCases { + var flags sql.Expression + if tt.flags != "" { + flags = expression.NewLiteral(tt.flags, sql.Text) + } + f, err := NewRegexpMatches( + expression.NewLiteral(tt.text, sql.Text), + expression.NewLiteral(tt.pattern, sql.Text), + flags, + ) + require.NoError(t, err) + + t.Run(f.String(), func(t *testing.T) { + require := require.New(t) + result, err := f.Eval(sql.NewEmptyContext(), nil) + if tt.err == nil { + require.NoError(err) + require.Equal(tt.expected, result) + } else { + require.Error(err) + require.True(tt.err.Is(err)) + } + }) + } + }) + + t.Run("not cacheable", func(t *testing.T) { + for _, tt := range testCases { + var flags sql.Expression + if tt.flags != "" { + flags = expression.NewGetField(2, sql.Text, "x", false) + } + f, err := NewRegexpMatches( + expression.NewGetField(0, sql.Text, "x", false), + expression.NewGetField(1, sql.Text, "x", false), + flags, + ) + require.NoError(t, err) + + t.Run(f.String(), func(t *testing.T) { + require := require.New(t) + result, err := f.Eval(sql.NewEmptyContext(), sql.Row{tt.text, tt.pattern, tt.flags}) + if tt.err == nil { + require.NoError(err) + require.Equal(tt.expected, result) + } else { + require.Error(err) + require.True(tt.err.Is(err)) + } + }) + } + }) +} diff --git a/sql/expression/function/registry.go b/sql/expression/function/registry.go index 6c10e2646..7d2586bba 100644 --- a/sql/expression/function/registry.go +++ b/sql/expression/function/registry.go @@ -3,70 +3,100 @@ package function import ( "math" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/aggregation" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression/function/aggregation" ) // Defaults is the function map with all the default functions. -var Defaults = sql.Functions{ - "count": sql.Function1(func(e sql.Expression) sql.Expression { - return aggregation.NewCount(e) - }), - "min": sql.Function1(func(e sql.Expression) sql.Expression { - return aggregation.NewMin(e) - }), - "max": sql.Function1(func(e sql.Expression) sql.Expression { - return aggregation.NewMax(e) - }), - "avg": sql.Function1(func(e sql.Expression) sql.Expression { - return aggregation.NewAvg(e) - }), - "sum": sql.Function1(func(e sql.Expression) sql.Expression { - return aggregation.NewSum(e) - }), - "is_binary": sql.Function1(NewIsBinary), - "substring": sql.FunctionN(NewSubstring), - "mid": sql.FunctionN(NewSubstring), - "substr": sql.FunctionN(NewSubstring), - "year": sql.Function1(NewYear), - "month": sql.Function1(NewMonth), - "day": sql.Function1(NewDay), - "weekday": sql.Function1(NewWeekday), - "hour": sql.Function1(NewHour), - "minute": sql.Function1(NewMinute), - "second": sql.Function1(NewSecond), - "dayofweek": sql.Function1(NewDayOfWeek), - "dayofyear": sql.Function1(NewDayOfYear), - "array_length": sql.Function1(NewArrayLength), - "split": sql.Function2(NewSplit), - "concat": sql.FunctionN(NewConcat), - "concat_ws": sql.FunctionN(NewConcatWithSeparator), - "coalesce": sql.FunctionN(NewCoalesce), - "lower": sql.Function1(NewLower), - "upper": sql.Function1(NewUpper), - "ceiling": sql.Function1(NewCeil), - "ceil": sql.Function1(NewCeil), - "floor": sql.Function1(NewFloor), - "round": sql.FunctionN(NewRound), - "connection_id": sql.Function0(NewConnectionID), - "soundex": sql.Function1(NewSoundex), - "json_extract": sql.FunctionN(NewJSONExtract), - "ln": sql.Function1(NewLogBaseFunc(float64(math.E))), - "log2": sql.Function1(NewLogBaseFunc(float64(2))), - "log10": sql.Function1(NewLogBaseFunc(float64(10))), - "log": sql.FunctionN(NewLog), - "rpad": sql.FunctionN(NewPadFunc(rPadType)), - "lpad": sql.FunctionN(NewPadFunc(lPadType)), - "sqrt": sql.Function1(NewSqrt), - "pow": sql.Function2(NewPower), - "power": sql.Function2(NewPower), - "ltrim": sql.Function1(NewTrimFunc(lTrimType)), - "rtrim": sql.Function1(NewTrimFunc(rTrimType)), - "trim": sql.Function1(NewTrimFunc(bTrimType)), - "reverse": sql.Function1(NewReverse), - "repeat": sql.Function2(NewRepeat), - "replace": sql.Function3(NewReplace), - "ifnull": sql.Function2(NewIfNull), - "nullif": sql.Function2(NewNullIf), - "now": sql.Function0(NewNow), +var Defaults = []sql.Function{ + sql.Function1{ + Name: "count", + Fn: func(e sql.Expression) sql.Expression { return aggregation.NewCount(e) }, + }, + sql.Function1{ + Name: "min", + Fn: func(e sql.Expression) sql.Expression { return aggregation.NewMin(e) }, + }, + sql.Function1{ + Name: "max", + Fn: func(e sql.Expression) sql.Expression { return aggregation.NewMax(e) }, + }, + sql.Function1{ + Name: "avg", + Fn: func(e sql.Expression) sql.Expression { return aggregation.NewAvg(e) }, + }, + sql.Function1{ + Name: "sum", + Fn: func(e sql.Expression) sql.Expression { return aggregation.NewSum(e) }, + }, + sql.Function1{ + Name: "first", + Fn: func(e sql.Expression) sql.Expression { return aggregation.NewFirst(e) }, + }, + sql.Function1{ + Name: "last", + Fn: func(e sql.Expression) sql.Expression { return aggregation.NewLast(e) }, + }, + sql.Function1{Name: "is_binary", Fn: NewIsBinary}, + sql.FunctionN{Name: "substring", Fn: NewSubstring}, + sql.Function3{Name: "substring_index", Fn: NewSubstringIndex}, + sql.FunctionN{Name: "mid", Fn: NewSubstring}, + sql.FunctionN{Name: "substr", Fn: NewSubstring}, + sql.Function1{Name: "date", Fn: NewDate}, + sql.Function1{Name: "year", Fn: NewYear}, + sql.Function1{Name: "month", Fn: NewMonth}, + sql.Function1{Name: "day", Fn: NewDay}, + sql.Function1{Name: "weekday", Fn: NewWeekday}, + sql.Function1{Name: "hour", Fn: NewHour}, + sql.Function1{Name: "minute", Fn: NewMinute}, + sql.Function1{Name: "second", Fn: NewSecond}, + sql.Function1{Name: "dayofweek", Fn: NewDayOfWeek}, + sql.Function1{Name: "dayofmonth", Fn: NewDay}, + sql.Function1{Name: "dayofyear", Fn: NewDayOfYear}, + sql.FunctionN{Name: "yearweek", Fn: NewYearWeek}, + sql.Function1{Name: "array_length", Fn: NewArrayLength}, + sql.Function2{Name: "split", Fn: NewSplit}, + sql.FunctionN{Name: "concat", Fn: NewConcat}, + sql.FunctionN{Name: "concat_ws", Fn: NewConcatWithSeparator}, + sql.FunctionN{Name: "coalesce", Fn: NewCoalesce}, + sql.Function1{Name: "lower", Fn: NewLower}, + sql.Function1{Name: "upper", Fn: NewUpper}, + sql.Function1{Name: "ceiling", Fn: NewCeil}, + sql.Function1{Name: "ceil", Fn: NewCeil}, + sql.Function1{Name: "floor", Fn: NewFloor}, + sql.FunctionN{Name: "round", Fn: NewRound}, + sql.Function0{Name: "connection_id", Fn: NewConnectionID}, + sql.Function1{Name: "soundex", Fn: NewSoundex}, + sql.FunctionN{Name: "json_extract", Fn: NewJSONExtract}, + sql.Function1{Name: "json_unquote", Fn: NewJSONUnquote}, + sql.Function1{Name: "ln", Fn: NewLogBaseFunc(float64(math.E))}, + sql.Function1{Name: "log2", Fn: NewLogBaseFunc(float64(2))}, + sql.Function1{Name: "log10", Fn: NewLogBaseFunc(float64(10))}, + sql.FunctionN{Name: "log", Fn: NewLog}, + sql.FunctionN{Name: "rpad", Fn: NewPadFunc(rPadType)}, + sql.FunctionN{Name: "lpad", Fn: NewPadFunc(lPadType)}, + sql.Function1{Name: "sqrt", Fn: NewSqrt}, + sql.Function2{Name: "pow", Fn: NewPower}, + sql.Function2{Name: "power", Fn: NewPower}, + sql.Function1{Name: "ltrim", Fn: NewTrimFunc(lTrimType)}, + sql.Function1{Name: "rtrim", Fn: NewTrimFunc(rTrimType)}, + sql.Function1{Name: "trim", Fn: NewTrimFunc(bTrimType)}, + sql.Function1{Name: "reverse", Fn: NewReverse}, + sql.Function2{Name: "repeat", Fn: NewRepeat}, + sql.Function3{Name: "replace", Fn: NewReplace}, + sql.Function2{Name: "ifnull", Fn: NewIfNull}, + sql.Function2{Name: "nullif", Fn: NewNullIf}, + sql.Function0{Name: "now", Fn: NewNow}, + sql.Function1{Name: "sleep", Fn: NewSleep}, + sql.Function1{Name: "to_base64", Fn: NewToBase64}, + sql.Function1{Name: "from_base64", Fn: NewFromBase64}, + sql.FunctionN{Name: "date_add", Fn: NewDateAdd}, + sql.FunctionN{Name: "date_sub", Fn: NewDateSub}, + sql.FunctionN{Name: "greatest", Fn: NewGreatest}, + sql.FunctionN{Name: "least", Fn: NewLeast}, + sql.Function1{Name: "length", Fn: NewLength}, + sql.Function1{Name: "char_length", Fn: NewCharLength}, + sql.Function1{Name: "character_length", Fn: NewCharLength}, + sql.Function1{Name: "explode", Fn: NewExplode}, + sql.FunctionN{Name: "regexp_matches", Fn: NewRegexpMatches}, } diff --git a/sql/expression/function/reverse_repeat_replace.go b/sql/expression/function/reverse_repeat_replace.go index 2bb99197e..cef9e9691 100644 --- a/sql/expression/function/reverse_repeat_replace.go +++ b/sql/expression/function/reverse_repeat_replace.go @@ -4,8 +4,8 @@ import ( "fmt" "strings" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "gopkg.in/src-d/go-errors.v1" ) @@ -39,7 +39,7 @@ func (r *Reverse) Eval( func reverseString(s string) string { r := []rune(s) - for i, j := 0, len(r) - 1; i < j; i, j = i+1, j-1 { + for i, j := 0, len(r)-1; i < j; i, j = i+1, j-1 { r[i], r[j] = r[j], r[i] } return string(r) @@ -49,13 +49,12 @@ func (r *Reverse) String() string { return fmt.Sprintf("reverse(%s)", r.Child) } -// TransformUp implements the Expression interface. -func (r *Reverse) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - child, err := r.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (r *Reverse) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(r, len(children), 1) } - return f(NewReverse(child)) + return NewReverse(children[0]), nil } // Type implements the Expression interface. @@ -84,18 +83,12 @@ func (r *Repeat) Type() sql.Type { return sql.Text } -// TransformUp implements the Expression interface. -func (r *Repeat) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - left, err := r.Left.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (r *Repeat) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 2 { + return nil, sql.ErrInvalidChildrenNumber.New(r, len(children), 2) } - - right, err := r.Right.TransformUp(f) - if err != nil { - return nil, err - } - return f(NewRepeat(left, right)) + return NewRepeat(children[0], children[1]), nil } // Eval implements the Expression interface. @@ -165,23 +158,12 @@ func (r *Replace) Type() sql.Type { return sql.Text } -// TransformUp implements the Expression interface. -func (r *Replace) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - str, err := r.str.TransformUp(f) - if err != nil { - return nil, err - } - - fromStr, err := r.fromStr.TransformUp(f) - if err != nil { - return nil, err - } - - toStr, err := r.toStr.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (r *Replace) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 3 { + return nil, sql.ErrInvalidChildrenNumber.New(r, len(children), 3) } - return f(NewReplace(str, fromStr, toStr)) + return NewReplace(children[0], children[1], children[2]), nil } // Eval implements the Expression interface. diff --git a/sql/expression/function/reverse_repeat_replace_test.go b/sql/expression/function/reverse_repeat_replace_test.go index 51266aa80..f04b16bc6 100644 --- a/sql/expression/function/reverse_repeat_replace_test.go +++ b/sql/expression/function/reverse_repeat_replace_test.go @@ -3,9 +3,9 @@ package function import ( "testing" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestReverse(t *testing.T) { diff --git a/sql/expression/function/rpad_lpad.go b/sql/expression/function/rpad_lpad.go index cc37eee34..12b33695b 100644 --- a/sql/expression/function/rpad_lpad.go +++ b/sql/expression/function/rpad_lpad.go @@ -5,8 +5,8 @@ import ( "reflect" "strings" + "github.com/src-d/go-mysql-server/sql" "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) var ErrDivisionByZero = errors.NewKind("division by zero") @@ -68,24 +68,9 @@ func (p *Pad) String() string { return fmt.Sprintf("rpad(%s, %s, %s)", p.str, p.length, p.padStr) } -// TransformUp implements the Expression interface. -func (p *Pad) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - str, err := p.str.TransformUp(f) - if err != nil { - return nil, err - } - - len, err := p.length.TransformUp(f) - if err != nil { - return nil, err - } - - padStr, err := p.padStr.TransformUp(f) - if err != nil { - return nil, err - } - padded, _ := NewPad(p.padType, str, len, padStr) - return f(padded) +// WithChildren implements the Expression interface. +func (p *Pad) WithChildren(children ...sql.Expression) (sql.Expression, error) { + return NewPad(p.padType, children...) } // Eval implements the Expression interface. diff --git a/sql/expression/function/rpad_lpad_test.go b/sql/expression/function/rpad_lpad_test.go index c6fd44cd3..81e1dbcae 100644 --- a/sql/expression/function/rpad_lpad_test.go +++ b/sql/expression/function/rpad_lpad_test.go @@ -3,8 +3,8 @@ package function import ( "testing" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" ) diff --git a/sql/expression/function/sleep.go b/sql/expression/function/sleep.go new file mode 100644 index 000000000..230a9596f --- /dev/null +++ b/sql/expression/function/sleep.go @@ -0,0 +1,73 @@ +package function + +import ( + "context" + "fmt" + "time" + + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" +) + +// Sleep is a function that just waits for the specified number of seconds +// and returns 0. +// It can be useful to test timeouts or long queries. +type Sleep struct { + expression.UnaryExpression +} + +// NewSleep creates a new Sleep expression. +func NewSleep(e sql.Expression) sql.Expression { + return &Sleep{expression.UnaryExpression{Child: e}} +} + +// Eval implements the Expression interface. +func (s *Sleep) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { + child, err := s.Child.Eval(ctx, row) + + if err != nil { + return nil, err + } + + if child == nil { + return nil, nil + } + + child, err = sql.Float64.Convert(child) + if err != nil { + return nil, err + } + + t := time.NewTimer(time.Duration(child.(float64)*1000) * time.Millisecond) + defer t.Stop() + + select { + case <-ctx.Done(): + return 0, context.Canceled + case <-t.C: + return 0, nil + } +} + +// String implements the Stringer interface. +func (s *Sleep) String() string { + return fmt.Sprintf("SLEEP(%s)", s.Child) +} + +// IsNullable implements the Expression interface. +func (s *Sleep) IsNullable() bool { + return false +} + +// WithChildren implements the Expression interface. +func (s *Sleep) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(s, len(children), 1) + } + return NewSleep(children[0]), nil +} + +// Type implements the Expression interface. +func (s *Sleep) Type() sql.Type { + return sql.Int32 +} diff --git a/sql/expression/function/sleep_test.go b/sql/expression/function/sleep_test.go new file mode 100644 index 000000000..65ce02478 --- /dev/null +++ b/sql/expression/function/sleep_test.go @@ -0,0 +1,50 @@ +package function + +import ( + "testing" + "time" + + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/stretchr/testify/require" +) + +func TestSleep(t *testing.T) { + f := NewSleep( + expression.NewGetField(0, sql.Text, "n", false), + ) + testCases := []struct { + name string + row sql.Row + expected interface{} + waitTime float64 + err bool + }{ + {"null input", sql.NewRow(nil), nil, 0, false}, + {"string input", sql.NewRow("foo"), nil, 0, true}, + {"int input", sql.NewRow(3), int(0), 3.0, false}, + {"number is zero", sql.NewRow(0), int(0), 0, false}, + {"negative number", sql.NewRow(-4), int(0), 0, false}, + {"positive number", sql.NewRow(4.48), int(0), 4.48, false}, + } + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + t.Helper() + require := require.New(t) + ctx := sql.NewEmptyContext() + + t1 := time.Now() + v, err := f.Eval(ctx, tt.row) + t2 := time.Now() + if tt.err { + require.Error(err) + } else { + require.NoError(err) + require.Equal(tt.expected, v) + + waited := t2.Sub(t1).Seconds() + require.InDelta(waited, tt.waitTime, 0.1) + } + }) + } +} diff --git a/sql/expression/function/soundex.go b/sql/expression/function/soundex.go index fb95dda8c..37774228e 100644 --- a/sql/expression/function/soundex.go +++ b/sql/expression/function/soundex.go @@ -5,8 +5,8 @@ import ( "strings" "unicode" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" ) // Soundex is a function that returns the soundex of a string. Two strings that @@ -87,13 +87,12 @@ func (s *Soundex) String() string { return fmt.Sprintf("SOUNDEX(%s)", s.Child) } -// TransformUp implements the Expression interface. -func (s *Soundex) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - child, err := s.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (s *Soundex) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(s, len(children), 1) } - return f(NewSoundex(child)) + return NewSoundex(children[0]), nil } // Type implements the Expression interface. diff --git a/sql/expression/function/soundex_test.go b/sql/expression/function/soundex_test.go index 32c1b65cc..eb5739e56 100644 --- a/sql/expression/function/soundex_test.go +++ b/sql/expression/function/soundex_test.go @@ -3,9 +3,9 @@ package function import ( "testing" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestSoundex(t *testing.T) { diff --git a/sql/expression/function/split.go b/sql/expression/function/split.go index 0c9955b7b..20e2a49f9 100644 --- a/sql/expression/function/split.go +++ b/sql/expression/function/split.go @@ -4,8 +4,8 @@ import ( "fmt" "regexp" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" ) // Split receives a string and returns the parts of it splitted by a @@ -76,17 +76,10 @@ func (f *Split) String() string { return fmt.Sprintf("split(%s, %s)", f.Left, f.Right) } -// TransformUp implements the Expression interface. -func (f *Split) TransformUp(fn sql.TransformExprFunc) (sql.Expression, error) { - left, err := f.Left.TransformUp(fn) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (f *Split) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 2 { + return nil, sql.ErrInvalidChildrenNumber.New(f, len(children), 2) } - - right, err := f.Right.TransformUp(fn) - if err != nil { - return nil, err - } - - return fn(NewSplit(left, right)) + return NewSplit(children[0], children[1]), nil } diff --git a/sql/expression/function/split_test.go b/sql/expression/function/split_test.go index 0de25db93..478d3b749 100644 --- a/sql/expression/function/split_test.go +++ b/sql/expression/function/split_test.go @@ -3,9 +3,9 @@ package function import ( "testing" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestSplit(t *testing.T) { diff --git a/sql/expression/function/sqrt_power.go b/sql/expression/function/sqrt_power.go index 60061c21a..020c8b7bf 100644 --- a/sql/expression/function/sqrt_power.go +++ b/sql/expression/function/sqrt_power.go @@ -4,8 +4,8 @@ import ( "fmt" "math" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" ) // Sqrt is a function that returns the square value of the number provided. @@ -32,13 +32,12 @@ func (s *Sqrt) IsNullable() bool { return s.Child.IsNullable() } -// TransformUp implements the Expression interface. -func (s *Sqrt) TransformUp(fn sql.TransformExprFunc) (sql.Expression, error) { - child, err := s.Child.TransformUp(fn) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (s *Sqrt) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(s, len(children), 1) } - return fn(NewSqrt(child)) + return NewSqrt(children[0]), nil } // Eval implements the Expression interface. @@ -86,19 +85,12 @@ func (p *Power) String() string { return fmt.Sprintf("power(%s, %s)", p.Left, p.Right) } -// TransformUp implements the Expression interface. -func (p *Power) TransformUp(fn sql.TransformExprFunc) (sql.Expression, error) { - left, err := p.Left.TransformUp(fn) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (p *Power) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 2 { + return nil, sql.ErrInvalidChildrenNumber.New(p, len(children), 2) } - - right, err := p.Right.TransformUp(fn) - if err != nil { - return nil, err - } - - return fn(NewPower(left, right)) + return NewPower(children[0], children[0]), nil } // Eval implements the Expression interface. diff --git a/sql/expression/function/sqrt_power_test.go b/sql/expression/function/sqrt_power_test.go index 8e2d8f337..c148a445b 100644 --- a/sql/expression/function/sqrt_power_test.go +++ b/sql/expression/function/sqrt_power_test.go @@ -1,12 +1,12 @@ package function import ( - "testing" "math" + "testing" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestSqrt(t *testing.T) { diff --git a/sql/expression/function/substring.go b/sql/expression/function/substring.go index 68f9c51fa..c5227b9bc 100644 --- a/sql/expression/function/substring.go +++ b/sql/expression/function/substring.go @@ -3,8 +3,9 @@ package function import ( "fmt" "reflect" + "strings" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // Substring is a function to return a part of a string. @@ -141,30 +142,118 @@ func (s *Substring) Resolved() bool { // Type implements the Expression interface. func (*Substring) Type() sql.Type { return sql.Text } -// TransformUp implements the Expression interface. -func (s *Substring) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - str, err := s.str.TransformUp(f) +/// WithChildren implements the Expression interface. +func (*Substring) WithChildren(children ...sql.Expression) (sql.Expression, error) { + return NewSubstring(children...) +} + +// SubstringIndex returns the substring from string str before count occurrences of the delimiter delim. +// If count is positive, everything to the left of the final delimiter (counting from the left) is returned. +// If count is negative, everything to the right of the final delimiter (counting from the right) is returned. +// SUBSTRING_INDEX() performs a case-sensitive match when searching for delim. +type SubstringIndex struct { + str sql.Expression + delim sql.Expression + count sql.Expression +} + +// NewSubstringIndex creates a new SubstringIndex UDF. +func NewSubstringIndex(str, delim, count sql.Expression) sql.Expression { + return &SubstringIndex{str, delim, count} +} + +// Children implements the Expression interface. +func (s *SubstringIndex) Children() []sql.Expression { + return []sql.Expression{s.str, s.delim, s.count} +} + +// Eval implements the Expression interface. +func (s *SubstringIndex) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { + ex, err := s.str.Eval(ctx, row) + if ex == nil || err != nil { + return nil, err + } + ex, err = sql.Text.Convert(ex) if err != nil { return nil, err } + str, ok := ex.(string) + if !ok { + return nil, sql.ErrInvalidType.New(reflect.TypeOf(ex).String()) + } - start, err := s.start.TransformUp(f) + ex, err = s.delim.Eval(ctx, row) + if ex == nil || err != nil { + return nil, err + } + ex, err = sql.Text.Convert(ex) if err != nil { return nil, err } + delim, ok := ex.(string) + if !ok { + return nil, sql.ErrInvalidType.New(reflect.TypeOf(ex).String()) + } - // It is safe to omit the errors of NewSubstring here because to be able to call - // this method, you need a valid instance of Substring, so the arity must be correct - // and that's the only error NewSubstring can return. - var sub sql.Expression - if s.len != nil { - len, err := s.len.TransformUp(f) - if err != nil { - return nil, err + ex, err = s.count.Eval(ctx, row) + if ex == nil || err != nil { + return nil, err + } + ex, err = sql.Int64.Convert(ex) + if err != nil { + return nil, err + } + count, ok := ex.(int64) + if !ok { + return nil, sql.ErrInvalidType.New(reflect.TypeOf(ex).String()) + } + + // Implementation taken from pingcap/tidb + // https://github.com/pingcap/tidb/blob/37c128b64f3ad2f08d52bc767b6e3320ecf429d8/expression/builtin_string.go#L1229 + strs := strings.Split(str, delim) + start, end := int64(0), int64(len(strs)) + if count > 0 { + // If count is positive, everything to the left of the final delimiter (counting from the left) is returned. + if count < end { + end = count } - sub, _ = NewSubstring(str, start, len) } else { - sub, _ = NewSubstring(str, start) + // If count is negative, everything to the right of the final delimiter (counting from the right) is returned. + count = -count + if count < 0 { + // -count overflows max int64, returns an empty string. + return "", nil + } + + if count < end { + start = end - count + } + } + + return strings.Join(strs[start:end], delim), nil +} + +// IsNullable implements the Expression interface. +func (s *SubstringIndex) IsNullable() bool { + return s.str.IsNullable() || s.delim.IsNullable() || s.count.IsNullable() +} + +func (s *SubstringIndex) String() string { + return fmt.Sprintf("SUBSTRING_INDEX(%s, %s, %d)", s.str, s.delim, s.count) +} + +// Resolved implements the Expression interface. +func (s *SubstringIndex) Resolved() bool { + return s.str.Resolved() && s.delim.Resolved() && s.count.Resolved() +} + +// Type implements the Expression interface. +func (*SubstringIndex) Type() sql.Type { return sql.Text } + +// WithChildren implements the Expression interface. +func (s *SubstringIndex) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 3 { + return nil, sql.ErrInvalidChildrenNumber.New(s, len(children), 3) } - return f(sub) + return NewSubstringIndex(children[0], children[1], children[2]), nil } diff --git a/sql/expression/function/substring_test.go b/sql/expression/function/substring_test.go index 826594155..e24cbb13f 100644 --- a/sql/expression/function/substring_test.go +++ b/sql/expression/function/substring_test.go @@ -3,9 +3,9 @@ package function import ( "testing" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestSubstring(t *testing.T) { @@ -51,3 +51,45 @@ func TestSubstring(t *testing.T) { }) } } + +func TestSubstringIndex(t *testing.T) { + f := NewSubstringIndex( + expression.NewGetField(0, sql.Text, "str", true), + expression.NewGetField(1, sql.Text, "delim", true), + expression.NewGetField(2, sql.Int64, "count", false), + ) + testCases := []struct { + name string + row sql.Row + expected interface{} + err bool + }{ + {"null string", sql.NewRow(nil, ".", 1), nil, false}, + {"null delim", sql.NewRow("foo", nil, 1), nil, false}, + {"null count", sql.NewRow("foo", 1, nil), nil, false}, + {"positive count", sql.NewRow("a.b.c.d.e.f", ".", 2), "a.b", false}, + {"negative count", sql.NewRow("a.b.c.d.e.f", ".", -2), "e.f", false}, + {"count 0", sql.NewRow("a.b.c", ".", 0), "", false}, + {"long delim", sql.NewRow("a.b.c.d.e.f", "..", 5), "a.b.c.d.e.f", false}, + {"count > len", sql.NewRow("a.b.c", ".", 10), "a.b.c", false}, + {"-count > -len", sql.NewRow("a.b.c", ".", -10), "a.b.c", false}, + {"remove suffix", sql.NewRow("source{d}", "{d}", 1), "source", false}, + {"remove suffix with negtive count", sql.NewRow("source{d}", "{d}", -1), "", false}, + {"wrong count type", sql.NewRow("", "", "foo"), "", true}, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + require := require.New(t) + ctx := sql.NewEmptyContext() + + v, err := f.Eval(ctx, tt.row) + if tt.err { + require.Error(err) + } else { + require.NoError(err) + require.Equal(tt.expected, v) + } + }) + } +} diff --git a/sql/expression/function/time.go b/sql/expression/function/time.go index 3f8594fdd..c0dcaf3d4 100644 --- a/sql/expression/function/time.go +++ b/sql/expression/function/time.go @@ -1,19 +1,18 @@ package function import ( + "errors" "fmt" "time" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" ) -func getDatePart( - ctx *sql.Context, +func getDate(ctx *sql.Context, u expression.UnaryExpression, - row sql.Row, - f func(interface{}) interface{}, -) (interface{}, error) { + row sql.Row) (interface{}, error) { + val, err := u.Child.Eval(ctx, row) if err != nil { return nil, err @@ -31,6 +30,19 @@ func getDatePart( } } + return date, nil +} + +func getDatePart(ctx *sql.Context, + u expression.UnaryExpression, + row sql.Row, + f func(interface{}) interface{}) (interface{}, error) { + + date, err := getDate(ctx, u, row) + if err != nil { + return nil, err + } + return f(date), nil } @@ -54,14 +66,12 @@ func (y *Year) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return getDatePart(ctx, y.UnaryExpression, row, year) } -// TransformUp implements the Expression interface. -func (y *Year) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - child, err := y.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (y *Year) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(y, len(children), 1) } - - return f(NewYear(child)) + return NewYear(children[0]), nil } // Month is a function that returns the month of a date. @@ -84,14 +94,12 @@ func (m *Month) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return getDatePart(ctx, m.UnaryExpression, row, month) } -// TransformUp implements the Expression interface. -func (m *Month) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - child, err := m.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (m *Month) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(m, len(children), 1) } - - return f(NewMonth(child)) + return NewMonth(children[0]), nil } // Day is a function that returns the day of a date. @@ -114,14 +122,12 @@ func (d *Day) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return getDatePart(ctx, d.UnaryExpression, row, day) } -// TransformUp implements the Expression interface. -func (d *Day) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - child, err := d.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (d *Day) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(d, len(children), 1) } - - return f(NewDay(child)) + return NewDay(children[0]), nil } // Weekday is a function that returns the weekday of a date where 0 = Monday, @@ -145,14 +151,12 @@ func (d *Weekday) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return getDatePart(ctx, d.UnaryExpression, row, weekday) } -// TransformUp implements the Expression interface. -func (d *Weekday) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - child, err := d.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (d *Weekday) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(d, len(children), 1) } - - return f(NewWeekday(child)) + return NewWeekday(children[0]), nil } // Hour is a function that returns the hour of a date. @@ -175,14 +179,12 @@ func (h *Hour) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return getDatePart(ctx, h.UnaryExpression, row, hour) } -// TransformUp implements the Expression interface. -func (h *Hour) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - child, err := h.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (h *Hour) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(h, len(children), 1) } - - return f(NewHour(child)) + return NewHour(children[0]), nil } // Minute is a function that returns the minute of a date. @@ -205,14 +207,12 @@ func (m *Minute) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return getDatePart(ctx, m.UnaryExpression, row, minute) } -// TransformUp implements the Expression interface. -func (m *Minute) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - child, err := m.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (m *Minute) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(m, len(children), 1) } - - return f(NewMinute(child)) + return NewMinute(children[0]), nil } // Second is a function that returns the second of a date. @@ -235,14 +235,12 @@ func (s *Second) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return getDatePart(ctx, s.UnaryExpression, row, second) } -// TransformUp implements the Expression interface. -func (s *Second) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - child, err := s.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (s *Second) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(s, len(children), 1) } - - return f(NewSecond(child)) + return NewSecond(children[0]), nil } // DayOfWeek is a function that returns the day of the week from a date where @@ -266,14 +264,12 @@ func (d *DayOfWeek) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return getDatePart(ctx, d.UnaryExpression, row, dayOfWeek) } -// TransformUp implements the Expression interface. -func (d *DayOfWeek) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - child, err := d.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (d *DayOfWeek) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(d, len(children), 1) } - - return f(NewDayOfWeek(child)) + return NewDayOfWeek(children[0]), nil } // DayOfYear is a function that returns the day of the year from a date. @@ -296,14 +292,12 @@ func (d *DayOfYear) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return getDatePart(ctx, d.UnaryExpression, row, dayOfYear) } -// TransformUp implements the Expression interface. -func (d *DayOfYear) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - child, err := d.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (d *DayOfYear) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(d, len(children), 1) } - - return f(NewDayOfYear(child)) + return NewDayOfYear(children[0]), nil } func datePartFunc(fn func(time.Time) int) func(interface{}) interface{} { @@ -316,6 +310,188 @@ func datePartFunc(fn func(time.Time) int) func(interface{}) interface{} { } } +// YearWeek is a function that returns year and week for a date. +// The year in the result may be different from the year in the date argument for the first and the last week of the year. +// Details: https://dev.mysql.com/doc/refman/5.5/en/date-and-time-functions.html#function_yearweek +type YearWeek struct { + date sql.Expression + mode sql.Expression +} + +// NewYearWeek creates a new YearWeek UDF +func NewYearWeek(args ...sql.Expression) (sql.Expression, error) { + if len(args) == 0 { + return nil, sql.ErrInvalidArgumentNumber.New("YEARWEEK", "1 or more", 0) + } + + yw := &YearWeek{date: args[0]} + if len(args) > 1 && args[1].Resolved() && sql.IsInteger(args[1].Type()) { + yw.mode = args[1] + } else { + yw.mode = expression.NewLiteral(0, sql.Int64) + } + return yw, nil +} + +func (d *YearWeek) String() string { return fmt.Sprintf("YEARWEEK(%s, %d)", d.date, d.mode) } + +// Type implements the Expression interface. +func (d *YearWeek) Type() sql.Type { return sql.Int32 } + +// Eval implements the Expression interface. +func (d *YearWeek) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { + date, err := getDate(ctx, expression.UnaryExpression{Child: d.date}, row) + if err != nil { + return nil, err + } + yyyy, ok := year(date).(int32) + if !ok { + return nil, errors.New("YEARWEEK: invalid year") + } + mm, ok := month(date).(int32) + if !ok { + return nil, errors.New("YEARWEEK: invalid month") + } + dd, ok := day(date).(int32) + if !ok { + return nil, errors.New("YEARWEEK: invalid day") + } + + mode := int64(0) + val, err := d.mode.Eval(ctx, row) + if err != nil { + return nil, err + } + if val != nil { + if i64, err := sql.Int64.Convert(val); err == nil { + if mode, ok = i64.(int64); ok { + mode %= 8 // mode in [0, 7] + } + } + } + yyyy, week := calcWeek(yyyy, mm, dd, weekMode(mode)|weekBehaviourYear) + + return (yyyy * 100) + week, nil +} + +// Resolved implements the Expression interface. +func (d *YearWeek) Resolved() bool { + return d.date.Resolved() && d.mode.Resolved() +} + +// Children implements the Expression interface. +func (d *YearWeek) Children() []sql.Expression { return []sql.Expression{d.date, d.mode} } + +// IsNullable implements the Expression interface. +func (d *YearWeek) IsNullable() bool { + return d.date.IsNullable() +} + +// WithChildren implements the Expression interface. +func (*YearWeek) WithChildren(children ...sql.Expression) (sql.Expression, error) { + return NewYearWeek(children...) +} + +// Following solution of YearWeek was taken from tidb: https://github.com/pingcap/tidb/blob/master/types/mytime.go +type weekBehaviour int64 + +const ( + // weekBehaviourMondayFirst set Monday as first day of week; otherwise Sunday is first day of week + weekBehaviourMondayFirst weekBehaviour = 1 << iota + // If set, Week is in range 1-53, otherwise Week is in range 0-53. + // Note that this flag is only relevant if WEEK_JANUARY is not set. + weekBehaviourYear + // If not set, Weeks are numbered according to ISO 8601:1988. + // If set, the week that contains the first 'first-day-of-week' is week 1. + weekBehaviourFirstWeekday +) + +func (v weekBehaviour) test(flag weekBehaviour) bool { + return (v & flag) != 0 +} + +func weekMode(mode int64) weekBehaviour { + weekFormat := weekBehaviour(mode & 7) + if (weekFormat & weekBehaviourMondayFirst) == 0 { + weekFormat ^= weekBehaviourFirstWeekday + } + return weekFormat +} + +// calcWeekday calculates weekday from daynr, returns 0 for Monday, 1 for Tuesday ... +func calcWeekday(daynr int32, sundayFirstDayOfWeek bool) int32 { + daynr += 5 + if sundayFirstDayOfWeek { + daynr++ + } + return daynr % 7 +} + +// calcWeek calculates week and year for the time. +func calcWeek(yyyy, mm, dd int32, wb weekBehaviour) (int32, int32) { + daynr := calcDaynr(yyyy, mm, dd) + firstDaynr := calcDaynr(yyyy, 1, 1) + mondayFirst := wb.test(weekBehaviourMondayFirst) + weekYear := wb.test(weekBehaviourYear) + firstWeekday := wb.test(weekBehaviourFirstWeekday) + weekday := calcWeekday(firstDaynr, !mondayFirst) + + week, days := int32(0), int32(0) + if mm == 1 && dd <= 7-weekday { + if !weekYear && + ((firstWeekday && weekday != 0) || (!firstWeekday && weekday >= 4)) { + return yyyy, week + } + weekYear = true + yyyy-- + days = calcDaysInYear(yyyy) + firstDaynr -= days + weekday = (weekday + 53*7 - days) % 7 + } + + if (firstWeekday && weekday != 0) || + (!firstWeekday && weekday >= 4) { + days = daynr - (firstDaynr + 7 - weekday) + } else { + days = daynr - (firstDaynr - weekday) + } + + if weekYear && days >= 52*7 { + weekday = (weekday + calcDaysInYear(yyyy)) % 7 + if (!firstWeekday && weekday < 4) || + (firstWeekday && weekday == 0) { + yyyy++ + week = 1 + return yyyy, week + } + } + week = days/7 + 1 + return yyyy, week +} + +// calcDaysInYear calculates days in one year, it works with 0 <= yyyy <= 99. +func calcDaysInYear(yyyy int32) int32 { + if (yyyy&3) == 0 && (yyyy%100 != 0 || (yyyy%400 == 0 && (yyyy != 0))) { + return 366 + } + return 365 +} + +// calcDaynr calculates days since 0000-00-00. +func calcDaynr(yyyy, mm, dd int32) int32 { + if yyyy == 0 && mm == 0 { + return 0 + } + + delsum := 365*yyyy + 31*(mm-1) + dd + if mm <= 2 { + yyyy-- + } else { + delsum -= (mm*4 + 23) / 10 + } + return delsum + yyyy/4 - ((yyyy/100+1)*3)/4 +} + var ( year = datePartFunc((time.Time).Year) month = datePartFunc(func(t time.Time) int { return int(t.Month()) }) @@ -361,7 +537,44 @@ func (n *Now) Eval(*sql.Context, sql.Row) (interface{}, error) { return n.clock(), nil } -// TransformUp implements the sql.Expression interface. -func (n *Now) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - return f(n) +// WithChildren implements the Expression interface. +func (n *Now) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(n, len(children), 0) + } + return n, nil +} + +// Date a function takes the DATE part out from a datetime expression. +type Date struct { + expression.UnaryExpression +} + +// NewDate returns a new Date node. +func NewDate(date sql.Expression) sql.Expression { + return &Date{expression.UnaryExpression{Child: date}} +} + +func (d *Date) String() string { return fmt.Sprintf("DATE(%s)", d.Child) } + +// Type implements the Expression interface. +func (d *Date) Type() sql.Type { return sql.Text } + +// Eval implements the Expression interface. +func (d *Date) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { + return getDatePart(ctx, d.UnaryExpression, row, func(v interface{}) interface{} { + if v == nil { + return nil + } + + return v.(time.Time).Format("2006-01-02") + }) +} + +// WithChildren implements the Expression interface. +func (d *Date) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(d, len(children), 1) + } + return NewDate(children[0]), nil } diff --git a/sql/expression/function/time_test.go b/sql/expression/function/time_test.go index a176dc82e..b9762ec1f 100644 --- a/sql/expression/function/time_test.go +++ b/sql/expression/function/time_test.go @@ -4,9 +4,9 @@ import ( "testing" "time" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) const ( @@ -292,15 +292,96 @@ func TestTime_DayOfYear(t *testing.T) { } } +func TestYearWeek(t *testing.T) { + f, err := NewYearWeek(expression.NewGetField(0, sql.Text, "foo", false)) + require.NoError(t, err) + ctx := sql.NewEmptyContext() + + testCases := []struct { + name string + row sql.Row + expected interface{} + err bool + }{ + {"null date", sql.NewRow(nil), nil, true}, + {"invalid type", sql.NewRow([]byte{0, 1, 2}), nil, true}, + {"date as string", sql.NewRow(stringDate), int32(200653), false}, + {"date as unix timestamp", sql.NewRow(int64(tsDate)), int32(200947), false}, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + require := require.New(t) + val, err := f.Eval(ctx, tt.row) + if tt.err { + require.Error(err) + } else { + require.NoError(err) + require.Equal(tt.expected, val) + } + }) + } +} + +func TestCalcDaynr(t *testing.T) { + require.EqualValues(t, calcDaynr(0, 0, 0), 0) + require.EqualValues(t, calcDaynr(9999, 12, 31), 3652424) + require.EqualValues(t, calcDaynr(1970, 1, 1), 719528) + require.EqualValues(t, calcDaynr(2006, 12, 16), 733026) + require.EqualValues(t, calcDaynr(10, 1, 2), 3654) + require.EqualValues(t, calcDaynr(2008, 2, 20), 733457) +} + +func TestCalcWeek(t *testing.T) { + _, w := calcWeek(2008, 2, 20, weekMode(0)) + + _, w = calcWeek(2008, 2, 20, weekMode(1)) + require.EqualValues(t, w, 8) + + _, w = calcWeek(2008, 12, 31, weekMode(1)) + require.EqualValues(t, w, 53) +} + func TestNow(t *testing.T) { require := require.New(t) date := time.Date(2018, time.December, 2, 16, 25, 0, 0, time.Local) - clock := clock(func() time.Time { + clk := clock(func() time.Time { return date }) - f := &Now{clock} + f := &Now{clk} result, err := f.Eval(nil, nil) require.NoError(err) require.Equal(date, result) } + +func TestDate(t *testing.T) { + f := NewDate(expression.NewGetField(0, sql.Text, "foo", false)) + ctx := sql.NewEmptyContext() + + testCases := []struct { + name string + row sql.Row + expected interface{} + err bool + }{ + {"null date", sql.NewRow(nil), nil, false}, + {"invalid type", sql.NewRow([]byte{0, 1, 2}), nil, false}, + {"date as string", sql.NewRow(stringDate), "2007-01-02", false}, + {"date as time", sql.NewRow(time.Now()), time.Now().Format("2006-01-02"), false}, + {"date as unix timestamp", sql.NewRow(int64(tsDate)), "2009-11-22", false}, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + require := require.New(t) + val, err := f.Eval(ctx, tt.row) + if tt.err { + require.Error(err) + } else { + require.NoError(err) + require.Equal(tt.expected, val) + } + }) + } +} diff --git a/sql/expression/function/tobase64_frombase64.go b/sql/expression/function/tobase64_frombase64.go new file mode 100644 index 000000000..f3c638983 --- /dev/null +++ b/sql/expression/function/tobase64_frombase64.go @@ -0,0 +1,145 @@ +package function + +import ( + "encoding/base64" + "fmt" + "reflect" + "strings" + + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" +) + +// ToBase64 is a function to encode a string to the Base64 format +// using the same dialect that MySQL's TO_BASE64 uses +type ToBase64 struct { + expression.UnaryExpression +} + +// NewToBase64 creates a new ToBase64 expression. +func NewToBase64(e sql.Expression) sql.Expression { + return &ToBase64{expression.UnaryExpression{Child: e}} +} + +// Eval implements the Expression interface. +func (t *ToBase64) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { + str, err := t.Child.Eval(ctx, row) + + if err != nil { + return nil, err + } + + if str == nil { + return nil, nil + } + + str, err = sql.Text.Convert(str) + if err != nil { + return nil, sql.ErrInvalidType.New(reflect.TypeOf(str)) + } + + encoded := base64.StdEncoding.EncodeToString([]byte(str.(string))) + + lenEncoded := len(encoded) + if lenEncoded <= 76 { + return encoded, nil + } + + // Split into max 76 chars lines + var out strings.Builder + start := 0 + end := 76 + for { + out.WriteString(encoded[start:end] + "\n") + start += 76 + end += 76 + if end >= lenEncoded { + out.WriteString(encoded[start:lenEncoded]) + break + } + } + + return out.String(), nil +} + +// String implements the Stringer interface. +func (t *ToBase64) String() string { + return fmt.Sprintf("TO_BASE64(%s)", t.Child) +} + +// IsNullable implements the Expression interface. +func (t *ToBase64) IsNullable() bool { + return t.Child.IsNullable() +} + +// WithChildren implements the Expression interface. +func (t *ToBase64) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(t, len(children), 1) + } + return NewToBase64(children[0]), nil +} + +// Type implements the Expression interface. +func (t *ToBase64) Type() sql.Type { + return sql.Text +} + +// FromBase64 is a function to decode a Base64-formatted string +// using the same dialect that MySQL's FROM_BASE64 uses +type FromBase64 struct { + expression.UnaryExpression +} + +// NewFromBase64 creates a new FromBase64 expression. +func NewFromBase64(e sql.Expression) sql.Expression { + return &FromBase64{expression.UnaryExpression{Child: e}} +} + +// Eval implements the Expression interface. +func (t *FromBase64) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { + str, err := t.Child.Eval(ctx, row) + + if err != nil { + return nil, err + } + + if str == nil { + return nil, nil + } + + str, err = sql.Text.Convert(str) + if err != nil { + return nil, sql.ErrInvalidType.New(reflect.TypeOf(str)) + } + + decoded, err := base64.StdEncoding.DecodeString(str.(string)) + if err != nil { + return nil, err + } + + return string(decoded), nil +} + +// String implements the Stringer interface. +func (t *FromBase64) String() string { + return fmt.Sprintf("FROM_BASE64(%s)", t.Child) +} + +// IsNullable implements the Expression interface. +func (t *FromBase64) IsNullable() bool { + return t.Child.IsNullable() +} + +// WithChildren implements the Expression interface. +func (t *FromBase64) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(t, len(children), 1) + } + return NewFromBase64(children[0]), nil +} + +// Type implements the Expression interface. +func (t *FromBase64) Type() sql.Type { + return sql.Text +} diff --git a/sql/expression/function/tobase64_frombase64_test.go b/sql/expression/function/tobase64_frombase64_test.go new file mode 100644 index 000000000..f234728d2 --- /dev/null +++ b/sql/expression/function/tobase64_frombase64_test.go @@ -0,0 +1,56 @@ +package function + +import ( + "testing" + + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/stretchr/testify/require" +) + +func TestBase64(t *testing.T) { + fTo := NewToBase64(expression.NewGetField(0, sql.Text, "", false)) + fFrom := NewFromBase64(expression.NewGetField(0, sql.Text, "", false)) + + testCases := []struct { + name string + row sql.Row + expected interface{} + err bool + }{ + // Use a MySQL server to get expected values if updating/adding to this! + {"null input", sql.NewRow(nil), nil, false}, + {"single_line", sql.NewRow("foo"), string("Zm9v"), false}, + {"multi_line", sql.NewRow( + "Gallia est omnis divisa in partes tres, quarum unam " + + "incolunt Belgae, aliam Aquitani, tertiam qui ipsorum lingua Celtae, " + + "nostra Galli appellantur"), + "R2FsbGlhIGVzdCBvbW5pcyBkaXZpc2EgaW4gcGFydGVzIHRyZXMsIHF1YXJ1bSB1bmFtIGluY29s\n" + + "dW50IEJlbGdhZSwgYWxpYW0gQXF1aXRhbmksIHRlcnRpYW0gcXVpIGlwc29ydW0gbGluZ3VhIENl\n" + + "bHRhZSwgbm9zdHJhIEdhbGxpIGFwcGVsbGFudHVy", false}, + {"empty_input", sql.NewRow(""), string(""), false}, + {"symbols", sql.NewRow("!@#$% %^&*()_+\r\n\t{};"), string("IUAjJCUgJV4mKigpXysNCgl7fTs="), + false}, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + t.Helper() + require := require.New(t) + ctx := sql.NewEmptyContext() + v, err := fTo.Eval(ctx, tt.row) + + if tt.err { + require.Error(err) + } else { + require.NoError(err) + require.Equal(tt.expected, v) + + ctx = sql.NewEmptyContext() + v2, err := fFrom.Eval(ctx, sql.NewRow(v)) + require.NoError(err) + require.Equal(sql.NewRow(v2), tt.row) + } + }) + } +} diff --git a/sql/expression/function/trim_ltrim_rtrim.go b/sql/expression/function/trim_ltrim_rtrim.go index 9f3d3104e..b08704dfb 100644 --- a/sql/expression/function/trim_ltrim_rtrim.go +++ b/sql/expression/function/trim_ltrim_rtrim.go @@ -6,11 +6,12 @@ import ( "strings" "unicode" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" ) type trimType rune + const ( lTrimType trimType = 'l' rTrimType trimType = 'r' @@ -54,14 +55,12 @@ func (t *Trim) IsNullable() bool { return t.Child.IsNullable() } -// TransformUp implements the Expression interface. -func (t *Trim) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - str, err := t.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (t *Trim) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(t, len(children), 1) } - - return f(NewTrim(t.trimType, str)) + return NewTrim(t.trimType, children[0]), nil } // Eval implements the Expression interface. diff --git a/sql/expression/function/trim_ltrim_rtrim_test.go b/sql/expression/function/trim_ltrim_rtrim_test.go index dabf7b02b..abfaa9f4e 100644 --- a/sql/expression/function/trim_ltrim_rtrim_test.go +++ b/sql/expression/function/trim_ltrim_rtrim_test.go @@ -3,13 +3,13 @@ package function import ( "testing" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" ) func TestTrim(t *testing.T) { - f := NewTrimFunc(bTrimType)(expression.NewGetField(0, sql.Text, "", false), ) + f := NewTrimFunc(bTrimType)(expression.NewGetField(0, sql.Text, "", false)) testCases := []struct { name string row sql.Row @@ -42,7 +42,7 @@ func TestTrim(t *testing.T) { } func TestLTrim(t *testing.T) { - f := NewTrimFunc(lTrimType)(expression.NewGetField(0, sql.Text, "", false), ) + f := NewTrimFunc(lTrimType)(expression.NewGetField(0, sql.Text, "", false)) testCases := []struct { name string row sql.Row @@ -75,7 +75,7 @@ func TestLTrim(t *testing.T) { } func TestRTrim(t *testing.T) { - f := NewTrimFunc(rTrimType)(expression.NewGetField(0, sql.Text, "", false), ) + f := NewTrimFunc(rTrimType)(expression.NewGetField(0, sql.Text, "", false)) testCases := []struct { name string row sql.Row diff --git a/sql/expression/function/version.go b/sql/expression/function/version.go index 2870d9409..eafeb4454 100644 --- a/sql/expression/function/version.go +++ b/sql/expression/function/version.go @@ -3,7 +3,7 @@ package function import ( "fmt" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) const mysqlVersion = "8.0.11" @@ -30,9 +30,12 @@ func (f Version) String() string { return "VERSION()" } -// TransformUp implements the Expression interface. -func (f Version) TransformUp(fn sql.TransformExprFunc) (sql.Expression, error) { - return fn(f) +// WithChildren implements the Expression interface. +func (f Version) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(f, len(children), 0) + } + return f, nil } // Resolved implements the Expression interface. diff --git a/sql/expression/function/version_test.go b/sql/expression/function/version_test.go index a35a3bbdb..0ec10cfdb 100644 --- a/sql/expression/function/version_test.go +++ b/sql/expression/function/version_test.go @@ -3,8 +3,8 @@ package function import ( "testing" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) const versionPostfix = "test" diff --git a/sql/expression/get_field.go b/sql/expression/get_field.go index 75690a5d2..e6a5884dc 100644 --- a/sql/expression/get_field.go +++ b/sql/expression/get_field.go @@ -3,8 +3,8 @@ package expression import ( "fmt" + "github.com/src-d/go-mysql-server/sql" errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) // GetField is an expression to get the field of a table. @@ -72,10 +72,12 @@ func (p *GetField) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return row[p.fieldIndex], nil } -// TransformUp implements the Expression interface. -func (p *GetField) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - n := *p - return f(&n) +// WithChildren implements the Expression interface. +func (p *GetField) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(p, len(children), 0) + } + return p, nil } func (p *GetField) String() string { @@ -124,7 +126,10 @@ func (f *GetSessionField) Resolved() bool { return true } // String implements the sql.Expression interface. func (f *GetSessionField) String() string { return "@@" + f.name } -// TransformUp implements the sql.Expression interface. -func (f *GetSessionField) TransformUp(fn sql.TransformExprFunc) (sql.Expression, error) { - return fn(f) +// WithChildren implements the Expression interface. +func (f *GetSessionField) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(f, len(children), 0) + } + return f, nil } diff --git a/sql/expression/interval.go b/sql/expression/interval.go new file mode 100644 index 000000000..a175d55b5 --- /dev/null +++ b/sql/expression/interval.go @@ -0,0 +1,284 @@ +package expression + +import ( + "fmt" + "regexp" + "strconv" + "strings" + "time" + + "github.com/src-d/go-mysql-server/sql" + errors "gopkg.in/src-d/go-errors.v1" +) + +// Interval defines a time duration. +type Interval struct { + UnaryExpression + Unit string +} + +// NewInterval creates a new interval expression. +func NewInterval(child sql.Expression, unit string) *Interval { + return &Interval{UnaryExpression{Child: child}, strings.ToUpper(unit)} +} + +// Type implements the sql.Expression interface. +func (i *Interval) Type() sql.Type { return sql.Uint64 } + +// IsNullable implements the sql.Expression interface. +func (i *Interval) IsNullable() bool { return i.Child.IsNullable() } + +// Eval implements the sql.Expression interface. +func (i *Interval) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { + panic("Interval.Eval is just a placeholder method and should not be called directly") +} + +var ( + errInvalidIntervalUnit = errors.NewKind("invalid interval unit: %s") + errInvalidIntervalFormat = errors.NewKind("invalid interval format for %q: %s") +) + +// EvalDelta evaluates the expression returning a TimeDelta. This method should +// be used instead of Eval, as this expression returns a TimeDelta, which is not +// a valid value that can be returned in Eval. +func (i *Interval) EvalDelta(ctx *sql.Context, row sql.Row) (*TimeDelta, error) { + val, err := i.Child.Eval(ctx, row) + if err != nil { + return nil, err + } + + if val == nil { + return nil, nil + } + + var td TimeDelta + + if r, ok := unitTextFormats[i.Unit]; ok { + val, err = sql.Text.Convert(val) + if err != nil { + return nil, err + } + + text := val.(string) + if !r.MatchString(text) { + return nil, errInvalidIntervalFormat.New(i.Unit, text) + } + + parts := textFormatParts(text, r) + + switch i.Unit { + case "DAY_HOUR": + td.Days = parts[0] + td.Hours = parts[1] + case "DAY_MICROSECOND": + td.Days = parts[0] + td.Hours = parts[1] + td.Minutes = parts[2] + td.Seconds = parts[3] + td.Microseconds = parts[4] + case "DAY_MINUTE": + td.Days = parts[0] + td.Hours = parts[1] + td.Minutes = parts[2] + case "DAY_SECOND": + td.Days = parts[0] + td.Hours = parts[1] + td.Minutes = parts[2] + td.Seconds = parts[3] + case "HOUR_MICROSECOND": + td.Hours = parts[0] + td.Minutes = parts[1] + td.Seconds = parts[2] + td.Microseconds = parts[3] + case "HOUR_SECOND": + td.Hours = parts[0] + td.Minutes = parts[1] + td.Seconds = parts[2] + case "HOUR_MINUTE": + td.Hours = parts[0] + td.Minutes = parts[1] + case "MINUTE_MICROSECOND": + td.Minutes = parts[0] + td.Seconds = parts[1] + td.Microseconds = parts[2] + case "MINUTE_SECOND": + td.Minutes = parts[0] + td.Seconds = parts[1] + case "SECOND_MICROSECOND": + td.Seconds = parts[0] + td.Microseconds = parts[1] + case "YEAR_MONTH": + td.Years = parts[0] + td.Months = parts[1] + default: + return nil, errInvalidIntervalUnit.New(i.Unit) + } + } else { + val, err = sql.Int64.Convert(val) + if err != nil { + return nil, err + } + + num := val.(int64) + + switch i.Unit { + case "DAY": + td.Days = num + case "HOUR": + td.Hours = num + case "MINUTE": + td.Minutes = num + case "SECOND": + td.Seconds = num + case "MICROSECOND": + td.Microseconds = num + case "QUARTER": + td.Months = num * 3 + case "MONTH": + td.Months = num + case "WEEK": + td.Days = num * 7 + case "YEAR": + td.Years = num + default: + return nil, errInvalidIntervalUnit.New(i.Unit) + } + } + + return &td, nil +} + +// WithChildren implements the Expression interface. +func (i *Interval) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(i, len(children), 1) + } + return NewInterval(children[0], i.Unit), nil +} + +func (i *Interval) String() string { + return fmt.Sprintf("INTERVAL %s %s", i.Child, i.Unit) +} + +var unitTextFormats = map[string]*regexp.Regexp{ + "DAY_HOUR": regexp.MustCompile(`^(\d+)\s+(\d+)$`), + "DAY_MICROSECOND": regexp.MustCompile(`^(\d+)\s+(\d+):(\d+):(\d+).(\d+)$`), + "DAY_MINUTE": regexp.MustCompile(`^(\d+)\s+(\d+):(\d+)$`), + "DAY_SECOND": regexp.MustCompile(`^(\d+)\s+(\d+):(\d+):(\d+)$`), + "HOUR_MICROSECOND": regexp.MustCompile(`^(\d+):(\d+):(\d+).(\d+)$`), + "HOUR_SECOND": regexp.MustCompile(`^(\d+):(\d+):(\d+)$`), + "HOUR_MINUTE": regexp.MustCompile(`^(\d+):(\d+)$`), + "MINUTE_MICROSECOND": regexp.MustCompile(`^(\d+):(\d+).(\d+)$`), + "MINUTE_SECOND": regexp.MustCompile(`^(\d+):(\d+)$`), + "SECOND_MICROSECOND": regexp.MustCompile(`^(\d+).(\d+)$`), + "YEAR_MONTH": regexp.MustCompile(`^(\d+)-(\d+)$`), +} + +func textFormatParts(text string, r *regexp.Regexp) []int64 { + parts := r.FindStringSubmatch(text) + var result []int64 + for _, p := range parts[1:] { + // It is safe to igore the error here, because at this point we know + // the string matches the regexp, and that means it can't be an + // invalid number. + n, _ := strconv.ParseInt(p, 10, 64) + result = append(result, n) + } + return result +} + +// TimeDelta is the difference between a time and another time. +type TimeDelta struct { + Years int64 + Months int64 + Days int64 + Hours int64 + Minutes int64 + Seconds int64 + Microseconds int64 +} + +// Add returns the given time plus the time delta. +func (td TimeDelta) Add(t time.Time) time.Time { + return td.apply(t, 1) +} + +// Sub returns the given time minus the time delta. +func (td TimeDelta) Sub(t time.Time) time.Time { + return td.apply(t, -1) +} + +const ( + day = 24 * time.Hour + week = 7 * day +) + +func (td TimeDelta) apply(t time.Time, sign int64) time.Time { + y := int64(t.Year()) + mo := int64(t.Month()) + d := t.Day() + h := t.Hour() + min := t.Minute() + s := t.Second() + ns := t.Nanosecond() + + if td.Years != 0 { + y += td.Years * sign + } + + if td.Months != 0 { + m := mo + td.Months*sign + if m < 1 { + mo = 12 + (m % 12) + y += m/12 - 1 + } else if m > 12 { + mo = m % 12 + y += m / 12 + } else { + mo = m + } + + // Due to the operations done before, month may be zero, which means it's + // december. + if mo == 0 { + mo = 12 + } + } + + if days := daysInMonth(time.Month(mo), int(y)); days < d { + d = days + } + + date := time.Date(int(y), time.Month(mo), d, h, min, s, ns, t.Location()) + + if td.Days != 0 { + date = date.Add(time.Duration(td.Days) * day * time.Duration(sign)) + } + + if td.Hours != 0 { + date = date.Add(time.Duration(td.Hours) * time.Hour * time.Duration(sign)) + } + + if td.Minutes != 0 { + date = date.Add(time.Duration(td.Minutes) * time.Minute * time.Duration(sign)) + } + + if td.Seconds != 0 { + date = date.Add(time.Duration(td.Seconds) * time.Second * time.Duration(sign)) + } + + if td.Microseconds != 0 { + date = date.Add(time.Duration(td.Microseconds) * time.Microsecond * time.Duration(sign)) + } + + return date +} + +func daysInMonth(month time.Month, year int) int { + if month == time.December { + return 31 + } + + date := time.Date(year, month+time.Month(1), 1, 0, 0, 0, 0, time.Local) + return date.Add(-1 * day).Day() +} diff --git a/sql/expression/interval_test.go b/sql/expression/interval_test.go new file mode 100644 index 000000000..24be71a5a --- /dev/null +++ b/sql/expression/interval_test.go @@ -0,0 +1,285 @@ +package expression + +import ( + "testing" + "time" + + "github.com/src-d/go-mysql-server/sql" + "github.com/stretchr/testify/require" +) + +func TestTimeDelta(t *testing.T) { + leapYear := date(2004, time.February, 29, 0, 0, 0, 0) + testCases := []struct { + name string + delta TimeDelta + date time.Time + output time.Time + }{ + { + "leap year minus one year", + TimeDelta{Years: -1}, + leapYear, + date(2003, time.February, 28, 0, 0, 0, 0), + }, + { + "leap year plus one year", + TimeDelta{Years: 1}, + leapYear, + date(2005, time.February, 28, 0, 0, 0, 0), + }, + { + "plus overflowing months", + TimeDelta{Months: 13}, + leapYear, + date(2005, time.March, 29, 0, 0, 0, 0), + }, + { + "plus overflowing until december", + TimeDelta{Months: 22}, + leapYear, + date(2006, time.December, 29, 0, 0, 0, 0), + }, + { + "minus overflowing months", + TimeDelta{Months: -13}, + leapYear, + date(2003, time.January, 29, 0, 0, 0, 0), + }, + { + "minus overflowing until december", + TimeDelta{Months: -14}, + leapYear, + date(2002, time.December, 29, 0, 0, 0, 0), + }, + { + "minus months", + TimeDelta{Months: -1}, + leapYear, + date(2004, time.January, 29, 0, 0, 0, 0), + }, + { + "plus months", + TimeDelta{Months: 1}, + leapYear, + date(2004, time.March, 29, 0, 0, 0, 0), + }, + { + "minus days", + TimeDelta{Days: -2}, + leapYear, + date(2004, time.February, 27, 0, 0, 0, 0), + }, + { + "plus days", + TimeDelta{Days: 1}, + leapYear, + date(2004, time.March, 1, 0, 0, 0, 0), + }, + { + "minus hours", + TimeDelta{Hours: -2}, + leapYear, + date(2004, time.February, 28, 22, 0, 0, 0), + }, + { + "plus hours", + TimeDelta{Hours: 26}, + leapYear, + date(2004, time.March, 1, 2, 0, 0, 0), + }, + { + "minus minutes", + TimeDelta{Minutes: -2}, + leapYear, + date(2004, time.February, 28, 23, 58, 0, 0), + }, + { + "plus minutes", + TimeDelta{Minutes: 26}, + leapYear, + date(2004, time.February, 29, 0, 26, 0, 0), + }, + { + "minus seconds", + TimeDelta{Seconds: -2}, + leapYear, + date(2004, time.February, 28, 23, 59, 58, 0), + }, + { + "plus seconds", + TimeDelta{Seconds: 26}, + leapYear, + date(2004, time.February, 29, 0, 0, 26, 0), + }, + { + "minus microseconds", + TimeDelta{Microseconds: -2}, + leapYear, + date(2004, time.February, 28, 23, 59, 59, 999998), + }, + { + "plus microseconds", + TimeDelta{Microseconds: 26}, + leapYear, + date(2004, time.February, 29, 0, 0, 0, 26), + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + result := tt.delta.Add(tt.date) + require.Equal(t, tt.output, result) + }) + } +} + +func TestIntervalEvalDelta(t *testing.T) { + testCases := []struct { + expr sql.Expression + unit string + row sql.Row + expected TimeDelta + }{ + { + NewGetField(0, sql.Int64, "foo", false), + "DAY", + sql.Row{int64(2)}, + TimeDelta{Days: 2}, + }, + { + NewLiteral(int64(2), sql.Int64), + "DAY", + nil, + TimeDelta{Days: 2}, + }, + { + NewLiteral(int64(2), sql.Int64), + "MONTH", + nil, + TimeDelta{Months: 2}, + }, + { + NewLiteral(int64(2), sql.Int64), + "YEAR", + nil, + TimeDelta{Years: 2}, + }, + { + NewLiteral(int64(2), sql.Int64), + "QUARTER", + nil, + TimeDelta{Months: 6}, + }, + { + NewLiteral(int64(2), sql.Int64), + "WEEK", + nil, + TimeDelta{Days: 14}, + }, + { + NewLiteral(int64(2), sql.Int64), + "HOUR", + nil, + TimeDelta{Hours: 2}, + }, + { + NewLiteral(int64(2), sql.Int64), + "MINUTE", + nil, + TimeDelta{Minutes: 2}, + }, + { + NewLiteral(int64(2), sql.Int64), + "SECOND", + nil, + TimeDelta{Seconds: 2}, + }, + { + NewLiteral(int64(2), sql.Int64), + "MICROSECOND", + nil, + TimeDelta{Microseconds: 2}, + }, + { + NewLiteral("2 3", sql.Text), + "DAY_HOUR", + nil, + TimeDelta{Days: 2, Hours: 3}, + }, + { + NewLiteral("2 3:04:05.06", sql.Text), + "DAY_MICROSECOND", + nil, + TimeDelta{Days: 2, Hours: 3, Minutes: 4, Seconds: 5, Microseconds: 6}, + }, + { + NewLiteral("2 3:04:05", sql.Text), + "DAY_SECOND", + nil, + TimeDelta{Days: 2, Hours: 3, Minutes: 4, Seconds: 5}, + }, + { + NewLiteral("2 3:04", sql.Text), + "DAY_MINUTE", + nil, + TimeDelta{Days: 2, Hours: 3, Minutes: 4}, + }, + { + NewLiteral("3:04:05.06", sql.Text), + "HOUR_MICROSECOND", + nil, + TimeDelta{Hours: 3, Minutes: 4, Seconds: 5, Microseconds: 6}, + }, + { + NewLiteral("3:04:05", sql.Text), + "HOUR_SECOND", + nil, + TimeDelta{Hours: 3, Minutes: 4, Seconds: 5}, + }, + { + NewLiteral("3:04", sql.Text), + "HOUR_MINUTE", + nil, + TimeDelta{Hours: 3, Minutes: 4}, + }, + { + NewLiteral("04:05.06", sql.Text), + "MINUTE_MICROSECOND", + nil, + TimeDelta{Minutes: 4, Seconds: 5, Microseconds: 6}, + }, + { + NewLiteral("04:05", sql.Text), + "MINUTE_SECOND", + nil, + TimeDelta{Minutes: 4, Seconds: 5}, + }, + { + NewLiteral("04.05", sql.Text), + "SECOND_MICROSECOND", + nil, + TimeDelta{Seconds: 4, Microseconds: 5}, + }, + { + NewLiteral("1-5", sql.Text), + "YEAR_MONTH", + nil, + TimeDelta{Years: 1, Months: 5}, + }, + } + + for _, tt := range testCases { + interval := NewInterval(tt.expr, tt.unit) + t.Run(interval.String(), func(t *testing.T) { + require := require.New(t) + result, err := interval.EvalDelta(sql.NewEmptyContext(), tt.row) + require.NoError(err) + require.Equal(tt.expected, *result) + }) + } +} + +func date(year int, month time.Month, day, hour, min, sec, micro int) time.Time { + return time.Date(year, month, day, hour, min, sec, micro*int(time.Microsecond), time.Local) +} diff --git a/sql/expression/isnull.go b/sql/expression/isnull.go index cf0c0c73d..a9ae575d5 100644 --- a/sql/expression/isnull.go +++ b/sql/expression/isnull.go @@ -1,6 +1,6 @@ package expression -import "gopkg.in/src-d/go-mysql-server.v0/sql" +import "github.com/src-d/go-mysql-server/sql" // IsNull is an expression that checks if an expression is null. type IsNull struct { @@ -36,11 +36,10 @@ func (e IsNull) String() string { return e.Child.String() + " IS NULL" } -// TransformUp implements the Expression interface. -func (e *IsNull) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - child, err := e.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (e *IsNull) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(e, len(children), 1) } - return f(NewIsNull(child)) + return NewIsNull(children[0]), nil } diff --git a/sql/expression/isnull_test.go b/sql/expression/isnull_test.go index f158455cf..5e638ba6c 100644 --- a/sql/expression/isnull_test.go +++ b/sql/expression/isnull_test.go @@ -3,7 +3,7 @@ package expression import ( "testing" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" ) diff --git a/sql/expression/istrue.go b/sql/expression/istrue.go new file mode 100644 index 000000000..ba3e0c75f --- /dev/null +++ b/sql/expression/istrue.go @@ -0,0 +1,78 @@ +package expression + +import ( + "errors" + "github.com/src-d/go-mysql-server/sql" +) + +// IsTrue is an expression that checks if an expression is true. +type IsTrue struct { + UnaryExpression + invert bool +} + +const IsTrueStr = "IS TRUE" +const IsFalseStr = "IS FALSE" + +// NewIsTrue creates a new IsTrue expression. +func NewIsTrue(child sql.Expression) *IsTrue { + return &IsTrue{UnaryExpression: UnaryExpression{child}} +} + +// NewIsFalse creates a new IsTrue expression with its boolean sense inverted (IsFalse, effectively). +func NewIsFalse(child sql.Expression) *IsTrue { + return &IsTrue{UnaryExpression: UnaryExpression{child}, invert: true} +} + +// Type implements the Expression interface. +func (*IsTrue) Type() sql.Type { + return sql.Boolean +} + +// IsNullable implements the Expression interface. +func (*IsTrue) IsNullable() bool { + return false +} + +// Eval implements the Expression interface. +func (e *IsTrue) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { + v, err := e.Child.Eval(ctx, row) + if err != nil { + return nil, err + } + + var boolVal interface{} + if v == nil { + return false, nil + } else { + boolVal, err = sql.Boolean.Convert(v) + if err != nil { + return nil, err + } + } + + if e.invert { + return !boolVal.(bool), nil + } + return boolVal, nil +} + +func (e *IsTrue) String() string { + isStr := IsTrueStr + if e.invert { + isStr = IsFalseStr + } + return e.Child.String() + " " + isStr +} + +// WithChildren implements the Expression interface. +func (e *IsTrue) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 1 { + return nil, errors.New("incorrect number of children") + } + + if e.invert { + return NewIsFalse(children[0]), nil + } + return NewIsTrue(children[0]), nil +} diff --git a/sql/expression/istrue_test.go b/sql/expression/istrue_test.go new file mode 100644 index 000000000..0cc4e41d1 --- /dev/null +++ b/sql/expression/istrue_test.go @@ -0,0 +1,85 @@ +package expression + +import ( + "github.com/src-d/go-mysql-server/sql" + "github.com/stretchr/testify/require" + "testing" +) + +func TestIsTrue(t *testing.T) { + require := require.New(t) + + boolF := NewGetField(0, sql.Boolean, "col1", true) + e := NewIsTrue(boolF) + require.Equal(sql.Boolean, e.Type()) + require.False(e.IsNullable()) + require.Equal(false, eval(t, e, sql.NewRow(nil))) + require.Equal(true, eval(t, e, sql.NewRow(true))) + require.Equal(false, eval(t, e, sql.NewRow(false))) + + intF := NewGetField(0, sql.Int64, "col1", true) + e = NewIsTrue(intF) + require.Equal(sql.Boolean, e.Type()) + require.False(e.IsNullable()) + require.Equal(false, eval(t, e, sql.NewRow(nil))) + require.Equal(true, eval(t, e, sql.NewRow(100))) + require.Equal(true, eval(t, e, sql.NewRow(-1))) + require.Equal(false, eval(t, e, sql.NewRow(0))) + + floatF := NewGetField(0, sql.Float64, "col1", true) + e = NewIsTrue(floatF) + require.Equal(sql.Boolean, e.Type()) + require.False(e.IsNullable()) + require.Equal(false, eval(t, e, sql.NewRow(nil))) + require.Equal(true, eval(t, e, sql.NewRow(1.5))) + require.Equal(true, eval(t, e, sql.NewRow(-1.5))) + require.Equal(false, eval(t, e, sql.NewRow(0))) + + stringF := NewGetField(0, sql.Text, "col1", true) + e = NewIsTrue(stringF) + require.Equal(sql.Boolean, e.Type()) + require.False(e.IsNullable()) + require.Equal(false, eval(t, e, sql.NewRow(nil))) + require.Equal(false, eval(t, e, sql.NewRow(""))) + require.Equal(false, eval(t, e, sql.NewRow("false"))) + require.Equal(false, eval(t, e, sql.NewRow("true"))) +} + +func TestIsFalse(t *testing.T) { + require := require.New(t) + + boolF := NewGetField(0, sql.Boolean, "col1", true) + e := NewIsFalse(boolF) + require.Equal(sql.Boolean, e.Type()) + require.False(e.IsNullable()) + require.Equal(false, eval(t, e, sql.NewRow(nil))) + require.Equal(false, eval(t, e, sql.NewRow(true))) + require.Equal(true, eval(t, e, sql.NewRow(false))) + + intF := NewGetField(0, sql.Int64, "col1", true) + e = NewIsFalse(intF) + require.Equal(sql.Boolean, e.Type()) + require.False(e.IsNullable()) + require.Equal(false, eval(t, e, sql.NewRow(nil))) + require.Equal(false, eval(t, e, sql.NewRow(100))) + require.Equal(false, eval(t, e, sql.NewRow(-1))) + require.Equal(true, eval(t, e, sql.NewRow(0))) + + floatF := NewGetField(0, sql.Float64, "col1", true) + e = NewIsFalse(floatF) + require.Equal(sql.Boolean, e.Type()) + require.False(e.IsNullable()) + require.Equal(false, eval(t, e, sql.NewRow(nil))) + require.Equal(false, eval(t, e, sql.NewRow(1.5))) + require.Equal(false, eval(t, e, sql.NewRow(-1.5))) + require.Equal(true, eval(t, e, sql.NewRow(0))) + + stringF := NewGetField(0, sql.Text, "col1", true) + e = NewIsFalse(stringF) + require.Equal(sql.Boolean, e.Type()) + require.False(e.IsNullable()) + require.Equal(false, eval(t, e, sql.NewRow(nil))) + require.Equal(true, eval(t, e, sql.NewRow(""))) + require.Equal(true, eval(t, e, sql.NewRow("false"))) + require.Equal(true, eval(t, e, sql.NewRow("true"))) +} diff --git a/sql/expression/like.go b/sql/expression/like.go index a99542da5..7913eb22c 100644 --- a/sql/expression/like.go +++ b/sql/expression/like.go @@ -7,8 +7,8 @@ import ( "strings" "sync" - "gopkg.in/src-d/go-mysql-server.v0/internal/regex" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/internal/regex" + "github.com/src-d/go-mysql-server/sql" ) // Like performs pattern matching against two strings. @@ -53,12 +53,14 @@ func (l *Like) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { } var ( - matcher regex.Matcher - right string + matcher regex.Matcher + disposer regex.Disposer + right string ) // eval right and convert to text if !l.cached || l.pool == nil { - v, err := l.Right.Eval(ctx, row) + var v interface{} + v, err = l.Right.Eval(ctx, row) if err != nil || v == nil { return nil, err } @@ -66,16 +68,16 @@ func (l *Like) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { if err != nil { return nil, err } - right = patternToRegex(v.(string)) + right = patternToGoRegex(v.(string)) } // for non-cached regex every time create a new matcher if !l.cached { - matcher, err = regex.New(regex.Default(), right) + matcher, disposer, err = regex.New("go", right) } else { if l.pool == nil { l.pool = &sync.Pool{ New: func() interface{} { - r, e := regex.New(regex.Default(), right) + r, _, e := regex.New(regex.Default(), right) if e != nil { err = e return nil @@ -91,9 +93,13 @@ func (l *Like) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { } ok := matcher.Match(left.(string)) - if l.pool != nil && l.cached { + if !l.cached { + disposer.Dispose() + } else if l.pool != nil { l.pool.Put(matcher) + } + return ok, nil } @@ -101,23 +107,17 @@ func (l *Like) String() string { return fmt.Sprintf("%s LIKE %s", l.Left, l.Right) } -// TransformUp implements the sql.Expression interface. -func (l *Like) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - left, err := l.Left.TransformUp(f) - if err != nil { - return nil, err - } - - right, err := l.Right.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (l *Like) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 2 { + return nil, sql.ErrInvalidChildrenNumber.New(l, len(children), 2) } - - return f(NewLike(left, right)) + return NewLike(children[0], children[1]), nil } -func patternToRegex(pattern string) string { +func patternToGoRegex(pattern string) string { var buf bytes.Buffer + buf.WriteString("(?s)") buf.WriteRune('^') var escaped bool for _, r := range strings.Replace(regexp.QuoteMeta(pattern), `\\`, `\`, -1) { diff --git a/sql/expression/like_test.go b/sql/expression/like_test.go index 93ac14d10..78b768bec 100644 --- a/sql/expression/like_test.go +++ b/sql/expression/like_test.go @@ -4,31 +4,31 @@ import ( "fmt" "testing" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestPatternToRegex(t *testing.T) { testCases := []struct { in, out string }{ - {`__`, `^..$`}, - {`_%_`, `^..*.$`}, - {`%_`, `^.*.$`}, - {`_%`, `^..*$`}, - {`a_b`, `^a.b$`}, - {`a%b`, `^a.*b$`}, - {`a.%b`, `^a\..*b$`}, - {`a\%b`, `^a%b$`}, - {`a\_b`, `^a_b$`}, - {`a\\b`, `^a\\b$`}, - {`a\\\_b`, `^a\\_b$`}, - {`(ab)`, `^\(ab\)$`}, + {`__`, `(?s)^..$`}, + {`_%_`, `(?s)^..*.$`}, + {`%_`, `(?s)^.*.$`}, + {`_%`, `(?s)^..*$`}, + {`a_b`, `(?s)^a.b$`}, + {`a%b`, `(?s)^a.*b$`}, + {`a.%b`, `(?s)^a\..*b$`}, + {`a\%b`, `(?s)^a%b$`}, + {`a\_b`, `(?s)^a_b$`}, + {`a\\b`, `(?s)^a\\b$`}, + {`a\\\_b`, `(?s)^a\\_b$`}, + {`(ab)`, `(?s)^\(ab\)$`}, } for _, tt := range testCases { t.Run(tt.in, func(t *testing.T) { - require.Equal(t, tt.out, patternToRegex(tt.in)) + require.Equal(t, tt.out, patternToGoRegex(tt.in)) }) } } diff --git a/sql/expression/literal.go b/sql/expression/literal.go index d263a79e2..02e88e0a9 100644 --- a/sql/expression/literal.go +++ b/sql/expression/literal.go @@ -3,7 +3,7 @@ package expression import ( "fmt" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // Literal represents a literal expression (string, number, bool, ...). @@ -14,6 +14,8 @@ type Literal struct { // NewLiteral creates a new Literal expression. func NewLiteral(value interface{}, fieldType sql.Type) *Literal { + // TODO(juanjux): we should probably check here if the type is sql.VarChar and the + // Capacity of the Type and the length of the value, but this can't return an error return &Literal{ value: value, fieldType: fieldType, @@ -51,10 +53,12 @@ func (p *Literal) String() string { } } -// TransformUp implements the Expression interface. -func (p *Literal) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - n := *p - return f(&n) +// WithChildren implements the Expression interface. +func (p *Literal) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(p, len(children), 0) + } + return p, nil } // Children implements the Expression interface. diff --git a/sql/expression/logic.go b/sql/expression/logic.go index 008de91fe..08a31c087 100644 --- a/sql/expression/logic.go +++ b/sql/expression/logic.go @@ -3,7 +3,7 @@ package expression import ( "fmt" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // And checks whether two expressions are true. @@ -68,19 +68,12 @@ func (a *And) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return true, nil } -// TransformUp implements the Expression interface. -func (a *And) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - left, err := a.Left.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (a *And) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 2 { + return nil, sql.ErrInvalidChildrenNumber.New(a, len(children), 2) } - - right, err := a.Right.TransformUp(f) - if err != nil { - return nil, err - } - - return f(NewAnd(left, right)) + return NewAnd(children[0], children[1]), nil } // Or checks whether one of the two given expressions is true. @@ -125,17 +118,10 @@ func (o *Or) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return rval == true, nil } -// TransformUp implements the Expression interface. -func (o *Or) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - left, err := o.Left.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Expression interface. +func (o *Or) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 2 { + return nil, sql.ErrInvalidChildrenNumber.New(o, len(children), 2) } - - right, err := o.Right.TransformUp(f) - if err != nil { - return nil, err - } - - return f(NewOr(left, right)) + return NewOr(children[0], children[1]), nil } diff --git a/sql/expression/logic_test.go b/sql/expression/logic_test.go index 399da8fff..b71c71b7e 100644 --- a/sql/expression/logic_test.go +++ b/sql/expression/logic_test.go @@ -3,8 +3,8 @@ package expression import ( "testing" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestAnd(t *testing.T) { diff --git a/sql/expression/set.go b/sql/expression/set.go new file mode 100644 index 000000000..d18bde374 --- /dev/null +++ b/sql/expression/set.go @@ -0,0 +1,61 @@ +package expression + +import ( + "fmt" + "github.com/src-d/go-mysql-server/sql" + "gopkg.in/src-d/go-errors.v1" +) + +var errCannotSetField = errors.NewKind("Expected GetField expression on left but got %T") + +// SetField updates the value of a field from a row. +type SetField struct { + BinaryExpression +} + +// NewSetField creates a new SetField expression. +func NewSetField(colName, expr sql.Expression) sql.Expression { + return &SetField{BinaryExpression{Left: colName, Right: expr}} +} + +func (s *SetField) String() string { + return fmt.Sprintf("SETFIELD %s = %s", s.Left, s.Right) +} + +// Type implements the Expression interface. +func (s *SetField) Type() sql.Type { + return s.Left.Type() +} + +// Eval implements the Expression interface. +// Returns a copy of the given row with an updated value. +func (s *SetField) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { + getField, ok := s.Left.(*GetField) + if !ok { + return nil, errCannotSetField.New(s.Left) + } + if getField.fieldIndex < 0 || getField.fieldIndex >= len(row) { + return nil, ErrIndexOutOfBounds.New(getField.fieldIndex, len(row)) + } + val, err := s.Right.Eval(ctx, row) + if err != nil { + return nil, err + } + if val != nil { + val, err = getField.fieldType.Convert(val) + if err != nil { + return nil, err + } + } + updatedRow := row.Copy() + updatedRow[getField.fieldIndex] = val + return updatedRow, nil +} + +// WithChildren implements the Expression interface. +func (s *SetField) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 2 { + return nil, sql.ErrInvalidChildrenNumber.New(s, len(children), 2) + } + return NewSetField(children[0], children[1]), nil +} \ No newline at end of file diff --git a/sql/expression/star.go b/sql/expression/star.go index 0fbd6d77b..5d6603be1 100644 --- a/sql/expression/star.go +++ b/sql/expression/star.go @@ -3,7 +3,7 @@ package expression import ( "fmt" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // Star represents the selection of all available fields. @@ -55,8 +55,10 @@ func (*Star) Eval(ctx *sql.Context, r sql.Row) (interface{}, error) { panic("star is just a placeholder node, but Eval was called") } -// TransformUp implements the Expression interface. -func (s *Star) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - n := *s - return f(&n) +// WithChildren implements the Expression interface. +func (s *Star) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(s, len(children), 0) + } + return s, nil } diff --git a/sql/expression/subquery.go b/sql/expression/subquery.go new file mode 100644 index 000000000..faae15aa1 --- /dev/null +++ b/sql/expression/subquery.go @@ -0,0 +1,125 @@ +package expression + +import ( + "fmt" + + "github.com/src-d/go-mysql-server/sql" + errors "gopkg.in/src-d/go-errors.v1" +) + +var errExpectedSingleRow = errors.NewKind("the subquery returned more than 1 row") + +// Subquery that is executed as an expression. +type Subquery struct { + Query sql.Node + value interface{} +} + +// NewSubquery returns a new subquery node. +func NewSubquery(node sql.Node) *Subquery { + return &Subquery{node, nil} +} + +// Eval implements the Expression interface. +func (s *Subquery) Eval(ctx *sql.Context, _ sql.Row) (interface{}, error) { + if s.value != nil { + if elems, ok := s.value.([]interface{}); ok { + if len(elems) > 1 { + return nil, errExpectedSingleRow.New() + } + return elems[0], nil + } + return s.value, nil + } + + iter, err := s.Query.RowIter(ctx) + if err != nil { + return nil, err + } + + rows, err := sql.RowIterToRows(iter) + if err != nil { + return nil, err + } + + if len(rows) == 0 { + s.value = nil + return nil, nil + } + + if len(rows) > 1 { + return nil, errExpectedSingleRow.New() + } + + s.value = rows[0][0] + return s.value, nil +} + +// EvalMultiple returns all rows returned by a subquery. +func (s *Subquery) EvalMultiple(ctx *sql.Context) ([]interface{}, error) { + if s.value != nil { + return s.value.([]interface{}), nil + } + + iter, err := s.Query.RowIter(ctx) + if err != nil { + return nil, err + } + + rows, err := sql.RowIterToRows(iter) + if err != nil { + return nil, err + } + + if len(rows) == 0 { + s.value = []interface{}{} + return nil, nil + } + + var result = make([]interface{}, len(rows)) + for i, row := range rows { + result[i] = row[0] + } + s.value = result + + return result, nil +} + +// IsNullable implements the Expression interface. +func (s *Subquery) IsNullable() bool { + return s.Query.Schema()[0].Nullable +} + +func (s *Subquery) String() string { + return fmt.Sprintf("(%s)", s.Query) +} + +// Resolved implements the Expression interface. +func (s *Subquery) Resolved() bool { + return s.Query.Resolved() +} + +// Type implements the Expression interface. +func (s *Subquery) Type() sql.Type { + return s.Query.Schema()[0].Type +} + +// WithChildren implements the Expression interface. +func (s *Subquery) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(s, len(children), 0) + } + return s, nil +} + +// Children implements the Expression interface. +func (s *Subquery) Children() []sql.Expression { + return nil +} + +// WithQuery returns the subquery with the query node changed. +func (s *Subquery) WithQuery(node sql.Node) *Subquery { + ns := *s + ns.Query = node + return &ns +} diff --git a/sql/expression/subquery_test.go b/sql/expression/subquery_test.go new file mode 100644 index 000000000..0d3f353be --- /dev/null +++ b/sql/expression/subquery_test.go @@ -0,0 +1,69 @@ +package expression_test + +import ( + "testing" + + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" + "github.com/stretchr/testify/require" +) + +func TestSubquery(t *testing.T) { + require := require.New(t) + table := memory.NewTable("", nil) + require.NoError(table.Insert(sql.NewEmptyContext(), nil)) + + subquery := expression.NewSubquery(plan.NewProject( + []sql.Expression{ + expression.NewLiteral("one", sql.Text), + }, + plan.NewResolvedTable(table), + )) + + value, err := subquery.Eval(sql.NewEmptyContext(), nil) + require.NoError(err) + require.Equal(value, "one") +} + +func TestSubqueryTooManyRows(t *testing.T) { + require := require.New(t) + table := memory.NewTable("", nil) + require.NoError(table.Insert(sql.NewEmptyContext(), nil)) + require.NoError(table.Insert(sql.NewEmptyContext(), nil)) + + subquery := expression.NewSubquery(plan.NewProject( + []sql.Expression{ + expression.NewLiteral("one", sql.Text), + }, + plan.NewResolvedTable(table), + )) + + _, err := subquery.Eval(sql.NewEmptyContext(), nil) + require.Error(err) +} + +func TestSubqueryMultipleRows(t *testing.T) { + require := require.New(t) + + ctx := sql.NewEmptyContext() + table := memory.NewTable("foo", sql.Schema{ + {Name: "t", Source: "foo", Type: sql.Text}, + }) + + require.NoError(table.Insert(ctx, sql.Row{"one"})) + require.NoError(table.Insert(ctx, sql.Row{"two"})) + require.NoError(table.Insert(ctx, sql.Row{"three"})) + + subquery := expression.NewSubquery(plan.NewProject( + []sql.Expression{ + expression.NewGetField(0, sql.Text, "t", false), + }, + plan.NewResolvedTable(table), + )) + + values, err := subquery.EvalMultiple(ctx) + require.NoError(err) + require.Equal(values, []interface{}{"one", "two", "three"}) +} diff --git a/sql/expression/transform.go b/sql/expression/transform.go new file mode 100644 index 000000000..05195c7fd --- /dev/null +++ b/sql/expression/transform.go @@ -0,0 +1,30 @@ +package expression + +import ( + "github.com/src-d/go-mysql-server/sql" +) + +// TransformUp applies a transformation function to the given expression from the +// bottom up. +func TransformUp(e sql.Expression, f sql.TransformExprFunc) (sql.Expression, error) { + children := e.Children() + if len(children) == 0 { + return f(e) + } + + newChildren := make([]sql.Expression, len(children)) + for i, c := range children { + c, err := TransformUp(c, f) + if err != nil { + return nil, err + } + newChildren[i] = c + } + + e, err := e.WithChildren(newChildren...) + if err != nil { + return nil, err + } + + return f(e) +} diff --git a/sql/expression/tuple.go b/sql/expression/tuple.go index 3f3dbaf19..11d35e1f5 100644 --- a/sql/expression/tuple.go +++ b/sql/expression/tuple.go @@ -4,7 +4,7 @@ import ( "fmt" "strings" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // Tuple is a fixed-size collection of expressions. @@ -77,18 +77,12 @@ func (t Tuple) Type() sql.Type { return sql.Tuple(types...) } -// TransformUp implements the Expression interface. -func (t Tuple) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - var exprs = make([]sql.Expression, len(t)) - for i, e := range t { - var err error - exprs[i], err = f(e) - if err != nil { - return nil, err - } +// WithChildren implements the Expression interface. +func (t Tuple) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != len(t) { + return nil, sql.ErrInvalidChildrenNumber.New(t, len(children), len(t)) } - - return f(Tuple(exprs)) + return NewTuple(children...), nil } // Children implements the Expression interface. diff --git a/sql/expression/tuple_test.go b/sql/expression/tuple_test.go index 947463d89..25fc411a7 100644 --- a/sql/expression/tuple_test.go +++ b/sql/expression/tuple_test.go @@ -3,8 +3,8 @@ package expression import ( "testing" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestTuple(t *testing.T) { diff --git a/sql/expression/unresolved.go b/sql/expression/unresolved.go index 4587edd48..6580655d2 100644 --- a/sql/expression/unresolved.go +++ b/sql/expression/unresolved.go @@ -4,7 +4,7 @@ import ( "fmt" "strings" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // UnresolvedColumn is an expression of a column that is not yet resolved. @@ -64,10 +64,12 @@ func (*UnresolvedColumn) Eval(ctx *sql.Context, r sql.Row) (interface{}, error) panic("unresolved column is a placeholder node, but Eval was called") } -// TransformUp implements the Expression interface. -func (uc *UnresolvedColumn) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - n := *uc - return f(&n) +// WithChildren implements the Expression interface. +func (uc *UnresolvedColumn) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(uc, len(children), 0) + } + return uc, nil } // UnresolvedFunction represents a function that is not yet resolved. @@ -126,16 +128,10 @@ func (*UnresolvedFunction) Eval(ctx *sql.Context, r sql.Row) (interface{}, error panic("unresolved function is a placeholder node, but Eval was called") } -// TransformUp implements the Expression interface. -func (uf *UnresolvedFunction) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { - var rc []sql.Expression - for _, c := range uf.Arguments { - c, err := c.TransformUp(f) - if err != nil { - return nil, err - } - rc = append(rc, c) +// WithChildren implements the Expression interface. +func (uf *UnresolvedFunction) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != len(uf.Arguments) { + return nil, sql.ErrInvalidChildrenNumber.New(uf, len(children), len(uf.Arguments)) } - - return f(NewUnresolvedFunction(uf.name, uf.IsAggregate, rc...)) + return NewUnresolvedFunction(uf.name, uf.IsAggregate, children...), nil } diff --git a/sql/expression/unresolved_test.go b/sql/expression/unresolved_test.go index 59ceb3551..f4b451b3e 100644 --- a/sql/expression/unresolved_test.go +++ b/sql/expression/unresolved_test.go @@ -3,8 +3,8 @@ package expression import ( "testing" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestUnresolvedExpression(t *testing.T) { diff --git a/sql/expression/walk.go b/sql/expression/walk.go index 36debed92..f04086781 100644 --- a/sql/expression/walk.go +++ b/sql/expression/walk.go @@ -1,6 +1,6 @@ package expression -import "gopkg.in/src-d/go-mysql-server.v0/sql" +import "github.com/src-d/go-mysql-server/sql" // Visitor visits exprs in the plan. type Visitor interface { diff --git a/sql/expression/walk_test.go b/sql/expression/walk_test.go index f093c4c6b..afbf4d7ad 100644 --- a/sql/expression/walk_test.go +++ b/sql/expression/walk_test.go @@ -3,8 +3,8 @@ package expression import ( "testing" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestWalk(t *testing.T) { diff --git a/sql/functionregistry.go b/sql/functionregistry.go index 6a6c0ef3a..0d9d6cfb9 100644 --- a/sql/functionregistry.go +++ b/sql/functionregistry.go @@ -1,125 +1,167 @@ package sql import ( + "github.com/src-d/go-mysql-server/internal/similartext" "gopkg.in/src-d/go-errors.v1" ) +// ErrFunctionAlreadyRegistered is thrown when a function is already registered +var ErrFunctionAlreadyRegistered = errors.NewKind("function '%s' is already registered") + // ErrFunctionNotFound is thrown when a function is not found -var ErrFunctionNotFound = errors.NewKind("function not found: %s") +var ErrFunctionNotFound = errors.NewKind("function: '%s' not found") // ErrInvalidArgumentNumber is returned when the number of arguments to call a // function is different from the function arity. -var ErrInvalidArgumentNumber = errors.NewKind("%s: expecting %v arguments for calling this function, %d received") +var ErrInvalidArgumentNumber = errors.NewKind("function '%s' expected %v arguments, %v received") -// Function is a function defined by the user that can be applied in a SQL -// query. +// Function is a function defined by the user that can be applied in a SQL query. type Function interface { // Call invokes the function. Call(...Expression) (Expression, error) + // Function name + name() string // isFunction will restrict implementations of Function isFunction() } type ( // Function0 is a function with 0 arguments. - Function0 func() Expression + Function0 struct { + Name string + Fn func() Expression + } // Function1 is a function with 1 argument. - Function1 func(e Expression) Expression + Function1 struct { + Name string + Fn func(e Expression) Expression + } // Function2 is a function with 2 arguments. - Function2 func(e1, e2 Expression) Expression + Function2 struct { + Name string + Fn func(e1, e2 Expression) Expression + } // Function3 is a function with 3 arguments. - Function3 func(e1, e2, e3 Expression) Expression + Function3 struct { + Name string + Fn func(e1, e2, e3 Expression) Expression + } // Function4 is a function with 4 arguments. - Function4 func(e1, e2, e3, e4 Expression) Expression + Function4 struct { + Name string + Fn func(e1, e2, e3, e4 Expression) Expression + } // Function5 is a function with 5 arguments. - Function5 func(e1, e2, e3, e4, e5 Expression) Expression + Function5 struct { + Name string + Fn func(e1, e2, e3, e4, e5 Expression) Expression + } // Function6 is a function with 6 arguments. - Function6 func(e1, e2, e3, e4, e5, e6 Expression) Expression + Function6 struct { + Name string + Fn func(e1, e2, e3, e4, e5, e6 Expression) Expression + } // Function7 is a function with 7 arguments. - Function7 func(e1, e2, e3, e4, e5, e6, e7 Expression) Expression + Function7 struct { + Name string + Fn func(e1, e2, e3, e4, e5, e6, e7 Expression) Expression + } // FunctionN is a function with variable number of arguments. This function // is expected to return ErrInvalidArgumentNumber if the arity does not // match, since the check has to be done in the implementation. - FunctionN func(...Expression) (Expression, error) + FunctionN struct { + Name string + Fn func(...Expression) (Expression, error) + } ) // Call implements the Function interface. func (fn Function0) Call(args ...Expression) (Expression, error) { if len(args) != 0 { - return nil, ErrInvalidArgumentNumber.New(0, len(args)) + return nil, ErrInvalidArgumentNumber.New(fn.Name, 0, len(args)) } - return fn(), nil + return fn.Fn(), nil } // Call implements the Function interface. func (fn Function1) Call(args ...Expression) (Expression, error) { if len(args) != 1 { - return nil, ErrInvalidArgumentNumber.New(1, len(args)) + return nil, ErrInvalidArgumentNumber.New(fn.Name, 1, len(args)) } - return fn(args[0]), nil + return fn.Fn(args[0]), nil } // Call implements the Function interface. func (fn Function2) Call(args ...Expression) (Expression, error) { if len(args) != 2 { - return nil, ErrInvalidArgumentNumber.New(2, len(args)) + return nil, ErrInvalidArgumentNumber.New(fn.Name, 2, len(args)) } - return fn(args[0], args[1]), nil + return fn.Fn(args[0], args[1]), nil } // Call implements the Function interface. func (fn Function3) Call(args ...Expression) (Expression, error) { if len(args) != 3 { - return nil, ErrInvalidArgumentNumber.New(3, len(args)) + return nil, ErrInvalidArgumentNumber.New(fn.Name, 3, len(args)) } - return fn(args[0], args[1], args[2]), nil + return fn.Fn(args[0], args[1], args[2]), nil } // Call implements the Function interface. func (fn Function4) Call(args ...Expression) (Expression, error) { if len(args) != 4 { - return nil, ErrInvalidArgumentNumber.New(4, len(args)) + return nil, ErrInvalidArgumentNumber.New(fn.Name, 4, len(args)) } - return fn(args[0], args[1], args[2], args[3]), nil + return fn.Fn(args[0], args[1], args[2], args[3]), nil } // Call implements the Function interface. func (fn Function5) Call(args ...Expression) (Expression, error) { if len(args) != 5 { - return nil, ErrInvalidArgumentNumber.New(5, len(args)) + return nil, ErrInvalidArgumentNumber.New(fn.Name, 5, len(args)) } - return fn(args[0], args[1], args[2], args[3], args[4]), nil + return fn.Fn(args[0], args[1], args[2], args[3], args[4]), nil } // Call implements the Function interface. func (fn Function6) Call(args ...Expression) (Expression, error) { if len(args) != 6 { - return nil, ErrInvalidArgumentNumber.New(6, len(args)) + return nil, ErrInvalidArgumentNumber.New(fn.Name, 6, len(args)) } - return fn(args[0], args[1], args[2], args[3], args[4], args[5]), nil + return fn.Fn(args[0], args[1], args[2], args[3], args[4], args[5]), nil } // Call implements the Function interface. func (fn Function7) Call(args ...Expression) (Expression, error) { if len(args) != 7 { - return nil, ErrInvalidArgumentNumber.New(7, len(args)) + return nil, ErrInvalidArgumentNumber.New(fn.Name, 7, len(args)) } - return fn(args[0], args[1], args[2], args[3], args[4], args[5], args[6]), nil + return fn.Fn(args[0], args[1], args[2], args[3], args[4], args[5], args[6]), nil } // Call implements the Function interface. func (fn FunctionN) Call(args ...Expression) (Expression, error) { - return fn(args...) + return fn.Fn(args...) } +func (fn Function0) name() string { return fn.Name } +func (fn Function1) name() string { return fn.Name } +func (fn Function2) name() string { return fn.Name } +func (fn Function3) name() string { return fn.Name } +func (fn Function4) name() string { return fn.Name } +func (fn Function5) name() string { return fn.Name } +func (fn Function6) name() string { return fn.Name } +func (fn Function7) name() string { return fn.Name } +func (fn FunctionN) name() string { return fn.Name } + func (Function0) isFunction() {} func (Function1) isFunction() {} func (Function2) isFunction() {} @@ -134,32 +176,41 @@ func (FunctionN) isFunction() {} // and User-Defined Functions. type FunctionRegistry map[string]Function -// Functions is a map of functions identified by their name. -type Functions map[string]Function - // NewFunctionRegistry creates a new FunctionRegistry. func NewFunctionRegistry() FunctionRegistry { return make(FunctionRegistry) } -// RegisterFunction registers a function with the given name. -func (r FunctionRegistry) RegisterFunction(name string, f Function) { - r[name] = f +// Register registers functions. +// If function with that name is already registered, +// the ErrFunctionAlreadyRegistered will be returned +func (r FunctionRegistry) Register(fn ...Function) error { + for _, f := range fn { + if _, ok := r[f.name()]; ok { + return ErrFunctionAlreadyRegistered.New(f.name()) + } + r[f.name()] = f + } + return nil } -// RegisterFunctions registers a map of functions. -func (r FunctionRegistry) RegisterFunctions(funcs Functions) { - for name, f := range funcs { - r[name] = f +// MustRegister registers functions. +// If function with that name is already registered, it will panic! +func (r FunctionRegistry) MustRegister(fn ...Function) { + if err := r.Register(fn...); err != nil { + panic(err) } } // Function returns a function with the given name. func (r FunctionRegistry) Function(name string) (Function, error) { - e, ok := r[name] - if !ok { + if len(r) == 0 { return nil, ErrFunctionNotFound.New(name) } - return e, nil + if fn, ok := r[name]; ok { + return fn, nil + } + similar := similartext.FindFromMap(r, name) + return nil, ErrFunctionNotFound.New(name + similar) } diff --git a/sql/functionregistry_test.go b/sql/functionregistry_test.go index 706779fcf..d60042331 100644 --- a/sql/functionregistry_test.go +++ b/sql/functionregistry_test.go @@ -3,9 +3,9 @@ package sql_test import ( "testing" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestFunctionRegistry(t *testing.T) { @@ -14,9 +14,10 @@ func TestFunctionRegistry(t *testing.T) { c := sql.NewCatalog() name := "func" var expected sql.Expression = expression.NewStar() - c.RegisterFunction(name, sql.Function1(func(arg sql.Expression) sql.Expression { - return expected - })) + c.MustRegister(sql.Function1{ + Name: name, + Fn: func(arg sql.Expression) sql.Expression { return expected }, + }) f, err := c.Function(name) require.NoError(err) diff --git a/sql/generator.go b/sql/generator.go new file mode 100644 index 000000000..218b1db66 --- /dev/null +++ b/sql/generator.go @@ -0,0 +1,57 @@ +package sql + +import ( + "io" + + "gopkg.in/src-d/go-errors.v1" +) + +// Generator will generate a set of values for a given row. +type Generator interface { + // Next value in the generator. + Next() (interface{}, error) + // Close the generator and dispose resources. + Close() error +} + +// ErrNotGenerator is returned when the value cannot be converted to a +// generator. +var ErrNotGenerator = errors.NewKind("cannot convert value of type %T to a generator") + +// ToGenerator converts a value to a generator if possible. +func ToGenerator(v interface{}) (Generator, error) { + switch v := v.(type) { + case Generator: + return v, nil + case []interface{}: + return NewArrayGenerator(v), nil + case nil: + return NewArrayGenerator(nil), nil + default: + return nil, ErrNotGenerator.New(v) + } +} + +// NewArrayGenerator creates a generator for a given array. +func NewArrayGenerator(array []interface{}) Generator { + return &arrayGenerator{array, 0} +} + +type arrayGenerator struct { + array []interface{} + pos int +} + +func (g *arrayGenerator) Next() (interface{}, error) { + if g.pos >= len(g.array) { + return nil, io.EOF + } + + g.pos++ + return g.array[g.pos-1], nil +} + +func (g *arrayGenerator) Close() error { + g.pos = len(g.array) + return nil +} diff --git a/sql/generator_test.go b/sql/generator_test.go new file mode 100644 index 000000000..145411333 --- /dev/null +++ b/sql/generator_test.go @@ -0,0 +1,54 @@ +package sql + +import ( + "fmt" + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestArrayGenerator(t *testing.T) { + require := require.New(t) + + expected := []interface{}{"a", "b", "c"} + gen := NewArrayGenerator(expected) + + var values []interface{} + for { + v, err := gen.Next() + if err != nil { + if err == io.EOF { + break + } + require.NoError(err) + } + values = append(values, v) + } + + require.Equal(expected, values) +} + +func TestToGenerator(t *testing.T) { + require := require.New(t) + + gen, err := ToGenerator([]interface{}{1, 2, 3}) + require.NoError(err) + require.Equal(NewArrayGenerator([]interface{}{1, 2, 3}), gen) + + gen, err = ToGenerator(new(fakeGen)) + require.NoError(err) + require.Equal(new(fakeGen), gen) + + gen, err = ToGenerator(nil) + require.NoError(err) + require.Equal(NewArrayGenerator(nil), gen) + + _, err = ToGenerator("foo") + require.Error(err) +} + +type fakeGen struct{} + +func (fakeGen) Next() (interface{}, error) { return nil, fmt.Errorf("not implemented") } +func (fakeGen) Close() error { return nil } diff --git a/sql/index.go b/sql/index.go index 92c0585f7..c673341ca 100644 --- a/sql/index.go +++ b/sql/index.go @@ -5,6 +5,8 @@ import ( "strings" "sync" + "github.com/src-d/go-mysql-server/internal/similartext" + "github.com/sirupsen/logrus" "gopkg.in/src-d/go-errors.v1" ) @@ -229,7 +231,7 @@ func (r *IndexRegistry) LoadIndexes(dbs Databases) error { } var checksum string - if c, ok := t.(Checksumable); ok { + if c, ok := t.(Checksumable); ok && len(indexes) != 0 { checksum, err = c.Checksum() if err != nil { return err @@ -253,11 +255,12 @@ func (r *IndexRegistry) LoadIndexes(dbs Databases) error { r.statuses[k] = IndexReady } else { logrus.Warnf( - "index %q is outdated and will not be used, you can remove it using `DROP INDEX %s`", + "index %q is outdated and will not be used, you can remove it using `DROP INDEX %s ON %s`", idx.ID(), idx.ID(), + idx.Table(), ) - r.statuses[k] = IndexOutdated + r.MarkOutdated(idx) } } } @@ -267,6 +270,12 @@ func (r *IndexRegistry) LoadIndexes(dbs Databases) error { return nil } +// MarkOutdated sets the index status as outdated. This method is not thread +// safe and should not be used directly except for testing. +func (r *IndexRegistry) MarkOutdated(idx Index) { + r.statuses[indexKey{idx.Database(), idx.ID()}] = IndexOutdated +} + func (r *IndexRegistry) retainIndex(db, id string) { r.rcmut.Lock() defer r.rcmut.Unlock() @@ -336,7 +345,7 @@ func (r *IndexRegistry) IndexesByTable(db, table string) []Index { r.mut.RLock() defer r.mut.RUnlock() - indexes := []Index{} + var indexes []Index for _, key := range r.indexOrder { idx := r.indexes[key] if idx.Database() == db && idx.Table() == table { @@ -541,6 +550,13 @@ func (r *IndexRegistry) AddIndex( func (r *IndexRegistry) DeleteIndex(db, id string, force bool) (<-chan struct{}, error) { r.mut.RLock() var key indexKey + + if len(r.indexes) == 0 { + return nil, ErrIndexNotFound.New(id) + } + + var indexNames []string + for k, idx := range r.indexes { if strings.ToLower(id) == idx.ID() { if !force && !r.CanRemoveIndex(idx) { @@ -551,11 +567,13 @@ func (r *IndexRegistry) DeleteIndex(db, id string, force bool) (<-chan struct{}, key = k break } + indexNames = append(indexNames, idx.ID()) } r.mut.RUnlock() if key.id == "" { - return nil, ErrIndexNotFound.New(id) + similar := similartext.Find(indexNames, id) + return nil, ErrIndexNotFound.New(id + similar) } var done = make(chan struct{}, 1) diff --git a/sql/index/config.go b/sql/index/config.go index dcf846216..7d5b7c9fd 100644 --- a/sql/index/config.go +++ b/sql/index/config.go @@ -24,7 +24,6 @@ func NewConfig( driverID string, driverConfig map[string]string, ) *Config { - cfg := &Config{ DB: db, Table: table, diff --git a/sql/index/pilosa/driver.go b/sql/index/pilosa/driver.go index 4b05e424a..4f9146b21 100644 --- a/sql/index/pilosa/driver.go +++ b/sql/index/pilosa/driver.go @@ -1,3 +1,5 @@ +// +build !windows + package pilosa import ( @@ -8,15 +10,21 @@ import ( "io/ioutil" "os" "path/filepath" + "runtime" + "strconv" "strings" + "sync" + "sync/atomic" "time" + "github.com/go-kit/kit/metrics/discard" opentracing "github.com/opentracing/opentracing-go" pilosa "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/syswrap" "github.com/sirupsen/logrus" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/index" errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/index" ) const ( @@ -35,8 +43,10 @@ const ( // ProcessingFileName is the extension of the lock/processing index file. ProcessingFileName = ".processing" - // MappingFileName is the extension of the mapping file. - MappingFileName = "mapping.db" + // MappingFileNamePrefix is the prefix in mapping file - + MappingFileNamePrefix = "map" + // MappingFileNameExtension is the extension in mapping file - + MappingFileNameExtension = ".db" ) const ( @@ -49,6 +59,11 @@ var ( errInvalidIndexType = errors.NewKind("expecting a pilosa index, instead got %T") ) +const ( + pilosaIndexThreadsKey = "PILOSA_INDEX_THREADS" + pilosaIndexThreadsVar = "pilosa_index_threads" +) + type ( bitBatch struct { size uint64 @@ -71,6 +86,17 @@ type ( } ) +var ( + // RowsGauge describes a metric that takes number of indexes rows over time. + RowsGauge = discard.NewGauge() + // TotalHistogram describes a metric that takes repeated observations of the total time to index values. + TotalHistogram = discard.NewHistogram() + // MappingHistogram describes a metric that takes repeated observations of the total time to map values. + MappingHistogram = discard.NewHistogram() + // BitmapHistogram describes a metric that takes repeated observations of the total time to store values in bitmaps + BitmapHistogram = discard.NewHistogram() +) + // NewDriver returns a new instance of pilosa.Driver // which satisfies sql.IndexDriver interface func NewDriver(root string) *Driver { @@ -84,6 +110,8 @@ func (*Driver) ID() string { return DriverID } +var errWriteConfigFile = errors.NewKind("unable to write indexes configuration file") + // Create a new index. func (d *Driver) Create( db, table, id string, @@ -107,7 +135,7 @@ func (d *Driver) Create( cfg := index.NewConfig(db, table, id, exprs, d.ID(), config) err = index.WriteConfigFile(d.configFilePath(db, table, id), cfg) if err != nil { - return nil, err + return nil, errWriteConfigFile.Wrap(err) } idx, err := d.newPilosaIndex(db, table) @@ -115,18 +143,19 @@ func (d *Driver) Create( return nil, err } - mapping := newMapping(d.mappingFilePath(db, table, id)) processingFile := d.processingFilePath(db, table, id) if err := index.WriteProcessingFile( processingFile, []byte{processingFileOnCreate}, ); err != nil { - return nil, err + return nil, errWriteConfigFile.Wrap(err) } - return newPilosaIndex(idx, mapping, cfg), nil + return newPilosaIndex(idx, cfg), nil } +var errReadIndexes = errors.NewKind("error loading all indexes for table %s of database %s: %s") + // LoadAll loads all indexes for given db and table func (d *Driver) LoadAll(db, table string) ([]sql.Index, error) { var ( @@ -140,7 +169,7 @@ func (d *Driver) LoadAll(db, table string) ([]sql.Index, error) { if os.IsNotExist(err) { return indexes, nil } - return nil, err + return nil, errReadIndexes.New(table, db, err) } for _, info := range dirs { if info.IsDir() && !strings.HasPrefix(info.Name(), ".") { @@ -162,6 +191,11 @@ func (d *Driver) LoadAll(db, table string) ([]sql.Index, error) { return indexes, nil } +var ( + errLoadingIndexConfig = errors.NewKind("unable to load index configuration") + errReadIndexConfig = errors.NewKind("unable to read index configuration") +) + func (d *Driver) loadIndex(db, table, id string) (*pilosaIndex, error) { idx, err := d.newPilosaIndex(db, table) if err != nil { @@ -174,15 +208,14 @@ func (d *Driver) loadIndex(db, table, id string) (*pilosaIndex, error) { dir := filepath.Join(d.root, db, table, id) config := d.configFilePath(db, table, id) - if _, err := os.Stat(config); err != nil { + if _, err = os.Stat(config); err != nil { return nil, errCorruptedIndex.New(dir) } - mapping := d.mappingFilePath(db, table, id) processing := d.processingFilePath(db, table, id) ok, err := index.ExistsProcessingFile(processing) if err != nil { - return nil, err + return nil, errLoadingIndexConfig.Wrap(err) } if ok { log := logrus.WithFields(logrus.Fields{ @@ -193,7 +226,7 @@ func (d *Driver) loadIndex(db, table, id string) (*pilosaIndex, error) { "dir": dir, }) log.Warn("could not read index file, index is corrupt and will be deleted") - if err := os.RemoveAll(dir); err != nil { + if err = os.RemoveAll(dir); err != nil { log.Warn("unable to remove corrupted index: " + dir) } @@ -202,13 +235,25 @@ func (d *Driver) loadIndex(db, table, id string) (*pilosaIndex, error) { cfg, err := index.ReadConfigFile(config) if err != nil { - return nil, err + return nil, errReadIndexConfig.Wrap(err) } - if cfg.Driver(DriverID) == nil { + cfgDriver := cfg.Driver(DriverID) + if cfgDriver == nil { return nil, errCorruptedIndex.New(dir) } - return newPilosaIndex(idx, newMapping(mapping), cfg), nil + pilosaIndex := newPilosaIndex(idx, cfg) + for k, v := range cfgDriver { + if strings.HasPrefix(v, MappingFileNamePrefix) && strings.HasSuffix(v, MappingFileNameExtension) { + path := d.mappingFilePath(db, table, id, k) + if _, err := os.Stat(path); err != nil { + continue + } + pilosaIndex.mapping[k] = newMapping(path) + } + } + + return pilosaIndex, nil } func (d *Driver) savePartition( @@ -217,13 +262,13 @@ func (d *Driver) savePartition( kviter sql.IndexKeyValueIter, idx *pilosaIndex, pilosaIndex *concurrentPilosaIndex, - offset uint64, b *batch, ) (uint64, error) { var ( colID uint64 err error ) + for i, e := range idx.Expressions() { name := fieldName(idx.ID(), e, p) pilosaIndex.DeleteField(name) @@ -236,27 +281,33 @@ func (d *Driver) savePartition( } rollback := true - if err := idx.mapping.openCreate(true); err != nil { + mk := mappingKey(p) + mapping, ok := idx.mapping[mk] + if !ok { + return 0, errMappingNotFound.New(mk) + } + if err := mapping.openCreate(true); err != nil { return 0, err } defer func() { if rollback { - idx.mapping.rollback() + mapping.rollback() } else { - e := d.saveMapping(ctx, idx.mapping, colID, false, b) + e := d.saveMapping(ctx, mapping, colID, false, b) if e != nil && err == nil { err = e } } - idx.mapping.close() + mapping.close() + kviter.Close() }() - for colID = offset; err == nil; colID++ { + for colID = 0; err == nil; colID++ { // commit each batch of objects (pilosa and boltdb) if colID%sql.IndexBatchSize == 0 && colID != 0 { - if err = d.saveBatch(ctx, idx.mapping, colID, b); err != nil { + if err = d.saveBatch(ctx, mapping, colID, b); err != nil { return 0, err } } @@ -264,28 +315,31 @@ func (d *Driver) savePartition( select { case <-ctx.Context.Done(): return 0, ctx.Context.Err() - default: - var ( - values []interface{} - location []byte - ) - if values, location, err = kviter.Next(); err != nil { - break - } + } - for i, field := range b.fields { - if values[i] == nil { - continue - } + values, location, err := kviter.Next() + if err != nil { + break + } - rowID, err := idx.mapping.getRowID(field.Name(), values[i]) - if err != nil { - return 0, err - } - b.bitBatches[i].Add(rowID, colID) + for i, field := range b.fields { + if values[i] == nil { + continue + } + + var rowID uint64 + rowID, err = mapping.getRowID(field.Name(), values[i]) + if err != nil { + return 0, err } - err = idx.mapping.putLocation(pilosaIndex.Name(), colID, location) + + b.bitBatches[i].Add(rowID, colID) + } + + err = mapping.putLocation(pilosaIndex.Name(), colID, location) + if err != nil { + return 0, err } } @@ -306,7 +360,7 @@ func (d *Driver) savePartition( } } - return colID - offset, err + return colID, err } // Save the given index (mapping and bitmap) @@ -330,46 +384,97 @@ func (d *Driver) Save( idx.wg.Add(1) defer idx.wg.Done() - var b = batch{ - fields: make([]*pilosa.Field, len(idx.Expressions())), - bitBatches: make([]*bitBatch, len(idx.Expressions())), - } - ctx.Context, idx.cancel = context.WithCancel(ctx.Context) processingFile := d.processingFilePath(i.Database(), i.Table(), i.ID()) - if err := index.WriteProcessingFile( + err = index.WriteProcessingFile( processingFile, []byte{processingFileOnSave}, - ); err != nil { - return err + ) + if err != nil { + return errWriteConfigFile.Wrap(err) } + cfgPath := d.configFilePath(i.Database(), i.Table(), i.ID()) + cfg, err := index.ReadConfigFile(cfgPath) + if err != nil { + return errReadIndexConfig.Wrap(err) + } + driverCfg := cfg.Driver(DriverID) + + defer iter.Close() pilosaIndex := idx.index - var rows uint64 + + var ( + rows, timePilosa, timeMapping uint64 + + wg sync.WaitGroup + tokens = make(chan struct{}, indexThreads(ctx)) + + errors []error + errmut sync.Mutex + ) + for { + select { + case <-ctx.Done(): + return + default: + } + p, kviter, err := iter.Next() if err != nil { if err == io.EOF { break } - return err - } - numRows, err := d.savePartition(ctx, p, kviter, idx, pilosaIndex, rows, &b) - if err != nil { + idx.cancel() + wg.Wait() return err } + mk := mappingKey(p) + driverCfg[mk] = mappingFileName(mk) + mapping := newMapping(d.mappingFilePath(idx.Database(), idx.Table(), idx.ID(), mk)) + idx.mapping[mk] = mapping + + wg.Add(1) + + go func() { + defer func() { + wg.Done() + <-tokens + }() + + tokens <- struct{}{} - rows += numRows + var b = &batch{ + fields: make([]*pilosa.Field, len(idx.Expressions())), + bitBatches: make([]*bitBatch, len(idx.Expressions())), + } + + numRows, err := d.savePartition(ctx, p, kviter, idx, pilosaIndex, b) + if err != nil { + errmut.Lock() + errors = append(errors, err) + idx.cancel() + errmut.Unlock() + return + } + + atomic.AddUint64(&timeMapping, uint64(b.timeMapping)) + atomic.AddUint64(&timePilosa, uint64(b.timePilosa)) + atomic.AddUint64(&rows, numRows) + }() } - logrus.WithFields(logrus.Fields{ - "duration": time.Since(start), - "pilosa": b.timePilosa, - "mapping": b.timeMapping, - "rows": rows, - "id": i.ID(), - }).Debugf("finished pilosa indexing") + wg.Wait() + if len(errors) > 0 { + return errors[0] + } + if err = index.WriteConfigFile(cfgPath, cfg); err != nil { + return errWriteConfigFile.Wrap(err) + } + + observeIndex(time.Since(start), timePilosa, timeMapping, rows) return index.RemoveProcessingFile(processingFile) } @@ -414,23 +519,25 @@ func (d *Driver) Delete(i sql.Index, partitions sql.PartitionIter) error { return err } } + mk := mappingKey(p) + delete(idx.mapping, mk) } return partitions.Close() } -func (d *Driver) saveBatch(ctx *sql.Context, m *mapping, colID uint64, b *batch) error { - err := d.savePilosa(ctx, colID, b) +func (d *Driver) saveBatch(ctx *sql.Context, m *mapping, cols uint64, b *batch) error { + err := d.savePilosa(ctx, cols, b) if err != nil { return err } - return d.saveMapping(ctx, m, colID, true, b) + return d.saveMapping(ctx, m, cols, true, b) } -func (d *Driver) savePilosa(ctx *sql.Context, colID uint64, b *batch) error { +func (d *Driver) savePilosa(ctx *sql.Context, cols uint64, b *batch) error { span, _ := ctx.Span("pilosa.Save.bitBatch", - opentracing.Tag{Key: "cols", Value: colID}, + opentracing.Tag{Key: "cols", Value: cols}, opentracing.Tag{Key: "fields", Value: len(b.fields)}, ) defer span.Finish() @@ -455,12 +562,12 @@ func (d *Driver) savePilosa(ctx *sql.Context, colID uint64, b *batch) error { func (d *Driver) saveMapping( ctx *sql.Context, m *mapping, - colID uint64, + cols uint64, cont bool, b *batch, ) error { span, _ := ctx.Span("pilosa.Save.mapping", - opentracing.Tag{Key: "cols", Value: colID}, + opentracing.Tag{Key: "cols", Value: cols}, opentracing.Tag{Key: "continues", Value: cont}, ) defer span.Finish() @@ -526,8 +633,13 @@ func (d *Driver) processingFilePath(db, table, id string) string { return filepath.Join(d.root, db, table, id, ProcessingFileName) } -func (d *Driver) mappingFilePath(db, table, id string) string { - return filepath.Join(d.root, db, table, id, MappingFileName) +func mappingFileName(key string) string { + h := sha1.New() + io.WriteString(h, key) + return fmt.Sprintf("%s-%x%s", MappingFileNamePrefix, h.Sum(nil), MappingFileNameExtension) +} +func (d *Driver) mappingFilePath(db, table, id string, key string) string { + return filepath.Join(d.root, db, table, id, mappingFileName(key)) } func (d *Driver) newPilosaIndex(db, table string) (*pilosa.Index, error) { @@ -539,3 +651,40 @@ func (d *Driver) newPilosaIndex(db, table string) (*pilosa.Index, error) { } return idx, nil } + +func indexThreads(ctx *sql.Context) int { + typ, val := ctx.Session.Get(pilosaIndexThreadsVar) + if val != nil && typ == sql.Int64 { + return int(val.(int64)) + } + + var value int + if v, ok := os.LookupEnv(pilosaIndexThreadsKey); ok { + value, _ = strconv.Atoi(v) + } + + if value <= 0 { + value = runtime.NumCPU() + } + + return value +} + +func observeIndex(timeTotal time.Duration, timePilosa, timeMapping, rows uint64) { + logrus.WithFields(logrus.Fields{ + "duration": timeTotal, + "pilosa": timePilosa, + "mapping": timeMapping, + "rows": rows, + "id": DriverID, + }).Debugf("finished pilosa indexing") + + TotalHistogram.With("driver", DriverID, "duration", "seconds").Observe(timeTotal.Seconds()) + BitmapHistogram.With("driver", DriverID, "duration", "seconds").Observe(float64(timePilosa)) + MappingHistogram.With("driver", DriverID, "duration", "seconds").Observe(float64(timeMapping)) + RowsGauge.With("driver", DriverID).Set(float64(rows)) +} + +func init() { + syswrap.SetMaxMapCount(0) +} diff --git a/sql/index/pilosa/driver_test.go b/sql/index/pilosa/driver_test.go index 8c06d680c..25bae3d7c 100644 --- a/sql/index/pilosa/driver_test.go +++ b/sql/index/pilosa/driver_test.go @@ -1,3 +1,5 @@ +// +build !windows + package pilosa import ( @@ -11,10 +13,10 @@ import ( "time" "github.com/pilosa/pilosa" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/test" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/test" ) var tmpDir string @@ -189,16 +191,20 @@ func TestSaveAndLoad(t *testing.T) { require.Equal(1, len(indexes)) var locations = make([][]string, len(it.records)) + for partition, records := range it.records { for _, r := range records { - lookup, err := sqlIdx.Get(r.values...) + var lookup sql.IndexLookup + lookup, err = sqlIdx.Get(r.values...) require.NoError(err) - lit, err := lookup.Values(testPartition(partition)) + var lit sql.IndexValueIter + lit, err = lookup.Values(testPartition(partition)) require.NoError(err) for { - loc, err := lit.Next() + var loc []byte + loc, err = lit.Next() if err == io.EOF { break } @@ -555,6 +561,7 @@ func TestIntersection(t *testing.T) { lookupLang, err := sqlIdxLang.Get(itLang.records[0][0].values...) require.NoError(err) + lookupPath, err := sqlIdxPath.Get(itPath.records[0][itPath.total-1].values...) require.NoError(err) @@ -867,7 +874,8 @@ func TestUnionDiffAsc(t *testing.T) { for partition, records := range it.records { ls[partition] = make([]*indexLookup, it.total) for i, r := range records { - l, err := pilosaIdx.Get(r.values...) + var l sql.IndexLookup + l, err = pilosaIdx.Get(r.values...) require.NoError(err) ls[partition][i], _ = l.(*indexLookup) } @@ -880,7 +888,8 @@ func TestUnionDiffAsc(t *testing.T) { require.NoError(err) for i := 1; i < it.total-1; i += 2 { - loc, err := diffIt.Next() + var loc []byte + loc, err = diffIt.Next() require.NoError(err) require.Equal(it.records[0][i].location, loc) @@ -934,7 +943,8 @@ func TestInterRanges(t *testing.T) { require.NoError(err) for i := ranges[0]; i < ranges[1]; i++ { - loc, err := interIt.Next() + var loc []byte + loc, err = interIt.Next() require.NoError(err) require.Equal(it.records[0][i].location, loc) } @@ -1291,10 +1301,10 @@ func (it *testIndexKeyValueIter) Next() ([]interface{}, []byte, error) { values[i] = e + "-" + loc + "-" + string(it.partition.Key()) } - *it.records = append(*it.records, testRecord{ + (*it.records)[it.offset] = testRecord{ values, []byte(loc), - }) + } it.offset++ return values, []byte(loc), nil @@ -1430,13 +1440,23 @@ type partitionKeyValueIter struct { records [][]testRecord } +func (i *partitionKeyValueIter) init() { + i.records = make([][]testRecord, i.partitions) + for j := 0; j < i.partitions; j++ { + i.records[j] = make([]testRecord, i.total) + } +} + func (i *partitionKeyValueIter) Next() (sql.Partition, sql.IndexKeyValueIter, error) { if i.pos >= i.partitions { return nil, nil, io.EOF } + if i.pos == 0 { + i.init() + } + i.pos++ - i.records = append(i.records, []testRecord{}) return testPartition(i.pos - 1), &testIndexKeyValueIter{ offset: i.offset, total: i.total, diff --git a/sql/index/pilosa/index.go b/sql/index/pilosa/index.go index 86448d548..d29312b86 100644 --- a/sql/index/pilosa/index.go +++ b/sql/index/pilosa/index.go @@ -1,3 +1,5 @@ +// +build !windows + package pilosa import ( @@ -5,9 +7,9 @@ import ( "sync" "github.com/pilosa/pilosa" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/index" errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/index" ) // concurrentPilosaIndex is a wrapper of pilosa.Index that can be opened and closed @@ -59,7 +61,7 @@ var ( // pilosaIndex is an pilosa implementation of sql.Index interface type pilosaIndex struct { index *concurrentPilosaIndex - mapping *mapping + mapping map[string]*mapping cancel context.CancelFunc wg sync.WaitGroup @@ -70,7 +72,7 @@ type pilosaIndex struct { checksum string } -func newPilosaIndex(idx *pilosa.Index, mapping *mapping, cfg *index.Config) *pilosaIndex { +func newPilosaIndex(idx *pilosa.Index, cfg *index.Config) *pilosaIndex { var checksum string for _, c := range cfg.Drivers { if ch, ok := c[sql.ChecksumKey]; ok { @@ -85,7 +87,7 @@ func newPilosaIndex(idx *pilosa.Index, mapping *mapping, cfg *index.Config) *pil table: cfg.Table, id: cfg.ID, expressions: cfg.Expressions, - mapping: mapping, + mapping: make(map[string]*mapping), checksum: checksum, } } @@ -116,15 +118,21 @@ func (idx *pilosaIndex) Get(keys ...interface{}) (sql.IndexLookup, error) { // Has checks if the given key is present in the index mapping func (idx *pilosaIndex) Has(p sql.Partition, key ...interface{}) (bool, error) { - if err := idx.mapping.open(); err != nil { + mk := mappingKey(p) + m, ok := idx.mapping[mk] + if !ok { + return false, errMappingNotFound.New(mk) + } + + if err := m.open(); err != nil { return false, err } - defer idx.mapping.close() + defer m.close() for i, expr := range idx.expressions { name := fieldName(idx.ID(), expr, p) - val, err := idx.mapping.get(name, key[i]) + val, err := m.get(name, key[i]) if err != nil || val == nil { return false, err } @@ -297,7 +305,8 @@ func newAscendLookup(f *filteredLookup, gte []interface{}, lt []interface{}) *as return false, err } - cmp, err := compare(v, l.gte[i]) + var cmp int + cmp, err = compare(v, l.gte[i]) if err != nil { return false, err } @@ -343,7 +352,8 @@ func newDescendLookup(f *filteredLookup, gt []interface{}, lte []interface{}) *d return false, err } - cmp, err := compare(v, l.gt[i]) + var cmp int + cmp, err = compare(v, l.gt[i]) if err != nil { return false, err } diff --git a/sql/index/pilosa/iterator.go b/sql/index/pilosa/iterator.go index a484f6684..d5b8eed41 100644 --- a/sql/index/pilosa/iterator.go +++ b/sql/index/pilosa/iterator.go @@ -1,10 +1,12 @@ +// +build !windows + package pilosa import ( "io" - "github.com/boltdb/bolt" "github.com/sirupsen/logrus" + bolt "go.etcd.io/bbolt" ) type locationValueIter struct { diff --git a/sql/index/pilosa/lookup.go b/sql/index/pilosa/lookup.go index 225c464a0..29173921b 100644 --- a/sql/index/pilosa/lookup.go +++ b/sql/index/pilosa/lookup.go @@ -1,3 +1,5 @@ +// +build !windows + package pilosa import ( @@ -9,14 +11,15 @@ import ( "time" "github.com/pilosa/pilosa" + "github.com/src-d/go-mysql-server/sql" errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) var ( errUnknownType = errors.NewKind("unknown type %T received as value") errTypeMismatch = errors.NewKind("cannot compare type %T with type %T") errUnmergeableType = errors.NewKind("unmergeable type %T") + errMappingNotFound = errors.NewKind("mapping not found for partition: %s") // operation functors // r1 AND r2 @@ -60,7 +63,7 @@ type ( indexLookup struct { id string index *concurrentPilosaIndex - mapping *mapping + mapping map[string]*mapping keys []interface{} expressions []string operations []*lookupOperation @@ -82,11 +85,12 @@ func (l *indexLookup) indexName() string { return l.index.Name() } -func (l *indexLookup) intersectExpressions(p sql.Partition) (*pilosa.Row, error) { +func (l *indexLookup) intersectExpressions(p sql.Partition, m *mapping) (*pilosa.Row, error) { var row *pilosa.Row + for i, expr := range l.expressions { field := l.index.Field(fieldName(l.id, expr, p)) - rowID, err := l.mapping.rowID(field.Name(), l.keys[i]) + rowID, err := m.rowID(field.Name(), l.keys[i]) if err == io.EOF { continue } @@ -105,15 +109,21 @@ func (l *indexLookup) intersectExpressions(p sql.Partition) (*pilosa.Row, error) } func (l *indexLookup) values(p sql.Partition) (*pilosa.Row, error) { - if err := l.mapping.open(); err != nil { + mk := mappingKey(p) + m, ok := l.mapping[mk] + if !ok { + return nil, errMappingNotFound.New(mk) + } + + if err := m.open(); err != nil { return nil, err } - defer l.mapping.close() + defer m.close() if err := l.index.Open(); err != nil { return nil, err } - row, err := l.intersectExpressions(p) + row, err := l.intersectExpressions(p, m) if e := l.index.Close(); e != nil { if err == nil { err = e @@ -148,20 +158,29 @@ func (l *indexLookup) values(p sql.Partition) (*pilosa.Row, error) { // Values implements sql.IndexLookup.Values func (l *indexLookup) Values(p sql.Partition) (sql.IndexValueIter, error) { + mk := mappingKey(p) + m, ok := l.mapping[mk] + if !ok { + return nil, errMappingNotFound.New(mk) + } + row, err := l.values(p) if err != nil { return nil, err } if row == nil { - return &indexValueIter{mapping: l.mapping, indexName: l.index.Name()}, nil + return &indexValueIter{ + mapping: m, + indexName: l.index.Name(), + }, nil } bits := row.Columns() return &indexValueIter{ total: uint64(len(bits)), bits: bits, - mapping: l.mapping, + mapping: m, indexName: l.index.Name(), }, nil } @@ -221,7 +240,7 @@ func (l *indexLookup) Difference(lookups ...sql.IndexLookup) sql.IndexLookup { type filteredLookup struct { id string index *concurrentPilosaIndex - mapping *mapping + mapping map[string]*mapping keys []interface{} expressions []string operations []*lookupOperation @@ -236,12 +255,12 @@ func (l *filteredLookup) indexName() string { } // evaluate Intersection of bitmaps -func (l *filteredLookup) intersectExpressions(p sql.Partition) (*pilosa.Row, error) { +func (l *filteredLookup) intersectExpressions(p sql.Partition, m *mapping) (*pilosa.Row, error) { var row *pilosa.Row for i, expr := range l.expressions { field := l.index.Field(fieldName(l.id, expr, p)) - rows, err := l.mapping.filter(field.Name(), func(b []byte) (bool, error) { + rows, err := m.filter(field.Name(), func(b []byte) (bool, error) { return l.filter(i, b) }) if err != nil { @@ -264,15 +283,21 @@ func (l *filteredLookup) intersectExpressions(p sql.Partition) (*pilosa.Row, err } func (l *filteredLookup) values(p sql.Partition) (*pilosa.Row, error) { - if err := l.mapping.open(); err != nil { + mk := mappingKey(p) + m, ok := l.mapping[mk] + if !ok { + return nil, errMappingNotFound.New(mk) + } + + if err := m.open(); err != nil { return nil, err } - defer l.mapping.close() + defer m.close() if err := l.index.Open(); err != nil { return nil, err } - row, err := l.intersectExpressions(p) + row, err := l.intersectExpressions(p, m) if e := l.index.Close(); e != nil { if err == nil { err = e @@ -309,21 +334,30 @@ func (l *filteredLookup) values(p sql.Partition) (*pilosa.Row, error) { } func (l *filteredLookup) Values(p sql.Partition) (sql.IndexValueIter, error) { + mk := mappingKey(p) + m, ok := l.mapping[mk] + if !ok { + return nil, errMappingNotFound.New(mk) + } row, err := l.values(p) if err != nil { return nil, err } if row == nil { - return &indexValueIter{mapping: l.mapping, indexName: l.index.Name()}, nil + return &indexValueIter{ + mapping: m, + indexName: l.index.Name(), + }, nil } bits := row.Columns() - if err := l.mapping.open(); err != nil { + if err = m.open(); err != nil { return nil, err } - defer l.mapping.close() - locations, err := l.mapping.sortedLocations(l.index.Name(), bits, l.reverse) + + defer m.close() + locations, err := m.sortedLocations(l.index.Name(), bits, l.reverse) if err != nil { return nil, err } @@ -397,7 +431,7 @@ type descendLookup struct { type negateLookup struct { id string index *concurrentPilosaIndex - mapping *mapping + mapping map[string]*mapping keys []interface{} expressions []string indexes map[string]struct{} @@ -406,12 +440,12 @@ type negateLookup struct { func (l *negateLookup) indexName() string { return l.index.Name() } -func (l *negateLookup) intersectExpressions(p sql.Partition) (*pilosa.Row, error) { +func (l *negateLookup) intersectExpressions(p sql.Partition, m *mapping) (*pilosa.Row, error) { var row *pilosa.Row for i, expr := range l.expressions { field := l.index.Field(fieldName(l.id, expr, p)) - maxRowID, err := l.mapping.getMaxRowID(field.Name()) + maxRowID, err := m.getMaxRowID(field.Name()) if err != nil { return nil, err } @@ -423,14 +457,15 @@ func (l *negateLookup) intersectExpressions(p sql.Partition) (*pilosa.Row, error var r *pilosa.Row // rowIDs start with 1 for ri := uint64(1); ri <= maxRowID; ri++ { - rr, err := field.Row(ri) + var rr *pilosa.Row + rr, err = field.Row(ri) if err != nil { return nil, err } r = union(r, rr) } - rowID, err := l.mapping.rowID(field.Name(), l.keys[i]) + rowID, err := m.rowID(field.Name(), l.keys[i]) if err != nil && err != io.EOF { return nil, err } @@ -447,15 +482,21 @@ func (l *negateLookup) intersectExpressions(p sql.Partition) (*pilosa.Row, error } func (l *negateLookup) values(p sql.Partition) (*pilosa.Row, error) { - if err := l.mapping.open(); err != nil { + mk := mappingKey(p) + m, ok := l.mapping[mk] + if !ok { + return nil, errMappingNotFound.New(mk) + } + + if err := m.open(); err != nil { return nil, err } - defer l.mapping.close() + defer m.close() if err := l.index.Open(); err != nil { return nil, err } - row, err := l.intersectExpressions(p) + row, err := l.intersectExpressions(p, m) if e := l.index.Close(); e != nil { if err == nil { err = e @@ -494,20 +535,28 @@ func (l *negateLookup) values(p sql.Partition) (*pilosa.Row, error) { // Values implements sql.IndexLookup.Values func (l *negateLookup) Values(p sql.Partition) (sql.IndexValueIter, error) { + mk := mappingKey(p) + m, ok := l.mapping[mk] + if !ok { + return nil, errMappingNotFound.New(mk) + } row, err := l.values(p) if err != nil { return nil, err } if row == nil { - return &indexValueIter{mapping: l.mapping, indexName: l.index.Name()}, nil + return &indexValueIter{ + mapping: m, + indexName: l.index.Name(), + }, nil } bits := row.Columns() return &indexValueIter{ total: uint64(len(bits)), bits: bits, - mapping: l.mapping, + mapping: m, indexName: l.index.Name(), }, nil } @@ -572,6 +621,14 @@ func decodeGob(k []byte, value interface{}) (interface{}, error) { var v string err := decoder.Decode(&v) return v, err + case int8: + var v int8 + err := decoder.Decode(&v) + return v, err + case int16: + var v int16 + err := decoder.Decode(&v) + return v, err case int32: var v int32 err := decoder.Decode(&v) @@ -580,6 +637,14 @@ func decodeGob(k []byte, value interface{}) (interface{}, error) { var v int64 err := decoder.Decode(&v) return v, err + case uint8: + var v uint8 + err := decoder.Decode(&v) + return v, err + case uint16: + var v uint16 + err := decoder.Decode(&v) + return v, err case uint32: var v uint32 err := decoder.Decode(&v) @@ -639,6 +704,36 @@ func compare(a, b interface{}) (int, error) { } return strings.Compare(a, v), nil + case int8: + v, ok := b.(int8) + if !ok { + return 0, errTypeMismatch.New(a, b) + } + + if a == v { + return 0, nil + } + + if a < v { + return -1, nil + } + + return 1, nil + case int16: + v, ok := b.(int16) + if !ok { + return 0, errTypeMismatch.New(a, b) + } + + if a == v { + return 0, nil + } + + if a < v { + return -1, nil + } + + return 1, nil case int32: v, ok := b.(int32) if !ok { @@ -668,6 +763,36 @@ func compare(a, b interface{}) (int, error) { return -1, nil } + return 1, nil + case uint8: + v, ok := b.(uint8) + if !ok { + return 0, errTypeMismatch.New(a, b) + } + + if a == v { + return 0, nil + } + + if a < v { + return -1, nil + } + + return 1, nil + case uint16: + v, ok := b.(uint16) + if !ok { + return 0, errTypeMismatch.New(a, b) + } + + if a == v { + return 0, nil + } + + if a < v { + return -1, nil + } + return 1, nil case uint32: v, ok := b.(uint32) diff --git a/sql/index/pilosa/lookup_test.go b/sql/index/pilosa/lookup_test.go index a3b05ff46..b93da3ca2 100644 --- a/sql/index/pilosa/lookup_test.go +++ b/sql/index/pilosa/lookup_test.go @@ -1,3 +1,5 @@ +// +build !windows + package pilosa import ( @@ -9,9 +11,9 @@ import ( "time" "github.com/pilosa/pilosa" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestCompare(t *testing.T) { @@ -96,8 +98,12 @@ func TestCompare(t *testing.T) { func TestDecodeGob(t *testing.T) { testCases := []interface{}{ "foo", + int8(1), + int16(1), int32(1), int64(1), + uint8(1), + uint16(1), uint32(1), uint64(1), float64(1), diff --git a/sql/index/pilosa/mapping.go b/sql/index/pilosa/mapping.go index bbe281ba9..8e33d3f2f 100644 --- a/sql/index/pilosa/mapping.go +++ b/sql/index/pilosa/mapping.go @@ -1,3 +1,5 @@ +// +build !windows + package pilosa import ( @@ -9,7 +11,8 @@ import ( "sort" "sync" - "github.com/boltdb/bolt" + "github.com/src-d/go-mysql-server/sql" + bolt "go.etcd.io/bbolt" ) // mapping @@ -127,20 +130,19 @@ func (m *mapping) rollback() error { } func (m *mapping) transaction(writable bool, f func(*bolt.Tx) error) error { + m.clientMut.Lock() + defer m.clientMut.Unlock() + var tx *bolt.Tx var err error if m.create { - m.clientMut.Lock() if m.tx == nil { m.tx, err = m.db.Begin(true) if err != nil { - m.clientMut.Unlock() return err } } - m.clientMut.Unlock() - tx = m.tx } else { tx, err = m.db.Begin(writable) @@ -150,7 +152,6 @@ func (m *mapping) transaction(writable bool, f func(*bolt.Tx) error) error { } err = f(tx) - if m.create { return err } @@ -217,7 +218,11 @@ func (m *mapping) getMaxRowID(fieldName string) (uint64, error) { return id, err } -func (m *mapping) putLocation(indexName string, colID uint64, location []byte) error { +func (m *mapping) putLocation( + indexName string, + colID uint64, + location []byte, +) error { return m.transaction(true, func(tx *bolt.Tx) error { b, err := tx.CreateBucketIfNotExists([]byte(indexName)) if err != nil { @@ -231,14 +236,19 @@ func (m *mapping) putLocation(indexName string, colID uint64, location []byte) e }) } -func (m *mapping) sortedLocations(indexName string, cols []uint64, reverse bool) ([][]byte, error) { +func (m *mapping) sortedLocations( + indexName string, + cols []uint64, + reverse bool, +) ([][]byte, error) { var result [][]byte m.mut.RLock() defer m.mut.RUnlock() err := m.db.View(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte(indexName)) + bucket := []byte(indexName) + b := tx.Bucket(bucket) if b == nil { - return fmt.Errorf("bucket %s not found", indexName) + return fmt.Errorf("bucket %s not found", bucket) } for _, col := range cols { @@ -274,13 +284,17 @@ func (b byBytes) Len() int { return len(b) } func (b byBytes) Swap(i, j int) { b[i], b[j] = b[j], b[i] } func (b byBytes) Less(i, j int) bool { return bytes.Compare(b[i], b[j]) < 0 } -func (m *mapping) getLocation(indexName string, colID uint64) ([]byte, error) { +func (m *mapping) getLocation( + indexName string, + colID uint64, +) ([]byte, error) { var location []byte err := m.transaction(true, func(tx *bolt.Tx) error { - b := tx.Bucket([]byte(indexName)) + bucket := []byte(indexName) + b := tx.Bucket(bucket) if b == nil { - return fmt.Errorf("bucket %s not found", indexName) + return fmt.Errorf("bucket %s not found", bucket) } key := make([]byte, 8) @@ -313,10 +327,11 @@ func (m *mapping) getBucket( return nil, err } - bucket = tx.Bucket([]byte(indexName)) + bu := []byte(indexName) + bucket = tx.Bucket(bu) if bucket == nil { _ = tx.Rollback() - return nil, fmt.Errorf("bucket %s not found", indexName) + return nil, fmt.Errorf("bucket %s not found", bu) } return bucket, err @@ -372,3 +387,7 @@ func (m *mapping) filter(name string, fn func([]byte) (bool, error)) ([]uint64, return result, err } + +func mappingKey(p sql.Partition) string { + return fmt.Sprintf("%x", p.Key()) +} diff --git a/sql/index/pilosa/mapping_test.go b/sql/index/pilosa/mapping_test.go index e1ef3ee50..e2a37c035 100644 --- a/sql/index/pilosa/mapping_test.go +++ b/sql/index/pilosa/mapping_test.go @@ -1,3 +1,5 @@ +// +build !windows + package pilosa import ( @@ -81,3 +83,9 @@ func TestGet(t *testing.T) { require.Equal(expected[i], val) } } + +type mockPartition string + +func (m mockPartition) Key() []byte { + return []byte(m) +} diff --git a/sql/index_test.go b/sql/index_test.go index da42c2464..1e631cba5 100644 --- a/sql/index_test.go +++ b/sql/index_test.go @@ -401,8 +401,8 @@ var _ Expression = (*dummyExpr)(nil) func (dummyExpr) Children() []Expression { return nil } func (dummyExpr) Eval(*Context, Row) (interface{}, error) { panic("not implemented") } -func (e dummyExpr) TransformUp(fn TransformExprFunc) (Expression, error) { - return fn(e) +func (e dummyExpr) WithChildren(children ...Expression) (Expression, error) { + return e, nil } func (e dummyExpr) String() string { return fmt.Sprintf("dummyExpr{%d, %s}", e.index, e.colName) } func (dummyExpr) IsNullable() bool { return false } diff --git a/sql/information_schema.go b/sql/information_schema.go index 5ec9aafb7..48147c7d5 100644 --- a/sql/information_schema.go +++ b/sql/information_schema.go @@ -1,9 +1,10 @@ -package sql // import "gopkg.in/src-d/go-mysql-server.v0/sql" +package sql import ( "bytes" "fmt" "io" + "strings" ) const ( @@ -214,27 +215,27 @@ func columnsRowIter(cat *Catalog) RowIter { collName = "utf8_bin" } rows = append(rows, Row{ - "def", // table_catalog - db.Name(), // table_schema - t.Name(), // table_name - c.Name, // column_name - uint64(i), // ordinal_position - c.Default, // column_default - nullable, // is_nullable - c.Type.String(), // data_type - nil, // character_maximum_length - nil, // character_octet_length - nil, // numeric_precision - nil, // numeric_scale - nil, // datetime_precision - charName, // character_set_name - collName, // collation_name - c.Type.String(), // column_type - "", // column_key - "", // extra - "select", // privileges - "", // column_comment - "", // generation_expression + "def", // table_catalog + db.Name(), // table_schema + t.Name(), // table_name + c.Name, // column_name + uint64(i), // ordinal_position + c.Default, // column_default + nullable, // is_nullable + strings.ToLower(MySQLTypeName(c.Type)), // data_type + nil, // character_maximum_length + nil, // character_octet_length + nil, // numeric_precision + nil, // numeric_scale + nil, // datetime_precision + charName, // character_set_name + collName, // collation_name + strings.ToLower(MySQLTypeName(c.Type)), // column_type + "", // column_key + "", // extra + "select", // privileges + "", // column_comment + "", // generation_expression }) } } diff --git a/sql/memory.go b/sql/memory.go new file mode 100644 index 000000000..35556ec99 --- /dev/null +++ b/sql/memory.go @@ -0,0 +1,194 @@ +package sql + +import ( + "os" + "runtime" + "strconv" + "sync" + + errors "gopkg.in/src-d/go-errors.v1" +) + +// Disposable objects can erase all their content when they're no longer in use. +// They should not be used again after they've been disposed. +type Disposable interface { + // Dispose the contents. + Dispose() +} + +// Freeable objects can free their memory. +type Freeable interface { + // Free the memory. + Free() +} + +// KeyValueCache is a cache of key value pairs. +type KeyValueCache interface { + // Put a new value in the cache. + Put(uint64, interface{}) error + // Get the value with the given key. + Get(uint64) (interface{}, error) +} + +// RowsCache is a cache of rows. +type RowsCache interface { + // Add a new row to the cache. If there is no memory available, it will try to + // free some memory. If after that there is still no memory available, it + // will return an error and erase all the content of the cache. + Add(Row) error + // Get all rows. + Get() []Row +} + +// ErrNoMemoryAvailable is returned when there is no more available memory. +var ErrNoMemoryAvailable = errors.NewKind("no memory available") + +const maxMemoryKey = "MAX_MEMORY" + +const ( + b = 1 + kib = 1024 * b + mib = 1024 * kib +) + +var maxMemory = func() uint64 { + val := os.Getenv(maxMemoryKey) + var v uint64 + if val != "" { + var err error + v, err = strconv.ParseUint(val, 10, 64) + if err != nil { + panic("MAX_MEMORY environment variable must be a number, but got: " + val) + } + } + + return v * uint64(mib) +}() + +// Reporter is a component that gives information about the memory usage. +type Reporter interface { + // MaxMemory returns the maximum number of memory allowed in bytes. + MaxMemory() uint64 + // UsedMemory returns the memory in use in bytes. + UsedMemory() uint64 +} + +// ProcessMemory is a reporter for the memory used by the process and the +// maximum amount of memory allowed controlled by the MAX_MEMORY environment +// variable. +var ProcessMemory Reporter = new(processReporter) + +type processReporter struct{} + +func (processReporter) UsedMemory() uint64 { + var s runtime.MemStats + runtime.ReadMemStats(&s) + return s.HeapInuse + s.StackInuse +} + +func (processReporter) MaxMemory() uint64 { return maxMemory } + +// HasAvailableMemory reports whether more memory is available to the program if +// it hasn't reached the max memory limit. +func HasAvailableMemory(r Reporter) bool { + maxMemory := r.MaxMemory() + if maxMemory == 0 { + return true + } + + return r.UsedMemory() < maxMemory +} + +// MemoryManager is in charge of keeping track and managing all the components that operate +// in memory. There should only be one instance of a memory manager running at the +// same time in each process. +type MemoryManager struct { + mu sync.RWMutex + reporter Reporter + caches map[uint64]Disposable + token uint64 +} + +// NewMemoryManager creates a new manager with the given memory reporter. If nil is given, +// then the Process reporter will be used by default. +func NewMemoryManager(r Reporter) *MemoryManager { + if r == nil { + r = ProcessMemory + } + + return &MemoryManager{ + reporter: r, + caches: make(map[uint64]Disposable), + } +} + +// HasAvailable reports whether the memory manager has any available memory. +func (m *MemoryManager) HasAvailable() bool { + return HasAvailableMemory(m.reporter) +} + +// DisposeFunc is a function to completely erase a cache and remove it from the manager. +type DisposeFunc func() + +// NewLRUCache returns an empty LRU cache and a function to dispose it when it's +// no longer needed. +func (m *MemoryManager) NewLRUCache(size uint) (KeyValueCache, DisposeFunc) { + c := newLRUCache(m, m.reporter, size) + pos := m.addCache(c) + return c, func() { + c.Dispose() + m.removeCache(pos) + } +} + +// NewHistoryCache returns an empty history cache and a function to dispose it when it's +// no longer needed. +func (m *MemoryManager) NewHistoryCache() (KeyValueCache, DisposeFunc) { + c := newHistoryCache(m, m.reporter) + pos := m.addCache(c) + return c, func() { + c.Dispose() + m.removeCache(pos) + } +} + +// NewRowsCache returns an empty rows cache and a function to dispose it when it's +// no longer needed. +func (m *MemoryManager) NewRowsCache() (RowsCache, DisposeFunc) { + c := newRowsCache(m, m.reporter) + pos := m.addCache(c) + return c, func() { + c.Dispose() + m.removeCache(pos) + } +} + +func (m *MemoryManager) addCache(c Disposable) (pos uint64) { + m.mu.Lock() + defer m.mu.Unlock() + m.token++ + m.caches[m.token] = c + return m.token +} + +func (m *MemoryManager) removeCache(pos uint64) { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.caches, pos) + + if len(m.caches) == 0 { + m.token = 0 + } +} + +// Free the memory of all freeable caches. +func (m *MemoryManager) Free() { + m.mu.RLock() + defer m.mu.RUnlock() + + for _, c := range m.caches { + if f, ok := c.(Freeable); ok { + f.Free() + } + } +} diff --git a/sql/memory_test.go b/sql/memory_test.go new file mode 100644 index 000000000..c4dcf7d57 --- /dev/null +++ b/sql/memory_test.go @@ -0,0 +1,79 @@ +package sql + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestManager(t *testing.T) { + require := require.New(t) + m := NewMemoryManager(nil) + + kv, dispose := m.NewLRUCache(5) + _, ok := kv.(*lruCache) + require.True(ok) + require.Len(m.caches, 1) + dispose() + require.Len(m.caches, 0) + + kv, dispose = m.NewHistoryCache() + _, ok = kv.(*historyCache) + require.True(ok) + require.Len(m.caches, 1) + dispose() + require.Len(m.caches, 0) + + rc, dispose := m.NewRowsCache() + _, ok = rc.(*rowsCache) + require.True(ok) + require.Len(m.caches, 1) + dispose() + require.Len(m.caches, 0) + + m.addCache(disposableCache{}) + f := new(freeableCache) + m.addCache(f) + m.Free() + require.True(f.freed) +} + +type disposableCache struct{} + +func (d disposableCache) Dispose() {} + +type freeableCache struct { + disposableCache + freed bool +} + +func (f *freeableCache) Free() { f.freed = true } + +func TestHasAvailable(t *testing.T) { + require.True(t, HasAvailableMemory(fixedReporter(2, 5))) + require.False(t, HasAvailableMemory(fixedReporter(6, 5))) +} + +type mockReporter struct { + f func() uint64 + max uint64 +} + +func (m mockReporter) UsedMemory() uint64 { return m.f() } +func (m mockReporter) MaxMemory() uint64 { return m.max } + +func fixedReporter(v, max uint64) mockReporter { + return mockReporter{func() uint64 { + return v + }, max} +} + +type mockMemory struct { + f func() +} + +func (m mockMemory) Free() { + if m.f != nil { + m.f() + } +} diff --git a/sql/parse/describe.go b/sql/parse/describe.go index d35cb0a69..e7993d49b 100644 --- a/sql/parse/describe.go +++ b/sql/parse/describe.go @@ -4,9 +4,9 @@ import ( "bufio" "strings" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/plan" errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" ) var ( diff --git a/sql/parse/describe_test.go b/sql/parse/describe_test.go index 5aedc68d6..c15b6af00 100644 --- a/sql/parse/describe_test.go +++ b/sql/parse/describe_test.go @@ -3,11 +3,11 @@ package parse import ( "testing" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" "github.com/stretchr/testify/require" errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" ) func TestParseDescribeQuery(t *testing.T) { diff --git a/sql/parse/indexes.go b/sql/parse/indexes.go index 70b6b2d3e..a435f7888 100644 --- a/sql/parse/indexes.go +++ b/sql/parse/indexes.go @@ -7,9 +7,9 @@ import ( "strings" "unicode" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" + "github.com/src-d/go-mysql-server/sql/plan" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) func parseShowIndex(s string) (sql.Node, error) { @@ -39,7 +39,7 @@ func parseShowIndex(s string) (sql.Node, error) { ), nil } -func parseCreateIndex(s string) (sql.Node, error) { +func parseCreateIndex(ctx *sql.Context, s string) (sql.Node, error) { r := bufio.NewReader(strings.NewReader(s)) var name, table, driver string @@ -78,7 +78,7 @@ func parseCreateIndex(s string) (sql.Node, error) { var indexExprs = make([]sql.Expression, len(exprs)) for i, e := range exprs { var err error - indexExprs[i], err = parseExpr(e) + indexExprs[i], err = parseExpr(ctx, e) if err != nil { return nil, err } diff --git a/sql/parse/indexes_test.go b/sql/parse/indexes_test.go index 5929fba65..a9d7df5a4 100644 --- a/sql/parse/indexes_test.go +++ b/sql/parse/indexes_test.go @@ -5,12 +5,12 @@ import ( "strings" "testing" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/plan" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestParseCreateIndex(t *testing.T) { @@ -159,7 +159,7 @@ func TestParseCreateIndex(t *testing.T) { t.Run(tt.query, func(t *testing.T) { require := require.New(t) - result, err := parseCreateIndex(strings.ToLower(tt.query)) + result, err := parseCreateIndex(sql.NewEmptyContext(), strings.ToLower(tt.query)) if tt.err != nil { require.Error(err) require.True(tt.err.Is(err)) diff --git a/sql/parse/lock.go b/sql/parse/lock.go index 6560253ad..b4d6d67b6 100644 --- a/sql/parse/lock.go +++ b/sql/parse/lock.go @@ -5,8 +5,8 @@ import ( "io" "strings" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/plan" ) func parseLockTables(ctx *sql.Context, query string) (sql.Node, error) { @@ -39,11 +39,12 @@ func readTableLocks(tables *[]*plan.TableLock) parseFunc { *tables = append(*tables, t) - if err := skipSpaces(rd); err != nil { + if err = skipSpaces(rd); err != nil { return err } - b, err := rd.Peek(1) + var b []byte + b, err = rd.Peek(1) if err == io.EOF { return nil } else if err != nil { diff --git a/sql/parse/parse.go b/sql/parse/parse.go index f3a80275c..f4dfe6f0a 100644 --- a/sql/parse/parse.go +++ b/sql/parse/parse.go @@ -1,4 +1,4 @@ -package parse // import "gopkg.in/src-d/go-mysql-server.v0/sql/parse" +package parse import ( "bufio" @@ -9,13 +9,14 @@ import ( "strconv" "strings" - opentracing "github.com/opentracing/opentracing-go" + "github.com/opentracing/opentracing-go" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/expression/function" + "github.com/src-d/go-mysql-server/sql/expression/function/aggregation" + "github.com/src-d/go-mysql-server/sql/plan" "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression/function" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" - "gopkg.in/src-d/go-vitess.v1/vt/sqlparser" + "vitess.io/vitess/go/vt/sqlparser" ) var ( @@ -25,14 +26,11 @@ var ( // ErrUnsupportedFeature is thrown when a feature is not already supported ErrUnsupportedFeature = errors.NewKind("unsupported feature: %s") - // ErrUnsupportedSubqueryExpression is thrown because subqueries are not supported, yet. - ErrUnsupportedSubqueryExpression = errors.NewKind("unsupported subquery expression") - // ErrInvalidSQLValType is returned when a SQLVal type is not valid. ErrInvalidSQLValType = errors.NewKind("invalid SQLVal of type: %d") // ErrInvalidSortOrder is returned when a sort order is not valid. - ErrInvalidSortOrder = errors.NewKind("invalod sort order: %s") + ErrInvalidSortOrder = errors.NewKind("invalid sort order: %s") ) var ( @@ -49,6 +47,17 @@ var ( unlockTablesRegex = regexp.MustCompile(`^unlock\s+tables$`) lockTablesRegex = regexp.MustCompile(`^lock\s+tables\s`) setRegex = regexp.MustCompile(`^set\s+`) + createViewRegex = regexp.MustCompile(`^create\s+view\s+`) +) + +// These constants aren't exported from vitess for some reason. This could be removed if we changed this. +const ( + colKeyNone sqlparser.ColumnKeyOption = iota + colKeyPrimary + colKeySpatialKey + colKeyUnique + colKeyUniqueKey + colKey ) // Parse parses the given SQL sentence and returns the corresponding node. @@ -72,7 +81,7 @@ func Parse(ctx *sql.Context, query string) (sql.Node, error) { case describeTablesRegex.MatchString(lowerQuery): return parseDescribeTables(lowerQuery) case createIndexRegex.MatchString(lowerQuery): - return parseCreateIndex(s) + return parseCreateIndex(ctx, s) case dropIndexRegex.MatchString(lowerQuery): return parseDropIndex(s) case showIndexRegex.MatchString(lowerQuery): @@ -84,7 +93,7 @@ func Parse(ctx *sql.Context, query string) (sql.Node, error) { case showWarningsRegex.MatchString(lowerQuery): return parseShowWarnings(ctx, s) case showCollationRegex.MatchString(lowerQuery): - return parseShowCollation(s) + return parseShowCollation(ctx, s) case describeRegex.MatchString(lowerQuery): return parseDescribeQuery(ctx, s) case fullProcessListRegex.MatchString(lowerQuery): @@ -95,6 +104,9 @@ func Parse(ctx *sql.Context, query string) (sql.Node, error) { return parseLockTables(ctx, s) case setRegex.MatchString(lowerQuery): s = fixSetQuery(s) + case createViewRegex.MatchString(lowerQuery): + // CREATE VIEW parses as a CREATE DDL statement with an empty table spec + return nil, ErrUnsupportedFeature.New("CREATE VIEW") } stmt, err := sqlparser.Parse(s) @@ -134,19 +146,34 @@ func convert(ctx *sql.Context, stmt sqlparser.Statement, query string) (sql.Node default: return nil, ErrUnsupportedSyntax.New(n) case *sqlparser.Show: - return convertShow(n, query) + // When a query is empty it means it comes from a subquery, as we don't + // have the query itself in a subquery. Hence, a SHOW could not be + // parsed. + if query == "" { + return nil, ErrUnsupportedFeature.New("SHOW in subquery") + } + return convertShow(ctx, n, query) case *sqlparser.Select: return convertSelect(ctx, n) case *sqlparser.Insert: return convertInsert(ctx, n) case *sqlparser.DDL: - return convertDDL(n) + // unlike other statements, DDL statements have loose parsing by default + ddl, err := sqlparser.ParseStrictDDL(query) + if err != nil { + return nil, err + } + return convertDDL(ddl.(*sqlparser.DDL)) case *sqlparser.Set: return convertSet(ctx, n) case *sqlparser.Use: return convertUse(n) case *sqlparser.Rollback: return plan.NewRollback(), nil + case *sqlparser.Delete: + return convertDelete(ctx, n) + case *sqlparser.Update: + return convertUpdate(ctx, n) } } @@ -162,13 +189,13 @@ func convertSet(ctx *sql.Context, n *sqlparser.Set) (sql.Node, error) { var variables = make([]plan.SetVariable, len(n.Exprs)) for i, e := range n.Exprs { - expr, err := exprToExpression(e.Expr) + expr, err := exprToExpression(ctx, e.Expr) if err != nil { return nil, err } name := strings.TrimSpace(e.Name.Lowered()) - if expr, err = expr.TransformUp(func(e sql.Expression) (sql.Expression, error) { + if expr, err = expression.TransformUp(expr, func(e sql.Expression) (sql.Expression, error) { if _, ok := e.(*expression.DefaultColumn); ok { return e, nil } @@ -212,7 +239,7 @@ func convertSet(ctx *sql.Context, n *sqlparser.Set) (sql.Node, error) { return plan.NewSet(variables...), nil } -func convertShow(s *sqlparser.Show, query string) (sql.Node, error) { +func convertShow(ctx *sql.Context, s *sqlparser.Show, query string) (sql.Node, error) { switch s.Type { case sqlparser.KeywordString(sqlparser.TABLES): var dbName string @@ -225,7 +252,7 @@ func convertShow(s *sqlparser.Show, query string) (sql.Node, error) { if s.ShowTablesOpt.Filter != nil { if s.ShowTablesOpt.Filter.Filter != nil { var err error - filter, err = exprToExpression(s.ShowTablesOpt.Filter.Filter) + filter, err = exprToExpression(ctx, s.ShowTablesOpt.Filter.Filter) if err != nil { return nil, err } @@ -244,7 +271,7 @@ func convertShow(s *sqlparser.Show, query string) (sql.Node, error) { } return node, nil - case sqlparser.KeywordString(sqlparser.DATABASES): + case sqlparser.KeywordString(sqlparser.DATABASES), sqlparser.KeywordString(sqlparser.SCHEMAS): return plan.NewShowDatabases(), nil case sqlparser.KeywordString(sqlparser.FIELDS), sqlparser.KeywordString(sqlparser.COLUMNS): // TODO(erizocosmico): vitess parser does not support EXTENDED. @@ -267,7 +294,7 @@ func convertShow(s *sqlparser.Show, query string) (sql.Node, error) { } if s.ShowTablesOpt.Filter.Filter != nil { - filter, err := exprToExpression(s.ShowTablesOpt.Filter.Filter) + filter, err := exprToExpression(ctx, s.ShowTablesOpt.Filter.Filter) if err != nil { return nil, err } @@ -278,7 +305,7 @@ func convertShow(s *sqlparser.Show, query string) (sql.Node, error) { return node, nil case sqlparser.KeywordString(sqlparser.TABLE): - return parseShowTableStatus(query) + return parseShowTableStatus(ctx, query) default: unsupportedShow := fmt.Sprintf("SHOW %s", s.Type) return nil, ErrUnsupportedFeature.New(unsupportedShow) @@ -291,48 +318,52 @@ func convertSelect(ctx *sql.Context, s *sqlparser.Select) (sql.Node, error) { return nil, err } - if s.Having != nil { - return nil, ErrUnsupportedFeature.New("HAVING") - } - if s.Where != nil { - node, err = whereToFilter(s.Where, node) + node, err = whereToFilter(ctx, s.Where, node) if err != nil { return nil, err } } - node, err = selectToProjectOrGroupBy(s.SelectExprs, s.GroupBy, node) + node, err = selectToProjectOrGroupBy(ctx, s.SelectExprs, s.GroupBy, node) if err != nil { return nil, err } + if s.Having != nil { + node, err = havingToHaving(ctx, s.Having, node) + if err != nil { + return nil, err + } + } + if s.Distinct != "" { node = plan.NewDistinct(node) } if len(s.OrderBy) != 0 { - node, err = orderByToSort(s.OrderBy, node) + node, err = orderByToSort(ctx, s.OrderBy, node) if err != nil { return nil, err } } - if s.Limit != nil { - node, err = limitToLimit(ctx, s.Limit.Rowcount, node) + // Limit must wrap offset, and not vice-versa, so that skipped rows don't count toward the returned row count. + if s.Limit != nil && s.Limit.Offset != nil { + node, err = offsetToOffset(ctx, s.Limit.Offset, node) if err != nil { return nil, err } - } else if ok, val := sql.HasDefaultValue(ctx.Session, "sql_select_limit"); !ok { - limit := val.(int64) - node = plan.NewLimit(int64(limit), node) } - if s.Limit != nil && s.Limit.Offset != nil { - node, err = offsetToOffset(ctx, s.Limit.Offset, node) + if s.Limit != nil { + node, err = limitToLimit(ctx, s.Limit.Rowcount, node) if err != nil { return nil, err } + } else if ok, val := sql.HasDefaultValue(ctx.Session, "sql_select_limit"); !ok { + limit := val.(int64) + node = plan.NewLimit(int64(limit), node) } return node, nil @@ -342,13 +373,23 @@ func convertDDL(c *sqlparser.DDL) (sql.Node, error) { switch c.Action { case sqlparser.CreateStr: return convertCreateTable(c) + case sqlparser.DropStr: + return convertDropTable(c) default: return nil, ErrUnsupportedSyntax.New(c) } } +func convertDropTable(c *sqlparser.DDL) (sql.Node, error) { + tableNames := make([]string, len(c.FromTables)) + for i, t := range c.FromTables { + tableNames[i] = t.Name.String() + } + return plan.NewDropTable(sql.UnresolvedDatabase(""), c.IfExists, tableNames...), nil +} + func convertCreateTable(c *sqlparser.DDL) (sql.Node, error) { - schema, err := columnDefinitionToSchema(c.TableSpec.Columns) + schema, err := tableSpecToSchema(c.TableSpec) if err != nil { return nil, err } @@ -366,6 +407,8 @@ func convertInsert(ctx *sql.Context, i *sqlparser.Insert) (sql.Node, error) { return nil, ErrUnsupportedSyntax.New(i) } + isReplace := i.Action == sqlparser.ReplaceStr + src, err := insertRowsToNode(ctx, i.Rows) if err != nil { return nil, err @@ -374,31 +417,143 @@ func convertInsert(ctx *sql.Context, i *sqlparser.Insert) (sql.Node, error) { return plan.NewInsertInto( plan.NewUnresolvedTable(i.Table.Name.String(), i.Table.Qualifier.String()), src, + isReplace, columnsToStrings(i.Columns), ), nil } -func columnDefinitionToSchema(colDef []*sqlparser.ColumnDefinition) (sql.Schema, error) { +func convertDelete(ctx *sql.Context, d *sqlparser.Delete) (sql.Node, error) { + node, err := tableExprsToTable(ctx, d.TableExprs) + if err != nil { + return nil, err + } + + if d.Where != nil { + node, err = whereToFilter(ctx, d.Where, node) + if err != nil { + return nil, err + } + } + + if len(d.OrderBy) != 0 { + node, err = orderByToSort(ctx, d.OrderBy, node) + if err != nil { + return nil, err + } + } + + // Limit must wrap offset, and not vice-versa, so that skipped rows don't count toward the returned row count. + if d.Limit != nil && d.Limit.Offset != nil { + node, err = offsetToOffset(ctx, d.Limit.Offset, node) + if err != nil { + return nil, err + } + } + + if d.Limit != nil { + node, err = limitToLimit(ctx, d.Limit.Rowcount, node) + if err != nil { + return nil, err + } + } + + return plan.NewDeleteFrom(node), nil +} + +func convertUpdate(ctx *sql.Context, d *sqlparser.Update) (sql.Node, error) { + node, err := tableExprsToTable(ctx, d.TableExprs) + if err != nil { + return nil, err + } + + updateExprs, err := updateExprsToExpressions(ctx, d.Exprs) + if err != nil { + return nil, err + } + + if d.Where != nil { + node, err = whereToFilter(ctx, d.Where, node) + if err != nil { + return nil, err + } + } + + if len(d.OrderBy) != 0 { + node, err = orderByToSort(ctx, d.OrderBy, node) + if err != nil { + return nil, err + } + } + + // Limit must wrap offset, and not vice-versa, so that skipped rows don't count toward the returned row count. + if d.Limit != nil && d.Limit.Offset != nil { + node, err = offsetToOffset(ctx, d.Limit.Offset, node) + if err != nil { + return nil, err + } + + } + + if d.Limit != nil { + node, err = limitToLimit(ctx, d.Limit.Rowcount, node) + if err != nil { + return nil, err + } + } + + return plan.NewUpdate(node, updateExprs), nil +} + +func tableSpecToSchema(tableSpec *sqlparser.TableSpec) (sql.Schema, error) { var schema sql.Schema - for _, cd := range colDef { - typ := cd.Type - internalTyp, err := sql.MysqlTypeToType(typ.SQLType()) + for _, cd := range tableSpec.Columns { + column, err := getColumn(cd, tableSpec.Indexes) if err != nil { return nil, err } - schema = append(schema, &sql.Column{ - Nullable: !bool(typ.NotNull), - Type: internalTyp, - Name: cd.Name.String(), - // TODO - Default: nil, - }) + schema = append(schema, column) } return schema, nil } +// getColumn returns the sql.Column for the column definition given, as part of a create table statement. +func getColumn(cd *sqlparser.ColumnDefinition, indexes []*sqlparser.IndexDefinition) (*sql.Column, error) { + typ := cd.Type + internalTyp, err := sql.MysqlTypeToType(typ.SQLType()) + if err != nil { + return nil, err + } + + // Primary key info can either be specified in the column's type info (for in-line declarations), or in a slice of + // indexes attached to the table def. We have to check both places to find if a column is part of the primary key + isPkey := cd.Type.KeyOpt == colKeyPrimary + + if !isPkey { + OuterLoop: + for _, index := range indexes { + if index.Info.Primary { + for _, indexCol := range index.Columns { + if indexCol.Column.Equal(cd.Name) { + isPkey = true + break OuterLoop + } + } + } + } + } + + return &sql.Column{ + Nullable: !bool(typ.NotNull), + Type: internalTyp, + Name: cd.Name.String(), + PrimaryKey: isPkey, + // TODO + Default: nil, + }, nil +} + func columnsToStrings(cols sqlparser.Columns) []string { res := make([]string, len(cols)) for i, c := range cols { @@ -415,19 +570,19 @@ func insertRowsToNode(ctx *sql.Context, ir sqlparser.InsertRows) (sql.Node, erro case *sqlparser.Union: return nil, ErrUnsupportedFeature.New("UNION") case sqlparser.Values: - return valuesToValues(v) + return valuesToValues(ctx, v) default: return nil, ErrUnsupportedSyntax.New(ir) } } -func valuesToValues(v sqlparser.Values) (sql.Node, error) { +func valuesToValues(ctx *sql.Context, v sqlparser.Values) (sql.Node, error) { exprTuples := make([][]sql.Expression, len(v)) for i, vt := range v { exprs := make([]sql.Expression, len(vt)) exprTuples[i] = exprs for j, e := range vt { - expr, err := exprToExpression(e) + expr, err := exprToExpression(ctx, e) if err != nil { return nil, err } @@ -501,15 +656,10 @@ func tableExprToTable( return nil, ErrUnsupportedSyntax.New(te) } case *sqlparser.JoinTableExpr: - // TODO: add support for the rest of joins - if t.Join != sqlparser.JoinStr && t.Join != sqlparser.NaturalJoinStr { - return nil, ErrUnsupportedFeature.New(t.Join) - } - // TODO: add support for using, once we have proper table // qualification of fields if len(t.Condition.Using) > 0 { - return nil, ErrUnsupportedFeature.New("using clause on join") + return nil, ErrUnsupportedFeature.New("USING clause on join") } left, err := tableExprToTable(ctx, t.LeftExpr) @@ -530,16 +680,26 @@ func tableExprToTable( return nil, ErrUnsupportedSyntax.New("missed ON clause for JOIN statement") } - cond, err := exprToExpression(t.Condition.On) + cond, err := exprToExpression(ctx, t.Condition.On) if err != nil { return nil, err } - return plan.NewInnerJoin(left, right, cond), nil + + switch t.Join { + case sqlparser.JoinStr: + return plan.NewInnerJoin(left, right, cond), nil + case sqlparser.LeftJoinStr: + return plan.NewLeftJoin(left, right, cond), nil + case sqlparser.RightJoinStr: + return plan.NewRightJoin(left, right, cond), nil + default: + return nil, ErrUnsupportedFeature.New(t.Join) + } } } -func whereToFilter(w *sqlparser.Where, child sql.Node) (*plan.Filter, error) { - c, err := exprToExpression(w.Expr) +func whereToFilter(ctx *sql.Context, w *sqlparser.Where, child sql.Node) (*plan.Filter, error) { + c, err := exprToExpression(ctx, w.Expr) if err != nil { return nil, err } @@ -547,10 +707,10 @@ func whereToFilter(w *sqlparser.Where, child sql.Node) (*plan.Filter, error) { return plan.NewFilter(c, child), nil } -func orderByToSort(ob sqlparser.OrderBy, child sql.Node) (*plan.Sort, error) { +func orderByToSort(ctx *sql.Context, ob sqlparser.OrderBy, child sql.Node) (*plan.Sort, error) { var sortFields []plan.SortField for _, o := range ob { - e, err := exprToExpression(o.Expr) + e, err := exprToExpression(ctx, o.Expr) if err != nil { return nil, err } @@ -577,21 +737,25 @@ func limitToLimit( limit sqlparser.Expr, child sql.Node, ) (*plan.Limit, error) { - e, err := exprToExpression(limit) + rowCount, err := getInt64Value(ctx, limit, "LIMIT with non-integer literal") if err != nil { return nil, err } - nl, ok := e.(*expression.Literal) - if !ok || nl.Type() != sql.Int64 { - return nil, ErrUnsupportedFeature.New("LIMIT with non-integer literal") + if rowCount < 0 { + return nil, ErrUnsupportedSyntax.New("LIMIT must be >= 0") } - n, err := nl.Eval(ctx, nil) + return plan.NewLimit(rowCount, child), nil +} + +func havingToHaving(ctx *sql.Context, having *sqlparser.Where, node sql.Node) (sql.Node, error) { + cond, err := exprToExpression(ctx, having.Expr) if err != nil { return nil, err } - return plan.NewLimit(n.(int64), child), nil + + return plan.NewHaving(cond, node), nil } func offsetToOffset( @@ -599,29 +763,64 @@ func offsetToOffset( offset sqlparser.Expr, child sql.Node, ) (*plan.Offset, error) { - e, err := exprToExpression(offset) + o, err := getInt64Value(ctx, offset, "OFFSET with non-integer literal") if err != nil { return nil, err } - nl, ok := e.(*expression.Literal) - if !ok || nl.Type() != sql.Int64 { - return nil, ErrUnsupportedFeature.New("OFFSET with non-integer literal") + if o < 0 { + return nil, ErrUnsupportedSyntax.New("OFFSET must be >= 0") } - n, err := nl.Eval(ctx, nil) + return plan.NewOffset(o, child), nil +} + +// getInt64Literal returns an int64 *expression.Literal for the value given, or an unsupported error with the string +// given if the expression doesn't represent an integer literal. +func getInt64Literal(ctx *sql.Context, expr sqlparser.Expr, errStr string) (*expression.Literal, error) { + e, err := exprToExpression(ctx, expr) if err != nil { return nil, err } - return plan.NewOffset(n.(int64), child), nil + + nl, ok := e.(*expression.Literal) + if !ok || !sql.IsInteger(nl.Type()) { + return nil, ErrUnsupportedFeature.New(errStr) + } else { + i64, err := sql.Int64.Convert(nl.Value()) + if err != nil { + return nil, ErrUnsupportedFeature.New(errStr) + } + return expression.NewLiteral(i64, sql.Int64), nil + } + + return nl, nil +} + +// getInt64Value returns the int64 literal value in the expression given, or an error with the errStr given if it +// cannot. +func getInt64Value(ctx *sql.Context, expr sqlparser.Expr, errStr string) (int64, error) { + ie, err := getInt64Literal(ctx, expr, errStr) + if err != nil { + return 0, err + } + + i, err := ie.Eval(ctx, nil) + if err != nil { + return 0, err + } + + return i.(int64), nil } func isAggregate(e sql.Expression) bool { var isAgg bool expression.Inspect(e, func(e sql.Expression) bool { - fn, ok := e.(*expression.UnresolvedFunction) - if ok { - isAgg = isAgg || fn.IsAggregate + switch e := e.(type) { + case *expression.UnresolvedFunction: + isAgg = isAgg || e.IsAggregate + case *aggregation.CountDistinct: + isAgg = true } return true @@ -629,8 +828,13 @@ func isAggregate(e sql.Expression) bool { return isAgg } -func selectToProjectOrGroupBy(se sqlparser.SelectExprs, g sqlparser.GroupBy, child sql.Node) (sql.Node, error) { - selectExprs, err := selectExprsToExpressions(se) +func selectToProjectOrGroupBy( + ctx *sql.Context, + se sqlparser.SelectExprs, + g sqlparser.GroupBy, + child sql.Node, +) (sql.Node, error) { + selectExprs, err := selectExprsToExpressions(ctx, se) if err != nil { return nil, err } @@ -646,7 +850,7 @@ func selectToProjectOrGroupBy(se sqlparser.SelectExprs, g sqlparser.GroupBy, chi } if isAgg { - groupingExprs, err := groupByToExpressions(g) + groupingExprs, err := groupByToExpressions(ctx, g) if err != nil { return nil, err } @@ -655,12 +859,14 @@ func selectToProjectOrGroupBy(se sqlparser.SelectExprs, g sqlparser.GroupBy, chi for i, ge := range groupingExprs { // if GROUP BY index if l, ok := ge.(*expression.Literal); ok && sql.IsNumber(l.Type()) { - if idx, ok := l.Value().(int64); ok && idx > 0 && idx <= agglen { - aggexpr := selectExprs[idx-1] - if alias, ok := aggexpr.(*expression.Alias); ok { - aggexpr = expression.NewUnresolvedColumn(alias.Name()) + if i64, err := sql.Int64.Convert(l.Value()); err == nil { + if idx, ok := i64.(int64); ok && idx > 0 && idx <= agglen { + aggexpr := selectExprs[idx-1] + if alias, ok := aggexpr.(*expression.Alias); ok { + aggexpr = expression.NewUnresolvedColumn(alias.Name()) + } + groupingExprs[i] = aggexpr } - groupingExprs[i] = aggexpr } } } @@ -671,10 +877,10 @@ func selectToProjectOrGroupBy(se sqlparser.SelectExprs, g sqlparser.GroupBy, chi return plan.NewProject(selectExprs, child), nil } -func selectExprsToExpressions(se sqlparser.SelectExprs) ([]sql.Expression, error) { +func selectExprsToExpressions(ctx *sql.Context, se sqlparser.SelectExprs) ([]sql.Expression, error) { var exprs []sql.Expression for _, e := range se { - pe, err := selectExprToExpression(e) + pe, err := selectExprToExpression(ctx, e) if err != nil { return nil, err } @@ -685,7 +891,7 @@ func selectExprsToExpressions(se sqlparser.SelectExprs) ([]sql.Expression, error return exprs, nil } -func exprToExpression(e sqlparser.Expr) (sql.Expression, error) { +func exprToExpression(ctx *sql.Context, e sqlparser.Expr) (sql.Expression, error) { switch v := e.(type) { default: return nil, ErrUnsupportedSyntax.New(e) @@ -697,14 +903,14 @@ func exprToExpression(e sqlparser.Expr) (sql.Expression, error) { err error ) if v.Name != nil { - name, err = exprToExpression(v.Name) + name, err = exprToExpression(ctx, v.Name) } else { - name, err = exprToExpression(v.StrVal) + name, err = exprToExpression(ctx, v.StrVal) } if err != nil { return nil, err } - from, err := exprToExpression(v.From) + from, err := exprToExpression(ctx, v.From) if err != nil { return nil, err } @@ -712,17 +918,17 @@ func exprToExpression(e sqlparser.Expr) (sql.Expression, error) { if v.To == nil { return function.NewSubstring(name, from) } - to, err := exprToExpression(v.To) + to, err := exprToExpression(ctx, v.To) if err != nil { return nil, err } return function.NewSubstring(name, from, to) case *sqlparser.ComparisonExpr: - return comparisonExprToExpression(v) + return comparisonExprToExpression(ctx, v) case *sqlparser.IsExpr: - return isExprToExpression(v) + return isExprToExpression(ctx, v) case *sqlparser.NotExpr: - c, err := exprToExpression(v.Expr) + c, err := exprToExpression(ctx, v.Expr) if err != nil { return nil, err } @@ -743,58 +949,70 @@ func exprToExpression(e sqlparser.Expr) (sql.Expression, error) { } return expression.NewUnresolvedColumn(v.Name.String()), nil case *sqlparser.FuncExpr: - exprs, err := selectExprsToExpressions(v.Exprs) + exprs, err := selectExprsToExpressions(ctx, v.Exprs) if err != nil { return nil, err } + if v.Distinct { + if v.Name.Lowered() != "count" { + return nil, ErrUnsupportedSyntax.New("DISTINCT on non-COUNT aggregations") + } + + if len(exprs) != 1 { + return nil, ErrUnsupportedSyntax.New("more than one expression in COUNT") + } + + return aggregation.NewCountDistinct(exprs[0]), nil + } + return expression.NewUnresolvedFunction(v.Name.Lowered(), - v.IsAggregate(), exprs...), nil + isAggregateFunc(v), exprs...), nil case *sqlparser.ParenExpr: - return exprToExpression(v.Expr) + return exprToExpression(ctx, v.Expr) case *sqlparser.AndExpr: - lhs, err := exprToExpression(v.Left) + lhs, err := exprToExpression(ctx, v.Left) if err != nil { return nil, err } - rhs, err := exprToExpression(v.Right) + rhs, err := exprToExpression(ctx, v.Right) if err != nil { return nil, err } return expression.NewAnd(lhs, rhs), nil case *sqlparser.OrExpr: - lhs, err := exprToExpression(v.Left) + lhs, err := exprToExpression(ctx, v.Left) if err != nil { return nil, err } - rhs, err := exprToExpression(v.Right) + rhs, err := exprToExpression(ctx, v.Right) if err != nil { return nil, err } return expression.NewOr(lhs, rhs), nil case *sqlparser.ConvertExpr: - expr, err := exprToExpression(v.Expr) + expr, err := exprToExpression(ctx, v.Expr) if err != nil { return nil, err } return expression.NewConvert(expr, v.Type.Type), nil case *sqlparser.RangeCond: - val, err := exprToExpression(v.Left) + val, err := exprToExpression(ctx, v.Left) if err != nil { return nil, err } - lower, err := exprToExpression(v.From) + lower, err := exprToExpression(ctx, v.From) if err != nil { return nil, err } - upper, err := exprToExpression(v.To) + upper, err := exprToExpression(ctx, v.To) if err != nil { return nil, err } @@ -810,7 +1028,7 @@ func exprToExpression(e sqlparser.Expr) (sql.Expression, error) { case sqlparser.ValTuple: var exprs = make([]sql.Expression, len(v)) for i, e := range v { - expr, err := exprToExpression(e) + expr, err := exprToExpression(ctx, e) if err != nil { return nil, err } @@ -819,14 +1037,63 @@ func exprToExpression(e sqlparser.Expr) (sql.Expression, error) { return expression.NewTuple(exprs...), nil case *sqlparser.BinaryExpr: - return binaryExprToExpression(v) + return binaryExprToExpression(ctx, v) case *sqlparser.UnaryExpr: - return unaryExprToExpression(v) + return unaryExprToExpression(ctx, v) case *sqlparser.Subquery: - return nil, ErrUnsupportedSubqueryExpression.New() + node, err := convert(ctx, v.Select, "") + if err != nil { + return nil, err + } + return expression.NewSubquery(node), nil case *sqlparser.CaseExpr: - return caseExprToExpression(v) + return caseExprToExpression(ctx, v) + case *sqlparser.IntervalExpr: + return intervalExprToExpression(ctx, v) + } +} + +func isAggregateFunc(v *sqlparser.FuncExpr) bool { + switch v.Name.Lowered() { + case "first", "last": + return true + } + + return v.IsAggregate() +} + +// Convert an integer, represented by the specified string in the specified +// base, to its smallest representation possible, out of: +// int8, uint8, int16, uint16, int32, uint32, int64 and uint64 +func convertInt(value string, base int) (sql.Expression, error) { + if i8, err := strconv.ParseInt(value, base, 8); err == nil { + return expression.NewLiteral(int8(i8), sql.Int8), nil } + if ui8, err := strconv.ParseUint(value, base, 8); err == nil { + return expression.NewLiteral(uint8(ui8), sql.Uint8), nil + } + if i16, err := strconv.ParseInt(value, base, 16); err == nil { + return expression.NewLiteral(int16(i16), sql.Int16), nil + } + if ui16, err := strconv.ParseUint(value, base, 16); err == nil { + return expression.NewLiteral(uint16(ui16), sql.Uint16), nil + } + if i32, err := strconv.ParseInt(value, base, 32); err == nil { + return expression.NewLiteral(int32(i32), sql.Int32), nil + } + if ui32, err := strconv.ParseUint(value, base, 32); err == nil { + return expression.NewLiteral(uint32(ui32), sql.Uint32), nil + } + if i64, err := strconv.ParseInt(value, base, 64); err == nil { + return expression.NewLiteral(int64(i64), sql.Int64), nil + } + + ui64, err := strconv.ParseUint(value, base, 64) + if err != nil { + return nil, err + } + + return expression.NewLiteral(uint64(ui64), sql.Uint64), nil } func convertVal(v *sqlparser.SQLVal) (sql.Expression, error) { @@ -834,12 +1101,7 @@ func convertVal(v *sqlparser.SQLVal) (sql.Expression, error) { case sqlparser.StrVal: return expression.NewLiteral(string(v.Val), sql.Text), nil case sqlparser.IntVal: - //TODO: Use smallest integer representation and widen later. - val, err := strconv.ParseInt(string(v.Val), 10, 64) - if err != nil { - return nil, err - } - return expression.NewLiteral(val, sql.Int64), nil + return convertInt(string(v.Val), 10) case sqlparser.FloatVal: val, err := strconv.ParseFloat(string(v.Val), 64) if err != nil { @@ -854,11 +1116,7 @@ func convertVal(v *sqlparser.SQLVal) (sql.Expression, error) { v = strings.Trim(v[1:], "'") } - val, err := strconv.ParseInt(v, 16, 64) - if err != nil { - return nil, err - } - return expression.NewLiteral(val, sql.Int64), nil + return convertInt(v, 16) case sqlparser.HexVal: val, err := v.HexDecode() if err != nil { @@ -874,8 +1132,8 @@ func convertVal(v *sqlparser.SQLVal) (sql.Expression, error) { return nil, ErrInvalidSQLValType.New(v.Type) } -func isExprToExpression(c *sqlparser.IsExpr) (sql.Expression, error) { - e, err := exprToExpression(c.Expr) +func isExprToExpression(ctx *sql.Context, c *sqlparser.IsExpr) (sql.Expression, error) { + e, err := exprToExpression(ctx, c.Expr) if err != nil { return nil, err } @@ -885,18 +1143,26 @@ func isExprToExpression(c *sqlparser.IsExpr) (sql.Expression, error) { return expression.NewIsNull(e), nil case sqlparser.IsNotNullStr: return expression.NewNot(expression.NewIsNull(e)), nil + case sqlparser.IsTrueStr: + return expression.NewIsTrue(e), nil + case sqlparser.IsFalseStr: + return expression.NewIsFalse(e), nil + case sqlparser.IsNotTrueStr: + return expression.NewNot(expression.NewIsTrue(e)), nil + case sqlparser.IsNotFalseStr: + return expression.NewNot(expression.NewIsFalse(e)), nil default: return nil, ErrUnsupportedSyntax.New(c) } } -func comparisonExprToExpression(c *sqlparser.ComparisonExpr) (sql.Expression, error) { - left, err := exprToExpression(c.Left) +func comparisonExprToExpression(ctx *sql.Context, c *sqlparser.ComparisonExpr) (sql.Expression, error) { + left, err := exprToExpression(ctx, c.Left) if err != nil { return nil, err } - right, err := exprToExpression(c.Right) + right, err := exprToExpression(ctx, c.Right) if err != nil { return nil, err } @@ -933,10 +1199,10 @@ func comparisonExprToExpression(c *sqlparser.ComparisonExpr) (sql.Expression, er } } -func groupByToExpressions(g sqlparser.GroupBy) ([]sql.Expression, error) { +func groupByToExpressions(ctx *sql.Context, g sqlparser.GroupBy) ([]sql.Expression, error) { es := make([]sql.Expression, len(g)) for i, ve := range g { - e, err := exprToExpression(ve) + e, err := exprToExpression(ctx, ve) if err != nil { return nil, err } @@ -947,7 +1213,7 @@ func groupByToExpressions(g sqlparser.GroupBy) ([]sql.Expression, error) { return es, nil } -func selectExprToExpression(se sqlparser.SelectExpr) (sql.Expression, error) { +func selectExprToExpression(ctx *sql.Context, se sqlparser.SelectExpr) (sql.Expression, error) { switch e := se.(type) { default: return nil, ErrUnsupportedSyntax.New(e) @@ -957,7 +1223,7 @@ func selectExprToExpression(se sqlparser.SelectExpr) (sql.Expression, error) { } return expression.NewQualifiedStar(e.TableName.Name.String()), nil case *sqlparser.AliasedExpr: - expr, err := exprToExpression(e.Expr) + expr, err := exprToExpression(ctx, e.Expr) if err != nil { return nil, err } @@ -971,10 +1237,10 @@ func selectExprToExpression(se sqlparser.SelectExpr) (sql.Expression, error) { } } -func unaryExprToExpression(e *sqlparser.UnaryExpr) (sql.Expression, error) { +func unaryExprToExpression(ctx *sql.Context, e *sqlparser.UnaryExpr) (sql.Expression, error) { switch e.Operator { case sqlparser.MinusStr: - expr, err := exprToExpression(e.Expr) + expr, err := exprToExpression(ctx, e.Expr) if err != nil { return nil, err } @@ -986,7 +1252,7 @@ func unaryExprToExpression(e *sqlparser.UnaryExpr) (sql.Expression, error) { } } -func binaryExprToExpression(be *sqlparser.BinaryExpr) (sql.Expression, error) { +func binaryExprToExpression(ctx *sql.Context, be *sqlparser.BinaryExpr) (sql.Expression, error) { switch be.Operator { case sqlparser.PlusStr, @@ -1001,16 +1267,26 @@ func binaryExprToExpression(be *sqlparser.BinaryExpr) (sql.Expression, error) { sqlparser.IntDivStr, sqlparser.ModStr: - l, err := exprToExpression(be.Left) + l, err := exprToExpression(ctx, be.Left) if err != nil { return nil, err } - r, err := exprToExpression(be.Right) + r, err := exprToExpression(ctx, be.Right) if err != nil { return nil, err } + _, lok := l.(*expression.Interval) + _, rok := r.(*expression.Interval) + if lok && be.Operator == "-" { + return nil, ErrUnsupportedSyntax.New("subtracting from an interval") + } else if (lok || rok) && be.Operator != "+" && be.Operator != "-" { + return nil, ErrUnsupportedSyntax.New("only + and - can be used to add of subtract intervals from dates") + } else if lok && rok { + return nil, ErrUnsupportedSyntax.New("intervals cannot be added or subtracted from other intervals") + } + return expression.NewArithmetic(l, r, be.Operator), nil default: @@ -1018,12 +1294,12 @@ func binaryExprToExpression(be *sqlparser.BinaryExpr) (sql.Expression, error) { } } -func caseExprToExpression(e *sqlparser.CaseExpr) (sql.Expression, error) { +func caseExprToExpression(ctx *sql.Context, e *sqlparser.CaseExpr) (sql.Expression, error) { var expr sql.Expression var err error if e.Expr != nil { - expr, err = exprToExpression(e.Expr) + expr, err = exprToExpression(ctx, e.Expr) if err != nil { return nil, err } @@ -1031,12 +1307,14 @@ func caseExprToExpression(e *sqlparser.CaseExpr) (sql.Expression, error) { var branches []expression.CaseBranch for _, w := range e.Whens { - cond, err := exprToExpression(w.Cond) + var cond sql.Expression + cond, err = exprToExpression(ctx, w.Cond) if err != nil { return nil, err } - val, err := exprToExpression(w.Val) + var val sql.Expression + val, err = exprToExpression(ctx, w.Val) if err != nil { return nil, err } @@ -1049,7 +1327,7 @@ func caseExprToExpression(e *sqlparser.CaseExpr) (sql.Expression, error) { var elseExpr sql.Expression if e.Else != nil { - elseExpr, err = exprToExpression(e.Else) + elseExpr, err = exprToExpression(ctx, e.Else) if err != nil { return nil, err } @@ -1058,6 +1336,31 @@ func caseExprToExpression(e *sqlparser.CaseExpr) (sql.Expression, error) { return expression.NewCase(expr, branches, elseExpr), nil } +func intervalExprToExpression(ctx *sql.Context, e *sqlparser.IntervalExpr) (sql.Expression, error) { + expr, err := exprToExpression(ctx, e.Expr) + if err != nil { + return nil, err + } + + return expression.NewInterval(expr, e.Unit), nil +} + +func updateExprsToExpressions(ctx *sql.Context, e sqlparser.UpdateExprs) ([]sql.Expression, error) { + res := make([]sql.Expression, len(e)) + for i, updateExpr := range e { + colName, err := exprToExpression(ctx, updateExpr.Name) + if err != nil { + return nil, err + } + innerExpr, err := exprToExpression(ctx, updateExpr.Expr) + if err != nil { + return nil, err + } + res[i] = expression.NewSetField(colName, innerExpr) + } + return res, nil +} + func removeComments(s string) string { r := bufio.NewReader(strings.NewReader(s)) var result []rune @@ -1157,7 +1460,7 @@ func readString(r *bufio.Reader, single bool) []rune { return result } -func parseShowTableStatus(query string) (sql.Node, error) { +func parseShowTableStatus(ctx *sql.Context, query string) (sql.Node, error) { buf := bufio.NewReader(strings.NewReader(query)) err := parseFuncs{ expect("show"), @@ -1199,7 +1502,7 @@ func parseShowTableStatus(query string) (sql.Node, error) { return nil, err } - expr, err := parseExpr(string(bs)) + expr, err := parseExpr(ctx, string(bs)) if err != nil { return nil, err } @@ -1223,7 +1526,7 @@ func parseShowTableStatus(query string) (sql.Node, error) { } } -func parseShowCollation(query string) (sql.Node, error) { +func parseShowCollation(ctx *sql.Context, query string) (sql.Node, error) { buf := bufio.NewReader(strings.NewReader(query)) err := parseFuncs{ expect("show"), @@ -1256,7 +1559,7 @@ func parseShowCollation(query string) (sql.Node, error) { return nil, err } - expr, err := parseExpr(string(bs)) + expr, err := parseExpr(ctx, string(bs)) if err != nil { return nil, err } diff --git a/sql/parse/parse_test.go b/sql/parse/parse_test.go index 5fa1e32f3..b66dca943 100644 --- a/sql/parse/parse_test.go +++ b/sql/parse/parse_test.go @@ -1,18 +1,20 @@ package parse import ( + "math" "testing" - errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/expression/function/aggregation" + "github.com/src-d/go-mysql-server/sql/plan" + "gopkg.in/src-d/go-errors.v1" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) var fixtures = map[string]sql.Node{ - `CREATE TABLE t1(a INTEGER, b TEXT, c DATE, d TIMESTAMP, e VARCHAR(20), f BLOB NOT NULL)`: plan.NewCreateTable( + `CREATE TABLE t1(a INTEGER, b TEXT, c DATE, d TIMESTAMP, e VARCHAR(20), f BLOB NOT NULL, g DATETIME, h CHAR(40))`: plan.NewCreateTable( sql.UnresolvedDatabase(""), "t1", sql.Schema{{ @@ -39,8 +41,70 @@ var fixtures = map[string]sql.Node{ Name: "f", Type: sql.Blob, Nullable: false, + }, { + Name: "g", + Type: sql.Datetime, + Nullable: true, + }, { + Name: "h", + Type: sql.Text, + Nullable: true, + }}, + ), + `CREATE TABLE t1(a INTEGER NOT NULL PRIMARY KEY, b TEXT)`: plan.NewCreateTable( + sql.UnresolvedDatabase(""), + "t1", + sql.Schema{{ + Name: "a", + Type: sql.Int32, + Nullable: false, + PrimaryKey: true, + }, { + Name: "b", + Type: sql.Text, + Nullable: true, + PrimaryKey: false, }}, ), + `CREATE TABLE t1(a INTEGER, b TEXT, PRIMARY KEY (a))`: plan.NewCreateTable( + sql.UnresolvedDatabase(""), + "t1", + sql.Schema{{ + Name: "a", + Type: sql.Int32, + Nullable: true, + PrimaryKey: true, + }, { + Name: "b", + Type: sql.Text, + Nullable: true, + PrimaryKey: false, + }}, + ), + `CREATE TABLE t1(a INTEGER, b TEXT, PRIMARY KEY (a, b))`: plan.NewCreateTable( + sql.UnresolvedDatabase(""), + "t1", + sql.Schema{{ + Name: "a", + Type: sql.Int32, + Nullable: true, + PrimaryKey: true, + }, { + Name: "b", + Type: sql.Text, + Nullable: true, + PrimaryKey: true, + }}, + ), + `DROP TABLE foo;`: plan.NewDropTable( + sql.UnresolvedDatabase(""), false, "foo", + ), + `DROP TABLE IF EXISTS foo;`: plan.NewDropTable( + sql.UnresolvedDatabase(""), true, "foo", + ), + `DROP TABLE IF EXISTS foo, bar, baz;`: plan.NewDropTable( + sql.UnresolvedDatabase(""), true, "foo", "bar", "baz", + ), `DESCRIBE TABLE foo;`: plan.NewDescribe( plan.NewUnresolvedTable("foo", ""), ), @@ -61,6 +125,13 @@ var fixtures = map[string]sql.Node{ }, plan.NewUnresolvedTable("foo", ""), ), + `SELECT foo IS TRUE, bar IS NOT FALSE FROM foo;`: plan.NewProject( + []sql.Expression{ + expression.NewIsTrue(expression.NewUnresolvedColumn("foo")), + expression.NewNot(expression.NewIsFalse(expression.NewUnresolvedColumn("bar"))), + }, + plan.NewUnresolvedTable("foo", ""), + ), `SELECT foo AS bar FROM foo;`: plan.NewProject( []sql.Expression{ expression.NewAlias( @@ -165,7 +236,7 @@ var fixtures = map[string]sql.Node{ plan.NewFilter( expression.NewEquals( expression.NewUnresolvedColumn("qux"), - expression.NewLiteral(int64(1), sql.Int64), + expression.NewLiteral(int8(1), sql.Int8), ), plan.NewUnresolvedTable("foo", ""), ), @@ -242,8 +313,18 @@ var fixtures = map[string]sql.Node{ plan.NewUnresolvedTable("t1", ""), plan.NewValues([][]sql.Expression{{ expression.NewLiteral("a", sql.Text), - expression.NewLiteral(int64(1), sql.Int64), + expression.NewLiteral(int8(1), sql.Int8), + }}), + false, + []string{"col1", "col2"}, + ), + `REPLACE INTO t1 (col1, col2) VALUES ('a', 1)`: plan.NewInsertInto( + plan.NewUnresolvedTable("t1", ""), + plan.NewValues([][]sql.Expression{{ + expression.NewLiteral("a", sql.Text), + expression.NewLiteral(int8(1), sql.Int8), }}), + true, []string{"col1", "col2"}, ), `SHOW TABLES`: plan.NewShowTables(sql.UnresolvedDatabase(""), false), @@ -309,8 +390,17 @@ var fixtures = map[string]sql.Node{ }, plan.NewUnresolvedTable("foo", ""), ), - `SELECT foo, bar FROM foo LIMIT 2 OFFSET 5;`: plan.NewOffset(5, - plan.NewLimit(2, plan.NewProject( + `SELECT foo, bar FROM foo LIMIT 2 OFFSET 5;`: plan.NewLimit(2, + plan.NewOffset(5, plan.NewProject( + []sql.Expression{ + expression.NewUnresolvedColumn("foo"), + expression.NewUnresolvedColumn("bar"), + }, + plan.NewUnresolvedTable("foo", ""), + )), + ), + `SELECT foo, bar FROM foo LIMIT 5,2;`: plan.NewLimit(2, + plan.NewOffset(5, plan.NewProject( []sql.Expression{ expression.NewUnresolvedColumn("foo"), expression.NewUnresolvedColumn("bar"), @@ -325,7 +415,7 @@ var fixtures = map[string]sql.Node{ plan.NewFilter( expression.NewEquals( expression.NewUnresolvedColumn("a"), - expression.NewLiteral(int64(1), sql.Int64), + expression.NewLiteral(int8(1), sql.Int8), ), plan.NewUnresolvedTable("foo", ""), ), @@ -397,9 +487,9 @@ var fixtures = map[string]sql.Node{ plan.NewFilter( expression.NewNot( expression.NewBetween( - expression.NewLiteral(int64(1), sql.Int64), - expression.NewLiteral(int64(2), sql.Int64), - expression.NewLiteral(int64(5), sql.Int64), + expression.NewLiteral(int8(1), sql.Int8), + expression.NewLiteral(int8(2), sql.Int8), + expression.NewLiteral(int8(5), sql.Int8), ), ), plan.NewUnresolvedTable("foo", ""), @@ -409,16 +499,16 @@ var fixtures = map[string]sql.Node{ []sql.Expression{expression.NewStar()}, plan.NewFilter( expression.NewBetween( - expression.NewLiteral(int64(1), sql.Int64), - expression.NewLiteral(int64(2), sql.Int64), - expression.NewLiteral(int64(5), sql.Int64), + expression.NewLiteral(int8(1), sql.Int8), + expression.NewLiteral(int8(2), sql.Int8), + expression.NewLiteral(int8(5), sql.Int8), ), plan.NewUnresolvedTable("foo", ""), ), ), `SELECT 0x01AF`: plan.NewProject( []sql.Expression{ - expression.NewLiteral(int64(431), sql.Int64), + expression.NewLiteral(int16(431), sql.Int16), }, plan.NewUnresolvedTable("dual", ""), ), @@ -435,12 +525,12 @@ var fixtures = map[string]sql.Node{ "somefunc", false, expression.NewTuple( - expression.NewLiteral(int64(1), sql.Int64), - expression.NewLiteral(int64(2), sql.Int64), + expression.NewLiteral(int8(1), sql.Int8), + expression.NewLiteral(int8(2), sql.Int8), ), expression.NewTuple( - expression.NewLiteral(int64(3), sql.Int64), - expression.NewLiteral(int64(4), sql.Int64), + expression.NewLiteral(int8(3), sql.Int8), + expression.NewLiteral(int8(4), sql.Int8), ), ), plan.NewUnresolvedTable("b", ""), @@ -451,7 +541,7 @@ var fixtures = map[string]sql.Node{ plan.NewFilter( expression.NewEquals( expression.NewLiteral(":foo_id", sql.Text), - expression.NewLiteral(int64(2), sql.Int64), + expression.NewLiteral(int8(2), sql.Int8), ), plan.NewUnresolvedTable("foo", ""), ), @@ -475,13 +565,13 @@ var fixtures = map[string]sql.Node{ ), `SELECT CAST(-3 AS UNSIGNED) FROM foo`: plan.NewProject( []sql.Expression{ - expression.NewConvert(expression.NewLiteral(int64(-3), sql.Int64), expression.ConvertToUnsigned), + expression.NewConvert(expression.NewLiteral(int8(-3), sql.Int8), expression.ConvertToUnsigned), }, plan.NewUnresolvedTable("foo", ""), ), `SELECT 2 = 2 FROM foo`: plan.NewProject( []sql.Expression{ - expression.NewEquals(expression.NewLiteral(int64(2), sql.Int64), expression.NewLiteral(int64(2), sql.Int64)), + expression.NewEquals(expression.NewLiteral(int8(2), sql.Int8), expression.NewLiteral(int8(2), sql.Int8)), }, plan.NewUnresolvedTable("foo", ""), ), @@ -525,10 +615,10 @@ var fixtures = map[string]sql.Node{ []sql.Expression{expression.NewStar()}, plan.NewFilter( expression.NewIn( - expression.NewLiteral(int64(1), sql.Int64), + expression.NewLiteral(int8(1), sql.Int8), expression.NewTuple( expression.NewLiteral("1", sql.Text), - expression.NewLiteral(int64(2), sql.Int64), + expression.NewLiteral(int8(2), sql.Int8), ), ), plan.NewUnresolvedTable("foo", ""), @@ -538,10 +628,10 @@ var fixtures = map[string]sql.Node{ []sql.Expression{expression.NewStar()}, plan.NewFilter( expression.NewNotIn( - expression.NewLiteral(int64(1), sql.Int64), + expression.NewLiteral(int8(1), sql.Int8), expression.NewTuple( expression.NewLiteral("1", sql.Text), - expression.NewLiteral(int64(2), sql.Int64), + expression.NewLiteral(int8(2), sql.Int8), ), ), plan.NewUnresolvedTable("foo", ""), @@ -550,12 +640,12 @@ var fixtures = map[string]sql.Node{ `SELECT a, b FROM t ORDER BY 2, 1`: plan.NewSort( []plan.SortField{ { - Column: expression.NewLiteral(int64(2), sql.Int64), + Column: expression.NewLiteral(int8(2), sql.Int8), Order: plan.Ascending, NullOrdering: plan.NullsFirst, }, { - Column: expression.NewLiteral(int64(1), sql.Int64), + Column: expression.NewLiteral(int8(1), sql.Int8), Order: plan.Ascending, NullOrdering: plan.NullsFirst, }, @@ -570,22 +660,22 @@ var fixtures = map[string]sql.Node{ ), `SELECT 1 + 1;`: plan.NewProject( []sql.Expression{ - expression.NewPlus(expression.NewLiteral(int64(1), sql.Int64), expression.NewLiteral(int64(1), sql.Int64)), + expression.NewPlus(expression.NewLiteral(int8(1), sql.Int8), expression.NewLiteral(int8(1), sql.Int8)), }, plan.NewUnresolvedTable("dual", ""), ), `SELECT 1 * (2 + 1);`: plan.NewProject( []sql.Expression{ - expression.NewMult(expression.NewLiteral(int64(1), sql.Int64), - expression.NewPlus(expression.NewLiteral(int64(2), sql.Int64), expression.NewLiteral(int64(1), sql.Int64))), + expression.NewMult(expression.NewLiteral(int8(1), sql.Int8), + expression.NewPlus(expression.NewLiteral(int8(2), sql.Int8), expression.NewLiteral(int8(1), sql.Int8))), }, plan.NewUnresolvedTable("dual", ""), ), `SELECT (0 - 1) * (1 | 1);`: plan.NewProject( []sql.Expression{ expression.NewMult( - expression.NewMinus(expression.NewLiteral(int64(0), sql.Int64), expression.NewLiteral(int64(1), sql.Int64)), - expression.NewBitOr(expression.NewLiteral(int64(1), sql.Int64), expression.NewLiteral(int64(1), sql.Int64)), + expression.NewMinus(expression.NewLiteral(int8(0), sql.Int8), expression.NewLiteral(int8(1), sql.Int8)), + expression.NewBitOr(expression.NewLiteral(int8(1), sql.Int8), expression.NewLiteral(int8(1), sql.Int8)), ), }, plan.NewUnresolvedTable("dual", ""), @@ -593,8 +683,8 @@ var fixtures = map[string]sql.Node{ `SELECT (1 << 3) % (2 div 1);`: plan.NewProject( []sql.Expression{ expression.NewMod( - expression.NewShiftLeft(expression.NewLiteral(int64(1), sql.Int64), expression.NewLiteral(int64(3), sql.Int64)), - expression.NewIntDiv(expression.NewLiteral(int64(2), sql.Int64), expression.NewLiteral(int64(1), sql.Int64))), + expression.NewShiftLeft(expression.NewLiteral(int8(1), sql.Int8), expression.NewLiteral(int8(3), sql.Int8)), + expression.NewIntDiv(expression.NewLiteral(int8(2), sql.Int8), expression.NewLiteral(int8(1), sql.Int8))), }, plan.NewUnresolvedTable("dual", ""), ), @@ -610,7 +700,7 @@ var fixtures = map[string]sql.Node{ `SELECT '1.0' + 2;`: plan.NewProject( []sql.Expression{ expression.NewPlus( - expression.NewLiteral("1.0", sql.Text), expression.NewLiteral(int64(2), sql.Int64), + expression.NewLiteral("1.0", sql.Text), expression.NewLiteral(int8(2), sql.Int8), ), }, plan.NewUnresolvedTable("dual", ""), @@ -679,7 +769,7 @@ var fixtures = map[string]sql.Node{ expression.NewUnresolvedFunction( "max", true, expression.NewUnresolvedColumn("i"), ), - expression.NewLiteral(int64(2), sql.Int64), + expression.NewLiteral(int8(2), sql.Int8), "/", ), }, @@ -707,7 +797,7 @@ var fixtures = map[string]sql.Node{ `SET autocommit=1, foo="bar"`: plan.NewSet( plan.SetVariable{ Name: "autocommit", - Value: expression.NewLiteral(int64(1), sql.Int64), + Value: expression.NewLiteral(int8(1), sql.Int8), }, plan.SetVariable{ Name: "foo", @@ -717,7 +807,7 @@ var fixtures = map[string]sql.Node{ `SET @@session.autocommit=1, foo="bar"`: plan.NewSet( plan.SetVariable{ Name: "@@session.autocommit", - Value: expression.NewLiteral(int64(1), sql.Int64), + Value: expression.NewLiteral(int8(1), sql.Int8), }, plan.SetVariable{ Name: "foo", @@ -783,11 +873,11 @@ var fixtures = map[string]sql.Node{ `SET SESSION NET_READ_TIMEOUT= 700, SESSION NET_WRITE_TIMEOUT= 700`: plan.NewSet( plan.SetVariable{ Name: "@@session.net_read_timeout", - Value: expression.NewLiteral(int64(700), sql.Int64), + Value: expression.NewLiteral(int16(700), sql.Int16), }, plan.SetVariable{ Name: "@@session.net_write_timeout", - Value: expression.NewLiteral(int64(700), sql.Int64), + Value: expression.NewLiteral(int16(700), sql.Int16), }, ), `SET gtid_mode=DEFAULT`: plan.NewSet( @@ -940,11 +1030,11 @@ var fixtures = map[string]sql.Node{ expression.NewUnresolvedColumn("foo"), []expression.CaseBranch{ { - Cond: expression.NewLiteral(int64(1), sql.Int64), + Cond: expression.NewLiteral(int8(1), sql.Int8), Value: expression.NewLiteral("foo", sql.Text), }, { - Cond: expression.NewLiteral(int64(2), sql.Int64), + Cond: expression.NewLiteral(int8(2), sql.Int8), Value: expression.NewLiteral("bar", sql.Text), }, }, @@ -957,11 +1047,11 @@ var fixtures = map[string]sql.Node{ expression.NewUnresolvedColumn("foo"), []expression.CaseBranch{ { - Cond: expression.NewLiteral(int64(1), sql.Int64), + Cond: expression.NewLiteral(int8(1), sql.Int8), Value: expression.NewLiteral("foo", sql.Text), }, { - Cond: expression.NewLiteral(int64(2), sql.Int64), + Cond: expression.NewLiteral(int8(2), sql.Int8), Value: expression.NewLiteral("bar", sql.Text), }, }, @@ -976,14 +1066,14 @@ var fixtures = map[string]sql.Node{ { Cond: expression.NewEquals( expression.NewUnresolvedColumn("foo"), - expression.NewLiteral(int64(1), sql.Int64), + expression.NewLiteral(int8(1), sql.Int8), ), Value: expression.NewLiteral("foo", sql.Text), }, { Cond: expression.NewEquals( expression.NewUnresolvedColumn("foo"), - expression.NewLiteral(int64(2), sql.Int64), + expression.NewLiteral(int8(2), sql.Int8), ), Value: expression.NewLiteral("bar", sql.Text), }, @@ -1007,7 +1097,172 @@ var fixtures = map[string]sql.Node{ ), plan.NewShowCollation(), ), - `ROLLBACK`: plan.NewRollback(), + `ROLLBACK`: plan.NewRollback(), + "SHOW CREATE TABLE `mytable`": plan.NewShowCreateTable("", nil, "mytable"), + "SHOW CREATE TABLE `mydb`.`mytable`": plan.NewShowCreateTable("mydb", nil, "mytable"), + "SHOW CREATE TABLE `my.table`": plan.NewShowCreateTable("", nil, "my.table"), + "SHOW CREATE TABLE `my.db`.`my.table`": plan.NewShowCreateTable("my.db", nil, "my.table"), + "SHOW CREATE TABLE `my``table`": plan.NewShowCreateTable("", nil, "my`table"), + "SHOW CREATE TABLE `my``db`.`my``table`": plan.NewShowCreateTable("my`db", nil, "my`table"), + "SHOW CREATE TABLE ````": plan.NewShowCreateTable("", nil, "`"), + "SHOW CREATE TABLE `.`": plan.NewShowCreateTable("", nil, "."), + `SELECT '2018-05-01' + INTERVAL 1 DAY`: plan.NewProject( + []sql.Expression{expression.NewArithmetic( + expression.NewLiteral("2018-05-01", sql.Text), + expression.NewInterval( + expression.NewLiteral(int8(1), sql.Int8), + "DAY", + ), + "+", + )}, + plan.NewUnresolvedTable("dual", ""), + ), + `SELECT '2018-05-01' - INTERVAL 1 DAY`: plan.NewProject( + []sql.Expression{expression.NewArithmetic( + expression.NewLiteral("2018-05-01", sql.Text), + expression.NewInterval( + expression.NewLiteral(int8(1), sql.Int8), + "DAY", + ), + "-", + )}, + plan.NewUnresolvedTable("dual", ""), + ), + `SELECT INTERVAL 1 DAY + '2018-05-01'`: plan.NewProject( + []sql.Expression{expression.NewArithmetic( + expression.NewInterval( + expression.NewLiteral(int8(1), sql.Int8), + "DAY", + ), + expression.NewLiteral("2018-05-01", sql.Text), + "+", + )}, + plan.NewUnresolvedTable("dual", ""), + ), + `SELECT '2018-05-01' + INTERVAL 1 DAY + INTERVAL 1 DAY`: plan.NewProject( + []sql.Expression{expression.NewArithmetic( + expression.NewArithmetic( + expression.NewLiteral("2018-05-01", sql.Text), + expression.NewInterval( + expression.NewLiteral(int8(1), sql.Int8), + "DAY", + ), + "+", + ), + expression.NewInterval( + expression.NewLiteral(int8(1), sql.Int8), + "DAY", + ), + "+", + )}, + plan.NewUnresolvedTable("dual", ""), + ), + `SELECT COUNT(*) FROM foo GROUP BY a HAVING COUNT(*) > 5`: plan.NewHaving( + expression.NewGreaterThan( + expression.NewUnresolvedFunction("count", true, expression.NewStar()), + expression.NewLiteral(int8(5), sql.Int8), + ), + plan.NewGroupBy( + []sql.Expression{expression.NewUnresolvedFunction("count", true, expression.NewStar())}, + []sql.Expression{expression.NewUnresolvedColumn("a")}, + plan.NewUnresolvedTable("foo", ""), + ), + ), + `SELECT DISTINCT COUNT(*) FROM foo GROUP BY a HAVING COUNT(*) > 5`: plan.NewDistinct( + plan.NewHaving( + expression.NewGreaterThan( + expression.NewUnresolvedFunction("count", true, expression.NewStar()), + expression.NewLiteral(int8(5), sql.Int8), + ), + plan.NewGroupBy( + []sql.Expression{expression.NewUnresolvedFunction("count", true, expression.NewStar())}, + []sql.Expression{expression.NewUnresolvedColumn("a")}, + plan.NewUnresolvedTable("foo", ""), + ), + ), + ), + `SELECT * FROM foo LEFT JOIN bar ON 1=1`: plan.NewProject( + []sql.Expression{expression.NewStar()}, + plan.NewLeftJoin( + plan.NewUnresolvedTable("foo", ""), + plan.NewUnresolvedTable("bar", ""), + expression.NewEquals( + expression.NewLiteral(int8(1), sql.Int8), + expression.NewLiteral(int8(1), sql.Int8), + ), + ), + ), + `SELECT * FROM foo LEFT OUTER JOIN bar ON 1=1`: plan.NewProject( + []sql.Expression{expression.NewStar()}, + plan.NewLeftJoin( + plan.NewUnresolvedTable("foo", ""), + plan.NewUnresolvedTable("bar", ""), + expression.NewEquals( + expression.NewLiteral(int8(1), sql.Int8), + expression.NewLiteral(int8(1), sql.Int8), + ), + ), + ), + `SELECT * FROM foo RIGHT JOIN bar ON 1=1`: plan.NewProject( + []sql.Expression{expression.NewStar()}, + plan.NewRightJoin( + plan.NewUnresolvedTable("foo", ""), + plan.NewUnresolvedTable("bar", ""), + expression.NewEquals( + expression.NewLiteral(int8(1), sql.Int8), + expression.NewLiteral(int8(1), sql.Int8), + ), + ), + ), + `SELECT * FROM foo RIGHT OUTER JOIN bar ON 1=1`: plan.NewProject( + []sql.Expression{expression.NewStar()}, + plan.NewRightJoin( + plan.NewUnresolvedTable("foo", ""), + plan.NewUnresolvedTable("bar", ""), + expression.NewEquals( + expression.NewLiteral(int8(1), sql.Int8), + expression.NewLiteral(int8(1), sql.Int8), + ), + ), + ), + `SELECT FIRST(i) FROM foo`: plan.NewGroupBy( + []sql.Expression{ + expression.NewUnresolvedFunction("first", true, expression.NewUnresolvedColumn("i")), + }, + []sql.Expression{}, + plan.NewUnresolvedTable("foo", ""), + ), + `SELECT LAST(i) FROM foo`: plan.NewGroupBy( + []sql.Expression{ + expression.NewUnresolvedFunction("last", true, expression.NewUnresolvedColumn("i")), + }, + []sql.Expression{}, + plan.NewUnresolvedTable("foo", ""), + ), + `SELECT COUNT(DISTINCT i) FROM foo`: plan.NewGroupBy( + []sql.Expression{ + aggregation.NewCountDistinct(expression.NewUnresolvedColumn("i")), + }, + []sql.Expression{}, + plan.NewUnresolvedTable("foo", ""), + ), + `SELECT -128, 127, 255, -32768, 32767, 65535, -2147483648, 2147483647, 4294967295, -9223372036854775808, 9223372036854775807, 18446744073709551615`: plan.NewProject( + []sql.Expression{ + expression.NewLiteral(int8(math.MinInt8), sql.Int8), + expression.NewLiteral(int8(math.MaxInt8), sql.Int8), + expression.NewLiteral(uint8(math.MaxUint8), sql.Uint8), + expression.NewLiteral(int16(math.MinInt16), sql.Int16), + expression.NewLiteral(int16(math.MaxInt16), sql.Int16), + expression.NewLiteral(uint16(math.MaxUint16), sql.Uint16), + expression.NewLiteral(int32(math.MinInt32), sql.Int32), + expression.NewLiteral(int32(math.MaxInt32), sql.Int32), + expression.NewLiteral(uint32(math.MaxUint32), sql.Uint32), + expression.NewLiteral(int64(math.MinInt64), sql.Int64), + expression.NewLiteral(int64(math.MaxInt64), sql.Int64), + expression.NewLiteral(uint64(math.MaxUint64), sql.Uint64), + }, + plan.NewUnresolvedTable("dual", ""), + ), } func TestParse(t *testing.T) { @@ -1016,7 +1271,7 @@ func TestParse(t *testing.T) { require := require.New(t) ctx := sql.NewEmptyContext() p, err := Parse(ctx, query) - require.Nil(err, "error for query '%s'", query) + require.NoError(err) require.Exactly(expectedPlan, p, "plans do not match for query '%s'", query) }) @@ -1025,14 +1280,23 @@ func TestParse(t *testing.T) { } var fixturesErrors = map[string]*errors.Kind{ - `SHOW METHEMONEY`: ErrUnsupportedFeature, - `LOCK TABLES foo AS READ`: errUnexpectedSyntax, - `LOCK TABLES foo LOW_PRIORITY READ`: errUnexpectedSyntax, - `SELECT * FROM mytable WHERE i IN (SELECT i FROM foo)`: ErrUnsupportedSubqueryExpression, + `SHOW METHEMONEY`: ErrUnsupportedFeature, + `LOCK TABLES foo AS READ`: errUnexpectedSyntax, + `LOCK TABLES foo LOW_PRIORITY READ`: errUnexpectedSyntax, + `SELECT * FROM mytable LIMIT -100`: ErrUnsupportedSyntax, + `SELECT * FROM mytable LIMIT 100 OFFSET -1`: ErrUnsupportedSyntax, `SELECT * FROM files JOIN commit_files JOIN refs `: ErrUnsupportedSyntax, + `SELECT INTERVAL 1 DAY - '2018-05-01'`: ErrUnsupportedSyntax, + `SELECT INTERVAL 1 DAY * '2018-05-01'`: ErrUnsupportedSyntax, + `SELECT '2018-05-01' * INTERVAL 1 DAY`: ErrUnsupportedSyntax, + `SELECT '2018-05-01' / INTERVAL 1 DAY`: ErrUnsupportedSyntax, + `SELECT INTERVAL 1 DAY + INTERVAL 1 DAY`: ErrUnsupportedSyntax, + `SELECT '2018-05-01' + (INTERVAL 1 DAY + INTERVAL 1 DAY)`: ErrUnsupportedSyntax, + `SELECT AVG(DISTINCT foo) FROM b`: ErrUnsupportedSyntax, + `CREATE VIEW view1 AS SELECT x FROM t1 WHERE x>0`: ErrUnsupportedFeature, } func TestParseErrors(t *testing.T) { diff --git a/sql/parse/show_create.go b/sql/parse/show_create.go index 68ea2ee7b..028b6024f 100644 --- a/sql/parse/show_create.go +++ b/sql/parse/show_create.go @@ -2,13 +2,12 @@ package parse import ( "bufio" + "io" "strings" -) -import ( + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/plan" "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" ) var errUnsupportedShowCreateQuery = errors.NewKind("Unsupported query: SHOW CREATE %s") @@ -31,10 +30,24 @@ func parseShowCreate(s string) (sql.Node, error) { switch strings.ToLower(thingToShow) { case "table": - var name string + var db, table string + + if err := readQuotableIdent(&table)(r); err != nil { + return nil, err + } + + ru, _, err := r.ReadRune() + if err != nil && err != io.EOF { + return nil, err + } else if err == nil && ru == '.' { + db = table + + if err = readQuotableIdent(&table)(r); err != nil { + return nil, err + } + } - err := parseFuncs{ - readQuotableIdent(&name), + err = parseFuncs{ skipSpaces, checkEOF, }.exec(r) @@ -43,9 +56,9 @@ func parseShowCreate(s string) (sql.Node, error) { } return plan.NewShowCreateTable( - sql.UnresolvedDatabase("").Name(), + db, nil, - name), nil + table), nil case "database", "schema": var ifNotExists bool var next string @@ -58,17 +71,17 @@ func parseShowCreate(s string) (sql.Node, error) { // If ` is the next character, it's a db name. Otherwise it may be // a table name or IF NOT EXISTS. if nextByte[0] == '`' { - if err := readQuotableIdent(&next)(r); err != nil { + if err = readQuotableIdent(&next)(r); err != nil { return nil, err } } else { - if err := readIdent(&next)(r); err != nil { + if err = readIdent(&next)(r); err != nil { return nil, err } if next == "if" { ifNotExists = true - err := parseFuncs{ + err = parseFuncs{ skipSpaces, expect("not"), skipSpaces, diff --git a/sql/parse/show_create_test.go b/sql/parse/show_create_test.go index 032d655c6..3831fc0fb 100644 --- a/sql/parse/show_create_test.go +++ b/sql/parse/show_create_test.go @@ -3,10 +3,10 @@ package parse import ( "testing" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/plan" "github.com/stretchr/testify/require" "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" ) func TestParseShowCreateTableQuery(t *testing.T) { @@ -32,9 +32,27 @@ func TestParseShowCreateTableQuery(t *testing.T) { }, { "SHOW CREATE TABLE mytable", - plan.NewShowCreateTable(sql.UnresolvedDatabase("").Name(), - nil, - "mytable"), + plan.NewShowCreateTable("", nil, "mytable"), + nil, + }, + { + "SHOW CREATE TABLE `mytable`", + plan.NewShowCreateTable("", nil, "mytable"), + nil, + }, + { + "SHOW CREATE TABLE mydb.`mytable`", + plan.NewShowCreateTable("mydb", nil, "mytable"), + nil, + }, + { + "SHOW CREATE TABLE `mydb`.mytable", + plan.NewShowCreateTable("mydb", nil, "mytable"), + nil, + }, + { + "SHOW CREATE TABLE `mydb`.`mytable`", + plan.NewShowCreateTable("mydb", nil, "mytable"), nil, }, } diff --git a/sql/parse/util.go b/sql/parse/util.go index e76de3771..bfb358b1d 100644 --- a/sql/parse/util.go +++ b/sql/parse/util.go @@ -9,9 +9,9 @@ import ( "strings" "unicode" + "github.com/src-d/go-mysql-server/sql" errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-vitess.v1/vt/sqlparser" + "vitess.io/vitess/go/vt/sqlparser" ) var ( @@ -144,6 +144,36 @@ func readValidIdentRune(r *bufio.Reader, buf *bytes.Buffer) error { return nil } +func readValidQuotedIdentRune(r *bufio.Reader, buf *bytes.Buffer) error { + bs, err := r.Peek(2) + if err != nil { + return err + } + + if bs[0] == '`' && bs[1] == '`' { + if _, _, err := r.ReadRune(); err != nil { + return err + } + if _, _, err := r.ReadRune(); err != nil { + return err + } + buf.WriteRune('`') + return nil + } + + if bs[0] == '`' && bs[1] != '`' { + return io.EOF + } + + if _, _, err := r.ReadRune(); err != nil { + return err + } + + buf.WriteByte(bs[0]) + + return nil +} + func unreadString(r *bufio.Reader, str string) { nr := *r r.Reset(io.MultiReader(strings.NewReader(str), &nr)) @@ -169,6 +199,26 @@ func readIdent(ident *string) parseFunc { } } +func readQuotedIdent(ident *string) parseFunc { + return func(r *bufio.Reader) error { + var buf bytes.Buffer + if err := readValidQuotedIdentRune(r, &buf); err != nil { + return err + } + + for { + if err := readValidQuotedIdentRune(r, &buf); err == io.EOF { + break + } else if err != nil { + return err + } + } + + *ident = strings.ToLower(buf.String()) + return nil + } +} + func oneOf(options ...string) parseFunc { return func(r *bufio.Reader) error { var ident string @@ -201,7 +251,7 @@ func readRemaining(val *string) parseFunc { } } -func parseExpr(str string) (sql.Expression, error) { +func parseExpr(ctx *sql.Context, str string) (sql.Expression, error) { stmt, err := sqlparser.Parse("SELECT " + str) if err != nil { return nil, err @@ -221,7 +271,7 @@ func parseExpr(str string) (sql.Expression, error) { return nil, errInvalidIndexExpression.New(str) } - return exprToExpression(selectExpr.Expr) + return exprToExpression(ctx, selectExpr.Expr) } func readQuotableIdent(ident *string) parseFunc { @@ -235,7 +285,7 @@ func readQuotableIdent(ident *string) parseFunc { if nextChar[0] == '`' { steps = parseFuncs{ expectQuote, - readIdent(ident), + readQuotedIdent(ident), expectQuote, } } else { diff --git a/sql/parse/variables.go b/sql/parse/variables.go index c92b5355d..df4844341 100644 --- a/sql/parse/variables.go +++ b/sql/parse/variables.go @@ -4,8 +4,8 @@ import ( "bufio" "strings" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/plan" ) func parseShowVariables(ctx *sql.Context, s string) (sql.Node, error) { diff --git a/sql/parse/warnings.go b/sql/parse/warnings.go index 10a01c331..9c88a3fd8 100644 --- a/sql/parse/warnings.go +++ b/sql/parse/warnings.go @@ -5,9 +5,9 @@ import ( "strconv" "strings" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/plan" errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/plan" ) var errInvalidIndex = errors.NewKind("invalid %s index %d (index must be non-negative)") diff --git a/sql/plan/common.go b/sql/plan/common.go index d089e2041..28e0dd935 100644 --- a/sql/plan/common.go +++ b/sql/plan/common.go @@ -1,6 +1,6 @@ -package plan // import "gopkg.in/src-d/go-mysql-server.v0/sql/plan" +package plan -import "gopkg.in/src-d/go-mysql-server.v0/sql" +import "github.com/src-d/go-mysql-server/sql" // IsUnary returns whether the node is unary or not. func IsUnary(node sql.Node) bool { @@ -57,20 +57,3 @@ func expressionsResolved(exprs ...sql.Expression) bool { return true } - -func transformExpressionsUp( - f sql.TransformExprFunc, - exprs []sql.Expression, -) ([]sql.Expression, error) { - - var es []sql.Expression - for _, e := range exprs { - te, err := e.TransformUp(f) - if err != nil { - return nil, err - } - es = append(es, te) - } - - return es, nil -} diff --git a/sql/plan/common_test.go b/sql/plan/common_test.go index d9d3a8677..483a0324c 100644 --- a/sql/plan/common_test.go +++ b/sql/plan/common_test.go @@ -6,12 +6,12 @@ import ( "io" "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) -var benchtable = func() *mem.Table { +var benchtable = func() *memory.Table { schema := sql.Schema{ {Name: "strfield", Type: sql.Text, Nullable: true}, {Name: "floatfield", Type: sql.Float64, Nullable: true}, @@ -20,7 +20,7 @@ var benchtable = func() *mem.Table { {Name: "bigintfield", Type: sql.Int64, Nullable: false}, {Name: "blobfield", Type: sql.Blob, Nullable: false}, } - t := mem.NewTable("test", schema) + t := memory.NewTable("test", schema) for i := 0; i < 100; i++ { n := fmt.Sprint(i) @@ -109,7 +109,7 @@ func collectRows(t *testing.T, node sql.Node) []sql.Row { func TestIsUnary(t *testing.T) { require := require.New(t) - table := mem.NewTable("foo", nil) + table := memory.NewTable("foo", nil) require.True(IsUnary(NewFilter(nil, NewResolvedTable(table)))) require.False(IsUnary(NewCrossJoin( @@ -120,7 +120,7 @@ func TestIsUnary(t *testing.T) { func TestIsBinary(t *testing.T) { require := require.New(t) - table := mem.NewTable("foo", nil) + table := memory.NewTable("foo", nil) require.False(IsBinary(NewFilter(nil, NewResolvedTable(table)))) require.True(IsBinary(NewCrossJoin( diff --git a/sql/plan/create_index.go b/sql/plan/create_index.go index f8553e36a..00455a127 100644 --- a/sql/plan/create_index.go +++ b/sql/plan/create_index.go @@ -8,9 +8,9 @@ import ( opentracing "github.com/opentracing/opentracing-go" otlog "github.com/opentracing/opentracing-go/log" "github.com/sirupsen/logrus" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) var ( @@ -260,56 +260,31 @@ func (c *CreateIndex) Expressions() []sql.Expression { return c.Exprs } -// TransformExpressions implements the Expressioner interface. -func (c *CreateIndex) TransformExpressions(fn sql.TransformExprFunc) (sql.Node, error) { - var exprs = make([]sql.Expression, len(c.Exprs)) - var err error - for i, e := range c.Exprs { - exprs[i], err = e.TransformUp(fn) - if err != nil { - return nil, err - } +// WithExpressions implements the Expressioner interface. +func (c *CreateIndex) WithExpressions(exprs ...sql.Expression) (sql.Node, error) { + if len(exprs) != len(c.Exprs) { + return nil, sql.ErrInvalidChildrenNumber.New(c, len(exprs), len(c.Exprs)) } nc := *c nc.Exprs = exprs - return &nc, nil } -// TransformExpressionsUp implements the Node interface. -func (c *CreateIndex) TransformExpressionsUp(fn sql.TransformExprFunc) (sql.Node, error) { - table, err := c.Table.TransformExpressionsUp(fn) - if err != nil { - return nil, err - } - - var exprs = make([]sql.Expression, len(c.Exprs)) - for i, e := range c.Exprs { - exprs[i], err = e.TransformUp(fn) - if err != nil { - return nil, err - } +// WithChildren implements the Node interface. +func (c *CreateIndex) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(c, len(children), 1) } nc := *c - nc.Table = table - nc.Exprs = exprs - + nc.Table = children[0] return &nc, nil } -// TransformUp implements the Node interface. -func (c *CreateIndex) TransformUp(fn sql.TransformNodeFunc) (sql.Node, error) { - table, err := c.Table.TransformUp(fn) - if err != nil { - return nil, err - } - - nc := *c - nc.Table = table - - return fn(&nc) +// IsAsync implements the AsyncNode interface. +func (c *CreateIndex) IsAsync() bool { + return c.Async } // getColumnsAndPrepareExpressions extracts the unique columns required by all @@ -323,15 +298,15 @@ func getColumnsAndPrepareExpressions( var expressions = make([]sql.Expression, len(exprs)) for i, e := range exprs { - ex, err := e.TransformUp(func(e sql.Expression) (sql.Expression, error) { + ex, err := expression.TransformUp(e, func(e sql.Expression) (sql.Expression, error) { gf, ok := e.(*expression.GetField) if !ok { return e, nil } var idx int - if i, ok := seen[gf.Name()]; ok { - idx = i + if j, ok := seen[gf.Name()]; ok { + idx = j } else { idx = len(columns) columns = append(columns, gf.Name()) diff --git a/sql/plan/create_index_test.go b/sql/plan/create_index_test.go index a4110f737..0cb509583 100644 --- a/sql/plan/create_index_test.go +++ b/sql/plan/create_index_test.go @@ -7,10 +7,10 @@ import ( "testing" "time" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/test" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/test" "github.com/stretchr/testify/require" ) @@ -18,7 +18,7 @@ import ( func TestCreateIndexAsync(t *testing.T) { require := require.New(t) - table := mem.NewTable("foo", sql.Schema{ + table := memory.NewTable("foo", sql.Schema{ {Name: "a", Source: "foo"}, {Name: "b", Source: "foo"}, {Name: "c", Source: "foo"}, @@ -27,7 +27,7 @@ func TestCreateIndexAsync(t *testing.T) { driver := new(mockDriver) catalog := sql.NewCatalog() catalog.RegisterIndexDriver(driver) - db := mem.NewDatabase("foo") + db := memory.NewDatabase("foo") db.AddTable("foo", table) catalog.AddDatabase(db) @@ -72,7 +72,7 @@ func TestCreateIndexAsync(t *testing.T) { func TestCreateIndexNotIndexableExprs(t *testing.T) { require := require.New(t) - table := mem.NewTable("foo", sql.Schema{ + table := memory.NewTable("foo", sql.Schema{ {Name: "a", Source: "foo", Type: sql.Blob}, {Name: "b", Source: "foo", Type: sql.JSON}, {Name: "c", Source: "foo", Type: sql.Text}, @@ -81,7 +81,7 @@ func TestCreateIndexNotIndexableExprs(t *testing.T) { driver := new(mockDriver) catalog := sql.NewCatalog() catalog.RegisterIndexDriver(driver) - db := mem.NewDatabase("foo") + db := memory.NewDatabase("foo") db.AddTable("foo", table) catalog.AddDatabase(db) @@ -121,7 +121,7 @@ func TestCreateIndexNotIndexableExprs(t *testing.T) { func TestCreateIndexSync(t *testing.T) { require := require.New(t) - table := mem.NewTable("foo", sql.Schema{ + table := memory.NewTable("foo", sql.Schema{ {Name: "a", Source: "foo"}, {Name: "b", Source: "foo"}, {Name: "c", Source: "foo"}, @@ -130,7 +130,7 @@ func TestCreateIndexSync(t *testing.T) { driver := new(mockDriver) catalog := sql.NewCatalog() catalog.RegisterIndexDriver(driver) - db := mem.NewDatabase("foo") + db := memory.NewDatabase("foo") db.AddTable("foo", table) catalog.AddDatabase(db) @@ -175,7 +175,7 @@ func TestCreateIndexChecksum(t *testing.T) { require := require.New(t) table := &checksumTable{ - mem.NewTable("foo", sql.Schema{ + memory.NewTable("foo", sql.Schema{ {Name: "a", Source: "foo"}, {Name: "b", Source: "foo"}, {Name: "c", Source: "foo"}, @@ -186,7 +186,7 @@ func TestCreateIndexChecksum(t *testing.T) { driver := new(mockDriver) catalog := sql.NewCatalog() catalog.RegisterIndexDriver(driver) - db := mem.NewDatabase("foo") + db := memory.NewDatabase("foo") db.AddTable("foo", table) catalog.AddDatabase(db) @@ -217,7 +217,7 @@ func TestCreateIndexChecksumWithUnderlying(t *testing.T) { &underlyingTable{ &underlyingTable{ &checksumTable{ - mem.NewTable("foo", sql.Schema{ + memory.NewTable("foo", sql.Schema{ {Name: "a", Source: "foo"}, {Name: "b", Source: "foo"}, {Name: "c", Source: "foo"}, @@ -253,7 +253,7 @@ func TestCreateIndexChecksumWithUnderlying(t *testing.T) { func TestCreateIndexWithIter(t *testing.T) { require := require.New(t) - foo := mem.NewPartitionedTable("foo", sql.Schema{ + foo := memory.NewPartitionedTable("foo", sql.Schema{ {Name: "one", Source: "foo", Type: sql.Int64}, {Name: "two", Source: "foo", Type: sql.Int64}, }, 2) @@ -277,7 +277,7 @@ func TestCreateIndexWithIter(t *testing.T) { driver := new(mockDriver) catalog := sql.NewCatalog() catalog.RegisterIndexDriver(driver) - db := mem.NewDatabase("foo") + db := memory.NewDatabase("foo") db.AddTable("foo", foo) catalog.AddDatabase(db) diff --git a/sql/plan/cross_join.go b/sql/plan/cross_join.go index 96887d12b..f7dc1d80d 100644 --- a/sql/plan/cross_join.go +++ b/sql/plan/cross_join.go @@ -5,7 +5,7 @@ import ( "reflect" opentracing "github.com/opentracing/opentracing-go" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // CrossJoin is a cross join between two tables. @@ -66,34 +66,13 @@ func (p *CrossJoin) RowIter(ctx *sql.Context) (sql.RowIter, error) { }), nil } -// TransformUp implements the Transformable interface. -func (p *CrossJoin) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - left, err := p.Left.TransformUp(f) - if err != nil { - return nil, err - } - - right, err := p.Right.TransformUp(f) - if err != nil { - return nil, err - } - - return f(NewCrossJoin(left, right)) -} - -// TransformExpressionsUp implements the Transformable interface. -func (p *CrossJoin) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { - left, err := p.Left.TransformExpressionsUp(f) - if err != nil { - return nil, err +// WithChildren implements the Node interface. +func (p *CrossJoin) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 2 { + return nil, sql.ErrInvalidChildrenNumber.New(p, len(children), 2) } - right, err := p.Right.TransformExpressionsUp(f) - if err != nil { - return nil, err - } - - return NewCrossJoin(left, right), nil + return NewCrossJoin(children[0], children[1]), nil } func (p *CrossJoin) String() string { @@ -155,17 +134,18 @@ func (i *crossJoinIterator) Next() (sql.Row, error) { } } -func (i *crossJoinIterator) Close() error { - if err := i.l.Close(); err != nil { - if i.r != nil { - _ = i.r.Close() - } - return err +func (i *crossJoinIterator) Close() (err error) { + if i.l != nil { + err = i.l.Close() } if i.r != nil { - return i.r.Close() + if err == nil { + err = i.r.Close() + } else { + i.r.Close() + } } - return nil + return err } diff --git a/sql/plan/cross_join_test.go b/sql/plan/cross_join_test.go index ab91704fa..f45548e99 100644 --- a/sql/plan/cross_join_test.go +++ b/sql/plan/cross_join_test.go @@ -4,9 +4,9 @@ import ( "io" "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) var lSchema = sql.Schema{ @@ -38,8 +38,8 @@ func TestCrossJoin(t *testing.T) { {Name: "rcol4", Type: sql.Int64}, } - ltable := mem.NewTable("left", lSchema) - rtable := mem.NewTable("right", rSchema) + ltable := memory.NewTable("left", lSchema) + rtable := memory.NewTable("right", rSchema) insertData(t, ltable) insertData(t, rtable) @@ -62,12 +62,12 @@ func TestCrossJoin(t *testing.T) { require.Equal("col1_1", row[0]) require.Equal("col2_1", row[1]) - require.Equal(int32(1111), row[2]) - require.Equal(int64(2222), row[3]) + require.Equal(int32(1), row[2]) + require.Equal(int64(2), row[3]) require.Equal("col1_1", row[4]) require.Equal("col2_1", row[5]) - require.Equal(int32(1111), row[6]) - require.Equal(int64(2222), row[7]) + require.Equal(int32(1), row[6]) + require.Equal(int64(2), row[7]) row, err = iter.Next() require.NoError(err) @@ -75,12 +75,12 @@ func TestCrossJoin(t *testing.T) { require.Equal("col1_1", row[0]) require.Equal("col2_1", row[1]) - require.Equal(int32(1111), row[2]) - require.Equal(int64(2222), row[3]) + require.Equal(int32(1), row[2]) + require.Equal(int64(2), row[3]) require.Equal("col1_2", row[4]) require.Equal("col2_2", row[5]) - require.Equal(int32(3333), row[6]) - require.Equal(int64(4444), row[7]) + require.Equal(int32(3), row[6]) + require.Equal(int64(4), row[7]) for i := 0; i < 2; i++ { row, err = iter.Next() @@ -99,8 +99,8 @@ func TestCrossJoin_Empty(t *testing.T) { require := require.New(t) ctx := sql.NewEmptyContext() - ltable := mem.NewTable("left", lSchema) - rtable := mem.NewTable("right", rSchema) + ltable := memory.NewTable("left", lSchema) + rtable := memory.NewTable("right", rSchema) insertData(t, ltable) j := NewCrossJoin( @@ -116,8 +116,8 @@ func TestCrossJoin_Empty(t *testing.T) { require.Equal(io.EOF, err) require.Nil(row) - ltable = mem.NewTable("left", lSchema) - rtable = mem.NewTable("right", rSchema) + ltable = memory.NewTable("left", lSchema) + rtable = memory.NewTable("right", rSchema) insertData(t, rtable) j = NewCrossJoin( @@ -134,13 +134,13 @@ func TestCrossJoin_Empty(t *testing.T) { require.Nil(row) } -func insertData(t *testing.T, table *mem.Table) { +func insertData(t *testing.T, table *memory.Table) { t.Helper() require := require.New(t) rows := []sql.Row{ - sql.NewRow("col1_1", "col2_1", int32(1111), int64(2222)), - sql.NewRow("col1_2", "col2_2", int32(3333), int64(4444)), + sql.NewRow("col1_1", "col2_1", int32(1), int64(2)), + sql.NewRow("col1_2", "col2_2", int32(3), int64(4)), } for _, r := range rows { diff --git a/sql/plan/ddl.go b/sql/plan/ddl.go index d097aa904..d9acf8a09 100644 --- a/sql/plan/ddl.go +++ b/sql/plan/ddl.go @@ -1,12 +1,14 @@ package plan import ( + "fmt" + "github.com/src-d/go-mysql-server/sql" "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) // ErrCreateTable is thrown when the database doesn't support table creation -var ErrCreateTable = errors.NewKind("tables cannot be created on database %s") +var ErrCreateTableNotSupported = errors.NewKind("tables cannot be created on database %s") +var ErrDropTableNotSupported = errors.NewKind("tables cannot be dropped on database %s") // CreateTable is a node describing the creation of some table. type CreateTable struct { @@ -50,12 +52,12 @@ func (c *CreateTable) Resolved() bool { // RowIter implements the Node interface. func (c *CreateTable) RowIter(s *sql.Context) (sql.RowIter, error) { - d, ok := c.db.(sql.Alterable) - if !ok { - return nil, ErrCreateTable.New(c.db.Name()) + creatable, ok := c.db.(sql.TableCreator) + if ok { + return sql.RowsToRowIter(), creatable.CreateTable(s, c.name, c.schema) } - return sql.RowsToRowIter(), d.Create(c.name, c.schema) + return nil, ErrCreateTableNotSupported.New(c.db.Name()) } // Schema implements the Node interface. @@ -64,16 +66,97 @@ func (c *CreateTable) Schema() sql.Schema { return nil } // Children implements the Node interface. func (c *CreateTable) Children() []sql.Node { return nil } -// TransformUp implements the Transformable interface. -func (c *CreateTable) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - return f(NewCreateTable(c.db, c.name, c.schema)) -} - -// TransformExpressionsUp implements the Transformable interface. -func (c *CreateTable) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { +// WithChildren implements the Node interface. +func (c *CreateTable) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(c, len(children), 0) + } return c, nil } func (c *CreateTable) String() string { return "CreateTable" } + +// DropTable is a node describing dropping one or more tables +type DropTable struct { + db sql.Database + names []string + ifExists bool +} + +// NewDropTable creates a new DropTable node +func NewDropTable(db sql.Database, ifExists bool, tableNames ...string) *DropTable { + return &DropTable{ + db: db, + names: tableNames, + ifExists: ifExists, + } +} + +var _ sql.Databaser = (*DropTable)(nil) + +// Database implements the sql.Databaser interface. +func (d *DropTable) Database() sql.Database { + return d.db +} + +// WithDatabase implements the sql.Databaser interface. +func (d *DropTable) WithDatabase(db sql.Database) (sql.Node, error) { + nc := *d + nc.db = db + return &nc, nil +} + +// Resolved implements the Resolvable interface. +func (d *DropTable) Resolved() bool { + _, ok := d.db.(sql.UnresolvedDatabase) + return !ok +} + +// RowIter implements the Node interface. +func (d *DropTable) RowIter(s *sql.Context) (sql.RowIter, error) { + droppable, ok := d.db.(sql.TableDropper) + if !ok { + return nil, ErrDropTableNotSupported.New(d.db.Name()) + } + + var err error + for _, tableName := range d.names { + _, ok := d.db.Tables()[tableName] + if !ok { + if d.ifExists { + continue + } + return nil, sql.ErrTableNotFound.New(tableName) + } + err = droppable.DropTable(s, tableName) + if err != nil { + break + } + } + + return sql.RowsToRowIter(), err +} + +// Schema implements the Node interface. +func (d *DropTable) Schema() sql.Schema { return nil } + +// Children implements the Node interface. +func (d *DropTable) Children() []sql.Node { return nil } + +// WithChildren implements the Node interface. +func (d *DropTable) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(d, len(children), 0) + } + return d, nil +} + +func (d *DropTable) String() string { + ifExists := "" + if d.ifExists { + ifExists = "if exists " + } + return fmt.Sprintf("Drop table %s%s", ifExists, d.names) +} diff --git a/sql/plan/ddl_test.go b/sql/plan/ddl_test.go index 8d97440bb..a226a631c 100644 --- a/sql/plan/ddl_test.go +++ b/sql/plan/ddl_test.go @@ -4,15 +4,15 @@ import ( "io" "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestCreateTable(t *testing.T) { require := require.New(t) - db := mem.NewDatabase("test") + db := memory.NewDatabase("test") tables := db.Tables() _, ok := tables["testTable"] require.False(ok) @@ -22,15 +22,7 @@ func TestCreateTable(t *testing.T) { {Name: "c2", Type: sql.Int32}, } - c := NewCreateTable(db, "testTable", s) - - rows, err := c.RowIter(sql.NewEmptyContext()) - - require.NoError(err) - - r, err := rows.Next() - require.Equal(err, io.EOF) - require.Nil(r) + createTable(t, db, "testTable", s) tables = db.Tables() @@ -43,3 +35,59 @@ func TestCreateTable(t *testing.T) { require.Equal("testTable", s.Source) } } + +func TestDropTable(t *testing.T) { + require := require.New(t) + + db := memory.NewDatabase("test") + + s := sql.Schema{ + {Name: "c1", Type: sql.Text}, + {Name: "c2", Type: sql.Int32}, + } + + createTable(t, db, "testTable1", s) + createTable(t, db, "testTable2", s) + createTable(t, db, "testTable3", s) + + d := NewDropTable(db, false, "testTable1", "testTable2") + rows, err := d.RowIter(sql.NewEmptyContext()) + require.NoError(err) + + r, err := rows.Next() + require.Equal(err, io.EOF) + require.Nil(r) + + _, ok := db.Tables()["testTable1"] + require.False(ok) + _, ok = db.Tables()["testTable2"] + require.False(ok) + _, ok = db.Tables()["testTable3"] + require.True(ok) + + d = NewDropTable(db, false, "testTable1") + _, err = d.RowIter(sql.NewEmptyContext()) + require.Error(err) + + d = NewDropTable(db, true, "testTable1") + _, err = d.RowIter(sql.NewEmptyContext()) + require.NoError(err) + + d = NewDropTable(db, true, "testTable1", "testTable2", "testTable3") + _, err = d.RowIter(sql.NewEmptyContext()) + require.NoError(err) + + _, ok = db.Tables()["testTable3"] + require.False(ok) +} + +func createTable(t *testing.T, db sql.Database, name string, schema sql.Schema) { + c := NewCreateTable(db, name, schema) + + rows, err := c.RowIter(sql.NewEmptyContext()) + require.NoError(t, err) + + r, err := rows.Next() + require.Equal(t, err, io.EOF) + require.Nil(t, r) +} \ No newline at end of file diff --git a/sql/plan/delete.go b/sql/plan/delete.go new file mode 100644 index 000000000..95498b397 --- /dev/null +++ b/sql/plan/delete.go @@ -0,0 +1,125 @@ +package plan + +import ( + "github.com/src-d/go-mysql-server/sql" + "gopkg.in/src-d/go-errors.v1" + "io" +) + +var ErrDeleteFromNotSupported = errors.NewKind("table doesn't support DELETE FROM") + +// DeleteFrom is a node describing a deletion from some table. +type DeleteFrom struct { + sql.Node +} + +// NewDeleteFrom creates a DeleteFrom node. +func NewDeleteFrom(n sql.Node) *DeleteFrom { + return &DeleteFrom{n} +} + +// Schema implements the Node interface. +func (p *DeleteFrom) Schema() sql.Schema { + return sql.Schema{{ + Name: "updated", + Type: sql.Int64, + Default: int64(0), + Nullable: false, + }} +} + +// Resolved implements the Resolvable interface. +func (p *DeleteFrom) Resolved() bool { + return p.Node.Resolved() +} + +func (p *DeleteFrom) Children() []sql.Node { + return []sql.Node{p.Node} +} + +func getDeletable(node sql.Node) (sql.Deleter, error) { + switch node := node.(type) { + case sql.Deleter: + return node, nil + case *ResolvedTable: + return getDeletableTable(node.Table) + } + for _, child := range node.Children() { + deleter, _ := getDeletable(child) + if deleter != nil { + return deleter, nil + } + } + return nil, ErrDeleteFromNotSupported.New() +} + +func getDeletableTable(t sql.Table) (sql.Deleter, error) { + switch t := t.(type) { + case sql.Deleter: + return t, nil + case sql.TableWrapper: + return getDeletableTable(t.Underlying()) + default: + return nil, ErrDeleteFromNotSupported.New() + } +} + +// Execute deletes the rows in the database. +func (p *DeleteFrom) Execute(ctx *sql.Context) (int, error) { + deletable, err := getDeletable(p.Node) + if err != nil { + return 0, err + } + + iter, err := p.Node.RowIter(ctx) + if err != nil { + return 0, err + } + + i := 0 + for { + row, err := iter.Next() + if err == io.EOF { + break + } + + if err != nil { + _ = iter.Close() + return i, err + } + + if err := deletable.Delete(ctx, row); err != nil { + _ = iter.Close() + return i, err + } + + i++ + } + + return i, nil +} + +// RowIter implements the Node interface. +func (p *DeleteFrom) RowIter(ctx *sql.Context) (sql.RowIter, error) { + n, err := p.Execute(ctx) + if err != nil { + return nil, err + } + + return sql.RowsToRowIter(sql.NewRow(int64(n))), nil +} + +// WithChildren implements the Node interface. +func (p *DeleteFrom) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(p, len(children), 1) + } + return NewDeleteFrom(children[0]), nil +} + +func (p DeleteFrom) String() string { + pr := sql.NewTreePrinter() + _ = pr.WriteNode("Delete") + _ = pr.WriteChildren(p.Node.String()) + return pr.String() +} diff --git a/sql/plan/describe.go b/sql/plan/describe.go index 5260ea8c0..e84cdc8a0 100644 --- a/sql/plan/describe.go +++ b/sql/plan/describe.go @@ -4,7 +4,7 @@ import ( "io" "strings" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // Describe is a node that describes its children. @@ -33,22 +33,13 @@ func (d *Describe) RowIter(ctx *sql.Context) (sql.RowIter, error) { return &describeIter{schema: d.Child.Schema()}, nil } -// TransformUp implements the Transformable interface. -func (d *Describe) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - child, err := d.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Node interface. +func (d *Describe) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(d, len(children), 1) } - return f(NewDescribe(child)) -} -// TransformExpressionsUp implements the Transformable interface. -func (d *Describe) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { - child, err := d.Child.TransformExpressionsUp(f) - if err != nil { - return nil, err - } - return NewDescribe(child), nil + return NewDescribe(children[0]), nil } func (d Describe) String() string { @@ -70,7 +61,7 @@ func (i *describeIter) Next() (sql.Row, error) { f := i.schema[i.i] i.i++ - return sql.NewRow(f.Name, f.Type.Type().String()), nil + return sql.NewRow(f.Name, sql.MySQLTypeName(f.Type)), nil } func (i *describeIter) Close() error { @@ -116,22 +107,11 @@ func (d *DescribeQuery) String() string { return pr.String() } -// TransformUp implements the Node interface. -func (d *DescribeQuery) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - child, err := d.Child.TransformUp(f) - if err != nil { - return nil, err - } - - return f(NewDescribeQuery(d.Format, child)) -} - -// TransformExpressionsUp implements the Node interface. -func (d *DescribeQuery) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { - child, err := d.Child.TransformExpressionsUp(f) - if err != nil { - return nil, err +// WithChildren implements the Node interface. +func (d *DescribeQuery) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(d, len(children), 1) } - return NewDescribeQuery(d.Format, child), nil + return NewDescribeQuery(d.Format, children[0]), nil } diff --git a/sql/plan/describe_test.go b/sql/plan/describe_test.go index d8d47deeb..18c0b9f84 100644 --- a/sql/plan/describe_test.go +++ b/sql/plan/describe_test.go @@ -4,17 +4,17 @@ import ( "io" "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestDescribe(t *testing.T) { require := require.New(t) ctx := sql.NewEmptyContext() - table := mem.NewTable("test", sql.Schema{ + table := memory.NewTable("test", sql.Schema{ {Name: "c1", Type: sql.Text}, {Name: "c2", Type: sql.Int32}, }) @@ -30,7 +30,7 @@ func TestDescribe(t *testing.T) { n, err = iter.Next() require.NoError(err) - require.Equal(sql.NewRow("c2", "INT32"), n) + require.Equal(sql.NewRow("c2", "INTEGER"), n) n, err = iter.Next() require.Equal(io.EOF, err) @@ -55,7 +55,7 @@ func TestDescribe_Empty(t *testing.T) { func TestDescribeQuery(t *testing.T) { require := require.New(t) - table := mem.NewTable("foo", sql.Schema{ + table := memory.NewTable("foo", sql.Schema{ {Source: "foo", Name: "a", Type: sql.Text}, {Source: "foo", Name: "b", Type: sql.Text}, }) diff --git a/sql/plan/distinct.go b/sql/plan/distinct.go index 345d6c9a0..8a3da753a 100644 --- a/sql/plan/distinct.go +++ b/sql/plan/distinct.go @@ -1,10 +1,9 @@ package plan import ( - "fmt" + "io" - "github.com/mitchellh/hashstructure" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // Distinct is a node that ensures all rows that come from it are unique. @@ -34,25 +33,16 @@ func (d *Distinct) RowIter(ctx *sql.Context) (sql.RowIter, error) { return nil, err } - return sql.NewSpanIter(span, newDistinctIter(it)), nil + return sql.NewSpanIter(span, newDistinctIter(ctx, it)), nil } -// TransformUp implements the Transformable interface. -func (d *Distinct) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - child, err := d.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Node interface. +func (d *Distinct) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(d, len(children), 1) } - return f(NewDistinct(child)) -} -// TransformExpressionsUp implements the Transformable interface. -func (d *Distinct) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { - child, err := d.Child.TransformExpressionsUp(f) - if err != nil { - return nil, err - } - return NewDistinct(child), nil + return NewDistinct(children[0]), nil } func (d Distinct) String() string { @@ -64,18 +54,21 @@ func (d Distinct) String() string { // distinctIter keeps track of the hashes of all rows that have been emitted. // It does not emit any rows whose hashes have been seen already. -// TODO: come up with a way to use less memory than keeping all hashes in mem. +// TODO: come up with a way to use less memory than keeping all hashes in memory. // Even though they are just 64-bit integers, this could be a problem in large // result sets. type distinctIter struct { childIter sql.RowIter - seen map[uint64]struct{} + seen sql.KeyValueCache + dispose sql.DisposeFunc } -func newDistinctIter(child sql.RowIter) *distinctIter { +func newDistinctIter(ctx *sql.Context, child sql.RowIter) *distinctIter { + cache, dispose := ctx.Memory.NewHistoryCache() return &distinctIter{ childIter: child, - seen: make(map[uint64]struct{}), + seen: cache, + dispose: dispose, } } @@ -83,29 +76,38 @@ func (di *distinctIter) Next() (sql.Row, error) { for { row, err := di.childIter.Next() if err != nil { + if err == io.EOF { + di.Dispose() + } return nil, err } - hash, err := hashstructure.Hash(row, nil) - if err != nil { - return nil, fmt.Errorf("unable to hash row: %s", err) + hash := sql.CacheKey(row) + if _, err := di.seen.Get(hash); err == nil { + continue } - if _, ok := di.seen[hash]; ok { - continue + if err := di.seen.Put(hash, struct{}{}); err != nil { + return nil, err } - di.seen[hash] = struct{}{} return row, nil } } func (di *distinctIter) Close() error { + di.Dispose() return di.childIter.Close() } +func (di *distinctIter) Dispose() { + if di.dispose != nil { + di.dispose() + } +} + // OrderedDistinct is a Distinct node optimized for sorted row sets. -// It's 2 orders of magnitude faster and uses 2 orders of magnitude less mem. +// It's 2 orders of magnitude faster and uses 2 orders of magnitude less memory. type OrderedDistinct struct { UnaryNode } @@ -135,22 +137,13 @@ func (d *OrderedDistinct) RowIter(ctx *sql.Context) (sql.RowIter, error) { return sql.NewSpanIter(span, newOrderedDistinctIter(it, d.Child.Schema())), nil } -// TransformUp implements the Transformable interface. -func (d *OrderedDistinct) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - child, err := d.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Node interface. +func (d *OrderedDistinct) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(d, len(children), 1) } - return f(NewOrderedDistinct(child)) -} -// TransformExpressionsUp implements the Transformable interface. -func (d *OrderedDistinct) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { - child, err := d.Child.TransformExpressionsUp(f) - if err != nil { - return nil, err - } - return NewOrderedDistinct(child), nil + return NewOrderedDistinct(children[0]), nil } func (d OrderedDistinct) String() string { diff --git a/sql/plan/distinct_test.go b/sql/plan/distinct_test.go index 7f7522449..098308d47 100644 --- a/sql/plan/distinct_test.go +++ b/sql/plan/distinct_test.go @@ -4,10 +4,10 @@ import ( "io" "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestDistinct(t *testing.T) { @@ -18,7 +18,7 @@ func TestDistinct(t *testing.T) { {Name: "name", Type: sql.Text, Nullable: true}, {Name: "email", Type: sql.Text, Nullable: true}, } - child := mem.NewTable("test", childSchema) + child := memory.NewTable("test", childSchema) rows := []sql.Row{ sql.NewRow("john", "john@doe.com"), @@ -65,7 +65,7 @@ func TestOrderedDistinct(t *testing.T) { {Name: "name", Type: sql.Text, Nullable: true}, {Name: "email", Type: sql.Text, Nullable: true}, } - child := mem.NewTable("test", childSchema) + child := memory.NewTable("test", childSchema) rows := []sql.Row{ sql.NewRow("jane", "jane@doe.com"), diff --git a/sql/plan/drop_index.go b/sql/plan/drop_index.go index 8b6066ada..f08caf711 100644 --- a/sql/plan/drop_index.go +++ b/sql/plan/drop_index.go @@ -1,8 +1,9 @@ package plan import ( - errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/internal/similartext" + "github.com/src-d/go-mysql-server/sql" + "gopkg.in/src-d/go-errors.v1" ) var ( @@ -51,9 +52,15 @@ func (d *DropIndex) RowIter(ctx *sql.Context) (sql.RowIter, error) { return nil, ErrTableNotNameable.New() } - table, ok := db.Tables()[n.Name()] + tables := db.Tables() + table, ok := tables[n.Name()] if !ok { - return nil, sql.ErrTableNotFound.New(n.Name()) + if len(tables) == 0 { + return nil, sql.ErrTableNotFound.New(n.Name()) + } + + similar := similartext.FindFromMap(tables, n.Name()) + return nil, sql.ErrTableNotFound.New(n.Name() + similar) } index := d.Catalog.Index(db.Name(), d.Name) @@ -62,7 +69,7 @@ func (d *DropIndex) RowIter(ctx *sql.Context) (sql.RowIter, error) { } d.Catalog.ReleaseIndex(index) - if !d.Catalog.CanUseIndex(index) { + if !d.Catalog.CanRemoveIndex(index) { return nil, ErrIndexNotAvailable.New(d.Name) } @@ -97,26 +104,13 @@ func (d *DropIndex) String() string { return pr.String() } -// TransformExpressionsUp implements the Node interface. -func (d *DropIndex) TransformExpressionsUp(fn sql.TransformExprFunc) (sql.Node, error) { - t, err := d.Table.TransformExpressionsUp(fn) - if err != nil { - return nil, err - } - - nc := *d - nc.Table = t - return &nc, nil -} - -// TransformUp implements the Node interface. -func (d *DropIndex) TransformUp(fn sql.TransformNodeFunc) (sql.Node, error) { - t, err := d.Table.TransformUp(fn) - if err != nil { - return nil, err +// WithChildren implements the Node interface. +func (d *DropIndex) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(d, len(children), 1) } - nc := *d - nc.Table = t - return fn(&nc) + nd := *d + nd.Table = children[0] + return &nd, nil } diff --git a/sql/plan/drop_index_test.go b/sql/plan/drop_index_test.go index 7c1c810e4..cd2e214fe 100644 --- a/sql/plan/drop_index_test.go +++ b/sql/plan/drop_index_test.go @@ -4,16 +4,16 @@ import ( "testing" "time" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestDeleteIndex(t *testing.T) { require := require.New(t) - table := mem.NewTable("foo", sql.Schema{ + table := memory.NewTable("foo", sql.Schema{ {Name: "a", Source: "foo"}, {Name: "b", Source: "foo"}, {Name: "c", Source: "foo"}, @@ -22,7 +22,7 @@ func TestDeleteIndex(t *testing.T) { driver := new(mockDriver) catalog := sql.NewCatalog() catalog.RegisterIndexDriver(driver) - db := mem.NewDatabase("foo") + db := memory.NewDatabase("foo") db.AddTable("foo", table) catalog.AddDatabase(db) @@ -56,7 +56,7 @@ func TestDeleteIndex(t *testing.T) { func TestDeleteIndexNotReady(t *testing.T) { require := require.New(t) - table := mem.NewTable("foo", sql.Schema{ + table := memory.NewTable("foo", sql.Schema{ {Name: "a", Source: "foo"}, {Name: "b", Source: "foo"}, {Name: "c", Source: "foo"}, @@ -65,7 +65,7 @@ func TestDeleteIndexNotReady(t *testing.T) { driver := new(mockDriver) catalog := sql.NewCatalog() catalog.RegisterIndexDriver(driver) - db := mem.NewDatabase("foo") + db := memory.NewDatabase("foo") db.AddTable("foo", table) catalog.AddDatabase(db) @@ -97,3 +97,47 @@ func TestDeleteIndexNotReady(t *testing.T) { close(done) <-ready } + +func TestDeleteIndexOutdated(t *testing.T) { + require := require.New(t) + + table := memory.NewTable("foo", sql.Schema{ + {Name: "a", Source: "foo"}, + {Name: "b", Source: "foo"}, + {Name: "c", Source: "foo"}, + }) + + driver := new(mockDriver) + catalog := sql.NewCatalog() + catalog.RegisterIndexDriver(driver) + db := memory.NewDatabase("foo") + db.AddTable("foo", table) + catalog.AddDatabase(db) + + var expressions = []sql.Expression{ + expression.NewGetFieldWithTable(0, sql.Int64, "foo", "c", true), + expression.NewGetFieldWithTable(1, sql.Int64, "foo", "a", true), + } + + done, ready, err := catalog.AddIndex(&mockIndex{id: "idx", db: "foo", table: "foo", exprs: expressions}) + require.NoError(err) + close(done) + <-ready + + idx := catalog.Index("foo", "idx") + require.NotNil(idx) + catalog.ReleaseIndex(idx) + catalog.MarkOutdated(idx) + + di := NewDropIndex("idx", NewResolvedTable(table)) + di.Catalog = catalog + di.CurrentDatabase = "foo" + + _, err = di.RowIter(sql.NewEmptyContext()) + require.NoError(err) + + time.Sleep(50 * time.Millisecond) + + require.Equal([]string{"idx"}, driver.deleted) + require.Nil(catalog.Index("foo", "idx")) +} diff --git a/sql/plan/empty_table.go b/sql/plan/empty_table.go index e2c9aa98b..198cef41d 100644 --- a/sql/plan/empty_table.go +++ b/sql/plan/empty_table.go @@ -1,6 +1,6 @@ package plan -import "gopkg.in/src-d/go-mysql-server.v0/sql" +import "github.com/src-d/go-mysql-server/sql" // EmptyTable is a node representing an empty table. var EmptyTable = new(emptyTable) @@ -16,12 +16,11 @@ func (emptyTable) RowIter(ctx *sql.Context) (sql.RowIter, error) { return sql.RowsToRowIter(), nil } -// TransformUp implements the Transformable interface. -func (e *emptyTable) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - return f(e) -} +// WithChildren implements the Node interface. +func (e *emptyTable) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(e, len(children), 0) + } -// TransformExpressionsUp implements the Transformable interface. -func (e *emptyTable) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { return e, nil } diff --git a/sql/plan/exchange.go b/sql/plan/exchange.go index bed9f115b..3eb59808d 100644 --- a/sql/plan/exchange.go +++ b/sql/plan/exchange.go @@ -6,8 +6,8 @@ import ( "io" "sync" + "github.com/src-d/go-mysql-server/sql" errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) // ErrNoPartitionable is returned when no Partitionable node is found @@ -61,24 +61,13 @@ func (e *Exchange) String() string { return p.String() } -// TransformUp implements the sql.Node interface. -func (e *Exchange) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - child, err := e.Child.TransformUp(f) - if err != nil { - return nil, err - } - - return f(NewExchange(e.Parallelism, child)) -} - -// TransformExpressionsUp implements the sql.Node interface. -func (e *Exchange) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { - child, err := e.Child.TransformExpressionsUp(f) - if err != nil { - return nil, err +// WithChildren implements the Node interface. +func (e *Exchange) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(e, len(children), 1) } - return NewExchange(e.Parallelism, child), nil + return NewExchange(e.Parallelism, children[0]), nil } type exchangeRowIter struct { @@ -86,12 +75,14 @@ type exchangeRowIter struct { parallelism int partitions sql.PartitionIter tree sql.Node - mut sync.Mutex - tokens chan struct{} + mut sync.RWMutex + tokensChan chan struct{} started bool rows chan sql.Row err chan error - quit chan struct{} + + quitMut sync.RWMutex + quitChan chan struct{} } func newExchangeRowIter( @@ -108,7 +99,7 @@ func newExchangeRowIter( started: false, tree: tree, partitions: iter, - quit: make(chan struct{}), + quitChan: make(chan struct{}), } } @@ -116,8 +107,8 @@ func (it *exchangeRowIter) releaseToken() { it.mut.Lock() defer it.mut.Unlock() - if it.tokens != nil { - it.tokens <- struct{}{} + if it.tokensChan != nil { + it.tokensChan <- struct{}{} } } @@ -125,17 +116,23 @@ func (it *exchangeRowIter) closeTokens() { it.mut.Lock() defer it.mut.Unlock() - close(it.tokens) - it.tokens = nil + close(it.tokensChan) + it.tokensChan = nil +} + +func (it *exchangeRowIter) tokens() chan struct{} { + it.mut.RLock() + defer it.mut.RUnlock() + return it.tokensChan } func (it *exchangeRowIter) fillTokens() { it.mut.Lock() defer it.mut.Unlock() - it.tokens = make(chan struct{}, it.parallelism) + it.tokensChan = make(chan struct{}, it.parallelism) for i := 0; i < it.parallelism; i++ { - it.tokens <- struct{}{} + it.tokensChan <- struct{}{} } } @@ -153,7 +150,7 @@ func (it *exchangeRowIter) start() { it.err <- context.Canceled it.closeTokens() return - case <-it.quit: + case <-it.quit(): it.closeTokens() return case p, ok := <-partitions: @@ -178,11 +175,11 @@ func (it *exchangeRowIter) start() { func (it *exchangeRowIter) iterPartitions(ch chan<- sql.Partition) { defer func() { - close(ch) - - if err := it.partitions.Close(); err != nil { - it.err <- err + if x := recover(); x != nil { + it.err <- fmt.Errorf("mysql_server caught panic:\n%v", x) } + + close(ch) }() for { @@ -190,9 +187,9 @@ func (it *exchangeRowIter) iterPartitions(ch chan<- sql.Partition) { case <-it.ctx.Done(): it.err <- context.Canceled return - case <-it.quit: + case <-it.quit(): return - case <-it.tokens: + case <-it.tokens(): } p, err := it.partitions.Next() @@ -208,7 +205,7 @@ func (it *exchangeRowIter) iterPartitions(ch chan<- sql.Partition) { } func (it *exchangeRowIter) iterPartition(p sql.Partition) { - node, err := it.tree.TransformUp(func(n sql.Node) (sql.Node, error) { + node, err := TransformUp(it.tree, func(n sql.Node) (sql.Node, error) { if t, ok := n.(sql.Table); ok { return &exchangePartition{p, t}, nil } @@ -237,7 +234,7 @@ func (it *exchangeRowIter) iterPartition(p sql.Partition) { case <-it.ctx.Done(): it.err <- context.Canceled return - case <-it.quit: + case <-it.quit(): return default: } @@ -274,12 +271,24 @@ func (it *exchangeRowIter) Next() (sql.Row, error) { } } +func (it *exchangeRowIter) quit() chan struct{} { + it.quitMut.RLock() + defer it.quitMut.RUnlock() + return it.quitChan +} + func (it *exchangeRowIter) Close() error { - if it.quit == nil { - return nil + it.quitMut.Lock() + if it.quitChan != nil { + close(it.quitChan) + it.quitChan = nil + } + it.quitMut.Unlock() + + if it.partitions != nil { + return it.partitions.Close() } - close(it.quit) return nil } @@ -306,10 +315,11 @@ func (p *exchangePartition) Schema() sql.Schema { return p.table.Schema() } -func (p *exchangePartition) TransformExpressionsUp(sql.TransformExprFunc) (sql.Node, error) { - return p, nil -} +// WithChildren implements the Node interface. +func (p *exchangePartition) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(p, len(children), 0) + } -func (p *exchangePartition) TransformUp(sql.TransformNodeFunc) (sql.Node, error) { return p, nil } diff --git a/sql/plan/exchange_test.go b/sql/plan/exchange_test.go index be172f05a..5a8e8e317 100644 --- a/sql/plan/exchange_test.go +++ b/sql/plan/exchange_test.go @@ -6,9 +6,9 @@ import ( "io" "testing" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestExchange(t *testing.T) { @@ -90,17 +90,28 @@ func TestExchangeCancelled(t *testing.T) { require.Equal(context.Canceled, err) } +func TestExchangePanicRecover(t *testing.T) { + ctx := sql.NewContext(context.Background()) + it := &partitionPanic{} + ex := newExchangeRowIter(ctx, 1, it, nil) + ex.start() + it.Close() + + require.True(t, it.closed) +} + type partitionable struct { sql.Node partitions int rowsPerPartition int } -func (p *partitionable) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - return f(p) -} +// WithChildren implements the Node interface. +func (p *partitionable) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(p, len(children), 0) + } -func (p *partitionable) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { return p, nil } @@ -165,3 +176,17 @@ func (r *partitionRows) Close() error { r.num = -1 return nil } + +type partitionPanic struct { + sql.Partition + closed bool +} + +func (*partitionPanic) Next() (sql.Partition, error) { + panic("partitionPanic.Next") +} + +func (p *partitionPanic) Close() error { + p.closed = true + return nil +} diff --git a/sql/plan/filter.go b/sql/plan/filter.go index ad3bae275..f160b3f66 100644 --- a/sql/plan/filter.go +++ b/sql/plan/filter.go @@ -1,7 +1,7 @@ package plan import ( - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // Filter skips rows that don't match a certain expression. @@ -36,28 +36,22 @@ func (p *Filter) RowIter(ctx *sql.Context) (sql.RowIter, error) { return sql.NewSpanIter(span, NewFilterIter(ctx, p.Expression, i)), nil } -// TransformUp implements the Transformable interface. -func (p *Filter) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - child, err := p.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Node interface. +func (p *Filter) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(p, len(children), 1) } - return f(NewFilter(p.Expression, child)) + + return NewFilter(p.Expression, children[0]), nil } -// TransformExpressionsUp implements the Transformable interface. -func (p *Filter) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { - expr, err := p.Expression.TransformUp(f) - if err != nil { - return nil, err +// WithExpressions implements the Expressioner interface. +func (p *Filter) WithExpressions(exprs ...sql.Expression) (sql.Node, error) { + if len(exprs) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(p, len(exprs), 1) } - child, err := p.Child.TransformExpressionsUp(f) - if err != nil { - return nil, err - } - - return NewFilter(expr, child), nil + return NewFilter(exprs[0], p.Child), nil } func (p *Filter) String() string { @@ -72,16 +66,6 @@ func (p *Filter) Expressions() []sql.Expression { return []sql.Expression{p.Expression} } -// TransformExpressions implements the Expressioner interface. -func (p *Filter) TransformExpressions(f sql.TransformExprFunc) (sql.Node, error) { - e, err := p.Expression.TransformUp(f) - if err != nil { - return nil, err - } - - return NewFilter(e, p.Child), nil -} - // FilterIter is an iterator that filters another iterator and skips rows that // don't match the given condition. type FilterIter struct { @@ -107,12 +91,12 @@ func (i *FilterIter) Next() (sql.Row, error) { return nil, err } - result, err := i.cond.Eval(i.ctx, row) + ok, err := sql.EvaluateCondition(i.ctx, i.cond, row) if err != nil { return nil, err } - if result == true { + if ok { return row, nil } } diff --git a/sql/plan/filter_test.go b/sql/plan/filter_test.go index 42dc63cdb..de5912fe7 100644 --- a/sql/plan/filter_test.go +++ b/sql/plan/filter_test.go @@ -3,10 +3,10 @@ package plan import ( "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestFilter(t *testing.T) { @@ -19,7 +19,7 @@ func TestFilter(t *testing.T) { {Name: "col3", Type: sql.Int32, Nullable: true}, {Name: "col4", Type: sql.Int64, Nullable: true}, } - child := mem.NewTable("test", childSchema) + child := memory.NewTable("test", childSchema) rows := []sql.Row{ sql.NewRow("col1_1", "col2_1", int32(1111), int64(2222)), diff --git a/sql/plan/generate.go b/sql/plan/generate.go new file mode 100644 index 000000000..c259b8d8d --- /dev/null +++ b/sql/plan/generate.go @@ -0,0 +1,134 @@ +package plan + +import ( + "fmt" + "io" + + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" +) + +// Generate will explode rows using a generator. +type Generate struct { + UnaryNode + Column *expression.GetField +} + +// NewGenerate creates a new generate node. +func NewGenerate(child sql.Node, col *expression.GetField) *Generate { + return &Generate{UnaryNode{child}, col} +} + +// Schema implements the sql.Node interface. +func (g *Generate) Schema() sql.Schema { + s := g.Child.Schema() + col := s[g.Column.Index()] + s[g.Column.Index()] = &sql.Column{ + Name: g.Column.Name(), + Type: sql.UnderlyingType(col.Type), + Nullable: col.Nullable, + } + return s +} + +// RowIter implements the sql.Node interface. +func (g *Generate) RowIter(ctx *sql.Context) (sql.RowIter, error) { + span, ctx := ctx.Span("plan.Generate") + + childIter, err := g.Child.RowIter(ctx) + if err != nil { + return nil, err + } + + return sql.NewSpanIter(span, &generateIter{ + child: childIter, + idx: g.Column.Index(), + }), nil +} + +// Expressions implements the Expressioner interface. +func (g *Generate) Expressions() []sql.Expression { return []sql.Expression{g.Column} } + +// WithChildren implements the Node interface. +func (g *Generate) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(g, len(children), 1) + } + + return NewGenerate(children[0], g.Column), nil +} + +// WithExpressions implements the Expressioner interface. +func (g *Generate) WithExpressions(exprs ...sql.Expression) (sql.Node, error) { + if len(exprs) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(g, len(exprs), 1) + } + + gf, ok := exprs[0].(*expression.GetField) + if !ok { + return nil, fmt.Errorf("Generate expects child to be expression.GetField, but is %T", exprs[0]) + } + + return NewGenerate(g.Child, gf), nil +} + +func (g *Generate) String() string { + tp := sql.NewTreePrinter() + _ = tp.WriteNode("Generate(%s)", g.Column) + _ = tp.WriteChildren(g.Child.String()) + return tp.String() +} + +type generateIter struct { + child sql.RowIter + idx int + + gen sql.Generator + row sql.Row +} + +func (i *generateIter) Next() (sql.Row, error) { + for { + if i.gen == nil { + var err error + i.row, err = i.child.Next() + if err != nil { + return nil, err + } + + i.gen, err = sql.ToGenerator(i.row[i.idx]) + if err != nil { + return nil, err + } + } + + val, err := i.gen.Next() + if err != nil { + if err == io.EOF { + if err := i.gen.Close(); err != nil { + return nil, err + } + + i.gen = nil + continue + } + return nil, err + } + + var row = make(sql.Row, len(i.row)) + copy(row, i.row) + row[i.idx] = val + return row, nil + } +} + +func (i *generateIter) Close() error { + if i.gen != nil { + if err := i.gen.Close(); err != nil { + _ = i.child.Close() + return err + } + } + + return i.child.Close() +} diff --git a/sql/plan/generate_test.go b/sql/plan/generate_test.go new file mode 100644 index 000000000..7f32db68c --- /dev/null +++ b/sql/plan/generate_test.go @@ -0,0 +1,85 @@ +package plan + +import ( + "testing" + + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/stretchr/testify/require" +) + +func TestGenerateRowIter(t *testing.T) { + require := require.New(t) + + child := newFakeNode( + sql.Schema{ + {Name: "a", Type: sql.Text, Source: "foo"}, + {Name: "b", Type: sql.Array(sql.Text), Source: "foo"}, + {Name: "c", Type: sql.Int64, Source: "foo"}, + }, + sql.RowsToRowIter( + sql.Row{"first", sql.NewArrayGenerator([]interface{}{"a", "b"}), int64(1)}, + sql.Row{"second", sql.NewArrayGenerator([]interface{}{"c", "d"}), int64(2)}, + ), + ) + + iter, err := NewGenerate( + child, + expression.NewGetFieldWithTable(1, sql.Array(sql.Text), "foo", "b", false), + ).RowIter(sql.NewEmptyContext()) + require.NoError(err) + + rows, err := sql.RowIterToRows(iter) + require.NoError(err) + + expected := []sql.Row{ + {"first", "a", int64(1)}, + {"first", "b", int64(1)}, + {"second", "c", int64(2)}, + {"second", "d", int64(2)}, + } + + require.Equal(expected, rows) +} + +func TestGenerateSchema(t *testing.T) { + require := require.New(t) + + schema := NewGenerate( + newFakeNode( + sql.Schema{ + {Name: "a", Type: sql.Text, Source: "foo"}, + {Name: "b", Type: sql.Array(sql.Text), Source: "foo"}, + {Name: "c", Type: sql.Int64, Source: "foo"}, + }, + nil, + ), + expression.NewGetField(1, sql.Array(sql.Text), "foobar", false), + ).Schema() + + expected := sql.Schema{ + {Name: "a", Type: sql.Text, Source: "foo"}, + {Name: "foobar", Type: sql.Text}, + {Name: "c", Type: sql.Int64, Source: "foo"}, + } + + require.Equal(expected, schema) +} + +type fakeNode struct { + schema sql.Schema + iter sql.RowIter +} + +func newFakeNode(s sql.Schema, iter sql.RowIter) *fakeNode { + return &fakeNode{s, iter} +} + +func (n *fakeNode) Children() []sql.Node { return nil } +func (n *fakeNode) Resolved() bool { return true } +func (n *fakeNode) Schema() sql.Schema { return n.schema } +func (n *fakeNode) RowIter(*sql.Context) (sql.RowIter, error) { return n.iter, nil } +func (n *fakeNode) String() string { return "fakeNode" } +func (*fakeNode) WithChildren(children ...sql.Node) (sql.Node, error) { + panic("placeholder") +} diff --git a/sql/plan/group_by.go b/sql/plan/group_by.go index cd0b294d7..4a9fb76b0 100644 --- a/sql/plan/group_by.go +++ b/sql/plan/group_by.go @@ -7,9 +7,9 @@ import ( "strings" opentracing "github.com/opentracing/opentracing-go" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) // ErrGroupBy is returned when the aggregation is not supported. @@ -28,7 +28,6 @@ func NewGroupBy( grouping []sql.Expression, child sql.Node, ) *GroupBy { - return &GroupBy{ UnaryNode: UnaryNode{Child: child}, Aggregate: aggregate, @@ -93,33 +92,34 @@ func (p *GroupBy) RowIter(ctx *sql.Context) (sql.RowIter, error) { return sql.NewSpanIter(span, iter), nil } -// TransformUp implements the Transformable interface. -func (p *GroupBy) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - child, err := p.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Node interface. +func (p *GroupBy) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(p, len(children), 1) } - return f(NewGroupBy(p.Aggregate, p.Grouping, child)) + + return NewGroupBy(p.Aggregate, p.Grouping, children[0]), nil } -// TransformExpressionsUp implements the Transformable interface. -func (p *GroupBy) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { - aggregate, err := transformExpressionsUp(f, p.Aggregate) - if err != nil { - return nil, err +// WithExpressions implements the Node interface. +func (p *GroupBy) WithExpressions(exprs ...sql.Expression) (sql.Node, error) { + expected := len(p.Aggregate) + len(p.Grouping) + if len(exprs) != expected { + return nil, sql.ErrInvalidChildrenNumber.New(p, len(exprs), expected) } - grouping, err := transformExpressionsUp(f, p.Grouping) - if err != nil { - return nil, err + var agg = make([]sql.Expression, len(p.Aggregate)) + for i := 0; i < len(p.Aggregate); i++ { + agg[i] = exprs[i] } - child, err := p.Child.TransformExpressionsUp(f) - if err != nil { - return nil, err + var grouping = make([]sql.Expression, len(p.Grouping)) + offset := len(p.Aggregate) + for i := 0; i < len(p.Grouping); i++ { + grouping[i] = exprs[i+offset] } - return NewGroupBy(aggregate, grouping, child), nil + return NewGroupBy(agg, grouping, p.Child), nil } func (p *GroupBy) String() string { @@ -152,21 +152,6 @@ func (p *GroupBy) Expressions() []sql.Expression { return exprs } -// TransformExpressions implements the Expressioner interface. -func (p *GroupBy) TransformExpressions(f sql.TransformExprFunc) (sql.Node, error) { - agg, err := transformExpressionsUp(f, p.Aggregate) - if err != nil { - return nil, err - } - - group, err := transformExpressionsUp(f, p.Grouping) - if err != nil { - return nil, err - } - - return NewGroupBy(agg, group, p.Child), nil -} - type groupByIter struct { aggregate []sql.Expression child sql.RowIter @@ -220,11 +205,12 @@ func (i *groupByIter) Close() error { type groupByGroupingIter struct { aggregate []sql.Expression grouping []sql.Expression - aggregation map[uint64][]sql.Row + aggregation sql.KeyValueCache keys []uint64 pos int child sql.RowIter ctx *sql.Context + dispose sql.DisposeFunc } func newGroupByGroupingIter( @@ -242,7 +228,7 @@ func newGroupByGroupingIter( func (i *groupByGroupingIter) Next() (sql.Row, error) { if i.aggregation == nil { - i.aggregation = make(map[uint64][]sql.Row) + i.aggregation, i.dispose = i.ctx.Memory.NewHistoryCache() if err := i.compute(); err != nil { return nil, err } @@ -252,9 +238,12 @@ func (i *groupByGroupingIter) Next() (sql.Row, error) { return nil, io.EOF } - buffers := i.aggregation[i.keys[i.pos]] + buffers, err := i.aggregation.Get(i.keys[i.pos]) + if err != nil { + return nil, err + } i.pos++ - return evalBuffers(i.ctx, buffers, i.aggregate) + return evalBuffers(i.ctx, buffers.([]sql.Row), i.aggregate) } func (i *groupByGroupingIter) compute() error { @@ -272,16 +261,25 @@ func (i *groupByGroupingIter) compute() error { return err } - if _, ok := i.aggregation[key]; !ok { + if _, err := i.aggregation.Get(key); err != nil { var buf = make([]sql.Row, len(i.aggregate)) for j, a := range i.aggregate { buf[j] = fillBuffer(a) } - i.aggregation[key] = buf + + if err := i.aggregation.Put(key, buf); err != nil { + return err + } + i.keys = append(i.keys, key) } - err = updateBuffers(i.ctx, i.aggregation[key], i.aggregate, row) + b, err := i.aggregation.Get(key) + if err != nil { + return err + } + + err = updateBuffers(i.ctx, b.([]sql.Row), i.aggregate, row) if err != nil { return err } @@ -353,15 +351,13 @@ func updateBuffer( return n.Update(ctx, buffers[idx], row) case *expression.Alias: return updateBuffer(ctx, buffers, idx, n.Child, row) - case *expression.GetField: + default: val, err := expr.Eval(ctx, row) if err != nil { return err } buffers[idx] = sql.NewRow(val) return nil - default: - return ErrGroupBy.New(n.String()) } } @@ -393,12 +389,10 @@ func evalBuffer( return n.Eval(ctx, buffer) case *expression.Alias: return evalBuffer(ctx, n.Child, buffer) - case *expression.GetField: + default: if len(buffer) > 0 { return buffer[0], nil } return nil, nil - default: - return nil, ErrGroupBy.New(n.String()) } } diff --git a/sql/plan/group_by_test.go b/sql/plan/group_by_test.go index 06441106a..69bbb83a5 100644 --- a/sql/plan/group_by_test.go +++ b/sql/plan/group_by_test.go @@ -3,17 +3,17 @@ package plan import ( "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/src-d/go-mysql-server/sql/expression/function/aggregation" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/aggregation" ) -func TestGroupBy_Schema(t *testing.T) { +func TestGroupBySchema(t *testing.T) { require := require.New(t) - child := mem.NewTable("test", nil) + child := memory.NewTable("test", nil) agg := []sql.Expression{ expression.NewAlias(expression.NewLiteral("s", sql.Text), "c1"), expression.NewAlias(aggregation.NewCount(expression.NewStar()), "c2"), @@ -21,14 +21,14 @@ func TestGroupBy_Schema(t *testing.T) { gb := NewGroupBy(agg, nil, NewResolvedTable(child)) require.Equal(sql.Schema{ {Name: "c1", Type: sql.Text}, - {Name: "c2", Type: sql.Int32}, + {Name: "c2", Type: sql.Int64}, }, gb.Schema()) } -func TestGroupBy_Resolved(t *testing.T) { +func TestGroupByResolved(t *testing.T) { require := require.New(t) - child := mem.NewTable("test", nil) + child := memory.NewTable("test", nil) agg := []sql.Expression{ expression.NewAlias(aggregation.NewCount(expression.NewStar()), "c2"), } @@ -42,7 +42,7 @@ func TestGroupBy_Resolved(t *testing.T) { require.False(gb.Resolved()) } -func TestGroupBy_RowIter(t *testing.T) { +func TestGroupByRowIter(t *testing.T) { require := require.New(t) ctx := sql.NewEmptyContext() @@ -50,7 +50,7 @@ func TestGroupBy_RowIter(t *testing.T) { {Name: "col1", Type: sql.Text}, {Name: "col2", Type: sql.Int64}, } - child := mem.NewTable("test", childSchema) + child := memory.NewTable("test", childSchema) rows := []sql.Row{ sql.NewRow("col1_1", int64(1111)), @@ -96,7 +96,7 @@ func TestGroupBy_RowIter(t *testing.T) { require.Equal(sql.NewRow("col1_2", int64(4444)), rows[1]) } -func TestGroupBy_EvalEmptyBuffer(t *testing.T) { +func TestGroupByEvalEmptyBuffer(t *testing.T) { require := require.New(t) ctx := sql.NewEmptyContext() @@ -105,7 +105,7 @@ func TestGroupBy_EvalEmptyBuffer(t *testing.T) { require.Nil(r) } -func TestGroupBy_Error(t *testing.T) { +func TestGroupByAggregationGrouping(t *testing.T) { require := require.New(t) ctx := sql.NewEmptyContext() @@ -114,7 +114,7 @@ func TestGroupBy_Error(t *testing.T) { {Name: "col2", Type: sql.Int64}, } - child := mem.NewTable("test", childSchema) + child := memory.NewTable("test", childSchema) rows := []sql.Row{ sql.NewRow("col1_1", int64(1111)), @@ -140,8 +140,15 @@ func TestGroupBy_Error(t *testing.T) { NewResolvedTable(child), ) - _, err := sql.NodeToRows(ctx, p) - require.Error(err) + rows, err := sql.NodeToRows(ctx, p) + require.NoError(err) + + expected := []sql.Row{ + {int64(3), false}, + {int64(2), false}, + } + + require.Equal(expected, rows) } func BenchmarkGroupBy(b *testing.B) { @@ -201,7 +208,7 @@ func benchmarkTable(t testing.TB) sql.Table { t.Helper() require := require.New(t) - table := mem.NewTable("test", sql.Schema{ + table := memory.NewTable("test", sql.Schema{ {Name: "a", Type: sql.Int64}, {Name: "b", Type: sql.Int64}, }) diff --git a/sql/plan/having.go b/sql/plan/having.go new file mode 100644 index 000000000..48a8bde25 --- /dev/null +++ b/sql/plan/having.go @@ -0,0 +1,62 @@ +package plan + +import "github.com/src-d/go-mysql-server/sql" + +// Having node is a filter that supports aggregate expressions. A having node +// is identical to a filter node in behaviour. The difference is that some +// analyzer rules work specifically on having clauses and not filters. For +// that reason, Having is a completely new node instead of using just filter. +type Having struct { + UnaryNode + Cond sql.Expression +} + +var _ sql.Expressioner = (*Having)(nil) + +// NewHaving creates a new having node. +func NewHaving(cond sql.Expression, child sql.Node) *Having { + return &Having{UnaryNode{Child: child}, cond} +} + +// Resolved implements the sql.Node interface. +func (h *Having) Resolved() bool { return h.Cond.Resolved() && h.Child.Resolved() } + +// Expressions implements the sql.Expressioner interface. +func (h *Having) Expressions() []sql.Expression { return []sql.Expression{h.Cond} } + +// WithChildren implements the Node interface. +func (h *Having) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(h, len(children), 1) + } + + return NewHaving(h.Cond, children[0]), nil +} + +// WithExpressions implements the Expressioner interface. +func (h *Having) WithExpressions(exprs ...sql.Expression) (sql.Node, error) { + if len(exprs) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(h, len(exprs), 1) + } + + return NewHaving(exprs[0], h.Child), nil +} + +// RowIter implements the sql.Node interface. +func (h *Having) RowIter(ctx *sql.Context) (sql.RowIter, error) { + span, ctx := ctx.Span("plan.Having") + iter, err := h.Child.RowIter(ctx) + if err != nil { + span.Finish() + return nil, err + } + + return sql.NewSpanIter(span, NewFilterIter(ctx, h.Cond, iter)), nil +} + +func (h *Having) String() string { + p := sql.NewTreePrinter() + _ = p.WriteNode("Having(%s)", h.Cond) + _ = p.WriteChildren(h.Child.String()) + return p.String() +} diff --git a/sql/plan/having_test.go b/sql/plan/having_test.go new file mode 100644 index 000000000..9651a631a --- /dev/null +++ b/sql/plan/having_test.go @@ -0,0 +1,95 @@ +package plan + +import ( + "testing" + + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/stretchr/testify/require" +) + +func TestHaving(t *testing.T) { + require := require.New(t) + ctx := sql.NewEmptyContext() + + childSchema := sql.Schema{ + {Name: "col1", Type: sql.Text, Nullable: true}, + {Name: "col2", Type: sql.Text, Nullable: true}, + {Name: "col3", Type: sql.Int32, Nullable: true}, + {Name: "col4", Type: sql.Int64, Nullable: true}, + } + child := memory.NewTable("test", childSchema) + + rows := []sql.Row{ + sql.NewRow("col1_1", "col2_1", int32(1111), int64(2222)), + sql.NewRow("col1_2", "col2_2", int32(3333), int64(4444)), + sql.NewRow("col1_3", "col2_3", nil, int64(4444)), + } + + for _, r := range rows { + require.NoError(child.Insert(sql.NewEmptyContext(), r)) + } + + f := NewHaving( + expression.NewEquals( + expression.NewGetField(0, sql.Text, "col1", true), + expression.NewLiteral("col1_1", sql.Text)), + NewResolvedTable(child), + ) + + require.Equal(1, len(f.Children())) + + iter, err := f.RowIter(ctx) + require.NoError(err) + require.NotNil(iter) + + row, err := iter.Next() + require.NoError(err) + require.NotNil(row) + + require.Equal("col1_1", row[0]) + require.Equal("col2_1", row[1]) + + row, err = iter.Next() + require.NotNil(err) + require.Nil(row) + + f = NewHaving( + expression.NewEquals( + expression.NewGetField(2, sql.Int32, "col3", true), + expression.NewLiteral(int32(1111), sql.Int32), + ), + NewResolvedTable(child), + ) + + iter, err = f.RowIter(ctx) + require.NoError(err) + require.NotNil(iter) + + row, err = iter.Next() + require.NoError(err) + require.NotNil(row) + + require.Equal(int32(1111), row[2]) + require.Equal(int64(2222), row[3]) + + f = NewHaving( + expression.NewEquals( + expression.NewGetField(3, sql.Int64, "col4", true), + expression.NewLiteral(int64(4444), sql.Int64), + ), + NewResolvedTable(child), + ) + + iter, err = f.RowIter(ctx) + require.NoError(err) + require.NotNil(iter) + + row, err = iter.Next() + require.NoError(err) + require.NotNil(row) + + require.Equal(int32(3333), row[2]) + require.Equal(int64(4444), row[3]) +} diff --git a/sql/plan/innerjoin.go b/sql/plan/innerjoin.go deleted file mode 100644 index 5645ade09..000000000 --- a/sql/plan/innerjoin.go +++ /dev/null @@ -1,303 +0,0 @@ -package plan - -import ( - "io" - "os" - "reflect" - - opentracing "github.com/opentracing/opentracing-go" - "gopkg.in/src-d/go-mysql-server.v0/sql" -) - -const experimentalInMemoryJoinKey = "EXPERIMENTAL_IN_MEMORY_JOIN" -const inMemoryJoinSessionVar = "inmemory_joins" - -var useInMemoryJoins = os.Getenv(experimentalInMemoryJoinKey) != "" - -// InnerJoin is an inner join between two tables. -type InnerJoin struct { - BinaryNode - Cond sql.Expression -} - -// NewInnerJoin creates a new inner join node from two tables. -func NewInnerJoin(left, right sql.Node, cond sql.Expression) *InnerJoin { - return &InnerJoin{ - BinaryNode: BinaryNode{ - Left: left, - Right: right, - }, - Cond: cond, - } -} - -// Schema implements the Node interface. -func (j *InnerJoin) Schema() sql.Schema { - return append(j.Left.Schema(), j.Right.Schema()...) -} - -// Resolved implements the Resolvable interface. -func (j *InnerJoin) Resolved() bool { - return j.Left.Resolved() && j.Right.Resolved() && j.Cond.Resolved() -} - -// RowIter implements the Node interface. -func (j *InnerJoin) RowIter(ctx *sql.Context) (sql.RowIter, error) { - var left, right string - if leftTable, ok := j.Left.(sql.Nameable); ok { - left = leftTable.Name() - } else { - left = reflect.TypeOf(j.Left).String() - } - - if rightTable, ok := j.Right.(sql.Nameable); ok { - right = rightTable.Name() - } else { - right = reflect.TypeOf(j.Right).String() - } - - span, ctx := ctx.Span("plan.InnerJoin", opentracing.Tags{ - "left": left, - "right": right, - }) - - l, err := j.Left.RowIter(ctx) - if err != nil { - span.Finish() - return nil, err - } - - var inMemorySession bool - _, val := ctx.Get(inMemoryJoinSessionVar) - if val != nil { - inMemorySession = true - } - - var iter sql.RowIter - if useInMemoryJoins || inMemorySession { - r, err := j.Right.RowIter(ctx) - if err != nil { - span.Finish() - return nil, err - } - - iter = &innerJoinMemoryIter{ - l: l, - r: r, - ctx: ctx, - cond: j.Cond, - } - } else { - iter = &innerJoinIter{ - l: l, - rp: j.Right, - ctx: ctx, - cond: j.Cond, - } - } - - return sql.NewSpanIter(span, iter), nil -} - -// TransformUp implements the Transformable interface. -func (j *InnerJoin) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - left, err := j.Left.TransformUp(f) - if err != nil { - return nil, err - } - - right, err := j.Right.TransformUp(f) - if err != nil { - return nil, err - } - - return f(NewInnerJoin(left, right, j.Cond)) -} - -// TransformExpressionsUp implements the Transformable interface. -func (j *InnerJoin) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { - left, err := j.Left.TransformExpressionsUp(f) - if err != nil { - return nil, err - } - - right, err := j.Right.TransformExpressionsUp(f) - if err != nil { - return nil, err - } - - cond, err := j.Cond.TransformUp(f) - if err != nil { - return nil, err - } - - return NewInnerJoin(left, right, cond), nil -} - -func (j *InnerJoin) String() string { - pr := sql.NewTreePrinter() - _ = pr.WriteNode("InnerJoin(%s)", j.Cond) - _ = pr.WriteChildren(j.Left.String(), j.Right.String()) - return pr.String() -} - -// Expressions implements the Expressioner interface. -func (j *InnerJoin) Expressions() []sql.Expression { - return []sql.Expression{j.Cond} -} - -// TransformExpressions implements the Expressioner interface. -func (j *InnerJoin) TransformExpressions(f sql.TransformExprFunc) (sql.Node, error) { - cond, err := j.Cond.TransformUp(f) - if err != nil { - return nil, err - } - - return NewInnerJoin(j.Left, j.Right, cond), nil -} - -type innerJoinIter struct { - l sql.RowIter - rp rowIterProvider - r sql.RowIter - ctx *sql.Context - cond sql.Expression - - leftRow sql.Row -} - -func (i *innerJoinIter) Next() (sql.Row, error) { - for { - if i.leftRow == nil { - r, err := i.l.Next() - if err != nil { - return nil, err - } - - i.leftRow = r - } - - if i.r == nil { - iter, err := i.rp.RowIter(i.ctx) - if err != nil { - return nil, err - } - - i.r = iter - } - - rightRow, err := i.r.Next() - if err == io.EOF { - i.r = nil - i.leftRow = nil - continue - } - - if err != nil { - return nil, err - } - - var row = make(sql.Row, len(i.leftRow)+len(rightRow)) - copy(row, i.leftRow) - copy(row[len(i.leftRow):], rightRow) - - v, err := i.cond.Eval(i.ctx, row) - if err != nil { - return nil, err - } - - if v == true { - return row, nil - } - } -} - -func (i *innerJoinIter) Close() error { - if err := i.l.Close(); err != nil { - if i.r != nil { - _ = i.r.Close() - } - return err - } - - if i.r != nil { - return i.r.Close() - } - - return nil -} - -type innerJoinMemoryIter struct { - l sql.RowIter - r sql.RowIter - ctx *sql.Context - cond sql.Expression - pos int - leftRow sql.Row - right []sql.Row -} - -func (i *innerJoinMemoryIter) Next() (sql.Row, error) { - for { - if i.leftRow == nil { - r, err := i.l.Next() - if err != nil { - return nil, err - } - - i.leftRow = r - } - - if i.r != nil { - for { - row, err := i.r.Next() - if err != nil { - if err == io.EOF { - break - } - return nil, err - } - - i.right = append(i.right, row) - } - i.r = nil - } - - if i.pos >= len(i.right) { - i.pos = 0 - i.leftRow = nil - continue - } - - rightRow := i.right[i.pos] - var row = make(sql.Row, len(i.leftRow)+len(rightRow)) - copy(row, i.leftRow) - copy(row[len(i.leftRow):], rightRow) - - i.pos++ - - v, err := i.cond.Eval(i.ctx, row) - if err != nil { - return nil, err - } - - if v == true { - return row, nil - } - } -} - -func (i *innerJoinMemoryIter) Close() error { - if err := i.l.Close(); err != nil { - if i.r != nil { - _ = i.r.Close() - } - return err - } - - if i.r != nil { - return i.r.Close() - } - - return nil -} diff --git a/sql/plan/innerjoin_test.go b/sql/plan/innerjoin_test.go deleted file mode 100644 index 51ca6f46a..000000000 --- a/sql/plan/innerjoin_test.go +++ /dev/null @@ -1,186 +0,0 @@ -package plan - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" -) - -func TestInnerJoin(t *testing.T) { - require := require.New(t) - finalSchema := append(lSchema, rSchema...) - - ltable := mem.NewTable("left", lSchema) - rtable := mem.NewTable("right", rSchema) - insertData(t, ltable) - insertData(t, rtable) - - j := NewInnerJoin( - NewResolvedTable(ltable), - NewResolvedTable(rtable), - expression.NewEquals( - expression.NewGetField(0, sql.Text, "lcol1", false), - expression.NewGetField(4, sql.Text, "rcol1", false), - )) - - require.Equal(finalSchema, j.Schema()) - - rows := collectRows(t, j) - require.Len(rows, 2) - - require.Equal([]sql.Row{ - {"col1_1", "col2_1", int32(1111), int64(2222), "col1_1", "col2_1", int32(1111), int64(2222)}, - {"col1_2", "col2_2", int32(3333), int64(4444), "col1_2", "col2_2", int32(3333), int64(4444)}, - }, rows) -} - -func TestInMemoryInnerJoin(t *testing.T) { - require := require.New(t) - finalSchema := append(lSchema, rSchema...) - - ltable := mem.NewTable("left", lSchema) - rtable := mem.NewTable("right", rSchema) - insertData(t, ltable) - insertData(t, rtable) - - j := NewInnerJoin( - NewResolvedTable(ltable), - NewResolvedTable(rtable), - expression.NewEquals( - expression.NewGetField(0, sql.Text, "lcol1", false), - expression.NewGetField(4, sql.Text, "rcol1", false), - )) - - require.Equal(finalSchema, j.Schema()) - - ctx := sql.NewEmptyContext() - ctx.Set(inMemoryJoinSessionVar, sql.Text, "true") - - iter, err := j.RowIter(ctx) - require.NoError(err) - - rows, err := sql.RowIterToRows(iter) - require.NoError(err) - require.Len(rows, 2) - - require.Equal([]sql.Row{ - {"col1_1", "col2_1", int32(1111), int64(2222), "col1_1", "col2_1", int32(1111), int64(2222)}, - {"col1_2", "col2_2", int32(3333), int64(4444), "col1_2", "col2_2", int32(3333), int64(4444)}, - }, rows) -} - -func TestInnerJoinEmpty(t *testing.T) { - require := require.New(t) - ctx := sql.NewEmptyContext() - - ltable := mem.NewTable("left", lSchema) - rtable := mem.NewTable("right", rSchema) - - j := NewInnerJoin( - NewResolvedTable(ltable), - NewResolvedTable(rtable), - expression.NewEquals( - expression.NewGetField(0, sql.Text, "lcol1", false), - expression.NewGetField(4, sql.Text, "rcol1", false), - )) - - iter, err := j.RowIter(ctx) - require.NoError(err) - - assertRows(t, iter, 0) -} - -func BenchmarkInnerJoin(b *testing.B) { - t1 := mem.NewTable("foo", sql.Schema{ - {Name: "a", Source: "foo", Type: sql.Int64}, - {Name: "b", Source: "foo", Type: sql.Text}, - }) - - t2 := mem.NewTable("bar", sql.Schema{ - {Name: "a", Source: "bar", Type: sql.Int64}, - {Name: "b", Source: "bar", Type: sql.Text}, - }) - - for i := 0; i < 5; i++ { - t1.Insert(sql.NewEmptyContext(), sql.NewRow(int64(i), fmt.Sprintf("t1_%d", i))) - t2.Insert(sql.NewEmptyContext(), sql.NewRow(int64(i), fmt.Sprintf("t2_%d", i))) - } - - n1 := NewInnerJoin( - NewResolvedTable(t1), - NewResolvedTable(t2), - expression.NewEquals( - expression.NewGetField(0, sql.Int64, "a", false), - expression.NewGetField(2, sql.Int64, "a", false), - ), - ) - - n2 := NewFilter( - expression.NewEquals( - expression.NewGetField(0, sql.Int64, "a", false), - expression.NewGetField(2, sql.Int64, "a", false), - ), - NewCrossJoin( - NewResolvedTable(t1), - NewResolvedTable(t2), - ), - ) - - expected := []sql.Row{ - {int64(0), "t1_0", int64(0), "t2_0"}, - {int64(1), "t1_1", int64(1), "t2_1"}, - {int64(2), "t1_2", int64(2), "t2_2"}, - {int64(3), "t1_3", int64(3), "t2_3"}, - {int64(4), "t1_4", int64(4), "t2_4"}, - } - - ctx := sql.NewEmptyContext() - b.Run("inner join", func(b *testing.B) { - require := require.New(b) - - for i := 0; i < b.N; i++ { - iter, err := n1.RowIter(ctx) - require.NoError(err) - - rows, err := sql.RowIterToRows(iter) - require.NoError(err) - - require.Equal(expected, rows) - } - }) - - b.Run("in memory inner join", func(b *testing.B) { - useInMemoryJoins = true - require := require.New(b) - - for i := 0; i < b.N; i++ { - iter, err := n1.RowIter(ctx) - require.NoError(err) - - rows, err := sql.RowIterToRows(iter) - require.NoError(err) - - require.Equal(expected, rows) - } - - useInMemoryJoins = false - }) - - b.Run("cross join with filter", func(b *testing.B) { - require := require.New(b) - - for i := 0; i < b.N; i++ { - iter, err := n2.RowIter(ctx) - require.NoError(err) - - rows, err := sql.RowIterToRows(iter) - require.NoError(err) - - require.Equal(expected, rows) - } - }) -} diff --git a/sql/plan/insert.go b/sql/plan/insert.go index 5df3f7b56..06e638d26 100644 --- a/sql/plan/insert.go +++ b/sql/plan/insert.go @@ -1,28 +1,36 @@ package plan import ( + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "gopkg.in/src-d/go-errors.v1" "io" "strings" - - "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) // ErrInsertIntoNotSupported is thrown when a table doesn't support inserts var ErrInsertIntoNotSupported = errors.NewKind("table doesn't support INSERT INTO") +var ErrReplaceIntoNotSupported = errors.NewKind("table doesn't support REPLACE INTO") +var ErrInsertIntoMismatchValueCount = errors.NewKind("number of values does not match number of columns provided") +var ErrInsertIntoUnsupportedValues = errors.NewKind("%T is unsupported for inserts") +var ErrInsertIntoDuplicateColumn = errors.NewKind("duplicate column name %v") +var ErrInsertIntoNonexistentColumn = errors.NewKind("invalid column name %v") +var ErrInsertIntoNonNullableDefaultNullColumn = errors.NewKind("column name '%v' is non-nullable but attempted to set default value of null") +var ErrInsertIntoNonNullableProvidedNull = errors.NewKind("column name '%v' is non-nullable but attempted to set a value of null") // InsertInto is a node describing the insertion into some table. type InsertInto struct { BinaryNode - Columns []string + Columns []string + IsReplace bool } // NewInsertInto creates an InsertInto node. -func NewInsertInto(dst, src sql.Node, cols []string) *InsertInto { +func NewInsertInto(dst, src sql.Node, isReplace bool, cols []string) *InsertInto { return &InsertInto{ BinaryNode: BinaryNode{Left: dst, Right: src}, Columns: cols, + IsReplace: isReplace, } } @@ -65,8 +73,36 @@ func (p *InsertInto) Execute(ctx *sql.Context) (int, error) { return 0, err } + var replaceable sql.Replacer + if p.IsReplace { + var ok bool + replaceable, ok = insertable.(sql.Replacer) + if !ok { + return 0, ErrReplaceIntoNotSupported.New() + } + } + dstSchema := p.Left.Schema() projExprs := make([]sql.Expression, len(dstSchema)) + + // If no columns are given, we assume the full schema in order + if len(p.Columns) == 0 { + p.Columns = make([]string, len(dstSchema)) + for i, f := range dstSchema { + p.Columns[i] = f.Name + } + } else { + err = p.validateColumns(ctx, dstSchema) + if err != nil { + return 0, err + } + } + + err = p.validateValueCount(ctx) + if err != nil { + return 0, err + } + for i, f := range dstSchema { found := false for j, col := range p.Columns { @@ -78,8 +114,10 @@ func (p *InsertInto) Execute(ctx *sql.Context) (int, error) { } if !found { - def, _ := f.Type.Convert(nil) - projExprs[i] = expression.NewLiteral(def, f.Type) + if !f.Nullable && f.Default == nil { + return 0, ErrInsertIntoNonNullableDefaultNullColumn.New(f.Name) + } + projExprs[i] = expression.NewLiteral(f.Default, f.Type) } } @@ -96,17 +134,51 @@ func (p *InsertInto) Execute(ctx *sql.Context) (int, error) { if err == io.EOF { break } - if err != nil { _ = iter.Close() return i, err } - if err := insertable.Insert(ctx, row); err != nil { + err = p.validateNullability(ctx, dstSchema, row) + if err != nil { _ = iter.Close() return i, err } + // Convert integer values in row to specified type in schema + for colIdx, oldValue := range row { + dstColType := projExprs[colIdx].Type() + + if sql.IsInteger(dstColType) && oldValue != nil { + newValue, err := dstColType.Convert(oldValue) + if err != nil { + return i, err + } + + row[colIdx] = newValue + } + } + + if replaceable != nil { + if err = replaceable.Delete(ctx, row); err != nil { + if err != sql.ErrDeleteRowNotFound { + _ = iter.Close() + return i, err + } + } else { + i++ + } + + if err = replaceable.Insert(ctx, row); err != nil { + _ = iter.Close() + return i, err + } + } else { + if err := insertable.Insert(ctx, row); err != nil { + _ = iter.Close() + return i, err + } + } i++ } @@ -123,39 +195,64 @@ func (p *InsertInto) RowIter(ctx *sql.Context) (sql.RowIter, error) { return sql.RowsToRowIter(sql.NewRow(int64(n))), nil } -// TransformUp implements the Transformable interface. -func (p *InsertInto) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - left, err := p.Left.TransformUp(f) - if err != nil { - return nil, err - } - - right, err := p.Right.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Node interface. +func (p *InsertInto) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 2 { + return nil, sql.ErrInvalidChildrenNumber.New(p, len(children), 2) } - return f(NewInsertInto(left, right, p.Columns)) + return NewInsertInto(children[0], children[1], p.IsReplace, p.Columns), nil } -// TransformExpressionsUp implements the Transformable interface. -func (p *InsertInto) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { - left, err := p.Left.TransformExpressionsUp(f) - if err != nil { - return nil, err +func (p InsertInto) String() string { + pr := sql.NewTreePrinter() + if p.IsReplace { + _ = pr.WriteNode("Replace(%s)", strings.Join(p.Columns, ", ")) + } else { + _ = pr.WriteNode("Insert(%s)", strings.Join(p.Columns, ", ")) } + _ = pr.WriteChildren(p.Left.String(), p.Right.String()) + return pr.String() +} - right, err := p.Right.TransformExpressionsUp(f) - if err != nil { - return nil, err +func (p *InsertInto) validateValueCount(ctx *sql.Context) error { + switch node := p.Right.(type) { + case *Values: + for _, exprTuple := range node.ExpressionTuples { + if len(exprTuple) != len(p.Columns) { + return ErrInsertIntoMismatchValueCount.New() + } + } + default: + return ErrInsertIntoUnsupportedValues.New(node) } + return nil +} - return NewInsertInto(left, right, p.Columns), nil +func (p *InsertInto) validateColumns(ctx *sql.Context, dstSchema sql.Schema) error { + dstColNames := make(map[string]struct{}) + for _, dstCol := range dstSchema { + dstColNames[dstCol.Name] = struct{}{} + } + columnNames := make(map[string]struct{}) + for _, columnName := range p.Columns { + if _, exists := dstColNames[columnName]; !exists { + return ErrInsertIntoNonexistentColumn.New(columnName) + } + if _, exists := columnNames[columnName]; !exists { + columnNames[columnName] = struct{}{} + } else { + return ErrInsertIntoDuplicateColumn.New(columnName) + } + } + return nil } -func (p InsertInto) String() string { - pr := sql.NewTreePrinter() - _ = pr.WriteNode("Insert(%s)", strings.Join(p.Columns, ", ")) - _ = pr.WriteChildren(p.Left.String(), p.Right.String()) - return pr.String() +func (p *InsertInto) validateNullability(ctx *sql.Context, dstSchema sql.Schema, row sql.Row) error { + for i, col := range dstSchema { + if !col.Nullable && row[i] == nil { + return ErrInsertIntoNonNullableProvidedNull.New(col.Name) + } + } + return nil } diff --git a/sql/plan/join.go b/sql/plan/join.go new file mode 100644 index 000000000..97f1abc13 --- /dev/null +++ b/sql/plan/join.go @@ -0,0 +1,549 @@ +package plan + +import ( + "io" + "os" + "reflect" + "strings" + + opentracing "github.com/opentracing/opentracing-go" + "github.com/src-d/go-mysql-server/sql" +) + +const ( + inMemoryJoinKey = "INMEMORY_JOINS" + inMemoryJoinSessionVar = "inmemory_joins" +) + +var useInMemoryJoins = shouldUseMemoryJoinsByEnv() + +func shouldUseMemoryJoinsByEnv() bool { + v := strings.TrimSpace(strings.ToLower(os.Getenv(inMemoryJoinKey))) + return v == "on" || v == "1" +} + +// InnerJoin is an inner join between two tables. +type InnerJoin struct { + BinaryNode + Cond sql.Expression +} + +// NewInnerJoin creates a new inner join node from two tables. +func NewInnerJoin(left, right sql.Node, cond sql.Expression) *InnerJoin { + return &InnerJoin{ + BinaryNode: BinaryNode{ + Left: left, + Right: right, + }, + Cond: cond, + } +} + +// Schema implements the Node interface. +func (j *InnerJoin) Schema() sql.Schema { + return append(j.Left.Schema(), j.Right.Schema()...) +} + +// Resolved implements the Resolvable interface. +func (j *InnerJoin) Resolved() bool { + return j.Left.Resolved() && j.Right.Resolved() && j.Cond.Resolved() +} + +// RowIter implements the Node interface. +func (j *InnerJoin) RowIter(ctx *sql.Context) (sql.RowIter, error) { + return joinRowIter(ctx, innerJoin, j.Left, j.Right, j.Cond) +} + +// WithChildren implements the Node interface. +func (j *InnerJoin) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 2 { + return nil, sql.ErrInvalidChildrenNumber.New(j, len(children), 2) + } + + return NewInnerJoin(children[0], children[1], j.Cond), nil +} + +// WithExpressions implements the Expressioner interface. +func (j *InnerJoin) WithExpressions(exprs ...sql.Expression) (sql.Node, error) { + if len(exprs) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(j, len(exprs), 1) + } + + return NewInnerJoin(j.Left, j.Right, exprs[0]), nil +} + +func (j *InnerJoin) String() string { + pr := sql.NewTreePrinter() + _ = pr.WriteNode("InnerJoin(%s)", j.Cond) + _ = pr.WriteChildren(j.Left.String(), j.Right.String()) + return pr.String() +} + +// Expressions implements the Expressioner interface. +func (j *InnerJoin) Expressions() []sql.Expression { + return []sql.Expression{j.Cond} +} + +// LeftJoin is a left join between two tables. +type LeftJoin struct { + BinaryNode + Cond sql.Expression +} + +// NewLeftJoin creates a new left join node from two tables. +func NewLeftJoin(left, right sql.Node, cond sql.Expression) *LeftJoin { + return &LeftJoin{ + BinaryNode: BinaryNode{ + Left: left, + Right: right, + }, + Cond: cond, + } +} + +// Schema implements the Node interface. +func (j *LeftJoin) Schema() sql.Schema { + return append(j.Left.Schema(), makeNullable(j.Right.Schema())...) +} + +// Resolved implements the Resolvable interface. +func (j *LeftJoin) Resolved() bool { + return j.Left.Resolved() && j.Right.Resolved() && j.Cond.Resolved() +} + +// RowIter implements the Node interface. +func (j *LeftJoin) RowIter(ctx *sql.Context) (sql.RowIter, error) { + return joinRowIter(ctx, leftJoin, j.Left, j.Right, j.Cond) +} + +// WithChildren implements the Node interface. +func (j *LeftJoin) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 2 { + return nil, sql.ErrInvalidChildrenNumber.New(j, len(children), 1) + } + + return NewLeftJoin(children[0], children[1], j.Cond), nil +} + +// WithExpressions implements the Expressioner interface. +func (j *LeftJoin) WithExpressions(exprs ...sql.Expression) (sql.Node, error) { + if len(exprs) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(j, len(exprs), 1) + } + + return NewLeftJoin(j.Left, j.Right, exprs[0]), nil +} + +func (j *LeftJoin) String() string { + pr := sql.NewTreePrinter() + _ = pr.WriteNode("LeftJoin(%s)", j.Cond) + _ = pr.WriteChildren(j.Left.String(), j.Right.String()) + return pr.String() +} + +// Expressions implements the Expressioner interface. +func (j *LeftJoin) Expressions() []sql.Expression { + return []sql.Expression{j.Cond} +} + +// RightJoin is a left join between two tables. +type RightJoin struct { + BinaryNode + Cond sql.Expression +} + +// NewRightJoin creates a new right join node from two tables. +func NewRightJoin(left, right sql.Node, cond sql.Expression) *RightJoin { + return &RightJoin{ + BinaryNode: BinaryNode{ + Left: left, + Right: right, + }, + Cond: cond, + } +} + +// Schema implements the Node interface. +func (j *RightJoin) Schema() sql.Schema { + return append(makeNullable(j.Left.Schema()), j.Right.Schema()...) +} + +// Resolved implements the Resolvable interface. +func (j *RightJoin) Resolved() bool { + return j.Left.Resolved() && j.Right.Resolved() && j.Cond.Resolved() +} + +// RowIter implements the Node interface. +func (j *RightJoin) RowIter(ctx *sql.Context) (sql.RowIter, error) { + return joinRowIter(ctx, rightJoin, j.Left, j.Right, j.Cond) +} + +// WithChildren implements the Node interface. +func (j *RightJoin) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 2 { + return nil, sql.ErrInvalidChildrenNumber.New(j, len(children), 2) + } + + return NewRightJoin(children[0], children[1], j.Cond), nil +} + +// WithExpressions implements the Expressioner interface. +func (j *RightJoin) WithExpressions(exprs ...sql.Expression) (sql.Node, error) { + if len(exprs) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(j, len(exprs), 1) + } + + return NewRightJoin(j.Left, j.Right, exprs[0]), nil +} + +func (j *RightJoin) String() string { + pr := sql.NewTreePrinter() + _ = pr.WriteNode("RightJoin(%s)", j.Cond) + _ = pr.WriteChildren(j.Left.String(), j.Right.String()) + return pr.String() +} + +// Expressions implements the Expressioner interface. +func (j *RightJoin) Expressions() []sql.Expression { + return []sql.Expression{j.Cond} +} + +type joinType byte + +const ( + innerJoin joinType = iota + leftJoin + rightJoin +) + +func (t joinType) String() string { + switch t { + case innerJoin: + return "InnerJoin" + case leftJoin: + return "LeftJoin" + case rightJoin: + return "RightJoin" + default: + return "INVALID" + } +} + +func joinRowIter( + ctx *sql.Context, + typ joinType, + left, right sql.Node, + cond sql.Expression, +) (sql.RowIter, error) { + var leftName, rightName string + if leftTable, ok := left.(sql.Nameable); ok { + leftName = leftTable.Name() + } else { + leftName = reflect.TypeOf(left).String() + } + + if rightTable, ok := right.(sql.Nameable); ok { + rightName = rightTable.Name() + } else { + rightName = reflect.TypeOf(right).String() + } + + span, ctx := ctx.Span("plan."+typ.String(), opentracing.Tags{ + "left": leftName, + "right": rightName, + }) + + var inMemorySession bool + _, val := ctx.Get(inMemoryJoinSessionVar) + if val != nil { + inMemorySession = true + } + + var mode = unknownMode + if useInMemoryJoins || inMemorySession { + mode = memoryMode + } + + cache, dispose := ctx.Memory.NewRowsCache() + if typ == rightJoin { + r, err := right.RowIter(ctx) + if err != nil { + span.Finish() + return nil, err + } + return sql.NewSpanIter(span, &joinIter{ + typ: typ, + primary: r, + secondaryProvider: left, + ctx: ctx, + cond: cond, + mode: mode, + secondaryRows: cache, + dispose: dispose, + }), nil + } + + l, err := left.RowIter(ctx) + if err != nil { + span.Finish() + return nil, err + } + + return sql.NewSpanIter(span, &joinIter{ + typ: typ, + primary: l, + secondaryProvider: right, + ctx: ctx, + cond: cond, + mode: mode, + secondaryRows: cache, + dispose: dispose, + }), nil +} + +// joinMode defines the mode in which a join will be performed. +type joinMode byte + +const ( + // unknownMode is the default mode. It will start iterating without really + // knowing in which mode it will end up computing the join. If it + // iterates the right side fully one time and so far it fits in memory, + // then it will switch to memory mode. Otherwise, if at some point during + // this first iteration it finds that it does not fit in memory, will + // switch to multipass mode. + unknownMode joinMode = iota + // memoryMode computes all the join directly in memory iterating each + // side of the join exactly once. + memoryMode + // multipassMode computes the join by iterating the left side once, + // and the right side one time for each row in the left side. + multipassMode +) + +// joinIter is a generic iterator for all join types. +type joinIter struct { + typ joinType + primary sql.RowIter + secondaryProvider rowIterProvider + secondary sql.RowIter + ctx *sql.Context + cond sql.Expression + + primaryRow sql.Row + foundMatch bool + rowSize int + + // used to compute in-memory + mode joinMode + secondaryRows sql.RowsCache + pos int + dispose sql.DisposeFunc +} + +func (i *joinIter) Dispose() { + if i.dispose != nil { + i.dispose() + i.dispose = nil + } +} + +func (i *joinIter) loadPrimary() error { + if i.primaryRow == nil { + r, err := i.primary.Next() + if err != nil { + if err == io.EOF { + i.Dispose() + } + return err + } + + i.primaryRow = r + i.foundMatch = false + } + + return nil +} + +func (i *joinIter) loadSecondaryInMemory() error { + iter, err := i.secondaryProvider.RowIter(i.ctx) + if err != nil { + return err + } + + for { + row, err := iter.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + + if err := i.secondaryRows.Add(row); err != nil { + return err + } + } + + if len(i.secondaryRows.Get()) == 0 { + return io.EOF + } + + return nil +} + +func (i *joinIter) loadSecondary() (row sql.Row, err error) { + if i.mode == memoryMode { + if len(i.secondaryRows.Get()) == 0 { + if err = i.loadSecondaryInMemory(); err != nil { + return nil, err + } + } + + if i.pos >= len(i.secondaryRows.Get()) { + i.primaryRow = nil + i.pos = 0 + return nil, io.EOF + } + + row := i.secondaryRows.Get()[i.pos] + i.pos++ + return row, nil + } + + if i.secondary == nil { + var iter sql.RowIter + iter, err = i.secondaryProvider.RowIter(i.ctx) + if err != nil { + return nil, err + } + + i.secondary = iter + } + + rightRow, err := i.secondary.Next() + if err != nil { + if err == io.EOF { + i.secondary = nil + i.primaryRow = nil + + // If we got to this point and the mode is still unknown it means + // the right side fits in memory, so the mode changes to memory + // join. + if i.mode == unknownMode { + i.mode = memoryMode + } + + return nil, io.EOF + } + return nil, err + } + + if i.mode == unknownMode { + var switchToMultipass bool + if !i.ctx.Memory.HasAvailable() { + switchToMultipass = true + } else { + err := i.secondaryRows.Add(rightRow) + if err != nil && !sql.ErrNoMemoryAvailable.Is(err) { + return nil, err + } + } + + if switchToMultipass { + i.Dispose() + i.secondaryRows = nil + i.mode = multipassMode + } + } + + return rightRow, nil +} + +func (i *joinIter) Next() (sql.Row, error) { + for { + if err := i.loadPrimary(); err != nil { + return nil, err + } + + primary := i.primaryRow + secondary, err := i.loadSecondary() + if err != nil { + if err == io.EOF { + if !i.foundMatch && (i.typ == leftJoin || i.typ == rightJoin) { + return i.buildRow(primary, nil), nil + } + continue + } + return nil, err + } + + row := i.buildRow(primary, secondary) + v, err := i.cond.Eval(i.ctx, row) + if err != nil { + return nil, err + } + + if v == false { + continue + } + + i.foundMatch = true + return row, nil + } +} + +// buildRow builds the resulting row using the rows from the primary and +// secondary branches depending on the join type. +func (i *joinIter) buildRow(primary, secondary sql.Row) sql.Row { + var row sql.Row + if i.rowSize > 0 { + row = make(sql.Row, i.rowSize) + } else { + row = make(sql.Row, len(primary)+len(secondary)) + i.rowSize = len(row) + } + + switch i.typ { + case rightJoin: + copy(row, secondary) + copy(row[i.rowSize-len(primary):], primary) + default: + copy(row, primary) + copy(row[len(primary):], secondary) + } + + return row +} + +func (i *joinIter) Close() (err error) { + i.Dispose() + i.secondary = nil + + if i.primary != nil { + if err = i.primary.Close(); err != nil { + if i.secondary != nil { + _ = i.secondary.Close() + } + return err + } + + } + + if i.secondary != nil { + err = i.secondary.Close() + } + + return err +} + +// makeNullable will return a copy of the received columns, but all of them +// will be turned into nullable columns. +func makeNullable(cols []*sql.Column) []*sql.Column { + var result = make([]*sql.Column, len(cols)) + for i := 0; i < len(cols); i++ { + col := *cols[i] + col.Nullable = true + result[i] = &col + } + return result +} diff --git a/sql/plan/join_test.go b/sql/plan/join_test.go new file mode 100644 index 000000000..07797602b --- /dev/null +++ b/sql/plan/join_test.go @@ -0,0 +1,288 @@ +package plan + +import ( + "context" + "fmt" + "testing" + + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/stretchr/testify/require" +) + +func TestJoinSchema(t *testing.T) { + t1 := NewResolvedTable(memory.NewTable("foo", sql.Schema{ + {Name: "a", Source: "foo", Type: sql.Int64}, + })) + + t2 := NewResolvedTable(memory.NewTable("bar", sql.Schema{ + {Name: "b", Source: "bar", Type: sql.Int64}, + })) + + t.Run("inner", func(t *testing.T) { + j := NewInnerJoin(t1, t2, nil) + result := j.Schema() + + require.Equal(t, sql.Schema{ + {Name: "a", Source: "foo", Type: sql.Int64}, + {Name: "b", Source: "bar", Type: sql.Int64}, + }, result) + }) + + t.Run("left", func(t *testing.T) { + j := NewLeftJoin(t1, t2, nil) + result := j.Schema() + + require.Equal(t, sql.Schema{ + {Name: "a", Source: "foo", Type: sql.Int64}, + {Name: "b", Source: "bar", Type: sql.Int64, Nullable: true}, + }, result) + }) + + t.Run("right", func(t *testing.T) { + j := NewRightJoin(t1, t2, nil) + result := j.Schema() + + require.Equal(t, sql.Schema{ + {Name: "a", Source: "foo", Type: sql.Int64, Nullable: true}, + {Name: "b", Source: "bar", Type: sql.Int64}, + }, result) + }) +} + +func TestInnerJoin(t *testing.T) { + testInnerJoin(t, sql.NewEmptyContext()) +} + +func TestInMemoryInnerJoin(t *testing.T) { + ctx := sql.NewEmptyContext() + ctx.Set(inMemoryJoinSessionVar, sql.Text, "true") + testInnerJoin(t, ctx) +} + +func TestMultiPassInnerJoin(t *testing.T) { + ctx := sql.NewContext(context.TODO(), sql.WithMemoryManager( + sql.NewMemoryManager(mockReporter{2, 1}), + )) + testInnerJoin(t, ctx) +} + +func testInnerJoin(t *testing.T, ctx *sql.Context) { + t.Helper() + + require := require.New(t) + ltable := memory.NewTable("left", lSchema) + rtable := memory.NewTable("right", rSchema) + insertData(t, ltable) + insertData(t, rtable) + + j := NewInnerJoin( + NewResolvedTable(ltable), + NewResolvedTable(rtable), + expression.NewEquals( + expression.NewGetField(0, sql.Text, "lcol1", false), + expression.NewGetField(4, sql.Text, "rcol1", false), + )) + + rows := collectRows(t, j) + require.Len(rows, 2) + + require.Equal([]sql.Row{ + {"col1_1", "col2_1", int32(1), int64(2), "col1_1", "col2_1", int32(1), int64(2)}, + {"col1_2", "col2_2", int32(3), int64(4), "col1_2", "col2_2", int32(3), int64(4)}, + }, rows) +} +func TestInnerJoinEmpty(t *testing.T) { + require := require.New(t) + ctx := sql.NewEmptyContext() + + ltable := memory.NewTable("left", lSchema) + rtable := memory.NewTable("right", rSchema) + + j := NewInnerJoin( + NewResolvedTable(ltable), + NewResolvedTable(rtable), + expression.NewEquals( + expression.NewGetField(0, sql.Text, "lcol1", false), + expression.NewGetField(4, sql.Text, "rcol1", false), + )) + + iter, err := j.RowIter(ctx) + require.NoError(err) + + assertRows(t, iter, 0) +} + +func BenchmarkInnerJoin(b *testing.B) { + t1 := memory.NewTable("foo", sql.Schema{ + {Name: "a", Source: "foo", Type: sql.Int64}, + {Name: "b", Source: "foo", Type: sql.Text}, + }) + + t2 := memory.NewTable("bar", sql.Schema{ + {Name: "a", Source: "bar", Type: sql.Int64}, + {Name: "b", Source: "bar", Type: sql.Text}, + }) + + for i := 0; i < 5; i++ { + t1.Insert(sql.NewEmptyContext(), sql.NewRow(int64(i), fmt.Sprintf("t1_%d", i))) + t2.Insert(sql.NewEmptyContext(), sql.NewRow(int64(i), fmt.Sprintf("t2_%d", i))) + } + + n1 := NewInnerJoin( + NewResolvedTable(t1), + NewResolvedTable(t2), + expression.NewEquals( + expression.NewGetField(0, sql.Int64, "a", false), + expression.NewGetField(2, sql.Int64, "a", false), + ), + ) + + n2 := NewFilter( + expression.NewEquals( + expression.NewGetField(0, sql.Int64, "a", false), + expression.NewGetField(2, sql.Int64, "a", false), + ), + NewCrossJoin( + NewResolvedTable(t1), + NewResolvedTable(t2), + ), + ) + + expected := []sql.Row{ + {int64(0), "t1_0", int64(0), "t2_0"}, + {int64(1), "t1_1", int64(1), "t2_1"}, + {int64(2), "t1_2", int64(2), "t2_2"}, + {int64(3), "t1_3", int64(3), "t2_3"}, + {int64(4), "t1_4", int64(4), "t2_4"}, + } + + ctx := sql.NewContext(context.TODO(), sql.WithMemoryManager( + sql.NewMemoryManager(mockReporter{1, 5}), + )) + b.Run("inner join", func(b *testing.B) { + require := require.New(b) + + for i := 0; i < b.N; i++ { + iter, err := n1.RowIter(ctx) + require.NoError(err) + + rows, err := sql.RowIterToRows(iter) + require.NoError(err) + + require.Equal(expected, rows) + } + }) + + b.Run("in memory inner join", func(b *testing.B) { + useInMemoryJoins = true + require := require.New(b) + + for i := 0; i < b.N; i++ { + iter, err := n1.RowIter(ctx) + require.NoError(err) + + rows, err := sql.RowIterToRows(iter) + require.NoError(err) + + require.Equal(expected, rows) + } + + useInMemoryJoins = false + }) + + b.Run("within memory threshold", func(b *testing.B) { + require := require.New(b) + + for i := 0; i < b.N; i++ { + iter, err := n1.RowIter(ctx) + require.NoError(err) + + rows, err := sql.RowIterToRows(iter) + require.NoError(err) + + require.Equal(expected, rows) + } + }) + + b.Run("cross join with filter", func(b *testing.B) { + require := require.New(b) + + for i := 0; i < b.N; i++ { + iter, err := n2.RowIter(ctx) + require.NoError(err) + + rows, err := sql.RowIterToRows(iter) + require.NoError(err) + + require.Equal(expected, rows) + } + }) +} + +func TestLeftJoin(t *testing.T) { + require := require.New(t) + + ltable := memory.NewTable("left", lSchema) + rtable := memory.NewTable("right", rSchema) + insertData(t, ltable) + insertData(t, rtable) + + j := NewLeftJoin( + NewResolvedTable(ltable), + NewResolvedTable(rtable), + expression.NewEquals( + expression.NewPlus( + expression.NewGetField(2, sql.Text, "lcol3", false), + expression.NewLiteral(int32(2), sql.Int32), + ), + expression.NewGetField(6, sql.Text, "rcol3", false), + )) + + iter, err := j.RowIter(sql.NewEmptyContext()) + require.NoError(err) + rows, err := sql.RowIterToRows(iter) + require.NoError(err) + require.ElementsMatch([]sql.Row{ + {"col1_1", "col2_1", int32(1), int64(2), "col1_2", "col2_2", int32(3), int64(4)}, + {"col1_2", "col2_2", int32(3), int64(4), nil, nil, nil, nil}, + }, rows) +} + +func TestRightJoin(t *testing.T) { + require := require.New(t) + + ltable := memory.NewTable("left", lSchema) + rtable := memory.NewTable("right", rSchema) + insertData(t, ltable) + insertData(t, rtable) + + j := NewRightJoin( + NewResolvedTable(ltable), + NewResolvedTable(rtable), + expression.NewEquals( + expression.NewPlus( + expression.NewGetField(2, sql.Text, "lcol3", false), + expression.NewLiteral(int32(2), sql.Int32), + ), + expression.NewGetField(6, sql.Text, "rcol3", false), + )) + + iter, err := j.RowIter(sql.NewEmptyContext()) + require.NoError(err) + rows, err := sql.RowIterToRows(iter) + require.NoError(err) + require.ElementsMatch([]sql.Row{ + {nil, nil, nil, nil, "col1_1", "col2_1", int32(1), int64(2)}, + {"col1_1", "col2_1", int32(1), int64(2), "col1_2", "col2_2", int32(3), int64(4)}, + }, rows) +} + +type mockReporter struct { + val uint64 + max uint64 +} + +func (m mockReporter) UsedMemory() uint64 { return m.val } +func (m mockReporter) MaxMemory() uint64 { return m.max } diff --git a/sql/plan/limit.go b/sql/plan/limit.go index b2831ef99..9ec72805e 100644 --- a/sql/plan/limit.go +++ b/sql/plan/limit.go @@ -4,22 +4,20 @@ import ( "io" opentracing "github.com/opentracing/opentracing-go" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) -var _ sql.Node = &Limit{} - // Limit is a node that only allows up to N rows to be retrieved. type Limit struct { UnaryNode - size int64 + Limit int64 } // NewLimit creates a new Limit node with the given size. func NewLimit(size int64, child sql.Node) *Limit { return &Limit{ UnaryNode: UnaryNode{Child: child}, - size: size, + Limit: size, } } @@ -30,7 +28,7 @@ func (l *Limit) Resolved() bool { // RowIter implements the Node interface. func (l *Limit) RowIter(ctx *sql.Context) (sql.RowIter, error) { - span, ctx := ctx.Span("plan.Limit", opentracing.Tag{Key: "limit", Value: l.size}) + span, ctx := ctx.Span("plan.Limit", opentracing.Tag{Key: "limit", Value: l.Limit}) li, err := l.Child.RowIter(ctx) if err != nil { @@ -40,27 +38,17 @@ func (l *Limit) RowIter(ctx *sql.Context) (sql.RowIter, error) { return sql.NewSpanIter(span, &limitIter{l, 0, li}), nil } -// TransformUp implements the Transformable interface. -func (l *Limit) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - child, err := l.Child.TransformUp(f) - if err != nil { - return nil, err - } - return f(NewLimit(l.size, child)) -} - -// TransformExpressionsUp implements the Transformable interface. -func (l *Limit) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { - child, err := l.Child.TransformExpressionsUp(f) - if err != nil { - return nil, err +// WithChildren implements the Node interface. +func (l *Limit) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(l, len(children), 1) } - return NewLimit(l.size, child), nil + return NewLimit(l.Limit, children[0]), nil } func (l Limit) String() string { pr := sql.NewTreePrinter() - _ = pr.WriteNode("Limit(%d)", l.size) + _ = pr.WriteNode("Limit(%d)", l.Limit) _ = pr.WriteChildren(l.Child.String()) return pr.String() } @@ -72,7 +60,7 @@ type limitIter struct { } func (li *limitIter) Next() (sql.Row, error) { - if li.currentPos >= li.l.size { + if li.currentPos >= li.l.Limit { return nil, io.EOF } diff --git a/sql/plan/limit_test.go b/sql/plan/limit_test.go index c5a682196..25fd94c21 100644 --- a/sql/plan/limit_test.go +++ b/sql/plan/limit_test.go @@ -8,11 +8,11 @@ import ( "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" ) -var testingTable *mem.Table +var testingTable *memory.Table var testingTableSize int func TestLimitPlan(t *testing.T) { @@ -38,31 +38,31 @@ func TestLimitImplementsNode(t *testing.T) { } func TestLimit0(t *testing.T) { - _, testingTableSize := getTestingTable(t) + _, size := getTestingTable(t) testingLimit := 0 iterator, _ := getLimitedIterator(t, int64(testingLimit)) - testLimitOverflow(t, iterator, testingLimit, testingTableSize) + testLimitOverflow(t, iterator, testingLimit, size) } func TestLimitLessThanTotal(t *testing.T) { - _, testingTableSize := getTestingTable(t) - testingLimit := testingTableSize - 1 + _, size := getTestingTable(t) + testingLimit := size - 1 iterator, _ := getLimitedIterator(t, int64(testingLimit)) - testLimitOverflow(t, iterator, testingLimit, testingTableSize) + testLimitOverflow(t, iterator, testingLimit, size) } func TestLimitEqualThanTotal(t *testing.T) { - _, testingTableSize := getTestingTable(t) - testingLimit := testingTableSize + _, size := getTestingTable(t) + testingLimit := size iterator, _ := getLimitedIterator(t, int64(testingLimit)) - testLimitOverflow(t, iterator, testingLimit, testingTableSize) + testLimitOverflow(t, iterator, testingLimit, size) } func TestLimitGreaterThanTotal(t *testing.T) { - _, testingTableSize := getTestingTable(t) - testingLimit := testingTableSize + 1 + _, size := getTestingTable(t) + testingLimit := size + 1 iterator, _ := getLimitedIterator(t, int64(testingLimit)) - testLimitOverflow(t, iterator, testingLimit, testingTableSize) + testLimitOverflow(t, iterator, testingLimit, size) } func testLimitOverflow(t *testing.T, iter sql.RowIter, limit int, dataSize int) { @@ -80,7 +80,7 @@ func testLimitOverflow(t *testing.T, iter sql.RowIter, limit int, dataSize int) } } -func getTestingTable(t *testing.T) (*mem.Table, int) { +func getTestingTable(t *testing.T) (*memory.Table, int) { t.Helper() if &testingTable == nil { return testingTable, testingTableSize @@ -89,7 +89,7 @@ func getTestingTable(t *testing.T) (*mem.Table, int) { childSchema := sql.Schema{ {Name: "col1", Type: sql.Text}, } - testingTable = mem.NewTable("test", childSchema) + testingTable = memory.NewTable("test", childSchema) rows := []sql.Row{ sql.NewRow("11a"), diff --git a/sql/plan/lock.go b/sql/plan/lock.go index 21aebe9ab..8e51edec6 100644 --- a/sql/plan/lock.go +++ b/sql/plan/lock.go @@ -3,8 +3,8 @@ package plan import ( "fmt" + "github.com/src-d/go-mysql-server/sql" errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) // TableLock is a read or write lock on a table. @@ -25,8 +25,6 @@ func NewLockTables(locks []*TableLock) *LockTables { return &LockTables{Locks: locks} } -var _ sql.Node = (*LockTables)(nil) - // Children implements the sql.Node interface. func (t *LockTables) Children() []sql.Node { var children = make([]sql.Node, len(t.Locks)) @@ -89,25 +87,21 @@ func (t *LockTables) String() string { return p.String() } -// TransformUp implements the sql.Node interface. -func (t *LockTables) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - var children = make([]*TableLock, len(t.Locks)) - for i, l := range t.Locks { - node, err := l.Table.TransformUp(f) - if err != nil { - return nil, err - } - children[i] = &TableLock{node, l.Write} +// WithChildren implements the Node interface. +func (t *LockTables) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != len(t.Locks) { + return nil, sql.ErrInvalidChildrenNumber.New(t, len(children), len(t.Locks)) } - nt := *t - nt.Locks = children - return f(&nt) -} + var locks = make([]*TableLock, len(t.Locks)) + for i, n := range children { + locks[i] = &TableLock{ + Table: n, + Write: t.Locks[i].Write, + } + } -// TransformExpressionsUp implements the sql.Node interface. -func (t *LockTables) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { - return t, nil + return &LockTables{t.Catalog, locks}, nil } // ErrTableNotLockable is returned whenever a lockable table can't be found. @@ -143,8 +137,6 @@ func NewUnlockTables() *UnlockTables { return new(UnlockTables) } -var _ sql.Node = (*UnlockTables)(nil) - // Children implements the sql.Node interface. func (t *UnlockTables) Children() []sql.Node { return nil } @@ -172,12 +164,11 @@ func (t *UnlockTables) String() string { return p.String() } -// TransformUp implements the sql.Node interface. -func (t *UnlockTables) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - return f(t) -} +// WithChildren implements the Node interface. +func (t *UnlockTables) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(t, len(children), 0) + } -// TransformExpressionsUp implements the sql.Node interface. -func (t *UnlockTables) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { return t, nil } diff --git a/sql/plan/lock_test.go b/sql/plan/lock_test.go index 6f95eda5b..63b806420 100644 --- a/sql/plan/lock_test.go +++ b/sql/plan/lock_test.go @@ -3,16 +3,16 @@ package plan import ( "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestLockTables(t *testing.T) { require := require.New(t) - t1 := newLockableTable(mem.NewTable("foo", nil)) - t2 := newLockableTable(mem.NewTable("bar", nil)) + t1 := newLockableTable(memory.NewTable("foo", nil)) + t2 := newLockableTable(memory.NewTable("bar", nil)) node := NewLockTables([]*TableLock{ {NewResolvedTable(t1), true}, {NewResolvedTable(t2), false}, @@ -31,10 +31,10 @@ func TestLockTables(t *testing.T) { func TestUnlockTables(t *testing.T) { require := require.New(t) - db := mem.NewDatabase("db") - t1 := newLockableTable(mem.NewTable("foo", nil)) - t2 := newLockableTable(mem.NewTable("bar", nil)) - t3 := newLockableTable(mem.NewTable("baz", nil)) + db := memory.NewDatabase("db") + t1 := newLockableTable(memory.NewTable("foo", nil)) + t2 := newLockableTable(memory.NewTable("bar", nil)) + t3 := newLockableTable(memory.NewTable("baz", nil)) db.AddTable("foo", t1) db.AddTable("bar", t2) db.AddTable("baz", t3) diff --git a/sql/plan/naturaljoin.go b/sql/plan/naturaljoin.go index 5a1a931d8..6ccf0182b 100644 --- a/sql/plan/naturaljoin.go +++ b/sql/plan/naturaljoin.go @@ -1,6 +1,6 @@ package plan -import "gopkg.in/src-d/go-mysql-server.v0/sql" +import "github.com/src-d/go-mysql-server/sql" // NaturalJoin is a join that automatically joins by all the columns with the // same name. @@ -35,32 +35,11 @@ func (j NaturalJoin) String() string { return pr.String() } -// TransformUp implements the Node interface. -func (j *NaturalJoin) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - left, err := j.Left.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Node interface. +func (j *NaturalJoin) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 2 { + return nil, sql.ErrInvalidChildrenNumber.New(j, len(children), 2) } - right, err := j.Right.TransformUp(f) - if err != nil { - return nil, err - } - - return f(NewNaturalJoin(left, right)) -} - -// TransformExpressionsUp implements the Node interface. -func (j *NaturalJoin) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { - left, err := j.Left.TransformExpressionsUp(f) - if err != nil { - return nil, err - } - - right, err := j.Right.TransformExpressionsUp(f) - if err != nil { - return nil, err - } - - return NewNaturalJoin(left, right), nil + return NewNaturalJoin(children[0], children[1]), nil } diff --git a/sql/plan/nothing.go b/sql/plan/nothing.go index aab307d48..43792405d 100644 --- a/sql/plan/nothing.go +++ b/sql/plan/nothing.go @@ -1,14 +1,12 @@ package plan -import "gopkg.in/src-d/go-mysql-server.v0/sql" +import "github.com/src-d/go-mysql-server/sql" // Nothing is a node that will return no rows. var Nothing nothing type nothing struct{} -var _ sql.Node = nothing{} - func (nothing) String() string { return "NOTHING" } func (nothing) Resolved() bool { return true } func (nothing) Schema() sql.Schema { return nil } @@ -16,9 +14,12 @@ func (nothing) Children() []sql.Node { return nil } func (nothing) RowIter(*sql.Context) (sql.RowIter, error) { return sql.RowsToRowIter(), nil } -func (nothing) TransformUp(sql.TransformNodeFunc) (sql.Node, error) { - return Nothing, nil -} -func (nothing) TransformExpressionsUp(sql.TransformExprFunc) (sql.Node, error) { + +// WithChildren implements the Node interface. +func (n nothing) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(n, len(children), 0) + } + return Nothing, nil } diff --git a/sql/plan/offset.go b/sql/plan/offset.go index a02357202..527b2c7cf 100644 --- a/sql/plan/offset.go +++ b/sql/plan/offset.go @@ -2,20 +2,20 @@ package plan import ( opentracing "github.com/opentracing/opentracing-go" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // Offset is a node that skips the first N rows. type Offset struct { UnaryNode - n int64 + Offset int64 } // NewOffset creates a new Offset node. func NewOffset(n int64, child sql.Node) *Offset { return &Offset{ UnaryNode: UnaryNode{Child: child}, - n: n, + Offset: n, } } @@ -26,37 +26,27 @@ func (o *Offset) Resolved() bool { // RowIter implements the Node interface. func (o *Offset) RowIter(ctx *sql.Context) (sql.RowIter, error) { - span, ctx := ctx.Span("plan.Offset", opentracing.Tag{Key: "offset", Value: o.n}) + span, ctx := ctx.Span("plan.Offset", opentracing.Tag{Key: "offset", Value: o.Offset}) it, err := o.Child.RowIter(ctx) if err != nil { span.Finish() return nil, err } - return sql.NewSpanIter(span, &offsetIter{o.n, it}), nil + return sql.NewSpanIter(span, &offsetIter{o.Offset, it}), nil } -// TransformUp implements the Transformable interface. -func (o *Offset) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - child, err := o.Child.TransformUp(f) - if err != nil { - return nil, err - } - return f(NewOffset(o.n, child)) -} - -// TransformExpressionsUp implements the Transformable interface. -func (o *Offset) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { - child, err := o.Child.TransformExpressionsUp(f) - if err != nil { - return nil, err +// WithChildren implements the Node interface. +func (o *Offset) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(o, len(children), 1) } - return NewOffset(o.n, child), nil + return NewOffset(o.Offset, children[0]), nil } func (o Offset) String() string { pr := sql.NewTreePrinter() - _ = pr.WriteNode("Offset(%d)", o.n) + _ = pr.WriteNode("Offset(%d)", o.Offset) _ = pr.WriteChildren(o.Child.String()) return pr.String() } diff --git a/sql/plan/offset_test.go b/sql/plan/offset_test.go index 6865ed175..607905536 100644 --- a/sql/plan/offset_test.go +++ b/sql/plan/offset_test.go @@ -3,8 +3,8 @@ package plan import ( "testing" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestOffsetPlan(t *testing.T) { diff --git a/sql/plan/process.go b/sql/plan/process.go index 3874ed487..2e0bec439 100644 --- a/sql/plan/process.go +++ b/sql/plan/process.go @@ -3,7 +3,7 @@ package plan import ( "io" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // QueryProcess represents a running query process node. It will use a callback @@ -21,28 +21,13 @@ func NewQueryProcess(node sql.Node, notify NotifyFunc) *QueryProcess { return &QueryProcess{UnaryNode{Child: node}, notify} } -// TransformUp implements the sql.Node interface. -func (p *QueryProcess) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - n, err := p.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Node interface. +func (p *QueryProcess) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(p, len(children), 1) } - np := *p - np.Child = n - return &np, nil -} - -// TransformExpressionsUp implements the sql.Node interface. -func (p *QueryProcess) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { - n, err := p.Child.TransformExpressionsUp(f) - if err != nil { - return nil, err - } - - np := *p - np.Child = n - return &np, nil + return NewQueryProcess(children[0], p.Notify), nil } // RowIter implements the sql.Node interface. @@ -52,7 +37,7 @@ func (p *QueryProcess) RowIter(ctx *sql.Context) (sql.RowIter, error) { return nil, err } - return &trackedRowIter{iter, p.Notify}, nil + return &trackedRowIter{iter: iter, onDone: p.Notify}, nil } func (p *QueryProcess) String() string { return p.Child.String() } @@ -63,12 +48,14 @@ func (p *QueryProcess) String() string { return p.Child.String() } // partition is processed. type ProcessIndexableTable struct { sql.IndexableTable - Notify NotifyFunc + OnPartitionDone NamedNotifyFunc + OnPartitionStart NamedNotifyFunc + OnRowNext NamedNotifyFunc } // NewProcessIndexableTable returns a new ProcessIndexableTable. -func NewProcessIndexableTable(t sql.IndexableTable, notify NotifyFunc) *ProcessIndexableTable { - return &ProcessIndexableTable{t, notify} +func NewProcessIndexableTable(t sql.IndexableTable, onPartitionDone, onPartitionStart, OnRowNext NamedNotifyFunc) *ProcessIndexableTable { + return &ProcessIndexableTable{t, onPartitionDone, onPartitionStart, OnRowNext} } // Underlying implements sql.TableWrapper interface. @@ -86,7 +73,7 @@ func (t *ProcessIndexableTable) IndexKeyValues( return nil, err } - return &trackedPartitionIndexKeyValueIter{iter, t.Notify}, nil + return &trackedPartitionIndexKeyValueIter{iter, t.OnPartitionDone, t.OnPartitionStart, t.OnRowNext}, nil } // PartitionRows implements the sql.Table interface. @@ -96,22 +83,46 @@ func (t *ProcessIndexableTable) PartitionRows(ctx *sql.Context, p sql.Partition) return nil, err } - return &trackedRowIter{iter, t.Notify}, nil + partitionName := partitionName(p) + if t.OnPartitionStart != nil { + t.OnPartitionStart(partitionName) + } + + var onDone NotifyFunc + if t.OnPartitionDone != nil { + onDone = func() { + t.OnPartitionDone(partitionName) + } + } + + var onNext NotifyFunc + if t.OnRowNext != nil { + onNext = func() { + t.OnRowNext(partitionName) + } + } + + return &trackedRowIter{iter: iter, onNext: onNext, onDone: onDone}, nil } var _ sql.IndexableTable = (*ProcessIndexableTable)(nil) +// NamedNotifyFunc is a function to notify about some event with a string argument. +type NamedNotifyFunc func(name string) + // ProcessTable is a wrapper for sql.Tables inside a query process. It // notifies the process manager about the status of a query when a partition // is processed. type ProcessTable struct { sql.Table - Notify NotifyFunc + OnPartitionDone NamedNotifyFunc + OnPartitionStart NamedNotifyFunc + OnRowNext NamedNotifyFunc } // NewProcessTable returns a new ProcessTable. -func NewProcessTable(t sql.Table, notify NotifyFunc) *ProcessTable { - return &ProcessTable{t, notify} +func NewProcessTable(t sql.Table, onPartitionDone, onPartitionStart, OnRowNext NamedNotifyFunc) *ProcessTable { + return &ProcessTable{t, onPartitionDone, onPartitionStart, OnRowNext} } // Underlying implements sql.TableWrapper interface. @@ -126,18 +137,38 @@ func (t *ProcessTable) PartitionRows(ctx *sql.Context, p sql.Partition) (sql.Row return nil, err } - return &trackedRowIter{iter, t.Notify}, nil + partitionName := partitionName(p) + if t.OnPartitionStart != nil { + t.OnPartitionStart(partitionName) + } + + var onDone NotifyFunc + if t.OnPartitionDone != nil { + onDone = func() { + t.OnPartitionDone(partitionName) + } + } + + var onNext NotifyFunc + if t.OnRowNext != nil { + onNext = func() { + t.OnRowNext(partitionName) + } + } + + return &trackedRowIter{iter: iter, onNext: onNext, onDone: onDone}, nil } type trackedRowIter struct { iter sql.RowIter - notify NotifyFunc + onDone NotifyFunc + onNext NotifyFunc } func (i *trackedRowIter) done() { - if i.notify != nil { - i.notify() - i.notify = nil + if i.onDone != nil { + i.onDone() + i.onDone = nil } } @@ -149,6 +180,11 @@ func (i *trackedRowIter) Next() (sql.Row, error) { } return nil, err } + + if i.onNext != nil { + i.onNext() + } + return row, nil } @@ -159,7 +195,9 @@ func (i *trackedRowIter) Close() error { type trackedPartitionIndexKeyValueIter struct { sql.PartitionIndexKeyValueIter - notify NotifyFunc + OnPartitionDone NamedNotifyFunc + OnPartitionStart NamedNotifyFunc + OnRowNext NamedNotifyFunc } func (i *trackedPartitionIndexKeyValueIter) Next() (sql.Partition, sql.IndexKeyValueIter, error) { @@ -168,24 +206,47 @@ func (i *trackedPartitionIndexKeyValueIter) Next() (sql.Partition, sql.IndexKeyV return nil, nil, err } - return p, &trackedIndexKeyValueIter{iter, i.notify}, nil + partitionName := partitionName(p) + if i.OnPartitionStart != nil { + i.OnPartitionStart(partitionName) + } + + var onDone NotifyFunc + if i.OnPartitionDone != nil { + onDone = func() { + i.OnPartitionDone(partitionName) + } + } + + var onNext NotifyFunc + if i.OnRowNext != nil { + onNext = func() { + i.OnRowNext(partitionName) + } + } + + return p, &trackedIndexKeyValueIter{iter, onDone, onNext}, nil } type trackedIndexKeyValueIter struct { iter sql.IndexKeyValueIter - notify NotifyFunc + onDone NotifyFunc + onNext NotifyFunc } func (i *trackedIndexKeyValueIter) done() { - if i.notify != nil { - i.notify() - i.notify = nil + if i.onDone != nil { + i.onDone() + i.onDone = nil } } -func (i *trackedIndexKeyValueIter) Close() error { +func (i *trackedIndexKeyValueIter) Close() (err error) { i.done() - return nil + if i.iter != nil { + err = i.iter.Close() + } + return err } func (i *trackedIndexKeyValueIter) Next() ([]interface{}, []byte, error) { @@ -197,5 +258,16 @@ func (i *trackedIndexKeyValueIter) Next() ([]interface{}, []byte, error) { return nil, nil, err } + if i.onNext != nil { + i.onNext() + } + return v, k, nil } + +func partitionName(p sql.Partition) string { + if n, ok := p.(sql.Nameable); ok { + return n.Name() + } + return string(p.Key()) +} diff --git a/sql/plan/process_test.go b/sql/plan/process_test.go index 0cc7c0485..de819edb0 100644 --- a/sql/plan/process_test.go +++ b/sql/plan/process_test.go @@ -4,16 +4,16 @@ import ( "io" "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestQueryProcess(t *testing.T) { require := require.New(t) - table := mem.NewTable("foo", sql.Schema{ + table := memory.NewTable("foo", sql.Schema{ {Name: "a", Type: sql.Int64}, }) @@ -52,7 +52,7 @@ func TestQueryProcess(t *testing.T) { func TestProcessTable(t *testing.T) { require := require.New(t) - table := mem.NewPartitionedTable("foo", sql.Schema{ + table := memory.NewPartitionedTable("foo", sql.Schema{ {Name: "a", Type: sql.Int64}, }, 2) @@ -61,7 +61,9 @@ func TestProcessTable(t *testing.T) { table.Insert(sql.NewEmptyContext(), sql.NewRow(int64(3))) table.Insert(sql.NewEmptyContext(), sql.NewRow(int64(4))) - var notifications int + var partitionDoneNotifications int + var partitionStartNotifications int + var rowNextNotifications int node := NewProject( []sql.Expression{ @@ -70,8 +72,14 @@ func TestProcessTable(t *testing.T) { NewResolvedTable( NewProcessTable( table, - func() { - notifications++ + func(partitionName string) { + partitionDoneNotifications++ + }, + func(partitionName string) { + partitionStartNotifications++ + }, + func(partitionName string) { + rowNextNotifications++ }, ), ), @@ -91,13 +99,15 @@ func TestProcessTable(t *testing.T) { } require.ElementsMatch(expected, rows) - require.Equal(2, notifications) + require.Equal(2, partitionDoneNotifications) + require.Equal(2, partitionStartNotifications) + require.Equal(4, rowNextNotifications) } func TestProcessIndexableTable(t *testing.T) { require := require.New(t) - table := mem.NewPartitionedTable("foo", sql.Schema{ + table := memory.NewPartitionedTable("foo", sql.Schema{ {Name: "a", Type: sql.Int64, Source: "foo"}, }, 2) @@ -106,12 +116,20 @@ func TestProcessIndexableTable(t *testing.T) { table.Insert(sql.NewEmptyContext(), sql.NewRow(int64(3))) table.Insert(sql.NewEmptyContext(), sql.NewRow(int64(4))) - var notifications int + var partitionDoneNotifications int + var partitionStartNotifications int + var rowNextNotifications int pt := NewProcessIndexableTable( table, - func() { - notifications++ + func(partitionName string) { + partitionDoneNotifications++ + }, + func(partitionName string) { + partitionStartNotifications++ + }, + func(partitionName string) { + rowNextNotifications++ }, ) @@ -144,5 +162,7 @@ func TestProcessIndexableTable(t *testing.T) { } require.ElementsMatch(expectedValues, values) - require.Equal(2, notifications) + require.Equal(2, partitionDoneNotifications) + require.Equal(2, partitionStartNotifications) + require.Equal(4, rowNextNotifications) } diff --git a/sql/plan/processlist.go b/sql/plan/processlist.go index ea75bb535..a447fb79d 100644 --- a/sql/plan/processlist.go +++ b/sql/plan/processlist.go @@ -1,11 +1,10 @@ package plan import ( - "fmt" "sort" "strings" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) type process struct { @@ -58,13 +57,12 @@ func (p *ShowProcessList) Children() []sql.Node { return nil } // Resolved implements the Node interface. func (p *ShowProcessList) Resolved() bool { return true } -// TransformUp implements the Node interface. -func (p *ShowProcessList) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - return f(p) -} +// WithChildren implements the Node interface. +func (p *ShowProcessList) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(p, len(children), 0) + } -// TransformExpressionsUp implements the Node interface. -func (p *ShowProcessList) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { return p, nil } @@ -78,21 +76,36 @@ func (p *ShowProcessList) RowIter(ctx *sql.Context) (sql.RowIter, error) { for i, proc := range processes { var status []string - for name, progress := range proc.Progress { - status = append(status, fmt.Sprintf("%s(%s)", name, progress)) + var names []string + for name := range proc.Progress { + names = append(names, name) + } + sort.Strings(names) + + for _, name := range names { + progress := proc.Progress[name] + + printer := sql.NewTreePrinter() + _ = printer.WriteNode("\n" + progress.String()) + children := []string{} + for _, partitionProgress := range progress.PartitionsProgress { + children = append(children, partitionProgress.String()) + } + sort.Strings(children) + _ = printer.WriteChildren(children...) + + status = append(status, printer.String()) } if len(status) == 0 { status = []string{"running"} } - sort.Strings(status) - rows[i] = process{ - id: int64(proc.Pid), + id: int64(proc.Connection), user: proc.User, time: int64(proc.Seconds()), - state: strings.Join(status, ", "), + state: strings.Join(status, ""), command: proc.Type.String(), host: ctx.Session.Client().Address, info: proc.Query, diff --git a/sql/plan/processlist_test.go b/sql/plan/processlist_test.go index ff204314a..12ee98b9a 100644 --- a/sql/plan/processlist_test.go +++ b/sql/plan/processlist_test.go @@ -4,8 +4,8 @@ import ( "context" "testing" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestShowProcessList(t *testing.T) { @@ -21,19 +21,21 @@ func TestShowProcessList(t *testing.T) { ctx, err := p.AddProcess(ctx, sql.QueryProcess, "SELECT foo") require.NoError(err) - p.AddProgressItem(ctx.Pid(), "a", 5) - p.AddProgressItem(ctx.Pid(), "b", 6) + p.AddTableProgress(ctx.Pid(), "a", 5) + p.AddTableProgress(ctx.Pid(), "b", 6) ctx = sql.NewContext(context.Background(), sql.WithPid(2), sql.WithSession(sess)) ctx, err = p.AddProcess(ctx, sql.CreateIndexProcess, "SELECT bar") require.NoError(err) - p.AddProgressItem(ctx.Pid(), "foo", 2) + p.AddTableProgress(ctx.Pid(), "foo", 2) - p.UpdateProgress(1, "a", 3) - p.UpdateProgress(1, "a", 1) - p.UpdateProgress(1, "b", 2) - p.UpdateProgress(2, "foo", 1) + p.UpdateTableProgress(1, "a", 3) + p.UpdateTableProgress(1, "a", 1) + p.UpdatePartitionProgress(1, "a", "a-1", 7) + p.UpdatePartitionProgress(1, "a", "a-2", 9) + p.UpdateTableProgress(1, "b", 2) + p.UpdateTableProgress(2, "foo", 1) n.ProcessList = p n.Database = "foo" @@ -44,8 +46,15 @@ func TestShowProcessList(t *testing.T) { require.NoError(err) expected := []sql.Row{ - {int64(1), "foo", addr, "foo", "query", int64(0), "a(4/5), b(2/6)", "SELECT foo"}, - {int64(2), "foo", addr, "foo", "create_index", int64(0), "foo(1/2)", "SELECT bar"}, + {int64(1), "foo", addr, "foo", "query", int64(0), + ` +a (4/5 partitions) + ├─ a-1 (7/? rows) + └─ a-2 (9/? rows) + +b (2/6 partitions) +`, "SELECT foo"}, + {int64(1), "foo", addr, "foo", "create_index", int64(0), "\nfoo (1/2 partitions)\n", "SELECT bar"}, } require.ElementsMatch(expected, rows) diff --git a/sql/plan/project.go b/sql/plan/project.go index 0e49315cd..8b166d449 100644 --- a/sql/plan/project.go +++ b/sql/plan/project.go @@ -4,7 +4,7 @@ import ( "strings" opentracing "github.com/opentracing/opentracing-go" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // Project is a projection of certain expression from the children node. @@ -69,30 +69,6 @@ func (p *Project) RowIter(ctx *sql.Context) (sql.RowIter, error) { return sql.NewSpanIter(span, &iter{p, i, ctx}), nil } -// TransformUp implements the Transformable interface. -func (p *Project) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - child, err := p.Child.TransformUp(f) - if err != nil { - return nil, err - } - return f(NewProject(p.Projections, child)) -} - -// TransformExpressionsUp implements the Transformable interface. -func (p *Project) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { - exprs, err := transformExpressionsUp(f, p.Projections) - if err != nil { - return nil, err - } - - child, err := p.Child.TransformExpressionsUp(f) - if err != nil { - return nil, err - } - - return NewProject(exprs, child), nil -} - func (p *Project) String() string { pr := sql.NewTreePrinter() var exprs = make([]string, len(p.Projections)) @@ -109,14 +85,22 @@ func (p *Project) Expressions() []sql.Expression { return p.Projections } -// TransformExpressions implements the Expressioner interface. -func (p *Project) TransformExpressions(f sql.TransformExprFunc) (sql.Node, error) { - projects, err := transformExpressionsUp(f, p.Projections) - if err != nil { - return nil, err +// WithChildren implements the Node interface. +func (p *Project) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(p, len(children), 1) + } + + return NewProject(p.Projections, children[0]), nil +} + +// WithExpressions implements the Expressioner interface. +func (p *Project) WithExpressions(exprs ...sql.Expression) (sql.Node, error) { + if len(exprs) != len(p.Projections) { + return nil, sql.ErrInvalidChildrenNumber.New(p, len(exprs), len(p.Projections)) } - return NewProject(projects, p.Child), nil + return NewProject(exprs, p.Child), nil } type iter struct { diff --git a/sql/plan/project_test.go b/sql/plan/project_test.go index 7ada8ab7f..84950b924 100644 --- a/sql/plan/project_test.go +++ b/sql/plan/project_test.go @@ -6,9 +6,9 @@ import ( "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" ) func TestProject(t *testing.T) { @@ -18,7 +18,7 @@ func TestProject(t *testing.T) { {Name: "col1", Type: sql.Text, Nullable: true}, {Name: "col2", Type: sql.Text, Nullable: true}, } - child := mem.NewTable("test", childSchema) + child := memory.NewTable("test", childSchema) child.Insert(sql.NewEmptyContext(), sql.NewRow("col1_1", "col2_1")) child.Insert(sql.NewEmptyContext(), sql.NewRow("col1_2", "col2_2")) p := NewProject( diff --git a/sql/plan/resolved_table.go b/sql/plan/resolved_table.go index d031db8b0..dbbd689a1 100644 --- a/sql/plan/resolved_table.go +++ b/sql/plan/resolved_table.go @@ -4,7 +4,7 @@ import ( "context" "io" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // ResolvedTable represents a resolved SQL Table. @@ -44,13 +44,12 @@ func (t *ResolvedTable) RowIter(ctx *sql.Context) (sql.RowIter, error) { }), nil } -// TransformUp implements the Transformable interface. -func (t *ResolvedTable) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - return f(NewResolvedTable(t.Table)) -} +// WithChildren implements the Node interface. +func (t *ResolvedTable) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(t, len(children), 0) + } -// TransformExpressionsUp implements the Transformable interface. -func (t *ResolvedTable) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { return t, nil } @@ -73,8 +72,8 @@ func (i *tableIter) Next() (sql.Row, error) { partition, err := i.partitions.Next() if err != nil { if err == io.EOF { - if err := i.partitions.Close(); err != nil { - return nil, err + if e := i.partitions.Close(); e != nil { + return nil, e } } @@ -95,7 +94,7 @@ func (i *tableIter) Next() (sql.Row, error) { row, err := i.rows.Next() if err != nil && err == io.EOF { - if err := i.rows.Close(); err != nil { + if err = i.rows.Close(); err != nil { return nil, err } diff --git a/sql/plan/resolved_table_test.go b/sql/plan/resolved_table_test.go index 25963935a..a90670468 100644 --- a/sql/plan/resolved_table_test.go +++ b/sql/plan/resolved_table_test.go @@ -6,8 +6,8 @@ import ( "io" "testing" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestResolvedTable(t *testing.T) { diff --git a/sql/plan/set.go b/sql/plan/set.go index 509c672ee..9b8101675 100644 --- a/sql/plan/set.go +++ b/sql/plan/set.go @@ -4,9 +4,9 @@ import ( "fmt" "strings" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" - "gopkg.in/src-d/go-vitess.v1/vt/sqlparser" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "vitess.io/vitess/go/vt/sqlparser" ) // Set configuration variables. Right now, only session variables are supported. @@ -42,41 +42,41 @@ func (s *Set) Resolved() bool { // Children implements the sql.Node interface. func (s *Set) Children() []sql.Node { return nil } -// TransformUp implements the sql.Node interface. -func (s *Set) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - return f(s) -} +// WithChildren implements the Node interface. +func (s *Set) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(s, len(children), 0) + } -// TransformExpressions implements sql.Expressioner interface. -func (s *Set) TransformExpressions(f sql.TransformExprFunc) (sql.Node, error) { - return s.TransformExpressionsUp(f) + return s, nil } -// Expressions implements the sql.Expressioner interface. -func (s *Set) Expressions() []sql.Expression { - var exprs = make([]sql.Expression, len(s.Variables)) - for i, v := range s.Variables { - exprs[i] = v.Value +// WithExpressions implements the Expressioner interface. +func (s *Set) WithExpressions(exprs ...sql.Expression) (sql.Node, error) { + if len(exprs) != len(s.Variables) { + return nil, sql.ErrInvalidChildrenNumber.New(s, len(exprs), len(s.Variables)) } - return exprs -} -// TransformExpressionsUp implements the sql.Node interface. -func (s *Set) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { var vars = make([]SetVariable, len(s.Variables)) for i, v := range s.Variables { - val, err := v.Value.TransformUp(f) - if err != nil { - return nil, err + vars[i] = SetVariable{ + Name: v.Name, + Value: exprs[i], } - - vars[i] = v - vars[i].Value = val } return NewSet(vars...), nil } +// Expressions implements the sql.Expressioner interface. +func (s *Set) Expressions() []sql.Expression { + var exprs = make([]sql.Expression, len(s.Variables)) + for i, v := range s.Variables { + exprs[i] = v.Value + } + return exprs +} + // RowIter implements the sql.Node interface. func (s *Set) RowIter(ctx *sql.Context) (sql.RowIter, error) { span, ctx := ctx.Span("plan.Set") @@ -94,7 +94,7 @@ func (s *Set) RowIter(ctx *sql.Context) (sql.RowIter, error) { ) name := strings.TrimPrefix( - strings.TrimPrefix(strings.TrimLeft(v.Name, "@"), sessionPrefix), + strings.TrimPrefix(strings.TrimLeft(v.Name, "@"), sessionPrefix), globalPrefix, ) diff --git a/sql/plan/set_test.go b/sql/plan/set_test.go index 7241b64ff..4af46b00e 100644 --- a/sql/plan/set_test.go +++ b/sql/plan/set_test.go @@ -4,9 +4,9 @@ import ( "context" "testing" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestSet(t *testing.T) { diff --git a/sql/plan/show_collation.go b/sql/plan/show_collation.go index 35b63feca..3c833b246 100644 --- a/sql/plan/show_collation.go +++ b/sql/plan/show_collation.go @@ -1,6 +1,6 @@ package plan -import "gopkg.in/src-d/go-mysql-server.v0/sql" +import "github.com/src-d/go-mysql-server/sql" // ShowCollation shows all available collations. type ShowCollation struct{} @@ -42,12 +42,11 @@ func (ShowCollation) RowIter(ctx *sql.Context) (sql.RowIter, error) { // Schema implements the sql.Node interface. func (ShowCollation) Schema() sql.Schema { return collationSchema } -// TransformUp implements the sql.Node interface. -func (ShowCollation) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - return f(ShowCollation{}) -} +// WithChildren implements the Node interface. +func (s ShowCollation) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(s, len(children), 0) + } -// TransformExpressionsUp implements the sql.Node interface. -func (ShowCollation) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { - return ShowCollation{}, nil + return s, nil } diff --git a/sql/plan/show_create_database.go b/sql/plan/show_create_database.go index 477aa132d..96de06175 100644 --- a/sql/plan/show_create_database.go +++ b/sql/plan/show_create_database.go @@ -4,7 +4,7 @@ import ( "bytes" "fmt" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // ShowCreateDatabase returns the SQL for creating a database. @@ -82,12 +82,11 @@ func (s *ShowCreateDatabase) Resolved() bool { return !ok } -// TransformExpressionsUp implements the sql.Node interface. -func (s *ShowCreateDatabase) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { - return s, nil -} +// WithChildren implements the Node interface. +func (s *ShowCreateDatabase) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(s, len(children), 0) + } -// TransformUp implements the sql.Node interface. -func (s *ShowCreateDatabase) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - return f(s) + return s, nil } diff --git a/sql/plan/show_create_database_test.go b/sql/plan/show_create_database_test.go index 4f0da72f6..c58fa065c 100644 --- a/sql/plan/show_create_database_test.go +++ b/sql/plan/show_create_database_test.go @@ -3,8 +3,8 @@ package plan import ( "testing" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestShowCreateDatabase(t *testing.T) { diff --git a/sql/plan/show_create_table.go b/sql/plan/show_create_table.go index 853613e62..500fb5930 100644 --- a/sql/plan/show_create_table.go +++ b/sql/plan/show_create_table.go @@ -5,12 +5,10 @@ import ( "io" "strings" - "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" -) + "github.com/src-d/go-mysql-server/internal/similartext" -// ErrTableNotFound is returned when the table could not be found. -var ErrTableNotFound = errors.NewKind("Table `%s` not found") + "github.com/src-d/go-mysql-server/sql" +) // ShowCreateTable is a node that shows the CREATE TABLE statement for a table. type ShowCreateTable struct { @@ -27,22 +25,20 @@ func (n *ShowCreateTable) Schema() sql.Schema { } } -// TransformExpressionsUp implements the Transformable interface. -func (n *ShowCreateTable) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { - return n, nil -} +// WithChildren implements the Node interface. +func (n *ShowCreateTable) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(n, len(children), 0) + } -// TransformUp implements the Transformable interface. -func (n *ShowCreateTable) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - return f(NewShowCreateTable(n.CurrentDatabase, n.Catalog, n.Table)) + return n, nil } // RowIter implements the Node interface func (n *ShowCreateTable) RowIter(*sql.Context) (sql.RowIter, error) { db, err := n.Catalog.Database(n.CurrentDatabase) - if err != nil { - return nil, sql.ErrDatabaseNotFound.New(n.CurrentDatabase) + return nil, err } return &showCreateTablesIter{ @@ -69,10 +65,16 @@ func (i *showCreateTablesIter) Next() (sql.Row, error) { i.didIteration = true - table, found := i.db.Tables()[i.table] + tables := i.db.Tables() + if len(tables) == 0 { + return nil, sql.ErrTableNotFound.New(i.table) + } + + table, found := tables[i.table] if !found { - return nil, ErrTableNotFound.New(i.table) + similar := similartext.FindFromMap(tables, i.table) + return nil, sql.ErrTableNotFound.New(i.table + similar) } composedCreateTableStatement := produceCreateStatement(table) @@ -85,35 +87,35 @@ func (i *showCreateTablesIter) Next() (sql.Row, error) { func produceCreateStatement(table sql.Table) string { schema := table.Schema() - colCreateStatements := make([]string, len(schema)) + colStmts := make([]string, len(schema)) // Statement creation parts for each column - for indx, col := range schema { - createStmtPart := fmt.Sprintf(" `%s` %s", col.Name, col.Type.Type()) + for i, col := range schema { + stmt := fmt.Sprintf(" `%s` %s", col.Name, strings.ToLower(sql.MySQLTypeName(col.Type))) if !col.Nullable { - createStmtPart = fmt.Sprintf("%s NOT NULL", createStmtPart) + stmt = fmt.Sprintf("%s NOT NULL", stmt) } switch def := col.Default.(type) { case string: if def != "" { - createStmtPart = fmt.Sprintf("%s DEFAULT %s", createStmtPart, def) + stmt = fmt.Sprintf("%s DEFAULT %q", stmt, def) } default: if def != nil { - createStmtPart = fmt.Sprintf("%s DEFAULT %v", createStmtPart, col.Default) + stmt = fmt.Sprintf("%s DEFAULT %v", stmt, col.Default) } } - colCreateStatements[indx] = createStmtPart + colStmts[i] = stmt } - prettyColCreateStmts := strings.Join(colCreateStatements, ",\n") - composedCreateTableStatement := - fmt.Sprintf("CREATE TABLE `%s` (\n%s\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", table.Name(), prettyColCreateStmts) - - return composedCreateTableStatement + return fmt.Sprintf( + "CREATE TABLE `%s` (\n%s\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", + table.Name(), + strings.Join(colStmts, ",\n"), + ) } func (i *showCreateTablesIter) Close() error { diff --git a/sql/plan/show_create_table_test.go b/sql/plan/show_create_table_test.go index de115f455..0da6418bf 100644 --- a/sql/plan/show_create_table_test.go +++ b/sql/plan/show_create_table_test.go @@ -3,22 +3,24 @@ package plan import ( "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestShowCreateTable(t *testing.T) { var require = require.New(t) - db := mem.NewDatabase("testdb") + db := memory.NewDatabase("testdb") - table := mem.NewTable( + table := memory.NewTable( "test-table", sql.Schema{ &sql.Column{Name: "baz", Type: sql.Text, Default: "", Nullable: false}, &sql.Column{Name: "zab", Type: sql.Int32, Default: int32(0), Nullable: true}, - &sql.Column{Name: "bza", Type: sql.Int64, Default: int64(0), Nullable: true}, + &sql.Column{Name: "bza", Type: sql.Uint64, Default: uint64(0), Nullable: true}, + &sql.Column{Name: "foo", Type: sql.VarChar(123), Default: "", Nullable: true}, + &sql.Column{Name: "pok", Type: sql.Char(123), Default: "", Nullable: true}, }) db.AddTable(table.Name(), table) @@ -37,9 +39,12 @@ func TestShowCreateTable(t *testing.T) { expected := sql.NewRow( table.Name(), - "CREATE TABLE `test-table` (\n `baz` TEXT NOT NULL,\n"+ - " `zab` INT32 DEFAULT 0,\n"+ - " `bza` INT64 DEFAULT 0\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", + "CREATE TABLE `test-table` (\n `baz` text NOT NULL,\n"+ + " `zab` integer DEFAULT 0,\n"+ + " `bza` bigint unsigned DEFAULT 0,\n"+ + " `foo` varchar(123),\n"+ + " `pok` char(123)\n"+ + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", ) require.Equal(expected, row) diff --git a/sql/plan/show_indexes.go b/sql/plan/show_indexes.go index 1bf9dd480..d427509ea 100644 --- a/sql/plan/show_indexes.go +++ b/sql/plan/show_indexes.go @@ -4,7 +4,7 @@ import ( "fmt" "io" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // ShowIndexes is a node that shows the indexes on a table. @@ -39,13 +39,12 @@ func (n *ShowIndexes) Resolved() bool { return !ok } -// TransformUp implements the Transformable interface. -func (n *ShowIndexes) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - return f(NewShowIndexes(n.db, n.Table, n.Registry)) -} +// WithChildren implements the Node interface. +func (n *ShowIndexes) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(n, len(children), 0) + } -// TransformExpressionsUp implements the Transformable interface. -func (n *ShowIndexes) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { return n, nil } diff --git a/sql/plan/show_indexes_test.go b/sql/plan/show_indexes_test.go index e5e3dd870..8ca0649e2 100644 --- a/sql/plan/show_indexes_test.go +++ b/sql/plan/show_indexes_test.go @@ -3,10 +3,10 @@ package plan import ( "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestShowIndexes(t *testing.T) { @@ -16,7 +16,7 @@ func TestShowIndexes(t *testing.T) { require.False(unresolved.Resolved()) require.Nil(unresolved.Children()) - db := mem.NewDatabase("test") + db := memory.NewDatabase("test") tests := []struct { name string @@ -25,7 +25,7 @@ func TestShowIndexes(t *testing.T) { }{ { name: "test1", - table: mem.NewTable( + table: memory.NewTable( "test1", sql.Schema{ &sql.Column{Name: "foo", Type: sql.Int32, Source: "test1", Default: int32(0), Nullable: false}, @@ -34,7 +34,7 @@ func TestShowIndexes(t *testing.T) { }, { name: "test2", - table: mem.NewTable( + table: memory.NewTable( "test2", sql.Schema{ &sql.Column{Name: "bar", Type: sql.Int64, Source: "test2", Default: int64(0), Nullable: true}, @@ -44,7 +44,7 @@ func TestShowIndexes(t *testing.T) { }, { name: "test3", - table: mem.NewTable( + table: memory.NewTable( "test3", sql.Schema{ &sql.Column{Name: "baz", Type: sql.Text, Source: "test3", Default: "", Nullable: false}, @@ -55,7 +55,7 @@ func TestShowIndexes(t *testing.T) { }, { name: "test4", - table: mem.NewTable( + table: memory.NewTable( "test4", sql.Schema{ &sql.Column{Name: "oof", Type: sql.Text, Source: "test4", Default: "", Nullable: false}, diff --git a/sql/plan/show_tables.go b/sql/plan/show_tables.go index 984c1e1eb..930cb20b4 100644 --- a/sql/plan/show_tables.go +++ b/sql/plan/show_tables.go @@ -3,7 +3,7 @@ package plan import ( "sort" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // ShowTables is a node that shows the database tables. @@ -84,13 +84,12 @@ func (p *ShowTables) RowIter(ctx *sql.Context) (sql.RowIter, error) { return sql.RowsToRowIter(rows...), nil } -// TransformUp implements the Transformable interface. -func (p *ShowTables) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - return f(NewShowTables(p.db, p.Full)) -} +// WithChildren implements the Node interface. +func (p *ShowTables) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(p, len(children), 0) + } -// TransformExpressionsUp implements the Transformable interface. -func (p *ShowTables) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { return p, nil } diff --git a/sql/plan/show_tables_test.go b/sql/plan/show_tables_test.go index 316483a73..153c91c50 100644 --- a/sql/plan/show_tables_test.go +++ b/sql/plan/show_tables_test.go @@ -4,9 +4,9 @@ import ( "io" "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestShowTables(t *testing.T) { @@ -18,10 +18,10 @@ func TestShowTables(t *testing.T) { require.False(unresolvedShowTables.Resolved()) require.Nil(unresolvedShowTables.Children()) - db := mem.NewDatabase("test") - db.AddTable("test1", mem.NewTable("test1", nil)) - db.AddTable("test2", mem.NewTable("test2", nil)) - db.AddTable("test3", mem.NewTable("test3", nil)) + db := memory.NewDatabase("test") + db.AddTable("test1", memory.NewTable("test1", nil)) + db.AddTable("test2", memory.NewTable("test2", nil)) + db.AddTable("test3", memory.NewTable("test3", nil)) resolvedShowTables := NewShowTables(db, false) require.True(resolvedShowTables.Resolved()) diff --git a/sql/plan/showcolumns.go b/sql/plan/showcolumns.go index 364e52762..d2094fe3f 100644 --- a/sql/plan/showcolumns.go +++ b/sql/plan/showcolumns.go @@ -3,7 +3,7 @@ package plan import ( "fmt" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // ShowColumns shows the columns details of a table. @@ -104,24 +104,13 @@ func (s *ShowColumns) RowIter(ctx *sql.Context) (sql.RowIter, error) { return sql.NewSpanIter(span, sql.RowsToRowIter(rows...)), nil } -// TransformUp creates a new ShowColumns node. -func (s *ShowColumns) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - child, err := s.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Node interface. +func (s *ShowColumns) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(s, len(children), 1) } - return f(NewShowColumns(s.Full, child)) -} - -// TransformExpressionsUp creates a new ShowColumns node. -func (s *ShowColumns) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { - child, err := s.Child.TransformExpressionsUp(f) - if err != nil { - return nil, err - } - - return NewShowColumns(s.Full, child), nil + return NewShowColumns(s.Full, children[0]), nil } func (s *ShowColumns) String() string { diff --git a/sql/plan/showcolumns_test.go b/sql/plan/showcolumns_test.go index c65450e60..696627247 100644 --- a/sql/plan/showcolumns_test.go +++ b/sql/plan/showcolumns_test.go @@ -3,15 +3,15 @@ package plan import ( "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestShowColumns(t *testing.T) { require := require.New(t) - table := NewResolvedTable(mem.NewTable("foo", sql.Schema{ + table := NewResolvedTable(memory.NewTable("foo", sql.Schema{ {Name: "a", Type: sql.Text}, {Name: "b", Type: sql.Int64, Nullable: true}, {Name: "c", Type: sql.Int64, Default: int64(1)}, @@ -34,7 +34,7 @@ func TestShowColumns(t *testing.T) { func TestShowColumnsFull(t *testing.T) { require := require.New(t) - table := NewResolvedTable(mem.NewTable("foo", sql.Schema{ + table := NewResolvedTable(memory.NewTable("foo", sql.Schema{ {Name: "a", Type: sql.Text}, {Name: "b", Type: sql.Int64, Nullable: true}, {Name: "c", Type: sql.Int64, Default: int64(1)}, diff --git a/sql/plan/showdatabases.go b/sql/plan/showdatabases.go index 309e9cac7..41873c7be 100644 --- a/sql/plan/showdatabases.go +++ b/sql/plan/showdatabases.go @@ -4,7 +4,7 @@ import ( "sort" "strings" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // ShowDatabases is a node that shows the databases. @@ -53,16 +53,15 @@ func (p *ShowDatabases) RowIter(ctx *sql.Context) (sql.RowIter, error) { return sql.RowsToRowIter(rows...), nil } -// TransformUp implements the Transformable interface. -func (p *ShowDatabases) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - np := *p - return f(&np) -} +// WithChildren implements the Node interface. +func (p *ShowDatabases) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(p, len(children), 0) + } -// TransformExpressionsUp implements the Transformable interface. -func (p *ShowDatabases) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { return p, nil } + func (p ShowDatabases) String() string { return "ShowDatabases" } diff --git a/sql/plan/showtablestatus.go b/sql/plan/showtablestatus.go index 604bc6b8d..158a1d65c 100644 --- a/sql/plan/showtablestatus.go +++ b/sql/plan/showtablestatus.go @@ -5,7 +5,7 @@ import ( "sort" "strings" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // ShowTableStatus returns the status of the tables in the databases. @@ -86,13 +86,12 @@ func (s *ShowTableStatus) String() string { return fmt.Sprintf("ShowTableStatus(%s)", strings.Join(s.Databases, ", ")) } -// TransformUp implements the sql.Node interface. -func (s *ShowTableStatus) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - return f(s) -} +// WithChildren implements the Node interface. +func (s *ShowTableStatus) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(s, len(children), 0) + } -// TransformExpressionsUp implements the sql.Node interface. -func (s *ShowTableStatus) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { return s, nil } diff --git a/sql/plan/showtablestatus_test.go b/sql/plan/showtablestatus_test.go index 28b56bcb7..7852409a9 100644 --- a/sql/plan/showtablestatus_test.go +++ b/sql/plan/showtablestatus_test.go @@ -3,9 +3,9 @@ package plan import ( "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestShowTableStatus(t *testing.T) { @@ -13,14 +13,14 @@ func TestShowTableStatus(t *testing.T) { catalog := sql.NewCatalog() - db1 := mem.NewDatabase("a") - db1.AddTable("t1", mem.NewTable("t1", nil)) - db1.AddTable("t2", mem.NewTable("t2", nil)) + db1 := memory.NewDatabase("a") + db1.AddTable("t1", memory.NewTable("t1", nil)) + db1.AddTable("t2", memory.NewTable("t2", nil)) catalog.AddDatabase(db1) - db2 := mem.NewDatabase("b") - db2.AddTable("t3", mem.NewTable("t3", nil)) - db2.AddTable("t4", mem.NewTable("t4", nil)) + db2 := memory.NewDatabase("b") + db2.AddTable("t3", memory.NewTable("t3", nil)) + db2.AddTable("t4", memory.NewTable("t4", nil)) catalog.AddDatabase(db2) node := NewShowTableStatus() diff --git a/sql/plan/showvariables.go b/sql/plan/showvariables.go index 151fa5385..8cb7fa0c1 100644 --- a/sql/plan/showvariables.go +++ b/sql/plan/showvariables.go @@ -3,8 +3,8 @@ package plan import ( "fmt" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" ) // ShowVariables is a node that shows the global and session variables @@ -28,13 +28,12 @@ func (sv *ShowVariables) Resolved() bool { return true } -// TransformUp implements the sq.Transformable interface. -func (sv *ShowVariables) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - return f(NewShowVariables(sv.config, sv.pattern)) -} +// WithChildren implements the Node interface. +func (sv *ShowVariables) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(sv, len(children), 0) + } -// TransformExpressionsUp implements the sql.Transformable interface. -func (sv *ShowVariables) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { return sv, nil } diff --git a/sql/plan/showvariables_test.go b/sql/plan/showvariables_test.go index e658c977b..1abbc7b96 100644 --- a/sql/plan/showvariables_test.go +++ b/sql/plan/showvariables_test.go @@ -4,8 +4,8 @@ import ( "io" "testing" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestShowVariables(t *testing.T) { diff --git a/sql/plan/showwarnings.go b/sql/plan/showwarnings.go index 388c46e7c..c990bfc81 100644 --- a/sql/plan/showwarnings.go +++ b/sql/plan/showwarnings.go @@ -1,7 +1,7 @@ package plan import ( - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // ShowWarnings is a node that shows the session warnings @@ -12,13 +12,12 @@ func (ShowWarnings) Resolved() bool { return true } -// TransformUp implements the sq.Transformable interface. -func (sw ShowWarnings) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - return f(sw) -} +// WithChildren implements the Node interface. +func (sw ShowWarnings) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(sw, len(children), 0) + } -// TransformExpressionsUp implements the sql.Transformable interface. -func (sw ShowWarnings) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { return sw, nil } diff --git a/sql/plan/showwarnings_test.go b/sql/plan/showwarnings_test.go index e8595f1f2..480cc7726 100644 --- a/sql/plan/showwarnings_test.go +++ b/sql/plan/showwarnings_test.go @@ -4,8 +4,8 @@ import ( "io" "testing" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestShowWarnings(t *testing.T) { diff --git a/sql/plan/sort.go b/sql/plan/sort.go index f33f8a837..fed1f7da3 100644 --- a/sql/plan/sort.go +++ b/sql/plan/sort.go @@ -6,8 +6,8 @@ import ( "sort" "strings" + "github.com/src-d/go-mysql-server/sql" "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) // ErrUnableSort is thrown when something happens on sorting @@ -68,18 +68,16 @@ func NewSort(sortFields []SortField, child sql.Node) *Sort { } } +var _ sql.Expressioner = (*Sort)(nil) + // Resolved implements the Resolvable interface. func (s *Sort) Resolved() bool { - return s.UnaryNode.Child.Resolved() && s.expressionsResolved() -} - -func (s *Sort) expressionsResolved() bool { for _, f := range s.SortFields { if !f.Column.Resolved() { return false } } - return true + return s.Child.Resolved() } // RowIter implements the Node interface. @@ -90,35 +88,7 @@ func (s *Sort) RowIter(ctx *sql.Context) (sql.RowIter, error) { span.Finish() return nil, err } - return sql.NewSpanIter(span, newSortIter(s, i)), nil -} - -// TransformUp implements the Transformable interface. -func (s *Sort) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - child, err := s.Child.TransformUp(f) - if err != nil { - return nil, err - } - return f(NewSort(s.SortFields, child)) -} - -// TransformExpressionsUp implements the Transformable interface. -func (s *Sort) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { - var sfs = make([]SortField, len(s.SortFields)) - for i, sf := range s.SortFields { - col, err := sf.Column.TransformUp(f) - if err != nil { - return nil, err - } - sfs[i] = SortField{col, sf.Order, sf.NullOrdering} - } - - child, err := s.Child.TransformExpressionsUp(f) - if err != nil { - return nil, err - } - - return NewSort(sfs, child), nil + return sql.NewSpanIter(span, newSortIter(ctx, s, i)), nil } func (s *Sort) String() string { @@ -141,37 +111,47 @@ func (s *Sort) Expressions() []sql.Expression { return exprs } -// TransformExpressions implements the Expressioner interface. -func (s *Sort) TransformExpressions(f sql.TransformExprFunc) (sql.Node, error) { - var sortFields = make([]SortField, len(s.SortFields)) - for i, field := range s.SortFields { - transformed, err := field.Column.TransformUp(f) - if err != nil { - return nil, err - } - sortFields[i] = SortField{ - Column: transformed, - Order: field.Order, - NullOrdering: field.NullOrdering, +// WithChildren implements the Node interface. +func (s *Sort) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(s, len(children), 1) + } + + return NewSort(s.SortFields, children[0]), nil +} + +// WithExpressions implements the Expressioner interface. +func (s *Sort) WithExpressions(exprs ...sql.Expression) (sql.Node, error) { + if len(exprs) != len(s.SortFields) { + return nil, sql.ErrInvalidChildrenNumber.New(s, len(exprs), len(s.SortFields)) + } + + var fields = make([]SortField, len(s.SortFields)) + for i, expr := range exprs { + fields[i] = SortField{ + Column: expr, + NullOrdering: s.SortFields[i].NullOrdering, + Order: s.SortFields[i].Order, } } - return NewSort(sortFields, s.Child), nil + return NewSort(fields, s.Child), nil } type sortIter struct { + ctx *sql.Context s *Sort childIter sql.RowIter sortedRows []sql.Row idx int } -func newSortIter(s *Sort, child sql.RowIter) *sortIter { +func newSortIter(ctx *sql.Context, s *Sort, child sql.RowIter) *sortIter { return &sortIter{ - s: s, - childIter: child, - sortedRows: nil, - idx: -1, + ctx: ctx, + s: s, + childIter: child, + idx: -1, } } @@ -183,6 +163,7 @@ func (i *sortIter) Next() (sql.Row, error) { } i.idx = 0 } + if i.idx >= len(i.sortedRows) { return nil, io.EOF } @@ -197,9 +178,11 @@ func (i *sortIter) Close() error { } func (i *sortIter) computeSortedRows() error { - var rows []sql.Row + cache, dispose := i.ctx.Memory.NewRowsCache() + defer dispose() + for { - childRow, err := i.childIter.Next() + row, err := i.childIter.Next() if err == io.EOF { break } @@ -207,9 +190,12 @@ func (i *sortIter) computeSortedRows() error { return err } - rows = append(rows, childRow) + if err := cache.Add(row); err != nil { + return err + } } + rows := cache.Get() sorter := &sorter{ sortFields: i.s.SortFields, rows: rows, diff --git a/sql/plan/sort_test.go b/sql/plan/sort_test.go index 583046b86..d5cb51ad8 100644 --- a/sql/plan/sort_test.go +++ b/sql/plan/sort_test.go @@ -3,9 +3,9 @@ package plan import ( "testing" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" ) @@ -27,7 +27,7 @@ func TestSort(t *testing.T) { {Name: "col2", Type: sql.Int32, Nullable: true}, } - child := mem.NewTable("test", schema) + child := memory.NewTable("test", schema) for _, row := range data { require.NoError(child.Insert(sql.NewEmptyContext(), row)) } @@ -68,7 +68,7 @@ func TestSortAscending(t *testing.T) { {Name: "col1", Type: sql.Text, Nullable: true}, } - child := mem.NewTable("test", schema) + child := memory.NewTable("test", schema) for _, row := range data { require.NoError(child.Insert(sql.NewEmptyContext(), row)) } @@ -108,7 +108,7 @@ func TestSortDescending(t *testing.T) { {Name: "col1", Type: sql.Text, Nullable: true}, } - child := mem.NewTable("test", schema) + child := memory.NewTable("test", schema) for _, row := range data { require.NoError(child.Insert(sql.NewEmptyContext(), row)) } diff --git a/sql/plan/subqueryalias.go b/sql/plan/subqueryalias.go index 47c42c19c..da1264c88 100644 --- a/sql/plan/subqueryalias.go +++ b/sql/plan/subqueryalias.go @@ -1,7 +1,7 @@ package plan import ( - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // SubqueryAlias is a node that gives a subquery a name. @@ -45,16 +45,22 @@ func (n *SubqueryAlias) RowIter(ctx *sql.Context) (sql.RowIter, error) { return sql.NewSpanIter(span, iter), nil } -// TransformUp implements the Node interface. -func (n *SubqueryAlias) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - return f(n) -} +// WithChildren implements the Node interface. +func (n *SubqueryAlias) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(n, len(children), 1) + } -// TransformExpressionsUp implements the Node interface. -func (n *SubqueryAlias) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { + nn := *n + nn.Child = children[0] return n, nil } +// Opaque implements the OpaqueNode interface. +func (n *SubqueryAlias) Opaque() bool { + return true +} + func (n SubqueryAlias) String() string { pr := sql.NewTreePrinter() _ = pr.WriteNode("SubqueryAlias(%s)", n.name) diff --git a/sql/plan/subqueryalias_test.go b/sql/plan/subqueryalias_test.go index a7b77242b..90c538e7f 100644 --- a/sql/plan/subqueryalias_test.go +++ b/sql/plan/subqueryalias_test.go @@ -3,10 +3,10 @@ package plan import ( "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) func TestSubqueryAliasSchema(t *testing.T) { @@ -22,7 +22,7 @@ func TestSubqueryAliasSchema(t *testing.T) { {Name: "baz", Type: sql.Text, Nullable: false, Source: "alias"}, } - table := mem.NewTable("bar", tableSchema) + table := memory.NewTable("bar", tableSchema) subquery := NewProject( []sql.Expression{ diff --git a/sql/plan/tablealias.go b/sql/plan/tablealias.go index aa68d2d2a..be37fb109 100644 --- a/sql/plan/tablealias.go +++ b/sql/plan/tablealias.go @@ -4,7 +4,7 @@ import ( "reflect" opentracing "github.com/opentracing/opentracing-go" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // TableAlias is a node that acts as a table with a given name. @@ -23,22 +23,13 @@ func (t *TableAlias) Name() string { return t.name } -// TransformUp implements the Transformable interface. -func (t *TableAlias) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - child, err := t.Child.TransformUp(f) - if err != nil { - return nil, err +// WithChildren implements the Node interface. +func (t *TableAlias) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(t, len(children), 1) } - return f(NewTableAlias(t.name, child)) -} -// TransformExpressionsUp implements the Transformable interface. -func (t *TableAlias) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { - child, err := t.Child.TransformExpressionsUp(f) - if err != nil { - return nil, err - } - return NewTableAlias(t.name, child), nil + return NewTableAlias(t.name, children[0]), nil } // RowIter implements the Node interface. diff --git a/sql/plan/tablealias_test.go b/sql/plan/tablealias_test.go index ced553f69..aee4d83c0 100644 --- a/sql/plan/tablealias_test.go +++ b/sql/plan/tablealias_test.go @@ -4,16 +4,16 @@ import ( "io" "testing" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestTableAlias(t *testing.T) { require := require.New(t) ctx := sql.NewEmptyContext() - table := mem.NewTable("bar", sql.Schema{ + table := memory.NewTable("bar", sql.Schema{ {Name: "a", Type: sql.Text, Nullable: true}, {Name: "b", Type: sql.Text, Nullable: true}, }) diff --git a/sql/plan/transaction.go b/sql/plan/transaction.go index 9e84cca04..3d09366ef 100644 --- a/sql/plan/transaction.go +++ b/sql/plan/transaction.go @@ -1,6 +1,6 @@ package plan -import "gopkg.in/src-d/go-mysql-server.v0/sql" +import "github.com/src-d/go-mysql-server/sql" // Rollback undoes the changes performed in a transaction. type Rollback struct{} @@ -15,13 +15,12 @@ func (*Rollback) RowIter(*sql.Context) (sql.RowIter, error) { func (*Rollback) String() string { return "ROLLBACK" } -// TransformUp implements the sql.Node interface. -func (r *Rollback) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - return f(r) -} +// WithChildren implements the Node interface. +func (r *Rollback) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(r, len(children), 0) + } -// TransformExpressionsUp implements the sql.Node interface. -func (r *Rollback) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { return r, nil } diff --git a/sql/plan/transform.go b/sql/plan/transform.go new file mode 100644 index 000000000..e437327ed --- /dev/null +++ b/sql/plan/transform.go @@ -0,0 +1,89 @@ +package plan + +import ( + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" +) + +// TransformUp applies a transformation function to the given tree from the +// bottom up. +func TransformUp(node sql.Node, f sql.TransformNodeFunc) (sql.Node, error) { + if o, ok := node.(sql.OpaqueNode); ok && o.Opaque() { + return f(node) + } + + children := node.Children() + if len(children) == 0 { + return f(node) + } + + newChildren := make([]sql.Node, len(children)) + for i, c := range children { + c, err := TransformUp(c, f) + if err != nil { + return nil, err + } + newChildren[i] = c + } + + node, err := node.WithChildren(newChildren...) + if err != nil { + return nil, err + } + + return f(node) +} + +// TransformExpressionsUp applies a transformation function to all expressions +// on the given tree from the bottom up. +func TransformExpressionsUp(node sql.Node, f sql.TransformExprFunc) (sql.Node, error) { + if o, ok := node.(sql.OpaqueNode); ok && o.Opaque() { + return TransformExpressions(node, f) + } + + children := node.Children() + if len(children) == 0 { + return TransformExpressions(node, f) + } + + newChildren := make([]sql.Node, len(children)) + for i, c := range children { + c, err := TransformExpressionsUp(c, f) + if err != nil { + return nil, err + } + newChildren[i] = c + } + + node, err := node.WithChildren(newChildren...) + if err != nil { + return nil, err + } + + return TransformExpressions(node, f) +} + +// TransformExpressions applies a transformation function to all expressions +// on the given node. +func TransformExpressions(node sql.Node, f sql.TransformExprFunc) (sql.Node, error) { + e, ok := node.(sql.Expressioner) + if !ok { + return node, nil + } + + exprs := e.Expressions() + if len(exprs) == 0 { + return node, nil + } + + newExprs := make([]sql.Expression, len(exprs)) + for i, e := range exprs { + e, err := expression.TransformUp(e, f) + if err != nil { + return nil, err + } + newExprs[i] = e + } + + return e.WithExpressions(newExprs...) +} diff --git a/sql/plan/transform_test.go b/sql/plan/transform_test.go index 250fc9fff..38d1bb0fb 100644 --- a/sql/plan/transform_test.go +++ b/sql/plan/transform_test.go @@ -3,9 +3,9 @@ package plan import ( "testing" - "gopkg.in/src-d/go-mysql-server.v0/mem" - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" + "github.com/src-d/go-mysql-server/memory" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "github.com/stretchr/testify/require" ) @@ -22,9 +22,9 @@ func TestTransformUp(t *testing.T) { {Name: "a", Type: sql.Text}, {Name: "b", Type: sql.Text}, } - table := mem.NewTable("resolved", schema) + table := memory.NewTable("resolved", schema) - pt, err := p.TransformUp(func(n sql.Node) (sql.Node, error) { + pt, err := TransformUp(p, func(n sql.Node) (sql.Node, error) { switch n.(type) { case *UnresolvedTable: return NewResolvedTable(table), nil diff --git a/sql/plan/unresolved.go b/sql/plan/unresolved.go index 11709fbcf..9bf70a639 100644 --- a/sql/plan/unresolved.go +++ b/sql/plan/unresolved.go @@ -3,8 +3,8 @@ package plan import ( "fmt" + "github.com/src-d/go-mysql-server/sql" errors "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) // ErrUnresolvedTable is thrown when a table cannot be resolved @@ -42,13 +42,12 @@ func (*UnresolvedTable) RowIter(ctx *sql.Context) (sql.RowIter, error) { return nil, ErrUnresolvedTable.New() } -// TransformUp implements the Transformable interface. -func (t *UnresolvedTable) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - return f(NewUnresolvedTable(t.name, t.Database)) -} +// WithChildren implements the Node interface. +func (t *UnresolvedTable) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(t, len(children), 0) + } -// TransformExpressionsUp implements the Transformable interface. -func (t *UnresolvedTable) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { return t, nil } diff --git a/sql/plan/unresolved_test.go b/sql/plan/unresolved_test.go index 3d0656cd9..3865fe40e 100644 --- a/sql/plan/unresolved_test.go +++ b/sql/plan/unresolved_test.go @@ -3,8 +3,8 @@ package plan import ( "testing" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestUnresolvedTable(t *testing.T) { diff --git a/sql/plan/update.go b/sql/plan/update.go new file mode 100644 index 000000000..a29fb7392 --- /dev/null +++ b/sql/plan/update.go @@ -0,0 +1,189 @@ +package plan + +import ( + "github.com/src-d/go-mysql-server/sql" + "gopkg.in/src-d/go-errors.v1" + "io" +) + +var ErrUpdateNotSupported = errors.NewKind("table doesn't support UPDATE") +var ErrUpdateUnexpectedSetResult = errors.NewKind("attempted to set field but expression returned %T") + +// Update is a node for updating rows on tables. +type Update struct { + sql.Node + UpdateExprs []sql.Expression +} + +// NewUpdate creates an Update node. +func NewUpdate(n sql.Node, updateExprs []sql.Expression) *Update { + return &Update{n, updateExprs} +} + +// Expressions implements the Expressioner interface. +func (p *Update) Expressions() []sql.Expression { + return p.UpdateExprs +} + +// Schema implements the Node interface. +func (p *Update) Schema() sql.Schema { + return sql.Schema{ + { + Name: "matched", + Type: sql.Int64, + Default: int64(0), + Nullable: false, + }, + { + Name: "updated", + Type: sql.Int64, + Default: int64(0), + Nullable: false, + }, + } +} + +// Resolved implements the Resolvable interface. +func (p *Update) Resolved() bool { + if !p.Node.Resolved() { + return false + } + for _, updateExpr := range p.UpdateExprs { + if !updateExpr.Resolved() { + return false + } + } + return true +} + +func (p *Update) Children() []sql.Node { + return []sql.Node{p.Node} +} + +func getUpdatable(node sql.Node) (sql.Updater, error) { + switch node := node.(type) { + case sql.Updater: + return node, nil + case *ResolvedTable: + return getUpdatableTable(node.Table) + } + for _, child := range node.Children() { + updater, _ := getUpdatable(child) + if updater != nil { + return updater, nil + } + } + return nil, ErrUpdateNotSupported.New() +} + +func getUpdatableTable(t sql.Table) (sql.Updater, error) { + switch t := t.(type) { + case sql.Updater: + return t, nil + case sql.TableWrapper: + return getUpdatableTable(t.Underlying()) + default: + return nil, ErrUpdateNotSupported.New() + } +} + +// Execute inserts the rows in the database. +func (p *Update) Execute(ctx *sql.Context) (int, int, error) { + updatable, err := getUpdatable(p.Node) + if err != nil { + return 0, 0, err + } + schema := p.Node.Schema() + + iter, err := p.Node.RowIter(ctx) + if err != nil { + return 0, 0, err + } + + rowsMatched := 0 + rowsUpdated := 0 + for { + oldRow, err := iter.Next() + if err == io.EOF { + break + } + if err != nil { + _ = iter.Close() + return rowsMatched, rowsUpdated, err + } + rowsMatched++ + + newRow, err := p.applyUpdates(ctx, oldRow) + if err != nil { + _ = iter.Close() + return rowsMatched, rowsUpdated, err + } + if equals, err := oldRow.Equals(newRow, schema); err == nil { + if !equals { + err = updatable.Update(ctx, oldRow, newRow) + if err != nil { + _ = iter.Close() + return rowsMatched, rowsUpdated, err + } + rowsUpdated++ + } + } else { + _ = iter.Close() + return rowsMatched, rowsUpdated, err + } + } + + return rowsMatched, rowsUpdated, nil +} + +// RowIter implements the Node interface. +func (p *Update) RowIter(ctx *sql.Context) (sql.RowIter, error) { + matched, updated, err := p.Execute(ctx) + if err != nil { + return nil, err + } + + return sql.RowsToRowIter(sql.NewRow(int64(matched), int64(updated))), nil +} + +// WithChildren implements the Node interface. +func (p *Update) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 1 { + return nil, sql.ErrInvalidChildrenNumber.New(p, len(children), 1) + } + return NewUpdate(children[0], p.UpdateExprs), nil +} + +// WithExpressions implements the Expressioner interface. +func (p *Update) WithExpressions(newExprs ...sql.Expression) (sql.Node, error) { + if len(newExprs) != len(p.UpdateExprs) { + return nil, sql.ErrInvalidChildrenNumber.New(p, len(p.UpdateExprs), 1) + } + return NewUpdate(p.Node, newExprs), nil +} + +func (p Update) String() string { + pr := sql.NewTreePrinter() + _ = pr.WriteNode("Update") + _ = pr.WriteChildren(p.Node.String()) + for _, updateExpr := range p.UpdateExprs { + _ = pr.WriteChildren(updateExpr.String()) + } + return pr.String() +} + +func (p *Update) applyUpdates(ctx *sql.Context, row sql.Row) (sql.Row, error) { + var ok bool + prev := row + for _, updateExpr := range p.UpdateExprs { + val, err := updateExpr.Eval(ctx, prev) + if err != nil { + return nil, err + } + prev, ok = val.(sql.Row) + if !ok { + return nil, ErrUpdateUnexpectedSetResult.New(val) + } + } + return prev, nil +} diff --git a/sql/plan/use.go b/sql/plan/use.go index 89553a583..b7eb517b7 100644 --- a/sql/plan/use.go +++ b/sql/plan/use.go @@ -3,7 +3,7 @@ package plan import ( "fmt" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // Use changes the current database. @@ -50,13 +50,12 @@ func (u *Use) RowIter(ctx *sql.Context) (sql.RowIter, error) { return sql.RowsToRowIter(), nil } -// TransformUp implements the sql.Node interface. -func (u *Use) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - return f(u) -} +// WithChildren implements the Node interface. +func (u *Use) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(u, len(children), 1) + } -// TransformExpressionsUp implements the sql.Node interface. -func (u *Use) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { return u, nil } diff --git a/sql/plan/values.go b/sql/plan/values.go index 65e06e78b..ea1d5be39 100644 --- a/sql/plan/values.go +++ b/sql/plan/values.go @@ -3,7 +3,7 @@ package plan import ( "fmt" - "gopkg.in/src-d/go-mysql-server.v0/sql" + "github.com/src-d/go-mysql-server/sql" ) // Values represents a set of tuples of expressions. @@ -76,25 +76,6 @@ func (p *Values) RowIter(ctx *sql.Context) (sql.RowIter, error) { return sql.RowsToRowIter(rows...), nil } -// TransformUp implements the Transformable interface. -func (p *Values) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) { - return f(p) -} - -// TransformExpressionsUp implements the Transformable interface. -func (p *Values) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) { - ets := make([][]sql.Expression, len(p.ExpressionTuples)) - var err error - for i, et := range p.ExpressionTuples { - ets[i], err = transformExpressionsUp(f, et) - if err != nil { - return nil, err - } - } - - return NewValues(ets), nil -} - func (p *Values) String() string { return fmt.Sprintf("Values(%d tuples)", len(p.ExpressionTuples)) } @@ -108,15 +89,33 @@ func (p *Values) Expressions() []sql.Expression { return exprs } -// TransformExpressions implements the Expressioner interface. -func (p *Values) TransformExpressions(f sql.TransformExprFunc) (sql.Node, error) { - tuples := [][]sql.Expression{} - for _, tuple := range p.ExpressionTuples { - transformed, err := transformExpressionsUp(f, tuple) - if err != nil { - return nil, err +// WithChildren implements the Node interface. +func (p *Values) WithChildren(children ...sql.Node) (sql.Node, error) { + if len(children) != 0 { + return nil, sql.ErrInvalidChildrenNumber.New(p, len(children), 0) + } + + return p, nil +} + +// WithExpressions implements the Expressioner interface. +func (p *Values) WithExpressions(exprs ...sql.Expression) (sql.Node, error) { + var expected int + for _, t := range p.ExpressionTuples { + expected += len(t) + } + + if len(exprs) != expected { + return nil, sql.ErrInvalidChildrenNumber.New(p, len(exprs), expected) + } + + var offset int + var tuples = make([][]sql.Expression, len(p.ExpressionTuples)) + for i, t := range p.ExpressionTuples { + for range t { + tuples[i] = append(tuples[i], exprs[offset]) + offset++ } - tuples = append(tuples, transformed) } return NewValues(tuples), nil diff --git a/sql/plan/walk.go b/sql/plan/walk.go index 3a940a0b4..6e43524e0 100644 --- a/sql/plan/walk.go +++ b/sql/plan/walk.go @@ -1,8 +1,8 @@ package plan import ( - "gopkg.in/src-d/go-mysql-server.v0/sql" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" ) // Visitor visits nodes in the plan. @@ -52,9 +52,9 @@ func Inspect(node sql.Node, f func(sql.Node) bool) { // expression it finds. func WalkExpressions(v expression.Visitor, node sql.Node) { Inspect(node, func(node sql.Node) bool { - if node, ok := node.(sql.Expressioner); ok { - for _, e := range node.Expressions() { - expression.Walk(v, e) + if n, ok := node.(sql.Expressioner); ok { + for _, err := range n.Expressions() { + expression.Walk(v, err) } } return true diff --git a/sql/plan/walk_test.go b/sql/plan/walk_test.go index 8d86981ed..a07230d96 100644 --- a/sql/plan/walk_test.go +++ b/sql/plan/walk_test.go @@ -3,8 +3,8 @@ package plan import ( "testing" + "github.com/src-d/go-mysql-server/sql" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-mysql-server.v0/sql" ) func TestWalk(t *testing.T) { diff --git a/sql/processlist.go b/sql/processlist.go index 425311a58..5bc1fecf6 100644 --- a/sql/processlist.go +++ b/sql/processlist.go @@ -6,22 +6,52 @@ import ( "sync" "time" + "github.com/sirupsen/logrus" "gopkg.in/src-d/go-errors.v1" ) // Progress between done items and total items. type Progress struct { + Name string Done int64 Total int64 } -func (p Progress) String() string { +func (p Progress) totalString() string { var total = "?" if p.Total > 0 { total = fmt.Sprint(p.Total) } + return total +} + +// TableProgress keeps track of a table progress, and for each of its partitions +type TableProgress struct { + Progress + PartitionsProgress map[string]PartitionProgress +} + +func NewTableProgress(name string, total int64) TableProgress { + return TableProgress{ + Progress: Progress{ + Name: name, + Total: total, + }, + PartitionsProgress: make(map[string]PartitionProgress), + } +} + +func (p TableProgress) String() string { + return fmt.Sprintf("%s (%d/%s partitions)", p.Name, p.Done, p.totalString()) +} + +// PartitionProgress keeps track of a partition progress +type PartitionProgress struct { + Progress +} - return fmt.Sprintf("%d/%s", p.Done, total) +func (p PartitionProgress) String() string { + return fmt.Sprintf("%s (%d/%s rows)", p.Name, p.Done, p.totalString()) } // ProcessType is the type of process. @@ -52,7 +82,7 @@ type Process struct { User string Type ProcessType Query string - Progress map[string]Progress + Progress map[string]TableProgress StartedAt time.Time Kill context.CancelFunc } @@ -107,7 +137,7 @@ func (pl *ProcessList) AddProcess( Connection: ctx.ID(), Type: typ, Query: query, - Progress: make(map[string]Progress), + Progress: make(map[string]TableProgress), User: ctx.Session.Client().User, StartedAt: time.Now(), Kill: cancel, @@ -116,9 +146,9 @@ func (pl *ProcessList) AddProcess( return ctx, nil } -// UpdateProgress updates the progress of the item with the given name for the +// UpdateTableProgress updates the progress of the table with the given name for the // process with the given pid. -func (pl *ProcessList) UpdateProgress(pid uint64, name string, delta int64) { +func (pl *ProcessList) UpdateTableProgress(pid uint64, name string, delta int64) { pl.mu.Lock() defer pl.mu.Unlock() @@ -129,16 +159,41 @@ func (pl *ProcessList) UpdateProgress(pid uint64, name string, delta int64) { progress, ok := p.Progress[name] if !ok { - progress = Progress{Total: -1} + progress = NewTableProgress(name, -1) } progress.Done += delta p.Progress[name] = progress } -// AddProgressItem adds a new item to track progress from to the process with +// UpdatePartitionProgress updates the progress of the table partition with the +// given name for the process with the given pid. +func (pl *ProcessList) UpdatePartitionProgress(pid uint64, tableName, partitionName string, delta int64) { + pl.mu.Lock() + defer pl.mu.Unlock() + + p, ok := pl.procs[pid] + if !ok { + return + } + + tablePg, ok := p.Progress[tableName] + if !ok { + return + } + + partitionPg, ok := tablePg.PartitionsProgress[partitionName] + if !ok { + partitionPg = PartitionProgress{Progress: Progress{Name: partitionName, Total: -1}} + } + + partitionPg.Done += delta + tablePg.PartitionsProgress[partitionName] = partitionPg +} + +// AddTableProgress adds a new item to track progress from to the process with // the given pid. If the pid does not exist, it will do nothing. -func (pl *ProcessList) AddProgressItem(pid uint64, name string, total int64) { +func (pl *ProcessList) AddTableProgress(pid uint64, name string, total int64) { pl.mu.Lock() defer pl.mu.Unlock() @@ -151,22 +206,91 @@ func (pl *ProcessList) AddProgressItem(pid uint64, name string, total int64) { pg.Total = total p.Progress[name] = pg } else { - p.Progress[name] = Progress{Total: total} + p.Progress[name] = NewTableProgress(name, total) + } +} + +// AddPartitionProgress adds a new item to track progress from to the process with +// the given pid. If the pid or the table does not exist, it will do nothing. +func (pl *ProcessList) AddPartitionProgress(pid uint64, tableName, partitionName string, total int64) { + pl.mu.Lock() + defer pl.mu.Unlock() + + p, ok := pl.procs[pid] + if !ok { + return + } + + tablePg, ok := p.Progress[tableName] + if !ok { + return + } + + if pg, ok := tablePg.PartitionsProgress[partitionName]; ok { + pg.Total = total + tablePg.PartitionsProgress[partitionName] = pg + } else { + tablePg.PartitionsProgress[partitionName] = + PartitionProgress{Progress: Progress{Name: partitionName, Total: total}} + } +} + +// RemoveTableProgress removes an existing item tracking progress from the +// process with the given pid, if it exists. +func (pl *ProcessList) RemoveTableProgress(pid uint64, name string) { + pl.mu.Lock() + defer pl.mu.Unlock() + + p, ok := pl.procs[pid] + if !ok { + return + } + + delete(p.Progress, name) +} + +// RemovePartitionProgress removes an existing item tracking progress from the +// process with the given pid, if it exists. +func (pl *ProcessList) RemovePartitionProgress(pid uint64, tableName, partitionName string) { + pl.mu.Lock() + defer pl.mu.Unlock() + + p, ok := pl.procs[pid] + if !ok { + return + } + + tablePg, ok := p.Progress[tableName] + if !ok { + return } + + delete(tablePg.PartitionsProgress, partitionName) } -// Kill terminates a process if it exists. -func (pl *ProcessList) Kill(pid uint64) { - pl.Done(pid) +// Kill terminates all queries for a given connection id. +func (pl *ProcessList) Kill(connID uint32) { + pl.mu.Lock() + defer pl.mu.Unlock() + + for pid, proc := range pl.procs { + if proc.Connection == connID { + logrus.Infof("kill query: pid %d", pid) + proc.Done() + delete(pl.procs, pid) + } + } } -// KillConnection kills all processes from the given connection. -func (pl *ProcessList) KillConnection(conn uint32) { +// KillOnlyQueries kills all queries, but not index creation queries, for a +// given connection id. +func (pl *ProcessList) KillOnlyQueries(connID uint32) { pl.mu.Lock() defer pl.mu.Unlock() for pid, proc := range pl.procs { - if proc.Connection == conn { + if proc.Connection == connID && proc.Type == QueryProcess { + logrus.Infof("kill query: pid %d", pid) proc.Done() delete(pl.procs, pid) } @@ -194,7 +318,7 @@ func (pl *ProcessList) Processes() []Process { for _, proc := range pl.procs { p := *proc - var progress = make(map[string]Progress, len(p.Progress)) + var progress = make(map[string]TableProgress, len(p.Progress)) for n, p := range p.Progress { progress[n] = p } diff --git a/sql/processlist_test.go b/sql/processlist_test.go index c01533ede..198f40b12 100644 --- a/sql/processlist_test.go +++ b/sql/processlist_test.go @@ -20,16 +20,16 @@ func TestProcessList(t *testing.T) { require.Equal(uint64(1), ctx.Pid()) require.Len(p.procs, 1) - p.AddProgressItem(ctx.Pid(), "a", 5) - p.AddProgressItem(ctx.Pid(), "b", 6) + p.AddTableProgress(ctx.Pid(), "a", 5) + p.AddTableProgress(ctx.Pid(), "b", 6) expectedProcess := &Process{ Pid: 1, Connection: 1, Type: QueryProcess, - Progress: map[string]Progress{ - "a": Progress{0, 5}, - "b": Progress{0, 6}, + Progress: map[string]TableProgress{ + "a": {Progress{Name: "a", Done: 0, Total: 5}, map[string]PartitionProgress{}}, + "b": {Progress{Name: "b", Done: 0, Total: 6}, map[string]PartitionProgress{}}, }, User: "foo", Query: "SELECT foo", @@ -39,19 +39,36 @@ func TestProcessList(t *testing.T) { p.procs[ctx.Pid()].Kill = nil require.Equal(expectedProcess, p.procs[ctx.Pid()]) + p.AddPartitionProgress(ctx.Pid(), "b", "b-1", -1) + p.AddPartitionProgress(ctx.Pid(), "b", "b-2", -1) + p.AddPartitionProgress(ctx.Pid(), "b", "b-3", -1) + + p.UpdatePartitionProgress(ctx.Pid(), "b", "b-2", 1) + + p.RemovePartitionProgress(ctx.Pid(), "b", "b-3") + + expectedProgress := map[string]TableProgress{ + "a": {Progress{Name: "a", Total: 5}, map[string]PartitionProgress{}}, + "b": {Progress{Name: "b", Total: 6}, map[string]PartitionProgress{ + "b-1": {Progress{Name: "b-1", Done: 0, Total: -1}}, + "b-2": {Progress{Name: "b-2", Done: 1, Total: -1}}, + }}, + } + require.Equal(expectedProgress, p.procs[ctx.Pid()].Progress) + ctx = NewContext(context.Background(), WithPid(2), WithSession(sess)) ctx, err = p.AddProcess(ctx, CreateIndexProcess, "SELECT bar") require.NoError(err) - p.AddProgressItem(ctx.Pid(), "foo", 2) + p.AddTableProgress(ctx.Pid(), "foo", 2) require.Equal(uint64(2), ctx.Pid()) require.Len(p.procs, 2) - p.UpdateProgress(1, "a", 3) - p.UpdateProgress(1, "a", 1) - p.UpdateProgress(1, "b", 2) - p.UpdateProgress(2, "foo", 1) + p.UpdateTableProgress(1, "a", 3) + p.UpdateTableProgress(1, "a", 1) + p.UpdateTableProgress(1, "b", 2) + p.UpdateTableProgress(2, "foo", 1) require.Equal(int64(4), p.procs[1].Progress["a"].Done) require.Equal(int64(2), p.procs[1].Progress["b"].Done) @@ -113,7 +130,7 @@ func TestKillConnection(t *testing.T) { } } - pl.KillConnection(1) + pl.Kill(1) require.Len(t, pl.procs, 1) // Odds should have been killed diff --git a/sql/session.go b/sql/session.go index 2449a492d..251467ba0 100644 --- a/sql/session.go +++ b/sql/session.go @@ -175,6 +175,8 @@ func DefaultSessionConfig() map[string]TypedValue { "ndbinfo_version": TypedValue{Text, ""}, "sql_select_limit": TypedValue{Int32, math.MaxInt32}, "transaction_isolation": TypedValue{Text, "READ UNCOMMITTED"}, + "version": TypedValue{Text, ""}, + "version_comment": TypedValue{Text, ""}, } } @@ -209,9 +211,11 @@ func NewBaseSession() Session { type Context struct { context.Context Session - pid uint64 - query string - tracer opentracing.Tracer + Memory *MemoryManager + pid uint64 + query string + tracer opentracing.Tracer + rootSpan opentracing.Span } // ContextOption is a function to configure the context. @@ -238,25 +242,44 @@ func WithPid(pid uint64) ContextOption { } } -// WithQuery add the given query to the context. +// WithQuery adds the given query to the context. func WithQuery(q string) ContextOption { return func(ctx *Context) { ctx.query = q } } +// WithMemoryManager adds the given memory manager to the context. +func WithMemoryManager(m *MemoryManager) ContextOption { + return func(ctx *Context) { + ctx.Memory = m + } +} + +// WithRootSpan sets the root span of the context. +func WithRootSpan(s opentracing.Span) ContextOption { + return func(ctx *Context) { + ctx.rootSpan = s + } +} + // NewContext creates a new query context. Options can be passed to configure // the context. If some aspect of the context is not configure, the default // value will be used. -// By default, the context will have an empty base session and a noop tracer. +// By default, the context will have an empty base session, a noop tracer and +// a memory manager using the process reporter. func NewContext( ctx context.Context, opts ...ContextOption, ) *Context { - c := &Context{ctx, NewBaseSession(), 0, "", opentracing.NoopTracer{}} + c := &Context{ctx, NewBaseSession(), nil, 0, "", opentracing.NoopTracer{}, nil} for _, opt := range opts { opt(c) } + + if c.Memory == nil { + c.Memory = NewMemoryManager(ProcessMemory) + } return c } @@ -283,12 +306,17 @@ func (c *Context) Span( span := c.tracer.StartSpan(opName, opts...) ctx := opentracing.ContextWithSpan(c.Context, span) - return span, &Context{ctx, c.Session, c.Pid(), c.Query(), c.tracer} + return span, &Context{ctx, c.Session, c.Memory, c.Pid(), c.Query(), c.tracer, c.rootSpan} } // WithContext returns a new context with the given underlying context. func (c *Context) WithContext(ctx context.Context) *Context { - return &Context{ctx, c.Session, c.Pid(), c.Query(), c.tracer} + return &Context{ctx, c.Session, c.Memory, c.Pid(), c.Query(), c.tracer, c.rootSpan} +} + +// RootSpan returns the root span, if any. +func (c *Context) RootSpan() opentracing.Span { + return c.rootSpan } // Error adds an error as warning to the session. diff --git a/sql/session_test.go b/sql/session_test.go index 3a109779f..315015ec5 100644 --- a/sql/session_test.go +++ b/sql/session_test.go @@ -51,27 +51,22 @@ func TestHasDefaultValue(t *testing.T) { type testNode struct{} -func (t *testNode) Resolved() bool { +func (*testNode) Resolved() bool { panic("not implemented") } - -func (t *testNode) TransformUp(func(Node) Node) Node { - panic("not implemented") -} - -func (t *testNode) TransformExpressionsUp(func(Expression) Expression) Node { +func (*testNode) WithChildren(...Node) (Node, error) { panic("not implemented") } -func (t *testNode) Schema() Schema { +func (*testNode) Schema() Schema { panic("not implemented") } -func (t *testNode) Children() []Node { +func (*testNode) Children() []Node { panic("not implemented") } -func (t *testNode) RowIter(ctx *Context) (RowIter, error) { +func (*testNode) RowIter(ctx *Context) (RowIter, error) { return newTestNodeIterator(ctx), nil } diff --git a/sql/type.go b/sql/type.go index 714b10b66..a93e7d83e 100644 --- a/sql/type.go +++ b/sql/type.go @@ -4,6 +4,8 @@ import ( "bytes" "encoding/json" "fmt" + "io" + "math" "reflect" "strconv" "strings" @@ -11,8 +13,8 @@ import ( "github.com/spf13/cast" "gopkg.in/src-d/go-errors.v1" - "gopkg.in/src-d/go-vitess.v1/sqltypes" - "gopkg.in/src-d/go-vitess.v1/vt/proto/query" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/proto/query" ) var ( @@ -25,10 +27,16 @@ var ( // ErrConvertingToTime is thrown when a value cannot be converted to a Time ErrConvertingToTime = errors.NewKind("value %q can't be converted to time.Time") + // ErrCharTruncation is thrown when a Char value is textually longer than the destination capacity + ErrCharTruncation = errors.NewKind("string value of %q is longer than destination capacity %d") + + // ErrVarCharTruncation is thrown when a VarChar value is textually longer than the destination capacity + ErrVarCharTruncation = errors.NewKind("string value of %q is longer than destination capacity %d") + // ErrValueNotNil is thrown when a value that was expected to be nil, is not ErrValueNotNil = errors.NewKind("value not nil: %#v") - // ErrNotTuple is retuned when the value is not a tuple. + // ErrNotTuple is returned when the value is not a tuple. ErrNotTuple = errors.NewKind("value of type %T is not a tuple") // ErrInvalidColumnNumber is returned when a tuple has an invalid number of @@ -116,6 +124,8 @@ type Column struct { Nullable bool // Source is the name of the table this column came from. Source string + // PrimaryKey is true if the column is part of the primary key for its table. + PrimaryKey bool } // Check ensures the value is correct for this column. @@ -147,33 +157,58 @@ type Type interface { // The result will be 0 if a==b, -1 if a < b, and +1 if a > b. Compare(interface{}, interface{}) (int, error) // SQL returns the sqltypes.Value for the given value. - SQL(interface{}) sqltypes.Value + SQL(interface{}) (sqltypes.Value, error) fmt.Stringer } +var maxTime = time.Date(9999, time.December, 31, 23, 59, 59, 0, time.UTC) + +// ValidateTime receives a time and returns either that time or nil if it's +// not a valid time. +func ValidateTime(t time.Time) interface{} { + if t.After(maxTime) { + return nil + } + return t +} + var ( // Null represents the null type. Null nullT // Numeric types + // Int8 is an integer of 8 bits + Int8 = numberT{t: sqltypes.Int8} + // Uint8 is an unsigned integer of 8 bits + Uint8 = numberT{t: sqltypes.Uint8} + // Int16 is an integer of 16 bits + Int16 = numberT{t: sqltypes.Int16} + // Uint16 is an unsigned integer of 16 bits + Uint16 = numberT{t: sqltypes.Uint16} + // Int24 is an integer of 24 bits. + Int24 = numberT{t: sqltypes.Int24} + // Uint24 is an unsigned integer of 24 bits. + Uint24 = numberT{t: sqltypes.Uint24} // Int32 is an integer of 32 bits. Int32 = numberT{t: sqltypes.Int32} + // Uint32 is an unsigned integer of 32 bits. + Uint32 = numberT{t: sqltypes.Uint32} // Int64 is an integer of 64 bytes. Int64 = numberT{t: sqltypes.Int64} - // Uint32 is an unsigned integer of 32 bytes. - Uint32 = numberT{t: sqltypes.Uint32} - // Uint64 is an unsigned integer of 64 bytes. + // Uint64 is an unsigned integer of 64 bits. Uint64 = numberT{t: sqltypes.Uint64} - // Float32 is a floating point number of 32 bytes. + // Float32 is a floating point number of 32 bits. Float32 = numberT{t: sqltypes.Float32} - // Float64 is a floating point number of 64 bytes. + // Float64 is a floating point number of 64 bits. Float64 = numberT{t: sqltypes.Float64} // Timestamp is an UNIX timestamp. Timestamp timestampT // Date is a date with day, month and year. Date dateT + // Datetime is a date and a time + Datetime datetimeT // Text is a string type. Text textT // Boolean is a boolean type. @@ -194,17 +229,39 @@ func Array(underlying Type) Type { return arrayT{underlying} } +// Char returns a new Char type of the given length. +func Char(length int) Type { + return charT{length: length} +} + +// VarChar returns a new VarChar type of the given length. +func VarChar(length int) Type { + return varCharT{length: length} +} + // MysqlTypeToType gets the column type using the mysql type func MysqlTypeToType(sql query.Type) (Type, error) { switch sql { case sqltypes.Null: return Null, nil + case sqltypes.Int8: + return Int8, nil + case sqltypes.Uint8: + return Uint8, nil + case sqltypes.Int16: + return Int16, nil + case sqltypes.Uint16: + return Uint16, nil + case sqltypes.Int24: + return Int24, nil + case sqltypes.Uint24: + return Uint24, nil case sqltypes.Int32: return Int32, nil - case sqltypes.Int64: - return Int64, nil case sqltypes.Uint32: return Uint32, nil + case sqltypes.Int64: + return Int64, nil case sqltypes.Uint64: return Uint64, nil case sqltypes.Float32: @@ -215,8 +272,18 @@ func MysqlTypeToType(sql query.Type) (Type, error) { return Timestamp, nil case sqltypes.Date: return Date, nil - case sqltypes.Text, sqltypes.VarChar: + case sqltypes.Text: + return Text, nil + case sqltypes.Char: + // Since we can't get the size of the sqltypes.Char to instantiate a + // specific Char(length) type we return a Text here + return Text, nil + case sqltypes.VarChar: + // Since we can't get the size of the sqltypes.VarChar to instantiate a + // specific VarChar(length) type we return a Text here return Text, nil + case sqltypes.Datetime: + return Datetime, nil case sqltypes.Bit: return Boolean, nil case sqltypes.TypeJSON: @@ -238,8 +305,8 @@ func (t nullT) Type() query.Type { } // SQL implements Type interface. -func (t nullT) SQL(interface{}) sqltypes.Value { - return sqltypes.NULL +func (t nullT) SQL(interface{}) (sqltypes.Value, error) { + return sqltypes.NULL, nil } // Convert implements Type interface. @@ -272,36 +339,56 @@ func (t numberT) Type() query.Type { } // SQL implements Type interface. -func (t numberT) SQL(v interface{}) sqltypes.Value { - if _, ok := v.(nullT); ok { - return sqltypes.NULL +func (t numberT) SQL(v interface{}) (sqltypes.Value, error) { + if v == nil { + return sqltypes.NULL, nil } switch t.t { + case sqltypes.Int8: + return sqltypes.MakeTrusted(t.t, strconv.AppendInt(nil, cast.ToInt64(v), 10)), nil + case sqltypes.Int16: + return sqltypes.MakeTrusted(t.t, strconv.AppendInt(nil, cast.ToInt64(v), 10)), nil case sqltypes.Int32: - return sqltypes.MakeTrusted(t.t, strconv.AppendInt(nil, cast.ToInt64(v), 10)) + return sqltypes.MakeTrusted(t.t, strconv.AppendInt(nil, cast.ToInt64(v), 10)), nil case sqltypes.Int64: - return sqltypes.MakeTrusted(t.t, strconv.AppendInt(nil, cast.ToInt64(v), 10)) + return sqltypes.MakeTrusted(t.t, strconv.AppendInt(nil, cast.ToInt64(v), 10)), nil + case sqltypes.Uint8: + return sqltypes.MakeTrusted(t.t, strconv.AppendUint(nil, cast.ToUint64(v), 10)), nil + case sqltypes.Uint16: + return sqltypes.MakeTrusted(t.t, strconv.AppendUint(nil, cast.ToUint64(v), 10)), nil case sqltypes.Uint32: - return sqltypes.MakeTrusted(t.t, strconv.AppendUint(nil, cast.ToUint64(v), 10)) + return sqltypes.MakeTrusted(t.t, strconv.AppendUint(nil, cast.ToUint64(v), 10)), nil case sqltypes.Uint64: - return sqltypes.MakeTrusted(t.t, strconv.AppendUint(nil, cast.ToUint64(v), 10)) + return sqltypes.MakeTrusted(t.t, strconv.AppendUint(nil, cast.ToUint64(v), 10)), nil case sqltypes.Float32: - return sqltypes.MakeTrusted(t.t, strconv.AppendFloat(nil, cast.ToFloat64(v), 'f', -1, 64)) + return sqltypes.MakeTrusted(t.t, strconv.AppendFloat(nil, cast.ToFloat64(v), 'f', -1, 64)), nil case sqltypes.Float64: - return sqltypes.MakeTrusted(t.t, strconv.AppendFloat(nil, cast.ToFloat64(v), 'f', -1, 64)) + return sqltypes.MakeTrusted(t.t, strconv.AppendFloat(nil, cast.ToFloat64(v), 'f', -1, 64)), nil default: - return sqltypes.MakeTrusted(t.t, []byte{}) + return sqltypes.MakeTrusted(t.t, []byte{}), nil } } // Convert implements Type interface. func (t numberT) Convert(v interface{}) (interface{}, error) { + if ti, ok := v.(time.Time); ok { + v = ti.Unix() + } + switch t.t { + case sqltypes.Int8: + return cast.ToInt8E(v) + case sqltypes.Int16: + return cast.ToInt16E(v) case sqltypes.Int32: return cast.ToInt32E(v) case sqltypes.Int64: return cast.ToInt64E(v) + case sqltypes.Uint8: + return cast.ToUint8E(v) + case sqltypes.Uint16: + return cast.ToUint16E(v) case sqltypes.Uint32: return cast.ToUint32E(v) case sqltypes.Uint64: @@ -318,15 +405,50 @@ func (t numberT) Convert(v interface{}) (interface{}, error) { // Compare implements Type interface. func (t numberT) Compare(a interface{}, b interface{}) (int, error) { if IsUnsigned(t) { - return compareUnsigned(a, b) + // only int types are unsigned + return compareUnsignedInts(a, b) } - return compareSigned(a, b) + switch t.t { + case sqltypes.Float64, sqltypes.Float32: + return compareFloats(a, b) + default: + return compareSignedInts(a, b) + } } func (t numberT) String() string { return t.t.String() } -func compareSigned(a interface{}, b interface{}) (int, error) { +func compareFloats(a interface{}, b interface{}) (int, error) { + if hasNulls, res := compareNulls(a, b); hasNulls { + return res, nil + } + + ca, err := cast.ToFloat64E(a) + if err != nil { + return 0, err + } + cb, err := cast.ToFloat64E(b) + if err != nil { + return 0, err + } + + if ca == cb { + return 0, nil + } + + if ca < cb { + return -1, nil + } + + return +1, nil +} + +func compareSignedInts(a interface{}, b interface{}) (int, error) { + if hasNulls, res := compareNulls(a, b); hasNulls { + return res, nil + } + ca, err := cast.ToInt64E(a) if err != nil { return 0, err @@ -347,7 +469,11 @@ func compareSigned(a interface{}, b interface{}) (int, error) { return +1, nil } -func compareUnsigned(a interface{}, b interface{}) (int, error) { +func compareUnsignedInts(a interface{}, b interface{}) (int, error) { + if hasNulls, res := compareNulls(a, b); hasNulls { + return res, nil + } + ca, err := cast.ToUint64E(a) if err != nil { return 0, err @@ -394,16 +520,20 @@ var TimestampLayouts = []string{ } // SQL implements Type interface. -func (t timestampT) SQL(v interface{}) sqltypes.Value { - if _, ok := v.(nullT); ok { - return sqltypes.NULL +func (t timestampT) SQL(v interface{}) (sqltypes.Value, error) { + if v == nil { + return sqltypes.NULL, nil + } + + v, err := t.Convert(v) + if err != nil { + return sqltypes.Value{}, err } - time := MustConvert(t, v).(time.Time) return sqltypes.MakeTrusted( sqltypes.Timestamp, - []byte(time.Format(TimestampLayout)), - ) + []byte(v.(time.Time).Format(TimestampLayout)), + ), nil } // Convert implements Type interface. @@ -440,6 +570,10 @@ func (t timestampT) Convert(v interface{}) (interface{}, error) { // Compare implements Type interface. func (t timestampT) Compare(a interface{}, b interface{}) (int, error) { + if hasNulls, res := compareNulls(a, b); hasNulls { + return res, nil + } + av := a.(time.Time) bv := b.(time.Time) if av.Before(bv) { @@ -466,16 +600,20 @@ func (t dateT) Type() query.Type { return sqltypes.Date } -func (t dateT) SQL(v interface{}) sqltypes.Value { - if _, ok := v.(nullT); ok { - return sqltypes.NULL +func (t dateT) SQL(v interface{}) (sqltypes.Value, error) { + if v == nil { + return sqltypes.NULL, nil + } + + v, err := t.Convert(v) + if err != nil { + return sqltypes.Value{}, err } - time := MustConvert(t, v).(time.Time) return sqltypes.MakeTrusted( sqltypes.Timestamp, - []byte(time.Format(DateLayout)), - ) + []byte(v.(time.Time).Format(DateLayout)), + ), nil } func (t dateT) Convert(v interface{}) (interface{}, error) { @@ -499,6 +637,10 @@ func (t dateT) Convert(v interface{}) (interface{}, error) { } func (t dateT) Compare(a, b interface{}) (int, error) { + if hasNulls, res := compareNulls(a, b); hasNulls { + return res, nil + } + av := truncateDate(a.(time.Time)) bv := truncateDate(b.(time.Time)) if av.Before(bv) { @@ -509,6 +651,156 @@ func (t dateT) Compare(a, b interface{}) (int, error) { return 0, nil } +type datetimeT struct{} + +// DatetimeLayout is the layout of the MySQL date format in the representation +// Go understands. +const DatetimeLayout = "2006-01-02 15:04:05" + +func (t datetimeT) String() string { return "DATETIME" } + +func (t datetimeT) Type() query.Type { + return sqltypes.Datetime +} + +func (t datetimeT) SQL(v interface{}) (sqltypes.Value, error) { + if v == nil { + return sqltypes.NULL, nil + } + + v, err := t.Convert(v) + if err != nil { + return sqltypes.Value{}, err + } + + return sqltypes.MakeTrusted( + sqltypes.Datetime, + []byte(v.(time.Time).Format(DatetimeLayout)), + ), nil +} + +func (t datetimeT) Convert(v interface{}) (interface{}, error) { + switch value := v.(type) { + case time.Time: + return value.UTC(), nil + case string: + t, err := time.Parse(DatetimeLayout, value) + if err != nil { + return nil, ErrConvertingToTime.Wrap(err, v) + } + return t.UTC(), nil + default: + ts, err := Int64.Convert(v) + if err != nil { + return nil, ErrInvalidType.New(reflect.TypeOf(v)) + } + + return time.Unix(ts.(int64), 0).UTC(), nil + } +} + +func (t datetimeT) Compare(a, b interface{}) (int, error) { + av := a.(time.Time) + bv := b.(time.Time) + if av.Before(bv) { + return -1, nil + } else if av.After(bv) { + return 1, nil + } + return 0, nil +} + +type charT struct { + length int +} + +func (t charT) Capacity() int { return t.length } + +func (t charT) String() string { return fmt.Sprintf("CHAR(%d)", t.length) } + +func (t charT) Type() query.Type { + return sqltypes.Char +} + +func (t charT) SQL(v interface{}) (sqltypes.Value, error) { + if v == nil { + return sqltypes.MakeTrusted(sqltypes.Char, nil), nil + } + + v, err := t.Convert(v) + if err != nil { + return sqltypes.Value{}, err + } + + return sqltypes.MakeTrusted(sqltypes.Char, []byte(v.(string))), nil +} + +// Converts any value that can be casted to a string +func (t charT) Convert(v interface{}) (interface{}, error) { + val, err := cast.ToStringE(v) + if err != nil { + return nil, ErrConvertToSQL.New(t) + } + + if len(val) > t.length { + return nil, ErrCharTruncation.New(val, t.length) + } + return val, nil +} + +// Compares two strings lexicographically +func (t charT) Compare(a interface{}, b interface{}) (int, error) { + return strings.Compare(a.(string), b.(string)), nil +} + +type varCharT struct { + length int +} + +func (t varCharT) Capacity() int { return t.length } + +func (t varCharT) String() string { return fmt.Sprintf("VARCHAR(%d)", t.length) } + +// Type implements Type interface +func (t varCharT) Type() query.Type { + return sqltypes.VarChar +} + +// SQL implements Type interface +func (t varCharT) SQL(v interface{}) (sqltypes.Value, error) { + if v == nil { + return sqltypes.MakeTrusted(sqltypes.VarChar, nil), nil + } + + v, err := t.Convert(v) + if err != nil { + return sqltypes.Value{}, err + } + + return sqltypes.MakeTrusted(sqltypes.VarChar, []byte(v.(string))), nil +} + +// Convert implements Type interface +func (t varCharT) Convert(v interface{}) (interface{}, error) { + val, err := cast.ToStringE(v) + if err != nil { + return nil, ErrConvertToSQL.New(t) + } + + if len(val) > t.length { + return nil, ErrVarCharTruncation.New(val, t.length) + } + return val, nil +} + +// Compare implements Type interface. +func (t varCharT) Compare(a interface{}, b interface{}) (int, error) { + if hasNulls, res := compareNulls(a, b); hasNulls { + return res, nil + } + return strings.Compare(a.(string), b.(string)), nil +} + type textT struct{} func (t textT) String() string { return "TEXT" } @@ -519,12 +811,17 @@ func (t textT) Type() query.Type { } // SQL implements Type interface. -func (t textT) SQL(v interface{}) sqltypes.Value { - if _, ok := v.(nullT); ok { - return sqltypes.NULL +func (t textT) SQL(v interface{}) (sqltypes.Value, error) { + if v == nil { + return sqltypes.NULL, nil } - return sqltypes.MakeTrusted(sqltypes.Text, []byte(MustConvert(t, v).(string))) + v, err := t.Convert(v) + if err != nil { + return sqltypes.Value{}, err + } + + return sqltypes.MakeTrusted(sqltypes.Text, []byte(v.(string))), nil } // Convert implements Type interface. @@ -538,6 +835,9 @@ func (t textT) Convert(v interface{}) (interface{}, error) { // Compare implements Type interface. func (t textT) Compare(a interface{}, b interface{}) (int, error) { + if hasNulls, res := compareNulls(a, b); hasNulls { + return res, nil + } return strings.Compare(a.(string), b.(string)), nil } @@ -551,9 +851,9 @@ func (t booleanT) Type() query.Type { } // SQL implements Type interface. -func (t booleanT) SQL(v interface{}) sqltypes.Value { - if _, ok := v.(nullT); ok { - return sqltypes.NULL +func (t booleanT) SQL(v interface{}) (sqltypes.Value, error) { + if v == nil { + return sqltypes.NULL, nil } b := []byte{'0'} @@ -561,20 +861,39 @@ func (t booleanT) SQL(v interface{}) sqltypes.Value { b[0] = '1' } - return sqltypes.MakeTrusted(sqltypes.Bit, b) + return sqltypes.MakeTrusted(sqltypes.Bit, b), nil } // Convert implements Type interface. func (t booleanT) Convert(v interface{}) (interface{}, error) { - val, err := cast.ToBoolE(v) - if err != nil { - return nil, ErrConvertToSQL.New(t) + switch b := v.(type) { + case bool: + return b, nil + case int, int64, int32, int16, int8, uint, uint64, uint32, uint16, uint8: + return b != 0, nil + case time.Duration: + return int64(b) != 0, nil + case time.Time: + return b.UnixNano() != 0, nil + case float32, float64: + return int(math.Round(v.(float64))) != 0, nil + case string: + return false, nil + + case nil: + return nil, fmt.Errorf("unable to cast nil to bool") + + default: + return nil, fmt.Errorf("unable to cast %#v of type %T to bool", v, v) } - return val, nil } // Compare implements Type interface. func (t booleanT) Compare(a interface{}, b interface{}) (int, error) { + if hasNulls, res := compareNulls(a, b); hasNulls { + return res, nil + } + if a == b { return 0, nil } @@ -596,12 +915,17 @@ func (t blobT) Type() query.Type { } // SQL implements Type interface. -func (t blobT) SQL(v interface{}) sqltypes.Value { - if _, ok := v.(nullT); ok { - return sqltypes.NULL +func (t blobT) SQL(v interface{}) (sqltypes.Value, error) { + if v == nil { + return sqltypes.NULL, nil } - return sqltypes.MakeTrusted(sqltypes.Blob, MustConvert(t, v).([]byte)) + v, err := t.Convert(v) + if err != nil { + return sqltypes.Value{}, err + } + + return sqltypes.MakeTrusted(sqltypes.Blob, v.([]byte)), nil } // Convert implements Type interface. @@ -622,6 +946,9 @@ func (t blobT) Convert(v interface{}) (interface{}, error) { // Compare implements Type interface. func (t blobT) Compare(a interface{}, b interface{}) (int, error) { + if hasNulls, res := compareNulls(a, b); hasNulls { + return res, nil + } return bytes.Compare(a.([]byte), b.([]byte)), nil } @@ -635,20 +962,38 @@ func (t jsonT) Type() query.Type { } // SQL implements Type interface. -func (t jsonT) SQL(v interface{}) sqltypes.Value { - if _, ok := v.(nullT); ok { - return sqltypes.NULL +func (t jsonT) SQL(v interface{}) (sqltypes.Value, error) { + if v == nil { + return sqltypes.NULL, nil } - return sqltypes.MakeTrusted(sqltypes.TypeJSON, MustConvert(t, v).([]byte)) + + v, err := t.Convert(v) + if err != nil { + return sqltypes.Value{}, err + } + + return sqltypes.MakeTrusted(sqltypes.TypeJSON, v.([]byte)), nil } // Convert implements Type interface. func (t jsonT) Convert(v interface{}) (interface{}, error) { - return json.Marshal(v) + switch v := v.(type) { + case string: + var doc interface{} + if err := json.Unmarshal([]byte(v), &doc); err != nil { + return json.Marshal(v) + } + return json.Marshal(doc) + default: + return json.Marshal(v) + } } // Compare implements Type interface. func (t jsonT) Compare(a interface{}, b interface{}) (int, error) { + if hasNulls, res := compareNulls(a, b); hasNulls { + return res, nil + } return bytes.Compare(a.([]byte), b.([]byte)), nil } @@ -666,12 +1011,8 @@ func (t tupleT) Type() query.Type { return sqltypes.Expression } -func (t tupleT) SQL(v interface{}) sqltypes.Value { - if _, ok := v.(nullT); ok { - return sqltypes.NULL - } - - panic("unable to convert tuple type to SQL") +func (t tupleT) SQL(v interface{}) (sqltypes.Value, error) { + return sqltypes.Value{}, fmt.Errorf("unable to convert tuple type to SQL") } func (t tupleT) Convert(v interface{}) (interface{}, error) { @@ -731,24 +1072,63 @@ func (t arrayT) Type() query.Type { return sqltypes.TypeJSON } -func (t arrayT) SQL(v interface{}) sqltypes.Value { - return JSON.SQL(v) +func (t arrayT) SQL(v interface{}) (sqltypes.Value, error) { + if v == nil { + return sqltypes.NULL, nil + } + + v, err := convertForJSON(t, v) + if err != nil { + return sqltypes.Value{}, err + } + + val, err := json.Marshal(v) + if err != nil { + return sqltypes.Value{}, err + } + + return sqltypes.MakeTrusted(sqltypes.TypeJSON, val), nil } func (t arrayT) Convert(v interface{}) (interface{}, error) { - if vals, ok := v.([]interface{}); ok { - var result = make([]interface{}, len(vals)) - for i, v := range vals { + switch v := v.(type) { + case []interface{}: + var result = make([]interface{}, len(v)) + for i, v := range v { var err error result[i], err = t.underlying.Convert(v) if err != nil { return nil, err } } - return result, nil + case Generator: + var values []interface{} + for { + val, err := v.Next() + if err != nil { + if err == io.EOF { + break + } + return nil, err + } + + val, err = t.underlying.Convert(val) + if err != nil { + return nil, err + } + + values = append(values, val) + } + + if err := v.Close(); err != nil { + return nil, err + } + + return values, nil + default: + return nil, ErrNotArray.New(v) } - return nil, ErrNotArray.New(v) } func (t arrayT) Compare(a, b interface{}) (int, error) { @@ -785,16 +1165,6 @@ func (t arrayT) Compare(a, b interface{}) (int, error) { return 0, nil } -// MustConvert calls the Convert function from a given Type, it err panics. -func MustConvert(t Type, v interface{}) interface{} { - c, err := t.Convert(v) - if err != nil { - panic(err) - } - - return c -} - // IsNumber checks if t is a number type func IsNumber(t Type) bool { return IsInteger(t) || IsDecimal(t) @@ -802,19 +1172,24 @@ func IsNumber(t Type) bool { // IsSigned checks if t is a signed type. func IsSigned(t Type) bool { - return t == Int32 || t == Int64 + return t == Int8 || t == Int16 || t == Int32 || t == Int64 } // IsUnsigned checks if t is an unsigned type. func IsUnsigned(t Type) bool { - return t == Uint32 || t == Uint64 + return t == Uint8 || t == Uint16 || t == Uint32 || t == Uint64 } -// IsInteger check if t is a (U)Int32/64 type +// IsInteger checks if t is a (U)Int32/64 type. func IsInteger(t Type) bool { return IsSigned(t) || IsUnsigned(t) } +// IsTime checks if t is a timestamp, date or datetime +func IsTime(t Type) bool { + return t == Timestamp || t == Date || t == Datetime +} + // IsDecimal checks if t is decimal type. func IsDecimal(t Type) bool { return t == Float32 || t == Float64 @@ -822,7 +1197,19 @@ func IsDecimal(t Type) bool { // IsText checks if t is a text type. func IsText(t Type) bool { - return t == Text || t == Blob || t == JSON + return t == Text || t == Blob || t == JSON || IsVarChar(t) || IsChar(t) +} + +// IsChar checks if t is a Char type. +func IsChar(t Type) bool { + _, ok := t.(charT) + return ok +} + +// IsVarChar checks if t is a varchar type. +func IsVarChar(t Type) bool { + _, ok := t.(varCharT) + return ok } // IsTuple checks if t is a tuple type. @@ -848,3 +1235,139 @@ func NumColumns(t Type) int { } return len(v) } + +// MySQLTypeName returns the MySQL display name for the given type. +func MySQLTypeName(t Type) string { + switch t.Type() { + case sqltypes.Int8: + return "TINYINT" + case sqltypes.Uint8: + return "TINYINT UNSIGNED" + case sqltypes.Int16: + return "SMALLINT" + case sqltypes.Uint16: + return "SMALLINT UNSIGNED" + case sqltypes.Int32: + return "INTEGER" + case sqltypes.Int64: + return "BIGINT" + case sqltypes.Uint32: + return "INTEGER UNSIGNED" + case sqltypes.Uint64: + return "BIGINT UNSIGNED" + case sqltypes.Float32: + return "FLOAT" + case sqltypes.Float64: + return "DOUBLE" + case sqltypes.Timestamp: + return "TIMESTAMP" + case sqltypes.Datetime: + return "DATETIME" + case sqltypes.Date: + return "DATE" + case sqltypes.Char: + return fmt.Sprintf("CHAR(%v)", t.(charT).Capacity()) + case sqltypes.VarChar: + return fmt.Sprintf("VARCHAR(%v)", t.(varCharT).Capacity()) + case sqltypes.Text: + return "TEXT" + case sqltypes.Bit: + return "BIT" + case sqltypes.TypeJSON: + return "JSON" + case sqltypes.Blob: + return "BLOB" + default: + return "UNKNOWN" + } +} + +// UnderlyingType returns the underlying type of an array if the type is an +// array, or the type itself in any other case. +func UnderlyingType(t Type) Type { + a, ok := t.(arrayT) + if !ok { + return t + } + + return a.underlying +} + +func convertForJSON(t Type, v interface{}) (interface{}, error) { + switch t := t.(type) { + case jsonT: + val, err := t.Convert(v) + if err != nil { + return nil, err + } + + var doc interface{} + err = json.Unmarshal(val.([]byte), &doc) + if err != nil { + return nil, err + } + + return doc, nil + case arrayT: + return convertArrayForJSON(t, v) + default: + return t.Convert(v) + } +} + +func convertArrayForJSON(t arrayT, v interface{}) (interface{}, error) { + switch v := v.(type) { + case []interface{}: + var result = make([]interface{}, len(v)) + for i, v := range v { + var err error + result[i], err = convertForJSON(t.underlying, v) + if err != nil { + return nil, err + } + } + return result, nil + case Generator: + var values []interface{} + for { + val, err := v.Next() + if err != nil { + if err == io.EOF { + break + } + return nil, err + } + + val, err = convertForJSON(t.underlying, val) + if err != nil { + return nil, err + } + + values = append(values, val) + } + + if err := v.Close(); err != nil { + return nil, err + } + + return values, nil + default: + return nil, ErrNotArray.New(v) + } +} + +// compareNulls compares two values, and returns true if either is null. +// The returned integer represents the ordering, with a rule that states nulls +// as being ordered before non-nulls. +func compareNulls(a interface{}, b interface{}) (bool, int) { + aIsNull := a == nil + bIsNull := b == nil + if aIsNull && bIsNull { + return true, 0 + } else if aIsNull && !bIsNull { + return true, -1 + } else if !aIsNull && bIsNull { + return true, 1 + } + return false, 0 +} diff --git a/sql/type_test.go b/sql/type_test.go index 0d4a01fd5..0dab60260 100644 --- a/sql/type_test.go +++ b/sql/type_test.go @@ -5,16 +5,16 @@ import ( "time" "github.com/stretchr/testify/require" - "gopkg.in/src-d/go-vitess.v1/sqltypes" - "gopkg.in/src-d/go-vitess.v1/vt/proto/query" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/proto/query" ) func TestIsNull(t *testing.T) { require.True(t, IsNull(nil)) n := numberT{sqltypes.Uint64} - require.Equal(t, sqltypes.NULL, n.SQL(Null)) - require.Equal(t, sqltypes.NewUint64(0), n.SQL(uint64(0))) + require.Equal(t, sqltypes.NULL, mustSQL(n.SQL(nil))) + require.Equal(t, sqltypes.NewUint64(0), mustSQL(n.SQL(uint64(0)))) } func TestText(t *testing.T) { @@ -24,18 +24,140 @@ func TestText(t *testing.T) { lt(t, Text, "a", "b") eq(t, Text, "a", "a") gt(t, Text, "b", "a") + + var3, err := VarChar(3).Convert("abc") + require.NoError(t, err) + convert(t, Text, var3, "abc") +} + +func TestBoolean(t *testing.T) { + convert(t, Boolean, "", false) + convert(t, Boolean, "true", false) + convert(t, Boolean, 0, false) + convert(t, Boolean, 1, true) + convert(t, Boolean, -1, true) + convert(t, Boolean, 0.0, false) + convert(t, Boolean, 0.4, false) + convert(t, Boolean, 0.5, true) + convert(t, Boolean, 1.0, true) + convert(t, Boolean, -1.0, true) + + eq(t, Boolean, true, true) + eq(t, Boolean, false, false) +} + +// Test conversion of all types of numbers to the specified signed integer type +// in typ, where minusOne, zero and one are the expected values with the +// same type as typ +func testSignedInt(t *testing.T, typ Type, minusOne, zero, one interface{}) { + t.Helper() + + convert(t, typ, -1, minusOne) + convert(t, typ, int8(-1), minusOne) + convert(t, typ, int16(-1), minusOne) + convert(t, typ, int32(-1), minusOne) + convert(t, typ, int64(-1), minusOne) + convert(t, typ, 0, zero) + convert(t, typ, int8(0), zero) + convert(t, typ, int16(0), zero) + convert(t, typ, int32(0), zero) + convert(t, typ, int64(0), zero) + convert(t, typ, uint8(0), zero) + convert(t, typ, uint16(0), zero) + convert(t, typ, uint32(0), zero) + convert(t, typ, uint64(0), zero) + convert(t, typ, 1, one) + convert(t, typ, int8(1), one) + convert(t, typ, int16(1), one) + convert(t, typ, int32(1), one) + convert(t, typ, int64(1), one) + convert(t, typ, uint8(1), one) + convert(t, typ, uint16(1), one) + convert(t, typ, uint32(1), one) + convert(t, typ, uint64(1), one) + convert(t, typ, "-1", minusOne) + convert(t, typ, "0", zero) + convert(t, typ, "1", one) + convertErr(t, typ, "") + + lt(t, Int8, minusOne, one) + eq(t, Int8, zero, zero) + eq(t, Int8, minusOne, minusOne) + eq(t, Int8, one, one) + gt(t, Int8, one, minusOne) +} + +// Test conversion of all types of numbers to the specified unsigned integer +// type in typ, where zero and one are the expected values with the same type +// as typ. The expected errors when converting from negative numbers are also +// tested +func testUnsignedInt(t *testing.T, typ Type, zero, one interface{}) { + t.Helper() + + convertErr(t, typ, -1) + convertErr(t, typ, int8(-1)) + convertErr(t, typ, int16(-1)) + convertErr(t, typ, int32(-1)) + convertErr(t, typ, int64(-1)) + convert(t, typ, 0, zero) + convert(t, typ, int8(0), zero) + convert(t, typ, int16(0), zero) + convert(t, typ, int32(0), zero) + convert(t, typ, int64(0), zero) + convert(t, typ, uint8(0), zero) + convert(t, typ, uint16(0), zero) + convert(t, typ, uint32(0), zero) + convert(t, typ, uint64(0), zero) + convert(t, typ, 1, one) + convert(t, typ, int8(1), one) + convert(t, typ, int16(1), one) + convert(t, typ, int32(1), one) + convert(t, typ, int64(1), one) + convert(t, typ, uint8(1), one) + convert(t, typ, uint16(1), one) + convert(t, typ, uint32(1), one) + convert(t, typ, uint64(1), one) + convertErr(t, typ, "-1") + convert(t, typ, "0", zero) + convert(t, typ, "1", one) + convertErr(t, typ, "") + + lt(t, Int8, zero, one) + eq(t, Int8, zero, zero) + eq(t, Int8, one, one) + gt(t, Int8, one, zero) +} + +func TestInt8(t *testing.T) { + testSignedInt(t, Int8, int8(-1), int8(0), int8(1)) +} + +func TestInt16(t *testing.T) { + testSignedInt(t, Int16, int16(-1), int16(0), int16(1)) } func TestInt32(t *testing.T) { - convert(t, Int32, int32(1), int32(1)) - convert(t, Int32, 1, int32(1)) - convert(t, Int32, int64(1), int32(1)) - convert(t, Int32, "5", int32(5)) - convertErr(t, Int32, "") + testSignedInt(t, Int32, int32(-1), int32(0), int32(1)) +} + +func TestInt64(t *testing.T) { + testSignedInt(t, Int64, int64(-1), int64(0), int64(1)) +} + +func TestUint8(t *testing.T) { + testUnsignedInt(t, Uint8, uint8(0), uint8(1)) +} + +func TestUint16(t *testing.T) { + testUnsignedInt(t, Uint16, uint16(0), uint16(1)) +} - lt(t, Int32, int32(1), int32(2)) - eq(t, Int32, int32(1), int32(1)) - gt(t, Int32, int32(3), int32(2)) +func TestUint32(t *testing.T) { + testUnsignedInt(t, Uint32, uint32(0), uint32(1)) +} + +func TestUint64(t *testing.T) { + testUnsignedInt(t, Uint64, uint64(0), uint64(1)) } func TestNumberComparison(t *testing.T) { @@ -50,18 +172,76 @@ func TestNumberComparison(t *testing.T) { gt(t, Uint32, int64(5), uint32(1)) gt(t, Uint32, uint32(5), int64(1)) lt(t, Uint32, uint64(1), int32(5)) -} -func TestInt64(t *testing.T) { - convert(t, Int64, int32(1), int64(1)) - convert(t, Int64, 1, int64(1)) - convert(t, Int64, int64(1), int64(1)) - convertErr(t, Int64, "") - convert(t, Int64, "5", int64(5)) + eq(t, Uint8, uint8(255), uint8(255)) + eq(t, Uint8, uint8(255), int32(255)) + eq(t, Uint8, uint8(255), int64(255)) + eq(t, Uint8, uint8(255), int64(255)) + gt(t, Uint8, uint8(255), int32(1)) + gt(t, Uint8, uint8(255), int64(1)) + lt(t, Uint8, uint8(255), int16(256)) + + // Exhaustive numeric type equality test + type typeAndValue struct { + t numberT + v interface{} + } + + allTypes := []typeAndValue{ + {Int8, int8(42)}, + {Uint8, uint8(42)}, + {Int16, int16(42)}, + {Uint16, uint16(42)}, + {Int24, int32(42)}, + {Uint24, uint32(42)}, + {Int32, int32(42)}, + {Uint32, uint32(42)}, + {Int64, int64(42)}, + {Uint64, uint64(42)}, + {Float32, float32(42)}, + {Float64, float64(42)}, + } + for _, a := range allTypes { + for _, b := range allTypes { + eq(t, a.t, a.v, b.v) + } + } - lt(t, Int64, int64(1), int64(2)) - eq(t, Int64, int64(1), int64(1)) - gt(t, Int64, int64(3), int64(2)) + // Float comparisons against other floats + greaterFloat := 7.5 + lesserFloat := 7.4 + gt(t, Float64, float64(greaterFloat), float64(lesserFloat)) + lt(t, Float64, float64(lesserFloat), float64(greaterFloat)) + eq(t, Float64, float64(greaterFloat), float64(greaterFloat)) + gt(t, Float64, float64(greaterFloat), float32(lesserFloat)) + lt(t, Float64, float64(lesserFloat), float32(greaterFloat)) + eq(t, Float64, float64(greaterFloat), float32(greaterFloat)) + gt(t, Float32, float32(greaterFloat), float32(lesserFloat)) + lt(t, Float32, float32(lesserFloat), float32(greaterFloat)) + eq(t, Float32, float32(greaterFloat), float32(greaterFloat)) + gt(t, Float32, float32(greaterFloat), float64(lesserFloat)) + lt(t, Float32, float32(lesserFloat), float64(greaterFloat)) + eq(t, Float32, float32(greaterFloat), float64(greaterFloat)) + + // Float comparisons against other types, testing comparison and truncation (when an int type is the left side of a + // comparison with a float type) + lessInt := 7 + floatComps := []typeAndValue{ + {Int8, int8(lessInt)}, + {Uint8, uint8(lessInt)}, + {Int16, int16(lessInt)}, + {Uint16, uint16(lessInt)}, + {Int32, int32(lessInt)}, + {Uint32, uint32(lessInt)}, + {Int64, int64(lessInt)}, + {Uint64, uint64(lessInt)}, + } + for _, a := range floatComps { + gt(t, Float64, float64(greaterFloat), a.v) + eq(t, a.t, float64(greaterFloat), a.v) + gt(t, Float32, float32(greaterFloat), a.v) + eq(t, a.t, float32(greaterFloat), a.v) + } } func TestFloat64(t *testing.T) { @@ -70,7 +250,8 @@ func TestFloat64(t *testing.T) { var f = numberT{ t: query.Type_FLOAT64, } - val := f.SQL(23.222) + val, err := f.SQL(23.222) + require.NoError(err) require.True(val.IsFloat()) require.Equal(sqltypes.NewFloat64(23.222), val) } @@ -97,7 +278,8 @@ func TestTimestamp(t *testing.T) { v.(time.Time).Format(TimestampLayout), ) - sql := Timestamp.SQL(now) + sql, err := Timestamp.SQL(now) + require.NoError(err) require.Equal([]byte(now.Format(TimestampLayout)), sql.Raw()) after := now.Add(time.Second) @@ -146,39 +328,57 @@ func TestExtraTimestamps(t *testing.T) { } } -func TestDate(t *testing.T) { +// Generic tests for Date and Datetime. +// typ should be Date or Datetime +func commonTestsDatesTypes(typ Type, layout string, t *testing.T) { require := require.New(t) now := time.Now().UTC() - v, err := Date.Convert(now) + v, err := typ.Convert(now) require.NoError(err) - require.Equal(now.Format(DateLayout), v.(time.Time).Format(DateLayout)) + require.Equal(now.Format(layout), v.(time.Time).Format(layout)) - v, err = Date.Convert(now.Format(DateLayout)) + v, err = typ.Convert(now.Format(layout)) require.NoError(err) require.Equal( - now.Format(DateLayout), - v.(time.Time).Format(DateLayout), + now.Format(layout), + v.(time.Time).Format(layout), ) - v, err = Date.Convert(now.Unix()) + v, err = typ.Convert(now.Unix()) require.NoError(err) require.Equal( - now.Format(DateLayout), - v.(time.Time).Format(DateLayout), + now.Format(layout), + v.(time.Time).Format(layout), ) - sql := Date.SQL(now) - require.Equal([]byte(now.Format(DateLayout)), sql.Raw()) + sql, err := typ.SQL(now) + require.NoError(err) + require.Equal([]byte(now.Format(layout)), sql.Raw()) + + after := now.Add(26 * time.Hour) + lt(t, typ, now, after) + eq(t, typ, now, now) + gt(t, typ, after, now) +} + +func TestDate(t *testing.T) { + commonTestsDatesTypes(Date, DateLayout, t) + now := time.Now().UTC() after := now.Add(time.Second) eq(t, Date, now, after) eq(t, Date, now, now) eq(t, Date, after, now) +} - after = now.Add(26 * time.Hour) - lt(t, Date, now, after) - eq(t, Date, now, now) - gt(t, Date, after, now) +func TestDatetime(t *testing.T) { + commonTestsDatesTypes(Datetime, DatetimeLayout, t) + + now := time.Now().UTC() + after := now.Add(time.Millisecond) + lt(t, Datetime, now, after) + eq(t, Datetime, now, now) + gt(t, Datetime, after, now) } func TestBlob(t *testing.T) { @@ -186,7 +386,6 @@ func TestBlob(t *testing.T) { convert(t, Blob, "", []byte{}) convert(t, Blob, nil, []byte(nil)) - MustConvert(Blob, nil) _, err := Blob.Convert(1) require.NotNil(err) @@ -200,6 +399,7 @@ func TestBlob(t *testing.T) { func TestJSON(t *testing.T) { convert(t, JSON, "", []byte(`""`)) convert(t, JSON, []int{1, 2}, []byte("[1,2]")) + convert(t, JSON, `{"a": true, "b": 3}`, []byte(`{"a":true,"b":3}`)) lt(t, JSON, []byte("A"), []byte("B")) eq(t, JSON, []byte("A"), []byte("A")) @@ -220,9 +420,8 @@ func TestTuple(t *testing.T) { convert(t, typ, []interface{}{1, 2, 3}, []interface{}{int32(1), "2", int64(3)}) - require.Panics(func() { - typ.SQL(nil) - }) + _, err = typ.SQL(nil) + require.Error(err) require.Equal(sqltypes.Expression, typ.Type()) @@ -235,6 +434,48 @@ func TestTuple(t *testing.T) { gt(t, typ, []interface{}{1, 2, 4}, []interface{}{1, 2, 3}) } +// Generic test for Char and VarChar types. +// genType should be sql.Char or sql.VarChar +func testCharTypes(genType func(int) Type, checkType func(Type) bool, t *testing.T) { + typ := genType(3) + require.True(t, checkType(typ)) + require.True(t, IsText(typ)) + convert(t, typ, "foo", "foo") + fooByte := []byte{'f', 'o', 'o'} + convert(t, typ, fooByte, "foo") + + typ = genType(1) + convertErr(t, typ, "foo") + convertErr(t, typ, fooByte) + convertErr(t, typ, 123) + + typ = genType(10) + convert(t, typ, 123, "123") + convertErr(t, typ, 1234567890123) + + convert(t, typ, "", "") + convert(t, typ, 1, "1") + + lt(t, typ, "a", "b") + eq(t, typ, "a", "a") + gt(t, typ, "b", "a") + + text, err := Text.Convert("abc") + require.NoError(t, err) + + convert(t, typ, text, "abc") + typ1 := genType(1) + convertErr(t, typ1, text) +} + +func TestChar(t *testing.T) { + testCharTypes(Char, IsChar, t) +} + +func TestVarChar(t *testing.T) { + testCharTypes(VarChar, IsVarChar, t) +} + func TestArray(t *testing.T) { require := require.New(t) @@ -244,6 +485,12 @@ func TestArray(t *testing.T) { require.True(ErrNotArray.Is(err)) convert(t, typ, []interface{}{1, 2, 3}, []interface{}{int64(1), int64(2), int64(3)}) + convert( + t, + typ, + NewArrayGenerator([]interface{}{1, 2, 3}), + []interface{}{int64(1), int64(2), int64(3)}, + ) require.Equal(sqltypes.TypeJSON, typ.Type()) @@ -256,6 +503,77 @@ func TestArray(t *testing.T) { gt(t, typ, []interface{}{1, 3, 3}, []interface{}{1, 2, 3}) gt(t, typ, []interface{}{1, 2, 4}, []interface{}{1, 2, 3}) gt(t, typ, []interface{}{1, 2, 4}, []interface{}{5, 6}) + + expected := []byte("[1,2,3]") + + v, err := Array(Int64).SQL([]interface{}{1, 2, 3}) + require.NoError(err) + require.Equal(expected, v.Raw()) + + v, err = Array(Int64).SQL(NewArrayGenerator([]interface{}{1, 2, 3})) + require.NoError(err) + require.Equal(expected, v.Raw()) +} + +func TestUnderlyingType(t *testing.T) { + require.Equal(t, Text, UnderlyingType(Array(Text))) + require.Equal(t, Text, UnderlyingType(Text)) +} + +type testJSONStruct struct { + A int + B string +} + +func TestJSONArraySQL(t *testing.T) { + require := require.New(t) + val, err := Array(JSON).SQL([]interface{}{ + testJSONStruct{1, "foo"}, + testJSONStruct{2, "bar"}, + }) + require.NoError(err) + expected := `[{"A":1,"B":"foo"},{"A":2,"B":"bar"}]` + require.Equal(expected, string(val.Raw())) +} + +func TestComparesWithNulls(t *testing.T) { + timeParse := func(layout string, value string) time.Time { + t, err := time.Parse(layout, value) + if err != nil { + panic(err) + } + return t + } + + var typeVals = []struct { + typ Type + val interface{} + }{ + {Int8, int8(0)}, + {Uint8, uint8(0)}, + {Int16, int16(0)}, + {Uint16, uint16(0)}, + {Int32, int32(0)}, + {Uint32, uint32(0)}, + {Int64, int64(0)}, + {Uint64, uint64(0)}, + {Float32, float32(0)}, + {Float64, float64(0)}, + {Timestamp, timeParse(TimestampLayout, "2132-04-05 12:51:36")}, + {Date, timeParse(DateLayout, "2231-11-07")}, + {Text, ""}, + {Boolean, false}, + {JSON, `{}`}, + {Blob, ""}, + } + + for _, typeVal := range typeVals { + t.Run(typeVal.typ.String(), func(t *testing.T) { + lt(t, typeVal.typ, nil, typeVal.val) + gt(t, typeVal.typ, typeVal.val, nil) + eq(t, typeVal.typ, nil, nil) + }) + } } func eq(t *testing.T, typ Type, a, b interface{}) { @@ -291,3 +609,10 @@ func convertErr(t *testing.T, typ Type, val interface{}) { _, err := typ.Convert(val) require.Error(t, err) } + +func mustSQL(v sqltypes.Value, err error) sqltypes.Value { + if err != nil { + panic(err) + } + return v +} diff --git a/test-server b/test-server deleted file mode 100755 index 192baec65..000000000 Binary files a/test-server and /dev/null differ diff --git a/test/mem_tracer.go b/test/mem_tracer.go index f11e00542..362e4321c 100644 --- a/test/mem_tracer.go +++ b/test/mem_tracer.go @@ -1,6 +1,8 @@ package test import ( + "sync" + opentracing "github.com/opentracing/opentracing-go" "github.com/opentracing/opentracing-go/log" ) @@ -8,6 +10,7 @@ import ( // MemTracer implements a simple tracer in memory for testing. type MemTracer struct { Spans []string + sync.Mutex } type memSpan struct { @@ -16,7 +19,9 @@ type memSpan struct { // StartSpan implements opentracing.Tracer interface. func (t *MemTracer) StartSpan(operationName string, opts ...opentracing.StartSpanOption) opentracing.Span { + t.Lock() t.Spans = append(t.Spans, operationName) + t.Unlock() return &memSpan{operationName} } pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy