From b143be7191ada425c6daf8adb2349a379ee17255 Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Tue, 22 Oct 2024 11:02:21 +0200 Subject: [PATCH 1/3] Add instructions --- docs/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/README.md b/docs/README.md index b6a00b9..011c574 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,6 +5,12 @@ This very experimental extension spawns an HTTP Client from within DuckDB resolv > Experimental: USE AT YOUR OWN RISK! +### 📦 Installation +```sql +INSTALL http_client FROM community; +LOAD http_client; +``` + ### Functions - `http_get(url)` - `http_post(url, headers, params)` From 7ac8ec956b7fd671599f8bd58e086e33dd957aa8 Mon Sep 17 00:00:00 2001 From: Alvaro Huarte Date: Wed, 23 Oct 2024 01:28:48 +0200 Subject: [PATCH 2/3] Funtions return response data as a JSON instead of the body --- docs/README.md | 92 ++++++++++++++------------- src/http_client_extension.cpp | 71 ++++++++++++++------- test/sql/httpclient.test | 115 ++++++++++++++++++---------------- 3 files changed, 158 insertions(+), 120 deletions(-) diff --git a/docs/README.md b/docs/README.md index 011c574..aba3ad6 100644 --- a/docs/README.md +++ b/docs/README.md @@ -19,62 +19,68 @@ LOAD http_client; #### GET ```sql D WITH __input AS ( - SELECT - http_get( + SELECT + http_get( 'https://httpbin.org/delay/0' - ) AS data - ), - __features AS ( - SELECT - unnest( from_json((data::JSON)->'headers', '{"Host": "VARCHAR"}') ) - AS features - FROM - __input - ) + ) AS res + ), + __response AS ( SELECT - __features.Host AS host, + (res->>'status')::INT AS status, + (res->>'reason') AS reason, + unnest( from_json(((res->>'body')::JSON)->'headers', '{"Host": "VARCHAR"}') ) AS features FROM - __features - ; -┌─────────────┐ -│ host │ -│ varchar │ -├─────────────┤ -│ httpbin.org │ -└─────────────┘ + __input + ) + SELECT + __response.status, + __response.reason, + __response.Host AS host, + FROM + __response + ; +┌────────┬─────────┬─────────────┐ +│ status │ reason │ host │ +│ int32 │ varchar │ varchar │ +├────────┼─────────┼─────────────┤ +│ 200 │ OK │ httpbin.org │ +└────────┴─────────┴─────────────┘ ``` #### POST ```sql -WITH __input AS ( - SELECT - http_post( +D WITH __input AS ( + SELECT + http_post( 'https://httpbin.org/delay/0', headers => MAP { 'accept': 'application/json', }, params => MAP { } - ) AS data - ), - __features AS ( - SELECT - unnest( from_json((data::JSON)->'headers', '{"Host": "VARCHAR"}') ) - AS features - FROM - __input - ) + ) AS res + ), + __response AS ( SELECT - __features.Host AS host, + (res->>'status')::INT AS status, + (res->>'reason') AS reason, + unnest( from_json(((res->>'body')::JSON)->'headers', '{"Host": "VARCHAR"}') ) AS features FROM - __features - ; -┌─────────────┐ -│ host │ -│ varchar │ -├─────────────┤ -│ httpbin.org │ -└─────────────┘ + __input + ) + SELECT + __response.status, + __response.reason, + __response.Host AS host, + FROM + __response + ; +┌────────┬─────────┬─────────────┐ +│ status │ reason │ host │ +│ int32 │ varchar │ varchar │ +├────────┼─────────┼─────────────┤ +│ 200 │ OK │ httpbin.org │ +└────────┴─────────┴─────────────┘ ``` #### Full Example w/ spatial data @@ -88,11 +94,11 @@ D WITH __input AS ( SELECT http_get( 'https://earth-search.aws.element84.com/v0/search') - AS data + AS res ), __features AS ( SELECT - unnest( from_json((data::JSON)->'features', '["json"]') ) + unnest( from_json(((res->>'body')::JSON)->'features', '["json"]') ) AS features FROM __input diff --git a/src/http_client_extension.cpp b/src/http_client_extension.cpp index c71c5e5..46109ce 100644 --- a/src/http_client_extension.cpp +++ b/src/http_client_extension.cpp @@ -44,7 +44,44 @@ static std::pair SetupHttpClient(co return std::make_pair(std::move(client), path); } -static void HandleHttpError(const duckdb_httplib_openssl::Result &res, const std::string &request_type) { +// Helper function to escape chars of a string representing a JSON object +std::string escape_json(const std::string &input) { + std::ostringstream output; + + for (auto c = input.cbegin(); c != input.cend(); c++) { + switch (*c) { + case '"' : output << "\\\""; break; + case '\\': output << "\\\\"; break; + case '\b': output << "\\b"; break; + case '\f': output << "\\f"; break; + case '\n': output << "\\n"; break; + case '\r': output << "\\r"; break; + case '\t': output << "\\t"; break; + default: + if ('\x00' <= *c && *c <= '\x1f') { + output << "\\u" + << std::hex << std::setw(4) << std::setfill('0') << static_cast(*c); + } else { + output << *c; + } + } + } + return output.str(); +} + +// Helper function to create a Response object as a string +static std::string GetJsonResponse(int status, const std::string &reason, const std::string &body) { + std::string response = StringUtil::Format( + "{ \"status\": %i, \"reason\": \"%s\", \"body\": \"%s\" }", + status, + escape_json(reason), + escape_json(body) + ); + return response; +} + +// Helper function to return the description of one HTTP error. +static std::string GetHttpErrorMessage(const duckdb_httplib_openssl::Result &res, const std::string &request_type) { std::string err_message = "HTTP " + request_type + " request failed. "; switch (res.error()) { @@ -85,7 +122,7 @@ static void HandleHttpError(const duckdb_httplib_openssl::Result &res, const std err_message += "Unknown error."; break; } - throw std::runtime_error(err_message); + return err_message; } @@ -103,17 +140,12 @@ static void HTTPGetRequestFunction(DataChunk &args, ExpressionState &state, Vect // Make the GET request auto res = client.Get(path.c_str()); if (res) { - if (res->status == 200) { - return StringVector::AddString(result, res->body); - } else { - throw std::runtime_error("HTTP GET error: " + std::to_string(res->status) + " - " + res->reason); - } + std::string response = GetJsonResponse(res->status, res->reason, res->body); + return StringVector::AddString(result, response); } else { - // Handle errors - HandleHttpError(res, "GET"); + std::string response = GetJsonResponse(-1, GetHttpErrorMessage(res, "POST"), ""); + return StringVector::AddString(result, response); } - // Ensure a return value in case of an error - return string_t(); }); } @@ -159,30 +191,25 @@ static void HTTPPostRequestFunction(DataChunk &args, ExpressionState &state, Vec // Make the POST request with headers and body auto res = client.Post(path.c_str(), header_map, body.val.GetString(), "application/json"); if (res) { - if (res->status == 200) { - return StringVector::AddString(result, res->body); - } else { - throw std::runtime_error("HTTP POST error: " + std::to_string(res->status) + " - " + res->reason); - } + std::string response = GetJsonResponse(res->status, res->reason, res->body); + return StringVector::AddString(result, response); } else { - // Handle errors - HandleHttpError(res, "POST"); + std::string response = GetJsonResponse(-1, GetHttpErrorMessage(res, "POST"), ""); + return StringVector::AddString(result, response); } - // Ensure a return value in case of an error - return string_t(); }); } static void LoadInternal(DatabaseInstance &instance) { ScalarFunctionSet http_get("http_get"); - http_get.AddFunction(ScalarFunction({LogicalType::VARCHAR}, LogicalType::VARCHAR, HTTPGetRequestFunction)); + http_get.AddFunction(ScalarFunction({LogicalType::VARCHAR}, LogicalType::JSON(), HTTPGetRequestFunction)); ExtensionUtil::RegisterFunction(instance, http_get); ScalarFunctionSet http_post("http_post"); http_post.AddFunction(ScalarFunction( {LogicalType::VARCHAR, LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR), LogicalType::JSON()}, - LogicalType::VARCHAR, HTTPPostRequestFunction)); + LogicalType::JSON(), HTTPPostRequestFunction)); ExtensionUtil::RegisterFunction(instance, http_post); } diff --git a/test/sql/httpclient.test b/test/sql/httpclient.test index 69049ab..c658552 100644 --- a/test/sql/httpclient.test +++ b/test/sql/httpclient.test @@ -14,55 +14,61 @@ require http_client require json # Confirm the GET extension works -query I +query III WITH __input AS ( - SELECT - http_get( - 'https://httpbin.org/delay/0' - ) AS data - ), - __features AS ( - SELECT - unnest( from_json((data::JSON)->'headers', '{"Host": "VARCHAR"}') ) - AS features - FROM - __input - ) - SELECT - __features.Host AS host, - FROM - __features - ; + SELECT + http_get( + 'https://httpbin.org/delay/0' + ) AS res +), +__response AS ( + SELECT + (res->>'status')::INT AS status, + (res->>'reason') AS reason, + unnest( from_json(((res->>'body')::JSON)->'headers', '{"Host": "VARCHAR"}') ) AS features + FROM + __input +) +SELECT + __response.status, + __response.reason, + __response.Host AS host +FROM + __response +; ---- -httpbin.org +200 OK httpbin.org # Confirm the POST extension works -query I +query III WITH __input AS ( - SELECT - http_post( - 'https://httpbin.org/delay/0', - headers => MAP { - 'accept': 'application/json', - }, - params => MAP { - } - ) AS data - ), - __features AS ( - SELECT - unnest( from_json((data::JSON)->'headers', '{"Host": "VARCHAR"}') ) - AS features - FROM - __input - ) - SELECT - __features.Host AS host, - FROM - __features - ; + SELECT + http_post( + 'https://httpbin.org/delay/0', + headers => MAP { + 'accept': 'application/json', + }, + params => MAP { + } + ) AS res +), +__response AS ( + SELECT + (res->>'status')::INT AS status, + (res->>'reason') AS reason, + unnest( from_json(((res->>'body')::JSON)->'headers', '{"Host": "VARCHAR"}') ) AS features + FROM + __input +) +SELECT + __response.status, + __response.reason, + __response.Host AS host +FROM + __response +; ---- -httpbin.org +200 OK httpbin.org # Confirm the POST extension works with headers and params query I @@ -81,19 +87,18 @@ WITH __input AS ( 'datetime': '2021-09-30/2021-09-30', 'limit': 10 } - ) AS data - ), - __features AS ( - SELECT - unnest( from_json((data::JSON)->'features', '["json"]') ) - AS features - FROM - __input - ) + ) AS res +), +__response AS ( SELECT - features->>'id' AS id + unnest( from_json(((res->>'body')::JSON)->'features', '["json"]') ) AS features FROM - __features - ; + __input +) +SELECT + features->>'id' AS id +FROM + __response +; ---- S2A_56LPN_20210930_0_L2A From 9eb63c009a0642a9fb26a4b4a1344b181ad93253 Mon Sep 17 00:00:00 2001 From: Alvaro Huarte Date: Wed, 23 Oct 2024 09:58:21 +0200 Subject: [PATCH 3/3] Fix typo --- src/http_client_extension.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/http_client_extension.cpp b/src/http_client_extension.cpp index 46109ce..131dba5 100644 --- a/src/http_client_extension.cpp +++ b/src/http_client_extension.cpp @@ -143,7 +143,7 @@ static void HTTPGetRequestFunction(DataChunk &args, ExpressionState &state, Vect std::string response = GetJsonResponse(res->status, res->reason, res->body); return StringVector::AddString(result, response); } else { - std::string response = GetJsonResponse(-1, GetHttpErrorMessage(res, "POST"), ""); + std::string response = GetJsonResponse(-1, GetHttpErrorMessage(res, "GET"), ""); return StringVector::AddString(result, response); } }); pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy