diff --git a/docs/content.zh/docs/connectors/datastream/http.md b/docs/content.zh/docs/connectors/datastream/http.md index e1e6cfa7..83349548 100644 --- a/docs/content.zh/docs/connectors/datastream/http.md +++ b/docs/content.zh/docs/connectors/datastream/http.md @@ -52,7 +52,7 @@ These options are specified on the builder using the setProperty method. | Option | Required | Description/Value | |---------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| connector | required | Specify what connector to use. For HTTP Sink it should be set to _'http-sink'_. | +| connector | required | Specify what connector to use. For HTTP Sink it should be set to _'http-async-sink'_. | | format | required | Specify what format to use. | | url | required | The base URL that should be used for HTTP requests. For example _http://localhost:8080/client_. | | insert-method | optional | Specify which HTTP method to use in the request. The value should be set either to `POST` or `PUT`. | diff --git a/docs/content.zh/docs/connectors/table/http.md b/docs/content.zh/docs/connectors/table/http.md index 7eddf094..6ab18129 100644 --- a/docs/content.zh/docs/connectors/table/http.md +++ b/docs/content.zh/docs/connectors/table/http.md @@ -59,6 +59,7 @@ The HTTP source connector supports [Lookup Joins](https://nightlies.apache.org/f * [Working with HTTP sink tables](#working-with-http-sink-tables) * [HTTP Sink](#http-sink) * [Sink Connector Options](#sink-connector-options) + * [Sink Http headers](#sink-http-headers) * [Sink table HTTP status codes](#sink-table-http-status-codes) * [Request submission](#request-submission) * [Batch submission mode](#batch-submission-mode) @@ -206,6 +207,7 @@ Note the options with the prefix _http_ are the HTTP connector specific options, | http.request.query-param-fields | optional | Used for the `http-generic-json-url` query creator. The names of the fields that will be mapped to query parameters. The parameters are separated by semicolons, such as `param1;param2`. | | http.request.body-template | optional | Used for the `http-generic-json-url` query creator. A JSON template string for constructing the request body for PUT and POST operations. Use `{{fieldName}}` placeholders to reference top-level columns from the lookup table. Supports creating complex nested JSON structures with both placeholders and literal values. See the [Body Template](#body-template) section for details and examples. | | http.request.url-map | optional | Used for the `http-generic-json-url` query creator. The map of insert names to column names used as url segments. Parses a string as a map of strings. For example if there are table columns called `customerId` and `orderId`, then specifying value `customerId:cid,orderID:oid` and a url of https://myendpoint/customers/{cid}/orders/{oid} will mean that the url used for the lookup query will dynamically pickup the values for `customerId`, `orderId` and use them in the url e.g. https://myendpoint/customers/cid1/orders/oid1. The expected format of the map is: `key1:value1,key2:value2`. | +| http.source.lookup.header.* | optional | Custom HTTP headers to be added to every lookup request. Each header is specified as an individual property using the prefix `http.source.lookup.header.`, followed by the header name. For example: `'http.source.lookup.header.Content-Type' = 'application/json'`. Multiple headers can be defined by adding multiple properties with this prefix. | ### Query Creators @@ -346,11 +348,14 @@ To create a custom format user has to implement Flink's `SerializationSchema` an `resources/META-INF.services/org.apache.flink.table.factories.Factory` file. This is common Flink mechanism for providing custom implementations for various factories. ### Http headers -It is possible to set HTTP headers that will be added to HTTP request sent by lookup source connector. -Headers are defined via property key `http.source.lookup.header.HEADER_NAME = header value` for example: -`http.source.lookup.header.X-Content-Type-Options = nosniff`. +It is possible to set custom HTTP headers that will be added to every HTTP request sent by the lookup source connector. +Headers are specified as individual properties using the prefix `http.source.lookup.header.`, followed by the header name. +For example: `'http.source.lookup.header.Content-Type' = 'application/json'`. -Headers can be set using http lookup source table DDL. In example below, HTTP request done for `http-lookup` table will contain three headers: +> **Note:** As of [FLINK-HTTP-3], custom HTTP headers are formally supported as a `ConfigOption` (`http.source.lookup.header.*`), +> enabling proper validation and discoverability of header configuration. + +Headers can be set using the HTTP lookup source table DDL. In the example below, every HTTP request for the `http-lookup` table will contain three headers: - `Origin` - `X-Content-Type-Options` - `Content-Type` @@ -431,7 +436,7 @@ CREATE TABLE http ( id bigint, some_field string ) WITH ( - 'connector' = 'http-sink', + 'connector' = 'http-async-sink', 'url' = 'http://example.com/myendpoint', 'format' = 'json' ) @@ -451,7 +456,7 @@ another format name. | Option | Required | Description/Value | |-------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| connector | required | Specify what connector to use. For HTTP Sink it should be set to _'http-sink'_. | +| connector | required | Specify what connector to use. For HTTP Sink it should be set to _'http-async-sink'_. | | format | required | Specify what format to use. | | url | required | The base URL that should be used for HTTP requests. For example _http://localhost:8080/client_. | | insert-method | optional | Specify which HTTP method to use in the request. The value should be set either to `POST` or `PUT`. | @@ -472,6 +477,30 @@ another format name. | http.sink.writer.thread-pool.size | optional | Sets the size of pool thread for HTTP Sink request processing. Increasing this value would mean that more concurrent requests can be processed in the same time. If not specified, the default value of 1 thread will be used. | | http.sink.writer.request.mode | optional | Sets the Http Sink request submission mode. Two modes are available: `single` and `batch`. Defaults to `batch` if not specified. | | http.sink.request.batch.size | optional | Applicable only for `http.sink.writer.request.mode = batch`. Sets number of individual events/requests that will be submitted as one HTTP request by HTTP sink. The default value is 500 which is same as HTTP Sink `maxBatchSize` | +| http.sink.header.* | optional | Custom HTTP headers to be added to every sink request. Each header is specified as an individual property using the prefix `http.sink.header.`, followed by the header name. For example: `'http.sink.header.Content-Type' = 'application/json'`. Multiple headers can be defined by adding multiple properties with this prefix. | + +### Sink Http headers +It is possible to set custom HTTP headers that will be added to every HTTP request sent by the HTTP sink connector. +Headers are specified as individual properties using the prefix `http.sink.header.`, followed by the header name. + +> **Note:** As of [FLINK-HTTP-3], custom HTTP headers are formally supported as a `ConfigOption` (`http.sink.header.*`), +> enabling proper validation and discoverability of header configuration. + +Headers can be set using the HTTP sink table DDL: + +```roomsql +CREATE TABLE http_sink ( + id bigint, + some_field string +) WITH ( + 'connector' = 'http-async-sink', + 'url' = 'http://example.com/myendpoint', + 'format' = 'json', + 'http.sink.header.Content-Type' = 'application/json', + 'http.sink.header.Authorization' = 'Bearer my-token', + 'http.sink.header.X-Custom-Header' = 'my-value' +) +``` ### Sink table HTTP status codes You can configure a list of HTTP status codes that should be treated as errors for HTTP sink table. @@ -541,7 +570,7 @@ CREATE TABLE http ( id bigint, some_field string ) WITH ( - 'connector' = 'http-sink', + 'connector' = 'http-async-sink', 'url' = 'http://example.com/myendpoint', 'format' = 'json', 'http.sink.request.batch.size' = '50' @@ -559,7 +588,7 @@ CREATE TABLE http ( id bigint, some_field string ) WITH ( - 'connector' = 'http-sink', + 'connector' = 'http-async-sink', 'url' = 'http://example.com/myendpoint', 'format' = 'json', 'http.sink.writer.request.mode' = 'single' diff --git a/docs/content/docs/connectors/datastream/http.md b/docs/content/docs/connectors/datastream/http.md index e1e6cfa7..83349548 100644 --- a/docs/content/docs/connectors/datastream/http.md +++ b/docs/content/docs/connectors/datastream/http.md @@ -52,7 +52,7 @@ These options are specified on the builder using the setProperty method. | Option | Required | Description/Value | |---------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| connector | required | Specify what connector to use. For HTTP Sink it should be set to _'http-sink'_. | +| connector | required | Specify what connector to use. For HTTP Sink it should be set to _'http-async-sink'_. | | format | required | Specify what format to use. | | url | required | The base URL that should be used for HTTP requests. For example _http://localhost:8080/client_. | | insert-method | optional | Specify which HTTP method to use in the request. The value should be set either to `POST` or `PUT`. | diff --git a/docs/content/docs/connectors/table/http.md b/docs/content/docs/connectors/table/http.md index 98572b62..c5ab9df0 100644 --- a/docs/content/docs/connectors/table/http.md +++ b/docs/content/docs/connectors/table/http.md @@ -59,6 +59,7 @@ The HTTP source connector supports [Lookup Joins](https://nightlies.apache.org/f * [Working with HTTP sink tables](#working-with-http-sink-tables) * [HTTP Sink](#http-sink) * [Sink Connector Options](#sink-connector-options) + * [Sink Http headers](#sink-http-headers) * [Sink table HTTP status codes](#sink-table-http-status-codes) * [Request submission](#request-submission) * [Batch submission mode](#batch-submission-mode) @@ -206,6 +207,7 @@ Note the options with the prefix _http_ are the HTTP connector specific options, | http.request.query-param-fields | optional | Used for the `http-generic-json-url` query creator. The names of the fields that will be mapped to query parameters. The parameters are separated by semicolons, such as `param1;param2`. | | http.request.body-template | optional | Used for the `http-generic-json-url` query creator. A JSON template string for constructing the request body for PUT and POST operations. Use `{{fieldName}}` placeholders to reference top-level columns from the lookup table. Supports creating complex nested JSON structures with both placeholders and literal values. See the [Body Template](#body-template) section for details and examples. | | http.request.url-map | optional | Used for the `http-generic-json-url` query creator. The map of insert names to column names used as url segments. Parses a string as a map of strings. For example if there are table columns called `customerId` and `orderId`, then specifying value `customerId:cid,orderID:oid` and a url of https://myendpoint/customers/{cid}/orders/{oid} will mean that the url used for the lookup query will dynamically pickup the values for `customerId`, `orderId` and use them in the url e.g. https://myendpoint/customers/cid1/orders/oid1. The expected format of the map is: `key1:value1,key2:value2`. | +| http.source.lookup.header.* | optional | Custom HTTP headers to be added to every lookup request. Each header is specified as an individual property using the prefix `http.source.lookup.header.`, followed by the header name. For example: `'http.source.lookup.header.Content-Type' = 'application/json'`. Multiple headers can be defined by adding multiple properties with this prefix. | ### Query Creators @@ -346,11 +348,14 @@ To create a custom format user has to implement Flink's `SerializationSchema` an `resources/META-INF.services/org.apache.flink.table.factories.Factory` file. This is common Flink mechanism for providing custom implementations for various factories. ### Http headers -It is possible to set HTTP headers that will be added to HTTP request sent by lookup source connector. -Headers are defined via property key `http.source.lookup.header.HEADER_NAME = header value` for example: -`http.source.lookup.header.X-Content-Type-Options = nosniff`. +It is possible to set custom HTTP headers that will be added to every HTTP request sent by the lookup source connector. +Headers are specified as individual properties using the prefix `http.source.lookup.header.`, followed by the header name. +For example: `'http.source.lookup.header.Content-Type' = 'application/json'`. -Headers can be set using http lookup source table DDL. In example below, HTTP request done for `http-lookup` table will contain three headers: +> **Note:** As of [FLINK-HTTP-3], custom HTTP headers are formally supported as a `ConfigOption` (`http.source.lookup.header.*`), +> enabling proper validation and discoverability of header configuration. + +Headers can be set using the HTTP lookup source table DDL. In the example below, every HTTP request for the `http-lookup` table will contain three headers: - `Origin` - `X-Content-Type-Options` - `Content-Type` @@ -431,7 +436,7 @@ CREATE TABLE http ( id bigint, some_field string ) WITH ( - 'connector' = 'http-sink', + 'connector' = 'http-async-sink', 'url' = 'http://example.com/myendpoint', 'format' = 'json' ) @@ -451,7 +456,7 @@ another format name. | Option | Required | Description/Value | |-------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| connector | required | Specify what connector to use. For HTTP Sink it should be set to _'http-sink'_. | +| connector | required | Specify what connector to use. For HTTP Sink it should be set to _'http-async-sink'_. | | format | required | Specify what format to use. | | url | required | The base URL that should be used for HTTP requests. For example _http://localhost:8080/client_. | | insert-method | optional | Specify which HTTP method to use in the request. The value should be set either to `POST` or `PUT`. | @@ -472,6 +477,30 @@ another format name. | http.sink.writer.thread-pool.size | optional | Sets the size of pool thread for HTTP Sink request processing. Increasing this value would mean that more concurrent requests can be processed in the same time. If not specified, the default value of 1 thread will be used. | | http.sink.writer.request.mode | optional | Sets the Http Sink request submission mode. Two modes are available: `single` and `batch`. Defaults to `batch` if not specified. | | http.sink.request.batch.size | optional | Applicable only for `http.sink.writer.request.mode = batch`. Sets number of individual events/requests that will be submitted as one HTTP request by HTTP sink. The default value is 500 which is same as HTTP Sink `maxBatchSize` | +| http.sink.header.* | optional | Custom HTTP headers to be added to every sink request. Each header is specified as an individual property using the prefix `http.sink.header.`, followed by the header name. For example: `'http.sink.header.Content-Type' = 'application/json'`. Multiple headers can be defined by adding multiple properties with this prefix. | + +### Sink Http headers +It is possible to set custom HTTP headers that will be added to every HTTP request sent by the HTTP sink connector. +Headers are specified as individual properties using the prefix `http.sink.header.`, followed by the header name. + +> **Note:** As of [FLINK-HTTP-3], custom HTTP headers are formally supported as a `ConfigOption` (`http.sink.header.*`), +> enabling proper validation and discoverability of header configuration. + +Headers can be set using the HTTP sink table DDL: + +```roomsql +CREATE TABLE http_sink ( + id bigint, + some_field string +) WITH ( + 'connector' = 'http-async-sink', + 'url' = 'http://example.com/myendpoint', + 'format' = 'json', + 'http.sink.header.Content-Type' = 'application/json', + 'http.sink.header.Authorization' = 'Bearer my-token', + 'http.sink.header.X-Custom-Header' = 'my-value' +) +``` ### Sink table HTTP status codes You can configure a list of HTTP status codes that should be treated as errors for HTTP sink table. @@ -541,7 +570,7 @@ CREATE TABLE http ( id bigint, some_field string ) WITH ( - 'connector' = 'http-sink', + 'connector' = 'http-async-sink', 'url' = 'http://example.com/myendpoint', 'format' = 'json', 'http.sink.request.batch.size' = '50' @@ -559,7 +588,7 @@ CREATE TABLE http ( id bigint, some_field string ) WITH ( - 'connector' = 'http-sink', + 'connector' = 'http-async-sink', 'url' = 'http://example.com/myendpoint', 'format' = 'json', 'http.sink.writer.request.mode' = 'single' diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/HttpPostRequestCallbackFactory.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/HttpPostRequestCallbackFactory.java index f48ff0d9..f5c8b729 100644 --- a/flink-connector-http/src/main/java/org/apache/flink/connector/http/HttpPostRequestCallbackFactory.java +++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/HttpPostRequestCallbackFactory.java @@ -49,7 +49,7 @@ * id bigint, * some_field string * ) with ( - * 'connector' = 'http-sink' + * 'connector' = 'http-async-sink' * 'url' = 'http://example.com/myendpoint' * 'format' = 'json', * 'http.sink.request-callback' = 'my-callback' diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupConnectorOptions.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupConnectorOptions.java index e5e7d58e..b0be1d79 100644 --- a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupConnectorOptions.java +++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupConnectorOptions.java @@ -23,6 +23,8 @@ import org.apache.flink.connector.http.retry.RetryStrategyType; import java.time.Duration; +import java.util.Collections; +import java.util.Map; import static org.apache.flink.connector.http.config.HttpConnectorConfigConstants.CONTINUE_ON_ERROR; import static org.apache.flink.connector.http.config.HttpConnectorConfigConstants.HTTP_LOGGING_LEVEL; @@ -227,4 +229,24 @@ public class HttpLookupConnectorOptions { + "Ignored responses togater with `" + SOURCE_RETRY_SUCCESS_CODES + "` are considered as successful."); + + /** + * Custom HTTP headers for lookup requests. + * + *

Headers are specified as key-value pairs using the prefix {@code + * http.source.lookup.header.}, for example: + * + * 

{@code
+     * 'http.source.lookup.header.Content-Type' = 'application/json',
+     * 'http.source.lookup.header.Authorization' = 'Bearer my-token'
+     * }
+ */ + public static final ConfigOption> SOURCE_LOOKUP_HEADERS = + ConfigOptions.key("http.source.lookup.header") + .mapType() + .defaultValue(Collections.emptyMap()) + .withDescription( + "Custom HTTP headers for lookup requests, specified as key-value pairs " + + "with the prefix 'http.source.lookup.header.'. " + + "For example, 'http.source.lookup.header.Content-Type' = 'application/json'."); } diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupTableSourceFactory.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupTableSourceFactory.java index 3a1e0e71..47f16192 100644 --- a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupTableSourceFactory.java +++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupTableSourceFactory.java @@ -53,6 +53,7 @@ import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.LOOKUP_REQUEST_FORMAT; import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.REQUEST_CALLBACK_IDENTIFIER; import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_CONNECTION_TIMEOUT; +import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_HEADERS; import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_HTTP_IGNORED_RESPONSE_CODES; import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_HTTP_RETRY_CODES; import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_HTTP_SUCCESS_CODES; @@ -204,8 +205,8 @@ public Set> optionalOptions() { SOURCE_LOOKUP_PROXY_PORT, SOURCE_LOOKUP_PROXY_USERNAME, SOURCE_LOOKUP_PROXY_PASSWORD, - SOURCE_LOOKUP_CONNECTION_TIMEOUT // TODO: add request timeout from properties - ); + SOURCE_LOOKUP_CONNECTION_TIMEOUT, // TODO: add request timeout from properties + SOURCE_LOOKUP_HEADERS); } private HttpLookupConfig getHttpLookupOptions(Context context, ReadableConfig readableConfig) { diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/sink/HttpDynamicSink.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/sink/HttpDynamicSink.java index 4d0e53ce..7af5e4ec 100644 --- a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/sink/HttpDynamicSink.java +++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/sink/HttpDynamicSink.java @@ -83,7 +83,7 @@ * id bigint, * some_field string * ) with ( - * 'connector' = 'http-sink' + * 'connector' = 'http-async-sink' * 'url' = 'http://example.com/myendpoint' * 'format' = 'json' * ) diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/sink/HttpDynamicSinkConnectorOptions.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/sink/HttpDynamicSinkConnectorOptions.java index 368447c2..48f67e6f 100644 --- a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/sink/HttpDynamicSinkConnectorOptions.java +++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/sink/HttpDynamicSinkConnectorOptions.java @@ -20,6 +20,9 @@ import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigOptions; +import java.util.Collections; +import java.util.Map; + import static org.apache.flink.connector.http.config.HttpConnectorConfigConstants.SINK_REQUEST_CALLBACK_IDENTIFIER; /** Table API options for {@link HttpDynamicSink}. */ @@ -41,4 +44,24 @@ public class HttpDynamicSinkConnectorOptions { ConfigOptions.key(SINK_REQUEST_CALLBACK_IDENTIFIER) .stringType() .defaultValue(Slf4jHttpPostRequestCallbackFactory.IDENTIFIER); + + /** + * Custom HTTP headers for sink requests. + * + *

Headers are specified as key-value pairs using the prefix {@code http.sink.header.}, for + * example: + * + * 

{@code
+     * 'http.sink.header.Content-Type' = 'application/json',
+     * 'http.sink.header.Authorization' = 'Bearer my-token'
+     * }
+ */ + public static final ConfigOption> SINK_HEADERS = + ConfigOptions.key("http.sink.header") + .mapType() + .defaultValue(Collections.emptyMap()) + .withDescription( + "Custom HTTP headers for sink requests, specified as key-value pairs " + + "with the prefix 'http.sink.header.'. " + + "For example, 'http.sink.header.Content-Type' = 'application/json'."); } diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/sink/HttpDynamicTableSinkFactory.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/sink/HttpDynamicTableSinkFactory.java index d3ef903e..f7d27237 100644 --- a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/sink/HttpDynamicTableSinkFactory.java +++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/sink/HttpDynamicTableSinkFactory.java @@ -33,6 +33,7 @@ import static org.apache.flink.connector.http.table.sink.HttpDynamicSinkConnectorOptions.INSERT_METHOD; import static org.apache.flink.connector.http.table.sink.HttpDynamicSinkConnectorOptions.REQUEST_CALLBACK_IDENTIFIER; +import static org.apache.flink.connector.http.table.sink.HttpDynamicSinkConnectorOptions.SINK_HEADERS; import static org.apache.flink.connector.http.table.sink.HttpDynamicSinkConnectorOptions.URL; /** Factory for creating {@link HttpDynamicSink}. */ @@ -96,6 +97,7 @@ public Set> optionalOptions() { var options = super.optionalOptions(); options.add(INSERT_METHOD); options.add(REQUEST_CALLBACK_IDENTIFIER); + options.add(SINK_HEADERS); return options; } diff --git a/flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/HttpLookupTableSourceFactoryTest.java b/flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/HttpLookupTableSourceFactoryTest.java index 94b9b4a1..41fb64b9 100644 --- a/flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/HttpLookupTableSourceFactoryTest.java +++ b/flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/HttpLookupTableSourceFactoryTest.java @@ -118,6 +118,31 @@ void shouldHandleEmptyUrlArgs() { assertThat(source).isInstanceOf(HttpLookupTableSource.class); } + @Test + void shouldAcceptCustomHeaders() { + Map options = + getOptions( + Map.of( + "http.source.lookup.header.Content-Type", "application/json", + "http.source.lookup.header.X-Custom-Header", "my-value")); + DynamicTableSource source = createTableSource(SCHEMA, options); + assertThat(source).isNotNull(); + assertThat(source).isInstanceOf(HttpLookupTableSource.class); + } + + @Test + void shouldAcceptMultipleCustomHeaders() { + Map options = + getOptions( + Map.of( + "http.source.lookup.header.Content-Type", "application/json", + "http.source.lookup.header.Authorization", "Bearer token123", + "http.source.lookup.header.Accept", "application/json")); + DynamicTableSource source = createTableSource(SCHEMA, options); + assertThat(source).isNotNull(); + assertThat(source).isInstanceOf(HttpLookupTableSource.class); + } + private Map getMandatoryOptions() { return Map.of( "connector", "http", diff --git a/flink-connector-http/src/test/java/org/apache/flink/connector/http/table/sink/HttpDynamicTableSinkFactoryTest.java b/flink-connector-http/src/test/java/org/apache/flink/connector/http/table/sink/HttpDynamicTableSinkFactoryTest.java index 5a835a86..c7bfde70 100644 --- a/flink-connector-http/src/test/java/org/apache/flink/connector/http/table/sink/HttpDynamicTableSinkFactoryTest.java +++ b/flink-connector-http/src/test/java/org/apache/flink/connector/http/table/sink/HttpDynamicTableSinkFactoryTest.java @@ -106,4 +106,22 @@ public void nonexistentOptionsTest() { assertThatThrownBy(() -> tEnv.executeSql("INSERT INTO http VALUES (1)").await()) .isInstanceOf(ValidationException.class); } + + @Test + public void shouldAcceptCustomHeadersTest() { + final String ddlWithHeaders = + String.format( + "CREATE TABLE httpWithHeaders (\n" + + " id bigint\n" + + ") with (\n" + + " 'connector' = '%s',\n" + + " 'url' = '%s',\n" + + " 'format' = 'json',\n" + + " 'http.sink.header.Content-Type' = 'application/json',\n" + + " 'http.sink.header.X-Custom-Header' = 'my-value'\n" + + ")", + HttpDynamicTableSinkFactory.IDENTIFIER, "http://localhost/"); + // Should not throw a ValidationException for unrecognized options + tEnv.executeSql(ddlWithHeaders); + } }