diff --git a/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml b/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml index 724f5a0b87ce3..63c870e1fabcf 100644 --- a/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml @@ -116,6 +116,7 @@ body: - alibaba - amazon - anthropic + - apache-arrow - apache-beam - apache-cassandra - apache-drill diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index 2848323162932..e44de3ece8cf9 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -33,6 +33,9 @@ labelPRBasedOnFilePath: provider:anthropic: - providers/anthropic/** + provider:apache-arrow: + - providers/apache/arrow/** + provider:apache-beam: - providers/apache/beam/** diff --git a/dev/breeze/doc/images/output_build-docs.svg b/dev/breeze/doc/images/output_build-docs.svg index ca5cb1775bcc0..0cb0bb2cc32f6 100644 --- a/dev/breeze/doc/images/output_build-docs.svg +++ b/dev/breeze/doc/images/output_build-docs.svg @@ -241,17 +241,17 @@ Usage:                                                                                                                 breeze build-docs                                                                                                      [OPTIONS] [airbyte | akeyless | alibaba | all-providers | amazon | anthropic | apache-airflow | apache-airflow-ctl |   -apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | -apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark -| apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  -dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github |    -google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins |   -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  -yandex | ydb | zendesk]...                                                                                             +apache-airflow-providers | apache.arrow | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs +| apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot +| apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant |   +cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks |        +datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook |   +ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk +| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa |      +weaviate | yandex | ydb | zendesk]...                                                                                  Build documents. diff --git a/dev/breeze/doc/images/output_build-docs.txt b/dev/breeze/doc/images/output_build-docs.txt index 7e86964622680..03bc03ff93e1c 100644 --- a/dev/breeze/doc/images/output_build-docs.txt +++ b/dev/breeze/doc/images/output_build-docs.txt @@ -1 +1 @@ -22cd66d36e80473f4fac0fa311c5e846 +861abb76ef1c8a276ab55350a81a512d diff --git a/dev/breeze/doc/images/output_release-management_add-back-references.svg b/dev/breeze/doc/images/output_release-management_add-back-references.svg index 09e78ba88111a..49af552e0b3c4 100644 --- a/dev/breeze/doc/images/output_release-management_add-back-references.svg +++ b/dev/breeze/doc/images/output_release-management_add-back-references.svg @@ -150,17 +150,17 @@ Usage:                                                                                                                 breeze release-management add-back-references                                                                          [OPTIONS] [airbyte | akeyless | alibaba | all-providers | amazon | anthropic | apache-airflow | apache-airflow-ctl |   -apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | -apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark -| apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  -dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github |    -google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins |   -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  -yandex | ydb | zendesk]...                                                                                             +apache-airflow-providers | apache.arrow | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs +| apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot +| apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant |   +cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks |        +datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook |   +ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk +| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa |      +weaviate | yandex | ydb | zendesk]...                                                                                  Command to add back references for documentation to make it backward compatible. diff --git a/dev/breeze/doc/images/output_release-management_add-back-references.txt b/dev/breeze/doc/images/output_release-management_add-back-references.txt index 672e05eb6b26f..818fbaa7abb71 100644 --- a/dev/breeze/doc/images/output_release-management_add-back-references.txt +++ b/dev/breeze/doc/images/output_release-management_add-back-references.txt @@ -1 +1 @@ -2700b8a52b9a04bb3cf5e365e70c2f40 +028f4178ccfd317ac075b24446884433 diff --git a/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg b/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg index cef6e4d3b1d14..de7a137ca904d 100644 --- a/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg +++ b/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg @@ -162,17 +162,17 @@ Usage:                                                                                                                 breeze release-management classify-provider-changes                                                                    -[OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        -apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy -| apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | -clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab |   -facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins |    -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    -ydb | zendesk]...                                                                                                      +[OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.arrow | apache.cassandra | apache.drill |        +apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin +| apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana |             +atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch +| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    +jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      +yandex | ydb | zendesk]...                                                                                             Classify each provider's unreleased changes with hard-coded, high-confidence rules, flagging ambiguous commits as  'needs_llm' for an agent/skill to assess. Outputs JSON - a deterministic alternative to the random '--non-interactive' diff --git a/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt b/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt index 6e02c700afdf5..44b287551c6e1 100644 --- a/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt +++ b/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt @@ -1 +1 @@ -4cac13b21eee8b732a46c5a15aec7a4b +0f0f937f1a0d86a58147723b1400b3ac diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg index 2f9692be54fa0..628d1a972bbf1 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg @@ -151,17 +151,17 @@ Usage:                                                                                                                 breeze release-management generate-issue-content-providers                                                             -[OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        -apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy -| apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | -clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab |   -facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins |    -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    -ydb | zendesk]...                                                                                                      +[OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.arrow | apache.cassandra | apache.drill |        +apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin +| apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana |             +atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch +| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    +jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      +yandex | ydb | zendesk]...                                                                                             Generates content for issue to test the release. diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt index 6afb289b5854f..91f9d9e152d75 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt @@ -1 +1 @@ -bd80bf2dd63a27111b8c5745df5dfb86 +465c7719885f58cdf5fc8bd0592ba7de diff --git a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg index 64efb0af6b65b..3a4f3e5295534 100644 --- a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg +++ b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg @@ -169,19 +169,19 @@ │--provider-id     Provider_id to generate metadata for. If not specified, metadata for all packages will be       │ │generated. This is debug-only option. When this option is specified, the metadata produced for  │ │the distribution will not be written to the file, but printed via console. (airbyte | akeyless |│ -│alibaba | amazon | anthropic | apache.beam | apache.cassandra | apache.drill | apache.druid | │ -│apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | │ -│apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | │ -│apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes│ -│| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | │ -│datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab | │ -│facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica│ -│| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | │ -│microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | │ -│opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | │ -│redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake │ -│| sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |│ -│yandex | ydb | zendesk)│ +│alibaba | amazon | anthropic | apache.arrow | apache.beam | apache.cassandra | apache.drill | │ +│apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | │ +│apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | │ +│apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | │ +│cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |│ +│common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | │ +│elasticsearch | exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http │ +│| imap | influxdb | informatica | jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql │ +│| microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | │ +│openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | │ +│postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | │ +│singularity | slack | smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata│ +│| trino | vertica | vespa | weaviate | yandex | ydb | zendesk)│ │--provider-versionProvider version to generate metadata for. Only used when --provider-id is specified. Limits    │ │running metadata generation to only this version of the provider. (TEXT)│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt index 7c14e04e63151..8b1eec0eef352 100644 --- a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt +++ b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt @@ -1 +1 @@ -5865810f5a4b12c9199267b7dd6459b3 +ed9d0b9e6709f6f00a87d5510d556590 diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg index 27a1d92de9a4b..7692bdd01e377 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg @@ -189,17 +189,17 @@ Usage:                                                                                                                 breeze release-management prepare-provider-distributions                                                               -[OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        -apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy -| apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | -clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab |   -facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins |    -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    -ydb | zendesk]...                                                                                                      +[OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.arrow | apache.cassandra | apache.drill |        +apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin +| apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana |             +atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch +| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    +jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      +yandex | ydb | zendesk]...                                                                                             Prepare sdist/whl distributions of Airflow Providers. Each provider directory is wiped with `git clean -fdx`  (preserving .venv, .idea, .vscode) before build to keep in-tree generated files out of the artifact. See dev/breeze  diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt index f054c03806e05..14515df724948 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt @@ -1 +1 @@ -a27c1726f5902e5fdb501ecdee226476 +7c0f4946a611ae3372292e08bac3a642 diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg index b9c71f9c81aec..4a2a8e8c474b9 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg @@ -210,17 +210,17 @@ Usage:                                                                                                                 breeze release-management prepare-provider-documentation                                                               -[OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        -apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy -| apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | -clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab |   -facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins |    -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    -ydb | zendesk]...                                                                                                      +[OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.arrow | apache.cassandra | apache.drill |        +apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin +| apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana |             +atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch +| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    +jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      +yandex | ydb | zendesk]...                                                                                             Prepare CHANGELOG, README and COMMITS information for providers. diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt index a54080b6bfbb5..5b7184cf9c60b 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt @@ -1 +1 @@ -c33a2f6d00a3a8dbec8b56c1c2d88d54 +57343f19b12400ac43ea8fc4b892ac99 diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.svg b/dev/breeze/doc/images/output_release-management_publish-docs.svg index 9416d6183c582..3bf45d018c232 100644 --- a/dev/breeze/doc/images/output_release-management_publish-docs.svg +++ b/dev/breeze/doc/images/output_release-management_publish-docs.svg @@ -189,17 +189,17 @@ Usage:                                                                                                                 breeze release-management publish-docs                                                                                 [OPTIONS] [airbyte | akeyless | alibaba | all-providers | amazon | anthropic | apache-airflow | apache-airflow-ctl |   -apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | -apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark -| apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  -dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github |    -google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins |   -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  -yandex | ydb | zendesk]...                                                                                             +apache-airflow-providers | apache.arrow | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs +| apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot +| apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant |   +cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks |        +datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook |   +ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk +| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa |      +weaviate | yandex | ydb | zendesk]...                                                                                  Command to publish generated documentation to airflow-site diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.txt b/dev/breeze/doc/images/output_release-management_publish-docs.txt index b6433c39bb9be..7c32d9f24bda4 100644 --- a/dev/breeze/doc/images/output_release-management_publish-docs.txt +++ b/dev/breeze/doc/images/output_release-management_publish-docs.txt @@ -1 +1 @@ -cc4ad6886b5ff0da1d4f819d2c5dc80d +fa238acb203848e7895db16c51ea9454 diff --git a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg index a9d2e173ef2b8..c8f7e0e905c68 100644 --- a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg +++ b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg @@ -183,11 +183,11 @@ │--python-versions Comma separate list of Python versions to update sbom from (defaults to all historical python   │ │versions) (3.6 | 3.7 | 3.8 | 3.9 | 3.10 | 3.11 | 3.12 | 3.13 | 3.14)│ │--provider-id     Provider id to generate the requirements for (airbyte | akeyless | alibaba | amazon | anthropic │ -│| apache.beam | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | │ -│apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | │ -│apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | │ -│atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | │ -│common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud | │ +│| apache.arrow | apache.beam | apache.cassandra | apache.drill | apache.druid | apache.flink | │ +│apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | │ +│apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb |│ +│asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai│ +│| common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |│ │dingding | discord | docker | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | │ │github | google | grpc | hashicorp | http | ibm.mq | imap | influxdb | informatica | jdbc | │ │jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | │ diff --git a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt index bb3262922cec4..055be564a9b6f 100644 --- a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt +++ b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt @@ -1 +1 @@ -84f322f2ffa9ded046c2d3678135a8f7 +277232e8cfc8662b921d404ceb1e879d diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg index dfb0027b3d7ef..09f1caaf71f81 100644 --- a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg +++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg @@ -204,17 +204,17 @@ Usage:                                                                                                                 breeze workflow-run publish-docs                                                                                       [OPTIONS] [airbyte | akeyless | alibaba | all-providers | amazon | anthropic | apache-airflow | apache-airflow-ctl |   -apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | -apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark -| apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  -dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github |    -google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins |   -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  -yandex | ydb | zendesk]...                                                                                             +apache-airflow-providers | apache.arrow | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs +| apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot +| apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant |   +cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks |        +datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook |   +ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk +| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa |      +weaviate | yandex | ydb | zendesk]...                                                                                  Trigger publish docs to S3 workflow diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.txt b/dev/breeze/doc/images/output_workflow-run_publish-docs.txt index 4c6ec79dba124..a60fc61ff5181 100644 --- a/dev/breeze/doc/images/output_workflow-run_publish-docs.txt +++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.txt @@ -1 +1 @@ -e784b5234a05d958f84df96cd5d8d562 +94e9f7708bb8a8b7c7d20266403455a1 diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 530e6ad74aa46..c7e75862633dc 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -2850,7 +2850,7 @@ def test_upgrade_to_newer_dependencies( pytest.param( ("providers/common/sql/src/airflow/providers/common/sql/common_sql_python.py",), { - "docs-list-as-string": "amazon apache.drill apache.druid apache.hive apache.iceberg " + "docs-list-as-string": "amazon apache.arrow apache.drill apache.druid apache.hive apache.iceberg " "apache.impala apache.pinot clickhousedb common.ai common.compat common.sql databricks elasticsearch " "exasol google informatica jdbc microsoft.mssql mysql odbc openlineage " "oracle pgvector postgres presto slack snowflake sqlite teradata trino vertica ydb", diff --git a/providers/apache/arrow/.gitignore b/providers/apache/arrow/.gitignore new file mode 100644 index 0000000000000..bff2d7629604d --- /dev/null +++ b/providers/apache/arrow/.gitignore @@ -0,0 +1 @@ +*.iml diff --git a/providers/apache/arrow/LICENSE b/providers/apache/arrow/LICENSE new file mode 100644 index 0000000000000..11069edd79019 --- /dev/null +++ b/providers/apache/arrow/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/providers/apache/arrow/NOTICE b/providers/apache/arrow/NOTICE new file mode 100644 index 0000000000000..a51bd9390d030 --- /dev/null +++ b/providers/apache/arrow/NOTICE @@ -0,0 +1,5 @@ +Apache Airflow +Copyright 2016-2026 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/providers/apache/arrow/README.rst b/providers/apache/arrow/README.rst new file mode 100644 index 0000000000000..04840fbc7e8c7 --- /dev/null +++ b/providers/apache/arrow/README.rst @@ -0,0 +1,114 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + +.. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE + ``PROVIDER_README_TEMPLATE.rst.jinja2`` IN the ``dev/breeze/src/airflow_breeze/templates`` DIRECTORY + +Package ``apache-airflow-providers-apache-arrow`` + +Release: ``0.1.0`` + + +`Apache Arrow `__ + + +Provider package +---------------- + +This is a provider package for ``apache.arrow`` provider. All classes for this provider package +are in ``airflow.providers.apache.arrow`` python package. + +You can find package information and changelog for the provider +in the `documentation `_. + +Installation +------------ + +You can install this package on top of an existing Airflow 2 installation (see ``Requirements`` below +for the minimum Airflow version supported) via +``pip install apache-airflow-providers-apache-arrow`` + +The package supports the following python versions: 3.9,3.10,3.11,3.12 + +Requirements +------------ + +======================================= ====================================== +PIP package Version required +======================================= ====================================== +``apache-airflow`` ``>=2.11.0`` +``apache-airflow-providers-common-sql`` ``>=1.28.2`` +``adbc-driver-manager`` ``>=1.7.0`` +``importlib-resources`` ``>=1.3`` +``more-itertools`` ``>=9.0.0`` +``pyarrow`` ``>=16.1.0; python_version < "3.13"`` +``pyarrow`` ``>=18.0.0; python_version >= "3.13"`` +======================================= ====================================== + +Cross provider package dependencies +----------------------------------- + +Those are dependencies that might be needed in order to use all the features of the package. +You need to install the specified providers in order to use them. + +You can install such cross-provider dependencies when installing from PyPI. For example: + +.. code-block:: bash + + pip install apache-airflow-providers-arrow[common.sql] + + +============================================================================================================ ============== +Dependent package Extra +============================================================================================================ ============== +`apache-airflow-providers-common-sql `_ ``common.sql`` +============================================================================================================ ============== + +Optional dependencies +---------------------- + +This provider supports multiple ADBC drivers for different databases. You can install the required driver(s) as extras: + +============== ===================================================================================================================================================================== +Extra Dependency +============== ===================================================================================================================================================================== +``sqlite`` ``adbc-driver-sqlite>=1.7.0`` +``postgresql`` ``adbc-driver-postgresql>=1.7.0`` +``snowflake`` ``adbc-driver-snowflake>=1.7.0`` +``bigquery`` ``adbc-driver-bigquery>=1.7.0`` +``flightsql`` ``adbc-driver-flightsql>=1.7.0`` +``all`` ``adbc-driver-sqlite>=1.7.0``, ``adbc-driver-postgresql>=1.7.0``, ``adbc-driver-snowflake>=1.7.0``, ``adbc-driver-bigquery>=1.7.0``, ``adbc-driver-flightsql>=1.7.0`` +============== ===================================================================================================================================================================== + +To use the provider with a specific database, install the provider with the corresponding extra. For example, to use SQLite: + +.. code-block:: bash + + pip install apache-airflow-providers-apache-arrow[sqlite] + +To install all available drivers: + +.. code-block:: bash + + pip install apache-airflow-providers-apache-arrow[all] + +These extras install the Python ADBC drivers required to connect to each database. Refer to the documentation for each driver for any additional system requirements. + +The changelog for the provider package can be found in the +`changelog `_. diff --git a/providers/apache/arrow/docs/.latest-doc-only-change.txt b/providers/apache/arrow/docs/.latest-doc-only-change.txt new file mode 100644 index 0000000000000..f41e3226a6f43 --- /dev/null +++ b/providers/apache/arrow/docs/.latest-doc-only-change.txt @@ -0,0 +1 @@ +7b2ec33c7ad4998d9c9735b79593fcdcd3b9dd1f diff --git a/providers/apache/arrow/docs/changelog.rst b/providers/apache/arrow/docs/changelog.rst new file mode 100644 index 0000000000000..c893ea7e832e1 --- /dev/null +++ b/providers/apache/arrow/docs/changelog.rst @@ -0,0 +1,49 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +.. NOTE TO CONTRIBUTORS: + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes + and you want to add an explanation to the users on how they are supposed to deal with them. + The changelog is updated and maintained semi-automatically by release manager. + +``apache-airflow-providers-apache-arrow`` + +Changelog +--------- + +0.1.0 +..... + +.. note:: + This release of provider is only available for Airflow 2.10+ as explained in the + Apache Airflow providers support policy _. + +Misc +~~~~ + +* ``Bump min Airflow version in providers to 2.10 (#49843)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``Update description of provider.yaml dependencies (#50231)`` + * ``Avoid committing history for providers (#49907)`` + +0.1.0 +..... + +Initial version of the provider. diff --git a/providers/apache/arrow/docs/commits.rst b/providers/apache/arrow/docs/commits.rst new file mode 100644 index 0000000000000..78554447218d2 --- /dev/null +++ b/providers/apache/arrow/docs/commits.rst @@ -0,0 +1,35 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + + .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE + `PROVIDER_COMMITS_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN! + +Package apache-airflow-providers-apache-arrow +------------------------------------------------------ + +`ADBC: Arrow Database Connectivity `__ + + +This is detailed commit list of changes for versions provider package: ``arrow``. +For high-level changelog, see :doc:`package information including changelog `. + +.. airflow-providers-commits:: diff --git a/providers/apache/arrow/docs/conf.py b/providers/apache/arrow/docs/conf.py new file mode 100644 index 0000000000000..29c8f3976f1d7 --- /dev/null +++ b/providers/apache/arrow/docs/conf.py @@ -0,0 +1,27 @@ +# Disable Flake8 because of all the sphinx imports +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Configuration of Providers docs building.""" + +from __future__ import annotations + +import os + +os.environ["AIRFLOW_PACKAGE_NAME"] = "apache-airflow-providers-apache-arrow" + +from docs.provider_conf import * # noqa: F403 diff --git a/providers/apache/arrow/docs/connections/adbc.rst b/providers/apache/arrow/docs/connections/adbc.rst new file mode 100644 index 0000000000000..dbfdf76031bdb --- /dev/null +++ b/providers/apache/arrow/docs/connections/adbc.rst @@ -0,0 +1,167 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _howto/connection:adbc: + +ADBC connection +=============== + +The ADBC connection type enables connection to any database that has an +`Arrow Database Connectivity (ADBC) `__ driver, +such as PostgreSQL, SQLite, DuckDB, Snowflake, and Flight SQL servers. + +Default Connection ID +--------------------- + +``adbc_default`` + +Configuring the Connection +-------------------------- + +Connection URL (Host field) + The database URI passed to the driver. For drivers that use a standard + connection string (e.g. ``postgresql://user:pass@host:5432/db`` or + ``file::memory:``), put it here. If the value contains ``::`` the hook + passes it through unchanged; otherwise the ``adbc://`` scheme prefix is + replaced with the dialect name. + +Login / Password + Convenience fields. When set, Airflow builds a URI of the form + ``://login:password@host/schema`` that is merged into + ``db_kwargs["uri"]``. Driver-specific URIs in the Host field take + precedence. + +Extra (JSON) + A JSON object with the following recognized keys: + + ``driver`` *(string)* + Python package name of the ADBC driver to load, e.g. + ``"adbc_driver_postgresql"`` or ``"adbc_driver_sqlite"``. + When omitted the hook derives the name from the dialect as + ``adbc_driver_``. + + ``entrypoint`` *(string, optional)* + Fully-qualified Python symbol used as the driver entrypoint, e.g. + ``"adbc_driver_sqlite.dbapi.connect"``. Most drivers do not need + this; it is only required when the automatic entrypoint discovery + built into ``adbc_driver_manager`` cannot locate the function. + + ``dialect`` *(string, optional)* + SQL dialect name used to build the URI scheme and derive the + default driver name. Defaults to ``"default"``. + + ``db_kwargs`` *(object, optional)* + Keyword arguments forwarded verbatim to the ADBC + ``AdbcDatabase`` constructor (database-level connection settings). + ``uri`` is always injected automatically from the Host field and + must not be repeated here. Common keys: + + .. list-table:: + :header-rows: 1 + :widths: 30 15 55 + + * - Key + - Type + - Description + * - ``username`` + - string + - Database user name (alternative to encoding it in the URI). + * - ``password`` + - string + - Database password (alternative to encoding it in the URI). + * - ``adbc.connection.autocommit`` + - bool + - Enable autocommit at the database level (driver-dependent). + * - ``adbc.sqlite.query.batch_rows`` + - int + - SQLite: number of rows fetched per Arrow batch. + + All other keys are driver-specific. Consult the documentation for + your ADBC driver for the full list. See also the + `ADBC driver documentation `__. + + ``conn_kwargs`` *(object, optional)* + Keyword arguments forwarded verbatim to the ADBC + ``AdbcConnection`` constructor (connection-level settings within the + already-open database). Common keys defined by the ADBC + specification: + + .. list-table:: + :header-rows: 1 + :widths: 30 15 55 + + * - Key + - Type + - Description + * - ``autocommit`` + - bool + - Enable autocommit for this connection. + * - ``read_only`` + - bool + - Open the connection in read-only mode. + * - ``current_catalog`` + - string + - Default catalog for unqualified table names. + * - ``current_db_schema`` + - string + - Default schema for unqualified table names. + + All other keys are driver-specific. + +.. warning:: **Security — blast radius of db_kwargs / conn_kwargs** + + Both ``db_kwargs`` and ``conn_kwargs`` are passed through to the ADBC + driver **without filtering**. Any user or role that can edit this + connection in the Airflow UI or via the Connections API can therefore + supply arbitrary driver-level options. Depending on the driver, this + may include options that: + + * load extensions from arbitrary file-system paths (e.g. SQLite's + ``load_extension`` or DuckDB's extension loading); + * configure network endpoints or proxy addresses; + * change authentication parameters. + + Restrict access to the Connections UI and API (Airflow RBAC ``Connections`` + resource) to trusted operators only. Do not expose connection-editing + permissions to untrusted Dag authors. + + Similarly, the ``driver`` and ``entrypoint`` extras load shared libraries + or Python symbols by name. Treat them with the same level of trust as + executable code. + +Examples +-------- + +SQLite in-memory database:: + + { + "driver": "adbc_driver_sqlite" + } + +PostgreSQL with extra driver options:: + + { + "driver": "adbc_driver_postgresql", + "dialect": "postgresql", + "db_kwargs": { + "adbc.postgresql.query.batch_rows": 1000 + }, + "conn_kwargs": { + "autocommit": true, + "read_only": false + } + } diff --git a/providers/apache/arrow/docs/index.rst b/providers/apache/arrow/docs/index.rst new file mode 100644 index 0000000000000..4a19f78cba452 --- /dev/null +++ b/providers/apache/arrow/docs/index.rst @@ -0,0 +1,139 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +``apache-airflow-providers-apache-arrow`` +========================================= + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Basics + + Home + Changelog + Security + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: References + + Python API <_api/airflow/providers/apache/arrow/index> + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: System tests + + System Tests <_api/tests/system/apache/arrow/index> + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Resources + + PyPI Repository + Example Dags + Installing from sources + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Guides + + Connection types + +.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Commits + + Detailed list of commits + + +apache-airflow-providers-apache-arrow package +------------------------------------------------------ + +`Apache Arrow `__ + + +Release: 0.1.0 + +Provider package +---------------- + +This package is for the ``apache.arrow`` provider. +All classes for this package are included in the ``airflow.providers.apache.arrow`` python package. + +Installation +------------ + +You can install this package on top of an existing Airflow installation via +``pip install apache-airflow-providers-apache-arrow``. +For the minimum Airflow version supported, see ``Requirements`` below. + +Requirements +------------ + +The minimum Apache Airflow version supported by this provider distribution is ``2.11.0``. + +======================================= ====================================== +PIP package Version required +======================================= ====================================== +``apache-airflow`` ``>=2.11.0`` +``apache-airflow-providers-common-sql`` ``>=1.28.2`` +``adbc-driver-manager`` ``>=1.7.0`` +``importlib-resources`` ``>=1.3`` +``more-itertools`` ``>=9.0.0`` +``pyarrow`` ``>=16.1.0; python_version < "3.13"`` +``pyarrow`` ``>=18.0.0; python_version >= "3.13"`` +======================================= ====================================== + +Optional dependencies +--------------------- + +These extras install optional third-party libraries that enable additional features of the provider. +Install them when installing from PyPI. For example: + +.. code-block:: bash + + pip install apache-airflow-providers-apache-arrow[sqlite] + + +============== ===================================================================================================================================================================== +Extra Dependencies +============== ===================================================================================================================================================================== +``sqlite`` ``adbc-driver-sqlite>=1.7.0`` +``snowflake`` ``adbc-driver-snowflake>=1.7.0`` +``bigquery`` ``adbc-driver-bigquery>=1.7.0`` +``flightsql`` ``adbc-driver-flightsql>=1.7.0`` +``postgresql`` ``adbc-driver-postgresql>=1.7.0`` +``all`` ``adbc-driver-sqlite>=1.7.0``, ``adbc-driver-snowflake>=1.7.0``, ``adbc-driver-bigquery>=1.7.0``, ``adbc-driver-flightsql>=1.7.0``, ``adbc-driver-postgresql>=1.7.0`` +============== ===================================================================================================================================================================== + +Downloading official packages +----------------------------- + +You can download officially released packages and verify their checksums and signatures from the +`Official Apache Download site `_ + +* `The apache-airflow-providers-apache-arrow 0.1.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-arrow 0.1.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/apache/arrow/docs/installing-providers-from-sources.rst b/providers/apache/arrow/docs/installing-providers-from-sources.rst new file mode 100644 index 0000000000000..fdbb17d017579 --- /dev/null +++ b/providers/apache/arrow/docs/installing-providers-from-sources.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: /../../../../devel-common/src/sphinx_exts/includes/installing-providers-from-sources.rst diff --git a/providers/apache/arrow/docs/integration-logos/apache-arrow.png b/providers/apache/arrow/docs/integration-logos/apache-arrow.png new file mode 100644 index 0000000000000..164401d1eeff8 Binary files /dev/null and b/providers/apache/arrow/docs/integration-logos/apache-arrow.png differ diff --git a/providers/apache/arrow/docs/redirects.txt b/providers/apache/arrow/docs/redirects.txt new file mode 100644 index 0000000000000..3a3c336a8474d --- /dev/null +++ b/providers/apache/arrow/docs/redirects.txt @@ -0,0 +1 @@ +connections/index.rst connections/adbc.rst diff --git a/providers/apache/arrow/docs/security.rst b/providers/apache/arrow/docs/security.rst new file mode 100644 index 0000000000000..351ff007ebf2f --- /dev/null +++ b/providers/apache/arrow/docs/security.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: /../../../../devel-common/src/sphinx_exts/includes/security.rst diff --git a/providers/apache/arrow/provider.yaml b/providers/apache/arrow/provider.yaml new file mode 100644 index 0000000000000..8ac0e11203828 --- /dev/null +++ b/providers/apache/arrow/provider.yaml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +--- +package-name: apache-airflow-providers-apache-arrow +name: Apache Arrow + +state: ready +lifecycle: incubation +source-date-epoch: 1744788833 +description: | + `Apache Arrow `__ +# Note that those versions are maintained by release manager - do not update them manually +# with the exception of case where other provider in sources has >= new provider version. +# In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have +# to be done in the same PR +versions: + - 0.1.0 + +integrations: + - integration-name: Apache Arrow + external-doc-url: https://arrow.apache.org/ + logo: /docs/integration-logos/apache-arrow.png + tags: [apache] + +hooks: + - integration-name: Apache Arrow + python-modules: + - airflow.providers.apache.arrow.hooks.adbc + +connection-types: + - hook-class-name: airflow.providers.apache.arrow.hooks.adbc.AdbcHook + connection-type: adbc diff --git a/providers/apache/arrow/pyproject.toml b/providers/apache/arrow/pyproject.toml new file mode 100644 index 0000000000000..a552dac33f877 --- /dev/null +++ b/providers/apache/arrow/pyproject.toml @@ -0,0 +1,155 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + +# IF YOU WANT TO MODIFY THIS FILE EXCEPT DEPENDENCIES, YOU SHOULD MODIFY THE TEMPLATE +# `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY +[build-system] +requires = ["flit_core==3.12.0"] +build-backend = "flit_core.buildapi" + +[project] +name = "apache-airflow-providers-apache-arrow" +version = "0.1.0" +description = "Provider package apache-airflow-providers-apache-arrow for Apache Airflow" +readme = "README.rst" +license = "Apache-2.0" +license-files = ['LICENSE', 'NOTICE'] +authors = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +maintainers = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +keywords = [ "airflow-provider", "apache.arrow", "airflow", "integration" ] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Framework :: Apache Airflow", + "Framework :: Apache Airflow :: Provider", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: System :: Monitoring", +] +requires-python = ">=3.10" + +# The dependencies should be modified in place in the generated file. +# Any change in the dependencies is preserved when the file is regenerated +# Make sure to run ``prek update-providers-dependencies --all-files`` +# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build`` +dependencies = [ + "apache-airflow>=2.11.0", + "apache-airflow-providers-common-sql>=1.28.2", + "adbc-driver-manager>=1.7.0", + "more-itertools>=9.0.0", + "pyarrow>=16.1.0; python_version < '3.13'", + "pyarrow>=18.0.0; python_version >= '3.13'", +] + +# The optional dependencies should be modified in place in the generated file +# Any change in the dependencies is preserved when the file is regenerated +[project.optional-dependencies] +"sqlite" = [ + "adbc-driver-sqlite>=1.7.0" +] +"snowflake" = [ + "adbc-driver-snowflake>=1.7.0" +] +"bigquery" = [ + "adbc-driver-bigquery>=1.7.0" +] +"flightsql" = [ + "adbc-driver-flightsql>=1.7.0" +] +"postgresql" = [ + "adbc-driver-postgresql>=1.7.0" +] +"all" = [ + "adbc-driver-sqlite>=1.7.0", + "adbc-driver-snowflake>=1.7.0", + "adbc-driver-bigquery>=1.7.0", + "adbc-driver-flightsql>=1.7.0", + "adbc-driver-postgresql>=1.7.0" +] + +[dependency-groups] +dev = [ + "apache-airflow", + "apache-airflow-task-sdk", + "apache-airflow-devel-common", + "apache-airflow-providers-common-sql", + # Additional devel dependencies (do not remove this line and add extra development dependencies) +] + +# To build docs: +# +# uv run --group docs build-docs +# +# To enable auto-refreshing build with server: +# +# uv run --group docs build-docs --autobuild +# +# To see more options: +# +# uv run --group docs build-docs --help +# +docs = [ + "apache-airflow-devel-common[docs]" +] + +[tool.uv.sources] +# These names must match the names as defined in the pyproject.toml of the workspace items, +# *not* the workspace folder paths +apache-airflow = {workspace = true} +apache-airflow-devel-common = {workspace = true} +apache-airflow-task-sdk = {workspace = true} +apache-airflow-providers-common-sql = {workspace = true} +apache-airflow-providers-standard = {workspace = true} + +[project.urls] +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-arrow/0.1.0" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-arrow/0.1.0/changelog.html" +"Bug Tracker" = "https://github.com/apache/airflow/issues" +"Source Code" = "https://github.com/apache/airflow" +"Slack Chat" = "https://s.apache.org/airflow-slack" +"Mastodon" = "https://fosstodon.org/@airflow" +"YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/" + +[project.entry-points."apache_airflow_provider"] +provider_info = "airflow.providers.apache.arrow.get_provider_info:get_provider_info" + +[tool.flit.module] +name = "airflow.providers.apache.arrow" + +# Explicit sdist contents so the build does not rely on VCS information +# (flit 4.0 makes --no-use-vcs the default — see https://github.com/pypa/flit/pull/782). +[tool.flit.sdist] +include = [ + "docs/", + "provider.yaml", + "src/airflow/__init__.py", + "src/airflow/providers/__init__.py", + "src/airflow/providers/apache/__init__.py", + "tests/", +] diff --git a/providers/apache/arrow/src/airflow/__init__.py b/providers/apache/arrow/src/airflow/__init__.py new file mode 100644 index 0000000000000..e8fd22856438c --- /dev/null +++ b/providers/apache/arrow/src/airflow/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/apache/arrow/src/airflow/providers/__init__.py b/providers/apache/arrow/src/airflow/providers/__init__.py new file mode 100644 index 0000000000000..e8fd22856438c --- /dev/null +++ b/providers/apache/arrow/src/airflow/providers/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/apache/arrow/src/airflow/providers/apache/__init__.py b/providers/apache/arrow/src/airflow/providers/apache/__init__.py new file mode 100644 index 0000000000000..e8fd22856438c --- /dev/null +++ b/providers/apache/arrow/src/airflow/providers/apache/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/apache/arrow/src/airflow/providers/apache/arrow/LICENSE b/providers/apache/arrow/src/airflow/providers/apache/arrow/LICENSE new file mode 100644 index 0000000000000..11069edd79019 --- /dev/null +++ b/providers/apache/arrow/src/airflow/providers/apache/arrow/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/providers/apache/arrow/src/airflow/providers/apache/arrow/__init__.py b/providers/apache/arrow/src/airflow/providers/apache/arrow/__init__.py new file mode 100644 index 0000000000000..e76bded9f7ffd --- /dev/null +++ b/providers/apache/arrow/src/airflow/providers/apache/arrow/__init__.py @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE +# OVERWRITTEN WHEN PREPARING DOCUMENTATION FOR THE PACKAGES. +# +# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE +# `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY +# +from __future__ import annotations + +import packaging.version + +from airflow import __version__ as airflow_version + +__all__ = ["__version__"] + +__version__ = "0.1.0" + +if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( + "2.11.0" +): + raise RuntimeError( + f"The package `apache-airflow-providers-apache-arrow:{__version__}` needs Apache Airflow 2.11.0+" + ) diff --git a/providers/apache/arrow/src/airflow/providers/apache/arrow/get_provider_info.py b/providers/apache/arrow/src/airflow/providers/apache/arrow/get_provider_info.py new file mode 100644 index 0000000000000..7ca21c842eaab --- /dev/null +++ b/providers/apache/arrow/src/airflow/providers/apache/arrow/get_provider_info.py @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! +# +# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE +# `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + +def get_provider_info(): + return { + "package-name": "apache-airflow-providers-apache-arrow", + "name": "Apache Arrow", + "description": "`Apache Arrow `__\n", + "integrations": [ + { + "integration-name": "Apache Arrow", + "external-doc-url": "https://arrow.apache.org/", + "logo": "/docs/integration-logos/apache-arrow.png", + "tags": ["apache"], + } + ], + "hooks": [ + { + "integration-name": "Apache Arrow", + "python-modules": ["airflow.providers.apache.arrow.hooks.adbc"], + } + ], + "connection-types": [ + { + "hook-class-name": "airflow.providers.apache.arrow.hooks.adbc.AdbcHook", + "connection-type": "adbc", + } + ], + } diff --git a/providers/apache/arrow/src/airflow/providers/apache/arrow/hooks/__init__.py b/providers/apache/arrow/src/airflow/providers/apache/arrow/hooks/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/apache/arrow/src/airflow/providers/apache/arrow/hooks/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/apache/arrow/src/airflow/providers/apache/arrow/hooks/adbc.py b/providers/apache/arrow/src/airflow/providers/apache/arrow/hooks/adbc.py new file mode 100644 index 0000000000000..c7b89d829f182 --- /dev/null +++ b/providers/apache/arrow/src/airflow/providers/apache/arrow/hooks/adbc.py @@ -0,0 +1,314 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import contextlib +import pathlib +import re +import sys +from collections.abc import Iterable, Mapping +from contextlib import closing +from functools import cached_property +from typing import Any + +from importlib import resources as importlib_resources +from adbc_driver_manager.dbapi import Connection, connect +from more_itertools import chunked +from pyarrow import RecordBatch, Schema, array, schema + +from airflow.providers.common.sql.dialects.dialect import Dialect +from airflow.providers.common.sql.hooks.sql import DbApiHook + + +def fetch_all_handler(cursor) -> list[tuple] | None: + """Return results for DbApiHook.run().""" + if not hasattr(cursor, "description"): + raise RuntimeError( + "The database we interact with does not support DBAPI 2.0. Use operator and " + "handlers that are specifically designed for your database." + ) + if cursor.description is not None: + return list(zip(*cursor.fetch_arrow_table().to_pydict().values())) + return None + + +def replace_placeholders(sql: str, placeholder: str) -> str: + # Replace each placeholder with $1, $2, $3 ... in order + counter = [1] + + def replacer(match): + replacement = f"${counter[0]}" + counter[0] += 1 + return replacement + + return re.sub(placeholder, replacer, sql) + + +# https://arrow.apache.org/adbc/current/python/api/adbc_driver_manager.html +# https://arrow.apache.org/docs/python/ +class AdbcHook(DbApiHook): + """ + General-purpose Airflow hook for interacting with databases via the Arrow Database Connectivity (ADBC) standard. + + This hook enables connections to any database supported by an ADBC driver, using the Python ADBC driver manager. + It provides methods for executing SQL queries, inserting rows in bulk, and handling Arrow-native data transfers. + + Key Features: + - Supports chunked and batched inserts using Apache Arrow RecordBatches for efficient data transfer. + - Discovers and loads ADBC drivers dynamically based on connection extras or naming conventions. + - Handles dialect-specific connection URIs and driver entrypoints. + - Integrates with Airflow's connection system (conn_id, extras, etc.). + - Provides custom placeholder replacement for parameterized SQL queries. + - Supports both native Arrow binding and DBAPI ``executemany`` for inserts. + - Exposes configuration via connection extras: driver, entrypoint, db_kwargs, conn_kwargs, dialect. + + Connection Extras: + - driver: Name of the ADBC driver to use (e.g., "adbc_driver_postgresql"). + - entrypoint: Optional Python entrypoint for the driver. + - db_kwargs: Dict of keyword arguments passed to the database connection. + - conn_kwargs: Dict of keyword arguments passed to the driver connect function. + - dialect: SQL dialect name (default: "default"). + + Example usage: + hook = AdbcHook(adbc_conn_id="my_adbc_conn") + records = hook.get_records("SELECT * FROM my_table") + + For more details, see: + - Apache Arrow ADBC Python API: https://arrow.apache.org/adbc/current/python/api/adbc_driver_manager.html + - Airflow SQL hooks: https://airflow.apache.org/docs/apache-airflow/stable/howto/custom-operator.html#hooks + """ + + conn_name_attr = "adbc_conn_id" + default_conn_name = "adbc_default" + conn_type = "adbc" + hook_name = "ADBC Connection" + supports_autocommit = True + + @classmethod + def get_ui_field_behaviour(cls) -> dict[str, Any]: + """Get custom field behaviour.""" + return { + "hidden_fields": ["port", "schema"], + "relabeling": {"host": "Connection URL"}, + } + + @cached_property + def _driver_path(self) -> str: + # Wheels bundle the shared library + root = importlib_resources.files(self.driver) + # The filename is always the same regardless of platform + entrypoint = root.joinpath(f"lib{self.driver}.so") + if entrypoint.is_file(): + return str(entrypoint) + + # Search sys.prefix + '/lib' (Unix, Conda on Unix) + root = pathlib.Path(sys.prefix) + for filename in (f"lib{self.driver}.so", f"lib{self.driver}.dylib"): + entrypoint = root.joinpath("lib", filename) + if entrypoint.is_file(): + return str(entrypoint) + + # Conda on Windows + entrypoint = root.joinpath("bin", f"{self.driver}.dll") + if entrypoint.is_file(): + return str(entrypoint) + + # Let the driver manager fall back to (DY)LD_LIBRARY_PATH/PATH + # (It will insert 'lib', 'so', etc. as needed) + return self.driver + + @cached_property + def uri(self) -> str: + host = self.connection.host + if host and "::" in str(host): + return str(host) + uri = self.get_uri() + return uri.replace( + f"{self.conn_type.lower().replace('_', '-')}://", + f"{self.dialect_name.lower().replace('_', '-')}://", + ) + + @cached_property + def driver(self) -> str: + return self.connection_extra_lower.get("driver") or f"adbc_driver_{self.dialect_name}" + + @cached_property + def entrypoint(self) -> str | None: + return self.connection_extra_lower.get("entrypoint") + + @cached_property + def db_kwargs(self) -> dict: + return {**{"uri": self.uri}, **self.connection_extra_lower.get("db_kwargs", {})} + + @cached_property + def conn_kwargs(self) -> dict: + return self.connection_extra_lower.get("conn_kwargs", {}) + + @cached_property + def dialect_name(self) -> str: + return self.connection_extra_lower.get("dialect", "default") + + def get_conn(self) -> Connection: + return connect( + driver=self._driver_path, + entrypoint=self.entrypoint, + db_kwargs=self.db_kwargs, + conn_kwargs=self.conn_kwargs, + autocommit=False, + ) + + def get_records( + self, + sql: str | list[str], + parameters: Iterable | Mapping[str, Any] | None = None, + ) -> Any: + """ + Execute the sql and return a set of records. + + :param sql: the sql statement to be executed (str) or a list of sql statements to execute + :param parameters: The parameters to render the SQL query with. + """ + return self.run(sql=sql, parameters=parameters, handler=fetch_all_handler) + + def _run_command(self, cur, sql_statement, parameters): + """Run a statement using an already open cursor.""" + if parameters: + sql_statement = replace_placeholders(sql_statement, re.escape(self.dialect.placeholder)) + + super()._run_command(cur, sql_statement, parameters) + + def _generate_insert_sql(self, table, values, target_fields=None, replace: bool = False, **kwargs) -> str: + sql_statement = super()._generate_insert_sql( + table, values, target_fields=target_fields, replace=replace, **kwargs + ) + sql_statement = replace_placeholders(sql_statement, re.escape(self.dialect.placeholder)) + + if self.log_sql: + self.log.info("Running statement: %s", sql_statement) + + return sql_statement + + @classmethod + def _to_record_batch(cls, rows, schema: Schema) -> RecordBatch: + return RecordBatch.from_arrays( + [array([row[index] for row in rows], type=field.type) for index, field in enumerate(schema)], + schema=schema, + ) + + @classmethod + def _execute_native_bind(cls, cursor, statement: str, record_batch: RecordBatch) -> None: + """Execute a statement using native Arrow bind on the cursor.""" + cursor.bind(record_batch) + cursor.execute(statement) + + @classmethod + def _execute_executemany(cls, cursor, statement: str, record_batch: RecordBatch) -> None: + """Execute a statement using cursor.executemany.""" + cursor.executemany(statement, record_batch) + + def insert_rows( + self, + table, + rows, + target_fields=None, + commit_every=1000, + replace=False, + *, + executemany=False, + fast_executemany=False, + autocommit=False, + **kwargs, + ): + """ + Insert a collection of tuples into a table. + + Rows are inserted in chunks, each chunk (of size ``commit_every``) is + done in a new transaction. + + :param table: Name of the target table + :param rows: The rows to insert into the table + :param target_fields: The names of the columns to fill in the table + :param commit_every: The maximum number of rows to insert in one + transaction. Set to 0 to insert all rows in one transaction. + :param replace: Whether to replace instead of insert + :param executemany: If True, all rows are inserted at once in + chunks defined by the commit_every parameter. This only works if all rows + have same number of column names, but leads to better performance. + :param fast_executemany: If True, the `fast_executemany` parameter will be set on the + cursor used by `executemany` which leads to better performance, if supported by driver. + :param autocommit: What to set the connection's autocommit setting to + before executing the query. + """ + nb_rows = 0 + + with self._create_autocommit_connection(autocommit) as conn: + table_name, schema_name = Dialect.extract_schema_from_table(table) + + table_schema = conn.adbc_get_table_schema( + table_name=table_name, + db_schema_filter=schema_name, + ) + + if not target_fields: + target_fields = table_schema.names + else: + table_schema = schema([field for field in table_schema if field.name in target_fields]) + + self.log.info("target fields: %s", target_fields) + self.log.info("table_schema: %s", table_schema) + + sql = self._generate_insert_sql( + table, + target_fields, # values not needed — parameters will come from RecordBatch + target_fields, + replace, + **kwargs, + ) + + with closing(conn.cursor()) as cur: + use_native_bind = hasattr(cur, "bind") + + # If native bind is not available, consider executemany path and + # try to enable fast_executemany if requested and supported. + if not use_native_bind and (self.supports_executemany or executemany): + if fast_executemany: + with contextlib.suppress(AttributeError): + # Try to set the fast_executemany attribute + cur.fast_executemany = True + self.log.info( + "Fast_executemany is enabled for conn_id '%s'!", + self.get_conn_id(), + ) + + # Choose the execution callable once based on cursor capability + if use_native_bind: + self.log.info("Native Arrow bind supported!") + execute_batch = self._execute_native_bind + else: + execute_batch = self._execute_executemany + + for chunked_rows in chunked(rows, commit_every): + batch = self._to_record_batch(rows=chunked_rows, schema=table_schema) + execute_batch(cur, sql, batch) + + conn.commit() + + nb_rows += batch.num_rows + self.log.info("Loaded %s rows into %s so far", nb_rows, table) + + self.log.info("Done loading. Loaded a total of %s rows into %s", nb_rows, table) diff --git a/providers/apache/arrow/tests/conftest.py b/providers/apache/arrow/tests/conftest.py new file mode 100644 index 0000000000000..f56ccce0a3f69 --- /dev/null +++ b/providers/apache/arrow/tests/conftest.py @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +pytest_plugins = "tests_common.pytest_plugin" diff --git a/providers/apache/arrow/tests/system/__init__.py b/providers/apache/arrow/tests/system/__init__.py new file mode 100644 index 0000000000000..e8fd22856438c --- /dev/null +++ b/providers/apache/arrow/tests/system/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/apache/arrow/tests/system/apache/__init__.py b/providers/apache/arrow/tests/system/apache/__init__.py new file mode 100644 index 0000000000000..e8fd22856438c --- /dev/null +++ b/providers/apache/arrow/tests/system/apache/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/apache/arrow/tests/system/apache/arrow/__init__.py b/providers/apache/arrow/tests/system/apache/arrow/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/apache/arrow/tests/system/apache/arrow/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/apache/arrow/tests/unit/__init__.py b/providers/apache/arrow/tests/unit/__init__.py new file mode 100644 index 0000000000000..e8fd22856438c --- /dev/null +++ b/providers/apache/arrow/tests/unit/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/apache/arrow/tests/unit/apache/__init__.py b/providers/apache/arrow/tests/unit/apache/__init__.py new file mode 100644 index 0000000000000..e8fd22856438c --- /dev/null +++ b/providers/apache/arrow/tests/unit/apache/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/apache/arrow/tests/unit/apache/arrow/__init__.py b/providers/apache/arrow/tests/unit/apache/arrow/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/apache/arrow/tests/unit/apache/arrow/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/apache/arrow/tests/unit/apache/arrow/hooks/__init__.py b/providers/apache/arrow/tests/unit/apache/arrow/hooks/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/apache/arrow/tests/unit/apache/arrow/hooks/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/apache/arrow/tests/unit/apache/arrow/hooks/test_adbc.py b/providers/apache/arrow/tests/unit/apache/arrow/hooks/test_adbc.py new file mode 100644 index 0000000000000..fcebcfbc7f577 --- /dev/null +++ b/providers/apache/arrow/tests/unit/apache/arrow/hooks/test_adbc.py @@ -0,0 +1,283 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import importlib +import json +import logging +from unittest import mock + +import pytest +from adbc_driver_manager import dbapi +from adbc_driver_manager.dbapi import Cursor +from pyarrow import field, schema, string + +from airflow.models import Connection +from airflow.providers.apache.arrow.hooks.adbc import AdbcHook +from airflow.providers.common.sql.dialects.dialect import Dialect + + +class TestAdbcHook: + def setup_method(self): + # Create a MagicMock cursor similar to DbApiHook tests + self.cur = mock.MagicMock(rowcount=0, fast_executemany=False) + self.conn = mock.MagicMock() + self.conn.cursor.return_value = self.cur + # Schema and extras that the hook might read + # Provide a real pyarrow Schema so _to_record_batch can build RecordBatch + self.conn.adbc_get_table_schema.return_value = schema([field("col", string())]) + self.conn.extra_dejson = {} + conn = self.conn + + logging.root.disabled = True + + # Instantiate the hook under test + self.hook = self.make_hook_for_conn(conn) + + # Ensure the cursor used in unit tests has a native bind method + # to simulate native Arrow bind support. + self.cur.bind = mock.MagicMock() + + # Make fetch_arrow_table().to_pydict() return a simple column mapping + arrow_table = mock.MagicMock() + arrow_table.to_pydict.return_value = {"col": [1, 2]} + self.cur.fetch_arrow_table.return_value = arrow_table + + def make_hook_for_conn(self, conn): + """ + Return an AdbcHook subclass instance bound to the provided conn. + + Tests previously redefined this subclass locally in multiple places. + This helper centralizes that logic so tests can simply call + `self.make_hook_for_conn(conn)`. + """ + + class AdbcHookMock(AdbcHook): + conn_name_attr = "adbc_default" + + @classmethod + def get_connection(cls, conn_id: str) -> Connection: + return conn + + def get_conn(self): + return conn + + @property + def dialect(self): + return Dialect(self) + + def get_db_log_messages(self, _conn) -> None: + return _conn.get_messages() + + return AdbcHookMock() + + def test_get_records_fetch_all_handler(self): + result = self.hook.get_records("SELECT 1") + assert result == [(1,), (2,)] + + def test_insert_rows_native_bind(self): + table = "table" + rows = [("a",), ("b",)] + + # Native bind supported (cursor has bind attribute) + self.hook.insert_rows(table, rows) + + assert self.cur.bind.called + assert self.cur.execute.called + assert self.conn.commit.call_count >= 1 + + def test_insert_rows_fast_executemany_not_supported(self): + # Cursor without native bind that doesn't support setting fast_executemany + class NoFastExecCursor(mock.MagicMock): + def __setattr__(self, name, value): + if name == "fast_executemany": + raise AttributeError("fast_executemany not supported") + super().__setattr__(name, value) + + cur = NoFastExecCursor(spec=Cursor) + delattr(cur, "bind") # Remove bind to simulate no native bind support + conn = mock.MagicMock() + conn.cursor.return_value = cur + conn.adbc_get_table_schema.return_value = schema([field("col", string())]) + conn.extra_dejson = {} + hook = self.make_hook_for_conn(conn) + + table = "table" + rows = [("x",), ("y",)] + + hook.insert_rows(table, rows, executemany=True, fast_executemany=True) + + assert cur.executemany.called + assert conn.commit.call_count >= 1 + + def test_insert_rows_fast_executemany_supported(self): + # Cursor without native bind but supports setting fast_executemany + cur = mock.MagicMock(spec=Cursor) + delattr(cur, "bind") # Remove bind to simulate no native bind support + conn = mock.MagicMock() + conn.cursor.return_value = cur + conn.adbc_get_table_schema.return_value = schema([field("col", string())]) + conn.extra_dejson = {} + hook = self.make_hook_for_conn(conn) + + table = "table" + rows = [("x",), ("y",)] + + hook.insert_rows(table, rows, executemany=True, fast_executemany=True) + + assert cur.fast_executemany + assert cur.executemany.called + assert conn.commit.call_count >= 1 + + @pytest.mark.skipif( + importlib.util.find_spec("adbc_driver_sqlite") is None, + reason="adbc_driver_sqlite not installed", + ) + def test_dbapi_connection(self, create_connection_without_db): + create_connection_without_db( + Connection( + conn_id="adbc_default", + conn_type="adbc", + host="file::memory:?cache=shared", + extra=json.dumps( + { + "driver": "adbc_driver_sqlite", + } + ), + ) + ) + + with AdbcHook()._create_autocommit_connection() as adbc_conn: + assert isinstance(adbc_conn, dbapi.Connection) + + def _make_real_conn_hook(self, connection: Connection) -> AdbcHook: + """Return an AdbcHook whose get_connection() returns *connection* but whose + get_conn() is the real implementation (needed to verify connect() kwargs).""" + + class AdbcHookWithRealGetConn(AdbcHook): + conn_name_attr = "adbc_default" + + @classmethod + def get_connection(cls, conn_id: str) -> Connection: + return connection + + return AdbcHookWithRealGetConn() + + @mock.patch("airflow.providers.apache.arrow.hooks.adbc.connect") + def test_get_conn_forwards_db_kwargs(self, mock_connect): + conn = Connection( + conn_id="adbc_default", + conn_type="adbc", + host="file::memory:", + extra=json.dumps( + { + "driver": "adbc_driver_sqlite", + "db_kwargs": {"timeout": 30, "username": "admin"}, + } + ), + ) + hook = self._make_real_conn_hook(conn) + hook.__dict__["_driver_path"] = "adbc_driver_sqlite" + + hook.get_conn() + + mock_connect.assert_called_once() + kw = mock_connect.call_args.kwargs + assert kw["db_kwargs"]["timeout"] == 30 + assert kw["db_kwargs"]["username"] == "admin" + assert kw["db_kwargs"]["uri"] == "file::memory:" + + @mock.patch("airflow.providers.apache.arrow.hooks.adbc.connect") + def test_get_conn_forwards_conn_kwargs(self, mock_connect): + conn = Connection( + conn_id="adbc_default", + conn_type="adbc", + host="file::memory:", + extra=json.dumps( + { + "driver": "adbc_driver_sqlite", + "conn_kwargs": {"autocommit": True, "read_only": True}, + } + ), + ) + hook = self._make_real_conn_hook(conn) + hook.__dict__["_driver_path"] = "adbc_driver_sqlite" + + hook.get_conn() + + mock_connect.assert_called_once() + kw = mock_connect.call_args.kwargs + assert kw["conn_kwargs"] == {"autocommit": True, "read_only": True} + + @mock.patch("airflow.providers.apache.arrow.hooks.adbc.connect") + def test_get_conn_forwards_entrypoint(self, mock_connect): + conn = Connection( + conn_id="adbc_default", + conn_type="adbc", + host="file::memory:", + extra=json.dumps( + { + "driver": "adbc_driver_sqlite", + "entrypoint": "adbc_driver_sqlite.dbapi.connect", + } + ), + ) + hook = self._make_real_conn_hook(conn) + hook.__dict__["_driver_path"] = "adbc_driver_sqlite" + + hook.get_conn() + + mock_connect.assert_called_once() + kw = mock_connect.call_args.kwargs + assert kw["entrypoint"] == "adbc_driver_sqlite.dbapi.connect" + + def test_dialect_name_from_extra(self): + conn = Connection( + conn_id="adbc_default", + conn_type="adbc", + host="file::memory:", + extra=json.dumps({"driver": "adbc_driver_postgresql", "dialect": "postgresql"}), + ) + hook = self.make_hook_for_conn(conn) + assert hook.dialect_name == "postgresql" + + def test_dialect_name_defaults_to_default(self): + conn = Connection( + conn_id="adbc_default", + conn_type="adbc", + host="file::memory:", + extra=json.dumps({"driver": "adbc_driver_sqlite"}), + ) + hook = self.make_hook_for_conn(conn) + assert hook.dialect_name == "default" + + @mock.patch("airflow.providers.apache.arrow.hooks.adbc.connect") + def test_get_conn_empty_extras_defaults(self, mock_connect): + conn = Connection( + conn_id="adbc_default", + conn_type="adbc", + host="file::memory:", + extra=json.dumps({"driver": "adbc_driver_sqlite"}), + ) + hook = self._make_real_conn_hook(conn) + hook.__dict__["_driver_path"] = "adbc_driver_sqlite" + + hook.get_conn() + + kw = mock_connect.call_args.kwargs + assert kw["conn_kwargs"] == {} + assert kw["entrypoint"] is None diff --git a/pyproject.toml b/pyproject.toml index 7e28ca58b1c1b..a22ebc6208753 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -113,6 +113,9 @@ apache-airflow = "airflow.__main__:main" "anthropic" = [ "apache-airflow-providers-anthropic>=0.1.0" ] +"apache.arrow" = [ + "apache-airflow-providers-apache-arrow>=0.1.0" +] "apache.cassandra" = [ "apache-airflow-providers-apache-cassandra>=3.7.0; python_version !=\"3.14\"" ] @@ -415,6 +418,7 @@ apache-airflow = "airflow.__main__:main" "apache-airflow-providers-alibaba>=3.0.0", "apache-airflow-providers-amazon>=9.0.0", "apache-airflow-providers-anthropic>=0.1.0", + "apache-airflow-providers-apache-arrow>=0.1.0", "apache-airflow-providers-apache-cassandra>=3.7.0; python_version !=\"3.14\"", "apache-airflow-providers-apache-drill>=2.8.1", "apache-airflow-providers-apache-druid>=3.12.0", @@ -1096,6 +1100,8 @@ mypy_path = [ "$MYPY_CONFIG_FILE_DIR/providers/amazon/tests", "$MYPY_CONFIG_FILE_DIR/providers/anthropic/src", "$MYPY_CONFIG_FILE_DIR/providers/anthropic/tests", + "$MYPY_CONFIG_FILE_DIR/providers/apache/arrow/src", + "$MYPY_CONFIG_FILE_DIR/providers/apache/arrow/tests", "$MYPY_CONFIG_FILE_DIR/providers/apache/cassandra/src", "$MYPY_CONFIG_FILE_DIR/providers/apache/cassandra/tests", "$MYPY_CONFIG_FILE_DIR/providers/apache/drill/src", @@ -1434,6 +1440,7 @@ apache-airflow-providers-akeyless = false apache-airflow-providers-alibaba = false apache-airflow-providers-amazon = false apache-airflow-providers-anthropic = false +apache-airflow-providers-apache-arrow = false apache-airflow-providers-apache-cassandra = false apache-airflow-providers-apache-drill = false apache-airflow-providers-apache-druid = false @@ -1579,6 +1586,7 @@ apache-airflow-providers-akeyless = false apache-airflow-providers-alibaba = false apache-airflow-providers-amazon = false apache-airflow-providers-anthropic = false +apache-airflow-providers-apache-arrow = false apache-airflow-providers-apache-cassandra = false apache-airflow-providers-apache-drill = false apache-airflow-providers-apache-druid = false @@ -1739,6 +1747,7 @@ apache-airflow-providers-akeyless = { workspace = true } apache-airflow-providers-alibaba = { workspace = true } apache-airflow-providers-amazon = { workspace = true } apache-airflow-providers-anthropic = { workspace = true } +apache-airflow-providers-apache-arrow = { workspace = true } apache-airflow-providers-apache-cassandra = { workspace = true } apache-airflow-providers-apache-drill = { workspace = true } apache-airflow-providers-apache-druid = { workspace = true } @@ -1879,6 +1888,7 @@ members = [ "providers/alibaba", "providers/amazon", "providers/anthropic", + "providers/apache/arrow", "providers/apache/cassandra", "providers/apache/drill", "providers/apache/druid", diff --git a/scripts/ci/docker-compose/remove-sources.yml b/scripts/ci/docker-compose/remove-sources.yml index ed2e4b1bed789..72c7f66ca5387 100644 --- a/scripts/ci/docker-compose/remove-sources.yml +++ b/scripts/ci/docker-compose/remove-sources.yml @@ -31,6 +31,7 @@ services: - ../../../empty:/opt/airflow/providers/alibaba/src - ../../../empty:/opt/airflow/providers/amazon/src - ../../../empty:/opt/airflow/providers/anthropic/src + - ../../../empty:/opt/airflow/providers/apache/arrow/src - ../../../empty:/opt/airflow/providers/apache/beam/src - ../../../empty:/opt/airflow/providers/apache/cassandra/src - ../../../empty:/opt/airflow/providers/apache/drill/src diff --git a/scripts/ci/docker-compose/tests-sources.yml b/scripts/ci/docker-compose/tests-sources.yml index 1f6e47ba961c2..68c036a07a36a 100644 --- a/scripts/ci/docker-compose/tests-sources.yml +++ b/scripts/ci/docker-compose/tests-sources.yml @@ -44,6 +44,7 @@ services: - ../../../providers/alibaba/tests:/opt/airflow/providers/alibaba/tests - ../../../providers/amazon/tests:/opt/airflow/providers/amazon/tests - ../../../providers/anthropic/tests:/opt/airflow/providers/anthropic/tests + - ../../../providers/apache/arrow/tests:/opt/airflow/providers/apache/arrow/tests - ../../../providers/apache/beam/tests:/opt/airflow/providers/apache/beam/tests - ../../../providers/apache/cassandra/tests:/opt/airflow/providers/apache/cassandra/tests - ../../../providers/apache/drill/tests:/opt/airflow/providers/apache/drill/tests diff --git a/uv.lock b/uv.lock index 2b08c898b6c38..90ac57215dcef 100644 --- a/uv.lock +++ b/uv.lock @@ -101,6 +101,7 @@ apache-airflow-shared-serialization = false apache-airflow-scripts = false apache-airflow-providers-exasol = false apache-airflow-providers-mongo = false +apache-airflow-providers-apache-arrow = false apache-airflow-providers-apprise = false apache-airflow-providers-apache-impala = false apache-airflow-ctl = false @@ -177,6 +178,7 @@ members = [ "apache-airflow-providers-alibaba", "apache-airflow-providers-amazon", "apache-airflow-providers-anthropic", + "apache-airflow-providers-apache-arrow", "apache-airflow-providers-apache-cassandra", "apache-airflow-providers-apache-drill", "apache-airflow-providers-apache-druid", @@ -322,6 +324,40 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/49/8d/58008a9a86075827f99aa8bb75d8db515bb9c34654f95e647cda31987db7/adal-1.2.7-py2.py3-none-any.whl", hash = "sha256:2a7451ed7441ddbc57703042204a3e30ef747478eea022c70f789fc7f084bc3d", size = 55539, upload-time = "2021-04-05T16:33:39.544Z" }, ] +[[package]] +name = "adbc-driver-bigquery" +version = "1.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "adbc-driver-manager" }, + { name = "importlib-resources" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/0d/3e7195500badfe3318d6409e3d7461354f7e3e57b3b8ee2cb5a4c0c78bbd/adbc_driver_bigquery-1.11.0.tar.gz", hash = "sha256:37fc2a90dfec1f741bc7475bdf50c744c22d05ec135d0858934117912c06077e", size = 19255, upload-time = "2026-04-07T00:17:26.014Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/0f/76baf47962bfa49608fcd5d38c3a46213155aeb1f58e5c8abf467ed7e4bc/adbc_driver_bigquery-1.11.0-py3-none-macosx_10_15_x86_64.whl", hash = "sha256:4153bb759dcfa6c45420c4ee9a1505f3c5daa7685d10cf068a91b8876e183732", size = 9736982, upload-time = "2026-04-07T00:14:31.386Z" }, + { url = "https://files.pythonhosted.org/packages/15/0b/b88fd632ea5b26ded6f11cbfc62d154249b54d4ae95ad8bdc2e681f3cbb0/adbc_driver_bigquery-1.11.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:023744303c4900fc8e3d12cd044835d406ee8a84880458766aff0800601d4999", size = 9049987, upload-time = "2026-04-07T00:14:37.48Z" }, + { url = "https://files.pythonhosted.org/packages/9e/0a/ce0b0772f479eb451993ed9a0d139fcdaddcab651d1ce09520003163aa54/adbc_driver_bigquery-1.11.0-py3-none-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3067b90ccbb6801936567bddcc56e79540efb7e09023f0300e9f6090cb7245e1", size = 18043100, upload-time = "2026-04-07T00:14:44.159Z" }, + { url = "https://files.pythonhosted.org/packages/74/2c/f71aaea78f4859935e334b1c5791e959cb24315201789bb9f7a8fd5a94c2/adbc_driver_bigquery-1.11.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dea28d663b622e1b4c803fd063b0b959dc867218461ecb6e34b557423e956527", size = 16537326, upload-time = "2026-04-07T00:14:49.284Z" }, + { url = "https://files.pythonhosted.org/packages/f2/41/595bdc39a6e8eea2be3ccd043a10e68243b8a682d45f52b5d982674269f3/adbc_driver_bigquery-1.11.0-py3-none-win_amd64.whl", hash = "sha256:75ff98aa9071707419c300f9da74e19da375cd6524ecf09493c8131960e31c3c", size = 17698043, upload-time = "2026-04-07T00:14:53.622Z" }, +] + +[[package]] +name = "adbc-driver-flightsql" +version = "1.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "adbc-driver-manager" }, + { name = "importlib-resources" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e2/d8/b0bf88456d9acf85812b1d28568f3c3f80019d2b52d2131d28daea259d1e/adbc_driver_flightsql-1.11.0.tar.gz", hash = "sha256:75f703eef1812c3932e5f4643c5e21b5690b23df7e09e88b727f7952aa559f2a", size = 34941, upload-time = "2026-04-07T00:17:26.829Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/af/3e40778760257b594df4efc0b12cbfe3c182bdcb69f67a9730cfad5e9cae/adbc_driver_flightsql-1.11.0-py3-none-macosx_10_15_x86_64.whl", hash = "sha256:6cceffd95af5cdf5f3be051629de6bfd65744e591177b0f8176cd6810725e8f0", size = 8014539, upload-time = "2026-04-07T00:14:57.149Z" }, + { url = "https://files.pythonhosted.org/packages/b2/06/6ae41136909c88fc85c913b008d3d9878683289a851ec78924ab11c9e3bc/adbc_driver_flightsql-1.11.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:2e716c15bfad65c489d13a49528de5bcce72179fb8024be19353b143af5749fd", size = 7437919, upload-time = "2026-04-07T00:15:04.504Z" }, + { url = "https://files.pythonhosted.org/packages/9a/37/d8d12630ebbe336517b42aed9d9d4deab91e2bee153a1cba6ea9af9c5026/adbc_driver_flightsql-1.11.0-py3-none-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cabff560abcf18cebef0bf24bb2886d145bae340e8eef18cf0b6642e09fa349d", size = 14696685, upload-time = "2026-04-07T00:15:07.644Z" }, + { url = "https://files.pythonhosted.org/packages/bd/d2/18147e9fda30d41644ef75c9b30a19c8bc39734b3308e127b17747079e93/adbc_driver_flightsql-1.11.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6252f0a4c8a6240d51b61dcefed88fb3d9fa1e5bcc67cb954c94907f447b744d", size = 13471551, upload-time = "2026-04-07T00:15:10.792Z" }, + { url = "https://files.pythonhosted.org/packages/97/e4/343f698d0377db4a99009af3cf4ce8dbdc36f1578d9be71e128ee2e3a225/adbc_driver_flightsql-1.11.0-py3-none-win_amd64.whl", hash = "sha256:955865ed9a5746073a7b78c291ebb59aaaeaf0244e394b1e5a037c53cf61900c", size = 14475054, upload-time = "2026-04-07T00:15:14.949Z" }, +] + [[package]] name = "adbc-driver-manager" version = "1.11.0" @@ -384,6 +420,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a7/47/dd2322a40537ee9e22aa9937c70614ca29e9259d1b8b17441b92fbb49d5c/adbc_driver_postgresql-1.11.0-py3-none-win_amd64.whl", hash = "sha256:95c13b3203615b816a258db91c375785750cb82395b985f16c0dc6af88b932e3", size = 3067900, upload-time = "2026-04-07T00:16:53.08Z" }, ] +[[package]] +name = "adbc-driver-snowflake" +version = "1.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "adbc-driver-manager" }, + { name = "importlib-resources" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fd/c1/561549e36abe8644bedc6b4848abb79ab41a6b0fc4b106ceef944b0c3493/adbc_driver_snowflake-1.11.0.tar.gz", hash = "sha256:7fce54ae7540c6c63aad6cc9ec2795a05647cc147c8af41ca2bd4e95455e8eae", size = 30570, upload-time = "2026-04-07T00:17:30.403Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/f3/1125c7929faf750e7a5bdcec33ca22259ce040338dae12f75f3eb8080def/adbc_driver_snowflake-1.11.0-py3-none-macosx_10_15_x86_64.whl", hash = "sha256:5ebddfa5fc0b5cd7c86b5cce17bb578889c8db5ff2b756903b64e684b56829e8", size = 15094744, upload-time = "2026-04-07T00:16:55.86Z" }, + { url = "https://files.pythonhosted.org/packages/d5/1d/88d564a8941607e6e555a3d3290ecb92852d4aa5a926a1cdbe86b17a8424/adbc_driver_snowflake-1.11.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:be862f8d0b6a507dae5c0644e13be79290180532c097d97c5a95a294c0bb5302", size = 13517204, upload-time = "2026-04-07T00:17:00.137Z" }, + { url = "https://files.pythonhosted.org/packages/6f/26/baa44b8b9faaf26846dd8c0e513c7ce6862d7f35de46ee61175ac976699f/adbc_driver_snowflake-1.11.0-py3-none-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7c7d7ecef7246303648a08cee3d78c383133bea8f2e1568cf6e8750ab94b5a57", size = 27574063, upload-time = "2026-04-07T00:17:04.862Z" }, + { url = "https://files.pythonhosted.org/packages/5d/41/fe557f0d8414037dfe86fcba6338eb094a51da894050ec9a4c9f4504513d/adbc_driver_snowflake-1.11.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:07b5edbeb1b85be0882d7095641df26a24b3b4c2ba4d8edda35e3bbe89e70a06", size = 24767291, upload-time = "2026-04-07T00:17:09.763Z" }, + { url = "https://files.pythonhosted.org/packages/e9/ca/542793fc9a164f013e3248061e922ac5d5246ce93718a5c0cb44bbd2219e/adbc_driver_snowflake-1.11.0-py3-none-win_amd64.whl", hash = "sha256:83b1a363a44ae4cb1e4956f9cc6c4e44dddd7c768a0f0a8b42d9cb8eb8906305", size = 27264722, upload-time = "2026-04-07T00:17:14.083Z" }, +] + [[package]] name = "adbc-driver-sqlite" version = "1.11.0" @@ -1006,6 +1059,7 @@ all = [ { name = "apache-airflow-providers-alibaba" }, { name = "apache-airflow-providers-amazon", extra = ["aiobotocore", "python3-saml", "s3fs"] }, { name = "apache-airflow-providers-anthropic" }, + { name = "apache-airflow-providers-apache-arrow" }, { name = "apache-airflow-providers-apache-cassandra", marker = "python_full_version != '3.14.*'" }, { name = "apache-airflow-providers-apache-drill" }, { name = "apache-airflow-providers-apache-druid" }, @@ -1127,6 +1181,9 @@ amazon-aws-auth = [ anthropic = [ { name = "apache-airflow-providers-anthropic" }, ] +apache-arrow = [ + { name = "apache-airflow-providers-apache-arrow" }, +] apache-atlas = [ { name = "atlasclient" }, ] @@ -1577,6 +1634,8 @@ requires-dist = [ { name = "apache-airflow-providers-amazon", extras = ["s3fs"], marker = "extra == 's3fs'", editable = "providers/amazon" }, { name = "apache-airflow-providers-anthropic", marker = "extra == 'all'", editable = "providers/anthropic" }, { name = "apache-airflow-providers-anthropic", marker = "extra == 'anthropic'", editable = "providers/anthropic" }, + { name = "apache-airflow-providers-apache-arrow", marker = "extra == 'all'", editable = "providers/apache/arrow" }, + { name = "apache-airflow-providers-apache-arrow", marker = "extra == 'apache-arrow'", editable = "providers/apache/arrow" }, { name = "apache-airflow-providers-apache-cassandra", marker = "python_full_version != '3.14.*' and extra == 'all'", editable = "providers/apache/cassandra" }, { name = "apache-airflow-providers-apache-cassandra", marker = "python_full_version != '3.14.*' and extra == 'apache-cassandra'", editable = "providers/apache/cassandra" }, { name = "apache-airflow-providers-apache-drill", marker = "extra == 'all'", editable = "providers/apache/drill" }, @@ -1788,7 +1847,7 @@ requires-dist = [ { name = "sentry-sdk", marker = "extra == 'sentry'", specifier = ">=2.30.0" }, { name = "uv", marker = "extra == 'uv'", specifier = ">=0.11.26" }, ] -provides-extras = ["all-core", "async", "graphviz", "gunicorn", "kerberos", "memray", "otel", "statsd", "all-task-sdk", "airbyte", "akeyless", "alibaba", "amazon", "anthropic", "apache-cassandra", "apache-drill", "apache-druid", "apache-flink", "apache-hdfs", "apache-hive", "apache-iceberg", "apache-impala", "apache-kafka", "apache-kylin", "apache-livy", "apache-pig", "apache-pinot", "apache-spark", "apache-tinkerpop", "apprise", "arangodb", "asana", "atlassian-jira", "celery", "clickhousedb", "cloudant", "cncf-kubernetes", "cohere", "common-ai", "common-compat", "common-io", "common-messaging", "common-sql", "databricks", "datadog", "dbt-cloud", "dingding", "discord", "docker", "edge3", "elasticsearch", "exasol", "fab", "facebook", "ftp", "git", "github", "google", "grpc", "hashicorp", "http", "ibm-mq", "imap", "influxdb", "informatica", "jdbc", "jenkins", "keycloak", "microsoft-azure", "microsoft-mssql", "microsoft-psrp", "microsoft-winrm", "mongo", "mysql", "neo4j", "odbc", "openai", "openfaas", "openlineage", "opensearch", "opsgenie", "oracle", "pagerduty", "papermill", "pgvector", "pinecone", "postgres", "presto", "qdrant", "redis", "salesforce", "samba", "segment", "sendgrid", "sftp", "singularity", "slack", "smtp", "snowflake", "sqlite", "ssh", "standard", "tableau", "telegram", "teradata", "trino", "vertica", "vespa", "weaviate", "yandex", "ydb", "zendesk", "all", "aiobotocore", "apache-atlas", "apache-webhdfs", "amazon-aws-auth", "cloudpickle", "github-enterprise", "google-auth", "ldap", "pandas", "polars", "rabbitmq", "sentry", "s3fs", "uv"] +provides-extras = ["all-core", "async", "graphviz", "gunicorn", "kerberos", "memray", "otel", "statsd", "all-task-sdk", "airbyte", "akeyless", "alibaba", "amazon", "anthropic", "apache-arrow", "apache-cassandra", "apache-drill", "apache-druid", "apache-flink", "apache-hdfs", "apache-hive", "apache-iceberg", "apache-impala", "apache-kafka", "apache-kylin", "apache-livy", "apache-pig", "apache-pinot", "apache-spark", "apache-tinkerpop", "apprise", "arangodb", "asana", "atlassian-jira", "celery", "clickhousedb", "cloudant", "cncf-kubernetes", "cohere", "common-ai", "common-compat", "common-io", "common-messaging", "common-sql", "databricks", "datadog", "dbt-cloud", "dingding", "discord", "docker", "edge3", "elasticsearch", "exasol", "fab", "facebook", "ftp", "git", "github", "google", "grpc", "hashicorp", "http", "ibm-mq", "imap", "influxdb", "informatica", "jdbc", "jenkins", "keycloak", "microsoft-azure", "microsoft-mssql", "microsoft-psrp", "microsoft-winrm", "mongo", "mysql", "neo4j", "odbc", "openai", "openfaas", "openlineage", "opensearch", "opsgenie", "oracle", "pagerduty", "papermill", "pgvector", "pinecone", "postgres", "presto", "qdrant", "redis", "salesforce", "samba", "segment", "sendgrid", "sftp", "singularity", "slack", "smtp", "snowflake", "sqlite", "ssh", "standard", "tableau", "telegram", "teradata", "trino", "vertica", "vespa", "weaviate", "yandex", "ydb", "zendesk", "all", "aiobotocore", "apache-atlas", "apache-webhdfs", "amazon-aws-auth", "cloudpickle", "github-enterprise", "google-auth", "ldap", "pandas", "polars", "rabbitmq", "sentry", "s3fs", "uv"] [package.metadata.requires-dev] ci-image = [ @@ -3281,6 +3340,85 @@ dev = [ ] docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] +[[package]] +name = "apache-airflow-providers-apache-arrow" +version = "0.1.0" +source = { editable = "providers/apache/arrow" } +dependencies = [ + { name = "adbc-driver-manager" }, + { name = "apache-airflow" }, + { name = "apache-airflow-providers-common-sql" }, + { name = "importlib-resources" }, + { name = "more-itertools" }, + { name = "pyarrow" }, +] + +[package.optional-dependencies] +all = [ + { name = "adbc-driver-bigquery" }, + { name = "adbc-driver-flightsql" }, + { name = "adbc-driver-postgresql" }, + { name = "adbc-driver-snowflake" }, + { name = "adbc-driver-sqlite" }, +] +bigquery = [ + { name = "adbc-driver-bigquery" }, +] +flightsql = [ + { name = "adbc-driver-flightsql" }, +] +postgresql = [ + { name = "adbc-driver-postgresql" }, +] +snowflake = [ + { name = "adbc-driver-snowflake" }, +] +sqlite = [ + { name = "adbc-driver-sqlite" }, +] + +[package.dev-dependencies] +dev = [ + { name = "apache-airflow" }, + { name = "apache-airflow-devel-common" }, + { name = "apache-airflow-providers-common-sql" }, + { name = "apache-airflow-task-sdk" }, +] +docs = [ + { name = "apache-airflow-devel-common", extra = ["docs"] }, +] + +[package.metadata] +requires-dist = [ + { name = "adbc-driver-bigquery", marker = "extra == 'all'", specifier = ">=1.7.0" }, + { name = "adbc-driver-bigquery", marker = "extra == 'bigquery'", specifier = ">=1.7.0" }, + { name = "adbc-driver-flightsql", marker = "extra == 'all'", specifier = ">=1.7.0" }, + { name = "adbc-driver-flightsql", marker = "extra == 'flightsql'", specifier = ">=1.7.0" }, + { name = "adbc-driver-manager", specifier = ">=1.7.0" }, + { name = "adbc-driver-postgresql", marker = "extra == 'all'", specifier = ">=1.7.0" }, + { name = "adbc-driver-postgresql", marker = "extra == 'postgresql'", specifier = ">=1.7.0" }, + { name = "adbc-driver-snowflake", marker = "extra == 'all'", specifier = ">=1.7.0" }, + { name = "adbc-driver-snowflake", marker = "extra == 'snowflake'", specifier = ">=1.7.0" }, + { name = "adbc-driver-sqlite", marker = "extra == 'all'", specifier = ">=1.7.0" }, + { name = "adbc-driver-sqlite", marker = "extra == 'sqlite'", specifier = ">=1.7.0" }, + { name = "apache-airflow", editable = "." }, + { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, + { name = "importlib-resources", specifier = ">=1.3" }, + { name = "more-itertools", specifier = ">=9.0.0" }, + { name = "pyarrow", marker = "python_full_version < '3.13'", specifier = ">=16.1.0" }, + { name = "pyarrow", marker = "python_full_version >= '3.13'", specifier = ">=18.0.0" }, +] +provides-extras = ["sqlite", "snowflake", "bigquery", "flightsql", "postgresql", "all"] + +[package.metadata.requires-dev] +dev = [ + { name = "apache-airflow", editable = "." }, + { name = "apache-airflow-devel-common", editable = "devel-common" }, + { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, + { name = "apache-airflow-task-sdk", editable = "task-sdk" }, +] +docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] + [[package]] name = "apache-airflow-providers-apache-cassandra" version = "3.9.5"