diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 09baf2a826c..9d6099646a7 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -65,7 +65,6 @@ body: label: Spark version description: Please provide the spark version in your environment. options: - - Spark-3.3.x - Spark-3.4.x - Spark-3.5.x - Spark-4.0.x diff --git a/.github/workflows/build_bundle_package.yml b/.github/workflows/build_bundle_package.yml index 4de2e943b67..1a5f9a832ef 100644 --- a/.github/workflows/build_bundle_package.yml +++ b/.github/workflows/build_bundle_package.yml @@ -27,7 +27,7 @@ on: workflow_dispatch: inputs: spark: - description: 'Spark version: spark-3.3, spark-3.4, spark-3.5 or spark-4.0' + description: 'Spark version: spark-3.4, spark-3.5, spark-4.0 or spark-4.1' required: true default: 'spark-3.5' hadoop: diff --git a/.github/workflows/util/install-spark-resources.sh b/.github/workflows/util/install-spark-resources.sh index bfbb55f55d7..98b0d1bd004 100755 --- a/.github/workflows/util/install-spark-resources.sh +++ b/.github/workflows/util/install-spark-resources.sh @@ -95,11 +95,6 @@ if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then mkdir -p ${INSTALL_DIR} case "$1" in - 3.3) - # Spark-3.3 - cd ${INSTALL_DIR} && \ - install_spark "3.3.1" "3" "2.12" - ;; 3.4) # Spark-3.4 cd ${INSTALL_DIR} && \ diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index e5b60c0fc8c..631c9c4b122 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -64,14 +64,14 @@ concurrency: jobs: # Detect which parts of the codebase were modified so downstream jobs can - # skip work that is irrelevant to the change (e.g. a Spark-3.3-shim-only change does - # not need to run spark-test-spark34/35/40/41; C++ changes still run spark-test-* + # skip work that is irrelevant to the change (e.g. a Spark-3.4-shim-only change does + # not need to run spark-test-spark35/40/41; C++ changes still run spark-test-* # jobs to validate JNI/native integration). # # Output flags: # cpp – C++ sources, Velox build scripts, or dev tooling changed # java – any cross-version Java/Scala/Maven source changed - # shims33/34/35/40/41 – version-specific shim layer changed + # shims34/35/40/41 – version-specific shim layer changed # tools_it – tools/gluten-it changed # # A job runs when ANY of the relevant flags is true, so: @@ -93,7 +93,6 @@ jobs: full_run: ${{ steps.filter.outputs.full_run }} cpp: ${{ steps.filter.outputs.cpp }} java: ${{ steps.filter.outputs.java }} - shims33: ${{ steps.filter.outputs.shims33 }} shims34: ${{ steps.filter.outputs.shims34 }} shims35: ${{ steps.filter.outputs.shims35 }} shims40: ${{ steps.filter.outputs.shims40 }} @@ -115,7 +114,7 @@ jobs: recent=$(git log --oneline --after="24 hours ago" HEAD | wc -l) if [ "$recent" -eq 0 ]; then echo "No commits in the last 24 hours -- skipping nightly run." - for flag in cpp java shims33 shims34 shims35 shims40 shims41 tools_it; do + for flag in cpp java shims34 shims35 shims40 shims41 tools_it; do echo "$flag=false" >> $GITHUB_OUTPUT done echo "full_run=false" >> $GITHUB_OUTPUT @@ -123,7 +122,7 @@ jobs: fi fi # Run everything, including the extended matrices PR runs skip. - for flag in cpp java shims33 shims34 shims35 shims40 shims41 tools_it; do + for flag in cpp java shims34 shims35 shims40 shims41 tools_it; do echo "$flag=true" >> $GITHUB_OUTPUT done echo "full_run=true" >> $GITHUB_OUTPUT @@ -139,7 +138,6 @@ jobs: echo "cpp=$(match '^(cpp/|ep/build-velox/|dev/)')" >> $GITHUB_OUTPUT echo "java=$(match '^(\.github/workflows/|pom\.xml|backends-velox/|gluten-(uniffle|celeborn|ras|core|substrait|arrow|delta|iceberg|hudi|paimon)/|gluten-ut/(common/|test/|pom\.xml)|package/|build/mvn)')" >> $GITHUB_OUTPUT - echo "shims33=$(match '^(shims/(spark33|common)/|gluten-ut/spark33/)')" >> $GITHUB_OUTPUT echo "shims34=$(match '^(shims/(spark34|common)/|gluten-ut/spark34/)')" >> $GITHUB_OUTPUT echo "shims35=$(match '^(shims/(spark35|common)/|gluten-ut/spark35/)')" >> $GITHUB_OUTPUT echo "shims40=$(match '^(shims/(spark40|common)/|gluten-ut/spark40/)')" >> $GITHUB_OUTPUT @@ -204,9 +202,9 @@ jobs: fail-fast: false matrix: os: [ "ubuntu:22.04" ] - spark: [ "spark-3.3", "spark-3.4", "spark-3.5", "spark-4.0", "spark-4.1" ] + spark: [ "spark-3.4", "spark-3.5", "spark-4.0", "spark-4.1" ] # PR runs test the primary JDK per Spark line (8 for 3.x, 17 for 4.x -- - # after the static excludes below that is 6 combos instead of 11). The + # after the static excludes below that is 4 combos instead of 8). The # alternative-JDK combos (11/21/25) validate build + TPC run under JDKs # that Gluten changes rarely break in a JDK-specific way; the nightly # full_run keeps covering them daily so a break surfaces within a day @@ -217,14 +215,10 @@ jobs: || '["java-8", "java-17"]') }} # Spark supports JDK17 since 3.3. exclude: - - spark: spark-3.3 - java: java-25 - spark: spark-3.4 java: java-25 - spark: spark-3.5 java: java-25 - - spark: spark-3.3 - java: java-21 - spark: spark-3.4 java: java-21 - spark: spark-3.5 @@ -233,8 +227,6 @@ jobs: java: java-17 - spark: spark-3.5 java: java-17 - - spark: spark-3.3 - java: java-11 - spark: spark-3.4 java: java-11 - spark: spark-4.0 @@ -348,12 +340,12 @@ jobs: # TPC run that tpc-test-ubuntu already exercises combo-by-combo, so PRs # test one representative combo per Spark line ({3.5, 4.1} x primary JDK # = 2 jobs after the excludes below) and the nightly full_run restores - # the whole 7-combo fan-out. + # the whole 5-combo fan-out. matrix: os: [ "centos:8" ] spark: >- ${{ fromJSON(needs.detect-changes.outputs.full_run == 'true' - && '["spark-3.3", "spark-3.4", "spark-3.5", "spark-4.0", "spark-4.1"]' + && '["spark-3.4", "spark-3.5", "spark-4.0", "spark-4.1"]' || '["spark-3.5", "spark-4.1"]') }} java: >- ${{ fromJSON(needs.detect-changes.outputs.full_run == 'true' @@ -365,8 +357,6 @@ jobs: java: java-17 - spark: spark-3.5 java: java-17 - - spark: spark-3.3 - java: java-11 - spark: spark-3.4 java: java-11 - spark: spark-4.0 @@ -493,7 +483,7 @@ jobs: strategy: fail-fast: false matrix: - spark: [ "spark-3.3" ] + spark: [ "spark-3.5" ] runs-on: ubuntu-22.04 steps: - name: Maximize build disk space @@ -528,7 +518,7 @@ jobs: cd $GITHUB_WORKSPACE/tools/gluten-it $GITHUB_WORKSPACE/$MVN_CMD clean install -P${{ matrix.spark }} GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh data-gen-only --local --benchmark-type=ds -s=30.0 --threads=12 - - name: TPC-DS SF30.0 Parquet local spark3.3 Q67/Q95 low memory, memory isolation off + - name: TPC-DS SF30.0 Parquet local spark3.5 Q67/Q95 low memory, memory isolation off run: | cd tools/gluten-it \ && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \ @@ -540,7 +530,7 @@ jobs: -d=OVER_ACQUIRE:0.3,spark.gluten.memory.overAcquiredMemoryRatio=0.3 \ -d=OVER_ACQUIRE:0.5,spark.gluten.memory.overAcquiredMemoryRatio=0.5 \ --excluded-dims=OFFHEAP_SIZE:4g - - name: TPC-DS SF30.0 Parquet local spark3.3 Q67 low memory, memory isolation on + - name: TPC-DS SF30.0 Parquet local spark3.5 Q67 low memory, memory isolation on run: | cd tools/gluten-it \ && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \ @@ -551,7 +541,7 @@ jobs: -d=OFFHEAP_SIZE:4g,spark.memory.offHeap.size=4g \ -d=OVER_ACQUIRE:0.3,spark.gluten.memory.overAcquiredMemoryRatio=0.3 \ -d=OVER_ACQUIRE:0.5,spark.gluten.memory.overAcquiredMemoryRatio=0.5 - - name: TPC-DS SF30.0 Parquet local spark3.3 Q95 low memory, memory isolation on + - name: TPC-DS SF30.0 Parquet local spark3.5 Q95 low memory, memory isolation on run: | cd tools/gluten-it \ && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \ @@ -562,7 +552,7 @@ jobs: -d=OFFHEAP_SIZE:4g,spark.memory.offHeap.size=4g \ -d=OVER_ACQUIRE:0.3,spark.gluten.memory.overAcquiredMemoryRatio=0.3 \ -d=OVER_ACQUIRE:0.5,spark.gluten.memory.overAcquiredMemoryRatio=0.5 - - name: TPC-DS SF30.0 Parquet local spark3.3 Q23A/Q23B low memory + - name: TPC-DS SF30.0 Parquet local spark3.5 Q23A/Q23B low memory run: | cd tools/gluten-it \ && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \ @@ -573,7 +563,7 @@ jobs: -d=FLUSH_MODE:DISABLED,spark.gluten.sql.columnar.backend.velox.flushablePartialAggregation=false,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=100,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 \ -d=FLUSH_MODE:ABANDONED,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 \ -d=FLUSH_MODE:FLUSHED,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=0.05,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=0.1,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=100,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 - - name: TPC-DS SF30.0 Parquet local spark3.3 Q23A/Q23B low memory, memory isolation on + - name: TPC-DS SF30.0 Parquet local spark3.5 Q23A/Q23B low memory, memory isolation on run: | cd tools/gluten-it \ && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \ @@ -584,7 +574,7 @@ jobs: -d=FLUSH_MODE:DISABLED,spark.gluten.sql.columnar.backend.velox.flushablePartialAggregation=false,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=100,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 \ -d=FLUSH_MODE:ABANDONED,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 \ -d=FLUSH_MODE:FLUSHED,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=0.05,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=0.1,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=100,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 - - name: TPC-DS SF30.0 Parquet local spark3.3 Q97 low memory + - name: TPC-DS SF30.0 Parquet local spark3.5 Q97 low memory run: | cd tools/gluten-it \ && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \ @@ -610,7 +600,7 @@ jobs: strategy: fail-fast: false matrix: - spark: [ "spark-3.3" ] + spark: [ "spark-3.5" ] shard: [ 1, 2, 3 ] runs-on: ubuntu-22.04 steps: @@ -646,7 +636,7 @@ jobs: cd $GITHUB_WORKSPACE/tools/gluten-it $GITHUB_WORKSPACE/$MVN_CMD clean install -P${{ matrix.spark }} GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh data-gen-only --local --benchmark-type=ds -s=30.0 --threads=12 - - name: TPC-DS SF30.0 Parquet local spark3.3 random kill tasks (shard ${{ matrix.shard }}/3) + - name: TPC-DS SF30.0 Parquet local spark3.5 random kill tasks (shard ${{ matrix.shard }}/3) run: | cd tools/gluten-it \ && GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh queries \ @@ -662,7 +652,7 @@ jobs: strategy: fail-fast: false matrix: - spark: [ "spark-3.3" ] + spark: [ "spark-3.5" ] uniffle: [ "0.10.0" ] hadoop: [ "2.10.2" ] runs-on: ubuntu-22.04 @@ -695,7 +685,7 @@ jobs: run: | cd $GITHUB_WORKSPACE/ && \ $MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox -Puniffle -DskipTests - - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.3 with uniffle-${{ matrix.uniffle }} + - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.5 with uniffle-${{ matrix.uniffle }} run: | export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \ cd $GITHUB_WORKSPACE/tools/gluten-it && \ @@ -712,7 +702,7 @@ jobs: strategy: fail-fast: false matrix: - spark: [ "spark-3.3" ] + spark: [ "spark-3.5" ] # Both shuffle writers stay per-PR (they exercise different Gluten # writer code paths); the older Celeborn release is client-compat # coverage that the nightly full_run re-checks daily instead of every @@ -745,7 +735,7 @@ jobs: run: | cd $GITHUB_WORKSPACE/ $MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox -Pceleborn -DskipTests - - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.3 with ${{ matrix.celeborn }} + - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.5 with ${{ matrix.celeborn }} run: | EXTRA_PROFILE="" if [ "${{ matrix.celeborn }}" = "celeborn-0.5.4" ]; then @@ -764,7 +754,7 @@ jobs: bash -c "echo -e 'CELEBORN_MASTER_MEMORY=8g\nCELEBORN_WORKER_MEMORY=8g\nCELEBORN_WORKER_OFFHEAP_MEMORY=16g' > ./conf/celeborn-env.sh" && \ bash -c "echo -e 'celeborn.worker.commitFiles.threads 32\nceleborn.worker.sortPartition.threads 16' > ./conf/celeborn-defaults.conf" && \ bash ./sbin/start-master.sh && bash ./sbin/start-worker.sh && \ - cd $GITHUB_WORKSPACE/tools/gluten-it && $GITHUB_WORKSPACE/$MVN_CMD clean install -Pspark-3.3 -Pceleborn ${EXTRA_PROFILE} && \ + cd $GITHUB_WORKSPACE/tools/gluten-it && $GITHUB_WORKSPACE/$MVN_CMD clean install -P${{ matrix.spark }} -Pceleborn ${EXTRA_PROFILE} && \ GLUTEN_IT_JVM_ARGS=-Xmx16G sbin/gluten-it.sh queries-compare \ --extra-conf=spark.celeborn.client.spark.shuffle.writer=${{ matrix.writer }} \ --extra-conf=spark.sql.shuffle.partitions=16 \ @@ -787,110 +777,6 @@ jobs: --off-heap-size=16g -s=1.0 --threads=16 --iterations=1 fi - spark-test-spark33: - needs: [detect-changes, build-native-lib-centos-7] - if: >- - needs.detect-changes.outputs.java == 'true' || - needs.detect-changes.outputs.shims33 == 'true' || - needs.detect-changes.outputs.cpp == 'true' - runs-on: ubuntu-22.04 - env: - SPARK_TESTING: true - container: apache/gluten:centos-8-jdk8 - steps: - - uses: actions/checkout@v7 - - name: Download All Artifacts - uses: actions/download-artifact@v8 - with: - name: velox-native-lib-centos-7-${{github.sha}} - path: ./cpp/build/releases/ - - name: Prepare - run: | - dnf module -y install python39 && \ - alternatives --set python3 /usr/bin/python3.9 && \ - pip3 install setuptools==77.0.3 && \ - pip3 install pyspark==3.3.1 cython && \ - pip3 install pandas==2.2.3 pyarrow==20.0.0 - - name: Build and Run unit test for Spark 3.3.1 (other tests) - run: | - cd $GITHUB_WORKSPACE/ - export SPARK_SCALA_VERSION=2.12 - yum install -y java-17-openjdk-devel - export JAVA_HOME=/usr/lib/jvm/java-17-openjdk - export PATH=$JAVA_HOME/bin:$PATH - java -version - $MVN_CMD clean test -Pspark-3.3 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ - -DargLine="-Dspark.test.home=/opt/shims/spark33/spark_home/" \ - -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.CudfTest,org.apache.gluten.tags.SkipTest - - name: Upload test report - if: always() - uses: actions/upload-artifact@v7 - with: - name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' - - name: Upload unit tests log files - if: ${{ !success() }} - uses: actions/upload-artifact@v7 - with: - name: ${{ github.job }}-test-log - path: | - **/target/*.log - **/gluten-ut/**/hs_err_*.log - **/gluten-ut/**/core.* - - name: Upload golden files - if: failure() - uses: actions/upload-artifact@v7 - with: - name: ${{ github.job }}-golden-files - path: /tmp/tpch-approved-plan/** - - spark-test-spark33-slow: - needs: [detect-changes, build-native-lib-centos-7] - if: >- - needs.detect-changes.outputs.java == 'true' || - needs.detect-changes.outputs.shims33 == 'true' || - needs.detect-changes.outputs.cpp == 'true' - runs-on: ubuntu-22.04 - env: - SPARK_TESTING: true - container: apache/gluten:centos-8-jdk8 - steps: - - uses: actions/checkout@v7 - - name: Download All Artifacts - uses: actions/download-artifact@v8 - with: - name: velox-native-lib-centos-7-${{github.sha}} - path: ./cpp/build/releases/ - - - name: Build and Run unit test for Spark 3.3.1 (slow tests) - run: | - cd $GITHUB_WORKSPACE/ - yum install -y java-17-openjdk-devel - export JAVA_HOME=/usr/lib/jvm/java-17-openjdk - export PATH=$JAVA_HOME/bin:$PATH - java -version - $MVN_CMD clean test -Pspark-3.3 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ - -DargLine="-Dspark.test.home=/opt/shims/spark33/spark_home/" \ - -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest - $MVN_CMD clean test -Pspark-3.3 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ - -DargLine="-Dspark.test.home=/opt/shims/spark33/spark_home/" \ - -DtagsToInclude=org.apache.spark.tags.SlowHiveTest - - name: Upload test report - if: always() - uses: actions/upload-artifact@v7 - with: - name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' - - name: Upload unit tests log files - if: ${{ !success() }} - uses: actions/upload-artifact@v7 - with: - name: ${{ github.job }}-test-log - path: | - **/target/*.log - **/gluten-ut/**/hs_err_*.log - **/gluten-ut/**/core.* - spark-test-spark34: needs: [detect-changes, build-native-lib-centos-7] if: >- diff --git a/.github/workflows/velox_nightly.yml b/.github/workflows/velox_nightly.yml index 6f040216941..a11a2d94cb5 100644 --- a/.github/workflows/velox_nightly.yml +++ b/.github/workflows/velox_nightly.yml @@ -97,7 +97,6 @@ jobs: - name: Build package for Spark run: | cd $GITHUB_WORKSPACE/ && \ - ./build/mvn clean install -Pspark-3.3 -Pbackends-velox -Pceleborn -Puniffle -DskipTests -Dmaven.source.skip ./build/mvn clean install -Pspark-3.4 -Pbackends-velox -Pceleborn -Puniffle -DskipTests -Dmaven.source.skip ./build/mvn clean install -Pspark-3.5 -Pbackends-velox -Pceleborn -Puniffle -DskipTests -Dmaven.source.skip - name: Upload bundle package @@ -248,7 +247,6 @@ jobs: - name: Build package for Spark run: | cd $GITHUB_WORKSPACE/ && \ - ./build/mvn clean install -Pspark-3.3 -Pbackends-velox -Pceleborn -Puniffle -DskipTests -Dmaven.source.skip ./build/mvn clean install -Pspark-3.4 -Pbackends-velox -Pceleborn -Puniffle -DskipTests -Dmaven.source.skip ./build/mvn clean install -Pspark-3.5 -Pbackends-velox -Pceleborn -Puniffle -DskipTests -Dmaven.source.skip - name: Upload bundle package diff --git a/AGENTS.md b/AGENTS.md index 8d71c4acad2..711103f1439 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,7 +31,7 @@ export PROMPT_ALWAYS_RESPOND=y # ClickHouse backend — build native first, then Maven. bash ./ep/build-clickhouse/src/build-clickhouse.sh -./build/mvn clean package -Pbackends-clickhouse -Pspark-3.3 -DskipTests +./build/mvn clean package -Pbackends-clickhouse -Pspark-3.5 -DskipTests ``` For incremental builds, cloud-FS flags, the Spark/Scala/JDK matrix, table-format diff --git a/LICENSE b/LICENSE index f2222e957c2..b42b619fdee 100644 --- a/LICENSE +++ b/LICENSE @@ -212,21 +212,9 @@ ./backends-velox/src/main/scala/org/apache/spark/sql/execution/SparkWriteFilesCommitProtocol.scala ./cpp-ch/local-engine/Parser/aggregate_function_parser/BloomFilterAggParser.cpp ./gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenExplainUtils.scala - ./shims/spark33/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala - ./shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/WriteFiles.scala - ./shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala - ./shims/spark33/src/main/scala/org/apache/spark/sql/execution/stat/StatFunctions.scala ./tools/gluten-it/common/src/main/scala/org/apache/spark/sql/TestUtils.scala Delta Lake(https://github.com/delta-io/delta) - ./backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/DeltaLog.scala - ./backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/Snapshot.scala - ./backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala - ./backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala - ./backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala - ./backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala - ./backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala - ./backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/stats/PrepareDeltaScan.scala ./backends-clickhouse/src-delta33/main/scala/org/apache/spark/sql/delta/DeltaLog.scala ./backends-clickhouse/src-delta33/main/scala/org/apache/spark/sql/delta/PreprocessTableWithDVs.scala ./backends-clickhouse/src-delta33/main/scala/org/apache/spark/sql/delta/Snapshot.scala diff --git a/README.md b/README.md index b20be1ce375..894bb581713 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Gluten's key components: * **Columnar Shuffle**: Handles shuffling of Gluten's columnar data. The shuffle service of Spark core is reused, while a columnar exchange operator is implemented to support Gluten's columnar data format. * **Fallback Mechanism**: Provides fallback to vanilla Spark for unsupported operators. Gluten's ColumnarToRow (C2R) and RowToColumnar (R2C) convert data between Gluten's columnar format and Spark's internal row format to support fallback transitions. * **Metrics**: Collected from Gluten native engine to help monitor execution, identify bugs, and diagnose performance bottlenecks. The metrics are displayed in Spark UI. -* **Shim Layer**: Ensures compatibility with multiple Spark versions. Gluten supports the latest 3–4 Spark releases during its development cycle, and currently supports Spark 3.3, 3.4, 3.5, 4.0, and 4.1. +* **Shim Layer**: Ensures compatibility with multiple Spark versions. Gluten supports the latest 3–4 Spark releases during its development cycle, and currently supports Spark 3.4, 3.5, 4.0, and 4.1. ## 3. User Guide diff --git a/backends-clickhouse/pom.xml b/backends-clickhouse/pom.xml index 816aeab6f87..0a64f1ac5fb 100644 --- a/backends-clickhouse/pom.xml +++ b/backends-clickhouse/pom.xml @@ -526,12 +526,6 @@ - - spark-3.3 - - false - - spark-3.5 diff --git a/backends-clickhouse/src-delta23/main/resources/META-INF/services/org.apache.gluten.sql.shims.DeltaShimProvider b/backends-clickhouse/src-delta23/main/resources/META-INF/services/org.apache.gluten.sql.shims.DeltaShimProvider deleted file mode 100644 index c512ce8a715..00000000000 --- a/backends-clickhouse/src-delta23/main/resources/META-INF/services/org.apache.gluten.sql.shims.DeltaShimProvider +++ /dev/null @@ -1 +0,0 @@ -org.apache.gluten.sql.shims.delta23.Delta23ShimProvider \ No newline at end of file diff --git a/backends-clickhouse/src-delta23/main/scala/io/delta/tables/ClickhouseTable.scala b/backends-clickhouse/src-delta23/main/scala/io/delta/tables/ClickhouseTable.scala deleted file mode 100644 index b5ba0cb44e5..00000000000 --- a/backends-clickhouse/src-delta23/main/scala/io/delta/tables/ClickhouseTable.scala +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.delta.tables - -import org.apache.spark.sql.{Dataset, Row, SparkSession} -import org.apache.spark.sql.delta.{DeltaErrors, DeltaTableIdentifier, DeltaTableUtils} -import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 - -import org.apache.hadoop.fs.Path - -import scala.collection.JavaConverters._ - -class ClickhouseTable( - @transient private val _df: Dataset[Row], - @transient private val table: ClickHouseTableV2) - extends DeltaTable(_df, table) { - - override def optimize(): DeltaOptimizeBuilder = { - DeltaOptimizeBuilder( - sparkSession, - table.tableIdentifier.getOrElse(s"clickhouse.`${deltaLog.dataPath.toString}`"), - table.options) - } -} - -object ClickhouseTable { - - /** - * Instantiate a [[DeltaTable]] object representing the data at the given path, If the given path - * is invalid (i.e. either no table exists or an existing table is not a Delta table), it throws a - * `not a Delta table` error. - * - * Note: This uses the active SparkSession in the current thread to read the table data. Hence, - * this throws error if active SparkSession has not been set, that is, - * `SparkSession.getActiveSession()` is empty. - * - * @since 0.3.0 - */ - def forPath(path: String): DeltaTable = { - val sparkSession = SparkSession.getActiveSession.getOrElse { - throw DeltaErrors.activeSparkSessionNotFound() - } - forPath(sparkSession, path) - } - - /** - * Instantiate a [[DeltaTable]] object representing the data at the given path, If the given path - * is invalid (i.e. either no table exists or an existing table is not a Delta table), it throws a - * `not a Delta table` error. - * - * @since 0.3.0 - */ - def forPath(sparkSession: SparkSession, path: String): DeltaTable = { - forPath(sparkSession, path, Map.empty[String, String]) - } - - /** - * Instantiate a [[DeltaTable]] object representing the data at the given path, If the given path - * is invalid (i.e. either no table exists or an existing table is not a Delta table), it throws a - * `not a Delta table` error. - * - * @param hadoopConf - * Hadoop configuration starting with "fs." or "dfs." will be picked up by `DeltaTable` to - * access the file system when executing queries. Other configurations will not be allowed. - * - * {{{ - * val hadoopConf = Map( - * "fs.s3a.access.key" -> "", - * "fs.s3a.secret.key" -> "" - * ) - * DeltaTable.forPath(spark, "/path/to/table", hadoopConf) - * }}} - * @since 2.2.0 - */ - def forPath( - sparkSession: SparkSession, - path: String, - hadoopConf: scala.collection.Map[String, String]): DeltaTable = { - // We only pass hadoopConf so that we won't pass any unsafe options to Delta. - val badOptions = hadoopConf.filterKeys { - k => !DeltaTableUtils.validDeltaTableHadoopPrefixes.exists(k.startsWith) - }.toMap - if (!badOptions.isEmpty) { - throw DeltaErrors.unsupportedDeltaTableForPathHadoopConf(badOptions) - } - val fileSystemOptions: Map[String, String] = hadoopConf.toMap - val hdpPath = new Path(path) - if (DeltaTableUtils.isDeltaTable(sparkSession, hdpPath, fileSystemOptions)) { - new ClickhouseTable( - sparkSession.read.format("clickhouse").options(fileSystemOptions).load(path), - new ClickHouseTableV2(spark = sparkSession, path = hdpPath, options = fileSystemOptions) - ) - } else { - throw DeltaErrors.notADeltaTableException(DeltaTableIdentifier(path = Some(path))) - } - } - - /** - * Java friendly API to instantiate a [[DeltaTable]] object representing the data at the given - * path, If the given path is invalid (i.e. either no table exists or an existing table is not a - * Delta table), it throws a `not a Delta table` error. - * - * @param hadoopConf - * Hadoop configuration starting with "fs." or "dfs." will be picked up by `DeltaTable` to - * access the file system when executing queries. Other configurations will be ignored. - * - * {{{ - * val hadoopConf = Map( - * "fs.s3a.access.key" -> "", - * "fs.s3a.secret.key", "" - * ) - * DeltaTable.forPath(spark, "/path/to/table", hadoopConf) - * }}} - * @since 2.2.0 - */ - def forPath( - sparkSession: SparkSession, - path: String, - hadoopConf: java.util.Map[String, String]): DeltaTable = { - val fsOptions = hadoopConf.asScala.toMap - forPath(sparkSession, path, fsOptions) - } -} diff --git a/backends-clickhouse/src-delta23/main/scala/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala b/backends-clickhouse/src-delta23/main/scala/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala deleted file mode 100644 index 9a0cde77284..00000000000 --- a/backends-clickhouse/src-delta23/main/scala/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.parser - -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} -import org.apache.spark.sql.catalyst.expressions.Expression -import org.apache.spark.sql.catalyst.parser.ParserInterface -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.types.{DataType, StructType} - -class GlutenCacheFilesSqlParser(spark: SparkSession, delegate: ParserInterface) - extends GlutenCacheFileSqlParserBase { - - override def parsePlan(sqlText: String): LogicalPlan = - parse(sqlText) { - parser => - astBuilder.visit(parser.singleStatement()) match { - case plan: LogicalPlan => plan - case _ => delegate.parsePlan(sqlText) - } - } - - override def parseExpression(sqlText: String): Expression = { - delegate.parseExpression(sqlText) - } - - override def parseTableIdentifier(sqlText: String): TableIdentifier = { - delegate.parseTableIdentifier(sqlText) - } - - override def parseFunctionIdentifier(sqlText: String): FunctionIdentifier = { - delegate.parseFunctionIdentifier(sqlText) - } - - override def parseMultipartIdentifier(sqlText: String): Seq[String] = { - delegate.parseMultipartIdentifier(sqlText) - } - - override def parseTableSchema(sqlText: String): StructType = { - delegate.parseTableSchema(sqlText) - } - - override def parseDataType(sqlText: String): DataType = { - delegate.parseDataType(sqlText) - } - - override def parseQuery(sqlText: String): LogicalPlan = { - delegate.parseQuery(sqlText) - } -} diff --git a/backends-clickhouse/src-delta23/main/scala/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala b/backends-clickhouse/src-delta23/main/scala/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala deleted file mode 100644 index 1f2dfe00767..00000000000 --- a/backends-clickhouse/src-delta23/main/scala/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.parser - -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} -import org.apache.spark.sql.catalyst.expressions.Expression -import org.apache.spark.sql.catalyst.parser.ParserInterface -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.types.{DataType, StructType} - -class GlutenClickhouseSqlParser(spark: SparkSession, delegate: ParserInterface) - extends GlutenClickhouseSqlParserBase { - - override def parsePlan(sqlText: String): LogicalPlan = - parse(sqlText) { - parser => - astBuilder.visit(parser.singleStatement()) match { - case plan: LogicalPlan => plan - case _ => delegate.parsePlan(sqlText) - } - } - - override def parseExpression(sqlText: String): Expression = { - delegate.parseExpression(sqlText) - } - - override def parseTableIdentifier(sqlText: String): TableIdentifier = { - delegate.parseTableIdentifier(sqlText) - } - - override def parseFunctionIdentifier(sqlText: String): FunctionIdentifier = { - delegate.parseFunctionIdentifier(sqlText) - } - - override def parseMultipartIdentifier(sqlText: String): Seq[String] = { - delegate.parseMultipartIdentifier(sqlText) - } - - override def parseTableSchema(sqlText: String): StructType = { - delegate.parseTableSchema(sqlText) - } - - override def parseDataType(sqlText: String): DataType = { - delegate.parseDataType(sqlText) - } - - override def parseQuery(sqlText: String): LogicalPlan = { - delegate.parseQuery(sqlText) - } -} diff --git a/backends-clickhouse/src-delta23/main/scala/org/apache/gluten/sql/shims/delta23/Delta23ShimProvider.scala b/backends-clickhouse/src-delta23/main/scala/org/apache/gluten/sql/shims/delta23/Delta23ShimProvider.scala deleted file mode 100644 index 854a996999a..00000000000 --- a/backends-clickhouse/src-delta23/main/scala/org/apache/gluten/sql/shims/delta23/Delta23ShimProvider.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.sql.shims.delta23 - -import org.apache.gluten.sql.shims.{DeltaShimProvider, DeltaShims} - -class Delta23ShimProvider extends DeltaShimProvider { - - override def createShim: DeltaShims = { - new Delta23Shims() - } - -} diff --git a/backends-clickhouse/src-delta23/main/scala/org/apache/gluten/sql/shims/delta23/Delta23Shims.scala b/backends-clickhouse/src-delta23/main/scala/org/apache/gluten/sql/shims/delta23/Delta23Shims.scala deleted file mode 100644 index 3528363a6e6..00000000000 --- a/backends-clickhouse/src-delta23/main/scala/org/apache/gluten/sql/shims/delta23/Delta23Shims.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.sql.shims.delta23 - -import org.apache.gluten.sql.shims.DeltaShims - -class Delta23Shims extends DeltaShims {} diff --git a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala b/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala deleted file mode 100644 index 2095a1e7e5b..00000000000 --- a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.delta - -import org.apache.gluten.backendsapi.clickhouse.CHConfig - -import org.apache.spark.SparkException -import org.apache.spark.sql.Dataset -import org.apache.spark.sql.delta.actions._ -import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 -import org.apache.spark.sql.delta.constraints.{Constraint, Constraints} -import org.apache.spark.sql.delta.files.MergeTreeDelayedCommitProtocol -import org.apache.spark.sql.delta.schema.InvariantViolationException -import org.apache.spark.sql.delta.sources.DeltaSQLConf -import org.apache.spark.sql.execution.SQLExecution -import org.apache.spark.sql.execution.datasources.{BasicWriteJobStatsTracker, FileFormatWriter, GlutenWriterColumnarRules, WriteJobStatsTracker} -import org.apache.spark.sql.execution.datasources.v1.MergeTreeWriterInjects -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig -import org.apache.spark.util.{Clock, SerializableConfiguration} - -import org.apache.commons.lang3.exception.ExceptionUtils - -import scala.collection.mutable.ListBuffer - -class ClickhouseOptimisticTransaction( - override val deltaLog: DeltaLog, - override val snapshot: Snapshot)(implicit override val clock: Clock) - extends OptimisticTransaction(deltaLog, snapshot) { - - def this(deltaLog: DeltaLog, snapshotOpt: Option[Snapshot] = None)(implicit clock: Clock) { - this( - deltaLog, - snapshotOpt.getOrElse(deltaLog.update()) - ) - } - - override def writeFiles( - inputData: Dataset[_], - writeOptions: Option[DeltaOptions], - additionalConstraints: Seq[Constraint]): Seq[FileAction] = { - if (ClickHouseConfig.isMergeTreeFormatEngine(metadata.configuration)) { - hasWritten = true - - val spark = inputData.sparkSession - val (data, partitionSchema) = performCDCPartition(inputData) - val outputPath = deltaLog.dataPath - - val (queryExecution, output, generatedColumnConstraints, _) = - normalizeData(deltaLog, data) - - val tableV2 = ClickHouseTableV2.getTable(deltaLog) - val committer = - new MergeTreeDelayedCommitProtocol( - outputPath.toString, - None, - tableV2.dataBaseName, - tableV2.tableName) - - // val (optionalStatsTracker, _) = - // getOptionalStatsTrackerAndStatsCollection(output, outputPath, partitionSchema, data) - val (optionalStatsTracker, _) = (None, None) - - val constraints = - Constraints.getAll(metadata, spark) ++ generatedColumnConstraints ++ additionalConstraints - - SQLExecution.withNewExecutionId(queryExecution, Option("deltaTransactionalWrite")) { - val queryPlan = queryExecution.executedPlan - val (newQueryPlan, newOutput) = - MergeTreeWriterInjects.insertFakeRowAdaptor(queryPlan, output) - val outputSpec = FileFormatWriter.OutputSpec(outputPath.toString, Map.empty, newOutput) - val partitioningColumns = getPartitioningColumns(partitionSchema, newOutput) - - val statsTrackers: ListBuffer[WriteJobStatsTracker] = ListBuffer() - - if (spark.conf.get(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED)) { - val basicWriteJobStatsTracker = new BasicWriteJobStatsTracker( - new SerializableConfiguration(deltaLog.newDeltaHadoopConf()), - BasicWriteJobStatsTracker.metrics) - // registerSQLMetrics(spark, basicWriteJobStatsTracker.driverSideMetrics) - statsTrackers.append(basicWriteJobStatsTracker) - } - - // Retain only a minimal selection of Spark writer options to avoid any potential - // compatibility issues - var options = writeOptions match { - case None => Map.empty[String, String] - case Some(writeOptions) => - writeOptions.options - .filterKeys { - key => - key.equalsIgnoreCase(DeltaOptions.MAX_RECORDS_PER_FILE) || - key.equalsIgnoreCase(DeltaOptions.COMPRESSION) - } - .map(identity) - } - - spark.conf.getAll.foreach( - entry => { - if ( - CHConfig.startWithSettingsPrefix(entry._1) - || entry._1.equalsIgnoreCase(DeltaSQLConf.DELTA_OPTIMIZE_MIN_FILE_SIZE.key) - ) { - options += (entry._1 -> entry._2) - } - }) - - try { - val format = tableV2.getFileFormat(metadata) - GlutenWriterColumnarRules.injectSparkLocalProperty(spark, Some(format.shortName()), None) - FileFormatWriter.write( - sparkSession = spark, - plan = newQueryPlan, - fileFormat = format, - // formats. - committer = committer, - outputSpec = outputSpec, - // scalastyle:off deltahadoopconfiguration - hadoopConf = spark.sessionState - .newHadoopConfWithOptions(metadata.configuration ++ deltaLog.options), - // scalastyle:on deltahadoopconfiguration - partitionColumns = partitioningColumns, - bucketSpec = - tableV2.normalizedBucketSpec(output.map(_.name), spark.sessionState.conf.resolver), - statsTrackers = optionalStatsTracker.toSeq ++ statsTrackers, - options = options - ) - } catch { - case s: SparkException => - // Pull an InvariantViolationException up to the top level if it was the root cause. - val violationException = ExceptionUtils.getRootCause(s) - if (violationException.isInstanceOf[InvariantViolationException]) { - throw violationException - } else { - throw s - } - } finally { - GlutenWriterColumnarRules.injectSparkLocalProperty(spark, None, None) - } - } - committer.addedStatuses.toSeq ++ committer.changeFiles - } else { - // TODO: support native delta parquet write - // 1. insert FakeRowAdaptor - // 2. DeltaInvariantCheckerExec transform - // 3. DeltaTaskStatisticsTracker collect null count / min values / max values - // 4. set the parameters 'staticPartitionWriteOnly', 'isNativeApplicable', - // 'nativeFormat' in the LocalProperty of the sparkcontext - super.writeFiles(inputData, writeOptions, additionalConstraints) - } - } -} diff --git a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/DeltaAdapter.scala b/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/DeltaAdapter.scala deleted file mode 100644 index f414ab8f285..00000000000 --- a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/DeltaAdapter.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.delta - -import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} -import org.apache.spark.sql.delta.stats.DeltaScan - -object DeltaAdapter extends DeltaAdapterTrait { - override def snapshot(deltaLog: DeltaLog): Snapshot = deltaLog.unsafeVolatileSnapshot - - override def snapshotFilesForScan( - snapshot: Snapshot, - projection: Seq[Attribute], - filters: Seq[Expression], - keepNumRecords: Boolean): DeltaScan = { - snapshot.filesForScan(filters, keepNumRecords) - } -} diff --git a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/DeltaLog.scala b/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/DeltaLog.scala deleted file mode 100644 index 78fbc3fcdb9..00000000000 --- a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/DeltaLog.scala +++ /dev/null @@ -1,1043 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.delta - -// scalastyle:off import.ordering.noEmptyLine -import java.io.File -import java.lang.ref.WeakReference -import java.net.URI -import java.util.concurrent.TimeUnit -import java.util.concurrent.locks.ReentrantLock - -import scala.collection.JavaConverters._ -import scala.collection.mutable -import scala.util.Try -import scala.util.control.NonFatal - -import com.databricks.spark.util.TagDefinitions._ -import org.apache.spark.sql.delta.actions._ -import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 -import org.apache.spark.sql.delta.commands.WriteIntoDelta -import org.apache.spark.sql.delta.commands.cdc.CDCReader -import org.apache.spark.sql.delta.files.{TahoeBatchFileIndex, TahoeLogFileIndex} -import org.apache.spark.sql.delta.metering.DeltaLogging -import org.apache.spark.sql.delta.schema.{SchemaMergingUtils, SchemaUtils} -import org.apache.spark.sql.delta.sources._ -import org.apache.spark.sql.delta.storage.LogStoreProvider -import com.google.common.cache.{CacheBuilder, RemovalNotification} -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.{FileStatus, FileSystem, Path} - -import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.analysis.{Resolver, UnresolvedAttribute} -import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTable} -import org.apache.spark.sql.catalyst.expressions.{And, Attribute, Cast, Expression, Literal} -import org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper -import org.apache.spark.sql.catalyst.util.FailFastMode -import org.apache.spark.sql.execution.datasources._ -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig -import org.apache.spark.sql.expressions.UserDefinedFunction -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.sources.{BaseRelation, InsertableRelation} -import org.apache.spark.sql.types.{StructField, StructType} -import org.apache.spark.sql.util.CaseInsensitiveStringMap -import org.apache.spark.util._ - -/** - * Gluten overwrite Delta: - * - * This file is copied from Delta 2.3.0, it is modified to overcome the following issues: - * 1. return ClickhouseOptimisticTransaction - * 2. return DeltaMergeTreeFileFormat - * 3. create HadoopFsRelation with the bucket options - */ -/** - * Used to query the current state of the log as well as modify it by adding - * new atomic collections of actions. - * - * Internally, this class implements an optimistic concurrency control - * algorithm to handle multiple readers or writers. Any single read - * is guaranteed to see a consistent snapshot of the table. - * - * @param logPath Path of the Delta log JSONs. - * @param dataPath Path of the data files. - * @param options Filesystem options filtered from `allOptions`. - * @param allOptions All options provided by the user, for example via `df.write.option()`. This - * includes but not limited to filesystem and table properties. - * @param clock Clock to be used when starting a new transaction. - */ -class DeltaLog private( - val logPath: Path, - val dataPath: Path, - val options: Map[String, String], - val allOptions: Map[String, String], - val clock: Clock - ) extends Checkpoints - with MetadataCleanup - with LogStoreProvider - with SnapshotManagement - with DeltaFileFormat - with ReadChecksum { - - import org.apache.spark.sql.delta.util.FileNames._ - - - private lazy implicit val _clock = clock - - protected def spark = SparkSession.active - - checkRequiredConfigurations() - - /** - * Keep a reference to `SparkContext` used to create `DeltaLog`. `DeltaLog` cannot be used when - * `SparkContext` is stopped. We keep the reference so that we can check whether the cache is - * still valid and drop invalid `DeltaLog`` objects. - */ - private val sparkContext = new WeakReference(spark.sparkContext) - - /** - * Returns the Hadoop [[Configuration]] object which can be used to access the file system. All - * Delta code should use this method to create the Hadoop [[Configuration]] object, so that the - * hadoop file system configurations specified in DataFrame options will come into effect. - */ - // scalastyle:off deltahadoopconfiguration - final def newDeltaHadoopConf(): Configuration = - spark.sessionState.newHadoopConfWithOptions(options) - // scalastyle:on deltahadoopconfiguration - - /** Used to read and write physical log files and checkpoints. */ - lazy val store = createLogStore(spark) - - /** Use ReentrantLock to allow us to call `lockInterruptibly` */ - protected val deltaLogLock = new ReentrantLock() - - /** Delta History Manager containing version and commit history. */ - lazy val history = new DeltaHistoryManager( - this, spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_HISTORY_PAR_SEARCH_THRESHOLD)) - - /* --------------- * - | Configuration | - * --------------- */ - - /** - * The max lineage length of a Snapshot before Delta forces to build a Snapshot from scratch. - * Delta will build a Snapshot on top of the previous one if it doesn't see a checkpoint. - * However, there is a race condition that when two writers are writing at the same time, - * a writer may fail to pick up checkpoints written by another one, and the lineage will grow - * and finally cause StackOverflowError. Hence we have to force to build a Snapshot from scratch - * when the lineage length is too large to avoid hitting StackOverflowError. - */ - def maxSnapshotLineageLength: Int = - spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_MAX_SNAPSHOT_LINEAGE_LENGTH) - - /** The unique identifier for this table. */ - def tableId: String = unsafeVolatileMetadata.id // safe because table id never changes - - /** - * Combines the tableId with the path of the table to ensure uniqueness. Normally `tableId` - * should be globally unique, but nothing stops users from copying a Delta table directly to - * a separate location, where the transaction log is copied directly, causing the tableIds to - * match. When users mutate the copied table, and then try to perform some checks joining the - * two tables, optimizations that depend on `tableId` alone may not be correct. Hence we use a - * composite id. - */ - private[delta] def compositeId: (String, Path) = tableId -> dataPath - - /** - * Run `body` inside `deltaLogLock` lock using `lockInterruptibly` so that the thread can be - * interrupted when waiting for the lock. - */ - def lockInterruptibly[T](body: => T): T = { - deltaLogLock.lockInterruptibly() - try { - body - } finally { - deltaLogLock.unlock() - } - } - - /** - * Creates a [[LogicalRelation]] for a given [[DeltaLogFileIndex]], with all necessary file source - * options taken from the Delta Log. All reads of Delta metadata files should use this method. - */ - def indexToRelation( - index: DeltaLogFileIndex, - schema: StructType = Action.logSchema): LogicalRelation = { - val formatSpecificOptions: Map[String, String] = index.format match { - case DeltaLogFileIndex.COMMIT_FILE_FORMAT => - DeltaLog.jsonCommitParseOption - case _ => Map.empty - } - // Delta should NEVER ignore missing or corrupt metadata files, because doing so can render the - // entire table unusable. Hard-wire that into the file source options so the user can't override - // it by setting spark.sql.files.ignoreCorruptFiles or spark.sql.files.ignoreMissingFiles. - // - // NOTE: This should ideally be [[FileSourceOptions.IGNORE_CORRUPT_FILES]] etc., but those - // constants are only available since spark-3.4. By hard-coding the values here instead, we - // preserve backward compatibility when compiling Delta against older spark versions (tho - // obviously the desired protection would be missing in that case). - val allOptions = options ++ formatSpecificOptions ++ Map( - "ignoreCorruptFiles" -> "false", - "ignoreMissingFiles" -> "false" - ) - // --- modified start - // Don't need to add the bucketOption here, it handles the delta log meta json file - // --- modified end - val fsRelation = HadoopFsRelation( - index, index.partitionSchema, schema, None, index.format, allOptions)(spark) - LogicalRelation(fsRelation) - } - - /** - * Load the data using the FileIndex. This allows us to skip many checks that add overhead, e.g. - * file existence checks, partitioning schema inference. - */ - def loadIndex( - index: DeltaLogFileIndex, - schema: StructType = Action.logSchema): DataFrame = { - Dataset.ofRows(spark, indexToRelation(index, schema)) - } - - /* ------------------ * - | Delta Management | - * ------------------ */ - - /** - * Returns a new [[OptimisticTransaction]] that can be used to read the current state of the - * log and then commit updates. The reads and updates will be checked for logical conflicts - * with any concurrent writes to the log. - * - * Note that all reads in a transaction must go through the returned transaction object, and not - * directly to the [[DeltaLog]] otherwise they will not be checked for conflicts. - */ - def startTransaction(): OptimisticTransaction = startTransaction(None) - - def startTransaction(snapshotOpt: Option[Snapshot]): OptimisticTransaction = { - // --- modified start - new ClickhouseOptimisticTransaction(this, snapshotOpt) - // --- modified end - } - - /** - * Execute a piece of code within a new [[OptimisticTransaction]]. Reads/write sets will - * be recorded for this table, and all other tables will be read - * at a snapshot that is pinned on the first access. - * - * @note This uses thread-local variable to make the active transaction visible. So do not use - * multi-threaded code in the provided thunk. - */ - def withNewTransaction[T](thunk: OptimisticTransaction => T): T = { - try { - val txn = startTransaction() - OptimisticTransaction.setActive(txn) - thunk(txn) - } finally { - OptimisticTransaction.clearActive() - } - } - - - /** - * Upgrade the table's protocol version, by default to the maximum recognized reader and writer - * versions in this DBR release. - */ - def upgradeProtocol( - snapshot: Snapshot, - newVersion: Protocol): Unit = { - val currentVersion = snapshot.protocol - if (newVersion == currentVersion) { - logConsole(s"Table $dataPath is already at protocol version $newVersion.") - return - } - - val txn = startTransaction(Some(snapshot)) - try { - SchemaMergingUtils.checkColumnNameDuplication(txn.metadata.schema, "in the table schema") - } catch { - case e: AnalysisException => - throw DeltaErrors.duplicateColumnsOnUpdateTable(e) - } - txn.commit(Seq(newVersion), DeltaOperations.UpgradeProtocol(newVersion)) - logConsole(s"Upgraded table at $dataPath to $newVersion.") - } - - // Test-only!! - private[delta] def upgradeProtocol(newVersion: Protocol): Unit = { - upgradeProtocol(unsafeVolatileSnapshot, newVersion) - } - - /** - * Get all actions starting from "startVersion" (inclusive). If `startVersion` doesn't exist, - * return an empty Iterator. - */ - def getChanges( - startVersion: Long, - failOnDataLoss: Boolean = false): Iterator[(Long, Seq[Action])] = { - val hadoopConf = newDeltaHadoopConf() - val deltas = store.listFrom(listingPrefix(logPath, startVersion), hadoopConf) - .filter(isDeltaFile) - // Subtract 1 to ensure that we have the same check for the inclusive startVersion - var lastSeenVersion = startVersion - 1 - deltas.map { status => - val p = status.getPath - val version = deltaVersion(p) - if (failOnDataLoss && version > lastSeenVersion + 1) { - throw DeltaErrors.failOnDataLossException(lastSeenVersion + 1, version) - } - lastSeenVersion = version - (version, store.read(status, hadoopConf).map(Action.fromJson)) - } - } - - /** - * Get access to all actions starting from "startVersion" (inclusive) via [[FileStatus]]. - * If `startVersion` doesn't exist, return an empty Iterator. - */ - def getChangeLogFiles( - startVersion: Long, - failOnDataLoss: Boolean = false): Iterator[(Long, FileStatus)] = { - val deltas = store.listFrom(listingPrefix(logPath, startVersion), newDeltaHadoopConf()) - .filter(isDeltaFile) - // Subtract 1 to ensure that we have the same check for the inclusive startVersion - var lastSeenVersion = startVersion - 1 - deltas.map { status => - val version = deltaVersion(status) - if (failOnDataLoss && version > lastSeenVersion + 1) { - throw DeltaErrors.failOnDataLossException(lastSeenVersion + 1, version) - } - lastSeenVersion = version - (version, status) - } - } - - /* --------------------- * - | Protocol validation | - * --------------------- */ - - /** - * Asserts the highest protocol supported by this client is not less than what required by the - * table for performing read or write operations. This ensures the client to support a - * greater-or-equal protocol versions and recognizes/supports all features enabled by the table. - * - * The operation type to be checked is passed as a string in `readOrWrite`. Valid values are - * `read` and `write`. - */ - private def protocolCheck(tableProtocol: Protocol, readOrWrite: String): Unit = { - val clientSupportedProtocol = Action.supportedProtocolVersion() - // Depending on the operation, pull related protocol versions out of Protocol objects. - // `getEnabledFeatures` is a pointer to pull reader/writer features out of a Protocol. - val (clientSupportedVersion, tableRequiredVersion, getEnabledFeatures) = readOrWrite match { - case "read" => ( - clientSupportedProtocol.minReaderVersion, - tableProtocol.minReaderVersion, - (f: Protocol) => f.readerFeatureNames) - case "write" => ( - clientSupportedProtocol.minWriterVersion, - tableProtocol.minWriterVersion, - (f: Protocol) => f.writerFeatureNames) - case _ => - throw new IllegalArgumentException("Table operation must be either `read` or `write`.") - } - - // Check is complete when both the protocol version and all referenced features are supported. - val clientSupportedFeatureNames = getEnabledFeatures(clientSupportedProtocol) - val tableEnabledFeatureNames = getEnabledFeatures(tableProtocol) - if (tableEnabledFeatureNames.subsetOf(clientSupportedFeatureNames) && - clientSupportedVersion >= tableRequiredVersion) { - return - } - - // Otherwise, either the protocol version, or few features referenced by the table, is - // unsupported. - val clientUnsupportedFeatureNames = - tableEnabledFeatureNames.diff(clientSupportedFeatureNames) - // Prepare event log constants and the appropriate error message handler. - val (opType, versionKey, unsupportedFeaturesException) = readOrWrite match { - case "read" => ( - "delta.protocol.failure.read", - "minReaderVersion", - DeltaErrors.unsupportedReaderTableFeaturesInTableException _) - case "write" => ( - "delta.protocol.failure.write", - "minWriterVersion", - DeltaErrors.unsupportedWriterTableFeaturesInTableException _) - } - recordDeltaEvent( - this, - opType, - data = Map( - "clientVersion" -> clientSupportedVersion, - versionKey -> tableRequiredVersion, - "clientFeatures" -> clientSupportedFeatureNames.mkString(","), - "clientUnsupportedFeatures" -> clientUnsupportedFeatureNames.mkString(","))) - if (clientSupportedVersion < tableRequiredVersion) { - throw new InvalidProtocolVersionException(tableRequiredVersion, clientSupportedVersion) - } else { - throw unsupportedFeaturesException(clientUnsupportedFeatureNames) - } - } - - /** - * Asserts that the table's protocol enabled all features that are active in the metadata. - * - * A mismatch shouldn't happen when the table has gone through a proper write process because we - * require all active features during writes. However, other clients may void this guarantee. - */ - def assertTableFeaturesMatchMetadata( - targetProtocol: Protocol, - targetMetadata: Metadata): Unit = { - if (!targetProtocol.supportsReaderFeatures && !targetProtocol.supportsWriterFeatures) return - - val protocolEnabledFeatures = targetProtocol.writerFeatureNames - .flatMap(TableFeature.featureNameToFeature) - val activeFeatures: Set[TableFeature] = - TableFeature.allSupportedFeaturesMap.values.collect { - case f: TableFeature with FeatureAutomaticallyEnabledByMetadata - if f.metadataRequiresFeatureToBeEnabled(targetMetadata, spark) => - f - }.toSet - val activeButNotEnabled = activeFeatures.diff(protocolEnabledFeatures) - if (activeButNotEnabled.nonEmpty) { - throw DeltaErrors.tableFeatureMismatchException(activeButNotEnabled.map(_.name)) - } - } - - /** - * Asserts that the client is up to date with the protocol and allowed to read the table that is - * using the given `protocol`. - */ - def protocolRead(protocol: Protocol): Unit = { - protocolCheck(protocol, "read") - } - - /** - * Asserts that the client is up to date with the protocol and allowed to write to the table - * that is using the given `protocol`. - */ - def protocolWrite(protocol: Protocol): Unit = { - protocolCheck(protocol, "write") - } - - /* ---------------------------------------- * - | Log Directory Management and Retention | - * ---------------------------------------- */ - - /** - * Whether a Delta table exists at this directory. - * It is okay to use the cached volatile snapshot here, since the worst case is that the table - * has recently started existing which hasn't been picked up here. If so, any subsequent command - * that updates the table will see the right value. - */ - def tableExists: Boolean = unsafeVolatileSnapshot.version >= 0 - - def isSameLogAs(otherLog: DeltaLog): Boolean = this.compositeId == otherLog.compositeId - - /** Creates the log directory if it does not exist. */ - def ensureLogDirectoryExist(): Unit = { - val fs = logPath.getFileSystem(newDeltaHadoopConf()) - if (!fs.exists(logPath)) { - if (!fs.mkdirs(logPath)) { - throw DeltaErrors.cannotCreateLogPathException(logPath.toString) - } - } - } - - /** - * Create the log directory. Unlike `ensureLogDirectoryExist`, this method doesn't check whether - * the log directory exists and it will ignore the return value of `mkdirs`. - */ - def createLogDirectory(): Unit = { - logPath.getFileSystem(newDeltaHadoopConf()).mkdirs(logPath) - } - - /* ------------ * - | Integration | - * ------------ */ - - /** - * Returns a [[org.apache.spark.sql.DataFrame]] containing the new files within the specified - * version range. - * - */ - def createDataFrame( - snapshot: Snapshot, - addFiles: Seq[AddFile], - isStreaming: Boolean = false, - actionTypeOpt: Option[String] = None - ): DataFrame = { - val actionType = actionTypeOpt.getOrElse(if (isStreaming) "streaming" else "batch") - val fileIndex = new TahoeBatchFileIndex(spark, actionType, addFiles, this, dataPath, snapshot) - - val hadoopOptions = snapshot.metadata.format.options ++ options - val partitionSchema = snapshot.metadata.partitionSchema - val metadata = snapshot.metadata - - - val relation = HadoopFsRelation( - fileIndex, - partitionSchema = DeltaColumnMapping.dropColumnMappingMetadata(partitionSchema), - // We pass all table columns as `dataSchema` so that Spark will preserve the partition column - // locations. Otherwise, for any partition columns not in `dataSchema`, Spark would just - // append them to the end of `dataSchema`. - dataSchema = DeltaColumnMapping.dropColumnMappingMetadata( - ColumnWithDefaultExprUtils.removeDefaultExpressions(metadata.schema)), - // --- modified start - // TODO: Don't add the bucketOption here, it will cause the OOM when the merge into update - // key is the bucket column, fix later - // --- modified end - bucketSpec = None, - fileFormat(metadata), - hadoopOptions)(spark) - - Dataset.ofRows(spark, LogicalRelation(relation, isStreaming = isStreaming)) - } - - /** - * Returns a [[BaseRelation]] that contains all of the data present - * in the table. This relation will be continually updated - * as files are added or removed from the table. However, new [[BaseRelation]] - * must be requested in order to see changes to the schema. - */ - def createRelation( - partitionFilters: Seq[Expression] = Nil, - snapshotToUseOpt: Option[Snapshot] = None, - isTimeTravelQuery: Boolean = false, - cdcOptions: CaseInsensitiveStringMap = CaseInsensitiveStringMap.empty): BaseRelation = { - - /** Used to link the files present in the table into the query planner. */ - // TODO: If snapshotToUse is unspecified, get the correct snapshot from update() - val snapshotToUse = snapshotToUseOpt.getOrElse(unsafeVolatileSnapshot) - if (snapshotToUse.version < 0) { - // A negative version here means the dataPath is an empty directory. Read query should error - // out in this case. - throw DeltaErrors.pathNotExistsException(dataPath.toString) - } - - // For CDC we have to return the relation that represents the change data instead of actual - // data. - if (!cdcOptions.isEmpty) { - recordDeltaEvent(this, "delta.cdf.read", data = cdcOptions.asCaseSensitiveMap()) - return CDCReader.getCDCRelation( - spark, snapshotToUse, isTimeTravelQuery, spark.sessionState.conf, cdcOptions) - } - - val fileIndex = TahoeLogFileIndex( - spark, this, dataPath, snapshotToUse, partitionFilters, isTimeTravelQuery) - // --- modified start - var bucketSpec: Option[BucketSpec] = - if (ClickHouseConfig.isMergeTreeFormatEngine(snapshotToUse.metadata.configuration)) { - ClickHouseTableV2.getTable(this).bucketOption - } else { - None - } - - new DeltaLog.DeltaHadoopFsRelation( - fileIndex, - partitionSchema = DeltaColumnMapping.dropColumnMappingMetadata( - snapshotToUse.metadata.partitionSchema), - // We pass all table columns as `dataSchema` so that Spark will preserve the partition column - // locations. Otherwise, for any partition columns not in `dataSchema`, Spark would just - // append them to the end of `dataSchema` - dataSchema = DeltaColumnMapping.dropColumnMappingMetadata( - ColumnWithDefaultExprUtils.removeDefaultExpressions( - SchemaUtils.dropNullTypeColumns(snapshotToUse.metadata.schema))), - bucketSpec = bucketSpec, - fileFormat(snapshotToUse.metadata), - // `metadata.format.options` is not set today. Even if we support it in future, we shouldn't - // store any file system options since they may contain credentials. Hence, it will never - // conflict with `DeltaLog.options`. - snapshotToUse.metadata.format.options ++ options - )( - spark, - this - ) - // --- modified end - } - - /** - * Verify the required Spark conf for delta - * Throw `DeltaErrors.configureSparkSessionWithExtensionAndCatalog` exception if - * `spark.sql.catalog.spark_catalog` config is missing. We do not check for - * `spark.sql.extensions` because DeltaSparkSessionExtension can alternatively - * be activated using the `.withExtension()` API. This check can be disabled - * by setting DELTA_CHECK_REQUIRED_SPARK_CONF to false. - */ - protected def checkRequiredConfigurations(): Unit = { - if (spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_REQUIRED_SPARK_CONFS_CHECK)) { - if (spark.conf.getOption( - SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION.key).isEmpty) { - throw DeltaErrors.configureSparkSessionWithExtensionAndCatalog(None) - } - } - } - - /** - * Returns a proper path canonicalization function for the current Delta log. - * - * If `runsOnExecutors` is true, the returned method will use a broadcast Hadoop Configuration - * so that the method is suitable for execution on executors. Otherwise, the returned method - * will use a local Hadoop Configuration and the method can only be executed on the driver. - */ - private[delta] def getCanonicalPathFunction(runsOnExecutors: Boolean): String => String = { - val hadoopConf = newDeltaHadoopConf() - // Wrap `hadoopConf` with a method to delay the evaluation to run on executors. - val getHadoopConf = if (runsOnExecutors) { - val broadcastHadoopConf = - spark.sparkContext.broadcast(new SerializableConfiguration(hadoopConf)) - () => broadcastHadoopConf.value.value - } else { - () => hadoopConf - } - - new DeltaLog.CanonicalPathFunction(getHadoopConf) - } - - /** - * Returns a proper path canonicalization UDF for the current Delta log. - * - * If `runsOnExecutors` is true, the returned UDF will use a broadcast Hadoop Configuration. - * Otherwise, the returned UDF will use a local Hadoop Configuration and the UDF can - * only be executed on the driver. - */ - private[delta] def getCanonicalPathUdf(runsOnExecutors: Boolean = true): UserDefinedFunction = { - DeltaUDF.stringFromString(getCanonicalPathFunction(runsOnExecutors)) - } - - override def fileFormat(metadata: Metadata): FileFormat = { - // --- modified start - if (ClickHouseConfig.isMergeTreeFormatEngine(metadata.configuration)) { - ClickHouseTableV2.getTable(this).getFileFormat(metadata) - } else { - super.fileFormat(metadata) - } - // --- modified end - } -} - -object DeltaLog extends DeltaLogging { - - // --- modified start - @SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) - private class DeltaHadoopFsRelation( - location: FileIndex, - partitionSchema: StructType, - // The top-level columns in `dataSchema` should match the actual physical file schema, - // otherwise the ORC data source may not work with the by-ordinal mode. - dataSchema: StructType, - bucketSpec: Option[BucketSpec], - fileFormat: FileFormat, - options: Map[String, String])(sparkSession: SparkSession, deltaLog: DeltaLog) - extends HadoopFsRelation( - location, - partitionSchema, - dataSchema, - bucketSpec, - fileFormat, - options)(sparkSession) - with InsertableRelation { - def insert(data: DataFrame, overwrite: Boolean): Unit = { - val mode = if (overwrite) SaveMode.Overwrite else SaveMode.Append - WriteIntoDelta( - deltaLog = deltaLog, - mode = mode, - new DeltaOptions(Map.empty[String, String], sparkSession.sessionState.conf), - partitionColumns = Seq.empty, - configuration = Map.empty, - data = data - ).run(sparkSession) - } - } - // --- modified end - - /** - * The key type of `DeltaLog` cache. It's a pair of the canonicalized table path and the file - * system options (options starting with "fs." or "dfs." prefix) passed into - * `DataFrameReader/Writer` - */ - private type DeltaLogCacheKey = (Path, Map[String, String]) - - /** The name of the subdirectory that holds Delta metadata files */ - private val LOG_DIR_NAME = "_delta_log" - - private[delta] def logPathFor(dataPath: String): Path = new Path(dataPath, LOG_DIR_NAME) - private[delta] def logPathFor(dataPath: Path): Path = new Path(dataPath, LOG_DIR_NAME) - private[delta] def logPathFor(dataPath: File): Path = logPathFor(dataPath.getAbsolutePath) - - /** - * We create only a single [[DeltaLog]] for any given `DeltaLogCacheKey` to avoid wasted work - * in reconstructing the log. - */ - private val deltaLogCache = { - val builder = CacheBuilder.newBuilder() - .expireAfterAccess(60, TimeUnit.MINUTES) - .removalListener((removalNotification: RemovalNotification[DeltaLogCacheKey, DeltaLog]) => { - val log = removalNotification.getValue - // TODO: We should use ref-counting to uncache snapshots instead of a manual timed op - try log.unsafeVolatileSnapshot.uncache() catch { - case _: java.lang.NullPointerException => - // Various layers will throw null pointer if the RDD is already gone. - } - }) - sys.props.get("delta.log.cacheSize") - .flatMap(v => Try(v.toLong).toOption) - .foreach(builder.maximumSize) - builder.build[DeltaLogCacheKey, DeltaLog]() - } - - - // Don't tolerate malformed JSON when parsing Delta log actions (default is PERMISSIVE) - val jsonCommitParseOption = Map("mode" -> FailFastMode.name) - - /** Helper for creating a log when it stored at the root of the data. */ - def forTable(spark: SparkSession, dataPath: String): DeltaLog = { - apply(spark, logPathFor(dataPath), Map.empty, new SystemClock) - } - - /** Helper for creating a log when it stored at the root of the data. */ - def forTable(spark: SparkSession, dataPath: String, options: Map[String, String]): DeltaLog = { - apply(spark, logPathFor(dataPath), options, new SystemClock) - } - - /** Helper for creating a log when it stored at the root of the data. */ - def forTable(spark: SparkSession, dataPath: File): DeltaLog = { - apply(spark, logPathFor(dataPath), new SystemClock) - } - - /** Helper for creating a log when it stored at the root of the data. */ - def forTable(spark: SparkSession, dataPath: Path): DeltaLog = { - apply(spark, logPathFor(dataPath), new SystemClock) - } - - /** Helper for creating a log when it stored at the root of the data. */ - def forTable(spark: SparkSession, dataPath: Path, options: Map[String, String]): DeltaLog = { - apply(spark, logPathFor(dataPath), options, new SystemClock) - } - - /** Helper for creating a log when it stored at the root of the data. */ - def forTable(spark: SparkSession, dataPath: String, clock: Clock): DeltaLog = { - apply(spark, logPathFor(dataPath), clock) - } - - /** Helper for creating a log when it stored at the root of the data. */ - def forTable(spark: SparkSession, dataPath: File, clock: Clock): DeltaLog = { - apply(spark, logPathFor(dataPath), clock) - } - - /** Helper for creating a log when it stored at the root of the data. */ - def forTable(spark: SparkSession, dataPath: Path, clock: Clock): DeltaLog = { - apply(spark, logPathFor(dataPath), clock) - } - - /** Helper for creating a log for the table. */ - def forTable(spark: SparkSession, tableName: TableIdentifier): DeltaLog = { - forTable(spark, tableName, new SystemClock) - } - - /** Helper for creating a log for the table. */ - def forTable(spark: SparkSession, table: CatalogTable): DeltaLog = { - forTable(spark, table, new SystemClock) - } - - /** Helper for creating a log for the table. */ - def forTable(spark: SparkSession, tableName: TableIdentifier, clock: Clock): DeltaLog = { - if (DeltaTableIdentifier.isDeltaPath(spark, tableName)) { - forTable(spark, new Path(tableName.table)) - } else { - forTable(spark, spark.sessionState.catalog.getTableMetadata(tableName), clock) - } - } - - /** Helper for creating a log for the table. */ - def forTable(spark: SparkSession, table: CatalogTable, clock: Clock): DeltaLog = { - apply(spark, logPathFor(new Path(table.location)), clock) - } - - /** Helper for creating a log for the table. */ - def forTable(spark: SparkSession, deltaTable: DeltaTableIdentifier): DeltaLog = { - forTable(spark, deltaTable, new SystemClock) - } - - /** Helper for creating a log for the table. */ - def forTable(spark: SparkSession, deltaTable: DeltaTableIdentifier, clock: Clock): DeltaLog = { - if (deltaTable.path.isDefined) { - forTable(spark, deltaTable.path.get, clock) - } else { - forTable(spark, deltaTable.table.get, clock) - } - } - - private def apply(spark: SparkSession, rawPath: Path, clock: Clock = new SystemClock): DeltaLog = - apply(spark, rawPath, Map.empty, clock) - - - /** Helper for getting a log, as well as the latest snapshot, of the table */ - def forTableWithSnapshot(spark: SparkSession, dataPath: String): (DeltaLog, Snapshot) = - withFreshSnapshot { forTable(spark, dataPath, _) } - - /** Helper for getting a log, as well as the latest snapshot, of the table */ - def forTableWithSnapshot(spark: SparkSession, dataPath: Path): (DeltaLog, Snapshot) = - withFreshSnapshot { forTable(spark, dataPath, _) } - - /** Helper for getting a log, as well as the latest snapshot, of the table */ - def forTableWithSnapshot( - spark: SparkSession, - tableName: TableIdentifier): (DeltaLog, Snapshot) = - withFreshSnapshot { forTable(spark, tableName, _) } - - /** Helper for getting a log, as well as the latest snapshot, of the table */ - def forTableWithSnapshot( - spark: SparkSession, - tableName: DeltaTableIdentifier): (DeltaLog, Snapshot) = - withFreshSnapshot { forTable(spark, tableName, _) } - - /** Helper for getting a log, as well as the latest snapshot, of the table */ - def forTableWithSnapshot( - spark: SparkSession, - dataPath: Path, - options: Map[String, String]): (DeltaLog, Snapshot) = - withFreshSnapshot { apply(spark, logPathFor(dataPath), options, _) } - - /** - * Helper function to be used with the forTableWithSnapshot calls. Thunk is a - * partially applied DeltaLog.forTable call, which we can then wrap around with a - * snapshot update. We use the system clock to avoid back-to-back updates. - */ - private[delta] def withFreshSnapshot(thunk: Clock => DeltaLog): (DeltaLog, Snapshot) = { - val clock = new SystemClock - val ts = clock.getTimeMillis() - val deltaLog = thunk(clock) - val snapshot = deltaLog.update(checkIfUpdatedSinceTs = Some(ts)) - (deltaLog, snapshot) - } - - private def apply( - spark: SparkSession, - rawPath: Path, - options: Map[String, String], - clock: Clock - ): DeltaLog = { - val fileSystemOptions: Map[String, String] = - if (spark.sessionState.conf.getConf( - DeltaSQLConf.LOAD_FILE_SYSTEM_CONFIGS_FROM_DATAFRAME_OPTIONS)) { - // We pick up only file system options so that we don't pass any parquet or json options to - // the code that reads Delta transaction logs. - options.filterKeys { k => - DeltaTableUtils.validDeltaTableHadoopPrefixes.exists(k.startsWith) - }.toMap - } else { - Map.empty - } - // scalastyle:off deltahadoopconfiguration - val hadoopConf = spark.sessionState.newHadoopConfWithOptions(fileSystemOptions) - // scalastyle:on deltahadoopconfiguration - val fs = rawPath.getFileSystem(hadoopConf) - val path = fs.makeQualified(rawPath) - def createDeltaLog(): DeltaLog = recordDeltaOperation( - null, - "delta.log.create", - Map(TAG_TAHOE_PATH -> path.getParent.toString)) { - AnalysisHelper.allowInvokingTransformsInAnalyzer { - new DeltaLog( - logPath = path, - dataPath = path.getParent, - options = fileSystemOptions, - allOptions = options, - clock = clock - ) - } - } - def getDeltaLogFromCache(): DeltaLog = { - // The following cases will still create a new ActionLog even if there is a cached - // ActionLog using a different format path: - // - Different `scheme` - // - Different `authority` (e.g., different user tokens in the path) - // - Different mount point. - try { - deltaLogCache.get(path -> fileSystemOptions, () => { - createDeltaLog() - } - ) - } catch { - case e: com.google.common.util.concurrent.UncheckedExecutionException => - throw e.getCause - } - } - - val deltaLog = getDeltaLogFromCache() - if (Option(deltaLog.sparkContext.get).map(_.isStopped).getOrElse(true)) { - // Invalid the cached `DeltaLog` and create a new one because the `SparkContext` of the cached - // `DeltaLog` has been stopped. - deltaLogCache.invalidate(path -> fileSystemOptions) - getDeltaLogFromCache() - } else { - deltaLog - } - } - - /** Invalidate the cached DeltaLog object for the given `dataPath`. */ - def invalidateCache(spark: SparkSession, dataPath: Path): Unit = { - try { - val rawPath = logPathFor(dataPath) - // scalastyle:off deltahadoopconfiguration - // This method cannot be called from DataFrameReader/Writer so it's safe to assume the user - // has set the correct file system configurations in the session configs. - val fs = rawPath.getFileSystem(spark.sessionState.newHadoopConf()) - // scalastyle:on deltahadoopconfiguration - val path = fs.makeQualified(rawPath) - - if (spark.sessionState.conf.getConf( - DeltaSQLConf.LOAD_FILE_SYSTEM_CONFIGS_FROM_DATAFRAME_OPTIONS)) { - // We rely on the fact that accessing the key set doesn't modify the entry access time. See - // `CacheBuilder.expireAfterAccess`. - val keysToBeRemoved = mutable.ArrayBuffer[DeltaLogCacheKey]() - val iter = deltaLogCache.asMap().keySet().iterator() - while (iter.hasNext) { - val key = iter.next() - if (key._1 == path) { - keysToBeRemoved += key - } - } - deltaLogCache.invalidateAll(keysToBeRemoved.asJava) - } else { - deltaLogCache.invalidate(path -> Map.empty) - } - } catch { - case NonFatal(e) => logWarning(e.getMessage, e) - } - } - - def clearCache(): Unit = { - deltaLogCache.invalidateAll() - } - - /** Return the number of cached `DeltaLog`s. Exposing for testing */ - private[delta] def cacheSize: Long = { - deltaLogCache.size() - } - - /** - * Filters the given [[Dataset]] by the given `partitionFilters`, returning those that match. - * @param files The active files in the DeltaLog state, which contains the partition value - * information - * @param partitionFilters Filters on the partition columns - * @param partitionColumnPrefixes The path to the `partitionValues` column, if it's nested - * @param shouldRewritePartitionFilters Whether to rewrite `partitionFilters` to be over the - * [[AddFile]] schema - */ - def filterFileList( - partitionSchema: StructType, - files: DataFrame, - partitionFilters: Seq[Expression], - partitionColumnPrefixes: Seq[String] = Nil, - shouldRewritePartitionFilters: Boolean = true): DataFrame = { - - val rewrittenFilters = if (shouldRewritePartitionFilters) { - rewritePartitionFilters( - partitionSchema, - files.sparkSession.sessionState.conf.resolver, - partitionFilters, - partitionColumnPrefixes) - } else { - partitionFilters - } - val expr = rewrittenFilters.reduceLeftOption(And).getOrElse(Literal.TrueLiteral) - val columnFilter = new Column(expr) - files.filter(columnFilter) - } - - /** - * Rewrite the given `partitionFilters` to be used for filtering partition values. - * We need to explicitly resolve the partitioning columns here because the partition columns - * are stored as keys of a Map type instead of attributes in the AddFile schema (below) and thus - * cannot be resolved automatically. - * - * @param partitionFilters Filters on the partition columns - * @param partitionColumnPrefixes The path to the `partitionValues` column, if it's nested - */ - def rewritePartitionFilters( - partitionSchema: StructType, - resolver: Resolver, - partitionFilters: Seq[Expression], - partitionColumnPrefixes: Seq[String] = Nil): Seq[Expression] = { - partitionFilters.map(_.transformUp { - case a: Attribute => - // If we have a special column name, e.g. `a.a`, then an UnresolvedAttribute returns - // the column name as '`a.a`' instead of 'a.a', therefore we need to strip the backticks. - val unquoted = a.name.stripPrefix("`").stripSuffix("`") - val partitionCol = partitionSchema.find { field => resolver(field.name, unquoted) } - partitionCol match { - case Some(f: StructField) => - val name = DeltaColumnMapping.getPhysicalName(f) - Cast( - UnresolvedAttribute(partitionColumnPrefixes ++ Seq("partitionValues", name)), - f.dataType) - case None => - // This should not be able to happen, but the case was present in the original code so - // we kept it to be safe. - log.error(s"Partition filter referenced column ${a.name} not in the partition schema") - UnresolvedAttribute(partitionColumnPrefixes ++ Seq("partitionValues", a.name)) - } - }) - } - - - /** - * Checks whether this table only accepts appends. If so it will throw an error in operations that - * can remove data such as DELETE/UPDATE/MERGE. - */ - def assertRemovable(snapshot: Snapshot): Unit = { - val metadata = snapshot.metadata - if (DeltaConfigs.IS_APPEND_ONLY.fromMetaData(metadata)) { - throw DeltaErrors.modifyAppendOnlyTableException(metadata.name) - } - } - - /** How long to keep around SetTransaction actions before physically deleting them. */ - def minSetTransactionRetentionInterval(metadata: Metadata): Option[Long] = { - DeltaConfigs.TRANSACTION_ID_RETENTION_DURATION - .fromMetaData(metadata) - .map(DeltaConfigs.getMilliSeconds) - } - /** How long to keep around logically deleted files before physically deleting them. */ - def tombstoneRetentionMillis(metadata: Metadata): Long = { - DeltaConfigs.getMilliSeconds(DeltaConfigs.TOMBSTONE_RETENTION.fromMetaData(metadata)) - } - - /** Get a function that canonicalizes a given `path`. */ - private[delta] class CanonicalPathFunction(getHadoopConf: () => Configuration) - extends Function[String, String] with Serializable { - // Mark it `@transient lazy val` so that de-serialization happens only once on every executor. - @transient - private lazy val fs = { - // scalastyle:off FileSystemGet - FileSystem.get(getHadoopConf()) - // scalastyle:on FileSystemGet - } - - override def apply(path: String): String = { - val hadoopPath = new Path(new URI(path)) - if (hadoopPath.isAbsoluteAndSchemeAuthorityNull) { - fs.makeQualified(hadoopPath).toUri.toString - } else { - // return untouched if it is a relative path or is already fully qualified - hadoopPath.toUri.toString - } - } - } -} diff --git a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/Snapshot.scala b/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/Snapshot.scala deleted file mode 100644 index b2b5ba42bb3..00000000000 --- a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/Snapshot.scala +++ /dev/null @@ -1,638 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.delta - -// scalastyle:off import.ordering.noEmptyLine -import scala.collection.mutable - -import org.apache.spark.sql.delta.actions._ -import org.apache.spark.sql.delta.actions.Action.logSchema -import org.apache.spark.sql.delta.metering.DeltaLogging -import org.apache.spark.sql.delta.schema.SchemaUtils -import org.apache.spark.sql.delta.sources.DeltaSQLConf -import org.apache.spark.sql.delta.stats.DataSkippingReader -import org.apache.spark.sql.delta.stats.DeltaScan -import org.apache.spark.sql.delta.stats.FileSizeHistogram -import org.apache.spark.sql.delta.stats.StatisticsCollection -import org.apache.spark.sql.delta.util.StateCache -import org.apache.hadoop.fs.{FileStatus, Path} - -import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.expressions.Expression -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig -import org.apache.spark.sql.functions._ -import org.apache.spark.sql.types.StructType -import org.apache.spark.util.Utils - -/** - * Gluten overwrite Delta: - * - * This file is copied from Delta 2.3.0. It is modified to overcome the following issues: - * 1. filesForScan() will cache the DeltaScan by the FilterExprsAsKey - * 2. filesForScan() should return DeltaScan of AddMergeTreeParts instead of AddFile - */ -/** - * A description of a Delta [[Snapshot]], including basic information such its [[DeltaLog]] - * metadata, protocol, and version. - */ -trait SnapshotDescriptor { - def deltaLog: DeltaLog - def version: Long - def metadata: Metadata - def protocol: Protocol - - def schema: StructType = metadata.schema -} - -/** - * An immutable snapshot of the state of the log at some delta version. Internally - * this class manages the replay of actions stored in checkpoint or delta files. - * - * After resolving any new actions, it caches the result and collects the - * following basic information to the driver: - * - Protocol Version - * - Metadata - * - Transaction state - * - * @param timestamp The timestamp of the latest commit in milliseconds. Can also be set to -1 if the - * timestamp of the commit is unknown or the table has not been initialized, i.e. - * `version = -1`. - * - */ -class Snapshot( - val path: Path, - override val version: Long, - val logSegment: LogSegment, - override val deltaLog: DeltaLog, - val timestamp: Long, - val checksumOpt: Option[VersionChecksum], - checkpointMetadataOpt: Option[CheckpointMetaData] = None) - extends SnapshotDescriptor - with StateCache - with StatisticsCollection - with DataSkippingReader - with DeltaLogging { - - import Snapshot._ - // For implicits which re-use Encoder: - import org.apache.spark.sql.delta.implicits._ - - protected def spark = SparkSession.active - - - /** Snapshot to scan by the DeltaScanGenerator for metadata query optimizations */ - override val snapshotToScan: Snapshot = this - - protected def getNumPartitions: Int = { - spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_SNAPSHOT_PARTITIONS) - .getOrElse(Snapshot.defaultNumSnapshotPartitions) - } - - /** Performs validations during initialization */ - protected def init(): Unit = { - deltaLog.protocolRead(protocol) - deltaLog.assertTableFeaturesMatchMetadata(protocol, metadata) - SchemaUtils.recordUndefinedTypes(deltaLog, metadata.schema) - } - - // Reconstruct the state by applying deltas in order to the checkpoint. - // We partition by path as it is likely the bulk of the data is add/remove. - // Non-path based actions will be collocated to a single partition. - private def stateReconstruction: Dataset[SingleAction] = { - recordFrameProfile("Delta", "snapshot.stateReconstruction") { - // for serializability - val localMinFileRetentionTimestamp = minFileRetentionTimestamp - val localMinSetTransactionRetentionTimestamp = minSetTransactionRetentionTimestamp - - val canonicalPath = deltaLog.getCanonicalPathUdf() - - // Canonicalize the paths so we can repartition the actions correctly, but only rewrite the - // add/remove actions themselves after partitioning and sorting are complete. Otherwise, the - // optimizer can generate a really bad plan that re-evaluates _EVERY_ field of the rewritten - // struct(...) projection every time we touch _ANY_ field of the rewritten struct. - // - // NOTE: We sort by [[ACTION_SORT_COL_NAME]] (provided by [[loadActions]]), to ensure that - // actions are presented to InMemoryLogReplay in the ascending version order it expects. - val ADD_PATH_CANONICAL_COL_NAME = "add_path_canonical" - val REMOVE_PATH_CANONICAL_COL_NAME = "remove_path_canonical" - loadActions - .withColumn(ADD_PATH_CANONICAL_COL_NAME, when( - col("add.path").isNotNull, canonicalPath(col("add.path")))) - .withColumn(REMOVE_PATH_CANONICAL_COL_NAME, when( - col("remove.path").isNotNull, canonicalPath(col("remove.path")))) - .repartition( - getNumPartitions, - coalesce(col(ADD_PATH_CANONICAL_COL_NAME), col(REMOVE_PATH_CANONICAL_COL_NAME))) - .sortWithinPartitions(ACTION_SORT_COL_NAME) - .withColumn("add", when( - col("add.path").isNotNull, - struct( - col(ADD_PATH_CANONICAL_COL_NAME).as("path"), - col("add.partitionValues"), - col("add.size"), - col("add.modificationTime"), - col("add.dataChange"), - col(ADD_STATS_TO_USE_COL_NAME).as("stats"), - col("add.tags"), - col("add.deletionVector") - ))) - .withColumn("remove", when( - col("remove.path").isNotNull, - col("remove").withField("path", col(REMOVE_PATH_CANONICAL_COL_NAME)))) - .as[SingleAction] - .mapPartitions { iter => - val state: LogReplay = - new InMemoryLogReplay( - localMinFileRetentionTimestamp, - localMinSetTransactionRetentionTimestamp) - state.append(0, iter.map(_.unwrap)) - state.checkpoint.map(_.wrap) - } - } - } - - /** - * Pulls the protocol and metadata of the table from the files that are used to compute the - * Snapshot directly--without triggering a full state reconstruction. This is important, because - * state reconstruction depends on protocol and metadata for correctness. - */ - protected def protocolAndMetadataReconstruction(): Array[(Protocol, Metadata)] = { - import implicits._ - - val schemaToUse = Action.logSchema(Set("protocol", "metaData")) - fileIndices.map(deltaLog.loadIndex(_, schemaToUse)) - .reduceOption(_.union(_)).getOrElse(emptyDF) - .withColumn(ACTION_SORT_COL_NAME, input_file_name()) - .select("protocol", "metaData", ACTION_SORT_COL_NAME) - .where("protocol.minReaderVersion is not null or metaData.id is not null") - .as[(Protocol, Metadata, String)] - .collect() - .sortBy(_._3) - .map { case (p, m, _) => p -> m } - } - - def redactedPath: String = - Utils.redact(spark.sessionState.conf.stringRedactionPattern, path.toUri.toString) - - @volatile private[delta] var stateReconstructionTriggered = false - private lazy val cachedState = recordFrameProfile("Delta", "snapshot.cachedState") { - stateReconstructionTriggered = true - cacheDS(stateReconstruction, s"Delta Table State #$version - $redactedPath") - } - - /** The current set of actions in this [[Snapshot]] as a typed Dataset. */ - def stateDS: Dataset[SingleAction] = recordFrameProfile("Delta", "stateDS") { - cachedState.getDS - } - - /** The current set of actions in this [[Snapshot]] as plain Rows */ - def stateDF: DataFrame = recordFrameProfile("Delta", "stateDF") { - cachedState.getDF - } - - /** - * A Map of alias to aggregations which needs to be done to calculate the `computedState` - */ - protected def aggregationsToComputeState: Map[String, Column] = { - Map( - // sum may return null for empty data set. - "sizeInBytes" -> coalesce(sum(col("add.size")), lit(0L)), - "numOfSetTransactions" -> count(col("txn")), - "numOfFiles" -> count(col("add")), - "numOfRemoves" -> count(col("remove")), - "numOfMetadata" -> count(col("metaData")), - "numOfProtocol" -> count(col("protocol")), - "setTransactions" -> collect_set(col("txn")), - "metadata" -> last(col("metaData"), ignoreNulls = true), - "protocol" -> last(col("protocol"), ignoreNulls = true), - "fileSizeHistogram" -> lit(null).cast(FileSizeHistogram.schema) - ) - } - - /** - * Computes some statistics around the transaction log, therefore on the actions made on this - * Delta table. - */ - protected lazy val computedState: State = { - withStatusCode("DELTA", s"Compute snapshot for version: $version") { - recordFrameProfile("Delta", "snapshot.computedState") { - val startTime = System.nanoTime() - val aggregations = - aggregationsToComputeState.map { case (alias, agg) => agg.as(alias) }.toSeq - val _computedState = recordFrameProfile("Delta", "snapshot.computedState.aggregations") { - stateDF.select(aggregations: _*).as[State].first() - } - if (_computedState.protocol == null) { - recordDeltaEvent( - deltaLog, - opType = "delta.assertions.missingAction", - data = Map( - "version" -> version.toString, "action" -> "Protocol", "source" -> "Snapshot")) - throw DeltaErrors.actionNotFoundException("protocol", version) - } else if (_computedState.protocol != protocol) { - recordDeltaEvent( - deltaLog, - opType = "delta.assertions.mismatchedAction", - data = Map( - "version" -> version.toString, "action" -> "Protocol", "source" -> "Snapshot", - "computedState.protocol" -> _computedState.protocol, - "extracted.protocol" -> protocol)) - throw DeltaErrors.actionNotFoundException("protocol", version) - } - - if (_computedState.metadata == null) { - recordDeltaEvent( - deltaLog, - opType = "delta.assertions.missingAction", - data = Map( - "version" -> version.toString, "action" -> "Metadata", "source" -> "Metadata")) - throw DeltaErrors.actionNotFoundException("metadata", version) - } else if (_computedState.metadata != metadata) { - recordDeltaEvent( - deltaLog, - opType = "delta.assertions.mismatchedAction", - data = Map( - "version" -> version.toString, "action" -> "Metadata", "source" -> "Snapshot", - "computedState.metadata" -> _computedState.metadata, - "extracted.metadata" -> metadata)) - throw DeltaErrors.actionNotFoundException("metadata", version) - } - - _computedState - } - } - } - - // Used by [[protocol]] and [[metadata]] below - private lazy val (_protocol, _metadata): (Protocol, Metadata) = { - // Should be small. At most 'checkpointInterval' rows, unless new commits are coming - // in before a checkpoint can be written - var protocol: Protocol = null - var metadata: Metadata = null - protocolAndMetadataReconstruction().foreach { - case (p: Protocol, _) => protocol = p - case (_, m: Metadata) => metadata = m - } - - if (protocol == null) { - recordDeltaEvent( - deltaLog, - opType = "delta.assertions.missingAction", - data = Map( - "version" -> version.toString, "action" -> "Protocol", "source" -> "Snapshot")) - throw DeltaErrors.actionNotFoundException("protocol", version) - } - - if (metadata == null) { - recordDeltaEvent( - deltaLog, - opType = "delta.assertions.missingAction", - data = Map( - "version" -> version.toString, "action" -> "Metadata", "source" -> "Snapshot")) - throw DeltaErrors.actionNotFoundException("metadata", version) - } - - protocol -> metadata - } - - def sizeInBytes: Long = computedState.sizeInBytes - def numOfSetTransactions: Long = computedState.numOfSetTransactions - def numOfFiles: Long = computedState.numOfFiles - def numOfRemoves: Long = computedState.numOfRemoves - def numOfMetadata: Long = computedState.numOfMetadata - def numOfProtocol: Long = computedState.numOfProtocol - def setTransactions: Seq[SetTransaction] = computedState.setTransactions - override def metadata: Metadata = _metadata - override def protocol: Protocol = _protocol - def fileSizeHistogram: Option[FileSizeHistogram] = computedState.fileSizeHistogram - private[delta] def sizeInBytesIfKnown: Option[Long] = Some(sizeInBytes) - private[delta] def setTransactionsIfKnown: Option[Seq[SetTransaction]] = Some(setTransactions) - private[delta] def numOfFilesIfKnown: Option[Long] = Some(numOfFiles) - - /** - * Tombstones before the [[minFileRetentionTimestamp]] timestamp will be dropped from the - * checkpoint. - */ - private[delta] def minFileRetentionTimestamp: Long = { - deltaLog.clock.getTimeMillis() - DeltaLog.tombstoneRetentionMillis(metadata) - } - - /** - * [[SetTransaction]]s before [[minSetTransactionRetentionTimestamp]] will be considered expired - * and dropped from the snapshot. - */ - private[delta] def minSetTransactionRetentionTimestamp: Option[Long] = { - DeltaLog.minSetTransactionRetentionInterval(metadata).map(deltaLog.clock.getTimeMillis() - _) - } - - /** - * Computes all the information that is needed by the checksum for the current snapshot. - * May kick off state reconstruction if needed by any of the underlying fields. - * Note that it's safe to set txnId to none, since the snapshot doesn't always have a txn - * attached. E.g. if a snapshot is created by reading a checkpoint, then no txnId is present. - */ - def computeChecksum: VersionChecksum = VersionChecksum( - txnId = None, - tableSizeBytes = sizeInBytes, - numFiles = numOfFiles, - numMetadata = numOfMetadata, - numProtocol = numOfProtocol, - setTransactions = checksumOpt.flatMap(_.setTransactions), - metadata = metadata, - protocol = protocol, - histogramOpt = fileSizeHistogram, - allFiles = checksumOpt.flatMap(_.allFiles)) - - /** A map to look up transaction version by appId. */ - lazy val transactions: Map[String, Long] = setTransactions.map(t => t.appId -> t.version).toMap - - // Here we need to bypass the ACL checks for SELECT anonymous function permissions. - /** All of the files present in this [[Snapshot]]. */ - def allFiles: Dataset[AddFile] = allFilesViaStateReconstruction - - private[delta] def allFilesViaStateReconstruction: Dataset[AddFile] = { - stateDS.where("add IS NOT NULL").select(col("add").as[AddFile]) - } - - /** All unexpired tombstones. */ - def tombstones: Dataset[RemoveFile] = { - stateDS.where("remove IS NOT NULL").select(col("remove").as[RemoveFile]) - } - - /** Returns the data schema of the table, used for reading stats */ - def tableDataSchema: StructType = metadata.dataSchema - - /** Returns the schema of the columns written out to file (overridden in write path) */ - def dataSchema: StructType = metadata.dataSchema - - /** Number of columns to collect stats on for data skipping */ - lazy val numIndexedCols: Int = DeltaConfigs.DATA_SKIPPING_NUM_INDEXED_COLS.fromMetaData(metadata) - - /** Return the set of properties of the table. */ - def getProperties: mutable.Map[String, String] = { - val base = new mutable.LinkedHashMap[String, String]() - metadata.configuration.foreach { case (k, v) => - if (k != "path") { - base.put(k, v) - } - } - base.put(Protocol.MIN_READER_VERSION_PROP, protocol.minReaderVersion.toString) - base.put(Protocol.MIN_WRITER_VERSION_PROP, protocol.minWriterVersion.toString) - if (protocol.supportsReaderFeatures || protocol.supportsWriterFeatures) { - val features = protocol.readerAndWriterFeatureNames.map(name => - s"${TableFeatureProtocolUtils.FEATURE_PROP_PREFIX}$name" -> - TableFeatureProtocolUtils.FEATURE_PROP_SUPPORTED) - base ++ features.toSeq.sorted - } else { - base - } - } - - // Given the list of files from `LogSegment`, create respective file indices to help create - // a DataFrame and short-circuit the many file existence and partition schema inference checks - // that exist in DataSource.resolveRelation(). - protected[delta] lazy val deltaFileIndexOpt: Option[DeltaLogFileIndex] = { - assertLogFilesBelongToTable(path, logSegment.deltas) - DeltaLogFileIndex(DeltaLogFileIndex.COMMIT_FILE_FORMAT, logSegment.deltas) - } - - protected lazy val checkpointFileIndexOpt: Option[DeltaLogFileIndex] = { - assertLogFilesBelongToTable(path, logSegment.checkpoint) - DeltaLogFileIndex(DeltaLogFileIndex.CHECKPOINT_FILE_FORMAT, logSegment.checkpoint) - } - - def getCheckpointMetadataOpt: Option[CheckpointMetaData] = checkpointMetadataOpt - - def deltaFileSizeInBytes(): Long = deltaFileIndexOpt.map(_.sizeInBytes).getOrElse(0L) - def checkpointSizeInBytes(): Long = checkpointFileIndexOpt.map(_.sizeInBytes).getOrElse(0L) - - protected lazy val fileIndices: Seq[DeltaLogFileIndex] = { - checkpointFileIndexOpt.toSeq ++ deltaFileIndexOpt.toSeq - } - - /** - * Loads the file indices into a DataFrame that can be used for LogReplay. - * - * In addition to the usual nested columns provided by the SingleAction schema, it should provide - * two additional columns to simplify the log replay process: [[ACTION_SORT_COL_NAME]] (which, - * when sorted in ascending order, will order older actions before newer ones, as required by - * [[InMemoryLogReplay]]); and [[ADD_STATS_TO_USE_COL_NAME]] (to handle certain combinations of - * config settings for delta.checkpoint.writeStatsAsJson and delta.checkpoint.writeStatsAsStruct). - */ - protected def loadActions: DataFrame = { - fileIndices.map(deltaLog.loadIndex(_)) - .reduceOption(_.union(_)).getOrElse(emptyDF) - .withColumn(ACTION_SORT_COL_NAME, input_file_name()) - .withColumn(ADD_STATS_TO_USE_COL_NAME, col("add.stats")) - } - - protected def emptyDF: DataFrame = - spark.createDataFrame(spark.sparkContext.emptyRDD[Row], logSchema) - - - override def logInfo(msg: => String): Unit = { - super.logInfo(s"[tableId=${deltaLog.tableId}] " + msg) - } - - override def logWarning(msg: => String): Unit = { - super.logWarning(s"[tableId=${deltaLog.tableId}] " + msg) - } - - override def logWarning(msg: => String, throwable: Throwable): Unit = { - super.logWarning(s"[tableId=${deltaLog.tableId}] " + msg, throwable) - } - - override def logError(msg: => String): Unit = { - super.logError(s"[tableId=${deltaLog.tableId}] " + msg) - } - - override def logError(msg: => String, throwable: Throwable): Unit = { - super.logError(s"[tableId=${deltaLog.tableId}] " + msg, throwable) - } - - override def toString: String = - s"${getClass.getSimpleName}(path=$path, version=$version, metadata=$metadata, " + - s"logSegment=$logSegment, checksumOpt=$checksumOpt)" - - // --- modified start - override def filesForScan(limit: Long): DeltaScan = { - val deltaScan = ClickhouseSnapshot.deltaScanCache.get( - FilterExprsAsKey(path, ClickhouseSnapshot.genSnapshotId(this), Seq.empty, Some(limit)), - () => { - super.filesForScan(limit) - }) - - replaceWithAddMergeTreeParts(deltaScan) - } - - override def filesForScan(filters: Seq[Expression], keepNumRecords: Boolean): DeltaScan = { - val deltaScan = ClickhouseSnapshot.deltaScanCache.get( - FilterExprsAsKey(path, ClickhouseSnapshot.genSnapshotId(this), filters, None), - () => { - super.filesForScan(filters, keepNumRecords) - }) - - replaceWithAddMergeTreeParts(deltaScan) - } - - override def filesForScan(limit: Long, partitionFilters: Seq[Expression]): DeltaScan = { - val deltaScan = ClickhouseSnapshot.deltaScanCache.get( - FilterExprsAsKey(path, ClickhouseSnapshot.genSnapshotId(this), partitionFilters, Some(limit)), - () => { - super.filesForScan(limit, partitionFilters) - }) - - replaceWithAddMergeTreeParts(deltaScan) - } - - private def replaceWithAddMergeTreeParts(deltaScan: DeltaScan) = { - if (ClickHouseConfig.isMergeTreeFormatEngine(metadata.configuration)) { - DeltaScan.apply( - deltaScan.version, - deltaScan.files - .map( - addFile => { - val addFileAsKey = AddFileAsKey(addFile) - - val ret = ClickhouseSnapshot.addFileToAddMTPCache.get(addFileAsKey) - // this is for later use - ClickhouseSnapshot.pathToAddMTPCache.put(ret.fullPartPath(), ret) - ret - }), - deltaScan.total, - deltaScan.partition, - deltaScan.scanned - )( - deltaScan.scannedSnapshot, - deltaScan.partitionFilters, - deltaScan.dataFilters, - deltaScan.unusedFilters, - deltaScan.scanDurationMs, - deltaScan.dataSkippingType - ) - } else { - deltaScan - } - } - // --- modified end - - logInfo(s"Created snapshot $this") - init() -} - -object Snapshot extends DeltaLogging { - - // Used by [[loadActions]] and [[stateReconstruction]] - val ACTION_SORT_COL_NAME = "action_sort_column" - val ADD_STATS_TO_USE_COL_NAME = "add_stats_to_use" - - private val defaultNumSnapshotPartitions: Int = 50 - - /** Verifies that a set of delta or checkpoint files to be read actually belongs to this table. */ - private def assertLogFilesBelongToTable(logBasePath: Path, files: Seq[FileStatus]): Unit = { - files.map(_.getPath).foreach { filePath => - if (new Path(filePath.toUri).getParent != new Path(logBasePath.toUri)) { - // scalastyle:off throwerror - throw new AssertionError(s"File ($filePath) doesn't belong in the " + - s"transaction log at $logBasePath. Please contact Databricks Support.") - // scalastyle:on throwerror - } - } - } - - /** - * Metrics and metadata computed around the Delta table. - * @param sizeInBytes The total size of the table (of active files, not including tombstones). - * @param numOfSetTransactions Number of streams writing to this table. - * @param numOfFiles The number of files in this table. - * @param numOfRemoves The number of tombstones in the state. - * @param numOfMetadata The number of metadata actions in the state. Should be 1. - * @param numOfProtocol The number of protocol actions in the state. Should be 1. - * @param setTransactions The streaming queries writing to this table. - * @param metadata The metadata of the table. - * @param protocol The protocol version of the Delta table. - * @param fileSizeHistogram A Histogram class tracking the file counts and total bytes - * in different size ranges. - */ - case class State( - sizeInBytes: Long, - numOfSetTransactions: Long, - numOfFiles: Long, - numOfRemoves: Long, - numOfMetadata: Long, - numOfProtocol: Long, - setTransactions: Seq[SetTransaction], - metadata: Metadata, - protocol: Protocol, - fileSizeHistogram: Option[FileSizeHistogram] = None - ) -} - -/** - * An initial snapshot with only metadata specified. Useful for creating a DataFrame from an - * existing parquet table during its conversion to delta. - * - * @param logPath the path to transaction log - * @param deltaLog the delta log object - * @param metadata the metadata of the table - */ -class InitialSnapshot( - val logPath: Path, - override val deltaLog: DeltaLog, - override val metadata: Metadata) - extends Snapshot( - path = logPath, - version = -1, - logSegment = LogSegment.empty(logPath), - deltaLog = deltaLog, - timestamp = -1, - checksumOpt = None - ) { - - def this(logPath: Path, deltaLog: DeltaLog) = this( - logPath, - deltaLog, - Metadata( - configuration = DeltaConfigs.mergeGlobalConfigs( - sqlConfs = SparkSession.active.sessionState.conf, - tableConf = Map.empty, - ignoreProtocolConfsOpt = Some( - DeltaConfigs.ignoreProtocolDefaultsIsSet( - sqlConfs = SparkSession.active.sessionState.conf, - tableConf = deltaLog.allOptions))), - createdTime = Some(System.currentTimeMillis()))) - - override def stateDS: Dataset[SingleAction] = emptyDF.as[SingleAction] - override def stateDF: DataFrame = emptyDF - override protected lazy val computedState: Snapshot.State = initialState - override def protocol: Protocol = computedState.protocol - private def initialState: Snapshot.State = { - val protocol = Protocol.forNewTable(spark, Some(metadata)) - Snapshot.State( - sizeInBytes = 0L, - numOfSetTransactions = 0L, - numOfFiles = 0L, - numOfRemoves = 0L, - numOfMetadata = 1L, - numOfProtocol = 1L, - setTransactions = Nil, - metadata = metadata, - protocol = protocol - ) - } -} diff --git a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala b/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala deleted file mode 100644 index 29caf77b631..00000000000 --- a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.delta.catalog - -import org.apache.spark.Partition -import org.apache.spark.internal.Logging -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.catalog.CatalogTable -import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} -import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder} -import org.apache.spark.sql.delta.{ClickhouseSnapshot, DeltaLog, DeltaTimeTravelSpec, Snapshot} -import org.apache.spark.sql.delta.actions.Metadata -import org.apache.spark.sql.delta.catalog.ClickHouseTableV2.deltaLog2Table -import org.apache.spark.sql.delta.sources.DeltaDataSource -import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, PartitionDirectory} -import org.apache.spark.sql.execution.datasources.clickhouse.utils.MergeTreePartsPartitionsUtil -import org.apache.spark.sql.execution.datasources.mergetree.StorageMeta -import org.apache.spark.sql.execution.datasources.v2.clickhouse.source.DeltaMergeTreeFileFormat -import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.util.CaseInsensitiveStringMap -import org.apache.spark.util.collection.BitSet - -import org.apache.hadoop.fs.Path - -import java.{util => ju} - -import scala.collection.JavaConverters._ - -@SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) -class ClickHouseTableV2( - override val spark: SparkSession, - override val path: Path, - override val catalogTable: Option[CatalogTable] = None, - override val tableIdentifier: Option[String] = None, - override val timeTravelOpt: Option[DeltaTimeTravelSpec] = None, - override val options: Map[String, String] = Map.empty, - override val cdcOptions: CaseInsensitiveStringMap = CaseInsensitiveStringMap.empty(), - val clickhouseExtensionOptions: Map[String, String] = Map.empty) - extends DeltaTableV2( - spark, - path, - catalogTable, - tableIdentifier, - timeTravelOpt, - options, - cdcOptions) - with ClickHouseTableV2Base { - - lazy val (rootPath, partitionFilters, timeTravelByPath) = { - if (catalogTable.isDefined) { - // Fast path for reducing path munging overhead - (new Path(catalogTable.get.location), Nil, None) - } else { - DeltaDataSource.parsePathIdentifier(spark, path.toString, options) - } - } - - override protected lazy val tableSchema: StructType = schema() - - override def name(): String = - catalogTable - .map(_.identifier.unquotedString) - .orElse(tableIdentifier) - .getOrElse(s"clickhouse.`${deltaLog.dataPath}`") - - override def properties(): ju.Map[String, String] = { - val ret = super.properties() - - // for file path based write - if (snapshot.version < 0 && clickhouseExtensionOptions.nonEmpty) { - ret.putAll(clickhouseExtensionOptions.asJava) - } - ret - } - - override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = { - new WriteIntoDeltaBuilder(deltaLog, info.options) - } - - def getFileFormat(meta: Metadata): DeltaMergeTreeFileFormat = { - new DeltaMergeTreeFileFormat( - StorageMeta - .withStorageID(meta, dataBaseName, tableName, ClickhouseSnapshot.genSnapshotId(snapshot))) - } - - override def deltaProperties: Map[String, String] = properties().asScala.toMap - - override def deltaCatalog: Option[CatalogTable] = catalogTable - - override def deltaPath: Path = path - - override def deltaSnapshot: Snapshot = snapshot - - def cacheThis(): Unit = { - deltaLog2Table.put(deltaLog, this) - } - - cacheThis() -} - -@SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) -class TempClickHouseTableV2( - override val spark: SparkSession, - override val catalogTable: Option[CatalogTable] = None) - extends ClickHouseTableV2(spark, null, catalogTable) { - import collection.JavaConverters._ - override def properties(): ju.Map[String, String] = catalogTable.get.properties.asJava - override protected def rawPartitionColumns: Seq[String] = catalogTable.get.partitionColumnNames - override def cacheThis(): Unit = {} -} - -object ClickHouseTableV2 extends Logging { - private val deltaLog2Table = - new scala.collection.concurrent.TrieMap[DeltaLog, ClickHouseTableV2]() - // for CTAS use - val temporalThreadLocalCHTable = new ThreadLocal[ClickHouseTableV2]() - - def getTable(deltaLog: DeltaLog): ClickHouseTableV2 = { - if (deltaLog2Table.contains(deltaLog)) { - deltaLog2Table(deltaLog) - } else if (temporalThreadLocalCHTable.get() != null) { - temporalThreadLocalCHTable.get() - } else { - throw new IllegalStateException( - s"Can not find ClickHouseTableV2 for deltalog ${deltaLog.dataPath}") - } - } - - def clearCache(): Unit = { - deltaLog2Table.clear() - temporalThreadLocalCHTable.remove() - } - - def partsPartitions( - deltaLog: DeltaLog, - relation: HadoopFsRelation, - selectedPartitions: Array[PartitionDirectory], - output: Seq[Attribute], - bucketedScan: Boolean, - optionalBucketSet: Option[BitSet], - optionalNumCoalescedBuckets: Option[Int], - disableBucketedScan: Boolean, - filterExprs: Seq[Expression]): Seq[Partition] = { - val tableV2 = ClickHouseTableV2.getTable(deltaLog) - - MergeTreePartsPartitionsUtil.getMergeTreePartsPartitions( - relation, - selectedPartitions, - output, - bucketedScan, - tableV2.spark, - tableV2, - optionalBucketSet, - optionalNumCoalescedBuckets, - disableBucketedScan, - filterExprs) - - } -} diff --git a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala b/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala deleted file mode 100644 index 88f2b208afd..00000000000 --- a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala +++ /dev/null @@ -1,525 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.delta.commands - -import org.apache.spark.sql.delta._ -import org.apache.spark.sql.delta.actions.{Action, AddCDCFile, AddFile, FileAction} -import org.apache.spark.sql.delta.commands.DeleteCommand.{rewritingFilesMsg, FINDING_TOUCHED_FILES_MSG} -import org.apache.spark.sql.delta.commands.MergeIntoCommand.totalBytesAndDistinctPartitionValues -import org.apache.spark.sql.delta.files.TahoeBatchFileIndex -import org.apache.spark.sql.delta.sources.DeltaSQLConf -import org.apache.spark.sql.delta.util.Utils -import com.fasterxml.jackson.databind.annotation.JsonDeserialize - -import org.apache.spark.SparkContext -import org.apache.spark.sql.{Column, DataFrame, Dataset, Row, SparkSession} -import org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases -import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, EqualNullSafe, Expression, If, Literal, Not} -import org.apache.spark.sql.catalyst.plans.QueryPlan -import org.apache.spark.sql.catalyst.plans.logical.{DeltaDelete, LogicalPlan} -import org.apache.spark.sql.execution.command.LeafRunnableCommand -import org.apache.spark.sql.execution.metric.SQLMetric -import org.apache.spark.sql.execution.metric.SQLMetrics.{createMetric, createTimingMetric} -import org.apache.spark.sql.functions.input_file_name -import org.apache.spark.sql.types.LongType - -/** - * Gluten overwrite Delta: - * - * This file is copied from Delta 2.3.0. - */ - -trait DeleteCommandMetrics { self: LeafRunnableCommand => - @transient private lazy val sc: SparkContext = SparkContext.getOrCreate() - - def createMetrics: Map[String, SQLMetric] = Map[String, SQLMetric]( - "numRemovedFiles" -> createMetric(sc, "number of files removed."), - "numAddedFiles" -> createMetric(sc, "number of files added."), - "numDeletedRows" -> createMetric(sc, "number of rows deleted."), - "numFilesBeforeSkipping" -> createMetric(sc, "number of files before skipping"), - "numBytesBeforeSkipping" -> createMetric(sc, "number of bytes before skipping"), - "numFilesAfterSkipping" -> createMetric(sc, "number of files after skipping"), - "numBytesAfterSkipping" -> createMetric(sc, "number of bytes after skipping"), - "numPartitionsAfterSkipping" -> createMetric(sc, "number of partitions after skipping"), - "numPartitionsAddedTo" -> createMetric(sc, "number of partitions added"), - "numPartitionsRemovedFrom" -> createMetric(sc, "number of partitions removed"), - "numCopiedRows" -> createMetric(sc, "number of rows copied"), - "numAddedBytes" -> createMetric(sc, "number of bytes added"), - "numRemovedBytes" -> createMetric(sc, "number of bytes removed"), - "executionTimeMs" -> - createTimingMetric(sc, "time taken to execute the entire operation"), - "scanTimeMs" -> - createTimingMetric(sc, "time taken to scan the files for matches"), - "rewriteTimeMs" -> - createTimingMetric(sc, "time taken to rewrite the matched files"), - "numAddedChangeFiles" -> createMetric(sc, "number of change data capture files generated"), - "changeFileBytes" -> createMetric(sc, "total size of change data capture files generated"), - "numTouchedRows" -> createMetric(sc, "number of rows touched") - ) - - def getDeletedRowsFromAddFilesAndUpdateMetrics(files: Seq[AddFile]) : Option[Long] = { - if (!conf.getConf(DeltaSQLConf.DELTA_DML_METRICS_FROM_METADATA)) { - return None; - } - // No file to get metadata, return none to be consistent with metadata stats disabled - if (files.isEmpty) { - return None - } - // Return None if any file does not contain numLogicalRecords status - var count: Long = 0 - for (file <- files) { - if (file.numLogicalRecords.isEmpty) { - return None - } - count += file.numLogicalRecords.get - } - metrics("numDeletedRows").set(count) - return Some(count) - } -} - -/** - * Performs a Delete based on the search condition - * - * Algorithm: - * 1) Scan all the files and determine which files have - * the rows that need to be deleted. - * 2) Traverse the affected files and rebuild the touched files. - * 3) Use the Delta protocol to atomically write the remaining rows to new files and remove - * the affected files that are identified in step 1. - */ -case class DeleteCommand( - deltaLog: DeltaLog, - target: LogicalPlan, - condition: Option[Expression]) - extends LeafRunnableCommand with DeltaCommand with DeleteCommandMetrics { - - override def innerChildren: Seq[QueryPlan[_]] = Seq(target) - - override val output: Seq[Attribute] = Seq(AttributeReference("num_affected_rows", LongType)()) - - override lazy val metrics = createMetrics - - final override def run(sparkSession: SparkSession): Seq[Row] = { - recordDeltaOperation(deltaLog, "delta.dml.delete") { - deltaLog.withNewTransaction { txn => - DeltaLog.assertRemovable(txn.snapshot) - if (hasBeenExecuted(txn, sparkSession)) { - sendDriverMetrics(sparkSession, metrics) - return Seq.empty - } - - val deleteActions = performDelete(sparkSession, deltaLog, txn) - txn.commitIfNeeded(deleteActions, DeltaOperations.Delete(condition.map(_.sql).toSeq)) - } - // Re-cache all cached plans(including this relation itself, if it's cached) that refer to - // this data source relation. - sparkSession.sharedState.cacheManager.recacheByPlan(sparkSession, target) - } - - // Adjust for deletes at partition boundaries. Deletes at partition boundaries is a metadata - // operation, therefore we don't actually have any information around how many rows were deleted - // While this info may exist in the file statistics, it's not guaranteed that we have these - // statistics. To avoid any performance regressions, we currently just return a -1 in such cases - if (metrics("numRemovedFiles").value > 0 && metrics("numDeletedRows").value == 0) { - Seq(Row(-1L)) - } else { - Seq(Row(metrics("numDeletedRows").value)) - } - } - - def performDelete( - sparkSession: SparkSession, - deltaLog: DeltaLog, - txn: OptimisticTransaction): Seq[Action] = { - import org.apache.spark.sql.delta.implicits._ - - var numRemovedFiles: Long = 0 - var numAddedFiles: Long = 0 - var numAddedChangeFiles: Long = 0 - var scanTimeMs: Long = 0 - var rewriteTimeMs: Long = 0 - var numAddedBytes: Long = 0 - var changeFileBytes: Long = 0 - var numRemovedBytes: Long = 0 - var numFilesBeforeSkipping: Long = 0 - var numBytesBeforeSkipping: Long = 0 - var numFilesAfterSkipping: Long = 0 - var numBytesAfterSkipping: Long = 0 - var numPartitionsAfterSkipping: Option[Long] = None - var numPartitionsRemovedFrom: Option[Long] = None - var numPartitionsAddedTo: Option[Long] = None - var numDeletedRows: Option[Long] = None - var numCopiedRows: Option[Long] = None - - val startTime = System.nanoTime() - val numFilesTotal = txn.snapshot.numOfFiles - - val deleteActions: Seq[Action] = condition match { - case None => - // Case 1: Delete the whole table if the condition is true - val reportRowLevelMetrics = conf.getConf(DeltaSQLConf.DELTA_DML_METRICS_FROM_METADATA) - val allFiles = txn.filterFiles(Nil, keepNumRecords = reportRowLevelMetrics) - - numRemovedFiles = allFiles.size - scanTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 - val (numBytes, numPartitions) = totalBytesAndDistinctPartitionValues(allFiles) - numRemovedBytes = numBytes - numFilesBeforeSkipping = numRemovedFiles - numBytesBeforeSkipping = numBytes - numFilesAfterSkipping = numRemovedFiles - numBytesAfterSkipping = numBytes - numDeletedRows = getDeletedRowsFromAddFilesAndUpdateMetrics(allFiles) - - if (txn.metadata.partitionColumns.nonEmpty) { - numPartitionsAfterSkipping = Some(numPartitions) - numPartitionsRemovedFrom = Some(numPartitions) - numPartitionsAddedTo = Some(0) - } - val operationTimestamp = System.currentTimeMillis() - allFiles.map(_.removeWithTimestamp(operationTimestamp)) - case Some(cond) => - val (metadataPredicates, otherPredicates) = - DeltaTableUtils.splitMetadataAndDataPredicates( - cond, txn.metadata.partitionColumns, sparkSession) - - numFilesBeforeSkipping = txn.snapshot.numOfFiles - numBytesBeforeSkipping = txn.snapshot.sizeInBytes - - if (otherPredicates.isEmpty) { - // Case 2: The condition can be evaluated using metadata only. - // Delete a set of files without the need of scanning any data files. - val operationTimestamp = System.currentTimeMillis() - val reportRowLevelMetrics = conf.getConf(DeltaSQLConf.DELTA_DML_METRICS_FROM_METADATA) - val candidateFiles = - txn.filterFiles(metadataPredicates, keepNumRecords = reportRowLevelMetrics) - - scanTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 - numRemovedFiles = candidateFiles.size - numRemovedBytes = candidateFiles.map(_.size).sum - numFilesAfterSkipping = candidateFiles.size - val (numCandidateBytes, numCandidatePartitions) = - totalBytesAndDistinctPartitionValues(candidateFiles) - numBytesAfterSkipping = numCandidateBytes - numDeletedRows = getDeletedRowsFromAddFilesAndUpdateMetrics(candidateFiles) - - if (txn.metadata.partitionColumns.nonEmpty) { - numPartitionsAfterSkipping = Some(numCandidatePartitions) - numPartitionsRemovedFrom = Some(numCandidatePartitions) - numPartitionsAddedTo = Some(0) - } - candidateFiles.map(_.removeWithTimestamp(operationTimestamp)) - } else { - // Case 3: Delete the rows based on the condition. - - // Should we write the DVs to represent the deleted rows? - val shouldWriteDVs = shouldWritePersistentDeletionVectors(sparkSession, txn) - - val candidateFiles = txn.filterFiles( - metadataPredicates ++ otherPredicates, - keepNumRecords = shouldWriteDVs) - // `candidateFiles` contains the files filtered using statistics and delete condition - // They may or may not contains any rows that need to be deleted. - - numFilesAfterSkipping = candidateFiles.size - val (numCandidateBytes, numCandidatePartitions) = - totalBytesAndDistinctPartitionValues(candidateFiles) - numBytesAfterSkipping = numCandidateBytes - if (txn.metadata.partitionColumns.nonEmpty) { - numPartitionsAfterSkipping = Some(numCandidatePartitions) - } - - val nameToAddFileMap = generateCandidateFileMap(deltaLog.dataPath, candidateFiles) - - val fileIndex = new TahoeBatchFileIndex( - sparkSession, "delete", candidateFiles, deltaLog, deltaLog.dataPath, txn.snapshot) - if (shouldWriteDVs) { - val targetDf = DeleteWithDeletionVectorsHelper.createTargetDfForScanningForMatches( - sparkSession, - target, - fileIndex) - - // Does the target table already has DVs enabled? If so, we need to read the table - // with deletion vectors. - val mustReadDeletionVectors = DeletionVectorUtils.deletionVectorsReadable(txn.snapshot) - - val touchedFiles = DeleteWithDeletionVectorsHelper.findTouchedFiles( - sparkSession, - txn, - mustReadDeletionVectors, - deltaLog, - targetDf, - fileIndex, - cond) - - if (touchedFiles.nonEmpty) { - DeleteWithDeletionVectorsHelper.processUnmodifiedData(touchedFiles) - } else { - Nil // Nothing to update - } - } else { - // Keep everything from the resolved target except a new TahoeFileIndex - // that only involves the affected files instead of all files. - val newTarget = DeltaTableUtils.replaceFileIndex(target, fileIndex) - val data = Dataset.ofRows(sparkSession, newTarget) - val deletedRowCount = metrics("numDeletedRows") - val deletedRowUdf = DeltaUDF.boolean { () => - deletedRowCount += 1 - true - }.asNondeterministic() - val filesToRewrite = - withStatusCode("DELTA", FINDING_TOUCHED_FILES_MSG) { - if (candidateFiles.isEmpty) { - Array.empty[String] - } else { - // --- modified start - data.filter(new Column(cond)) - .select(input_file_name().as("input_files")) - .filter(deletedRowUdf()) - .distinct() - .as[String] - .collect() - // --- modified end - } - } - - numRemovedFiles = filesToRewrite.length - scanTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 - if (filesToRewrite.isEmpty) { - // Case 3.1: no row matches and no delete will be triggered - if (txn.metadata.partitionColumns.nonEmpty) { - numPartitionsRemovedFrom = Some(0) - numPartitionsAddedTo = Some(0) - } - Nil - } else { - // Case 3.2: some files need an update to remove the deleted files - // Do the second pass and just read the affected files - val baseRelation = buildBaseRelation( - sparkSession, txn, "delete", deltaLog.dataPath, filesToRewrite, nameToAddFileMap) - // Keep everything from the resolved target except a new TahoeFileIndex - // that only involves the affected files instead of all files. - val newTarget = DeltaTableUtils.replaceFileIndex(target, baseRelation.location) - val targetDF = Dataset.ofRows(sparkSession, newTarget) - val filterCond = Not(EqualNullSafe(cond, Literal.TrueLiteral)) - val rewrittenActions = rewriteFiles(txn, targetDF, filterCond, filesToRewrite.length) - val (changeFiles, rewrittenFiles) = rewrittenActions - .partition(_.isInstanceOf[AddCDCFile]) - numAddedFiles = rewrittenFiles.size - val removedFiles = filesToRewrite.map(f => - getTouchedFile(deltaLog.dataPath, f, nameToAddFileMap)) - val (removedBytes, removedPartitions) = - totalBytesAndDistinctPartitionValues(removedFiles) - numRemovedBytes = removedBytes - val (rewrittenBytes, rewrittenPartitions) = - totalBytesAndDistinctPartitionValues(rewrittenFiles) - numAddedBytes = rewrittenBytes - if (txn.metadata.partitionColumns.nonEmpty) { - numPartitionsRemovedFrom = Some(removedPartitions) - numPartitionsAddedTo = Some(rewrittenPartitions) - } - numAddedChangeFiles = changeFiles.size - changeFileBytes = changeFiles.collect { case f: AddCDCFile => f.size }.sum - rewriteTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 - scanTimeMs - numDeletedRows = Some(metrics("numDeletedRows").value) - numCopiedRows = - Some(metrics("numTouchedRows").value - metrics("numDeletedRows").value) - - val operationTimestamp = System.currentTimeMillis() - removeFilesFromPaths( - deltaLog, nameToAddFileMap, filesToRewrite, operationTimestamp) ++ rewrittenActions - } - } - } - } - metrics("numRemovedFiles").set(numRemovedFiles) - metrics("numAddedFiles").set(numAddedFiles) - val executionTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 - metrics("executionTimeMs").set(executionTimeMs) - metrics("scanTimeMs").set(scanTimeMs) - metrics("rewriteTimeMs").set(rewriteTimeMs) - metrics("numAddedChangeFiles").set(numAddedChangeFiles) - metrics("changeFileBytes").set(changeFileBytes) - metrics("numAddedBytes").set(numAddedBytes) - metrics("numRemovedBytes").set(numRemovedBytes) - metrics("numFilesBeforeSkipping").set(numFilesBeforeSkipping) - metrics("numBytesBeforeSkipping").set(numBytesBeforeSkipping) - metrics("numFilesAfterSkipping").set(numFilesAfterSkipping) - metrics("numBytesAfterSkipping").set(numBytesAfterSkipping) - numPartitionsAfterSkipping.foreach(metrics("numPartitionsAfterSkipping").set) - numPartitionsAddedTo.foreach(metrics("numPartitionsAddedTo").set) - numPartitionsRemovedFrom.foreach(metrics("numPartitionsRemovedFrom").set) - numCopiedRows.foreach(metrics("numCopiedRows").set) - txn.registerSQLMetrics(sparkSession, metrics) - sendDriverMetrics(sparkSession, metrics) - - recordDeltaEvent( - deltaLog, - "delta.dml.delete.stats", - data = DeleteMetric( - condition = condition.map(_.sql).getOrElse("true"), - numFilesTotal, - numFilesAfterSkipping, - numAddedFiles, - numRemovedFiles, - numAddedFiles, - numAddedChangeFiles = numAddedChangeFiles, - numFilesBeforeSkipping, - numBytesBeforeSkipping, - numFilesAfterSkipping, - numBytesAfterSkipping, - numPartitionsAfterSkipping, - numPartitionsAddedTo, - numPartitionsRemovedFrom, - numCopiedRows, - numDeletedRows, - numAddedBytes, - numRemovedBytes, - changeFileBytes = changeFileBytes, - scanTimeMs, - rewriteTimeMs) - ) - - if (deleteActions.nonEmpty) { - createSetTransaction(sparkSession, deltaLog).toSeq ++ deleteActions - } else { - Seq.empty - } - } - - /** - * Returns the list of [[AddFile]]s and [[AddCDCFile]]s that have been re-written. - */ - private def rewriteFiles( - txn: OptimisticTransaction, - baseData: DataFrame, - filterCondition: Expression, - numFilesToRewrite: Long): Seq[FileAction] = { - val shouldWriteCdc = DeltaConfigs.CHANGE_DATA_FEED.fromMetaData(txn.metadata) - - // number of total rows that we have seen / are either copying or deleting (sum of both). - val numTouchedRows = metrics("numTouchedRows") - val numTouchedRowsUdf = DeltaUDF.boolean { () => - numTouchedRows += 1 - true - }.asNondeterministic() - - withStatusCode( - "DELTA", rewritingFilesMsg(numFilesToRewrite)) { - val dfToWrite = if (shouldWriteCdc) { - import org.apache.spark.sql.delta.commands.cdc.CDCReader._ - // The logic here ends up being surprisingly elegant, with all source rows ending up in - // the output. Recall that we flipped the user-provided delete condition earlier, before the - // call to `rewriteFiles`. All rows which match this latest `filterCondition` are retained - // as table data, while all rows which don't match are removed from the rewritten table data - // but do get included in the output as CDC events. - baseData - .filter(numTouchedRowsUdf()) - .withColumn( - CDC_TYPE_COLUMN_NAME, - new Column(If(filterCondition, CDC_TYPE_NOT_CDC, CDC_TYPE_DELETE)) - ) - } else { - baseData - .filter(numTouchedRowsUdf()) - .filter(new Column(filterCondition)) - } - - txn.writeFiles(dfToWrite) - } - } - - def shouldWritePersistentDeletionVectors( - spark: SparkSession, txn: OptimisticTransaction): Boolean = { - // DELETE with DVs only enabled for tests. - Utils.isTesting && - spark.conf.get(DeltaSQLConf.DELETE_USE_PERSISTENT_DELETION_VECTORS) && - DeletionVectorUtils.deletionVectorsWritable(txn.snapshot) - } -} - -object DeleteCommand { - def apply(delete: DeltaDelete): DeleteCommand = { - val index = EliminateSubqueryAliases(delete.child) match { - case DeltaFullTable(tahoeFileIndex) => - tahoeFileIndex - case o => - throw DeltaErrors.notADeltaSourceException("DELETE", Some(o)) - } - DeleteCommand(index.deltaLog, delete.child, delete.condition) - } - - val FILE_NAME_COLUMN: String = "_input_file_name_" - val FINDING_TOUCHED_FILES_MSG: String = "Finding files to rewrite for DELETE operation" - - def rewritingFilesMsg(numFilesToRewrite: Long): String = - s"Rewriting $numFilesToRewrite files for DELETE operation" -} - -/** - * Used to report details about delete. - * - * @param condition: what was the delete condition - * @param numFilesTotal: how big is the table - * @param numTouchedFiles: how many files did we touch. Alias for `numFilesAfterSkipping` - * @param numRewrittenFiles: how many files had to be rewritten. Alias for `numAddedFiles` - * @param numRemovedFiles: how many files we removed. Alias for `numTouchedFiles` - * @param numAddedFiles: how many files we added. Alias for `numRewrittenFiles` - * @param numAddedChangeFiles: how many change files were generated - * @param numFilesBeforeSkipping: how many candidate files before skipping - * @param numBytesBeforeSkipping: how many candidate bytes before skipping - * @param numFilesAfterSkipping: how many candidate files after skipping - * @param numBytesAfterSkipping: how many candidate bytes after skipping - * @param numPartitionsAfterSkipping: how many candidate partitions after skipping - * @param numPartitionsAddedTo: how many new partitions were added - * @param numPartitionsRemovedFrom: how many partitions were removed - * @param numCopiedRows: how many rows were copied - * @param numDeletedRows: how many rows were deleted - * @param numBytesAdded: how many bytes were added - * @param numBytesRemoved: how many bytes were removed - * @param changeFileBytes: total size of change files generated - * @param scanTimeMs: how long did finding take - * @param rewriteTimeMs: how long did rewriting take - * - * @note All the time units are milliseconds. - */ -case class DeleteMetric( - condition: String, - numFilesTotal: Long, - numTouchedFiles: Long, - numRewrittenFiles: Long, - numRemovedFiles: Long, - numAddedFiles: Long, - numAddedChangeFiles: Long, - numFilesBeforeSkipping: Long, - numBytesBeforeSkipping: Long, - numFilesAfterSkipping: Long, - numBytesAfterSkipping: Long, - numPartitionsAfterSkipping: Option[Long], - numPartitionsAddedTo: Option[Long], - numPartitionsRemovedFrom: Option[Long], - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - numCopiedRows: Option[Long], - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - numDeletedRows: Option[Long], - numBytesAdded: Long, - numBytesRemoved: Long, - changeFileBytes: Long, - scanTimeMs: Long, - rewriteTimeMs: Long -) diff --git a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala b/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala deleted file mode 100644 index 86bd9a42333..00000000000 --- a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala +++ /dev/null @@ -1,1227 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.delta.commands - -import java.util.concurrent.TimeUnit - -import scala.collection.JavaConverters._ -import scala.collection.mutable - -import org.apache.spark.sql.delta._ -import org.apache.spark.sql.delta.actions.{AddCDCFile, AddFile, FileAction} -import org.apache.spark.sql.delta.commands.merge.MergeIntoMaterializeSource -import org.apache.spark.sql.delta.files._ -import org.apache.spark.sql.delta.schema.{ImplicitMetadataOperation, SchemaUtils} -import org.apache.spark.sql.delta.sources.DeltaSQLConf -import org.apache.spark.sql.delta.util.{AnalysisHelper, SetAccumulator} -import com.fasterxml.jackson.databind.annotation.JsonDeserialize - -import org.apache.spark.SparkContext -import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute -import org.apache.spark.sql.catalyst.encoders.{ExpressionEncoder, RowEncoder} -import org.apache.spark.sql.catalyst.expressions.{Alias, And, Attribute, AttributeReference, BasePredicate, Expression, Literal, NamedExpression, PredicateHelper, UnsafeProjection} -import org.apache.spark.sql.catalyst.expressions.codegen._ -import org.apache.spark.sql.catalyst.plans.logical._ -import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap -import org.apache.spark.sql.execution.command.LeafRunnableCommand -import org.apache.spark.sql.execution.datasources.LogicalRelation -import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} -import org.apache.spark.sql.functions._ -import org.apache.spark.sql.types.{DataTypes, LongType, StructType} - -/** - * Gluten overwrite Delta: - * - * This file is copied from Delta 2.3.0. - */ - -case class MergeDataSizes( - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - rows: Option[Long] = None, - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - files: Option[Long] = None, - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - bytes: Option[Long] = None, - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - partitions: Option[Long] = None) - -/** - * Represents the state of a single merge clause: - * - merge clause's (optional) predicate - * - action type (insert, update, delete) - * - action's expressions - */ -case class MergeClauseStats( - condition: Option[String], - actionType: String, - actionExpr: Seq[String]) - -object MergeClauseStats { - def apply(mergeClause: DeltaMergeIntoClause): MergeClauseStats = { - MergeClauseStats( - condition = mergeClause.condition.map(_.sql), - mergeClause.clauseType.toLowerCase(), - actionExpr = mergeClause.actions.map(_.sql)) - } -} - -/** State for a merge operation */ -case class MergeStats( - // Merge condition expression - conditionExpr: String, - - // Expressions used in old MERGE stats, now always Null - updateConditionExpr: String, - updateExprs: Seq[String], - insertConditionExpr: String, - insertExprs: Seq[String], - deleteConditionExpr: String, - - // Newer expressions used in MERGE with any number of MATCHED/NOT MATCHED/NOT MATCHED BY SOURCE - matchedStats: Seq[MergeClauseStats], - notMatchedStats: Seq[MergeClauseStats], - notMatchedBySourceStats: Seq[MergeClauseStats], - - // Timings - executionTimeMs: Long, - scanTimeMs: Long, - rewriteTimeMs: Long, - - // Data sizes of source and target at different stages of processing - source: MergeDataSizes, - targetBeforeSkipping: MergeDataSizes, - targetAfterSkipping: MergeDataSizes, - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - sourceRowsInSecondScan: Option[Long], - - // Data change sizes - targetFilesRemoved: Long, - targetFilesAdded: Long, - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - targetChangeFilesAdded: Option[Long], - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - targetChangeFileBytes: Option[Long], - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - targetBytesRemoved: Option[Long], - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - targetBytesAdded: Option[Long], - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - targetPartitionsRemovedFrom: Option[Long], - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - targetPartitionsAddedTo: Option[Long], - targetRowsCopied: Long, - targetRowsUpdated: Long, - targetRowsMatchedUpdated: Long, - targetRowsNotMatchedBySourceUpdated: Long, - targetRowsInserted: Long, - targetRowsDeleted: Long, - targetRowsMatchedDeleted: Long, - targetRowsNotMatchedBySourceDeleted: Long, - - // MergeMaterializeSource stats - materializeSourceReason: Option[String] = None, - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - materializeSourceAttempts: Option[Long] = None -) - -object MergeStats { - - def fromMergeSQLMetrics( - metrics: Map[String, SQLMetric], - condition: Expression, - matchedClauses: Seq[DeltaMergeIntoMatchedClause], - notMatchedClauses: Seq[DeltaMergeIntoNotMatchedClause], - notMatchedBySourceClauses: Seq[DeltaMergeIntoNotMatchedBySourceClause], - isPartitioned: Boolean): MergeStats = { - - def metricValueIfPartitioned(metricName: String): Option[Long] = { - if (isPartitioned) Some(metrics(metricName).value) else None - } - - MergeStats( - // Merge condition expression - conditionExpr = condition.sql, - - // Newer expressions used in MERGE with any number of MATCHED/NOT MATCHED/ - // NOT MATCHED BY SOURCE - matchedStats = matchedClauses.map(MergeClauseStats(_)), - notMatchedStats = notMatchedClauses.map(MergeClauseStats(_)), - notMatchedBySourceStats = notMatchedBySourceClauses.map(MergeClauseStats(_)), - - // Timings - executionTimeMs = metrics("executionTimeMs").value, - scanTimeMs = metrics("scanTimeMs").value, - rewriteTimeMs = metrics("rewriteTimeMs").value, - - // Data sizes of source and target at different stages of processing - source = MergeDataSizes(rows = Some(metrics("numSourceRows").value)), - targetBeforeSkipping = - MergeDataSizes( - files = Some(metrics("numTargetFilesBeforeSkipping").value), - bytes = Some(metrics("numTargetBytesBeforeSkipping").value)), - targetAfterSkipping = - MergeDataSizes( - files = Some(metrics("numTargetFilesAfterSkipping").value), - bytes = Some(metrics("numTargetBytesAfterSkipping").value), - partitions = metricValueIfPartitioned("numTargetPartitionsAfterSkipping")), - sourceRowsInSecondScan = - metrics.get("numSourceRowsInSecondScan").map(_.value).filter(_ >= 0), - - // Data change sizes - targetFilesAdded = metrics("numTargetFilesAdded").value, - targetChangeFilesAdded = metrics.get("numTargetChangeFilesAdded").map(_.value), - targetChangeFileBytes = metrics.get("numTargetChangeFileBytes").map(_.value), - targetFilesRemoved = metrics("numTargetFilesRemoved").value, - targetBytesAdded = Some(metrics("numTargetBytesAdded").value), - targetBytesRemoved = Some(metrics("numTargetBytesRemoved").value), - targetPartitionsRemovedFrom = metricValueIfPartitioned("numTargetPartitionsRemovedFrom"), - targetPartitionsAddedTo = metricValueIfPartitioned("numTargetPartitionsAddedTo"), - targetRowsCopied = metrics("numTargetRowsCopied").value, - targetRowsUpdated = metrics("numTargetRowsUpdated").value, - targetRowsMatchedUpdated = metrics("numTargetRowsMatchedUpdated").value, - targetRowsNotMatchedBySourceUpdated = metrics("numTargetRowsNotMatchedBySourceUpdated").value, - targetRowsInserted = metrics("numTargetRowsInserted").value, - targetRowsDeleted = metrics("numTargetRowsDeleted").value, - targetRowsMatchedDeleted = metrics("numTargetRowsMatchedDeleted").value, - targetRowsNotMatchedBySourceDeleted = metrics("numTargetRowsNotMatchedBySourceDeleted").value, - - // Deprecated fields - updateConditionExpr = null, - updateExprs = null, - insertConditionExpr = null, - insertExprs = null, - deleteConditionExpr = null) - } -} - -/** - * Performs a merge of a source query/table into a Delta table. - * - * Issues an error message when the ON search_condition of the MERGE statement can match - * a single row from the target table with multiple rows of the source table-reference. - * - * Algorithm: - * - * Phase 1: Find the input files in target that are touched by the rows that satisfy - * the condition and verify that no two source rows match with the same target row. - * This is implemented as an inner-join using the given condition. See [[findTouchedFiles]] - * for more details. - * - * Phase 2: Read the touched files again and write new files with updated and/or inserted rows. - * - * Phase 3: Use the Delta protocol to atomically remove the touched files and add the new files. - * - * @param source Source data to merge from - * @param target Target table to merge into - * @param targetFileIndex TahoeFileIndex of the target table - * @param condition Condition for a source row to match with a target row - * @param matchedClauses All info related to matched clauses. - * @param notMatchedClauses All info related to not matched clauses. - * @param notMatchedBySourceClauses All info related to not matched by source clauses. - * @param migratedSchema The final schema of the target - may be changed by schema - * evolution. - */ -case class MergeIntoCommand( - @transient source: LogicalPlan, - @transient target: LogicalPlan, - @transient targetFileIndex: TahoeFileIndex, - condition: Expression, - matchedClauses: Seq[DeltaMergeIntoMatchedClause], - notMatchedClauses: Seq[DeltaMergeIntoNotMatchedClause], - notMatchedBySourceClauses: Seq[DeltaMergeIntoNotMatchedBySourceClause], - migratedSchema: Option[StructType]) extends LeafRunnableCommand - with DeltaCommand - with PredicateHelper - with AnalysisHelper - with ImplicitMetadataOperation - with MergeIntoMaterializeSource { - - import MergeIntoCommand._ - - import SQLMetrics._ - import org.apache.spark.sql.delta.commands.cdc.CDCReader._ - - override val canMergeSchema: Boolean = conf.getConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE) - override val canOverwriteSchema: Boolean = false - - override val output: Seq[Attribute] = Seq( - AttributeReference("num_affected_rows", LongType)(), - AttributeReference("num_updated_rows", LongType)(), - AttributeReference("num_deleted_rows", LongType)(), - AttributeReference("num_inserted_rows", LongType)()) - - @transient private lazy val sc: SparkContext = SparkContext.getOrCreate() - @transient private lazy val targetDeltaLog: DeltaLog = targetFileIndex.deltaLog - /** - * Map to get target output attributes by name. - * The case sensitivity of the map is set accordingly to Spark configuration. - */ - @transient private lazy val targetOutputAttributesMap: Map[String, Attribute] = { - val attrMap: Map[String, Attribute] = target - .outputSet.view - .map(attr => attr.name -> attr).toMap - if (conf.caseSensitiveAnalysis) { - attrMap - } else { - CaseInsensitiveMap(attrMap) - } - } - - /** Whether this merge statement has only a single insert (NOT MATCHED) clause. */ - private def isSingleInsertOnly: Boolean = - matchedClauses.isEmpty && notMatchedBySourceClauses.isEmpty && notMatchedClauses.length == 1 - /** Whether this merge statement has no insert (NOT MATCHED) clause. */ - private def hasNoInserts: Boolean = notMatchedClauses.isEmpty - - // We over-count numTargetRowsDeleted when there are multiple matches; - // this is the amount of the overcount, so we can subtract it to get a correct final metric. - private var multipleMatchDeleteOnlyOvercount: Option[Long] = None - - override lazy val metrics = Map[String, SQLMetric]( - "numSourceRows" -> createMetric(sc, "number of source rows"), - "numSourceRowsInSecondScan" -> - createMetric(sc, "number of source rows (during repeated scan)"), - "numTargetRowsCopied" -> createMetric(sc, "number of target rows rewritten unmodified"), - "numTargetRowsInserted" -> createMetric(sc, "number of inserted rows"), - "numTargetRowsUpdated" -> createMetric(sc, "number of updated rows"), - "numTargetRowsMatchedUpdated" -> - createMetric(sc, "number of rows updated by a matched clause"), - "numTargetRowsNotMatchedBySourceUpdated" -> - createMetric(sc, "number of rows updated by a not matched by source clause"), - "numTargetRowsDeleted" -> createMetric(sc, "number of deleted rows"), - "numTargetRowsMatchedDeleted" -> - createMetric(sc, "number of rows deleted by a matched clause"), - "numTargetRowsNotMatchedBySourceDeleted" -> - createMetric(sc, "number of rows deleted by a not matched by source clause"), - "numTargetFilesBeforeSkipping" -> createMetric(sc, "number of target files before skipping"), - "numTargetFilesAfterSkipping" -> createMetric(sc, "number of target files after skipping"), - "numTargetFilesRemoved" -> createMetric(sc, "number of files removed to target"), - "numTargetFilesAdded" -> createMetric(sc, "number of files added to target"), - "numTargetChangeFilesAdded" -> - createMetric(sc, "number of change data capture files generated"), - "numTargetChangeFileBytes" -> - createMetric(sc, "total size of change data capture files generated"), - "numTargetBytesBeforeSkipping" -> createMetric(sc, "number of target bytes before skipping"), - "numTargetBytesAfterSkipping" -> createMetric(sc, "number of target bytes after skipping"), - "numTargetBytesRemoved" -> createMetric(sc, "number of target bytes removed"), - "numTargetBytesAdded" -> createMetric(sc, "number of target bytes added"), - "numTargetPartitionsAfterSkipping" -> - createMetric(sc, "number of target partitions after skipping"), - "numTargetPartitionsRemovedFrom" -> - createMetric(sc, "number of target partitions from which files were removed"), - "numTargetPartitionsAddedTo" -> - createMetric(sc, "number of target partitions to which files were added"), - "executionTimeMs" -> - createTimingMetric(sc, "time taken to execute the entire operation"), - "scanTimeMs" -> - createTimingMetric(sc, "time taken to scan the files for matches"), - "rewriteTimeMs" -> - createTimingMetric(sc, "time taken to rewrite the matched files")) - - override def run(spark: SparkSession): Seq[Row] = { - metrics("executionTimeMs").set(0) - metrics("scanTimeMs").set(0) - metrics("rewriteTimeMs").set(0) - - if (migratedSchema.isDefined) { - // Block writes of void columns in the Delta log. Currently void columns are not properly - // supported and are dropped on read, but this is not enough for merge command that is also - // reading the schema from the Delta log. Until proper support we prefer to fail merge - // queries that add void columns. - val newNullColumn = SchemaUtils.findNullTypeColumn(migratedSchema.get) - if (newNullColumn.isDefined) { - throw new AnalysisException( - s"""Cannot add column '${newNullColumn.get}' with type 'void'. Please explicitly specify a - |non-void type.""".stripMargin.replaceAll("\n", " ") - ) - } - } - val (materializeSource, _) = shouldMaterializeSource(spark, source, isSingleInsertOnly) - if (!materializeSource) { - runMerge(spark) - } else { - // If it is determined that source should be materialized, wrap the execution with retries, - // in case the data of the materialized source is lost. - runWithMaterializedSourceLostRetries( - spark, targetFileIndex.deltaLog, metrics, runMerge) - } - } - - protected def runMerge(spark: SparkSession): Seq[Row] = { - recordDeltaOperation(targetDeltaLog, "delta.dml.merge") { - val startTime = System.nanoTime() - targetDeltaLog.withNewTransaction { deltaTxn => - if (hasBeenExecuted(deltaTxn, spark)) { - sendDriverMetrics(spark, metrics) - return Seq.empty - } - if (target.schema.size != deltaTxn.metadata.schema.size) { - throw DeltaErrors.schemaChangedSinceAnalysis( - atAnalysis = target.schema, latestSchema = deltaTxn.metadata.schema) - } - - if (canMergeSchema) { - updateMetadata( - spark, deltaTxn, migratedSchema.getOrElse(target.schema), - deltaTxn.metadata.partitionColumns, deltaTxn.metadata.configuration, - isOverwriteMode = false, rearrangeOnly = false) - } - - // If materialized, prepare the DF reading the materialize source - // Otherwise, prepare a regular DF from source plan. - val materializeSourceReason = prepareSourceDFAndReturnMaterializeReason( - spark, - source, - condition, - matchedClauses, - notMatchedClauses, - isSingleInsertOnly) - - val deltaActions = { - if (isSingleInsertOnly && spark.conf.get(DeltaSQLConf.MERGE_INSERT_ONLY_ENABLED)) { - writeInsertsOnlyWhenNoMatchedClauses(spark, deltaTxn) - } else { - val filesToRewrite = findTouchedFiles(spark, deltaTxn) - val newWrittenFiles = withStatusCode("DELTA", "Writing merged data") { - writeAllChanges(spark, deltaTxn, filesToRewrite) - } - filesToRewrite.map(_.remove) ++ newWrittenFiles - } - } - - val finalActions = createSetTransaction(spark, targetDeltaLog).toSeq ++ deltaActions - // Metrics should be recorded before commit (where they are written to delta logs). - metrics("executionTimeMs").set((System.nanoTime() - startTime) / 1000 / 1000) - deltaTxn.registerSQLMetrics(spark, metrics) - - // This is a best-effort sanity check. - if (metrics("numSourceRowsInSecondScan").value >= 0 && - metrics("numSourceRows").value != metrics("numSourceRowsInSecondScan").value) { - log.warn(s"Merge source has ${metrics("numSourceRows")} rows in initial scan but " + - s"${metrics("numSourceRowsInSecondScan")} rows in second scan") - if (conf.getConf(DeltaSQLConf.MERGE_FAIL_IF_SOURCE_CHANGED)) { - throw DeltaErrors.sourceNotDeterministicInMergeException(spark) - } - } - - deltaTxn.commitIfNeeded( - finalActions, - DeltaOperations.Merge( - Option(condition.sql), - matchedClauses.map(DeltaOperations.MergePredicate(_)), - notMatchedClauses.map(DeltaOperations.MergePredicate(_)), - notMatchedBySourceClauses.map(DeltaOperations.MergePredicate(_)))) - - // Record metrics - var stats = MergeStats.fromMergeSQLMetrics( - metrics, - condition, - matchedClauses, - notMatchedClauses, - notMatchedBySourceClauses, - deltaTxn.metadata.partitionColumns.nonEmpty) - stats = stats.copy( - materializeSourceReason = Some(materializeSourceReason.toString), - materializeSourceAttempts = Some(attempt)) - - recordDeltaEvent(targetFileIndex.deltaLog, "delta.dml.merge.stats", data = stats) - - } - spark.sharedState.cacheManager.recacheByPlan(spark, target) - } - sendDriverMetrics(spark, metrics) - Seq(Row(metrics("numTargetRowsUpdated").value + metrics("numTargetRowsDeleted").value + - metrics("numTargetRowsInserted").value, metrics("numTargetRowsUpdated").value, - metrics("numTargetRowsDeleted").value, metrics("numTargetRowsInserted").value)) - } - - /** - * Find the target table files that contain the rows that satisfy the merge condition. This is - * implemented as an inner-join between the source query/table and the target table using - * the merge condition. - */ - private def findTouchedFiles( - spark: SparkSession, - deltaTxn: OptimisticTransaction - ): Seq[AddFile] = recordMergeOperation(sqlMetricName = "scanTimeMs") { - - // Accumulator to collect all the distinct touched files - val touchedFilesAccum = new SetAccumulator[String]() - spark.sparkContext.register(touchedFilesAccum, TOUCHED_FILES_ACCUM_NAME) - - // UDFs to records touched files names and add them to the accumulator - val recordTouchedFileName = DeltaUDF.intFromString { fileName => - // --- modified start - fileName.split(",").foreach(name => touchedFilesAccum.add(name)) - // --- modified end - 1 - }.asNondeterministic() - - // Prune non-matching files if we don't need to collect them for NOT MATCHED BY SOURCE clauses. - val dataSkippedFiles = - if (notMatchedBySourceClauses.isEmpty) { - val targetOnlyPredicates = - splitConjunctivePredicates(condition).filter(_.references.subsetOf(target.outputSet)) - deltaTxn.filterFiles(targetOnlyPredicates) - } else { - deltaTxn.filterFiles() - } - - // UDF to increment metrics - val incrSourceRowCountExpr = makeMetricUpdateUDF("numSourceRows") - val sourceDF = getSourceDF() - .filter(new Column(incrSourceRowCountExpr)) - - // Join the source and target table using the merge condition to find touched files. An inner - // join collects all candidate files for MATCHED clauses, a right outer join also includes - // candidates for NOT MATCHED BY SOURCE clauses. - // In addition, we attach two columns - // - a monotonically increasing row id for target rows to later identify whether the same - // target row is modified by multiple user or not - // - the target file name the row is from to later identify the files touched by matched rows - val joinType = if (notMatchedBySourceClauses.isEmpty) "inner" else "right_outer" - val targetDF = buildTargetPlanWithFiles(spark, deltaTxn, dataSkippedFiles) - .withColumn(ROW_ID_COL, monotonically_increasing_id()) - .withColumn(FILE_NAME_COL, input_file_name()) - val joinToFindTouchedFiles = sourceDF.join(targetDF, new Column(condition), joinType) - - // Process the matches from the inner join to record touched files and find multiple matches - val collectTouchedFiles = joinToFindTouchedFiles - .select(col(ROW_ID_COL), recordTouchedFileName(col(FILE_NAME_COL)).as("one")) - - // Calculate frequency of matches per source row - val matchedRowCounts = collectTouchedFiles.groupBy(ROW_ID_COL).agg(sum("one").as("count")) - - // Get multiple matches and simultaneously collect (using touchedFilesAccum) the file names - // multipleMatchCount = # of target rows with more than 1 matching source row (duplicate match) - // multipleMatchSum = total # of duplicate matched rows - import org.apache.spark.sql.delta.implicits._ - val (multipleMatchCount, multipleMatchSum) = matchedRowCounts - .filter("count > 1") - .select(coalesce(count(new Column("*")), lit(0)), coalesce(sum("count"), lit(0))) - .as[(Long, Long)] - .collect() - .head - - val hasMultipleMatches = multipleMatchCount > 0 - - // Throw error if multiple matches are ambiguous or cannot be computed correctly. - val canBeComputedUnambiguously = { - // Multiple matches are not ambiguous when there is only one unconditional delete as - // all the matched row pairs in the 2nd join in `writeAllChanges` will get deleted. - val isUnconditionalDelete = matchedClauses.headOption match { - case Some(DeltaMergeIntoMatchedDeleteClause(None)) => true - case _ => false - } - matchedClauses.size == 1 && isUnconditionalDelete - } - - if (hasMultipleMatches && !canBeComputedUnambiguously) { - throw DeltaErrors.multipleSourceRowMatchingTargetRowInMergeException(spark) - } - - if (hasMultipleMatches) { - // This is only allowed for delete-only queries. - // This query will count the duplicates for numTargetRowsDeleted in Job 2, - // because we count matches after the join and not just the target rows. - // We have to compensate for this by subtracting the duplicates later, - // so we need to record them here. - val duplicateCount = multipleMatchSum - multipleMatchCount - multipleMatchDeleteOnlyOvercount = Some(duplicateCount) - } - - // Get the AddFiles using the touched file names. - val touchedFileNames = touchedFilesAccum.value.iterator().asScala.toSeq - logTrace(s"findTouchedFiles: matched files:\n\t${touchedFileNames.mkString("\n\t")}") - - val nameToAddFileMap = generateCandidateFileMap(targetDeltaLog.dataPath, dataSkippedFiles) - val touchedAddFiles = touchedFileNames.map(f => - getTouchedFile(targetDeltaLog.dataPath, f, nameToAddFileMap)) - - // When the target table is empty, and the optimizer optimized away the join entirely - // numSourceRows will be incorrectly 0. We need to scan the source table once to get the correct - // metric here. - if (metrics("numSourceRows").value == 0 && - (dataSkippedFiles.isEmpty || targetDF.take(1).isEmpty)) { - val numSourceRows = sourceDF.count() - metrics("numSourceRows").set(numSourceRows) - } - - // Update metrics - metrics("numTargetFilesBeforeSkipping") += deltaTxn.snapshot.numOfFiles - metrics("numTargetBytesBeforeSkipping") += deltaTxn.snapshot.sizeInBytes - val (afterSkippingBytes, afterSkippingPartitions) = - totalBytesAndDistinctPartitionValues(dataSkippedFiles) - metrics("numTargetFilesAfterSkipping") += dataSkippedFiles.size - metrics("numTargetBytesAfterSkipping") += afterSkippingBytes - metrics("numTargetPartitionsAfterSkipping") += afterSkippingPartitions - val (removedBytes, removedPartitions) = totalBytesAndDistinctPartitionValues(touchedAddFiles) - metrics("numTargetFilesRemoved") += touchedAddFiles.size - metrics("numTargetBytesRemoved") += removedBytes - metrics("numTargetPartitionsRemovedFrom") += removedPartitions - touchedAddFiles - } - - /** - * This is an optimization of the case when there is no update clause for the merge. - * We perform an left anti join on the source data to find the rows to be inserted. - * - * This will currently only optimize for the case when there is a _single_ notMatchedClause. - */ - private def writeInsertsOnlyWhenNoMatchedClauses( - spark: SparkSession, - deltaTxn: OptimisticTransaction - ): Seq[FileAction] = recordMergeOperation(sqlMetricName = "rewriteTimeMs") { - - // UDFs to update metrics - val incrSourceRowCountExpr = makeMetricUpdateUDF("numSourceRows") - val incrInsertedCountExpr = makeMetricUpdateUDF("numTargetRowsInserted") - - val outputColNames = getTargetOutputCols(deltaTxn).map(_.name) - // we use head here since we know there is only a single notMatchedClause - val outputExprs = notMatchedClauses.head.resolvedActions.map(_.expr) - val outputCols = outputExprs.zip(outputColNames).map { case (expr, name) => - new Column(Alias(expr, name)()) - } - - // source DataFrame - val sourceDF = getSourceDF() - .filter(new Column(incrSourceRowCountExpr)) - .filter(new Column(notMatchedClauses.head.condition.getOrElse(Literal.TrueLiteral))) - - // Skip data based on the merge condition - val conjunctivePredicates = splitConjunctivePredicates(condition) - val targetOnlyPredicates = - conjunctivePredicates.filter(_.references.subsetOf(target.outputSet)) - val dataSkippedFiles = deltaTxn.filterFiles(targetOnlyPredicates) - - // target DataFrame - val targetDF = buildTargetPlanWithFiles(spark, deltaTxn, dataSkippedFiles) - - val insertDf = sourceDF.join(targetDF, new Column(condition), "leftanti") - .select(outputCols: _*) - .filter(new Column(incrInsertedCountExpr)) - - val newFiles = deltaTxn - .writeFiles(repartitionIfNeeded(spark, insertDf, deltaTxn.metadata.partitionColumns)) - .filter { - // In some cases (e.g. insert-only when all rows are matched, insert-only with an empty - // source, insert-only with an unsatisfied condition) we can write out an empty insertDf. - // This is hard to catch before the write without collecting the DF ahead of time. Instead, - // we can just accept only the AddFiles that actually add rows or - // when we don't know the number of records - case a: AddFile => a.numLogicalRecords.forall(_ > 0) - case _ => true - } - - // Update metrics - metrics("numTargetFilesBeforeSkipping") += deltaTxn.snapshot.numOfFiles - metrics("numTargetBytesBeforeSkipping") += deltaTxn.snapshot.sizeInBytes - val (afterSkippingBytes, afterSkippingPartitions) = - totalBytesAndDistinctPartitionValues(dataSkippedFiles) - metrics("numTargetFilesAfterSkipping") += dataSkippedFiles.size - metrics("numTargetBytesAfterSkipping") += afterSkippingBytes - metrics("numTargetPartitionsAfterSkipping") += afterSkippingPartitions - metrics("numTargetFilesRemoved") += 0 - metrics("numTargetBytesRemoved") += 0 - metrics("numTargetPartitionsRemovedFrom") += 0 - val (addedBytes, addedPartitions) = totalBytesAndDistinctPartitionValues(newFiles) - metrics("numTargetFilesAdded") += newFiles.count(_.isInstanceOf[AddFile]) - metrics("numTargetBytesAdded") += addedBytes - metrics("numTargetPartitionsAddedTo") += addedPartitions - newFiles - } - - /** - * Write new files by reading the touched files and updating/inserting data using the source - * query/table. This is implemented using a full|right-outer-join using the merge condition. - * - * Note that unlike the insert-only code paths with just one control column INCR_ROW_COUNT_COL, - * this method has two additional control columns ROW_DROPPED_COL for dropping deleted rows and - * CDC_TYPE_COL_NAME used for handling CDC when enabled. - */ - private def writeAllChanges( - spark: SparkSession, - deltaTxn: OptimisticTransaction, - filesToRewrite: Seq[AddFile] - ): Seq[FileAction] = recordMergeOperation(sqlMetricName = "rewriteTimeMs") { - import org.apache.spark.sql.catalyst.expressions.Literal.{TrueLiteral, FalseLiteral} - - val cdcEnabled = DeltaConfigs.CHANGE_DATA_FEED.fromMetaData(deltaTxn.metadata) - - var targetOutputCols = getTargetOutputCols(deltaTxn) - var outputRowSchema = deltaTxn.metadata.schema - - // When we have duplicate matches (only allowed when the whenMatchedCondition is a delete with - // no match condition) we will incorrectly generate duplicate CDC rows. - // Duplicate matches can be due to: - // - Duplicate rows in the source w.r.t. the merge condition - // - A target-only or source-only merge condition, which essentially turns our join into a cross - // join with the target/source satisfiying the merge condition. - // These duplicate matches are dropped from the main data output since this is a delete - // operation, but the duplicate CDC rows are not removed by default. - // See https://github.com/delta-io/delta/issues/1274 - - // We address this specific scenario by adding row ids to the target before performing our join. - // There should only be one CDC delete row per target row so we can use these row ids to dedupe - // the duplicate CDC delete rows. - - // We also need to address the scenario when there are duplicate matches with delete and we - // insert duplicate rows. Here we need to additionally add row ids to the source before the - // join to avoid dropping these valid duplicate inserted rows and their corresponding cdc rows. - - // When there is an insert clause, we set SOURCE_ROW_ID_COL=null for all delete rows because we - // need to drop the duplicate matches. - val isDeleteWithDuplicateMatchesAndCdc = multipleMatchDeleteOnlyOvercount.nonEmpty && cdcEnabled - - // Generate a new target dataframe that has same output attributes exprIds as the target plan. - // This allows us to apply the existing resolved update/insert expressions. - val baseTargetDF = buildTargetPlanWithFiles(spark, deltaTxn, filesToRewrite) - val joinType = if (hasNoInserts && - spark.conf.get(DeltaSQLConf.MERGE_MATCHED_ONLY_ENABLED)) { - "rightOuter" - } else { - "fullOuter" - } - - logDebug(s"""writeAllChanges using $joinType join: - | source.output: ${source.outputSet} - | target.output: ${target.outputSet} - | condition: $condition - | newTarget.output: ${baseTargetDF.queryExecution.logical.outputSet} - """.stripMargin) - - // UDFs to update metrics - val incrSourceRowCountExpr = makeMetricUpdateUDF("numSourceRowsInSecondScan") - val incrUpdatedCountExpr = makeMetricUpdateUDF("numTargetRowsUpdated") - val incrUpdatedMatchedCountExpr = makeMetricUpdateUDF("numTargetRowsMatchedUpdated") - val incrUpdatedNotMatchedBySourceCountExpr = - makeMetricUpdateUDF("numTargetRowsNotMatchedBySourceUpdated") - val incrInsertedCountExpr = makeMetricUpdateUDF("numTargetRowsInserted") - val incrNoopCountExpr = makeMetricUpdateUDF("numTargetRowsCopied") - val incrDeletedCountExpr = makeMetricUpdateUDF("numTargetRowsDeleted") - val incrDeletedMatchedCountExpr = makeMetricUpdateUDF("numTargetRowsMatchedDeleted") - val incrDeletedNotMatchedBySourceCountExpr = - makeMetricUpdateUDF("numTargetRowsNotMatchedBySourceDeleted") - - // Apply an outer join to find both, matches and non-matches. We are adding two boolean fields - // with value `true`, one to each side of the join. Whether this field is null or not after - // the outer join, will allow us to identify whether the resultant joined row was a - // matched inner result or an unmatched result with null on one side. - // We add row IDs to the targetDF if we have a delete-when-matched clause with duplicate - // matches and CDC is enabled, and additionally add row IDs to the source if we also have an - // insert clause. See above at isDeleteWithDuplicateMatchesAndCdc definition for more details. - var sourceDF = getSourceDF() - .withColumn(SOURCE_ROW_PRESENT_COL, new Column(incrSourceRowCountExpr)) - var targetDF = baseTargetDF - .withColumn(TARGET_ROW_PRESENT_COL, lit(true)) - if (isDeleteWithDuplicateMatchesAndCdc) { - targetDF = targetDF.withColumn(TARGET_ROW_ID_COL, monotonically_increasing_id()) - if (notMatchedClauses.nonEmpty) { // insert clause - sourceDF = sourceDF.withColumn(SOURCE_ROW_ID_COL, monotonically_increasing_id()) - } - } - val joinedDF = sourceDF.join(targetDF, new Column(condition), joinType) - val joinedPlan = joinedDF.queryExecution.analyzed - - def resolveOnJoinedPlan(exprs: Seq[Expression]): Seq[Expression] = { - tryResolveReferencesForExpressions(spark, exprs, joinedPlan) - } - - // ==== Generate the expressions to process full-outer join output and generate target rows ==== - // If there are N columns in the target table, there will be N + 3 columns after processing - // - N columns for target table - // - ROW_DROPPED_COL to define whether the generated row should dropped or written - // - INCR_ROW_COUNT_COL containing a UDF to update the output row row counter - // - CDC_TYPE_COLUMN_NAME containing the type of change being performed in a particular row - - // To generate these N + 3 columns, we will generate N + 3 expressions and apply them to the - // rows in the joinedDF. The CDC column will be either used for CDC generation or dropped before - // performing the final write, and the other two will always be dropped after executing the - // metrics UDF and filtering on ROW_DROPPED_COL. - - // We produce rows for both the main table data (with CDC_TYPE_COLUMN_NAME = CDC_TYPE_NOT_CDC), - // and rows for the CDC data which will be output to CDCReader.CDC_LOCATION. - // See [[CDCReader]] for general details on how partitioning on the CDC type column works. - - // In the following functions `updateOutput`, `deleteOutput` and `insertOutput`, we - // produce a Seq[Expression] for each intended output row. - // Depending on the clause and whether CDC is enabled, we output between 0 and 3 rows, as a - // Seq[Seq[Expression]] - - // There is one corner case outlined above at isDeleteWithDuplicateMatchesAndCdc definition. - // When we have a delete-ONLY merge with duplicate matches we have N + 4 columns: - // N target cols, TARGET_ROW_ID_COL, ROW_DROPPED_COL, INCR_ROW_COUNT_COL, CDC_TYPE_COLUMN_NAME - // When we have a delete-when-matched merge with duplicate matches + an insert clause, we have - // N + 5 columns: - // N target cols, TARGET_ROW_ID_COL, SOURCE_ROW_ID_COL, ROW_DROPPED_COL, INCR_ROW_COUNT_COL, - // CDC_TYPE_COLUMN_NAME - // These ROW_ID_COL will always be dropped before the final write. - - if (isDeleteWithDuplicateMatchesAndCdc) { - targetOutputCols = targetOutputCols :+ UnresolvedAttribute(TARGET_ROW_ID_COL) - outputRowSchema = outputRowSchema.add(TARGET_ROW_ID_COL, DataTypes.LongType) - if (notMatchedClauses.nonEmpty) { // there is an insert clause, make SRC_ROW_ID_COL=null - targetOutputCols = targetOutputCols :+ Alias(Literal(null), SOURCE_ROW_ID_COL)() - outputRowSchema = outputRowSchema.add(SOURCE_ROW_ID_COL, DataTypes.LongType) - } - } - - if (cdcEnabled) { - outputRowSchema = outputRowSchema - .add(ROW_DROPPED_COL, DataTypes.BooleanType) - .add(INCR_ROW_COUNT_COL, DataTypes.BooleanType) - .add(CDC_TYPE_COLUMN_NAME, DataTypes.StringType) - } - - def updateOutput(resolvedActions: Seq[DeltaMergeAction], incrMetricExpr: Expression) - : Seq[Seq[Expression]] = { - val updateExprs = { - // Generate update expressions and set ROW_DELETED_COL = false and - // CDC_TYPE_COLUMN_NAME = CDC_TYPE_NOT_CDC - val mainDataOutput = resolvedActions.map(_.expr) :+ FalseLiteral :+ - incrMetricExpr :+ CDC_TYPE_NOT_CDC - if (cdcEnabled) { - // For update preimage, we have do a no-op copy with ROW_DELETED_COL = false and - // CDC_TYPE_COLUMN_NAME = CDC_TYPE_UPDATE_PREIMAGE and INCR_ROW_COUNT_COL as a no-op - // (because the metric will be incremented in `mainDataOutput`) - val preImageOutput = targetOutputCols :+ FalseLiteral :+ TrueLiteral :+ - Literal(CDC_TYPE_UPDATE_PREIMAGE) - // For update postimage, we have the same expressions as for mainDataOutput but with - // INCR_ROW_COUNT_COL as a no-op (because the metric will be incremented in - // `mainDataOutput`), and CDC_TYPE_COLUMN_NAME = CDC_TYPE_UPDATE_POSTIMAGE - val postImageOutput = mainDataOutput.dropRight(2) :+ TrueLiteral :+ - Literal(CDC_TYPE_UPDATE_POSTIMAGE) - Seq(mainDataOutput, preImageOutput, postImageOutput) - } else { - Seq(mainDataOutput) - } - } - updateExprs.map(resolveOnJoinedPlan) - } - - def deleteOutput(incrMetricExpr: Expression): Seq[Seq[Expression]] = { - val deleteExprs = { - // Generate expressions to set the ROW_DELETED_COL = true and CDC_TYPE_COLUMN_NAME = - // CDC_TYPE_NOT_CDC - val mainDataOutput = targetOutputCols :+ TrueLiteral :+ incrMetricExpr :+ - CDC_TYPE_NOT_CDC - if (cdcEnabled) { - // For delete we do a no-op copy with ROW_DELETED_COL = false, INCR_ROW_COUNT_COL as a - // no-op (because the metric will be incremented in `mainDataOutput`) and - // CDC_TYPE_COLUMN_NAME = CDC_TYPE_DELETE - val deleteCdcOutput = targetOutputCols :+ FalseLiteral :+ TrueLiteral :+ CDC_TYPE_DELETE - Seq(mainDataOutput, deleteCdcOutput) - } else { - Seq(mainDataOutput) - } - } - deleteExprs.map(resolveOnJoinedPlan) - } - - def insertOutput(resolvedActions: Seq[DeltaMergeAction], incrMetricExpr: Expression) - : Seq[Seq[Expression]] = { - // Generate insert expressions and set ROW_DELETED_COL = false and - // CDC_TYPE_COLUMN_NAME = CDC_TYPE_NOT_CDC - val insertExprs = resolvedActions.map(_.expr) - val mainDataOutput = resolveOnJoinedPlan( - if (isDeleteWithDuplicateMatchesAndCdc) { - // Must be delete-when-matched merge with duplicate matches + insert clause - // Therefore we must keep the target row id and source row id. Since this is a not-matched - // clause we know the target row-id will be null. See above at - // isDeleteWithDuplicateMatchesAndCdc definition for more details. - insertExprs :+ - Alias(Literal(null), TARGET_ROW_ID_COL)() :+ UnresolvedAttribute(SOURCE_ROW_ID_COL) :+ - FalseLiteral :+ incrMetricExpr :+ CDC_TYPE_NOT_CDC - } else { - insertExprs :+ FalseLiteral :+ incrMetricExpr :+ CDC_TYPE_NOT_CDC - } - ) - if (cdcEnabled) { - // For insert we have the same expressions as for mainDataOutput, but with - // INCR_ROW_COUNT_COL as a no-op (because the metric will be incremented in - // `mainDataOutput`), and CDC_TYPE_COLUMN_NAME = CDC_TYPE_INSERT - val insertCdcOutput = mainDataOutput.dropRight(2) :+ TrueLiteral :+ Literal(CDC_TYPE_INSERT) - Seq(mainDataOutput, insertCdcOutput) - } else { - Seq(mainDataOutput) - } - } - - def clauseOutput(clause: DeltaMergeIntoClause): Seq[Seq[Expression]] = clause match { - case u: DeltaMergeIntoMatchedUpdateClause => - updateOutput(u.resolvedActions, And(incrUpdatedCountExpr, incrUpdatedMatchedCountExpr)) - case _: DeltaMergeIntoMatchedDeleteClause => - deleteOutput(And(incrDeletedCountExpr, incrDeletedMatchedCountExpr)) - case i: DeltaMergeIntoNotMatchedInsertClause => - insertOutput(i.resolvedActions, incrInsertedCountExpr) - case u: DeltaMergeIntoNotMatchedBySourceUpdateClause => - updateOutput( - u.resolvedActions, - And(incrUpdatedCountExpr, incrUpdatedNotMatchedBySourceCountExpr)) - case _: DeltaMergeIntoNotMatchedBySourceDeleteClause => - deleteOutput(And(incrDeletedCountExpr, incrDeletedNotMatchedBySourceCountExpr)) - } - - def clauseCondition(clause: DeltaMergeIntoClause): Expression = { - // if condition is None, then expression always evaluates to true - val condExpr = clause.condition.getOrElse(TrueLiteral) - resolveOnJoinedPlan(Seq(condExpr)).head - } - - val joinedRowEncoder = RowEncoder(joinedPlan.schema) - val outputRowEncoder = RowEncoder(outputRowSchema).resolveAndBind() - - val processor = new JoinedRowProcessor( - targetRowHasNoMatch = resolveOnJoinedPlan(Seq(col(SOURCE_ROW_PRESENT_COL).isNull.expr)).head, - sourceRowHasNoMatch = resolveOnJoinedPlan(Seq(col(TARGET_ROW_PRESENT_COL).isNull.expr)).head, - matchedConditions = matchedClauses.map(clauseCondition), - matchedOutputs = matchedClauses.map(clauseOutput), - notMatchedConditions = notMatchedClauses.map(clauseCondition), - notMatchedOutputs = notMatchedClauses.map(clauseOutput), - notMatchedBySourceConditions = notMatchedBySourceClauses.map(clauseCondition), - notMatchedBySourceOutputs = notMatchedBySourceClauses.map(clauseOutput), - noopCopyOutput = - resolveOnJoinedPlan(targetOutputCols :+ FalseLiteral :+ incrNoopCountExpr :+ - CDC_TYPE_NOT_CDC), - deleteRowOutput = - resolveOnJoinedPlan(targetOutputCols :+ TrueLiteral :+ TrueLiteral :+ CDC_TYPE_NOT_CDC), - joinedAttributes = joinedPlan.output, - joinedRowEncoder = joinedRowEncoder, - outputRowEncoder = outputRowEncoder) - - var outputDF = - Dataset.ofRows(spark, joinedPlan).mapPartitions(processor.processPartition)(outputRowEncoder) - - if (isDeleteWithDuplicateMatchesAndCdc) { - // When we have a delete when matched clause with duplicate matches we have to remove - // duplicate CDC rows. This scenario is further explained at - // isDeleteWithDuplicateMatchesAndCdc definition. - - // To remove duplicate CDC rows generated by the duplicate matches we dedupe by - // TARGET_ROW_ID_COL since there should only be one CDC delete row per target row. - // When there is an insert clause in addition to the delete clause we additionally dedupe by - // SOURCE_ROW_ID_COL and CDC_TYPE_COLUMN_NAME to avoid dropping valid duplicate inserted rows - // and their corresponding CDC rows. - val columnsToDedupeBy = if (notMatchedClauses.nonEmpty) { // insert clause - Seq(TARGET_ROW_ID_COL, SOURCE_ROW_ID_COL, CDC_TYPE_COLUMN_NAME) - } else { - Seq(TARGET_ROW_ID_COL) - } - outputDF = outputDF - .dropDuplicates(columnsToDedupeBy) - .drop(ROW_DROPPED_COL, INCR_ROW_COUNT_COL, TARGET_ROW_ID_COL, SOURCE_ROW_ID_COL) - } else { - outputDF = outputDF.drop(ROW_DROPPED_COL, INCR_ROW_COUNT_COL) - } - - logDebug("writeAllChanges: join output plan:\n" + outputDF.queryExecution) - - // Write to Delta - val newFiles = deltaTxn - .writeFiles(repartitionIfNeeded(spark, outputDF, deltaTxn.metadata.partitionColumns)) - - // Update metrics - val (addedBytes, addedPartitions) = totalBytesAndDistinctPartitionValues(newFiles) - metrics("numTargetFilesAdded") += newFiles.count(_.isInstanceOf[AddFile]) - metrics("numTargetChangeFilesAdded") += newFiles.count(_.isInstanceOf[AddCDCFile]) - metrics("numTargetChangeFileBytes") += newFiles.collect{ case f: AddCDCFile => f.size }.sum - metrics("numTargetBytesAdded") += addedBytes - metrics("numTargetPartitionsAddedTo") += addedPartitions - if (multipleMatchDeleteOnlyOvercount.isDefined) { - // Compensate for counting duplicates during the query. - val actualRowsDeleted = - metrics("numTargetRowsDeleted").value - multipleMatchDeleteOnlyOvercount.get - assert(actualRowsDeleted >= 0) - metrics("numTargetRowsDeleted").set(actualRowsDeleted) - val actualRowsMatchedDeleted = - metrics("numTargetRowsMatchedDeleted").value - multipleMatchDeleteOnlyOvercount.get - assert(actualRowsMatchedDeleted >= 0) - metrics("numTargetRowsMatchedDeleted").set(actualRowsMatchedDeleted) - } - - newFiles - } - - - /** - * Build a new logical plan using the given `files` that has the same output columns (exprIds) - * as the `target` logical plan, so that existing update/insert expressions can be applied - * on this new plan. - */ - private def buildTargetPlanWithFiles( - spark: SparkSession, - deltaTxn: OptimisticTransaction, - files: Seq[AddFile]): DataFrame = { - val targetOutputCols = getTargetOutputCols(deltaTxn) - val targetOutputColsMap = { - val colsMap: Map[String, NamedExpression] = targetOutputCols.view - .map(col => col.name -> col).toMap - if (conf.caseSensitiveAnalysis) { - colsMap - } else { - CaseInsensitiveMap(colsMap) - } - } - - val plan = { - // We have to do surgery to use the attributes from `targetOutputCols` to scan the table. - // In cases of schema evolution, they may not be the same type as the original attributes. - val original = - deltaTxn.deltaLog.createDataFrame(deltaTxn.snapshot, files).queryExecution.analyzed - val transformed = original.transform { - case LogicalRelation(base, output, catalogTbl, isStreaming) => - LogicalRelation( - base, - // We can ignore the new columns which aren't yet AttributeReferences. - targetOutputCols.collect { case a: AttributeReference => a }, - catalogTbl, - isStreaming) - } - - // In case of schema evolution & column mapping, we would also need to rebuild the file format - // because under column mapping, the reference schema within DeltaParquetFileFormat - // that is used to populate metadata needs to be updated - if (deltaTxn.metadata.columnMappingMode != NoMapping) { - val updatedFileFormat = deltaTxn.deltaLog.fileFormat(deltaTxn.metadata) - DeltaTableUtils.replaceFileFormat(transformed, updatedFileFormat) - } else { - transformed - } - } - - // For each plan output column, find the corresponding target output column (by name) and - // create an alias - val aliases = plan.output.map { - case newAttrib: AttributeReference => - val existingTargetAttrib = targetOutputColsMap.get(newAttrib.name) - .getOrElse { - throw DeltaErrors.failedFindAttributeInOutputColumns( - newAttrib.name, targetOutputCols.mkString(",")) - }.asInstanceOf[AttributeReference] - - if (existingTargetAttrib.exprId == newAttrib.exprId) { - // It's not valid to alias an expression to its own exprId (this is considered a - // non-unique exprId by the analyzer), so we just use the attribute directly. - newAttrib - } else { - Alias(newAttrib, existingTargetAttrib.name)(exprId = existingTargetAttrib.exprId) - } - } - - Dataset.ofRows(spark, Project(aliases, plan)) - } - - /** Expressions to increment SQL metrics */ - private def makeMetricUpdateUDF(name: String): Expression = { - // only capture the needed metric in a local variable - val metric = metrics(name) - DeltaUDF.boolean { () => metric += 1; true }.asNondeterministic().apply().expr - } - - private def getTargetOutputCols(txn: OptimisticTransaction): Seq[NamedExpression] = { - txn.metadata.schema.map { col => - targetOutputAttributesMap - .get(col.name) - .map { a => - AttributeReference(col.name, col.dataType, col.nullable)(a.exprId) - } - .getOrElse(Alias(Literal(null), col.name)() - ) - } - } - - /** - * Repartitions the output DataFrame by the partition columns if table is partitioned - * and `merge.repartitionBeforeWrite.enabled` is set to true. - */ - protected def repartitionIfNeeded( - spark: SparkSession, - df: DataFrame, - partitionColumns: Seq[String]): DataFrame = { - if (partitionColumns.nonEmpty && spark.conf.get(DeltaSQLConf.MERGE_REPARTITION_BEFORE_WRITE)) { - df.repartition(partitionColumns.map(col): _*) - } else { - df - } - } - - /** - * Execute the given `thunk` and return its result while recording the time taken to do it. - * - * @param sqlMetricName name of SQL metric to update with the time taken by the thunk - * @param thunk the code to execute - */ - private def recordMergeOperation[A](sqlMetricName: String = null)(thunk: => A): A = { - val startTimeNs = System.nanoTime() - val r = thunk - val timeTakenMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNs) - if (sqlMetricName != null && timeTakenMs > 0) { - metrics(sqlMetricName) += timeTakenMs - } - r - } -} - -object MergeIntoCommand { - /** - * Spark UI will track all normal accumulators along with Spark tasks to show them on Web UI. - * However, the accumulator used by `MergeIntoCommand` can store a very large value since it - * tracks all files that need to be rewritten. We should ask Spark UI to not remember it, - * otherwise, the UI data may consume lots of memory. Hence, we use the prefix `internal.metrics.` - * to make this accumulator become an internal accumulator, so that it will not be tracked by - * Spark UI. - */ - val TOUCHED_FILES_ACCUM_NAME = "internal.metrics.MergeIntoDelta.touchedFiles" - - val ROW_ID_COL = "_row_id_" - val TARGET_ROW_ID_COL = "_target_row_id_" - val SOURCE_ROW_ID_COL = "_source_row_id_" - val FILE_NAME_COL = "_file_name_" - val SOURCE_ROW_PRESENT_COL = "_source_row_present_" - val TARGET_ROW_PRESENT_COL = "_target_row_present_" - val ROW_DROPPED_COL = "_row_dropped_" - val INCR_ROW_COUNT_COL = "_incr_row_count_" - - /** - * @param targetRowHasNoMatch whether a joined row is a target row with no match in the source - * table - * @param sourceRowHasNoMatch whether a joined row is a source row with no match in the target - * table - * @param matchedConditions condition for each match clause - * @param matchedOutputs corresponding output for each match clause. for each clause, we - * have 1-3 output rows, each of which is a sequence of expressions - * to apply to the joined row - * @param notMatchedConditions condition for each not-matched clause - * @param notMatchedOutputs corresponding output for each not-matched clause. for each clause, - * we have 1-2 output rows, each of which is a sequence of - * expressions to apply to the joined row - * @param notMatchedBySourceConditions condition for each not-matched-by-source clause - * @param notMatchedBySourceOutputs corresponding output for each not-matched-by-source - * clause. for each clause, we have 1-3 output rows, each of - * which is a sequence of expressions to apply to the joined - * row - * @param noopCopyOutput no-op expression to copy a target row to the output - * @param deleteRowOutput expression to drop a row from the final output. this is used for - * source rows that don't match any not-matched clauses - * @param joinedAttributes schema of our outer-joined dataframe - * @param joinedRowEncoder joinedDF row encoder - * @param outputRowEncoder final output row encoder - */ - class JoinedRowProcessor( - targetRowHasNoMatch: Expression, - sourceRowHasNoMatch: Expression, - matchedConditions: Seq[Expression], - matchedOutputs: Seq[Seq[Seq[Expression]]], - notMatchedConditions: Seq[Expression], - notMatchedOutputs: Seq[Seq[Seq[Expression]]], - notMatchedBySourceConditions: Seq[Expression], - notMatchedBySourceOutputs: Seq[Seq[Seq[Expression]]], - noopCopyOutput: Seq[Expression], - deleteRowOutput: Seq[Expression], - joinedAttributes: Seq[Attribute], - joinedRowEncoder: ExpressionEncoder[Row], - outputRowEncoder: ExpressionEncoder[Row]) extends Serializable { - - private def generateProjection(exprs: Seq[Expression]): UnsafeProjection = { - UnsafeProjection.create(exprs, joinedAttributes) - } - - private def generatePredicate(expr: Expression): BasePredicate = { - GeneratePredicate.generate(expr, joinedAttributes) - } - - def processPartition(rowIterator: Iterator[Row]): Iterator[Row] = { - - val targetRowHasNoMatchPred = generatePredicate(targetRowHasNoMatch) - val sourceRowHasNoMatchPred = generatePredicate(sourceRowHasNoMatch) - val matchedPreds = matchedConditions.map(generatePredicate) - val matchedProjs = matchedOutputs.map(_.map(generateProjection)) - val notMatchedPreds = notMatchedConditions.map(generatePredicate) - val notMatchedProjs = notMatchedOutputs.map(_.map(generateProjection)) - val notMatchedBySourcePreds = notMatchedBySourceConditions.map(generatePredicate) - val notMatchedBySourceProjs = notMatchedBySourceOutputs.map(_.map(generateProjection)) - val noopCopyProj = generateProjection(noopCopyOutput) - val deleteRowProj = generateProjection(deleteRowOutput) - val outputProj = UnsafeProjection.create(outputRowEncoder.schema) - - // this is accessing ROW_DROPPED_COL. If ROW_DROPPED_COL is not in outputRowEncoder.schema - // then CDC must be disabled and it's the column after our output cols - def shouldDeleteRow(row: InternalRow): Boolean = { - row.getBoolean( - outputRowEncoder.schema.getFieldIndex(ROW_DROPPED_COL) - .getOrElse(outputRowEncoder.schema.fields.size) - ) - } - - def processRow(inputRow: InternalRow): Iterator[InternalRow] = { - // Identify which set of clauses to execute: matched, not-matched or not-matched-by-source - val (predicates, projections, noopAction) = if (targetRowHasNoMatchPred.eval(inputRow)) { - // Target row did not match any source row, so update the target row. - (notMatchedBySourcePreds, notMatchedBySourceProjs, noopCopyProj) - } else if (sourceRowHasNoMatchPred.eval(inputRow)) { - // Source row did not match with any target row, so insert the new source row - (notMatchedPreds, notMatchedProjs, deleteRowProj) - } else { - // Source row matched with target row, so update the target row - (matchedPreds, matchedProjs, noopCopyProj) - } - - // find (predicate, projection) pair whose predicate satisfies inputRow - val pair = (predicates zip projections).find { - case (predicate, _) => predicate.eval(inputRow) - } - - pair match { - case Some((_, projections)) => - projections.map(_.apply(inputRow)).iterator - case None => Iterator(noopAction.apply(inputRow)) - } - } - - val toRow = joinedRowEncoder.createSerializer() - val fromRow = outputRowEncoder.createDeserializer() - rowIterator - .map(toRow) - .flatMap(processRow) - .filter(!shouldDeleteRow(_)) - .map { notDeletedInternalRow => - fromRow(outputProj(notDeletedInternalRow)) - } - } - } - - /** Count the number of distinct partition values among the AddFiles in the given set. */ - def totalBytesAndDistinctPartitionValues(files: Seq[FileAction]): (Long, Int) = { - val distinctValues = new mutable.HashSet[Map[String, String]]() - var bytes = 0L - val iter = files.collect { case a: AddFile => a }.iterator - while (iter.hasNext) { - val file = iter.next() - distinctValues += file.partitionValues - bytes += file.size - } - // If the only distinct value map is an empty map, then it must be an unpartitioned table. - // Return 0 in that case. - val numDistinctValues = - if (distinctValues.size == 1 && distinctValues.head.isEmpty) 0 else distinctValues.size - (bytes, numDistinctValues) - } -} diff --git a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala b/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala deleted file mode 100644 index 7fa2c97d900..00000000000 --- a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala +++ /dev/null @@ -1,501 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.delta.commands - -import java.util.ConcurrentModificationException - -import scala.collection.mutable.ArrayBuffer - -import org.apache.spark.sql.delta.skipping.MultiDimClustering -import org.apache.spark.sql.delta._ -import org.apache.spark.sql.delta.DeltaOperations.Operation -import org.apache.spark.sql.delta.actions.{Action, AddFile, DeletionVectorDescriptor, FileAction, RemoveFile} -import org.apache.spark.sql.delta.commands.OptimizeTableCommandOverwrites.{getDeltaLogClickhouse, groupFilesIntoBinsClickhouse, runOptimizeBinJobClickhouse} -import org.apache.spark.sql.delta.commands.optimize._ -import org.apache.spark.sql.delta.files.SQLMetricsReporting -import org.apache.spark.sql.delta.schema.SchemaUtils -import org.apache.spark.sql.delta.sources.DeltaSQLConf - -import org.apache.spark.SparkContext -import org.apache.spark.SparkContext.SPARK_JOB_GROUP_ID -import org.apache.spark.sql.{AnalysisException, Encoders, Row, SparkSession} -import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute -import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Expression} -import org.apache.spark.sql.execution.command.{LeafRunnableCommand, RunnableCommand} -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig -import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.AddMergeTreeParts -import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils -import org.apache.spark.sql.execution.metric.SQLMetric -import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric -import org.apache.spark.sql.types._ -import org.apache.spark.util.{SystemClock, ThreadUtils} - -/** - * Gluten overwrite Delta: - * - * This file is copied from Delta 2.3.0. It is modified in: - * 1. getDeltaLogClickhouse - * 2. runOptimizeBinJobClickhouse - * 3. groupFilesIntoBinsClickhouse - */ - -/** Base class defining abstract optimize command */ -abstract class OptimizeTableCommandBase extends RunnableCommand with DeltaCommand { - - override val output: Seq[Attribute] = Seq( - AttributeReference("path", StringType)(), - AttributeReference("metrics", Encoders.product[OptimizeMetrics].schema)()) - - /** - * Validates ZOrderBy columns - * - validates that partitions columns are not used in `unresolvedZOrderByCols` - * - validates that we already collect stats for all the columns used in `unresolvedZOrderByCols` - * - * @param spark [[SparkSession]] to use - * @param txn the [[OptimisticTransaction]] being used to optimize - * @param unresolvedZOrderByCols Seq of [[UnresolvedAttribute]] corresponding to zOrderBy columns - */ - def validateZorderByColumns( - spark: SparkSession, - txn: OptimisticTransaction, - unresolvedZOrderByCols: Seq[UnresolvedAttribute]): Unit = { - if (unresolvedZOrderByCols.isEmpty) return - val metadata = txn.snapshot.metadata - val partitionColumns = metadata.partitionColumns.toSet - val dataSchema = - StructType(metadata.schema.filterNot(c => partitionColumns.contains(c.name))) - val df = spark.createDataFrame(new java.util.ArrayList[Row](), dataSchema) - val checkColStat = spark.sessionState.conf.getConf( - DeltaSQLConf.DELTA_OPTIMIZE_ZORDER_COL_STAT_CHECK) - val statCollectionSchema = txn.snapshot.statCollectionSchema - val colsWithoutStats = ArrayBuffer[String]() - - unresolvedZOrderByCols.foreach { colAttribute => - val colName = colAttribute.name - if (checkColStat) { - try { - SchemaUtils.findColumnPosition(colAttribute.nameParts, statCollectionSchema) - } catch { - case e: AnalysisException if e.getMessage.contains("Couldn't find column") => - colsWithoutStats.append(colName) - } - } - val isNameEqual = spark.sessionState.conf.resolver - if (partitionColumns.find(isNameEqual(_, colName)).nonEmpty) { - throw DeltaErrors.zOrderingOnPartitionColumnException(colName) - } - if (df.queryExecution.analyzed.resolve(colAttribute.nameParts, isNameEqual).isEmpty) { - throw DeltaErrors.zOrderingColumnDoesNotExistException(colName) - } - } - if (checkColStat && colsWithoutStats.nonEmpty) { - throw DeltaErrors.zOrderingOnColumnWithNoStatsException( - colsWithoutStats.toSeq, spark) - } - } -} - -/** - * The `optimize` command implementation for Spark SQL. Example SQL: - * {{{ - * OPTIMIZE ('/path/to/dir' | delta.table) [WHERE part = 25]; - * }}} - */ -case class OptimizeTableCommand( - path: Option[String], - tableId: Option[TableIdentifier], - userPartitionPredicates: Seq[String], - options: Map[String, String])(val zOrderBy: Seq[UnresolvedAttribute]) - extends OptimizeTableCommandBase with LeafRunnableCommand { - - override val otherCopyArgs: Seq[AnyRef] = zOrderBy :: Nil - - override def run(sparkSession: SparkSession): Seq[Row] = { - // --- modified start - CHDataSourceUtils.ensureClickHouseTableV2(tableId, sparkSession) - val deltaLog = getDeltaLogClickhouse(sparkSession, path, tableId, "OPTIMIZE", options) - // --- modified end - - val txn = deltaLog.startTransaction() - if (txn.readVersion == -1) { - throw DeltaErrors.notADeltaTableException(deltaLog.dataPath.toString) - } - - val partitionColumns = txn.snapshot.metadata.partitionColumns - // Parse the predicate expression into Catalyst expression and verify only simple filters - // on partition columns are present - - val partitionPredicates = userPartitionPredicates.flatMap { predicate => - val predicates = parsePredicates(sparkSession, predicate) - verifyPartitionPredicates( - sparkSession, - partitionColumns, - predicates) - predicates - } - - validateZorderByColumns(sparkSession, txn, zOrderBy) - val zOrderByColumns = zOrderBy.map(_.name).toSeq - - new OptimizeExecutor(sparkSession, txn, partitionPredicates, zOrderByColumns).optimize() - } -} - -/** - * Optimize job which compacts small files into larger files to reduce - * the number of files and potentially allow more efficient reads. - * - * @param sparkSession Spark environment reference. - * @param txn The transaction used to optimize this table - * @param partitionPredicate List of partition predicates to select subset of files to optimize. - */ -class OptimizeExecutor( - sparkSession: SparkSession, - txn: OptimisticTransaction, - partitionPredicate: Seq[Expression], - zOrderByColumns: Seq[String]) - extends DeltaCommand with SQLMetricsReporting with Serializable { - - /** Timestamp to use in [[FileAction]] */ - private val operationTimestamp = new SystemClock().getTimeMillis() - - private val isMultiDimClustering = zOrderByColumns.nonEmpty - - def optimize(): Seq[Row] = { - recordDeltaOperation(txn.deltaLog, "delta.optimize") { - // --- modified start - val isMergeTreeFormat = ClickHouseConfig - .isMergeTreeFormatEngine(txn.deltaLog.unsafeVolatileMetadata.configuration) - // --- modified end - val minFileSize = sparkSession.sessionState.conf.getConf( - DeltaSQLConf.DELTA_OPTIMIZE_MIN_FILE_SIZE) - val maxFileSize = sparkSession.sessionState.conf.getConf( - DeltaSQLConf.DELTA_OPTIMIZE_MAX_FILE_SIZE) - require(minFileSize > 0, "minFileSize must be > 0") - require(maxFileSize > 0, "maxFileSize must be > 0") - - val candidateFiles = txn.filterFiles(partitionPredicate, keepNumRecords = true) - val partitionSchema = txn.metadata.partitionSchema - - val maxDeletedRowsRatio = sparkSession.sessionState.conf.getConf( - DeltaSQLConf.DELTA_OPTIMIZE_MAX_DELETED_ROWS_RATIO) - val filesToProcess = pruneCandidateFileList(minFileSize, maxDeletedRowsRatio, candidateFiles) - // --- modified start - val maxThreads = - sparkSession.sessionState.conf.getConf(DeltaSQLConf.DELTA_OPTIMIZE_MAX_THREADS) - val (updates, jobs) = if (isMergeTreeFormat) { - val partitionsToCompact = filesToProcess - .groupBy(file => (file.asInstanceOf[AddMergeTreeParts].bucketNum, file.partitionValues)) - .toSeq - val jobs = groupFilesIntoBinsClickhouse(partitionsToCompact, maxFileSize) - (ThreadUtils.parmap(jobs, "OptimizeJob", maxThreads) { partitionBinGroup => - // --- modified start - runOptimizeBinJobClickhouse( - txn, - partitionBinGroup._1._2, - partitionBinGroup._1._1, - partitionBinGroup._2, - maxFileSize) - // --- modified end - }.flatten, jobs) - } else { - val partitionsToCompact = filesToProcess.groupBy(_.partitionValues).toSeq - val jobs = groupFilesIntoBins(partitionsToCompact, maxFileSize) - (ThreadUtils.parmap(jobs, "OptimizeJob", maxThreads) { partitionBinGroup => - runOptimizeBinJob(txn, partitionBinGroup._1, partitionBinGroup._2, maxFileSize) - }.flatten, jobs) - } - // --- modified end - - val addedFiles = updates.collect { case a: AddFile => a } - val removedFiles = updates.collect { case r: RemoveFile => r } - val removedDVs = filesToProcess.filter(_.deletionVector != null).map(_.deletionVector).toSeq - if (addedFiles.size > 0) { - val operation = DeltaOperations.Optimize(partitionPredicate.map(_.sql), zOrderByColumns) - val metrics = createMetrics(sparkSession.sparkContext, addedFiles, removedFiles, removedDVs) - commitAndRetry(txn, operation, updates, metrics) { newTxn => - val newPartitionSchema = newTxn.metadata.partitionSchema - val candidateSetOld = candidateFiles.map(_.path).toSet - val candidateSetNew = newTxn.filterFiles(partitionPredicate).map(_.path).toSet - - // As long as all of the files that we compacted are still part of the table, - // and the partitioning has not changed it is valid to continue to try - // and commit this checkpoint. - if (candidateSetOld.subsetOf(candidateSetNew) && partitionSchema == newPartitionSchema) { - true - } else { - val deleted = candidateSetOld -- candidateSetNew - logWarning(s"The following compacted files were delete " + - s"during checkpoint ${deleted.mkString(",")}. Aborting the compaction.") - false - } - } - } - - val optimizeStats = OptimizeStats() - optimizeStats.addedFilesSizeStats.merge(addedFiles) - optimizeStats.removedFilesSizeStats.merge(removedFiles) - optimizeStats.numPartitionsOptimized = jobs.map(j => j._1).distinct.size - optimizeStats.numBatches = jobs.size - optimizeStats.totalConsideredFiles = candidateFiles.size - optimizeStats.totalFilesSkipped = optimizeStats.totalConsideredFiles - removedFiles.size - optimizeStats.totalClusterParallelism = sparkSession.sparkContext.defaultParallelism - val numTableColumns = txn.snapshot.metadata.schema.size - optimizeStats.numTableColumns = numTableColumns - optimizeStats.numTableColumnsWithStats = - DeltaConfigs.DATA_SKIPPING_NUM_INDEXED_COLS.fromMetaData(txn.snapshot.metadata) - .min(numTableColumns) - if (removedDVs.size > 0) { - optimizeStats.deletionVectorStats = Some(DeletionVectorStats( - numDeletionVectorsRemoved = removedDVs.size, - numDeletionVectorRowsRemoved = removedDVs.map(_.cardinality).sum)) - } - - if (isMultiDimClustering) { - val inputFileStats = - ZOrderFileStats(removedFiles.size, removedFiles.map(_.size.getOrElse(0L)).sum) - optimizeStats.zOrderStats = Some(ZOrderStats( - strategyName = "all", // means process all files in a partition - inputCubeFiles = ZOrderFileStats(0, 0), - inputOtherFiles = inputFileStats, - inputNumCubes = 0, - mergedFiles = inputFileStats, - // There will one z-cube for each partition - numOutputCubes = optimizeStats.numPartitionsOptimized)) - } - - return Seq(Row(txn.deltaLog.dataPath.toString, optimizeStats.toOptimizeMetrics)) - } - } - - /** - * Helper method to prune the list of selected files based on fileSize and ratio of - * deleted rows according to the deletion vector in [[AddFile]]. - */ - private def pruneCandidateFileList( - minFileSize: Long, maxDeletedRowsRatio: Double, files: Seq[AddFile]): Seq[AddFile] = { - - // Select all files in case of multi-dimensional clustering - if (isMultiDimClustering) return files - - def shouldCompactBecauseOfDeletedRows(file: AddFile): Boolean = { - // Always compact files with DVs but without numRecords stats. - // This may be overly aggressive, but it fixes the problem in the long-term, - // as the compacted files will have stats. - (file.deletionVector != null && file.numPhysicalRecords.isEmpty) || - file.deletedToPhysicalRecordsRatio.getOrElse(0d) > maxDeletedRowsRatio - } - - // Select files that are small or have too many deleted rows - files.filter( - addFile => addFile.size < minFileSize || shouldCompactBecauseOfDeletedRows(addFile)) - } - - /** - * Utility methods to group files into bins for optimize. - * - * @param partitionsToCompact List of files to compact group by partition. - * Partition is defined by the partition values (partCol -> partValue) - * @param maxTargetFileSize Max size (in bytes) of the compaction output file. - * @return Sequence of bins. Each bin contains one or more files from the same - * partition and targeted for one output file. - */ - private def groupFilesIntoBins( - partitionsToCompact: Seq[(Map[String, String], Seq[AddFile])], - maxTargetFileSize: Long): Seq[(Map[String, String], Seq[AddFile])] = { - partitionsToCompact.flatMap { - case (partition, files) => - val bins = new ArrayBuffer[Seq[AddFile]]() - - val currentBin = new ArrayBuffer[AddFile]() - var currentBinSize = 0L - - files.sortBy(_.size).foreach { file => - // Generally, a bin is a group of existing files, whose total size does not exceed the - // desired maxFileSize. They will be coalesced into a single output file. - // However, if isMultiDimClustering = true, all files in a partition will be read by the - // same job, the data will be range-partitioned and numFiles = totalFileSize / maxFileSize - // will be produced. See below. - if (file.size + currentBinSize > maxTargetFileSize && !isMultiDimClustering) { - bins += currentBin.toVector - currentBin.clear() - currentBin += file - currentBinSize = file.size - } else { - currentBin += file - currentBinSize += file.size - } - } - - if (currentBin.nonEmpty) { - bins += currentBin.toVector - } - - bins.filter { bin => - bin.size > 1 || // bin has more than one file or - (bin.size == 1 && bin(0).deletionVector != null) || // single file in the bin has a DV or - isMultiDimClustering // multi-clustering - }.map(b => (partition, b)) - } - } - - /** - * Utility method to run a Spark job to compact the files in given bin - * - * @param txn [[OptimisticTransaction]] instance in use to commit the changes to DeltaLog. - * @param partition Partition values of the partition that files in [[bin]] belongs to. - * @param bin List of files to compact into one large file. - * @param maxFileSize Targeted output file size in bytes - */ - private def runOptimizeBinJob( - txn: OptimisticTransaction, - partition: Map[String, String], - bin: Seq[AddFile], - maxFileSize: Long): Seq[FileAction] = { - val baseTablePath = txn.deltaLog.dataPath - - val input = txn.deltaLog.createDataFrame(txn.snapshot, bin, actionTypeOpt = Some("Optimize")) - val repartitionDF = if (isMultiDimClustering) { - val totalSize = bin.map(_.size).sum - val approxNumFiles = Math.max(1, totalSize / maxFileSize).toInt - MultiDimClustering.cluster( - input, - approxNumFiles, - zOrderByColumns) - } else { - val useRepartition = sparkSession.sessionState.conf.getConf( - DeltaSQLConf.DELTA_OPTIMIZE_REPARTITION_ENABLED) - if (useRepartition) { - input.repartition(numPartitions = 1) - } else { - input.coalesce(numPartitions = 1) - } - } - - val partitionDesc = partition.toSeq.map(entry => entry._1 + "=" + entry._2).mkString(",") - - val partitionName = if (partition.isEmpty) "" else s" in partition ($partitionDesc)" - val description = s"$baseTablePath
Optimizing ${bin.size} files" + partitionName - sparkSession.sparkContext.setJobGroup( - sparkSession.sparkContext.getLocalProperty(SPARK_JOB_GROUP_ID), - description) - - val addFiles = txn.writeFiles(repartitionDF).collect { - case a: AddFile => - a.copy(dataChange = false) - case other => - throw new IllegalStateException( - s"Unexpected action $other with type ${other.getClass}. File compaction job output" + - s"should only have AddFiles") - } - val removeFiles = bin.map(f => f.removeWithTimestamp(operationTimestamp, dataChange = false)) - val updates = addFiles ++ removeFiles - updates - } - - /** - * Attempts to commit the given actions to the log. In the case of a concurrent update, - * the given function will be invoked with a new transaction to allow custom conflict - * detection logic to indicate it is safe to try again, by returning `true`. - * - * This function will continue to try to commit to the log as long as `f` returns `true`, - * otherwise throws a subclass of [[ConcurrentModificationException]]. - */ - private def commitAndRetry( - txn: OptimisticTransaction, - optimizeOperation: Operation, - actions: Seq[Action], - metrics: Map[String, SQLMetric])(f: OptimisticTransaction => Boolean): Unit = { - try { - txn.registerSQLMetrics(sparkSession, metrics) - txn.commit(actions, optimizeOperation) - } catch { - case e: ConcurrentModificationException => - val newTxn = txn.deltaLog.startTransaction() - if (f(newTxn)) { - logInfo("Retrying commit after checking for semantic conflicts with concurrent updates.") - commitAndRetry(newTxn, optimizeOperation, actions, metrics)(f) - } else { - logWarning("Semantic conflicts detected. Aborting operation.") - throw e - } - } - } - - /** Create a map of SQL metrics for adding to the commit history. */ - private def createMetrics( - sparkContext: SparkContext, - addedFiles: Seq[AddFile], - removedFiles: Seq[RemoveFile], - removedDVs: Seq[DeletionVectorDescriptor]): Map[String, SQLMetric] = { - - def setAndReturnMetric(description: String, value: Long) = { - val metric = createMetric(sparkContext, description) - metric.set(value) - metric - } - - def totalSize(actions: Seq[FileAction]): Long = { - var totalSize = 0L - actions.foreach { file => - val fileSize = file match { - case addFile: AddFile => addFile.size - case removeFile: RemoveFile => removeFile.size.getOrElse(0L) - case default => - throw new IllegalArgumentException(s"Unknown FileAction type: ${default.getClass}") - } - totalSize += fileSize - } - totalSize - } - - val (deletionVectorRowsRemoved, deletionVectorBytesRemoved) = - removedDVs.map(dv => (dv.cardinality, dv.sizeInBytes.toLong)) - .reduceLeftOption((dv1, dv2) => (dv1._1 + dv2._1, dv1._2 + dv2._2)) - .getOrElse((0L, 0L)) - - val dvMetrics: Map[String, SQLMetric] = Map( - "numDeletionVectorsRemoved" -> - setAndReturnMetric( - "total number of deletion vectors removed", - removedDVs.size), - "numDeletionVectorRowsRemoved" -> - setAndReturnMetric( - "total number of deletion vector rows removed", - deletionVectorRowsRemoved), - "numDeletionVectorBytesRemoved" -> - setAndReturnMetric( - "total number of bytes of removed deletion vectors", - deletionVectorBytesRemoved)) - - val sizeStats = FileSizeStatsWithHistogram.create(addedFiles.map(_.size).sorted) - Map[String, SQLMetric]( - "minFileSize" -> setAndReturnMetric("minimum file size", sizeStats.get.min), - "p25FileSize" -> setAndReturnMetric("25th percentile file size", sizeStats.get.p25), - "p50FileSize" -> setAndReturnMetric("50th percentile file size", sizeStats.get.p50), - "p75FileSize" -> setAndReturnMetric("75th percentile file size", sizeStats.get.p75), - "maxFileSize" -> setAndReturnMetric("maximum file size", sizeStats.get.max), - "numAddedFiles" -> setAndReturnMetric("total number of files added.", addedFiles.size), - "numRemovedFiles" -> setAndReturnMetric("total number of files removed.", removedFiles.size), - "numAddedBytes" -> setAndReturnMetric("total number of bytes added", totalSize(addedFiles)), - "numRemovedBytes" -> - setAndReturnMetric("total number of bytes removed", totalSize(removedFiles)) - ) ++ dvMetrics - } -} diff --git a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala b/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala deleted file mode 100644 index ef8157ffafa..00000000000 --- a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala +++ /dev/null @@ -1,323 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.delta.commands - -import org.apache.gluten.memory.CHThreadGroup -import org.apache.spark.{TaskContext, TaskOutputFileAlreadyExistException} -import org.apache.spark.internal.Logging -import org.apache.spark.internal.io.FileCommitProtocol.TaskCommitMessage -import org.apache.spark.internal.io.SparkHadoopWriterUtils -import org.apache.spark.shuffle.FetchFailedException -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier} -import org.apache.spark.sql.catalyst.catalog.CatalogTableType -import org.apache.spark.sql.delta._ -import org.apache.spark.sql.delta.actions.{AddFile, FileAction} -import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 -import org.apache.spark.sql.errors.QueryExecutionErrors -import org.apache.spark.sql.execution.datasources.{CHDatasourceJniWrapper, WriteTaskResult} -import org.apache.spark.sql.execution.datasources.v1.CHMergeTreeWriterInjects -import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.{AddFileTags, AddMergeTreeParts} -import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.StructType -import org.apache.spark.util.{SerializableConfiguration, SystemClock, Utils} -import org.apache.hadoop.fs.{FileAlreadyExistsException, Path} -import org.apache.hadoop.mapreduce.{TaskAttemptContext, TaskAttemptID, TaskID, TaskType} -import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl - -import java.util.Date -import scala.collection.mutable.ArrayBuffer - -object OptimizeTableCommandOverwrites extends Logging { - - case class TaskDescription( - path: String, - database: String, - tableName: String, - snapshotId: String, - orderByKey: String, - lowCardKey: String, - minmaxIndexKey: String, - bfIndexKey: String, - setIndexKey: String, - primaryKey: String, - partitionColumns: Seq[String], - partList: Seq[String], - tableSchema: StructType, - clickhouseTableConfigs: Map[String, String], - serializableHadoopConf: SerializableConfiguration, - jobIdInstant: Long, - partitionDir: Option[String], - bucketDir: Option[String] - ) - - private def executeTask( - description: TaskDescription, - sparkStageId: Int, - sparkPartitionId: Int, - sparkAttemptNumber: Int - ): WriteTaskResult = { - CHThreadGroup.registerNewThreadGroup() - val jobId = SparkHadoopWriterUtils.createJobID(new Date(description.jobIdInstant), sparkStageId) - val taskId = new TaskID(jobId, TaskType.MAP, sparkPartitionId) - val taskAttemptId = new TaskAttemptID(taskId, sparkAttemptNumber) - - // Set up the attempt context required to use in the output committer. - val taskAttemptContext: TaskAttemptContext = { - // Set up the configuration object - val hadoopConf = description.serializableHadoopConf.value - hadoopConf.set("mapreduce.job.id", jobId.toString) - hadoopConf.set("mapreduce.task.id", taskAttemptId.getTaskID.toString) - hadoopConf.set("mapreduce.task.attempt.id", taskAttemptId.toString) - hadoopConf.setBoolean("mapreduce.task.ismap", true) - hadoopConf.setInt("mapreduce.task.partition", 0) - - new TaskAttemptContextImpl(hadoopConf, taskAttemptId) - } - - try { - Utils.tryWithSafeFinallyAndFailureCallbacks(block = { - - val planWithSplitInfo = CHMergeTreeWriterInjects.genMergeTreeWriteRel( - description.path, - description.database, - description.tableName, - description.snapshotId, - description.orderByKey, - description.lowCardKey, - description.minmaxIndexKey, - description.bfIndexKey, - description.setIndexKey, - description.primaryKey, - description.partitionColumns, - description.partList, - description.tableSchema, - description.clickhouseTableConfigs, - description.tableSchema.toAttributes - ) - - val returnedMetrics = - CHDatasourceJniWrapper.nativeMergeMTParts( - planWithSplitInfo.splitInfo, - description.partitionDir.getOrElse(""), - description.bucketDir.getOrElse("") - ) - if (returnedMetrics != null && returnedMetrics.nonEmpty) { - val addFiles = AddFileTags.partsMetricsToAddFile( - description.database, - description.tableName, - description.path, - returnedMetrics, - Seq(Utils.localHostName())) - - val (taskCommitMessage, taskCommitTime) = Utils.timeTakenMs { - // committer.commitTask(taskAttemptContext) - new TaskCommitMessage(addFiles.toSeq) - } - -// val summary = MergeTreeExecutedWriteSummary( -// updatedPartitions = updatedPartitions.toSet, -// stats = statsTrackers.map(_.getFinalStats(taskCommitTime))) - WriteTaskResult(taskCommitMessage, null) - } else { - throw new IllegalStateException() - } - })( - catchBlock = { - // If there is an error, abort the task - logError(s"Job $jobId aborted.") - }, - finallyBlock = {}) - } catch { - case e: FetchFailedException => - throw e - case f: FileAlreadyExistsException if SQLConf.get.fastFailFileFormatOutput => - // If any output file to write already exists, it does not make sense to re-run this task. - // We throw the exception and let Executor throw ExceptionFailure to abort the job. - throw new TaskOutputFileAlreadyExistException(f) - case t: Throwable => - throw QueryExecutionErrors.taskFailedWhileWritingRowsError(t) - } - - } - - def runOptimizeBinJobClickhouse( - txn: OptimisticTransaction, - partitionValues: Map[String, String], - bucketNum: String, - bin: Seq[AddFile], - maxFileSize: Long): Seq[FileAction] = { - val tableV2 = ClickHouseTableV2.getTable(txn.deltaLog) - - val sparkSession = SparkSession.getActiveSession.get - - val rddWithNonEmptyPartitions = - sparkSession.sparkContext.parallelize(Array.empty[InternalRow], 1) - - val jobIdInstant = new Date().getTime - val ret = new Array[WriteTaskResult](rddWithNonEmptyPartitions.partitions.length) - - val serializableHadoopConf = new SerializableConfiguration( - sparkSession.sessionState.newHadoopConfWithOptions( - txn.metadata.configuration ++ txn.deltaLog.options)) - - val partitionDir = if (tableV2.partitionColumns.isEmpty) { - None - } else { - Some(tableV2.partitionColumns.map(c => c + "=" + partitionValues(c)).mkString("/")) - } - - val bucketDir = if (tableV2.bucketOption.isEmpty) { - None - } else { - Some(bucketNum) - } - - val description = TaskDescription.apply( - txn.deltaLog.dataPath.toString, - tableV2.dataBaseName, - tableV2.tableName, - ClickhouseSnapshot.genSnapshotId(tableV2.snapshot), - tableV2.orderByKey, - tableV2.lowCardKey, - tableV2.minmaxIndexKey, - tableV2.bfIndexKey, - tableV2.setIndexKey, - tableV2.primaryKey, - tableV2.partitionColumns, - bin.map(_.asInstanceOf[AddMergeTreeParts].name), - tableV2.schema(), - tableV2.clickhouseTableConfigs, - serializableHadoopConf, - jobIdInstant, - partitionDir, - bucketDir - ) - sparkSession.sparkContext.runJob( - rddWithNonEmptyPartitions, - (taskContext: TaskContext, _: Iterator[InternalRow]) => { - executeTask( - description, - taskContext.stageId(), - taskContext.partitionId(), - taskContext.taskAttemptId().toInt & Integer.MAX_VALUE - ) - }, - rddWithNonEmptyPartitions.partitions.indices, - (index, res: WriteTaskResult) => { - ret(index) = res - } - ) - - val addFiles = ret - .flatMap(_.commitMsg.obj.asInstanceOf[Seq[AddFile]]) - .toSeq - - val removeFiles = - bin.map(f => f.removeWithTimestamp(new SystemClock().getTimeMillis(), dataChange = false)) - addFiles ++ removeFiles - - } - - def getDeltaLogClickhouse( - spark: SparkSession, - path: Option[String], - tableIdentifier: Option[TableIdentifier], - operationName: String, - hadoopConf: Map[String, String] = Map.empty): DeltaLog = { - val tablePath = - if (path.nonEmpty) { - new Path(path.get) - } else if (tableIdentifier.nonEmpty) { - val sessionCatalog = spark.sessionState.catalog - lazy val metadata = sessionCatalog.getTableMetadata(tableIdentifier.get) - - if (CHDataSourceUtils.isClickhousePath(spark, tableIdentifier.get)) { - new Path(tableIdentifier.get.table) - } else if (CHDataSourceUtils.isClickHouseTable(spark, tableIdentifier.get)) { - new Path(metadata.location) - } else { - DeltaTableIdentifier(spark, tableIdentifier.get) match { - case Some(id) if id.path.nonEmpty => - new Path(id.path.get) - case Some(id) if id.table.nonEmpty => - new Path(metadata.location) - case _ => - if (metadata.tableType == CatalogTableType.VIEW) { - throw DeltaErrors.viewNotSupported(operationName) - } - throw DeltaErrors.notADeltaTableException(operationName) - } - } - } else { - throw DeltaErrors.missingTableIdentifierException(operationName) - } - - val startTime = Some(System.currentTimeMillis) - val deltaLog = DeltaLog.forTable(spark, tablePath, hadoopConf) - if (deltaLog.update(checkIfUpdatedSinceTs = startTime).version < 0) { - throw DeltaErrors.notADeltaTableException( - operationName, - DeltaTableIdentifier(path, tableIdentifier)) - } - deltaLog - } - - def groupFilesIntoBinsClickhouse( - partitionsToCompact: Seq[((String, Map[String, String]), Seq[AddFile])], - maxTargetFileSize: Long): Seq[((String, Map[String, String]), Seq[AddFile])] = { - partitionsToCompact.flatMap { - case (partition, files) => - val bins = new ArrayBuffer[Seq[AddFile]]() - - val currentBin = new ArrayBuffer[AddFile]() - var currentBinSize = 0L - - files.sortBy(_.size).foreach { - file => - // Generally, a bin is a group of existing files, whose total size does not exceed the - // desired maxFileSize. They will be coalesced into a single output file. - // However, if isMultiDimClustering = true, all files in a partition will be read by the - // same job, the data will be range-partitioned and - // numFiles = totalFileSize / maxFileSize - // will be produced. See below. - - // isMultiDimClustering is always false for Gluten Clickhouse for now - if (file.size + currentBinSize > maxTargetFileSize /* && !isMultiDimClustering */ ) { - bins += currentBin.toVector - currentBin.clear() - currentBin += file - currentBinSize = file.size - } else { - currentBin += file - currentBinSize += file.size - } - } - - if (currentBin.nonEmpty) { - bins += currentBin.toVector - } - - bins - .map(b => (partition, b)) - // select bins that have at least two files or in case of multi-dim clustering - // select all bins - .filter(_._2.size > 1 /* || isMultiDimClustering */ ) - } - } -} diff --git a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala b/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala deleted file mode 100644 index b39bcd5ba8d..00000000000 --- a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala +++ /dev/null @@ -1,404 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.delta.commands - -// scalastyle:off import.ordering.noEmptyLine -import org.apache.spark.sql.delta.{DeltaConfigs, DeltaLog, DeltaOperations, DeltaTableUtils, DeltaUDF, OptimisticTransaction} -import org.apache.spark.sql.delta.actions.{AddCDCFile, AddFile, FileAction} -import org.apache.spark.sql.delta.commands.cdc.CDCReader.{CDC_TYPE_COLUMN_NAME, CDC_TYPE_NOT_CDC, CDC_TYPE_UPDATE_POSTIMAGE, CDC_TYPE_UPDATE_PREIMAGE} -import org.apache.spark.sql.delta.files.{TahoeBatchFileIndex, TahoeFileIndex} -import org.apache.hadoop.fs.Path - -import org.apache.spark.SparkContext -import org.apache.spark.sql.{Column, DataFrame, Dataset, Row, SparkSession} -import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute -import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeReference, Expression, If, Literal} -import org.apache.spark.sql.catalyst.plans.QueryPlan -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.execution.command.LeafRunnableCommand -import org.apache.spark.sql.execution.metric.SQLMetric -import org.apache.spark.sql.execution.metric.SQLMetrics.{createMetric, createTimingMetric} -import org.apache.spark.sql.functions._ -import org.apache.spark.sql.types.LongType - -/** - * Gluten overwrite Delta: - * - * This file is copied from Delta 2.3.0. - */ - -/** - * Performs an Update using `updateExpression` on the rows that match `condition` - * - * Algorithm: - * 1) Identify the affected files, i.e., the files that may have the rows to be updated. - * 2) Scan affected files, apply the updates, and generate a new DF with updated rows. - * 3) Use the Delta protocol to atomically write the new DF as new files and remove - * the affected files that are identified in step 1. - */ -case class UpdateCommand( - tahoeFileIndex: TahoeFileIndex, - target: LogicalPlan, - updateExpressions: Seq[Expression], - condition: Option[Expression]) - extends LeafRunnableCommand with DeltaCommand { - - override val output: Seq[Attribute] = { - Seq(AttributeReference("num_affected_rows", LongType)()) - } - - override def innerChildren: Seq[QueryPlan[_]] = Seq(target) - - @transient private lazy val sc: SparkContext = SparkContext.getOrCreate() - - override lazy val metrics = Map[String, SQLMetric]( - "numAddedFiles" -> createMetric(sc, "number of files added."), - "numAddedBytes" -> createMetric(sc, "number of bytes added"), - "numRemovedFiles" -> createMetric(sc, "number of files removed."), - "numRemovedBytes" -> createMetric(sc, "number of bytes removed"), - "numUpdatedRows" -> createMetric(sc, "number of rows updated."), - "numCopiedRows" -> createMetric(sc, "number of rows copied."), - "executionTimeMs" -> - createTimingMetric(sc, "time taken to execute the entire operation"), - "scanTimeMs" -> - createTimingMetric(sc, "time taken to scan the files for matches"), - "rewriteTimeMs" -> - createTimingMetric(sc, "time taken to rewrite the matched files"), - "numAddedChangeFiles" -> createMetric(sc, "number of change data capture files generated"), - "changeFileBytes" -> createMetric(sc, "total size of change data capture files generated"), - "numTouchedRows" -> createMetric(sc, "number of rows touched (copied + updated)") - ) - - final override def run(sparkSession: SparkSession): Seq[Row] = { - recordDeltaOperation(tahoeFileIndex.deltaLog, "delta.dml.update") { - val deltaLog = tahoeFileIndex.deltaLog - deltaLog.withNewTransaction { txn => - DeltaLog.assertRemovable(txn.snapshot) - if (hasBeenExecuted(txn, sparkSession)) { - sendDriverMetrics(sparkSession, metrics) - return Seq.empty - } - performUpdate(sparkSession, deltaLog, txn) - } - // Re-cache all cached plans(including this relation itself, if it's cached) that refer to - // this data source relation. - sparkSession.sharedState.cacheManager.recacheByPlan(sparkSession, target) - } - Seq(Row(metrics("numUpdatedRows").value)) - } - - private def performUpdate( - sparkSession: SparkSession, deltaLog: DeltaLog, txn: OptimisticTransaction): Unit = { - import org.apache.spark.sql.delta.implicits._ - - var numTouchedFiles: Long = 0 - var numRewrittenFiles: Long = 0 - var numAddedBytes: Long = 0 - var numRemovedBytes: Long = 0 - var numAddedChangeFiles: Long = 0 - var changeFileBytes: Long = 0 - var scanTimeMs: Long = 0 - var rewriteTimeMs: Long = 0 - - val startTime = System.nanoTime() - val numFilesTotal = txn.snapshot.numOfFiles - - val updateCondition = condition.getOrElse(Literal.TrueLiteral) - val (metadataPredicates, dataPredicates) = - DeltaTableUtils.splitMetadataAndDataPredicates( - updateCondition, txn.metadata.partitionColumns, sparkSession) - val candidateFiles = txn.filterFiles(metadataPredicates ++ dataPredicates) - val nameToAddFile = generateCandidateFileMap(deltaLog.dataPath, candidateFiles) - - scanTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 - - val filesToRewrite: Seq[AddFile] = if (candidateFiles.isEmpty) { - // Case 1: Do nothing if no row qualifies the partition predicates - // that are part of Update condition - Nil - } else if (dataPredicates.isEmpty) { - // Case 2: Update all the rows from the files that are in the specified partitions - // when the data filter is empty - candidateFiles - } else { - // Case 3: Find all the affected files using the user-specified condition - val fileIndex = new TahoeBatchFileIndex( - sparkSession, "update", candidateFiles, deltaLog, tahoeFileIndex.path, txn.snapshot) - // Keep everything from the resolved target except a new TahoeFileIndex - // that only involves the affected files instead of all files. - val newTarget = DeltaTableUtils.replaceFileIndex(target, fileIndex) - val data = Dataset.ofRows(sparkSession, newTarget) - val updatedRowCount = metrics("numUpdatedRows") - val updatedRowUdf = DeltaUDF.boolean { () => - updatedRowCount += 1 - true - }.asNondeterministic() - val pathsToRewrite = - withStatusCode("DELTA", UpdateCommand.FINDING_TOUCHED_FILES_MSG) { - // --- modified start - data.filter(new Column(updateCondition)) - .select(input_file_name().as("input_files")) - .filter(updatedRowUdf()) - .distinct() - .as[String] - .collect() - // --- modified end - } - - scanTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 - - pathsToRewrite.map(getTouchedFile(deltaLog.dataPath, _, nameToAddFile)).toSeq - } - - numTouchedFiles = filesToRewrite.length - - val newActions = if (filesToRewrite.isEmpty) { - // Do nothing if no row qualifies the UPDATE condition - Nil - } else { - // Generate the new files containing the updated values - withStatusCode("DELTA", UpdateCommand.rewritingFilesMsg(filesToRewrite.size)) { - rewriteFiles(sparkSession, txn, tahoeFileIndex.path, - filesToRewrite.map(_.path), nameToAddFile, updateCondition) - } - } - - rewriteTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 - scanTimeMs - - val (changeActions, addActions) = newActions.partition(_.isInstanceOf[AddCDCFile]) - numRewrittenFiles = addActions.size - numAddedBytes = addActions.map(_.getFileSize).sum - numAddedChangeFiles = changeActions.size - changeFileBytes = changeActions.collect { case f: AddCDCFile => f.size }.sum - - val totalActions = if (filesToRewrite.isEmpty) { - // Do nothing if no row qualifies the UPDATE condition - Nil - } else { - // Delete the old files and return those delete actions along with the new AddFile actions for - // files containing the updated values - val operationTimestamp = System.currentTimeMillis() - val deleteActions = filesToRewrite.map(_.removeWithTimestamp(operationTimestamp)) - numRemovedBytes = filesToRewrite.map(_.getFileSize).sum - deleteActions ++ newActions - } - - metrics("numAddedFiles").set(numRewrittenFiles) - metrics("numAddedBytes").set(numAddedBytes) - metrics("numAddedChangeFiles").set(numAddedChangeFiles) - metrics("changeFileBytes").set(changeFileBytes) - metrics("numRemovedFiles").set(numTouchedFiles) - metrics("numRemovedBytes").set(numRemovedBytes) - metrics("executionTimeMs").set((System.nanoTime() - startTime) / 1000 / 1000) - metrics("scanTimeMs").set(scanTimeMs) - metrics("rewriteTimeMs").set(rewriteTimeMs) - // In the case where the numUpdatedRows is not captured, we can siphon out the metrics from - // the BasicWriteStatsTracker. This is for case 2 where the update condition contains only - // metadata predicates and so the entire partition is re-written. - val outputRows = txn.getMetric("numOutputRows").map(_.value).getOrElse(-1L) - if (metrics("numUpdatedRows").value == 0 && outputRows != 0 && - metrics("numCopiedRows").value == 0) { - // We know that numTouchedRows = numCopiedRows + numUpdatedRows. - // Since an entire partition was re-written, no rows were copied. - // So numTouchedRows == numUpdateRows - metrics("numUpdatedRows").set(metrics("numTouchedRows").value) - } else { - // This is for case 3 where the update condition contains both metadata and data predicates - // so relevant files will have some rows updated and some rows copied. We don't need to - // consider case 1 here, where no files match the update condition, as we know that - // `totalActions` is empty. - metrics("numCopiedRows").set( - metrics("numTouchedRows").value - metrics("numUpdatedRows").value) - } - txn.registerSQLMetrics(sparkSession, metrics) - - val finalActions = createSetTransaction(sparkSession, deltaLog).toSeq ++ totalActions - txn.commitIfNeeded(finalActions, DeltaOperations.Update(condition.map(_.toString))) - sendDriverMetrics(sparkSession, metrics) - - recordDeltaEvent( - deltaLog, - "delta.dml.update.stats", - data = UpdateMetric( - condition = condition.map(_.sql).getOrElse("true"), - numFilesTotal, - numTouchedFiles, - numRewrittenFiles, - numAddedChangeFiles, - changeFileBytes, - scanTimeMs, - rewriteTimeMs) - ) - } - - /** - * Scan all the affected files and write out the updated files. - * - * When CDF is enabled, includes the generation of CDC preimage and postimage columns for - * changed rows. - * - * @return the list of [[AddFile]]s and [[AddCDCFile]]s that have been written. - */ - private def rewriteFiles( - spark: SparkSession, - txn: OptimisticTransaction, - rootPath: Path, - inputLeafFiles: Seq[String], - nameToAddFileMap: Map[String, AddFile], - condition: Expression): Seq[FileAction] = { - // Containing the map from the relative file path to AddFile - val baseRelation = buildBaseRelation( - spark, txn, "update", rootPath, inputLeafFiles, nameToAddFileMap) - val newTarget = DeltaTableUtils.replaceFileIndex(target, baseRelation.location) - val targetDf = Dataset.ofRows(spark, newTarget) - - // Number of total rows that we have seen, i.e. are either copying or updating (sum of both). - // This will be used later, along with numUpdatedRows, to determine numCopiedRows. - val numTouchedRows = metrics("numTouchedRows") - val numTouchedRowsUdf = DeltaUDF.boolean { () => - numTouchedRows += 1 - true - }.asNondeterministic() - - val updatedDataFrame = UpdateCommand.withUpdatedColumns( - target, - updateExpressions, - condition, - targetDf - .filter(numTouchedRowsUdf()) - .withColumn(UpdateCommand.CONDITION_COLUMN_NAME, new Column(condition)), - UpdateCommand.shouldOutputCdc(txn)) - - txn.writeFiles(updatedDataFrame) - } -} - -object UpdateCommand { - val FILE_NAME_COLUMN = "_input_file_name_" - val CONDITION_COLUMN_NAME = "__condition__" - val FINDING_TOUCHED_FILES_MSG: String = "Finding files to rewrite for UPDATE operation" - - def rewritingFilesMsg(numFilesToRewrite: Long): String = - s"Rewriting $numFilesToRewrite files for UPDATE operation" - - /** - * Whether or not CDC is enabled on this table and, thus, if we should output CDC data during this - * UPDATE operation. - */ - def shouldOutputCdc(txn: OptimisticTransaction): Boolean = { - DeltaConfigs.CHANGE_DATA_FEED.fromMetaData(txn.metadata) - } - - /** - * Build the new columns. If the condition matches, generate the new value using - * the corresponding UPDATE EXPRESSION; otherwise, keep the original column value. - * - * When CDC is enabled, includes the generation of CDC pre-image and post-image columns for - * changed rows. - * - * @param target target we are updating into - * @param updateExpressions the update transformation to perform on the input DataFrame - * @param dfWithEvaluatedCondition source DataFrame on which we will apply the update expressions - * with an additional column CONDITION_COLUMN_NAME which is the - * true/false value of if the update condition is satisfied - * @param condition update condition - * @param shouldOutputCdc if we should output CDC data during this UPDATE operation. - * @return the updated DataFrame, with extra CDC columns if CDC is enabled - */ - def withUpdatedColumns( - target: LogicalPlan, - updateExpressions: Seq[Expression], - condition: Expression, - dfWithEvaluatedCondition: DataFrame, - shouldOutputCdc: Boolean): DataFrame = { - val resultDf = if (shouldOutputCdc) { - val namedUpdateCols = updateExpressions.zip(target.output).map { - case (expr, targetCol) => new Column(expr).as(targetCol.name) - } - - // Build an array of output rows to be unpacked later. If the condition is matched, we - // generate CDC pre and postimages in addition to the final output row; if the condition - // isn't matched, we just generate a rewritten no-op row without any CDC events. - val preimageCols = target.output.map(new Column(_)) :+ - lit(CDC_TYPE_UPDATE_PREIMAGE).as(CDC_TYPE_COLUMN_NAME) - val postimageCols = namedUpdateCols :+ - lit(CDC_TYPE_UPDATE_POSTIMAGE).as(CDC_TYPE_COLUMN_NAME) - val notCdcCol = new Column(CDC_TYPE_NOT_CDC).as(CDC_TYPE_COLUMN_NAME) - val updatedDataCols = namedUpdateCols :+ notCdcCol - val noopRewriteCols = target.output.map(new Column(_)) :+ notCdcCol - val packedUpdates = array( - struct(preimageCols: _*), - struct(postimageCols: _*), - struct(updatedDataCols: _*) - ).expr - - val packedData = if (condition == Literal.TrueLiteral) { - packedUpdates - } else { - If( - UnresolvedAttribute(CONDITION_COLUMN_NAME), - packedUpdates, // if it should be updated, then use `packagedUpdates` - array(struct(noopRewriteCols: _*)).expr) // else, this is a noop rewrite - } - - // Explode the packed array, and project back out the final data columns. - val finalColNames = target.output.map(_.name) :+ CDC_TYPE_COLUMN_NAME - dfWithEvaluatedCondition - .select(explode(new Column(packedData)).as("packedData")) - .select(finalColNames.map { n => col(s"packedData.`$n`").as(s"$n") }: _*) - } else { - val finalCols = updateExpressions.zip(target.output).map { case (update, original) => - val updated = if (condition == Literal.TrueLiteral) { - update - } else { - If(UnresolvedAttribute(CONDITION_COLUMN_NAME), update, original) - } - new Column(Alias(updated, original.name)()) - } - - dfWithEvaluatedCondition.select(finalCols: _*) - } - - resultDf.drop(CONDITION_COLUMN_NAME) - } -} - -/** - * Used to report details about update. - * - * @param condition: what was the update condition - * @param numFilesTotal: how big is the table - * @param numTouchedFiles: how many files did we touch - * @param numRewrittenFiles: how many files had to be rewritten - * @param numAddedChangeFiles: how many change files were generated - * @param changeFileBytes: total size of change files generated - * @param scanTimeMs: how long did finding take - * @param rewriteTimeMs: how long did rewriting take - * - * @note All the time units are milliseconds. - */ -case class UpdateMetric( - condition: String, - numFilesTotal: Long, - numTouchedFiles: Long, - numRewrittenFiles: Long, - numAddedChangeFiles: Long, - changeFileBytes: Long, - scanTimeMs: Long, - rewriteTimeMs: Long -) diff --git a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala b/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala deleted file mode 100644 index e59645f58c2..00000000000 --- a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala +++ /dev/null @@ -1,591 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.delta.commands - -// scalastyle:off import.ordering.noEmptyLine -import java.net.URI -import java.util.Date -import java.util.concurrent.TimeUnit -import scala.collection.JavaConverters._ -import org.apache.spark.sql.delta._ -import org.apache.spark.sql.delta.actions.{FileAction, RemoveFile} -import org.apache.spark.sql.delta.sources.DeltaSQLConf -import org.apache.spark.sql.delta.util.DeltaFileOperations -import org.apache.spark.sql.delta.util.DeltaFileOperations.tryDeleteNonRecursive -import com.fasterxml.jackson.databind.annotation.JsonDeserialize -import org.apache.gluten.extension.GlutenSessionExtensions -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.{FileSystem, Path} -import org.apache.spark.broadcast.Broadcast -import org.apache.spark.sql.{Column, DataFrame, Dataset, SparkSession} -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig -import org.apache.spark.sql.execution.metric.SQLMetric -import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric -import org.apache.spark.sql.functions._ -import org.apache.spark.util.{Clock, SerializableConfiguration, SystemClock} - -/** - * Gluten overwrite Delta: - * - * This file is copied from Delta 2.3.0. It is modified to overcome the following issues: - * 1. In Gluten, part is a directory, but VacuumCommand assumes part is a file. So we need some - * modifications to make it work. - */ - -/** - * Vacuums the table by clearing all untracked files and folders within this table. - * First lists all the files and directories in the table, and gets the relative paths with - * respect to the base of the table. Then it gets the list of all tracked files for this table, - * which may or may not be within the table base path, and gets the relative paths of - * all the tracked files with respect to the base of the table. Files outside of the table path - * will be ignored. Then we take a diff of the files and delete directories that were already empty, - * and all files that are within the table that are no longer tracked. - */ -object VacuumCommand extends VacuumCommandImpl with Serializable { - - // --- modified start - case class FileNameAndSize(path: String, length: Long, isDir: Boolean = false) - // --- modified end - /** - * Additional check on retention duration to prevent people from shooting themselves in the foot. - */ - protected def checkRetentionPeriodSafety( - spark: SparkSession, - retentionMs: Option[Long], - configuredRetention: Long): Unit = { - require(retentionMs.forall(_ >= 0), "Retention for Vacuum can't be less than 0.") - val checkEnabled = - spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_RETENTION_CHECK_ENABLED) - val retentionSafe = retentionMs.forall(_ >= configuredRetention) - var configuredRetentionHours = TimeUnit.MILLISECONDS.toHours(configuredRetention) - if (TimeUnit.HOURS.toMillis(configuredRetentionHours) < configuredRetention) { - configuredRetentionHours += 1 - } - require(!checkEnabled || retentionSafe, - s"""Are you sure you would like to vacuum files with such a low retention period? If you have - |writers that are currently writing to this table, there is a risk that you may corrupt the - |state of your Delta table. - | - |If you are certain that there are no operations being performed on this table, such as - |insert/upsert/delete/optimize, then you may turn off this check by setting: - |spark.databricks.delta.retentionDurationCheck.enabled = false - | - |If you are not sure, please use a value not less than "$configuredRetentionHours hours". - """.stripMargin) - } - - /** - * Clears all untracked files and folders within this table. First lists all the files and - * directories in the table, and gets the relative paths with respect to the base of the - * table. Then it gets the list of all tracked files for this table, which may or may not - * be within the table base path, and gets the relative paths of all the tracked files with - * respect to the base of the table. Files outside of the table path will be ignored. - * Then we take a diff of the files and delete directories that were already empty, and all files - * that are within the table that are no longer tracked. - * - * @param dryRun If set to true, no files will be deleted. Instead, we will list all files and - * directories that will be cleared. - * @param retentionHours An optional parameter to override the default Delta tombstone retention - * period - * @return A Dataset containing the paths of the files/folders to delete in dryRun mode. Otherwise - * returns the base path of the table. - */ - def gc( - spark: SparkSession, - deltaLog: DeltaLog, - dryRun: Boolean = true, - retentionHours: Option[Double] = None, - clock: Clock = new SystemClock): DataFrame = { - recordDeltaOperation(deltaLog, "delta.gc") { - - val path = deltaLog.dataPath - val deltaHadoopConf = deltaLog.newDeltaHadoopConf() - val fs = path.getFileSystem(deltaHadoopConf) - - import org.apache.spark.sql.delta.implicits._ - - val snapshot = deltaLog.update() - - require(snapshot.version >= 0, "No state defined for this table. Is this really " + - "a Delta table? Refusing to garbage collect.") - - // --- modified start - val isMergeTreeFormat = ClickHouseConfig - .isMergeTreeFormatEngine(deltaLog.unsafeVolatileMetadata.configuration) - // --- modified end - - DeletionVectorUtils.assertDeletionVectorsNotReadable( - spark, snapshot.metadata, snapshot.protocol) - - val snapshotTombstoneRetentionMillis = DeltaLog.tombstoneRetentionMillis(snapshot.metadata) - val retentionMillis = retentionHours.map(h => TimeUnit.HOURS.toMillis(math.round(h))) - checkRetentionPeriodSafety(spark, retentionMillis, snapshotTombstoneRetentionMillis) - - val deleteBeforeTimestamp = retentionMillis.map { millis => - clock.getTimeMillis() - millis - }.getOrElse(snapshot.minFileRetentionTimestamp) - // --- modified start: toGMTString is a deprecated function - logInfo(s"Starting garbage collection (dryRun = $dryRun) of untracked files older than " + - s"${new Date(deleteBeforeTimestamp).toString} in $path") - // --- modified end - val hadoopConf = spark.sparkContext.broadcast( - new SerializableConfiguration(deltaHadoopConf)) - val basePath = fs.makeQualified(path).toString - var isBloomFiltered = false - val parallelDeleteEnabled = - spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_PARALLEL_DELETE_ENABLED) - val parallelDeletePartitions = - spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_PARALLEL_DELETE_PARALLELISM) - .getOrElse(spark.sessionState.conf.numShufflePartitions) - val relativizeIgnoreError = - spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_RELATIVIZE_IGNORE_ERROR) - val startTimeToIdentifyEligibleFiles = System.currentTimeMillis() - - // --- modified start - val originalEnabledGluten = - spark.sparkContext.getLocalProperty(GlutenSessionExtensions.GLUTEN_ENABLE_FOR_THREAD_KEY) - // gluten can not support vacuum command - spark.sparkContext.setLocalProperty(GlutenSessionExtensions.GLUTEN_ENABLE_FOR_THREAD_KEY, "false") - // --- modified end - - val validFiles = snapshot.stateDS - .mapPartitions { actions => - val reservoirBase = new Path(basePath) - val fs = reservoirBase.getFileSystem(hadoopConf.value.value) - actions.flatMap { - _.unwrap match { - case tombstone: RemoveFile if tombstone.delTimestamp < deleteBeforeTimestamp => - Nil - case fa: FileAction => - getValidRelativePathsAndSubdirs( - fa, - fs, - reservoirBase, - relativizeIgnoreError, - isBloomFiltered) - case _ => Nil - } - } - }.toDF("path") - - val partitionColumns = snapshot.metadata.partitionSchema.fieldNames - val parallelism = spark.sessionState.conf.parallelPartitionDiscoveryParallelism - - val allFilesAndDirs = DeltaFileOperations.recursiveListDirs( - spark, - Seq(basePath), - hadoopConf, - hiddenDirNameFilter = DeltaTableUtils.isHiddenDirectory(partitionColumns, _), - hiddenFileNameFilter = DeltaTableUtils.isHiddenDirectory(partitionColumns, _), - fileListingParallelism = Option(parallelism) - ) - .groupByKey(_.path) - .mapGroups { (k, v) => - val duplicates = v.toSeq - // of all the duplicates we can return the newest file. - duplicates.maxBy(_.modificationTime) - } - - try { - allFilesAndDirs.cache() - - implicit val fileNameAndSizeEncoder = org.apache.spark.sql.Encoders.product[FileNameAndSize] - - val dirCounts = allFilesAndDirs.where(col("isDir")).count() + 1 // +1 for the base path - - // The logic below is as follows: - // 1. We take all the files and directories listed in our reservoir - // 2. We filter all files older than our tombstone retention period and directories - // 3. We get the subdirectories of all files so that we can find non-empty directories - // 4. We groupBy each path, and count to get how many files are in each sub-directory - // 5. We subtract all the valid files and tombstones in our state - // 6. We filter all paths with a count of 1, which will correspond to files not in the - // state, and empty directories. We can safely delete all of these - // --- modified start - val diff = if (isMergeTreeFormat) { - val diff_tmp = allFilesAndDirs - .where(col("modificationTime") < deleteBeforeTimestamp || col("isDir")) - .mapPartitions { fileStatusIterator => - val reservoirBase = new Path(basePath) - val fs = reservoirBase.getFileSystem(hadoopConf.value.value) - fileStatusIterator.flatMap { fileStatus => - if (fileStatus.isDir) { - Iterator.single(FileNameAndSize( - relativize(fileStatus.getHadoopPath, fs, reservoirBase, isDir = true), - 0L, - true)) - } else { - val dirs = getAllSubdirs(basePath, fileStatus.path, fs) - val dirsWithSlash = dirs.map { p => - val relativizedPath = relativize(new Path(p), fs, reservoirBase, isDir = true) - FileNameAndSize(relativizedPath, 0L, true) - } - dirsWithSlash ++ Iterator( - FileNameAndSize(relativize( - fileStatus.getHadoopPath, fs, reservoirBase, isDir = false), - fileStatus.length)) - } - } - } - .withColumn( - "dir", - when(col("isDir"), col("path")) - .otherwise(expr("substring_index(path, '/',size(split(path, '/')) -1)"))) - .groupBy(col("path"), col("dir")) - .agg(count(new Column("*")).as("count"), sum("length").as("length")) - - diff_tmp - .join(validFiles, diff_tmp("dir") === validFiles("path"), "leftanti") - .where(col("count") === 1) - } else { - allFilesAndDirs - .where(col("modificationTime") < deleteBeforeTimestamp || col("isDir")) - .mapPartitions { fileStatusIterator => - val reservoirBase = new Path(basePath) - val fs = reservoirBase.getFileSystem(hadoopConf.value.value) - fileStatusIterator.flatMap { fileStatus => - if (fileStatus.isDir) { - Iterator.single(FileNameAndSize( - relativize(fileStatus.getHadoopPath, fs, reservoirBase, isDir = true), 0L)) - } else { - val dirs = getAllSubdirs(basePath, fileStatus.path, fs) - val dirsWithSlash = dirs.map { p => - val relativizedPath = relativize(new Path(p), fs, reservoirBase, isDir = true) - FileNameAndSize(relativizedPath, 0L) - } - dirsWithSlash ++ Iterator( - FileNameAndSize(relativize( - fileStatus.getHadoopPath, fs, reservoirBase, isDir = false), - fileStatus.length)) - } - } - } - .groupBy(col("path")) - .agg(count(new Column("*")).as("count"), sum("length").as("length")) - .join(validFiles, Seq("path"), "leftanti") - .where(col("count") === 1) - } - // --- modified end - - val sizeOfDataToDeleteRow = diff.agg(sum("length").cast("long")).first - val sizeOfDataToDelete = if (sizeOfDataToDeleteRow.isNullAt(0)) { - 0L - } else { - sizeOfDataToDeleteRow.getLong(0) - } - - val diffFiles = diff - .select(col("path")) - .as[String] - .map { relativePath => - assert(!stringToPath(relativePath).isAbsolute, - "Shouldn't have any absolute paths for deletion here.") - pathToString(DeltaFileOperations.absolutePath(basePath, relativePath)) - } - val timeTakenToIdentifyEligibleFiles = - System.currentTimeMillis() - startTimeToIdentifyEligibleFiles - - val numFiles = diffFiles.count() - if (dryRun) { - val stats = DeltaVacuumStats( - isDryRun = true, - specifiedRetentionMillis = retentionMillis, - defaultRetentionMillis = snapshotTombstoneRetentionMillis, - minRetainedTimestamp = deleteBeforeTimestamp, - dirsPresentBeforeDelete = dirCounts, - objectsDeleted = numFiles, - sizeOfDataToDelete = sizeOfDataToDelete, - timeTakenToIdentifyEligibleFiles = timeTakenToIdentifyEligibleFiles, - timeTakenForDelete = 0L) - - recordDeltaEvent(deltaLog, "delta.gc.stats", data = stats) - logConsole(s"Found $numFiles files ($sizeOfDataToDelete bytes) and directories in " + - s"a total of $dirCounts directories that are safe to delete.") - - return diffFiles.map(f => stringToPath(f).toString).toDF("path") - } - logVacuumStart( - spark, - deltaLog, - path, - diffFiles, - sizeOfDataToDelete, - retentionMillis, - snapshotTombstoneRetentionMillis) - - val deleteStartTime = System.currentTimeMillis() - val filesDeleted = try { - delete(diffFiles, spark, basePath, - hadoopConf, parallelDeleteEnabled, parallelDeletePartitions) - } catch { - case t: Throwable => - logVacuumEnd(deltaLog, spark, path) - throw t - } - val timeTakenForDelete = System.currentTimeMillis() - deleteStartTime - val stats = DeltaVacuumStats( - isDryRun = false, - specifiedRetentionMillis = retentionMillis, - defaultRetentionMillis = snapshotTombstoneRetentionMillis, - minRetainedTimestamp = deleteBeforeTimestamp, - dirsPresentBeforeDelete = dirCounts, - objectsDeleted = filesDeleted, - sizeOfDataToDelete = sizeOfDataToDelete, - timeTakenToIdentifyEligibleFiles = timeTakenToIdentifyEligibleFiles, - timeTakenForDelete = timeTakenForDelete) - recordDeltaEvent(deltaLog, "delta.gc.stats", data = stats) - logVacuumEnd(deltaLog, spark, path, Some(filesDeleted), Some(dirCounts)) - - - spark.createDataset(Seq(basePath)).toDF("path") - } finally { - allFilesAndDirs.unpersist() - - // --- modified start - if (originalEnabledGluten != null) { - spark.sparkContext.setLocalProperty( - GlutenSessionExtensions.GLUTEN_ENABLE_FOR_THREAD_KEY, originalEnabledGluten) - } else { - spark.sparkContext.setLocalProperty( - GlutenSessionExtensions.GLUTEN_ENABLE_FOR_THREAD_KEY, "true") - } - // --- modified end - } - } - } -} - -trait VacuumCommandImpl extends DeltaCommand { - - private val supportedFsForLogging = Seq( - "wasbs", "wasbss", "abfs", "abfss", "adl", "gs", "file", "hdfs" - ) - - /** - * Returns whether we should record vacuum metrics in the delta log. - */ - private def shouldLogVacuum( - spark: SparkSession, - deltaLog: DeltaLog, - hadoopConf: Configuration, - path: Path): Boolean = { - val logVacuumConf = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_LOGGING_ENABLED) - - if (logVacuumConf.nonEmpty) { - return logVacuumConf.get - } - - val logStore = deltaLog.store - - try { - val rawResolvedUri: URI = logStore.resolvePathOnPhysicalStorage(path, hadoopConf).toUri - val scheme = rawResolvedUri.getScheme - supportedFsForLogging.contains(scheme) - } catch { - case _: UnsupportedOperationException => - logWarning("Vacuum event logging" + - " not enabled on this file system because we cannot detect your cloud storage type.") - false - } - } - - /** - * Record Vacuum specific metrics in the commit log at the START of vacuum. - * - * @param spark - spark session - * @param deltaLog - DeltaLog of the table - * @param path - the (data) path to the root of the table - * @param diff - the list of paths (files, directories) that are safe to delete - * @param sizeOfDataToDelete - the amount of data (bytes) to be deleted - * @param specifiedRetentionMillis - the optional override retention period (millis) to keep - * logically removed files before deleting them - * @param defaultRetentionMillis - the default retention period (millis) - */ - protected def logVacuumStart( - spark: SparkSession, - deltaLog: DeltaLog, - path: Path, - diff: Dataset[String], - sizeOfDataToDelete: Long, - specifiedRetentionMillis: Option[Long], - defaultRetentionMillis: Long): Unit = { - logInfo(s"Deleting untracked files and empty directories in $path. The amount of data to be " + - s"deleted is $sizeOfDataToDelete (in bytes)") - - // We perform an empty commit in order to record information about the Vacuum - if (shouldLogVacuum(spark, deltaLog, deltaLog.newDeltaHadoopConf(), path)) { - val checkEnabled = - spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_RETENTION_CHECK_ENABLED) - val txn = deltaLog.startTransaction() - val metrics = Map[String, SQLMetric]( - "numFilesToDelete" -> createMetric(spark.sparkContext, "number of files to deleted"), - "sizeOfDataToDelete" -> createMetric(spark.sparkContext, - "The total amount of data to be deleted in bytes") - ) - metrics("numFilesToDelete").set(diff.count()) - metrics("sizeOfDataToDelete").set(sizeOfDataToDelete) - txn.registerSQLMetrics(spark, metrics) - txn.commit(actions = Seq(), DeltaOperations.VacuumStart( - checkEnabled, - specifiedRetentionMillis, - defaultRetentionMillis - )) - } - } - - /** - * Record Vacuum specific metrics in the commit log at the END of vacuum. - * - * @param deltaLog - DeltaLog of the table - * @param spark - spark session - * @param path - the (data) path to the root of the table - * @param filesDeleted - if the vacuum completed this will contain the number of files deleted. - * if the vacuum failed, this will be None. - * @param dirCounts - if the vacuum completed this will contain the number of directories - * vacuumed. if the vacuum failed, this will be None. - */ - protected def logVacuumEnd( - deltaLog: DeltaLog, - spark: SparkSession, - path: Path, - filesDeleted: Option[Long] = None, - dirCounts: Option[Long] = None): Unit = { - if (shouldLogVacuum(spark, deltaLog, deltaLog.newDeltaHadoopConf(), path)) { - val txn = deltaLog.startTransaction() - val status = if (filesDeleted.isEmpty && dirCounts.isEmpty) { "FAILED" } else { "COMPLETED" } - if (filesDeleted.nonEmpty && dirCounts.nonEmpty) { - val metrics = Map[String, SQLMetric]( - "numDeletedFiles" -> createMetric(spark.sparkContext, "number of files deleted."), - "numVacuumedDirectories" -> - createMetric(spark.sparkContext, "num of directories vacuumed."), - "status" -> createMetric(spark.sparkContext, "status of vacuum") - ) - metrics("numDeletedFiles").set(filesDeleted.get) - metrics("numVacuumedDirectories").set(dirCounts.get) - txn.registerSQLMetrics(spark, metrics) - } - txn.commit(actions = Seq(), DeltaOperations.VacuumEnd( - status - )) - } - - if (filesDeleted.nonEmpty) { - logConsole(s"Deleted ${filesDeleted.get} files and directories in a total " + - s"of ${dirCounts.get} directories.") - } - } - - /** - * Attempts to relativize the `path` with respect to the `reservoirBase` and converts the path to - * a string. - */ - protected def relativize( - path: Path, - fs: FileSystem, - reservoirBase: Path, - isDir: Boolean): String = { - pathToString(DeltaFileOperations.tryRelativizePath(fs, reservoirBase, path)) - } - - /** - * Wrapper function for DeltaFileOperations.getAllSubDirectories - * returns all subdirectories that `file` has with respect to `base`. - */ - protected def getAllSubdirs(base: String, file: String, fs: FileSystem): Iterator[String] = { - DeltaFileOperations.getAllSubDirectories(base, file)._1 - } - - /** - * Attempts to delete the list of candidate files. Returns the number of files deleted. - */ - protected def delete( - diff: Dataset[String], - spark: SparkSession, - basePath: String, - hadoopConf: Broadcast[SerializableConfiguration], - parallel: Boolean, - parallelPartitions: Int): Long = { - import org.apache.spark.sql.delta.implicits._ - - if (parallel) { - diff.repartition(parallelPartitions).mapPartitions { files => - val fs = new Path(basePath).getFileSystem(hadoopConf.value.value) - val filesDeletedPerPartition = - files.map(p => stringToPath(p)).count(f => tryDeleteNonRecursive(fs, f)) - Iterator(filesDeletedPerPartition) - }.collect().sum - } else { - val fs = new Path(basePath).getFileSystem(hadoopConf.value.value) - val fileResultSet = diff.toLocalIterator().asScala - fileResultSet.map(p => stringToPath(p)).count(f => tryDeleteNonRecursive(fs, f)) - } - } - - protected def stringToPath(path: String): Path = new Path(new URI(path)) - - protected def pathToString(path: Path): String = path.toUri.toString - - /** Returns the relative path of a file action or None if the file lives outside of the table. */ - protected def getActionRelativePath( - action: FileAction, - fs: FileSystem, - basePath: Path, - relativizeIgnoreError: Boolean): Option[String] = { - val filePath = stringToPath(action.path) - if (filePath.isAbsolute) { - val maybeRelative = - DeltaFileOperations.tryRelativizePath(fs, basePath, filePath, relativizeIgnoreError) - if (maybeRelative.isAbsolute) { - // This file lives outside the directory of the table. - None - } else { - Some(pathToString(maybeRelative)) - } - } else { - Some(pathToString(filePath)) - } - } - - - /** - * Returns the relative paths of all files and subdirectories for this action that must be - * retained during GC. - */ - protected def getValidRelativePathsAndSubdirs( - action: FileAction, - fs: FileSystem, - basePath: Path, - relativizeIgnoreError: Boolean, - isBloomFiltered: Boolean): Seq[String] = { - getActionRelativePath(action, fs, basePath, relativizeIgnoreError).map { relativePath => - Seq(relativePath) ++ getAllSubdirs("/", relativePath, fs) - }.getOrElse(Seq.empty) - } -} - -case class DeltaVacuumStats( - isDryRun: Boolean, - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - specifiedRetentionMillis: Option[Long], - defaultRetentionMillis: Long, - minRetainedTimestamp: Long, - dirsPresentBeforeDelete: Long, - objectsDeleted: Long, - sizeOfDataToDelete: Long, - timeTakenToIdentifyEligibleFiles: Long, - timeTakenForDelete: Long) diff --git a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala b/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala deleted file mode 100644 index 99360ed0f18..00000000000 --- a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.delta.files - -class MergeTreeDelayedCommitProtocol( - val outputPath: String, - randomPrefixLength: Option[Int], - val database: String, - val tableName: String) - extends DelayedCommitProtocol("delta-mergetree", outputPath, randomPrefixLength) - with MergeTreeFileCommitProtocol {} diff --git a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala b/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala deleted file mode 100644 index dbb5c4050a2..00000000000 --- a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.delta.rules - -import org.apache.gluten.backendsapi.clickhouse.CHBackendSettings - -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, V2WriteCommand} -import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.delta.{OptimisticTransaction, Snapshot, SubqueryTransformerHelper} -import org.apache.spark.sql.delta.files.TahoeLogFileIndex -import org.apache.spark.sql.delta.metering.DeltaLogging -import org.apache.spark.sql.delta.perf.OptimizeMetadataOnlyDeltaQuery -import org.apache.spark.sql.delta.sources.DeltaSQLConf -import org.apache.spark.sql.delta.stats.DeltaScanGenerator - -import org.apache.hadoop.fs.Path - -class CHOptimizeMetadataOnlyDeltaQuery(protected val spark: SparkSession) - extends Rule[LogicalPlan] - with DeltaLogging - with SubqueryTransformerHelper - with OptimizeMetadataOnlyDeltaQuery { - - private val scannedSnapshots = - new java.util.concurrent.ConcurrentHashMap[(String, Path), Snapshot] - - protected def getDeltaScanGenerator(index: TahoeLogFileIndex): DeltaScanGenerator = { - // The first case means that we've fixed the table snapshot for time travel - if (index.isTimeTravelQuery) return index.getSnapshot - OptimisticTransaction - .getActive() - .map(_.getDeltaScanGenerator(index)) - .getOrElse { - // Will be called only when the log is accessed the first time - scannedSnapshots.computeIfAbsent(index.deltaLog.compositeId, _ => index.getSnapshot) - } - } - - override def apply(plan: LogicalPlan): LogicalPlan = { - // Should not be applied to subqueries to avoid duplicate delta jobs. - val isSubquery = isSubqueryRoot(plan) - // Should not be applied to DataSourceV2 write plans, because they'll be planned later - // through a V1 fallback and only that later planning takes place within the transaction. - val isDataSourceV2 = plan.isInstanceOf[V2WriteCommand] - if (isSubquery || isDataSourceV2) { - return plan - } - // when 'stats.skipping' is off, it still use the metadata to optimize query for count/min/max - if ( - spark.sessionState.conf - .getConfString( - CHBackendSettings.GLUTEN_CLICKHOUSE_DELTA_METADATA_OPTIMIZE, - CHBackendSettings.GLUTEN_CLICKHOUSE_DELTA_METADATA_OPTIMIZE_DEFAULT_VALUE) - .toBoolean && - !spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_STATS_SKIPPING, true) - ) { - optimizeQueryWithMetadata(plan) - } else { - plan - } - } -} diff --git a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/stats/PrepareDeltaScan.scala b/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/stats/PrepareDeltaScan.scala deleted file mode 100644 index 21e31d35411..00000000000 --- a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/delta/stats/PrepareDeltaScan.scala +++ /dev/null @@ -1,406 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.delta.stats - -import java.util.Objects - -import scala.collection.mutable - -import org.apache.spark.sql.delta._ -import org.apache.spark.sql.delta.actions.AddFile -import org.apache.spark.sql.delta.files.{TahoeFileIndexWithSnapshot, TahoeLogFileIndex} -import org.apache.spark.sql.delta.metering.DeltaLogging -import org.apache.spark.sql.delta.perf.OptimizeMetadataOnlyDeltaQuery -import org.apache.spark.sql.delta.sources.DeltaSQLConf -import org.apache.hadoop.fs.Path - -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.planning.PhysicalOperation -import org.apache.spark.sql.catalyst.plans.logical._ -import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.catalyst.trees.TreePattern.PROJECT -import org.apache.spark.sql.execution.datasources.LogicalRelation - -/** - * Gluten overwrite Delta: - * - * This file is copied from Delta 2.3.0, it is modified to overcome the following issues: - * 1. Returns the plan directly even if stats.skipping is turned off - */ - -/** - * Before query planning, we prepare any scans over delta tables by pushing - * any projections or filters in allowing us to gather more accurate statistics - * for CBO and metering. - * - * Note the following - * - This rule also ensures that all reads from the same delta log use the same snapshot of log - * thus providing snapshot isolation. - * - If this rule is invoked within an active [[OptimisticTransaction]], then the scans are - * generated using the transaction. - */ -trait PrepareDeltaScanBase extends Rule[LogicalPlan] - with PredicateHelper - with DeltaLogging - with OptimizeMetadataOnlyDeltaQuery - with PreprocessTableWithDVs { self: PrepareDeltaScan => - - /** - * Tracks the first-access snapshots of other logs planned by this rule. The snapshots are - * the keyed by the log's unique id. Note that the lifetime of this rule is a single - * query, therefore, the map tracks the snapshots only within a query. - */ - private val scannedSnapshots = - new java.util.concurrent.ConcurrentHashMap[(String, Path), Snapshot] - - /** - * Gets the [[DeltaScanGenerator]] for the given log, which will be used to generate - * [[DeltaScan]]s. Every time this method is called on a log within the lifetime of this - * rule (i.e., the lifetime of the query for which this rule was instantiated), the returned - * generator will read a snapshot that is pinned on the first access for that log. - * - * Internally, it will use the snapshot of the file index, the snapshot of the active transaction - * (if any), or the latest snapshot of the given log. - */ - protected def getDeltaScanGenerator(index: TahoeLogFileIndex): DeltaScanGenerator = { - // The first case means that we've fixed the table snapshot for time travel - if (index.isTimeTravelQuery) return index.getSnapshot - val scanGenerator = OptimisticTransaction.getActive() - .map(_.getDeltaScanGenerator(index)) - .getOrElse { - // Will be called only when the log is accessed the first time - scannedSnapshots.computeIfAbsent(index.deltaLog.compositeId, _ => index.getSnapshot) - } - import PrepareDeltaScanBase._ - if (onGetDeltaScanGeneratorCallback != null) onGetDeltaScanGeneratorCallback(scanGenerator) - scanGenerator - } - - /** - * Helper method to generate a [[PreparedDeltaFileIndex]] - */ - protected def getPreparedIndex( - preparedScan: DeltaScan, - fileIndex: TahoeLogFileIndex): PreparedDeltaFileIndex = { - assert(fileIndex.partitionFilters.isEmpty, - "Partition filters should have been extracted by DeltaAnalysis.") - PreparedDeltaFileIndex( - spark, - fileIndex.deltaLog, - fileIndex.path, - preparedScan, - fileIndex.versionToUse) - } - - /** - * Scan files using the given `filters` and return `DeltaScan`. - * - * Note: when `limitOpt` is non empty, `filters` must contain only partition filters. Otherwise, - * it can contain arbitrary filters. See `DeltaTableScan` for more details. - */ - protected def filesForScan( - scanGenerator: DeltaScanGenerator, - limitOpt: Option[Int], - filters: Seq[Expression], - delta: LogicalRelation): DeltaScan = { - withStatusCode("DELTA", "Filtering files for query") { - if (limitOpt.nonEmpty) { - // If we trigger limit push down, the filters must be partition filters. Since - // there are no data filters, we don't need to apply Generated Columns - // optimization. See `DeltaTableScan` for more details. - return scanGenerator.filesForScan(limitOpt.get, filters) - } - val filtersForScan = - if (!GeneratedColumn.partitionFilterOptimizationEnabled(spark)) { - filters - } else { - val generatedPartitionFilters = GeneratedColumn.generatePartitionFilters( - spark, scanGenerator.snapshotToScan, filters, delta) - filters ++ generatedPartitionFilters - } - scanGenerator.filesForScan(filtersForScan) - } - } - - /** - * Prepares delta scans sequentially. - */ - protected def prepareDeltaScan(plan: LogicalPlan): LogicalPlan = { - // A map from the canonicalized form of a DeltaTableScan operator to its corresponding delta - // scan. This map is used to avoid fetching duplicate delta indexes for structurally-equal - // delta scans. - val deltaScans = new mutable.HashMap[LogicalPlan, DeltaScan]() - - transformWithSubqueries(plan) { - case scan @ DeltaTableScan(planWithRemovedProjections, filters, fileIndex, - limit, delta) => - val scanGenerator = getDeltaScanGenerator(fileIndex) - val preparedScan = deltaScans.getOrElseUpdate(planWithRemovedProjections.canonicalized, - filesForScan(scanGenerator, limit, filters, delta)) - val preparedIndex = getPreparedIndex(preparedScan, fileIndex) - optimizeGeneratedColumns(scan, preparedIndex, filters, limit, delta) - } - } - - protected def optimizeGeneratedColumns( - scan: LogicalPlan, - preparedIndex: PreparedDeltaFileIndex, - filters: Seq[Expression], - limit: Option[Int], - delta: LogicalRelation): LogicalPlan = { - if (limit.nonEmpty) { - // If we trigger limit push down, the filters must be partition filters. Since - // there are no data filters, we don't need to apply Generated Columns - // optimization. See `DeltaTableScan` for more details. - return DeltaTableUtils.replaceFileIndex(scan, preparedIndex) - } - if (!GeneratedColumn.partitionFilterOptimizationEnabled(spark)) { - DeltaTableUtils.replaceFileIndex(scan, preparedIndex) - } else { - val generatedPartitionFilters = - GeneratedColumn.generatePartitionFilters(spark, preparedIndex, filters, delta) - val scanWithFilters = - if (generatedPartitionFilters.nonEmpty) { - scan transformUp { - case delta @ DeltaTable(_: TahoeLogFileIndex) => - Filter(generatedPartitionFilters.reduceLeft(And), delta) - } - } else { - scan - } - DeltaTableUtils.replaceFileIndex(scanWithFilters, preparedIndex) - } - } - - override def apply(_plan: LogicalPlan): LogicalPlan = { - var plan = _plan - - // --- modified start - // Should not be applied to subqueries to avoid duplicate delta jobs. - val isSubquery = isSubqueryRoot(plan) - // Should not be applied to DataSourceV2 write plans, because they'll be planned later - // through a V1 fallback and only that later planning takes place within the transaction. - val isDataSourceV2 = plan.isInstanceOf[V2WriteCommand] - if (isSubquery || isDataSourceV2) { - return plan - } - - val shouldPrepareDeltaScan = ( - spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_STATS_SKIPPING) - ) - val updatedPlan = if (shouldPrepareDeltaScan) { - if (spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_OPTIMIZE_METADATA_QUERY_ENABLED)) { - plan = optimizeQueryWithMetadata(plan) - } - prepareDeltaScan(plan) - } else { - // If this query is running inside an active transaction and is touching the same table - // as the transaction, then mark that the entire table as tainted to be safe. - OptimisticTransaction.getActive.foreach { txn => - val logsInPlan = plan.collect { case DeltaTable(fileIndex) => fileIndex.deltaLog } - if (logsInPlan.exists(_.isSameLogAs(txn.deltaLog))) { - txn.readWholeTable() - } - } - - // Just return the plan if statistics based skipping is off. - // It will fall back to just partition pruning at planning time. - plan - } - // --- modified end - preprocessTablesWithDVs(updatedPlan) - } - - /** - * This is an extractor object. See https://docs.scala-lang.org/tour/extractor-objects.html. - */ - object DeltaTableScan { - - /** - * The components of DeltaTableScanType are: - * - the plan with removed projections. We remove projections as a plan differentiator - * because it does not affect file listing results. - * - filter expressions collected by `PhysicalOperation` - * - the `TahoeLogFileIndex` of the matched DeltaTable` - * - integer value of limit expression, if any - * - matched `DeltaTable` - */ - private type DeltaTableScanType = - (LogicalPlan, Seq[Expression], TahoeLogFileIndex, Option[Int], LogicalRelation) - - /** - * This is an extractor method (basically, the opposite of a constructor) which takes in an - * object `plan` and tries to give back the arguments as a [[DeltaTableScanType]]. - */ - def unapply(plan: LogicalPlan): Option[DeltaTableScanType] = { - val limitPushdownEnabled = spark.conf.get(DeltaSQLConf.DELTA_LIMIT_PUSHDOWN_ENABLED) - - // Remove projections as a plan differentiator because it does not affect file listing - // results. Plans with the same filters but different projections therefore will not have - // duplicate delta indexes. - def canonicalizePlanForDeltaFileListing(plan: LogicalPlan): LogicalPlan = { - val planWithRemovedProjections = plan.transformWithPruning(_.containsPattern(PROJECT)) { - case p: Project if p.projectList.forall(_.isInstanceOf[AttributeReference]) => p.child - } - planWithRemovedProjections - } - - plan match { - case LocalLimit(IntegerLiteral(limit), - PhysicalOperation(_, filters, delta @ DeltaTable(fileIndex: TahoeLogFileIndex))) - if limitPushdownEnabled && containsPartitionFiltersOnly(filters, fileIndex) => - Some((canonicalizePlanForDeltaFileListing(plan), filters, fileIndex, Some(limit), delta)) - case PhysicalOperation( - _, - filters, - delta @ DeltaTable(fileIndex: TahoeLogFileIndex)) => - val allFilters = fileIndex.partitionFilters ++ filters - Some((canonicalizePlanForDeltaFileListing(plan), allFilters, fileIndex, None, delta)) - - case _ => None - } - } - - private def containsPartitionFiltersOnly( - filters: Seq[Expression], - fileIndex: TahoeLogFileIndex): Boolean = { - val partitionColumns = fileIndex.snapshotAtAnalysis.metadata.partitionColumns - import DeltaTableUtils._ - filters.forall(expr => !containsSubquery(expr) && - isPredicatePartitionColumnsOnly(expr, partitionColumns, spark)) - } - } -} - -class PrepareDeltaScan(protected val spark: SparkSession) - extends PrepareDeltaScanBase - -object PrepareDeltaScanBase { - - /** - * Optional callback function that is called after `getDeltaScanGenerator` is called - * by the PrepareDeltaScan rule. This is primarily used for testing purposes. - */ - @volatile private var onGetDeltaScanGeneratorCallback: DeltaScanGenerator => Unit = _ - - /** - * Run a thunk of code with the given callback function injected into the PrepareDeltaScan rule. - * The callback function is called after `getDeltaScanGenerator` is called - * by the PrepareDeltaScan rule. This is primarily used for testing purposes. - */ - private[delta] def withCallbackOnGetDeltaScanGenerator[T]( - callback: DeltaScanGenerator => Unit)(thunk: => T): T = { - try { - onGetDeltaScanGeneratorCallback = callback - thunk - } finally { - onGetDeltaScanGeneratorCallback = null - } - } -} - -/** - * A [[TahoeFileIndex]] that uses a prepared scan to return the list of relevant files. - * This is injected into a query right before query planning by [[PrepareDeltaScan]] so that - * CBO and metering can accurately understand how much data will be read. - * - * @param versionScanned The version of the table that is being scanned, if a specific version - * has specifically been requested, e.g. by time travel. - */ -case class PreparedDeltaFileIndex( - override val spark: SparkSession, - override val deltaLog: DeltaLog, - override val path: Path, - preparedScan: DeltaScan, - versionScanned: Option[Long]) - extends TahoeFileIndexWithSnapshot(spark, deltaLog, path, preparedScan.scannedSnapshot) - with DeltaLogging { - - /** - * Returns all matching/valid files by the given `partitionFilters` and `dataFilters` - */ - override def matchingFiles( - partitionFilters: Seq[Expression], - dataFilters: Seq[Expression]): Seq[AddFile] = { - val currentFilters = ExpressionSet(partitionFilters ++ dataFilters) - val (addFiles, eventData) = if (currentFilters == preparedScan.allFilters || - currentFilters == preparedScan.filtersUsedForSkipping) { - // [[DeltaScan]] was created using `allFilters` out of which only `filtersUsedForSkipping` - // filters were used for skipping while creating the DeltaScan. - // If currentFilters is same as allFilters, then no need to recalculate files and we can use - // previous results. - // If currentFilters is same as filtersUsedForSkipping, then also we don't need to recalculate - // files as [[DeltaScan.files]] were calculates using filtersUsedForSkipping only. So if we - // recalculate, we will get same result. So we should use previous result in this case also. - val eventData = Map( - "reused" -> true, - "currentFiltersSameAsPreparedAllFilters" -> (currentFilters == preparedScan.allFilters), - "currentFiltersSameAsPreparedFiltersUsedForSkipping" -> - (currentFilters == preparedScan.filtersUsedForSkipping) - ) - (preparedScan.files.distinct, eventData) - } else { - logInfo( - s""" - |Prepared scan does not match actual filters. Reselecting files to query. - |Prepared: ${preparedScan.allFilters} - |Actual: ${currentFilters} - """.stripMargin) - val eventData = Map( - "reused" -> false, - "preparedAllFilters" -> preparedScan.allFilters.mkString(","), - "preparedFiltersUsedForSkipping" -> preparedScan.filtersUsedForSkipping.mkString(","), - "currentFilters" -> currentFilters.mkString(",") - ) - val files = preparedScan.scannedSnapshot.filesForScan(partitionFilters ++ dataFilters).files - (files, eventData) - } - recordDeltaEvent(deltaLog, - opType = "delta.preparedDeltaFileIndex.reuseSkippingResult", - data = eventData) - addFiles - } - - /** - * Returns the list of files that will be read when scanning this relation. This call may be - * very expensive for large tables. - */ - override def inputFiles: Array[String] = - preparedScan.files.map(f => absolutePath(f.path).toString).toArray - - /** Refresh any cached file listings */ - override def refresh(): Unit = { } - - /** Sum of table file sizes, in bytes */ - override def sizeInBytes: Long = - preparedScan.scanned.bytesCompressed - .getOrElse(spark.sessionState.conf.defaultSizeInBytes) - - override def equals(other: Any): Boolean = other match { - case p: PreparedDeltaFileIndex => - p.deltaLog == deltaLog && p.path == path && p.preparedScan == preparedScan && - p.partitionSchema == partitionSchema && p.versionScanned == versionScanned - case _ => false - } - - override def hashCode(): Int = { - Objects.hash(deltaLog, path, preparedScan, partitionSchema, versionScanned) - } - -} diff --git a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/execution/CHDeltaColumnarWrite.scala b/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/execution/CHDeltaColumnarWrite.scala deleted file mode 100644 index 0a1aee5c4bf..00000000000 --- a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/execution/CHDeltaColumnarWrite.scala +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution - -import org.apache.gluten.exception.GlutenNotSupportException - -import org.apache.spark.internal.io.FileCommitProtocol -import org.apache.spark.sql.execution.datasources.WriteJobDescription - -object CHDeltaColumnarWrite { - def apply( - jobTrackerID: String, - description: WriteJobDescription, - committer: FileCommitProtocol): CHColumnarWrite[FileCommitProtocol] = - throw new GlutenNotSupportException("Delta Native is not supported in Spark 3.3") -} diff --git a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala b/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala deleted file mode 100644 index 8c1062f4c7b..00000000000 --- a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.v2.clickhouse - -import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap -import org.apache.spark.sql.connector.catalog.Table -import org.apache.spark.sql.connector.expressions.Transform -import org.apache.spark.sql.delta._ -import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 -import org.apache.spark.sql.delta.commands.WriteIntoDelta -import org.apache.spark.sql.delta.commands.cdc.CDCReader -import org.apache.spark.sql.delta.sources.{DeltaDataSource, DeltaSourceUtils, DeltaSQLConf} -import org.apache.spark.sql.sources.BaseRelation -import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.util.CaseInsensitiveStringMap - -import org.apache.hadoop.fs.Path - -import scala.collection.JavaConverters._ -import scala.collection.mutable - -/** A DataSource V1 for integrating Delta into Spark SQL batch and Streaming APIs. */ -class ClickHouseDataSource extends DeltaDataSource { - - override def shortName(): String = { - ClickHouseConfig.NAME - } - - override def getTable( - schema: StructType, - partitioning: Array[Transform], - properties: java.util.Map[String, String]): Table = { - val options = new CaseInsensitiveStringMap(properties) - val path = options.get("path") - if (path == null) throw DeltaErrors.pathNotSpecifiedException - new ClickHouseTableV2( - SparkSession.active, - new Path(path), - options = properties.asScala.toMap, - clickhouseExtensionOptions = ClickHouseConfig - .createMergeTreeConfigurations( - ClickHouseConfig - .getMergeTreeConfigurations(properties) - .asJava) - ) - } - - override def createRelation( - sqlContext: SQLContext, - mode: SaveMode, - parameters: Map[String, String], - data: DataFrame): BaseRelation = { - val path = parameters.getOrElse("path", throw DeltaErrors.pathNotSpecifiedException) - val partitionColumns = parameters - .get(DeltaSourceUtils.PARTITIONING_COLUMNS_KEY) - .map(DeltaDataSource.decodePartitioningColumns) - .getOrElse(Nil) - - val deltaLog = DeltaLog.forTable(sqlContext.sparkSession, path, parameters) - // need to use the latest snapshot - val configs = if (deltaLog.update().version < 0) { - // when creating table, save the clickhouse config to the delta metadata - val clickHouseTableV2 = ClickHouseTableV2.getTable(deltaLog) - clickHouseTableV2.properties().asScala.toMap ++ DeltaConfigs - .validateConfigurations(parameters.filterKeys(_.startsWith("delta.")).toMap) - } else { - DeltaConfigs.validateConfigurations(parameters.filterKeys(_.startsWith("delta.")).toMap) - } - WriteIntoDelta( - deltaLog = deltaLog, - mode = mode, - new DeltaOptions(parameters, sqlContext.sparkSession.sessionState.conf), - partitionColumns = partitionColumns, - configuration = configs, - data = data - ).run(sqlContext.sparkSession) - - deltaLog.createRelation() - } - - override def createRelation( - sqlContext: SQLContext, - parameters: Map[String, String]): BaseRelation = { - recordFrameProfile("Delta", "DeltaDataSource.createRelation") { - val maybePath = parameters.getOrElse("path", throw DeltaErrors.pathNotSpecifiedException) - - // Log any invalid options that are being passed in - DeltaOptions.verifyOptions(CaseInsensitiveMap(parameters)) - - val timeTravelByParams = DeltaDataSource.getTimeTravelVersion(parameters) - var cdcOptions: mutable.Map[String, String] = mutable.Map.empty - val caseInsensitiveParams = new CaseInsensitiveStringMap(parameters.asJava) - if (CDCReader.isCDCRead(caseInsensitiveParams)) { - cdcOptions = mutable.Map[String, String](DeltaDataSource.CDC_ENABLED_KEY -> "true") - if (caseInsensitiveParams.containsKey(DeltaDataSource.CDC_START_VERSION_KEY)) { - cdcOptions(DeltaDataSource.CDC_START_VERSION_KEY) = - caseInsensitiveParams.get(DeltaDataSource.CDC_START_VERSION_KEY) - } - if (caseInsensitiveParams.containsKey(DeltaDataSource.CDC_START_TIMESTAMP_KEY)) { - cdcOptions(DeltaDataSource.CDC_START_TIMESTAMP_KEY) = - caseInsensitiveParams.get(DeltaDataSource.CDC_START_TIMESTAMP_KEY) - } - if (caseInsensitiveParams.containsKey(DeltaDataSource.CDC_END_VERSION_KEY)) { - cdcOptions(DeltaDataSource.CDC_END_VERSION_KEY) = - caseInsensitiveParams.get(DeltaDataSource.CDC_END_VERSION_KEY) - } - if (caseInsensitiveParams.containsKey(DeltaDataSource.CDC_END_TIMESTAMP_KEY)) { - cdcOptions(DeltaDataSource.CDC_END_TIMESTAMP_KEY) = - caseInsensitiveParams.get(DeltaDataSource.CDC_END_TIMESTAMP_KEY) - } - } - val dfOptions: Map[String, String] = - if ( - sqlContext.sparkSession.sessionState.conf.getConf( - DeltaSQLConf.LOAD_FILE_SYSTEM_CONFIGS_FROM_DATAFRAME_OPTIONS) - ) { - parameters - } else { - Map.empty - } - (new ClickHouseTableV2( - sqlContext.sparkSession, - new Path(maybePath), - timeTravelOpt = timeTravelByParams, - options = dfOptions, - cdcOptions = new CaseInsensitiveStringMap(cdcOptions.asJava) - )).toBaseRelation - } - } -} diff --git a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala b/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala deleted file mode 100644 index 47b2ae2bd1a..00000000000 --- a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala +++ /dev/null @@ -1,661 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.v2.clickhouse - -import org.apache.spark.sql.{AnalysisException, DataFrame, SparkSession} -import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.analysis.{NoSuchDatabaseException, NoSuchNamespaceException, NoSuchTableException} -import org.apache.spark.sql.catalyst.catalog._ -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.connector.catalog._ -import org.apache.spark.sql.connector.catalog.TableCapability.V1_BATCH_WRITE -import org.apache.spark.sql.connector.expressions.Transform -import org.apache.spark.sql.connector.write.{LogicalWriteInfo, V1Write, WriteBuilder} -import org.apache.spark.sql.delta.{DeltaConfigs, DeltaErrors, DeltaLog, DeltaOptions, DeltaTableUtils} -import org.apache.spark.sql.delta.DeltaTableIdentifier.gluePermissionError -import org.apache.spark.sql.delta.catalog.{ClickHouseTableV2, DeltaTableV2, TempClickHouseTableV2} -import org.apache.spark.sql.delta.commands.{CreateDeltaTableCommand, TableCreationModes, WriteIntoDelta} -import org.apache.spark.sql.delta.metering.DeltaLogging -import org.apache.spark.sql.delta.sources.{DeltaSourceUtils, DeltaSQLConf} -import org.apache.spark.sql.execution.datasources.{DataSource, PartitioningUtils} -import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils -import org.apache.spark.sql.execution.datasources.v2.utils.CatalogUtil -import org.apache.spark.sql.sources.InsertableRelation -import org.apache.spark.sql.types.StructType - -import org.apache.hadoop.fs.Path - -import java.util -import java.util.Locale - -import scala.collection.JavaConverters._ - -class ClickHouseSparkCatalog - extends DelegatingCatalogExtension - with StagingTableCatalog - with SupportsPathIdentifier - with DeltaLogging { - - val spark = SparkSession.active - - private def createCatalogTable( - ident: Identifier, - schema: StructType, - partitions: Array[Transform], - properties: util.Map[String, String] - ): Table = { - super.createTable(ident, schema, partitions, properties) - } - - override def createTable( - ident: Identifier, - schema: StructType, - partitions: Array[Transform], - properties: util.Map[String, String]): Table = { - if (CHDataSourceUtils.isClickHouseDataSourceName(getProvider(properties))) { - createClickHouseTable( - ident, - schema, - partitions, - properties, - Map.empty, - sourceQuery = None, - TableCreationModes.Create) - } else if (DeltaSourceUtils.isDeltaDataSourceName(getProvider(properties))) { - createDeltaTable( - ident, - schema, - partitions, - properties, - Map.empty, - sourceQuery = None, - TableCreationModes.Create - ) - } else { - createCatalogTable(ident, schema, partitions, properties) - } - } - - /** - * Creates a ClickHouse table - * - * @param ident - * The identifier of the table - * @param schema - * The schema of the table - * @param partitions - * The partition transforms for the table - * @param allTableProperties - * The table properties that configure the behavior of the table or provide information about - * the table - * @param writeOptions - * Options specific to the write during table creation or replacement - * @param sourceQuery - * A query if this CREATE request came from a CTAS or RTAS - * @param operation - * The specific table creation mode, whether this is a Create/Replace/Create or Replace - */ - private def createClickHouseTable( - ident: Identifier, - schema: StructType, - partitions: Array[Transform], - allTableProperties: util.Map[String, String], - writeOptions: Map[String, String], - sourceQuery: Option[DataFrame], - operation: TableCreationModes.CreationMode): Table = { - val (partitionColumns, maybeBucketSpec) = - CatalogUtil.convertPartitionTransforms(partitions) - var newSchema = schema - var newPartitionColumns = partitionColumns - var newBucketSpec = maybeBucketSpec - - // Delta does not support bucket feature, so save the bucket infos into properties if exists. - val tableProperties = - ClickHouseConfig.createMergeTreeConfigurations(allTableProperties, newBucketSpec) - - val isByPath = isPathIdentifier(ident) - val location = if (isByPath) { - Option(ident.name()) - } else { - Option(allTableProperties.get("location")) - } - val locUriOpt = location.map(CatalogUtils.stringToURI) - val storage = DataSource - .buildStorageFormatFromOptions(writeOptions) - .copy(locationUri = locUriOpt) - val tableType = - if (location.isDefined) CatalogTableType.EXTERNAL else CatalogTableType.MANAGED - val id = { - TableIdentifier(ident.name(), ident.namespace().lastOption) - } - val existingTableOpt = getExistingTableIfExists(id) - val loc = new Path(locUriOpt.getOrElse(spark.sessionState.catalog.defaultTablePath(id))) - val commentOpt = Option(allTableProperties.get("comment")) - - val tableDesc = new CatalogTable( - identifier = id, - tableType = tableType, - storage = storage, - schema = newSchema, - provider = Some(ClickHouseConfig.ALT_NAME), - partitionColumnNames = newPartitionColumns, - bucketSpec = newBucketSpec, - properties = tableProperties, - comment = commentOpt - ) - - val withDb = verifyTableAndSolidify(tableDesc, None, true) - - val writer = sourceQuery.map { - df => - WriteIntoDelta( - DeltaLog.forTable(spark, loc), - operation.mode, - new DeltaOptions(withDb.storage.properties, spark.sessionState.conf), - withDb.partitionColumnNames, - withDb.properties ++ commentOpt.map("comment" -> _), - df, - schemaInCatalog = if (newSchema != schema) Some(newSchema) else None - ) - } - try { - ClickHouseTableV2.temporalThreadLocalCHTable.set( - new TempClickHouseTableV2(spark, Some(withDb))) - - CreateDeltaTableCommand( - withDb, - existingTableOpt, - operation.mode, - writer, - operation = operation, - tableByPath = isByPath).run(spark) - } finally { - ClickHouseTableV2.temporalThreadLocalCHTable.remove() - } - - logInfo(s"create table ${ident.toString} successfully.") - loadTable(ident) - } - - /** - * Creates a Delta table - * - * @param ident - * The identifier of the table - * @param schema - * The schema of the table - * @param partitions - * The partition transforms for the table - * @param allTableProperties - * The table properties that configure the behavior of the table or provide information about - * the table - * @param writeOptions - * Options specific to the write during table creation or replacement - * @param sourceQuery - * A query if this CREATE request came from a CTAS or RTAS - * @param operation - * The specific table creation mode, whether this is a Create/Replace/Create or Replace - */ - private def createDeltaTable( - ident: Identifier, - schema: StructType, - partitions: Array[Transform], - allTableProperties: util.Map[String, String], - writeOptions: Map[String, String], - sourceQuery: Option[DataFrame], - operation: TableCreationModes.CreationMode - ): Table = { - // These two keys are tableProperties in data source v2 but not in v1, so we have to filter - // them out. Otherwise property consistency checks will fail. - val tableProperties = allTableProperties.asScala.filterKeys { - case TableCatalog.PROP_LOCATION => false - case TableCatalog.PROP_PROVIDER => false - case TableCatalog.PROP_COMMENT => false - case TableCatalog.PROP_OWNER => false - case TableCatalog.PROP_EXTERNAL => false - case "path" => false - case _ => true - }.toMap - val (partitionColumns, maybeBucketSpec) = - CatalogUtil.convertPartitionTransforms(partitions) - var newSchema = schema - var newPartitionColumns = partitionColumns - var newBucketSpec = maybeBucketSpec - val conf = spark.sessionState.conf - - val isByPath = isPathIdentifier(ident) - if ( - isByPath && !conf.getConf(DeltaSQLConf.DELTA_LEGACY_ALLOW_AMBIGUOUS_PATHS) - && allTableProperties.containsKey("location") - // The location property can be qualified and different from the path in the identifier, so - // we check `endsWith` here. - && Option(allTableProperties.get("location")).exists(!_.endsWith(ident.name())) - ) { - throw DeltaErrors.ambiguousPathsInCreateTableException( - ident.name(), - allTableProperties.get("location")) - } - val location = if (isByPath) { - Option(ident.name()) - } else { - Option(allTableProperties.get("location")) - } - val id = { - TableIdentifier(ident.name(), ident.namespace().lastOption) - } - var locUriOpt = location.map(CatalogUtils.stringToURI) - val existingTableOpt = getExistingTableIfExists(id) - val loc = locUriOpt - .orElse(existingTableOpt.flatMap(_.storage.locationUri)) - .getOrElse(spark.sessionState.catalog.defaultTablePath(id)) - val storage = DataSource - .buildStorageFormatFromOptions(writeOptions) - .copy(locationUri = Option(loc)) - val tableType = - if (location.isDefined) CatalogTableType.EXTERNAL else CatalogTableType.MANAGED - val commentOpt = Option(allTableProperties.get("comment")) - - var tableDesc = new CatalogTable( - identifier = id, - tableType = tableType, - storage = storage, - schema = newSchema, - provider = Some(DeltaSourceUtils.ALT_NAME), - partitionColumnNames = newPartitionColumns, - bucketSpec = newBucketSpec, - properties = tableProperties, - comment = commentOpt - ) - - val withDb = verifyTableAndSolidify(tableDesc, None) - - val writer = sourceQuery.map { - df => - WriteIntoDelta( - DeltaLog.forTable(spark, new Path(loc)), - operation.mode, - new DeltaOptions(withDb.storage.properties, spark.sessionState.conf), - withDb.partitionColumnNames, - withDb.properties ++ commentOpt.map("comment" -> _), - df, - schemaInCatalog = if (newSchema != schema) Some(newSchema) else None - ) - } - - CreateDeltaTableCommand( - withDb, - existingTableOpt, - operation.mode, - writer, - operation, - tableByPath = isByPath).run(spark) - - loadTable(ident) - } - - /** Performs checks on the parameters provided for table creation for a ClickHouse table. */ - private def verifyTableAndSolidify( - tableDesc: CatalogTable, - query: Option[LogicalPlan], - isMergeTree: Boolean = false): CatalogTable = { - - if (!isMergeTree && tableDesc.bucketSpec.isDefined) { - throw DeltaErrors.operationNotSupportedException("Bucketing", tableDesc.identifier) - } - - val schema = query - .map { - plan => - assert(tableDesc.schema.isEmpty, "Can't specify table schema in CTAS.") - plan.schema.asNullable - } - .getOrElse(tableDesc.schema) - - PartitioningUtils.validatePartitionColumn( - schema, - tableDesc.partitionColumnNames, - caseSensitive = false - ) // Delta is case insensitive - - val validatedConfigurations = if (isMergeTree) { - tableDesc.properties - } else { - DeltaConfigs.validateConfigurations(tableDesc.properties) - } - - val db = tableDesc.identifier.database.getOrElse(catalog.getCurrentDatabase) - val tableIdentWithDB = tableDesc.identifier.copy(database = Some(db)) - tableDesc.copy( - identifier = tableIdentWithDB, - schema = schema, - properties = validatedConfigurations) - } - - /** Checks if a table already exists for the provided identifier. */ - def getExistingTableIfExists(table: TableIdentifier): Option[CatalogTable] = { - // If this is a path identifier, we cannot return an existing CatalogTable. The Create command - // will check the file system itself - if (isPathIdentifier(table)) return None - val tableExists = catalog.tableExists(table) - if (tableExists) { - val oldTable = catalog.getTableMetadata(table) - if (oldTable.tableType == CatalogTableType.VIEW) { - throw new AnalysisException(s"$table is a view. You may not write data into a view.") - } - if ( - !DeltaSourceUtils.isDeltaTable(oldTable.provider) && - !CHDataSourceUtils.isClickHouseTable(oldTable.provider) - ) { - throw DeltaErrors.notADeltaTable(table.table) - } - Some(oldTable) - } else { - None - } - } - - private def getProvider(properties: util.Map[String, String]): String = { - Option(properties.get("provider")).getOrElse(ClickHouseConfig.NAME) - } - - override def loadTable(ident: Identifier): Table = { - try { - super.loadTable(ident) match { - case v1: V1Table if CHDataSourceUtils.isClickHouseTable(v1.catalogTable) => - new ClickHouseTableV2( - spark, - new Path(v1.catalogTable.location), - catalogTable = Some(v1.catalogTable), - tableIdentifier = Some(ident.toString)) - case v1: V1Table if DeltaTableUtils.isDeltaTable(v1.catalogTable) => - DeltaTableV2( - spark, - new Path(v1.catalogTable.location), - catalogTable = Some(v1.catalogTable), - tableIdentifier = Some(ident.toString)) - case o => - o - } - } catch { - case _: NoSuchDatabaseException | _: NoSuchNamespaceException | _: NoSuchTableException - if isPathIdentifier(ident) => - newDeltaPathTable(ident) - case e: AnalysisException if gluePermissionError(e) && isPathIdentifier(ident) => - logWarning( - "Received an access denied error from Glue. Assuming this " + - s"identifier ($ident) is path based.", - e) - newDeltaPathTable(ident) - } - } - - private def newDeltaPathTable(ident: Identifier): DeltaTableV2 = { - if (hasClickHouseNamespace(ident)) { - new ClickHouseTableV2(spark, new Path(ident.name())) - } else { - DeltaTableV2(spark, new Path(ident.name())) - } - } - - /** support to delete mergetree data from the external table */ - override def purgeTable(ident: Identifier): Boolean = { - try { - loadTable(ident) match { - case t: ClickHouseTableV2 => - val tableType = t.properties().getOrDefault("Type", "") - // file-based or external table - val isExternal = tableType.isEmpty || tableType.equalsIgnoreCase("external") - val tablePath = t.rootPath - // first delete the table metadata - val deletedTable = super.dropTable(ident) - if (deletedTable && isExternal) { - val fs = tablePath.getFileSystem(spark.sessionState.newHadoopConf()) - // delete all data if there is a external table - fs.delete(tablePath, true) - } - true - case _ => super.purgeTable(ident) - } - } catch { - case _: Exception => - false - } - } - - override def stageReplace( - ident: Identifier, - schema: StructType, - partitions: Array[Transform], - properties: util.Map[String, String]): StagedTable = - recordFrameProfile("DeltaCatalog", "stageReplace") { - if ( - CHDataSourceUtils.isClickHouseDataSourceName(getProvider(properties)) || - DeltaSourceUtils.isDeltaDataSourceName(getProvider(properties)) - ) { - new StagedDeltaTableV2(ident, schema, partitions, properties, TableCreationModes.Replace) - } else { - super.dropTable(ident) - val table = createCatalogTable(ident, schema, partitions, properties) - BestEffortStagedTable(ident, table, this) - } - } - - override def stageCreateOrReplace( - ident: Identifier, - schema: StructType, - partitions: Array[Transform], - properties: util.Map[String, String]): StagedTable = - recordFrameProfile("DeltaCatalog", "stageCreateOrReplace") { - if ( - CHDataSourceUtils.isClickHouseDataSourceName(getProvider(properties)) || - DeltaSourceUtils.isDeltaDataSourceName(getProvider(properties)) - ) { - new StagedDeltaTableV2( - ident, - schema, - partitions, - properties, - TableCreationModes.CreateOrReplace) - } else { - try super.dropTable(ident) - catch { - case _: NoSuchDatabaseException => // this is fine - case _: NoSuchTableException => // this is fine - } - val table = createCatalogTable(ident, schema, partitions, properties) - BestEffortStagedTable(ident, table, this) - } - } - - override def stageCreate( - ident: Identifier, - schema: StructType, - partitions: Array[Transform], - properties: util.Map[String, String]): StagedTable = - recordFrameProfile("DeltaCatalog", "stageCreate") { - if ( - CHDataSourceUtils.isClickHouseDataSourceName(getProvider(properties)) || - DeltaSourceUtils.isDeltaDataSourceName(getProvider(properties)) - ) { - new StagedDeltaTableV2(ident, schema, partitions, properties, TableCreationModes.Create) - } else { - val table = createCatalogTable(ident, schema, partitions, properties) - BestEffortStagedTable(ident, table, this) - } - } - - /** - * A staged delta table, which creates a HiveMetaStore entry and appends data if this was a - * CTAS/RTAS command. We have a ugly way of using this API right now, but it's the best way to - * maintain old behavior compatibility between Databricks Runtime and OSS Delta Lake. - */ - private class StagedDeltaTableV2( - ident: Identifier, - override val schema: StructType, - val partitions: Array[Transform], - override val properties: util.Map[String, String], - operation: TableCreationModes.CreationMode) - extends StagedTable - with SupportsWrite { - - private var asSelectQuery: Option[DataFrame] = None - private var writeOptions: Map[String, String] = Map.empty - - override def commitStagedChanges(): Unit = - recordFrameProfile("DeltaCatalog", "commitStagedChanges") { - val conf = spark.sessionState.conf - val props = new util.HashMap[String, String]() - // Options passed in through the SQL API will show up both with an "option." prefix and - // without in Spark 3.1, so we need to remove those from the properties - val optionsThroughProperties = properties.asScala.collect { - case (k, _) if k.startsWith("option.") => k.stripPrefix("option.") - }.toSet - val sqlWriteOptions = new util.HashMap[String, String]() - properties.asScala.foreach { - case (k, v) => - if (!k.startsWith("option.") && !optionsThroughProperties.contains(k)) { - // Do not add to properties - props.put(k, v) - } else if (optionsThroughProperties.contains(k)) { - sqlWriteOptions.put(k, v) - } - } - if (writeOptions.isEmpty && !sqlWriteOptions.isEmpty) { - writeOptions = sqlWriteOptions.asScala.toMap - } - if (conf.getConf(DeltaSQLConf.DELTA_LEGACY_STORE_WRITER_OPTIONS_AS_PROPS)) { - // Legacy behavior - writeOptions.foreach { case (k, v) => props.put(k, v) } - } else { - writeOptions.foreach { - case (k, v) => - // Continue putting in Delta prefixed options to avoid breaking workloads - if (k.toLowerCase(Locale.ROOT).startsWith("delta.")) { - props.put(k, v) - } - } - } - if (CHDataSourceUtils.isClickHouseDataSourceName(getProvider(properties))) { - createClickHouseTable( - ident, - schema, - partitions, - props, - writeOptions, - asSelectQuery, - operation) - } else { - createDeltaTable(ident, schema, partitions, props, writeOptions, asSelectQuery, operation) - } - } - - override def name(): String = ident.name() - - override def abortStagedChanges(): Unit = {} - - override def capabilities(): util.Set[TableCapability] = Set(V1_BATCH_WRITE).asJava - - override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = { - writeOptions = info.options.asCaseSensitiveMap().asScala.toMap - new DeltaV1WriteBuilder - } - - /* - * WriteBuilder for creating a Delta table. - */ - private class DeltaV1WriteBuilder extends WriteBuilder { - override def build(): V1Write = new V1Write { - override def toInsertableRelation(): InsertableRelation = { - new InsertableRelation { - override def insert(data: DataFrame, overwrite: Boolean): Unit = { - asSelectQuery = Option(data) - } - } - } - } - } - } - - private case class BestEffortStagedTable(ident: Identifier, table: Table, catalog: TableCatalog) - extends StagedTable - with SupportsWrite { - override def abortStagedChanges(): Unit = catalog.dropTable(ident) - - override def commitStagedChanges(): Unit = {} - - // Pass through - override def name(): String = table.name() - override def schema(): StructType = table.schema() - override def partitioning(): Array[Transform] = table.partitioning() - override def capabilities(): util.Set[TableCapability] = table.capabilities() - override def properties(): util.Map[String, String] = table.properties() - - override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = table match { - case supportsWrite: SupportsWrite => supportsWrite.newWriteBuilder(info) - case _ => throw DeltaErrors.unsupportedWriteStagedTable(name) - } - } -} - -/** - * A trait for handling table access through clickhouse.`/some/path`. This is a stop-gap solution - * until PathIdentifiers are implemented in Apache Spark. - */ -trait SupportsPathIdentifier extends TableCatalog { - self: ClickHouseSparkCatalog => - - protected lazy val catalog: SessionCatalog = spark.sessionState.catalog - - override def tableExists(ident: Identifier): Boolean = { - if (isPathIdentifier(ident)) { - val path = new Path(ident.name()) - val fs = path.getFileSystem(spark.sessionState.newHadoopConf()) - fs.exists(path) && fs.listStatus(path).nonEmpty - } else { - super.tableExists(ident) - } - } - - protected def isPathIdentifier(ident: Identifier): Boolean = { - // Should be a simple check of a special PathIdentifier class in the future - try { - supportSQLOnFile && (hasClickHouseNamespace(ident) || hasDeltaNamespace(ident)) && - new Path(ident.name()).isAbsolute - } catch { - case _: IllegalArgumentException => false - } - } - - protected def isPathIdentifier(table: CatalogTable): Boolean = { - isPathIdentifier(table.identifier) - } - - protected def isPathIdentifier(tableIdentifier: TableIdentifier): Boolean = { - isPathIdentifier(Identifier.of(tableIdentifier.database.toArray, tableIdentifier.table)) - } - - private def supportSQLOnFile: Boolean = spark.sessionState.conf.runSQLonFile - - protected def hasClickHouseNamespace(ident: Identifier): Boolean = { - ident.namespace().length == 1 && - CHDataSourceUtils.isClickHouseDataSourceName(ident.namespace().head) - } - - protected def hasDeltaNamespace(ident: Identifier): Boolean = { - ident.namespace().length == 1 && DeltaSourceUtils.isDeltaDataSourceName(ident.namespace().head) - } -} diff --git a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/metadata/AddFileTags.scala b/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/metadata/AddFileTags.scala deleted file mode 100644 index c336c2fd7ec..00000000000 --- a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/metadata/AddFileTags.scala +++ /dev/null @@ -1,250 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata - -import org.apache.spark.sql.delta.actions.{AddFile, DeletionVectorDescriptor} -import org.apache.spark.sql.delta.util.MergeTreePartitionUtils -import org.apache.spark.sql.execution.datasources.clickhouse.WriteReturnedMetric - -import com.fasterxml.jackson.core.`type`.TypeReference -import com.fasterxml.jackson.databind.ObjectMapper -import org.apache.hadoop.fs.Path - -import java.util.{List => JList} - -import scala.collection.JavaConverters._ - -@SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) -class AddMergeTreeParts( - val database: String, - val table: String, - val engine: String, // default is "MergeTree" - val tablePath: String, // table path - val targetNode: String, // the node which the current part is generated - val name: String, // part name - val uuid: String, - val rows: Long, // row count - override val size: Long, // the size of the part - val dataCompressedBytes: Long, - val dataUncompressedBytes: Long, - override val modificationTime: Long, - val partitionId: String, - val minBlockNumber: Long, - val maxBlockNumber: Long, - val level: Int, - val dataVersion: Long, - val bucketNum: String, - val dirName: String, - override val dataChange: Boolean, - val partition: String = "", - val defaultCompressionCodec: String = "LZ4", - override val stats: String = "", - override val partitionValues: Map[String, String] = Map.empty[String, String], - val partType: String = "Wide", - val active: Int = 1, - val marks: Long = -1L, // mark count - val marksBytes: Long = -1L, - val removeTime: Long = -1L, - val refcount: Int = -1, - val minDate: Int = -1, - val maxDate: Int = -1, - val minTime: Long = -1L, - val maxTime: Long = -1L, - val primaryKeyBytesInMemory: Long = -1L, - val primaryKeyBytesInMemoryAllocated: Long = -1L, - val isFrozen: Int = 0, - val diskName: String = "default", - val hashOfAllFiles: String = "", - val hashOfUncompressedFiles: String = "", - val uncompressedHashOfCompressedFiles: String = "", - val deleteTtlInfoMin: Long = -1L, - val deleteTtlInfoMax: Long = -1L, - val moveTtlInfoExpression: String = "", - val moveTtlInfoMin: Long = -1L, - val moveTtlInfoMax: Long = -1L, - val recompressionTtlInfoExpression: String = "", - val recompressionTtlInfoMin: Long = -1L, - val recompressionTtlInfoMax: Long = -1L, - val groupByTtlInfoExpression: String = "", - val groupByTtlInfoMin: Long = -1L, - val groupByTtlInfoMax: Long = -1L, - val rowsWhereTtlInfoExpression: String = "", - val rowsWhereTtlInfoMin: Long = -1L, - val rowsWhereTtlInfoMax: Long = -1L, - override val tags: Map[String, String] = null, - override val deletionVector: DeletionVectorDescriptor = null) - extends AddFile( - name, - partitionValues, - size, - modificationTime, - dataChange, - stats, - tags, - deletionVector) { - - def fullPartPath(): String = { - dirName + "/" + name - } -} - -object AddFileTags { - // scalastyle:off argcount - private def partsInfoToAddFile( - database: String, - table: String, - engine: String, - tablePath: String, - targetNode: String, - name: String, - uuid: String, - rows: Long, - bytesOnDisk: Long, - dataCompressedBytes: Long, - dataUncompressedBytes: Long, - modificationTime: Long, - partitionId: String, - minBlockNumber: Long, - maxBlockNumber: Long, - level: Int, - dataVersion: Long, - bucketNum: String, - dirName: String, - dataChange: Boolean, - partition: String = "", - defaultCompressionCodec: String = "LZ4", - stats: String = "", - partitionValues: Map[String, String] = Map.empty[String, String], - marks: Long = -1L): AddFile = { - // scalastyle:on argcount - val tags = Map[String, String]( - "database" -> database, - "table" -> table, - "engine" -> engine, - "path" -> tablePath, - "targetNode" -> targetNode, - "partition" -> partition, - "uuid" -> uuid, - "rows" -> rows.toString, - "bytesOnDisk" -> bytesOnDisk.toString, - "dataCompressedBytes" -> dataCompressedBytes.toString, - "dataUncompressedBytes" -> dataUncompressedBytes.toString, - "modificationTime" -> modificationTime.toString, - "partitionId" -> partitionId, - "minBlockNumber" -> minBlockNumber.toString, - "maxBlockNumber" -> maxBlockNumber.toString, - "level" -> level.toString, - "dataVersion" -> dataVersion.toString, - "defaultCompressionCodec" -> defaultCompressionCodec, - "bucketNum" -> bucketNum, - "dirName" -> dirName, - "marks" -> marks.toString - ) - val mapper: ObjectMapper = new ObjectMapper() - val rootNode = mapper.createObjectNode() - rootNode.put("numRecords", rows) - rootNode.put("minValues", "") - rootNode.put("maxValues", "") - rootNode.put("nullCount", "") - // Add the `stats` into delta meta log - val metricsStats = mapper.writeValueAsString(rootNode) - val uriName = new Path(name).toUri.toString - AddFile(uriName, partitionValues, bytesOnDisk, modificationTime, dataChange, metricsStats, tags) - } - - def addFileToAddMergeTreeParts(addFile: AddFile): AddMergeTreeParts = { - assert(addFile.tags != null && addFile.tags.nonEmpty) - new AddMergeTreeParts( - addFile.tags("database"), - addFile.tags("table"), - addFile.tags("engine"), - addFile.tags("path"), - addFile.tags("targetNode"), - addFile.path, - addFile.tags("uuid"), - addFile.tags("rows").toLong, - addFile.size, - addFile.tags("dataCompressedBytes").toLong, - addFile.tags("dataUncompressedBytes").toLong, - addFile.modificationTime, - addFile.tags("partitionId"), - addFile.tags("minBlockNumber").toLong, - addFile.tags("maxBlockNumber").toLong, - addFile.tags("level").toInt, - addFile.tags("dataVersion").toLong, - addFile.tags("bucketNum"), - addFile.tags("dirName"), - addFile.dataChange, - addFile.tags("partition"), - addFile.tags("defaultCompressionCodec"), - addFile.stats, - addFile.partitionValues, - marks = addFile.tags("marks").toLong, - tags = addFile.tags, - deletionVector = addFile.deletionVector - ) - } - - def partsMetricsToAddFile( - database: String, - tableName: String, - originPathStr: String, - returnedMetrics: String, - hostName: Seq[String]): Seq[AddFile] = { - - val mapper: ObjectMapper = new ObjectMapper() - val values: JList[WriteReturnedMetric] = - mapper.readValue(returnedMetrics, new TypeReference[JList[WriteReturnedMetric]]() {}) - val path = new Path(originPathStr) - val modificationTime = System.currentTimeMillis() - - values.asScala.map { - value => - val partitionValues = if (value.getPartitionValues.isEmpty) { - Map.empty[String, String] - } else { - MergeTreePartitionUtils.parsePartitions(value.getPartitionValues) - } - - AddFileTags.partsInfoToAddFile( - database, - tableName, - "MergeTree", - path.toUri.getPath, - hostName.map(_.trim).mkString(","), - value.getPartName, - "", - value.getRowCount, - value.getDiskSize, - -1L, - -1L, - modificationTime, - "", - -1L, - -1L, - -1, - -1L, - value.getBucketId, - path.toString, - dataChange = true, - "", - partitionValues = partitionValues, - marks = value.getMarkCount - ) - }.toSeq - } -} diff --git a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala b/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala deleted file mode 100644 index 266b1324d19..00000000000 --- a/backends-clickhouse/src-delta23/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.v2.clickhouse.source - -import org.apache.spark.sql.delta.{DeltaParquetFileFormat, MergeTreeFileFormat} -import org.apache.spark.sql.delta.actions.Metadata - -@SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) -class DeltaMergeTreeFileFormat(metadata: Metadata) - extends DeltaParquetFileFormat(metadata) - with MergeTreeFileFormat { - - override def equals(other: Any): Boolean = { - other match { - case ff: DeltaMergeTreeFileFormat => - ff.columnMappingMode == columnMappingMode && - ff.referenceSchema == referenceSchema && - ff.isSplittable == isSplittable && - ff.disablePushDowns == disablePushDowns - case _ => false - } - } - - override def hashCode(): Int = getClass.getCanonicalName.hashCode() -} diff --git a/backends-clickhouse/src-delta23/test/scala/org/apache/spark/gluten/delta/DeltaStatsUtils.scala b/backends-clickhouse/src-delta23/test/scala/org/apache/spark/gluten/delta/DeltaStatsUtils.scala deleted file mode 100644 index 5d9f761e8a7..00000000000 --- a/backends-clickhouse/src-delta23/test/scala/org/apache/spark/gluten/delta/DeltaStatsUtils.scala +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.gluten.delta - -import org.apache.spark.sql.{DataFrame, SparkSession} - -object DeltaStatsUtils { - - def statsDF( - sparkSession: SparkSession, - deltaJson: String, - schema: String - ): DataFrame = { - throw new IllegalAccessException("Method not used below spark 3.5") - } -} diff --git a/backends-clickhouse/src-iceberg-spark33/test/java/org/apache/gluten/execution/iceberg/TestFlinkUpsert.java b/backends-clickhouse/src-iceberg-spark33/test/java/org/apache/gluten/execution/iceberg/TestFlinkUpsert.java deleted file mode 100644 index 0c734b152f4..00000000000 --- a/backends-clickhouse/src-iceberg-spark33/test/java/org/apache/gluten/execution/iceberg/TestFlinkUpsert.java +++ /dev/null @@ -1,539 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.execution.iceberg; - -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.table.api.EnvironmentSettings; -import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; -import org.apache.flink.types.Row; -import org.apache.iceberg.FileFormat; -import org.apache.iceberg.Parameter; -import org.apache.iceberg.Parameters; -import org.apache.iceberg.TableProperties; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.flink.CatalogTestBase; -import org.apache.iceberg.flink.MiniClusterResource; -import org.apache.iceberg.flink.TestHelpers; -import org.apache.iceberg.relocated.com.google.common.collect.Iterables; -import org.apache.iceberg.relocated.com.google.common.collect.Lists; -import org.apache.iceberg.relocated.com.google.common.collect.Maps; -import org.apache.spark.sql.Dataset; -import org.apache.spark.sql.SparkSession; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.TestTemplate; - -import java.time.LocalDate; -import java.time.ZoneId; -import java.util.Date; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.stream.Collectors; - -public class TestFlinkUpsert extends CatalogTestBase { - - @Parameter(index = 2) - private FileFormat format; - - @Parameter(index = 3) - private boolean isStreamingJob; - - private final Map tableUpsertProps = Maps.newHashMap(); - private TableEnvironment tEnv; - private SparkSession spark; - private ClickHouseIcebergHiveTableSupport hiveTableSupport; - - @Parameters(name = "catalogName={0}, baseNamespace={1}, format={2}, isStreaming={3}") - public static List parameters() { - List parameters = Lists.newArrayList(); - // ignore ORC and AVRO, ch backend only support PARQUET - for (FileFormat format : new FileFormat[] {FileFormat.PARQUET}) { - for (Boolean isStreaming : new Boolean[] {true, false}) { - // Only test with one catalog as this is a file operation concern. - // FlinkCatalogTestBase requires the catalog name start with testhadoop if using hadoop - // catalog. - String catalogName = "testhive"; - Namespace baseNamespace = Namespace.empty(); - parameters.add(new Object[] {catalogName, baseNamespace, format, isStreaming}); - } - } - return parameters; - } - - @Override - protected TableEnvironment getTableEnv() { - if (tEnv == null) { - synchronized (this) { - EnvironmentSettings.Builder settingsBuilder = EnvironmentSettings.newInstance(); - if (isStreamingJob) { - settingsBuilder.inStreamingMode(); - StreamExecutionEnvironment env = - StreamExecutionEnvironment.getExecutionEnvironment( - MiniClusterResource.DISABLE_CLASSLOADER_CHECK_CONFIG); - env.enableCheckpointing(400); - env.setMaxParallelism(2); - env.setParallelism(2); - tEnv = StreamTableEnvironment.create(env, settingsBuilder.build()); - } else { - settingsBuilder.inBatchMode(); - tEnv = TableEnvironment.create(settingsBuilder.build()); - } - } - } - return tEnv; - } - - @Override - @BeforeEach - public void before() { - super.before(); - sql("CREATE DATABASE IF NOT EXISTS %s", flinkDatabase); - sql("USE CATALOG %s", catalogName); - sql("USE %s", DATABASE); - tableUpsertProps.put(TableProperties.FORMAT_VERSION, "2"); - tableUpsertProps.put(TableProperties.UPSERT_ENABLED, "true"); - tableUpsertProps.put(TableProperties.DEFAULT_FILE_FORMAT, format.name()); - - hiveTableSupport = new ClickHouseIcebergHiveTableSupport(); - hiveTableSupport.initSparkConf( - hiveConf.get("hive.metastore.uris"), - catalogName, - String.format("file://%s", this.warehouseRoot())); - hiveTableSupport.initializeSession(); - spark = hiveTableSupport.spark(); - } - - @Override - @AfterEach - public void clean() { - sql("DROP DATABASE IF EXISTS %s", flinkDatabase); - super.clean(); - - hiveTableSupport.clean(); - } - - static String toWithClause(Map props) { - StringBuilder builder = new StringBuilder(); - builder.append("("); - int propCount = 0; - for (Map.Entry entry : props.entrySet()) { - if (propCount > 0) { - builder.append(","); - } - builder - .append("'") - .append(entry.getKey()) - .append("'") - .append("=") - .append("'") - .append(entry.getValue()) - .append("'"); - propCount++; - } - builder.append(")"); - return builder.toString(); - } - - @TestTemplate - public void testUpsertAndQuery() { - String tableName = "test_upsert_query"; - LocalDate dt20220301 = LocalDate.of(2022, 3, 1); - LocalDate dt20220302 = LocalDate.of(2022, 3, 2); - Date dt20220301Spark = - Date.from(dt20220301.atStartOfDay().atZone(ZoneId.systemDefault()).toInstant()); - Date dt20220302Spark = - Date.from(dt20220302.atStartOfDay().atZone(ZoneId.systemDefault()).toInstant()); - - sql( - "CREATE TABLE %s(id INT NOT NULL, name STRING NOT NULL, dt DATE, " - + "PRIMARY KEY(id,dt) NOT ENFORCED) " - + "PARTITIONED BY (dt) WITH %s", - tableName, toWithClause(tableUpsertProps)); - - try { - sql( - "INSERT INTO %s VALUES " - + "(1, 'Bill', DATE '2022-03-01')," - + "(1, 'Jane', DATE '2022-03-01')," - + "(2, 'Jane', DATE '2022-03-01')", - tableName); - - sql( - "INSERT INTO %s VALUES " - + "(2, 'Bill', DATE '2022-03-01')," - + "(1, 'Jane', DATE '2022-03-02')," - + "(2, 'Jane', DATE '2022-03-02')", - tableName); - - List rowsOn20220301 = - Lists.newArrayList(Row.of(1, "Jane", dt20220301), Row.of(2, "Bill", dt20220301)); - TestHelpers.assertRows( - sql("SELECT * FROM %s WHERE dt < '2022-03-02'", tableName), rowsOn20220301); - List rowsOn20220301Spark = - Lists.newArrayList( - Row.of(1, "Jane", dt20220301Spark), Row.of(2, "Bill", dt20220301Spark)); - TestHelpers.assertRows( - convertToFlinkRows( - spark.sql( - String.format( - Locale.ROOT, - "SELECT * FROM %s.db.%s WHERE dt < '2022-03-02'", - catalogName, - tableName)), - 3), - rowsOn20220301Spark); - - List rowsOn20220302 = - Lists.newArrayList(Row.of(1, "Jane", dt20220302), Row.of(2, "Jane", dt20220302)); - TestHelpers.assertRows( - sql("SELECT * FROM %s WHERE dt = '2022-03-02'", tableName), rowsOn20220302); - List rowsOn20220302Spark = - Lists.newArrayList( - Row.of(1, "Jane", dt20220302Spark), Row.of(2, "Jane", dt20220302Spark)); - TestHelpers.assertRows( - convertToFlinkRows( - spark.sql( - String.format( - Locale.ROOT, - "SELECT * FROM %s.db.%s WHERE dt = '2022-03-02'", - catalogName, - tableName)), - 3), - rowsOn20220302Spark); - - TestHelpers.assertRows( - sql("SELECT * FROM %s", tableName), - Lists.newArrayList(Iterables.concat(rowsOn20220301, rowsOn20220302))); - TestHelpers.assertRows( - convertToFlinkRows( - spark.sql( - String.format(Locale.ROOT, "SELECT * FROM %s.db.%s", catalogName, tableName)), - 3), - Lists.newArrayList(Iterables.concat(rowsOn20220301Spark, rowsOn20220302Spark))); - } finally { - sql("DROP TABLE IF EXISTS %s.%s", flinkDatabase, tableName); - } - } - - private List convertToFlinkRows(Dataset rows, int columnCount) { - return rows.collectAsList().stream() - .map( - r -> { - switch (columnCount) { - case 1: - return Row.of(r.get(0)); - case 2: - return Row.of(r.get(0), r.get(1)); - case 3: - return Row.of(r.get(0), r.get(1), r.get(2)); - default: - throw new IllegalArgumentException("Unsupported column count: " + columnCount); - } - }) - .collect(Collectors.toList()); - } - - @TestTemplate - public void testUpsertOptions() { - String tableName = "test_upsert_options"; - LocalDate dt20220301 = LocalDate.of(2022, 3, 1); - LocalDate dt20220302 = LocalDate.of(2022, 3, 2); - Date dt20220301Spark = - Date.from(dt20220301.atStartOfDay().atZone(ZoneId.systemDefault()).toInstant()); - Date dt20220302Spark = - Date.from(dt20220302.atStartOfDay().atZone(ZoneId.systemDefault()).toInstant()); - - Map optionsUpsertProps = Maps.newHashMap(tableUpsertProps); - optionsUpsertProps.remove(TableProperties.UPSERT_ENABLED); - sql( - "CREATE TABLE %s(id INT NOT NULL, name STRING NOT NULL, dt DATE, " - + "PRIMARY KEY(id,dt) NOT ENFORCED) " - + "PARTITIONED BY (dt) WITH %s", - tableName, toWithClause(optionsUpsertProps)); - - try { - sql( - "INSERT INTO %s /*+ OPTIONS('upsert-enabled'='true')*/ VALUES " - + "(1, 'Bill', DATE '2022-03-01')," - + "(1, 'Jane', DATE '2022-03-01')," - + "(2, 'Jane', DATE '2022-03-01')", - tableName); - - sql( - "INSERT INTO %s /*+ OPTIONS('upsert-enabled'='true')*/ VALUES " - + "(2, 'Bill', DATE '2022-03-01')," - + "(1, 'Jane', DATE '2022-03-02')," - + "(2, 'Jane', DATE '2022-03-02')", - tableName); - - List rowsOn20220301 = - Lists.newArrayList(Row.of(1, "Jane", dt20220301), Row.of(2, "Bill", dt20220301)); - TestHelpers.assertRows( - sql("SELECT * FROM %s WHERE dt < '2022-03-02'", tableName), rowsOn20220301); - List rowsOn20220301Spark = - Lists.newArrayList( - Row.of(1, "Jane", dt20220301Spark), Row.of(2, "Bill", dt20220301Spark)); - TestHelpers.assertRows( - convertToFlinkRows( - spark.sql( - String.format( - Locale.ROOT, - "SELECT * FROM %s.db.%s WHERE dt < '2022-03-02'", - catalogName, - tableName)), - 3), - rowsOn20220301Spark); - - List rowsOn20220302 = - Lists.newArrayList(Row.of(1, "Jane", dt20220302), Row.of(2, "Jane", dt20220302)); - TestHelpers.assertRows( - sql("SELECT * FROM %s WHERE dt = '2022-03-02'", tableName), rowsOn20220302); - List rowsOn20220302Spark = - Lists.newArrayList( - Row.of(1, "Jane", dt20220302Spark), Row.of(2, "Jane", dt20220302Spark)); - TestHelpers.assertRows( - convertToFlinkRows( - spark.sql( - String.format( - Locale.ROOT, - "SELECT * FROM %s.db.%s WHERE dt = '2022-03-02'", - catalogName, - tableName)), - 3), - rowsOn20220302Spark); - - TestHelpers.assertRows( - sql("SELECT * FROM %s", tableName), - Lists.newArrayList(Iterables.concat(rowsOn20220301, rowsOn20220302))); - TestHelpers.assertRows( - convertToFlinkRows( - spark.sql( - String.format(Locale.ROOT, "SELECT * FROM %s.db.%s", catalogName, tableName)), - 3), - Lists.newArrayList(Iterables.concat(rowsOn20220301Spark, rowsOn20220302Spark))); - } finally { - sql("DROP TABLE IF EXISTS %s.%s", flinkDatabase, tableName); - } - } - - @TestTemplate - public void testPrimaryKeyEqualToPartitionKey() { - // This is an SQL based reproduction of TestFlinkIcebergSinkV2#testUpsertOnDataKey - String tableName = "upsert_on_id_key"; - try { - sql( - "CREATE TABLE %s(id INT NOT NULL, name STRING NOT NULL, PRIMARY KEY(id) NOT ENFORCED) " - + "PARTITIONED BY (id) WITH %s", - tableName, toWithClause(tableUpsertProps)); - - sql("INSERT INTO %s VALUES " + "(1, 'Bill')," + "(1, 'Jane')," + "(2, 'Bill')", tableName); - - List rows = Lists.newArrayList(Row.of(1, "Jane"), Row.of(2, "Bill")); - TestHelpers.assertRows(sql("SELECT * FROM %s", tableName), rows); - TestHelpers.assertRows( - convertToFlinkRows( - spark.sql( - String.format(Locale.ROOT, "SELECT * FROM %s.db.%s", catalogName, tableName)), - 2), - rows); - - sql("INSERT INTO %s VALUES " + "(1, 'Bill')," + "(2, 'Jane')", tableName); - - List rows2 = Lists.newArrayList(Row.of(1, "Bill"), Row.of(2, "Jane")); - TestHelpers.assertRows(sql("SELECT * FROM %s", tableName), rows2); - spark.sql(String.format(Locale.ROOT, "REFRESH TABLE %s.db.%s", catalogName, tableName)); - TestHelpers.assertRows( - convertToFlinkRows( - spark.sql( - String.format(Locale.ROOT, "SELECT * FROM %s.db.%s", catalogName, tableName)), - 2), - rows2); - - sql("INSERT INTO %s VALUES " + "(3, 'Bill')," + "(4, 'Jane')", tableName); - - List rows3 = - Lists.newArrayList( - Row.of(1, "Bill"), Row.of(2, "Jane"), Row.of(3, "Bill"), Row.of(4, "Jane")); - TestHelpers.assertRows(sql("SELECT * FROM %s", tableName), rows3); - spark.sql(String.format(Locale.ROOT, "REFRESH TABLE %s.db.%s", catalogName, tableName)); - TestHelpers.assertRows( - convertToFlinkRows( - spark.sql( - String.format(Locale.ROOT, "SELECT * FROM %s.db.%s", catalogName, tableName)), - 2), - rows3); - } finally { - sql("DROP TABLE IF EXISTS %s.%s", flinkDatabase, tableName); - } - } - - @TestTemplate - public void testPrimaryKeyFieldsAtBeginningOfSchema() { - String tableName = "upsert_on_pk_at_schema_start"; - LocalDate dt = LocalDate.of(2022, 3, 1); - Date dtSpark = Date.from(dt.atStartOfDay().atZone(ZoneId.systemDefault()).toInstant()); - - try { - sql( - "CREATE TABLE %s(id INT, dt DATE NOT NULL, name STRING NOT NULL, " - + "PRIMARY KEY(id,dt) NOT ENFORCED) " - + "PARTITIONED BY (dt) WITH %s", - tableName, toWithClause(tableUpsertProps)); - - sql( - "INSERT INTO %s VALUES " - + "(1, DATE '2022-03-01', 'Andy')," - + "(1, DATE '2022-03-01', 'Bill')," - + "(2, DATE '2022-03-01', 'Jane')", - tableName); - - TestHelpers.assertRows( - sql("SELECT * FROM %s", tableName), - Lists.newArrayList(Row.of(1, dt, "Bill"), Row.of(2, dt, "Jane"))); - TestHelpers.assertRows( - convertToFlinkRows( - spark.sql( - String.format(Locale.ROOT, "SELECT * FROM %s.db.%s", catalogName, tableName)), - 3), - Lists.newArrayList(Row.of(1, dtSpark, "Bill"), Row.of(2, dtSpark, "Jane"))); - - sql( - "INSERT INTO %s VALUES " - + "(1, DATE '2022-03-01', 'Jane')," - + "(2, DATE '2022-03-01', 'Bill')", - tableName); - - TestHelpers.assertRows( - sql("SELECT * FROM %s", tableName), - Lists.newArrayList(Row.of(1, dt, "Jane"), Row.of(2, dt, "Bill"))); - spark.sql(String.format(Locale.ROOT, "REFRESH TABLE %s.db.%s", catalogName, tableName)); - TestHelpers.assertRows( - convertToFlinkRows( - spark.sql( - String.format(Locale.ROOT, "SELECT * FROM %s.db.%s", catalogName, tableName)), - 3), - Lists.newArrayList(Row.of(1, dtSpark, "Jane"), Row.of(2, dtSpark, "Bill"))); - - sql( - "INSERT INTO %s VALUES " - + "(3, DATE '2022-03-01', 'Duke')," - + "(4, DATE '2022-03-01', 'Leon')", - tableName); - - TestHelpers.assertRows( - sql("SELECT * FROM %s", tableName), - Lists.newArrayList( - Row.of(1, dt, "Jane"), - Row.of(2, dt, "Bill"), - Row.of(3, dt, "Duke"), - Row.of(4, dt, "Leon"))); - spark.sql(String.format(Locale.ROOT, "REFRESH TABLE %s.db.%s", catalogName, tableName)); - TestHelpers.assertRows( - convertToFlinkRows( - spark.sql( - String.format(Locale.ROOT, "SELECT * FROM %s.db.%s", catalogName, tableName)), - 3), - Lists.newArrayList( - Row.of(1, dtSpark, "Jane"), - Row.of(2, dtSpark, "Bill"), - Row.of(3, dtSpark, "Duke"), - Row.of(4, dtSpark, "Leon"))); - } finally { - sql("DROP TABLE IF EXISTS %s.%s", flinkDatabase, tableName); - } - } - - @TestTemplate - public void testPrimaryKeyFieldsAtEndOfTableSchema() { - // This is the same test case as testPrimaryKeyFieldsAtBeginningOfSchema, but the primary key - // fields - // are located at the end of the flink schema. - String tableName = "upsert_on_pk_at_schema_end"; - LocalDate dt = LocalDate.of(2022, 3, 1); - Date dtSpark = Date.from(dt.atStartOfDay().atZone(ZoneId.systemDefault()).toInstant()); - try { - sql( - "CREATE TABLE %s(name STRING NOT NULL, id INT, dt DATE NOT NULL, " - + "PRIMARY KEY(id,dt) NOT ENFORCED) " - + "PARTITIONED BY (dt) WITH %s", - tableName, toWithClause(tableUpsertProps)); - - sql( - "INSERT INTO %s VALUES " - + "('Andy', 1, DATE '2022-03-01')," - + "('Bill', 1, DATE '2022-03-01')," - + "('Jane', 2, DATE '2022-03-01')", - tableName); - - TestHelpers.assertRows( - sql("SELECT * FROM %s", tableName), - Lists.newArrayList(Row.of("Bill", 1, dt), Row.of("Jane", 2, dt))); - TestHelpers.assertRows( - convertToFlinkRows( - spark.sql( - String.format(Locale.ROOT, "SELECT * FROM %s.db.%s", catalogName, tableName)), - 3), - Lists.newArrayList(Row.of("Bill", 1, dtSpark), Row.of("Jane", 2, dtSpark))); - - sql( - "INSERT INTO %s VALUES " - + "('Jane', 1, DATE '2022-03-01')," - + "('Bill', 2, DATE '2022-03-01')", - tableName); - - TestHelpers.assertRows( - sql("SELECT * FROM %s", tableName), - Lists.newArrayList(Row.of("Jane", 1, dt), Row.of("Bill", 2, dt))); - spark.sql(String.format(Locale.ROOT, "REFRESH TABLE %s.db.%s", catalogName, tableName)); - TestHelpers.assertRows( - convertToFlinkRows( - spark.sql( - String.format(Locale.ROOT, "SELECT * FROM %s.db.%s", catalogName, tableName)), - 3), - Lists.newArrayList(Row.of("Jane", 1, dtSpark), Row.of("Bill", 2, dtSpark))); - - sql( - "INSERT INTO %s VALUES " - + "('Duke', 3, DATE '2022-03-01')," - + "('Leon', 4, DATE '2022-03-01')", - tableName); - - TestHelpers.assertRows( - sql("SELECT * FROM %s", tableName), - Lists.newArrayList( - Row.of("Jane", 1, dt), - Row.of("Bill", 2, dt), - Row.of("Duke", 3, dt), - Row.of("Leon", 4, dt))); - spark.sql(String.format(Locale.ROOT, "REFRESH TABLE %s.db.%s", catalogName, tableName)); - TestHelpers.assertRows( - convertToFlinkRows( - spark.sql( - String.format(Locale.ROOT, "SELECT * FROM %s.db.%s", catalogName, tableName)), - 3), - Lists.newArrayList( - Row.of("Jane", 1, dtSpark), - Row.of("Bill", 2, dtSpark), - Row.of("Duke", 3, dtSpark), - Row.of("Leon", 4, dtSpark))); - } finally { - sql("DROP TABLE IF EXISTS %s.%s", flinkDatabase, tableName); - } - } -} diff --git a/backends-clickhouse/src-iceberg-spark33/test/java/org/apache/gluten/execution/iceberg/TestPositionDeletesTableGluten.java b/backends-clickhouse/src-iceberg-spark33/test/java/org/apache/gluten/execution/iceberg/TestPositionDeletesTableGluten.java deleted file mode 100644 index 0dbb871e7b7..00000000000 --- a/backends-clickhouse/src-iceberg-spark33/test/java/org/apache/gluten/execution/iceberg/TestPositionDeletesTableGluten.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.execution.iceberg; - -import org.apache.iceberg.CatalogUtil; -import org.apache.iceberg.FileFormat; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.exceptions.AlreadyExistsException; -import org.apache.iceberg.hive.HiveCatalog; -import org.apache.iceberg.hive.TestHiveMetastore; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; -import org.apache.iceberg.spark.SparkCatalogConfig; -import org.apache.iceberg.spark.source.TestPositionDeletesTable; -import org.apache.spark.api.java.JavaSparkContext; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.runners.Parameterized; - -import java.util.Map; - -public class TestPositionDeletesTableGluten extends TestPositionDeletesTable { - private static final Map CATALOG_PROPS = - ImmutableMap.of("type", "hive", "default-namespace", "default", "cache-enabled", "false"); - private static ClickHouseIcebergHiveTableSupport hiveTableSupport; - - @BeforeClass - public static void startMetastoreAndSpark() { - metastore = new TestHiveMetastore(); - metastore.start(); - hiveConf = metastore.hiveConf(); - hiveTableSupport = new ClickHouseIcebergHiveTableSupport(); - hiveTableSupport.initSparkConf( - hiveConf.get("hive.metastore.uris"), SparkCatalogConfig.HIVE.catalogName(), null); - hiveTableSupport.initializeSession(); - spark = hiveTableSupport.spark(); - sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); - catalog = - (HiveCatalog) - CatalogUtil.loadCatalog( - HiveCatalog.class.getName(), "hive", ImmutableMap.of(), hiveConf); - - try { - catalog.createNamespace(Namespace.of(new String[] {"default"})); - } catch (AlreadyExistsException ignore) { - } - } - - @AfterClass - public static void stopMetastoreAndSpark() throws Exception { - catalog = null; - if (metastore != null) { - metastore.stop(); - metastore = null; - } - hiveTableSupport.clean(); - } - - @Parameterized.Parameters( - name = - "formatVersion = {0}, catalogName = {1}, implementation = {2}, config = {3}, fileFormat = {4}") - public static Object[][] parameters() { - // ignore ORC and AVRO, ch backend only support PARQUET - return new Object[][] { - { - SparkCatalogConfig.HIVE.catalogName(), - SparkCatalogConfig.HIVE.implementation(), - CATALOG_PROPS, - FileFormat.PARQUET - } - }; - } - - public TestPositionDeletesTableGluten( - String catalogName, String implementation, Map config, FileFormat format) { - super(catalogName, implementation, config, format); - } -} diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/1.txt deleted file mode 100644 index 874b24f6c2a..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/1.txt +++ /dev/null @@ -1,160 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (30) -+- == Final Plan == - VeloxColumnarToRow (21) - +- ^ SortExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- VeloxResizeBatches (14) - +- ^ RegularHashAggregateExecTransformer (12) - +- ^ InputIteratorTransformer (11) - +- ShuffleQueryStage (9), Statistics(X) - +- ColumnarExchange (8) - +- VeloxResizeBatches (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ FileSourceScanExecTransformer parquet (1) -+- == Initial Plan == - Sort (29) - +- Exchange (28) - +- HashAggregate (27) - +- Exchange (26) - +- HashAggregate (25) - +- Project (24) - +- Filter (23) - +- Scan parquet (22) - - -(1) FileSourceScanExecTransformer parquet -Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -Arguments: (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) - -(3) ProjectExecTransformer -Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_returnflag#X, l_linestatus#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X, CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)) AS _pre_X#X] -Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] - -(4) FlushableHashAggregateExecTransformer -Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_returnflag#X, l_linestatus#X, _pre_X#X, _pre_X#X] -Keys [2]: [l_returnflag#X, l_linestatus#X] -Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(_pre_X#X), partial_sum(_pre_X#X), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] -Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] - -(5) ProjectExecTransformer -Output [18]: [hash(l_returnflag#X, l_linestatus#X, 42) AS hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] - -(6) WholeStageCodegenTransformer (X) -Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -Arguments: false - -(7) VeloxResizeBatches -Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] - -(8) ColumnarExchange -Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [shuffle_writer_type=hash] - -(9) ShuffleQueryStage -Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -Arguments: X - -(10) InputAdapter -Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] - -(11) InputIteratorTransformer -Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] - -(12) RegularHashAggregateExecTransformer -Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -Keys [2]: [l_returnflag#X, l_linestatus#X] -Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] -Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] -Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] - -(13) WholeStageCodegenTransformer (X) -Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -Arguments: false - -(14) VeloxResizeBatches -Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] - -(15) ColumnarExchange -Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(16) ShuffleQueryStage -Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -Arguments: X - -(17) InputAdapter -Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] - -(18) InputIteratorTransformer -Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] - -(19) SortExecTransformer -Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 - -(20) WholeStageCodegenTransformer (X) -Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -Arguments: false - -(21) VeloxColumnarToRow -Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] - -(22) Scan parquet -Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] -ReadSchema: struct - -(23) Filter -Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) - -(24) Project -Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] -Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] - -(25) HashAggregate -Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] -Keys [2]: [l_returnflag#X, l_linestatus#X] -Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] -Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] - -(26) Exchange -Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(27) HashAggregate -Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -Keys [2]: [l_returnflag#X, l_linestatus#X] -Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] -Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] -Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] - -(28) Exchange -Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(29) Sort -Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 - -(30) AdaptiveSparkPlan -Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt deleted file mode 100644 index 79dc111aaa2..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt +++ /dev/null @@ -1,367 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (68) -+- == Final Plan == - VeloxColumnarToRow (44) - +- TakeOrderedAndProjectExecTransformer (43) - +- ^ ProjectExecTransformer (41) - +- ^ RegularHashAggregateExecTransformer (40) - +- ^ InputIteratorTransformer (39) - +- ShuffleQueryStage (37), Statistics(X) - +- ColumnarExchange (36) - +- VeloxResizeBatches (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - : : :- ^ FilterExecTransformer (2) - : : : +- ^ FileSourceScanExecTransformer parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- BroadcastQueryStage (8), Statistics(X) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ FilterExecTransformer (4) - : : +- ^ FileSourceScanExecTransformer parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- BroadcastQueryStage (18), Statistics(X) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ FilterExecTransformer (14) - : +- ^ FileSourceScanExecTransformer parquet (13) - +- ^ InputIteratorTransformer (29) - +- BroadcastQueryStage (27), Statistics(X) - +- ColumnarBroadcastExchange (26) - +- ^ FilterExecTransformer (24) - +- ^ FileSourceScanExecTransformer parquet (23) -+- == Initial Plan == - TakeOrderedAndProject (67) - +- HashAggregate (66) - +- Exchange (65) - +- HashAggregate (64) - +- Project (63) - +- BroadcastHashJoin Inner BuildRight (62) - :- Project (58) - : +- BroadcastHashJoin Inner BuildRight (57) - : :- Project (52) - : : +- BroadcastHashJoin Inner BuildRight (51) - : : :- Filter (46) - : : : +- Scan parquet (45) - : : +- BroadcastExchange (50) - : : +- Project (49) - : : +- Filter (48) - : : +- Scan parquet (47) - : +- BroadcastExchange (56) - : +- Project (55) - : +- Filter (54) - : +- Scan parquet (53) - +- BroadcastExchange (61) - +- Filter (60) - +- Scan parquet (59) - - -(1) FileSourceScanExecTransformer parquet -Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(3) FileSourceScanExecTransformer parquet -Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] -ReadSchema: struct - -(4) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(5) ProjectExecTransformer -Output [2]: [o_orderkey#X, o_custkey#X] -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] - -(6) WholeStageCodegenTransformer (X) -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: false - -(7) ColumnarBroadcastExchange -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] - -(8) BroadcastQueryStage -Output [2]: [o_orderkey#X, o_custkey#X] -Arguments: X - -(9) InputAdapter -Input [2]: [o_orderkey#X, o_custkey#X] - -(10) InputIteratorTransformer -Input [2]: [o_orderkey#X, o_custkey#X] - -(11) BroadcastHashJoinExecTransformer -Left keys [1]: [c_custkey#X] -Right keys [1]: [o_custkey#X] -Join condition: None - -(12) ProjectExecTransformer -Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] - -(13) FileSourceScanExecTransformer parquet -Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] -ReadSchema: struct - -(14) FilterExecTransformer -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -Arguments: ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) - -(15) ProjectExecTransformer -Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] - -(16) WholeStageCodegenTransformer (X) -Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -Arguments: false - -(17) ColumnarBroadcastExchange -Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(18) BroadcastQueryStage -Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -Arguments: X - -(19) InputAdapter -Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] - -(20) InputIteratorTransformer -Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] - -(21) BroadcastHashJoinExecTransformer -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(22) ProjectExecTransformer -Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] - -(23) FileSourceScanExecTransformer parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_nationkey)] -ReadSchema: struct - -(24) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: isnotnull(n_nationkey#X) - -(25) WholeStageCodegenTransformer (X) -Input [2]: [n_nationkey#X, n_name#X] -Arguments: false - -(26) ColumnarBroadcastExchange -Input [2]: [n_nationkey#X, n_name#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(27) BroadcastQueryStage -Output [2]: [n_nationkey#X, n_name#X] -Arguments: X - -(28) InputAdapter -Input [2]: [n_nationkey#X, n_name#X] - -(29) InputIteratorTransformer -Input [2]: [n_nationkey#X, n_name#X] - -(30) BroadcastHashJoinExecTransformer -Left keys [1]: [c_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(31) ProjectExecTransformer -Output [8]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] -Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] - -(32) FlushableHashAggregateExecTransformer -Input [8]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, n_name#X, _pre_X#X] -Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] -Functions [1]: [partial_sum(_pre_X#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] - -(33) ProjectExecTransformer -Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] - -(34) WholeStageCodegenTransformer (X) -Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -Arguments: false - -(35) VeloxResizeBatches -Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] - -(36) ColumnarExchange -Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] - -(37) ShuffleQueryStage -Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -Arguments: X - -(38) InputAdapter -Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] - -(39) InputIteratorTransformer -Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] - -(40) RegularHashAggregateExecTransformer -Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] -Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] - -(41) ProjectExecTransformer -Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] - -(42) WholeStageCodegenTransformer (X) -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: false - -(43) TakeOrderedAndProjectExecTransformer -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 - -(44) VeloxColumnarToRow -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] - -(45) Scan parquet -Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] -ReadSchema: struct - -(46) Filter -Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(47) Scan parquet -Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] -ReadSchema: struct - -(48) Filter -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(49) Project -Output [2]: [o_orderkey#X, o_custkey#X] -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] - -(50) BroadcastExchange -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] - -(51) BroadcastHashJoin -Left keys [1]: [c_custkey#X] -Right keys [1]: [o_custkey#X] -Join condition: None - -(52) Project -Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] - -(53) Scan parquet -Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] -ReadSchema: struct - -(54) Filter -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) - -(55) Project -Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] - -(56) BroadcastExchange -Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(57) BroadcastHashJoin -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(58) Project -Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] - -(59) Scan parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_nationkey)] -ReadSchema: struct - -(60) Filter -Input [2]: [n_nationkey#X, n_name#X] -Condition : isnotnull(n_nationkey#X) - -(61) BroadcastExchange -Input [2]: [n_nationkey#X, n_name#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(62) BroadcastHashJoin -Left keys [1]: [c_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(63) Project -Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] -Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] - -(64) HashAggregate -Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] -Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] -Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] - -(65) Exchange -Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(66) HashAggregate -Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] -Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] - -(67) TakeOrderedAndProject -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] - -(68) AdaptiveSparkPlan -Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/11.txt deleted file mode 100644 index 96a23c07840..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/11.txt +++ /dev/null @@ -1,548 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (60) -+- == Final Plan == - VeloxColumnarToRow (40) - +- ^ SortExecTransformer (38) - +- ^ InputIteratorTransformer (37) - +- ShuffleQueryStage (35), Statistics(X) - +- ColumnarExchange (34) - +- VeloxResizeBatches (33) - +- ^ FilterExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- VeloxResizeBatches (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ FilterExecTransformer (2) - : : +- ^ FileSourceScanExecTransformer parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- BroadcastQueryStage (7), Statistics(X) - : +- ColumnarBroadcastExchange (6) - : +- ^ FilterExecTransformer (4) - : +- ^ FileSourceScanExecTransformer parquet (3) - +- ^ InputIteratorTransformer (19) - +- BroadcastQueryStage (17), Statistics(X) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FilterExecTransformer (13) - +- ^ FileSourceScanExecTransformer parquet (12) -+- == Initial Plan == - Sort (59) - +- Exchange (58) - +- Filter (57) - +- HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- BroadcastHashJoin Inner BuildRight (52) - :- Project (47) - : +- BroadcastHashJoin Inner BuildRight (46) - : :- Filter (42) - : : +- Scan parquet (41) - : +- BroadcastExchange (45) - : +- Filter (44) - : +- Scan parquet (43) - +- BroadcastExchange (51) - +- Project (50) - +- Filter (49) - +- Scan parquet (48) - - -(1) FileSourceScanExecTransformer parquet -Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(ps_suppkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: isnotnull(ps_suppkey#X) - -(3) FileSourceScanExecTransformer parquet -Output [2]: [s_suppkey#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] -ReadSchema: struct - -(4) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(5) WholeStageCodegenTransformer (X) -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: false - -(6) ColumnarBroadcastExchange -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(7) BroadcastQueryStage -Output [2]: [s_suppkey#X, s_nationkey#X] -Arguments: X - -(8) InputAdapter -Input [2]: [s_suppkey#X, s_nationkey#X] - -(9) InputIteratorTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] - -(10) BroadcastHashJoinExecTransformer -Left keys [1]: [ps_suppkey#X] -Right keys [1]: [s_suppkey#X] -Join condition: None - -(11) ProjectExecTransformer -Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] - -(12) FileSourceScanExecTransformer parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] -ReadSchema: struct - -(13) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) - -(14) ProjectExecTransformer -Output [1]: [n_nationkey#X] -Input [2]: [n_nationkey#X, n_name#X] - -(15) WholeStageCodegenTransformer (X) -Input [1]: [n_nationkey#X] -Arguments: false - -(16) ColumnarBroadcastExchange -Input [1]: [n_nationkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(17) BroadcastQueryStage -Output [1]: [n_nationkey#X] -Arguments: X - -(18) InputAdapter -Input [1]: [n_nationkey#X] - -(19) InputIteratorTransformer -Input [1]: [n_nationkey#X] - -(20) BroadcastHashJoinExecTransformer -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(21) ProjectExecTransformer -Output [2]: [ps_partkey#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] -Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] - -(22) FlushableHashAggregateExecTransformer -Input [2]: [ps_partkey#X, _pre_X#X] -Keys [1]: [ps_partkey#X] -Functions [1]: [partial_sum(_pre_X#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [3]: [ps_partkey#X, sum#X, isEmpty#X] - -(23) ProjectExecTransformer -Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] -Input [3]: [ps_partkey#X, sum#X, isEmpty#X] - -(24) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] -Arguments: false - -(25) VeloxResizeBatches -Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] - -(26) ColumnarExchange -Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] - -(27) ShuffleQueryStage -Output [3]: [ps_partkey#X, sum#X, isEmpty#X] -Arguments: X - -(28) InputAdapter -Input [3]: [ps_partkey#X, sum#X, isEmpty#X] - -(29) InputIteratorTransformer -Input [3]: [ps_partkey#X, sum#X, isEmpty#X] - -(30) RegularHashAggregateExecTransformer -Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -Keys [1]: [ps_partkey#X] -Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] - -(31) FilterExecTransformer -Input [2]: [ps_partkey#X, value#X] -Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) - -(32) WholeStageCodegenTransformer (X) -Input [2]: [ps_partkey#X, value#X] -Arguments: false - -(33) VeloxResizeBatches -Input [2]: [ps_partkey#X, value#X] - -(34) ColumnarExchange -Input [2]: [ps_partkey#X, value#X] -Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(35) ShuffleQueryStage -Output [2]: [ps_partkey#X, value#X] -Arguments: X - -(36) InputAdapter -Input [2]: [ps_partkey#X, value#X] - -(37) InputIteratorTransformer -Input [2]: [ps_partkey#X, value#X] - -(38) SortExecTransformer -Input [2]: [ps_partkey#X, value#X] -Arguments: [value#X DESC NULLS LAST], true, 0 - -(39) WholeStageCodegenTransformer (X) -Input [2]: [ps_partkey#X, value#X] -Arguments: false - -(40) VeloxColumnarToRow -Input [2]: [ps_partkey#X, value#X] - -(41) Scan parquet -Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(ps_suppkey)] -ReadSchema: struct - -(42) Filter -Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Condition : isnotnull(ps_suppkey#X) - -(43) Scan parquet -Output [2]: [s_suppkey#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] -ReadSchema: struct - -(44) Filter -Input [2]: [s_suppkey#X, s_nationkey#X] -Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(45) BroadcastExchange -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(46) BroadcastHashJoin -Left keys [1]: [ps_suppkey#X] -Right keys [1]: [s_suppkey#X] -Join condition: None - -(47) Project -Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] - -(48) Scan parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] -ReadSchema: struct - -(49) Filter -Input [2]: [n_nationkey#X, n_name#X] -Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) - -(50) Project -Output [1]: [n_nationkey#X] -Input [2]: [n_nationkey#X, n_name#X] - -(51) BroadcastExchange -Input [1]: [n_nationkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(52) BroadcastHashJoin -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(53) Project -Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] -Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] - -(54) HashAggregate -Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] -Keys [1]: [ps_partkey#X] -Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [3]: [ps_partkey#X, sum#X, isEmpty#X] - -(55) Exchange -Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(56) HashAggregate -Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -Keys [1]: [ps_partkey#X] -Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] - -(57) Filter -Input [2]: [ps_partkey#X, value#X] -Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) - -(58) Exchange -Input [2]: [ps_partkey#X, value#X] -Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(59) Sort -Input [2]: [ps_partkey#X, value#X] -Arguments: [value#X DESC NULLS LAST], true, 0 - -(60) AdaptiveSparkPlan -Output [2]: [ps_partkey#X, value#X] -Arguments: isFinalPlan=true - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 31 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (102) -+- == Final Plan == - VeloxColumnarToRow (85) - +- ^ ProjectExecTransformer (83) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- VeloxResizeBatches (77) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (73) - :- ^ ProjectExecTransformer (68) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (67) - : :- ^ FilterExecTransformer (62) - : : +- ^ FileSourceScanExecTransformer parquet (61) - : +- ^ InputIteratorTransformer (66) - : +- BroadcastQueryStage (64), Statistics(X) - : +- ReusedExchange (63) - +- ^ InputIteratorTransformer (72) - +- BroadcastQueryStage (70), Statistics(X) - +- ReusedExchange (69) -+- == Initial Plan == - HashAggregate (101) - +- Exchange (100) - +- HashAggregate (99) - +- Project (98) - +- BroadcastHashJoin Inner BuildRight (97) - :- Project (92) - : +- BroadcastHashJoin Inner BuildRight (91) - : :- Filter (87) - : : +- Scan parquet (86) - : +- BroadcastExchange (90) - : +- Filter (89) - : +- Scan parquet (88) - +- BroadcastExchange (96) - +- Project (95) - +- Filter (94) - +- Scan parquet (93) - - -(61) FileSourceScanExecTransformer parquet -Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(ps_suppkey)] -ReadSchema: struct - -(62) FilterExecTransformer -Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: isnotnull(ps_suppkey#X) - -(63) ReusedExchange [Reuses operator id: 6] -Output [2]: [s_suppkey#X, s_nationkey#X] - -(64) BroadcastQueryStage -Output [2]: [s_suppkey#X, s_nationkey#X] -Arguments: X - -(65) InputAdapter -Input [2]: [s_suppkey#X, s_nationkey#X] - -(66) InputIteratorTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] - -(67) BroadcastHashJoinExecTransformer -Left keys [1]: [ps_suppkey#X] -Right keys [1]: [s_suppkey#X] -Join condition: None - -(68) ProjectExecTransformer -Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] - -(69) ReusedExchange [Reuses operator id: 16] -Output [1]: [n_nationkey#X] - -(70) BroadcastQueryStage -Output [1]: [n_nationkey#X] -Arguments: X - -(71) InputAdapter -Input [1]: [n_nationkey#X] - -(72) InputIteratorTransformer -Input [1]: [n_nationkey#X] - -(73) BroadcastHashJoinExecTransformer -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(74) ProjectExecTransformer -Output [1]: [CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] -Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] - -(75) FlushableHashAggregateExecTransformer -Input [1]: [_pre_X#X] -Keys: [] -Functions [1]: [partial_sum(_pre_X#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [2]: [sum#X, isEmpty#X] - -(76) WholeStageCodegenTransformer (X) -Input [2]: [sum#X, isEmpty#X] -Arguments: false - -(77) VeloxResizeBatches -Input [2]: [sum#X, isEmpty#X] - -(78) ColumnarExchange -Input [2]: [sum#X, isEmpty#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(79) ShuffleQueryStage -Output [2]: [sum#X, isEmpty#X] -Arguments: X - -(80) InputAdapter -Input [2]: [sum#X, isEmpty#X] - -(81) InputIteratorTransformer -Input [2]: [sum#X, isEmpty#X] - -(82) RegularHashAggregateExecTransformer -Input [2]: [sum#X, isEmpty#X] -Keys: [] -Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -Results [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] - -(83) ProjectExecTransformer -Output [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -Input [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] - -(84) WholeStageCodegenTransformer (X) -Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -Arguments: false - -(85) VeloxColumnarToRow -Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] - -(86) Scan parquet -Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(ps_suppkey)] -ReadSchema: struct - -(87) Filter -Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Condition : isnotnull(ps_suppkey#X) - -(88) Scan parquet -Output [2]: [s_suppkey#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] -ReadSchema: struct - -(89) Filter -Input [2]: [s_suppkey#X, s_nationkey#X] -Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(90) BroadcastExchange -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(91) BroadcastHashJoin -Left keys [1]: [ps_suppkey#X] -Right keys [1]: [s_suppkey#X] -Join condition: None - -(92) Project -Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] - -(93) Scan parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] -ReadSchema: struct - -(94) Filter -Input [2]: [n_nationkey#X, n_name#X] -Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) - -(95) Project -Output [1]: [n_nationkey#X] -Input [2]: [n_nationkey#X, n_name#X] - -(96) BroadcastExchange -Input [1]: [n_nationkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(97) BroadcastHashJoin -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(98) Project -Output [2]: [ps_availqty#X, ps_supplycost#X] -Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] - -(99) HashAggregate -Input [2]: [ps_availqty#X, ps_supplycost#X] -Keys: [] -Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [2]: [sum#X, isEmpty#X] - -(100) Exchange -Input [2]: [sum#X, isEmpty#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] - -(101) HashAggregate -Input [2]: [sum#X, isEmpty#X] -Keys: [] -Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -Results [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] - -(102) AdaptiveSparkPlan -Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/12.txt deleted file mode 100644 index 77e8ab1e5d0..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/12.txt +++ /dev/null @@ -1,234 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (44) -+- == Final Plan == - VeloxColumnarToRow (30) - +- ^ SortExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- VeloxResizeBatches (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ShuffleQueryStage (18), Statistics(X) - +- ColumnarExchange (17) - +- VeloxResizeBatches (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ FileSourceScanExecTransformer parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ FileSourceScanExecTransformer parquet (8) -+- == Initial Plan == - Sort (43) - +- Exchange (42) - +- HashAggregate (41) - +- Exchange (40) - +- HashAggregate (39) - +- Project (38) - +- BroadcastHashJoin Inner BuildLeft (37) - :- BroadcastExchange (33) - : +- Filter (32) - : +- Scan parquet (31) - +- Project (36) - +- Filter (35) - +- Scan parquet (34) - - -(1) FileSourceScanExecTransformer parquet -Output [2]: [o_orderkey#X, o_orderpriority#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [2]: [o_orderkey#X, o_orderpriority#X] -Arguments: isnotnull(o_orderkey#X) - -(3) WholeStageCodegenTransformer (X) -Input [2]: [o_orderkey#X, o_orderpriority#X] -Arguments: false - -(4) ColumnarBroadcastExchange -Input [2]: [o_orderkey#X, o_orderpriority#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(5) BroadcastQueryStage -Output [2]: [o_orderkey#X, o_orderpriority#X] -Arguments: X - -(6) InputAdapter -Input [2]: [o_orderkey#X, o_orderpriority#X] - -(7) InputIteratorTransformer -Input [2]: [o_orderkey#X, o_orderpriority#X] - -(8) FileSourceScanExecTransformer parquet -Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] -ReadSchema: struct - -(9) FilterExecTransformer -Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -Arguments: ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) - -(10) ProjectExecTransformer -Output [2]: [l_orderkey#X, l_shipmode#X] -Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] - -(11) BroadcastHashJoinExecTransformer -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(12) ProjectExecTransformer -Output [3]: [l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] -Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] - -(13) FlushableHashAggregateExecTransformer -Input [3]: [l_shipmode#X, _pre_X#X, _pre_X#X] -Keys [1]: [l_shipmode#X] -Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] -Aggregate Attributes [2]: [sum#X, sum#X] -Results [3]: [l_shipmode#X, sum#X, sum#X] - -(14) ProjectExecTransformer -Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] -Input [3]: [l_shipmode#X, sum#X, sum#X] - -(15) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] -Arguments: false - -(16) VeloxResizeBatches -Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] - -(17) ColumnarExchange -Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] -Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [shuffle_writer_type=hash] - -(18) ShuffleQueryStage -Output [3]: [l_shipmode#X, sum#X, sum#X] -Arguments: X - -(19) InputAdapter -Input [3]: [l_shipmode#X, sum#X, sum#X] - -(20) InputIteratorTransformer -Input [3]: [l_shipmode#X, sum#X, sum#X] - -(21) RegularHashAggregateExecTransformer -Input [3]: [l_shipmode#X, sum#X, sum#X] -Keys [1]: [l_shipmode#X] -Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] -Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] -Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] - -(22) WholeStageCodegenTransformer (X) -Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -Arguments: false - -(23) VeloxResizeBatches -Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] - -(24) ColumnarExchange -Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(25) ShuffleQueryStage -Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -Arguments: X - -(26) InputAdapter -Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] - -(27) InputIteratorTransformer -Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] - -(28) SortExecTransformer -Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 - -(29) WholeStageCodegenTransformer (X) -Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -Arguments: false - -(30) VeloxColumnarToRow -Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] - -(31) Scan parquet -Output [2]: [o_orderkey#X, o_orderpriority#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderkey)] -ReadSchema: struct - -(32) Filter -Input [2]: [o_orderkey#X, o_orderpriority#X] -Condition : isnotnull(o_orderkey#X) - -(33) BroadcastExchange -Input [2]: [o_orderkey#X, o_orderpriority#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(34) Scan parquet -Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] -ReadSchema: struct - -(35) Filter -Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) - -(36) Project -Output [2]: [l_orderkey#X, l_shipmode#X] -Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] - -(37) BroadcastHashJoin -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(38) Project -Output [2]: [o_orderpriority#X, l_shipmode#X] -Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] - -(39) HashAggregate -Input [2]: [o_orderpriority#X, l_shipmode#X] -Keys [1]: [l_shipmode#X] -Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] -Aggregate Attributes [2]: [sum#X, sum#X] -Results [3]: [l_shipmode#X, sum#X, sum#X] - -(40) Exchange -Input [3]: [l_shipmode#X, sum#X, sum#X] -Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(41) HashAggregate -Input [3]: [l_shipmode#X, sum#X, sum#X] -Keys [1]: [l_shipmode#X] -Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] -Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] -Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] - -(42) Exchange -Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(43) Sort -Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 - -(44) AdaptiveSparkPlan -Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/13.txt deleted file mode 100644 index c5f217ff2b7..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/13.txt +++ /dev/null @@ -1,294 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (55) -+- == Final Plan == - VeloxColumnarToRow (39) - +- ^ SortExecTransformer (37) - +- ^ InputIteratorTransformer (36) - +- ShuffleQueryStage (34), Statistics(X) - +- ColumnarExchange (33) - +- VeloxResizeBatches (32) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- VeloxResizeBatches (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- VeloxResizeBatches (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ FileSourceScanExecTransformer parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FilterExecTransformer (3) - +- ^ FileSourceScanExecTransformer parquet (2) -+- == Initial Plan == - Sort (54) - +- Exchange (53) - +- HashAggregate (52) - +- Exchange (51) - +- HashAggregate (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- Project (46) - +- BroadcastHashJoin LeftOuter BuildRight (45) - :- Scan parquet (40) - +- BroadcastExchange (44) - +- Project (43) - +- Filter (42) - +- Scan parquet (41) - - -(1) FileSourceScanExecTransformer parquet -Output [1]: [c_custkey#X] -Batched: true -Location: InMemoryFileIndex [*] -ReadSchema: struct - -(2) FileSourceScanExecTransformer parquet -Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] -ReadSchema: struct - -(3) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -Arguments: ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) - -(4) ProjectExecTransformer -Output [2]: [o_orderkey#X, o_custkey#X] -Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] - -(5) WholeStageCodegenTransformer (X) -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: false - -(6) ColumnarBroadcastExchange -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] - -(7) BroadcastQueryStage -Output [2]: [o_orderkey#X, o_custkey#X] -Arguments: X - -(8) InputAdapter -Input [2]: [o_orderkey#X, o_custkey#X] - -(9) InputIteratorTransformer -Input [2]: [o_orderkey#X, o_custkey#X] - -(10) BroadcastHashJoinExecTransformer -Left keys [1]: [c_custkey#X] -Right keys [1]: [o_custkey#X] -Join condition: None - -(11) ProjectExecTransformer -Output [2]: [c_custkey#X, o_orderkey#X] -Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] - -(12) FlushableHashAggregateExecTransformer -Input [2]: [c_custkey#X, o_orderkey#X] -Keys [1]: [c_custkey#X] -Functions [1]: [partial_count(o_orderkey#X)] -Aggregate Attributes [1]: [count#X] -Results [2]: [c_custkey#X, count#X] - -(13) ProjectExecTransformer -Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, count#X] -Input [2]: [c_custkey#X, count#X] - -(14) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, c_custkey#X, count#X] -Arguments: false - -(15) VeloxResizeBatches -Input [3]: [hash_partition_key#X, c_custkey#X, count#X] - -(16) ColumnarExchange -Input [3]: [hash_partition_key#X, c_custkey#X, count#X] -Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, count#X], [plan_id=X], [shuffle_writer_type=hash] - -(17) ShuffleQueryStage -Output [2]: [c_custkey#X, count#X] -Arguments: X - -(18) InputAdapter -Input [2]: [c_custkey#X, count#X] - -(19) InputIteratorTransformer -Input [2]: [c_custkey#X, count#X] - -(20) RegularHashAggregateExecTransformer -Input [2]: [c_custkey#X, count#X] -Keys [1]: [c_custkey#X] -Functions [1]: [count(o_orderkey#X)] -Aggregate Attributes [1]: [count(o_orderkey#X)#X] -Results [2]: [c_custkey#X, count(o_orderkey#X)#X] - -(21) ProjectExecTransformer -Output [1]: [count(o_orderkey#X)#X AS c_count#X] -Input [2]: [c_custkey#X, count(o_orderkey#X)#X] - -(22) FlushableHashAggregateExecTransformer -Input [1]: [c_count#X] -Keys [1]: [c_count#X] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#X] -Results [2]: [c_count#X, count#X] - -(23) ProjectExecTransformer -Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] -Input [2]: [c_count#X, count#X] - -(24) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, c_count#X, count#X] -Arguments: false - -(25) VeloxResizeBatches -Input [3]: [hash_partition_key#X, c_count#X, count#X] - -(26) ColumnarExchange -Input [3]: [hash_partition_key#X, c_count#X, count#X] -Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [shuffle_writer_type=hash] - -(27) ShuffleQueryStage -Output [2]: [c_count#X, count#X] -Arguments: X - -(28) InputAdapter -Input [2]: [c_count#X, count#X] - -(29) InputIteratorTransformer -Input [2]: [c_count#X, count#X] - -(30) RegularHashAggregateExecTransformer -Input [2]: [c_count#X, count#X] -Keys [1]: [c_count#X] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#X] -Results [2]: [c_count#X, count(1)#X AS custdist#X] - -(31) WholeStageCodegenTransformer (X) -Input [2]: [c_count#X, custdist#X] -Arguments: false - -(32) VeloxResizeBatches -Input [2]: [c_count#X, custdist#X] - -(33) ColumnarExchange -Input [2]: [c_count#X, custdist#X] -Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(34) ShuffleQueryStage -Output [2]: [c_count#X, custdist#X] -Arguments: X - -(35) InputAdapter -Input [2]: [c_count#X, custdist#X] - -(36) InputIteratorTransformer -Input [2]: [c_count#X, custdist#X] - -(37) SortExecTransformer -Input [2]: [c_count#X, custdist#X] -Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 - -(38) WholeStageCodegenTransformer (X) -Input [2]: [c_count#X, custdist#X] -Arguments: false - -(39) VeloxColumnarToRow -Input [2]: [c_count#X, custdist#X] - -(40) Scan parquet -Output [1]: [c_custkey#X] -Batched: true -Location: InMemoryFileIndex [*] -ReadSchema: struct - -(41) Scan parquet -Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] -ReadSchema: struct - -(42) Filter -Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) - -(43) Project -Output [2]: [o_orderkey#X, o_custkey#X] -Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] - -(44) BroadcastExchange -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] - -(45) BroadcastHashJoin -Left keys [1]: [c_custkey#X] -Right keys [1]: [o_custkey#X] -Join condition: None - -(46) Project -Output [2]: [c_custkey#X, o_orderkey#X] -Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] - -(47) HashAggregate -Input [2]: [c_custkey#X, o_orderkey#X] -Keys [1]: [c_custkey#X] -Functions [1]: [partial_count(o_orderkey#X)] -Aggregate Attributes [1]: [count#X] -Results [2]: [c_custkey#X, count#X] - -(48) Exchange -Input [2]: [c_custkey#X, count#X] -Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(49) HashAggregate -Input [2]: [c_custkey#X, count#X] -Keys [1]: [c_custkey#X] -Functions [1]: [count(o_orderkey#X)] -Aggregate Attributes [1]: [count(o_orderkey#X)#X] -Results [1]: [count(o_orderkey#X)#X AS c_count#X] - -(50) HashAggregate -Input [1]: [c_count#X] -Keys [1]: [c_count#X] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#X] -Results [2]: [c_count#X, count#X] - -(51) Exchange -Input [2]: [c_count#X, count#X] -Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(52) HashAggregate -Input [2]: [c_count#X, count#X] -Keys [1]: [c_count#X] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#X] -Results [2]: [c_count#X, count(1)#X AS custdist#X] - -(53) Exchange -Input [2]: [c_count#X, custdist#X] -Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(54) Sort -Input [2]: [c_count#X, custdist#X] -Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 - -(55) AdaptiveSparkPlan -Output [2]: [c_count#X, custdist#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/14.txt deleted file mode 100644 index 2030306af8a..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/14.txt +++ /dev/null @@ -1,194 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (35) -+- == Final Plan == - VeloxColumnarToRow (23) - +- ^ ProjectExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- VeloxResizeBatches (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ FileSourceScanExecTransformer parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ FileSourceScanExecTransformer parquet (4) -+- == Initial Plan == - HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- BroadcastHashJoin Inner BuildRight (30) - :- Project (26) - : +- Filter (25) - : +- Scan parquet (24) - +- BroadcastExchange (29) - +- Filter (28) - +- Scan parquet (27) - - -(1) FileSourceScanExecTransformer parquet -Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) - -(3) ProjectExecTransformer -Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(4) FileSourceScanExecTransformer parquet -Output [2]: [p_partkey#X, p_type#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_partkey)] -ReadSchema: struct - -(5) FilterExecTransformer -Input [2]: [p_partkey#X, p_type#X] -Arguments: isnotnull(p_partkey#X) - -(6) WholeStageCodegenTransformer (X) -Input [2]: [p_partkey#X, p_type#X] -Arguments: false - -(7) ColumnarBroadcastExchange -Input [2]: [p_partkey#X, p_type#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(8) BroadcastQueryStage -Output [2]: [p_partkey#X, p_type#X] -Arguments: X - -(9) InputAdapter -Input [2]: [p_partkey#X, p_type#X] - -(10) InputIteratorTransformer -Input [2]: [p_partkey#X, p_type#X] - -(11) BroadcastHashJoinExecTransformer -Left keys [1]: [l_partkey#X] -Right keys [1]: [p_partkey#X] -Join condition: None - -(12) ProjectExecTransformer -Output [2]: [CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] -Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] - -(13) FlushableHashAggregateExecTransformer -Input [2]: [_pre_X#X, _pre_X#X] -Keys: [] -Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] -Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] - -(14) WholeStageCodegenTransformer (X) -Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -Arguments: false - -(15) VeloxResizeBatches -Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] - -(16) ColumnarExchange -Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(17) ShuffleQueryStage -Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -Arguments: X - -(18) InputAdapter -Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] - -(19) InputIteratorTransformer -Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] - -(20) RegularHashAggregateExecTransformer -Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -Keys: [] -Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] - -(21) ProjectExecTransformer -Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] -Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] - -(22) WholeStageCodegenTransformer (X) -Input [1]: [promo_revenue#X] -Arguments: false - -(23) VeloxColumnarToRow -Input [1]: [promo_revenue#X] - -(24) Scan parquet -Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] -ReadSchema: struct - -(25) Filter -Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) - -(26) Project -Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(27) Scan parquet -Output [2]: [p_partkey#X, p_type#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_partkey)] -ReadSchema: struct - -(28) Filter -Input [2]: [p_partkey#X, p_type#X] -Condition : isnotnull(p_partkey#X) - -(29) BroadcastExchange -Input [2]: [p_partkey#X, p_type#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(30) BroadcastHashJoin -Left keys [1]: [l_partkey#X] -Right keys [1]: [p_partkey#X] -Join condition: None - -(31) Project -Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] -Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] - -(32) HashAggregate -Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] -Keys: [] -Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] - -(33) Exchange -Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] - -(34) HashAggregate -Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -Keys: [] -Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] - -(35) AdaptiveSparkPlan -Output [1]: [promo_revenue#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/15.txt deleted file mode 100644 index a5a88d3bea9..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/15.txt +++ /dev/null @@ -1,385 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (43) -+- == Final Plan == - VeloxColumnarToRow (28) - +- AQEShuffleRead (27) - +- ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- VeloxResizeBatches (24) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (21) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ FileSourceScanExecTransformer parquet (1) - +- ^ FilterExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- VeloxResizeBatches (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ FileSourceScanExecTransformer parquet (8) -+- == Initial Plan == - Sort (42) - +- Exchange (41) - +- Project (40) - +- BroadcastHashJoin Inner BuildLeft (39) - :- BroadcastExchange (31) - : +- Filter (30) - : +- Scan parquet (29) - +- Filter (38) - +- HashAggregate (37) - +- Exchange (36) - +- HashAggregate (35) - +- Project (34) - +- Filter (33) - +- Scan parquet (32) - - -(1) FileSourceScanExecTransformer parquet -Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -Arguments: isnotnull(s_suppkey#X) - -(3) WholeStageCodegenTransformer (X) -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -Arguments: false - -(4) ColumnarBroadcastExchange -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(5) BroadcastQueryStage -Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -Arguments: X - -(6) InputAdapter -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] - -(7) InputIteratorTransformer -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] - -(8) FileSourceScanExecTransformer parquet -Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] -ReadSchema: struct - -(9) FilterExecTransformer -Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) - -(10) ProjectExecTransformer -Output [2]: [l_suppkey#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] -Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(11) FlushableHashAggregateExecTransformer -Input [2]: [l_suppkey#X, _pre_X#X] -Keys [1]: [l_suppkey#X] -Functions [1]: [partial_sum(_pre_X#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [3]: [l_suppkey#X, sum#X, isEmpty#X] - -(12) ProjectExecTransformer -Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] -Input [3]: [l_suppkey#X, sum#X, isEmpty#X] - -(13) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] -Arguments: false - -(14) VeloxResizeBatches -Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] - -(15) ColumnarExchange -Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] - -(16) ShuffleQueryStage -Output [3]: [l_suppkey#X, sum#X, isEmpty#X] -Arguments: X - -(17) InputAdapter -Input [3]: [l_suppkey#X, sum#X, isEmpty#X] - -(18) InputIteratorTransformer -Input [3]: [l_suppkey#X, sum#X, isEmpty#X] - -(19) RegularHashAggregateExecTransformer -Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -Keys [1]: [l_suppkey#X] -Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] - -(20) FilterExecTransformer -Input [2]: [supplier_no#X, total_revenue#X] -Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) - -(21) BroadcastHashJoinExecTransformer -Left keys [1]: [s_suppkey#X] -Right keys [1]: [supplier_no#X] -Join condition: None - -(22) ProjectExecTransformer -Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] - -(23) WholeStageCodegenTransformer (X) -Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -Arguments: false - -(24) VeloxResizeBatches -Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] - -(25) ColumnarExchange -Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(26) ShuffleQueryStage -Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -Arguments: X - -(27) AQEShuffleRead -Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -Arguments: local - -(28) VeloxColumnarToRow -Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] - -(29) Scan parquet -Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey)] -ReadSchema: struct - -(30) Filter -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -Condition : isnotnull(s_suppkey#X) - -(31) BroadcastExchange -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(32) Scan parquet -Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] -ReadSchema: struct - -(33) Filter -Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) - -(34) Project -Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] -Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(35) HashAggregate -Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] -Keys [1]: [l_suppkey#X] -Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [3]: [l_suppkey#X, sum#X, isEmpty#X] - -(36) Exchange -Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(37) HashAggregate -Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -Keys [1]: [l_suppkey#X] -Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] - -(38) Filter -Input [2]: [supplier_no#X, total_revenue#X] -Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) - -(39) BroadcastHashJoin -Left keys [1]: [s_suppkey#X] -Right keys [1]: [supplier_no#X] -Join condition: None - -(40) Project -Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] - -(41) Exchange -Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(42) Sort -Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 - -(43) AdaptiveSparkPlan -Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -Arguments: isFinalPlan=true - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (68) -+- == Final Plan == - VeloxColumnarToRow (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ ProjectExecTransformer (56) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ InputIteratorTransformer (54) - +- ShuffleQueryStage (52), Statistics(X) - +- ColumnarExchange (51) - +- VeloxResizeBatches (50) - +- ^ ProjectExecTransformer (48) - +- ^ FlushableHashAggregateExecTransformer (47) - +- ^ ProjectExecTransformer (46) - +- ^ FilterExecTransformer (45) - +- ^ FileSourceScanExecTransformer parquet (44) -+- == Initial Plan == - HashAggregate (67) - +- HashAggregate (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- Filter (61) - +- Scan parquet (60) - - -(44) FileSourceScanExecTransformer parquet -Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] -ReadSchema: struct - -(45) FilterExecTransformer -Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) - -(46) ProjectExecTransformer -Output [2]: [l_suppkey#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] -Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(47) FlushableHashAggregateExecTransformer -Input [2]: [l_suppkey#X, _pre_X#X] -Keys [1]: [l_suppkey#X] -Functions [1]: [partial_sum(_pre_X#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [3]: [l_suppkey#X, sum#X, isEmpty#X] - -(48) ProjectExecTransformer -Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] -Input [3]: [l_suppkey#X, sum#X, isEmpty#X] - -(49) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] -Arguments: false - -(50) VeloxResizeBatches -Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] - -(51) ColumnarExchange -Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] - -(52) ShuffleQueryStage -Output [3]: [l_suppkey#X, sum#X, isEmpty#X] -Arguments: X - -(53) InputAdapter -Input [3]: [l_suppkey#X, sum#X, isEmpty#X] - -(54) InputIteratorTransformer -Input [3]: [l_suppkey#X, sum#X, isEmpty#X] - -(55) RegularHashAggregateExecTransformer -Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -Keys [1]: [l_suppkey#X] -Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] - -(56) ProjectExecTransformer -Output [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -Input [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] - -(57) RegularHashAggregateExecTransformer -Input [1]: [total_revenue#X] -Keys: [] -Functions [1]: [max(total_revenue#X)] -Aggregate Attributes [1]: [max(total_revenue#X)#X] -Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] - -(58) WholeStageCodegenTransformer (X) -Input [1]: [max(total_revenue)#X] -Arguments: false - -(59) VeloxColumnarToRow -Input [1]: [max(total_revenue)#X] - -(60) Scan parquet -Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] -ReadSchema: struct - -(61) Filter -Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) - -(62) Project -Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] -Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(63) HashAggregate -Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] -Keys [1]: [l_suppkey#X] -Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [3]: [l_suppkey#X, sum#X, isEmpty#X] - -(64) Exchange -Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(65) HashAggregate -Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -Keys [1]: [l_suppkey#X] -Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] - -(66) HashAggregate -Input [1]: [total_revenue#X] -Keys: [] -Functions [1]: [partial_max(total_revenue#X)] -Aggregate Attributes [1]: [max#X] -Results [1]: [max#X] - -(67) HashAggregate -Input [1]: [max#X] -Keys: [] -Functions [1]: [max(total_revenue#X)] -Aggregate Attributes [1]: [max(total_revenue#X)#X] -Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] - -(68) AdaptiveSparkPlan -Output [1]: [max(total_revenue)#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/16.txt deleted file mode 100644 index 29f8d071c0f..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/16.txt +++ /dev/null @@ -1,320 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (59) -+- == Final Plan == - VeloxColumnarToRow (38) - +- ^ SortExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- VeloxResizeBatches (31) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- VeloxResizeBatches (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- VeloxResizeBatches (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ FilterExecTransformer (2) - : +- ^ FileSourceScanExecTransformer parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ FilterExecTransformer (4) - +- ^ FileSourceScanExecTransformer parquet (3) -+- == Initial Plan == - Sort (58) - +- Exchange (57) - +- HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- Project (50) - +- BroadcastHashJoin Inner BuildRight (49) - :- BroadcastHashJoin LeftAnti BuildRight (45) - : :- Filter (40) - : : +- Scan parquet (39) - : +- BroadcastExchange (44) - : +- Project (43) - : +- Filter (42) - : +- Scan parquet (41) - +- BroadcastExchange (48) - +- Filter (47) - +- Scan parquet (46) - - -(1) FileSourceScanExecTransformer parquet -Output [2]: [ps_partkey#X, ps_suppkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(ps_partkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [2]: [ps_partkey#X, ps_suppkey#X] -Arguments: isnotnull(ps_partkey#X) - -(3) FileSourceScanExecTransformer parquet -Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] -ReadSchema: struct - -(4) FilterExecTransformer -Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -Arguments: (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) - -(5) WholeStageCodegenTransformer (X) -Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -Arguments: false - -(6) ColumnarBroadcastExchange -Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(7) BroadcastQueryStage -Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -Arguments: X - -(8) InputAdapter -Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] - -(9) InputIteratorTransformer -Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] - -(10) BroadcastHashJoinExecTransformer -Left keys [1]: [ps_partkey#X] -Right keys [1]: [p_partkey#X] -Join condition: None - -(11) ProjectExecTransformer -Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] -Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] - -(12) FlushableHashAggregateExecTransformer -Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] -Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -Functions: [] -Aggregate Attributes: [] -Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] - -(13) ProjectExecTransformer -Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] - -(14) WholeStageCodegenTransformer (X) -Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -Arguments: false - -(15) VeloxResizeBatches -Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] - -(16) ColumnarExchange -Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(17) ShuffleQueryStage -Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -Arguments: X - -(18) InputAdapter -Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] - -(19) InputIteratorTransformer -Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] - -(20) RegularHashAggregateExecTransformer -Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -Functions: [] -Aggregate Attributes: [] -Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] - -(21) FlushableHashAggregateExecTransformer -Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -Keys [3]: [p_brand#X, p_type#X, p_size#X] -Functions [1]: [partial_count(distinct ps_suppkey#X)] -Aggregate Attributes [1]: [count(ps_suppkey#X)#X] -Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] - -(22) ProjectExecTransformer -Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] -Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] - -(23) WholeStageCodegenTransformer (X) -Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] -Arguments: false - -(24) VeloxResizeBatches -Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] - -(25) ColumnarExchange -Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] -Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [shuffle_writer_type=hash] - -(26) ShuffleQueryStage -Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] -Arguments: X - -(27) InputAdapter -Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] - -(28) InputIteratorTransformer -Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] - -(29) RegularHashAggregateExecTransformer -Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -Keys [3]: [p_brand#X, p_type#X, p_size#X] -Functions [1]: [count(distinct ps_suppkey#X)] -Aggregate Attributes [1]: [count(ps_suppkey#X)#X] -Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] - -(30) WholeStageCodegenTransformer (X) -Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -Arguments: false - -(31) VeloxResizeBatches -Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] - -(32) ColumnarExchange -Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(33) ShuffleQueryStage -Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -Arguments: X - -(34) InputAdapter -Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] - -(35) InputIteratorTransformer -Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] - -(36) SortExecTransformer -Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 - -(37) WholeStageCodegenTransformer (X) -Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -Arguments: false - -(38) VeloxColumnarToRow -Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] - -(39) Scan parquet -Output [2]: [ps_partkey#X, ps_suppkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(ps_partkey)] -ReadSchema: struct - -(40) Filter -Input [2]: [ps_partkey#X, ps_suppkey#X] -Condition : isnotnull(ps_partkey#X) - -(41) Scan parquet -Output [2]: [s_suppkey#X, s_comment#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_comment)] -ReadSchema: struct - -(42) Filter -Input [2]: [s_suppkey#X, s_comment#X] -Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) - -(43) Project -Output [1]: [s_suppkey#X] -Input [2]: [s_suppkey#X, s_comment#X] - -(44) BroadcastExchange -Input [1]: [s_suppkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] - -(45) BroadcastHashJoin -Left keys [1]: [ps_suppkey#X] -Right keys [1]: [s_suppkey#X] -Join condition: None - -(46) Scan parquet -Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] -ReadSchema: struct - -(47) Filter -Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) - -(48) BroadcastExchange -Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(49) BroadcastHashJoin -Left keys [1]: [ps_partkey#X] -Right keys [1]: [p_partkey#X] -Join condition: None - -(50) Project -Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] -Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] - -(51) HashAggregate -Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] -Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -Functions: [] -Aggregate Attributes: [] -Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] - -(52) Exchange -Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(53) HashAggregate -Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -Functions: [] -Aggregate Attributes: [] -Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] - -(54) HashAggregate -Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -Keys [3]: [p_brand#X, p_type#X, p_size#X] -Functions [1]: [partial_count(distinct ps_suppkey#X)] -Aggregate Attributes [1]: [count(ps_suppkey#X)#X] -Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] - -(55) Exchange -Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(56) HashAggregate -Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -Keys [3]: [p_brand#X, p_type#X, p_size#X] -Functions [1]: [count(distinct ps_suppkey#X)] -Aggregate Attributes [1]: [count(ps_suppkey#X)#X] -Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] - -(57) Exchange -Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(58) Sort -Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 - -(59) AdaptiveSparkPlan -Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/17.txt deleted file mode 100644 index 142dbad3cb4..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/17.txt +++ /dev/null @@ -1,202 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (36) -+- == Final Plan == - VeloxColumnarToRow (15) - +- ^ ProjectExecTransformer (13) - +- ^ RegularHashAggregateExecTransformer (12) - +- ^ InputIteratorTransformer (11) - +- ShuffleQueryStage (9), Statistics(X) - +- ColumnarExchange (8) - +- VeloxResizeBatches (7) - +- ^ FlushableHashAggregateExecTransformer (5) - +- ^ InputIteratorTransformer (4) - +- RowToVeloxColumnar (2) - +- LocalTableScan (1) -+- == Initial Plan == - HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- BroadcastHashJoin Inner BuildRight (31) - :- Project (23) - : +- BroadcastHashJoin Inner BuildRight (22) - : :- Filter (17) - : : +- Scan parquet (16) - : +- BroadcastExchange (21) - : +- Project (20) - : +- Filter (19) - : +- Scan parquet (18) - +- BroadcastExchange (30) - +- Filter (29) - +- HashAggregate (28) - +- Exchange (27) - +- HashAggregate (26) - +- Filter (25) - +- Scan parquet (24) - - -(1) LocalTableScan -Output [1]: [l_extendedprice#X] -Arguments: , [l_extendedprice#X] - -(2) RowToVeloxColumnar -Input [1]: [l_extendedprice#X] - -(3) InputAdapter -Input [1]: [l_extendedprice#X] - -(4) InputIteratorTransformer -Input [1]: [l_extendedprice#X] - -(5) FlushableHashAggregateExecTransformer -Input [1]: [l_extendedprice#X] -Keys: [] -Functions [1]: [partial_sum(l_extendedprice#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [2]: [sum#X, isEmpty#X] - -(6) WholeStageCodegenTransformer (X) -Input [2]: [sum#X, isEmpty#X] -Arguments: false - -(7) VeloxResizeBatches -Input [2]: [sum#X, isEmpty#X] - -(8) ColumnarExchange -Input [2]: [sum#X, isEmpty#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(9) ShuffleQueryStage -Output [2]: [sum#X, isEmpty#X] -Arguments: X - -(10) InputAdapter -Input [2]: [sum#X, isEmpty#X] - -(11) InputIteratorTransformer -Input [2]: [sum#X, isEmpty#X] - -(12) RegularHashAggregateExecTransformer -Input [2]: [sum#X, isEmpty#X] -Keys: [] -Functions [1]: [sum(l_extendedprice#X)] -Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] -Results [1]: [sum(l_extendedprice#X)#X] - -(13) ProjectExecTransformer -Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] -Input [1]: [sum(l_extendedprice#X)#X] - -(14) WholeStageCodegenTransformer (X) -Input [1]: [avg_yearly#X] -Arguments: false - -(15) VeloxColumnarToRow -Input [1]: [avg_yearly#X] - -(16) Scan parquet -Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] -ReadSchema: struct - -(17) Filter -Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) - -(18) Scan parquet -Output [3]: [p_partkey#X, p_brand#X, p_container#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] -ReadSchema: struct - -(19) Filter -Input [3]: [p_partkey#X, p_brand#X, p_container#X] -Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) - -(20) Project -Output [1]: [p_partkey#X] -Input [3]: [p_partkey#X, p_brand#X, p_container#X] - -(21) BroadcastExchange -Input [1]: [p_partkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(22) BroadcastHashJoin -Left keys [1]: [l_partkey#X] -Right keys [1]: [p_partkey#X] -Join condition: None - -(23) Project -Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] -Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] - -(24) Scan parquet -Output [2]: [l_partkey#X, l_quantity#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_partkey)] -ReadSchema: struct - -(25) Filter -Input [2]: [l_partkey#X, l_quantity#X] -Condition : isnotnull(l_partkey#X) - -(26) HashAggregate -Input [2]: [l_partkey#X, l_quantity#X] -Keys [1]: [l_partkey#X] -Functions [1]: [partial_avg(l_quantity#X)] -Aggregate Attributes [2]: [sum#X, count#X] -Results [3]: [l_partkey#X, sum#X, count#X] - -(27) Exchange -Input [3]: [l_partkey#X, sum#X, count#X] -Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(28) HashAggregate -Input [3]: [l_partkey#X, sum#X, count#X] -Keys [1]: [l_partkey#X] -Functions [1]: [avg(l_quantity#X)] -Aggregate Attributes [1]: [avg(l_quantity#X)#X] -Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] - -(29) Filter -Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] -Condition : isnotnull((0.2 * avg(l_quantity))#X) - -(30) BroadcastExchange -Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] -Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] - -(31) BroadcastHashJoin -Left keys [1]: [p_partkey#X] -Right keys [1]: [l_partkey#X] -Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) - -(32) Project -Output [1]: [l_extendedprice#X] -Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] - -(33) HashAggregate -Input [1]: [l_extendedprice#X] -Keys: [] -Functions [1]: [partial_sum(l_extendedprice#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [2]: [sum#X, isEmpty#X] - -(34) Exchange -Input [2]: [sum#X, isEmpty#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] - -(35) HashAggregate -Input [2]: [sum#X, isEmpty#X] -Keys: [] -Functions [1]: [sum(l_extendedprice#X)] -Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] -Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] - -(36) AdaptiveSparkPlan -Output [1]: [avg_yearly#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt deleted file mode 100644 index bee21f3f208..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt +++ /dev/null @@ -1,478 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (88) -+- == Final Plan == - VeloxColumnarToRow (55) - +- TakeOrderedAndProjectExecTransformer (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- VeloxResizeBatches (47) - +- ^ ProjectExecTransformer (45) - +- ^ FlushableHashAggregateExecTransformer (44) - +- ^ ProjectExecTransformer (43) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (42) - :- ^ ProjectExecTransformer (29) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (28) - : :- ^ InputIteratorTransformer (7) - : : +- BroadcastQueryStage (5), Statistics(X) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ FilterExecTransformer (2) - : : +- ^ FileSourceScanExecTransformer parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (27) - : :- ^ FilterExecTransformer (9) - : : +- ^ FileSourceScanExecTransformer parquet (8) - : +- ^ InputIteratorTransformer (26) - : +- BroadcastQueryStage (24), Statistics(X) - : +- ColumnarBroadcastExchange (23) - : +- ^ ProjectExecTransformer (21) - : +- ^ FilterExecTransformer (20) - : +- ^ RegularHashAggregateExecTransformer (19) - : +- ^ InputIteratorTransformer (18) - : +- ShuffleQueryStage (16), Statistics(X) - : +- ColumnarExchange (15) - : +- VeloxResizeBatches (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ FileSourceScanExecTransformer parquet (10) - +- ^ InputIteratorTransformer (41) - +- BroadcastQueryStage (39), Statistics(X) - +- ColumnarBroadcastExchange (38) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (36) - :- ^ FilterExecTransformer (31) - : +- ^ FileSourceScanExecTransformer parquet (30) - +- ^ InputIteratorTransformer (35) - +- BroadcastQueryStage (33), Statistics(X) - +- ReusedExchange (32) -+- == Initial Plan == - TakeOrderedAndProject (87) - +- HashAggregate (86) - +- Exchange (85) - +- HashAggregate (84) - +- Project (83) - +- BroadcastHashJoin Inner BuildRight (82) - :- Project (70) - : +- BroadcastHashJoin Inner BuildLeft (69) - : :- BroadcastExchange (58) - : : +- Filter (57) - : : +- Scan parquet (56) - : +- BroadcastHashJoin LeftSemi BuildRight (68) - : :- Filter (60) - : : +- Scan parquet (59) - : +- BroadcastExchange (67) - : +- Project (66) - : +- Filter (65) - : +- HashAggregate (64) - : +- Exchange (63) - : +- HashAggregate (62) - : +- Scan parquet (61) - +- BroadcastExchange (81) - +- BroadcastHashJoin LeftSemi BuildRight (80) - :- Filter (72) - : +- Scan parquet (71) - +- BroadcastExchange (79) - +- Project (78) - +- Filter (77) - +- HashAggregate (76) - +- Exchange (75) - +- HashAggregate (74) - +- Scan parquet (73) - - -(1) FileSourceScanExecTransformer parquet -Output [2]: [c_custkey#X, c_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_custkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [2]: [c_custkey#X, c_name#X] -Arguments: isnotnull(c_custkey#X) - -(3) WholeStageCodegenTransformer (X) -Input [2]: [c_custkey#X, c_name#X] -Arguments: false - -(4) ColumnarBroadcastExchange -Input [2]: [c_custkey#X, c_name#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(5) BroadcastQueryStage -Output [2]: [c_custkey#X, c_name#X] -Arguments: X - -(6) InputAdapter -Input [2]: [c_custkey#X, c_name#X] - -(7) InputIteratorTransformer -Input [2]: [c_custkey#X, c_name#X] - -(8) FileSourceScanExecTransformer parquet -Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] -ReadSchema: struct - -(9) FilterExecTransformer -Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Arguments: (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) - -(10) FileSourceScanExecTransformer parquet -Output [2]: [l_orderkey#X, l_quantity#X] -Batched: true -Location: InMemoryFileIndex [*] -ReadSchema: struct - -(11) FlushableHashAggregateExecTransformer -Input [2]: [l_orderkey#X, l_quantity#X] -Keys [1]: [l_orderkey#X] -Functions [1]: [partial_sum(l_quantity#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [3]: [l_orderkey#X, sum#X, isEmpty#X] - -(12) ProjectExecTransformer -Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] -Input [3]: [l_orderkey#X, sum#X, isEmpty#X] - -(13) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] -Arguments: false - -(14) VeloxResizeBatches -Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] - -(15) ColumnarExchange -Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] - -(16) ShuffleQueryStage -Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -Arguments: X - -(17) InputAdapter -Input [3]: [l_orderkey#X, sum#X, isEmpty#X] - -(18) InputIteratorTransformer -Input [3]: [l_orderkey#X, sum#X, isEmpty#X] - -(19) RegularHashAggregateExecTransformer -Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -Keys [1]: [l_orderkey#X] -Functions [1]: [sum(l_quantity#X)] -Aggregate Attributes [1]: [sum(l_quantity#X)#X] -Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] - -(20) FilterExecTransformer -Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) - -(21) ProjectExecTransformer -Output [1]: [l_orderkey#X] -Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] - -(22) WholeStageCodegenTransformer (X) -Input [1]: [l_orderkey#X] -Arguments: false - -(23) ColumnarBroadcastExchange -Input [1]: [l_orderkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(24) BroadcastQueryStage -Output [1]: [l_orderkey#X] -Arguments: X - -(25) InputAdapter -Input [1]: [l_orderkey#X] - -(26) InputIteratorTransformer -Input [1]: [l_orderkey#X] - -(27) BroadcastHashJoinExecTransformer -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(28) BroadcastHashJoinExecTransformer -Left keys [1]: [c_custkey#X] -Right keys [1]: [o_custkey#X] -Join condition: None - -(29) ProjectExecTransformer -Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] - -(30) FileSourceScanExecTransformer parquet -Output [2]: [l_orderkey#X, l_quantity#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_orderkey)] -ReadSchema: struct - -(31) FilterExecTransformer -Input [2]: [l_orderkey#X, l_quantity#X] -Arguments: isnotnull(l_orderkey#X) - -(32) ReusedExchange [Reuses operator id: 23] -Output [1]: [l_orderkey#X] - -(33) BroadcastQueryStage -Output [1]: [l_orderkey#X] -Arguments: X - -(34) InputAdapter -Input [1]: [l_orderkey#X] - -(35) InputIteratorTransformer -Input [1]: [l_orderkey#X] - -(36) BroadcastHashJoinExecTransformer -Left keys [1]: [l_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(37) WholeStageCodegenTransformer (X) -Input [2]: [l_orderkey#X, l_quantity#X] -Arguments: false - -(38) ColumnarBroadcastExchange -Input [2]: [l_orderkey#X, l_quantity#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(39) BroadcastQueryStage -Output [2]: [l_orderkey#X, l_quantity#X] -Arguments: X - -(40) InputAdapter -Input [2]: [l_orderkey#X, l_quantity#X] - -(41) InputIteratorTransformer -Input [2]: [l_orderkey#X, l_quantity#X] - -(42) BroadcastHashJoinExecTransformer -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(43) ProjectExecTransformer -Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] -Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] - -(44) FlushableHashAggregateExecTransformer -Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] -Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] -Functions [1]: [partial_sum(l_quantity#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] - -(45) ProjectExecTransformer -Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] - -(46) WholeStageCodegenTransformer (X) -Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -Arguments: false - -(47) VeloxResizeBatches -Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] - -(48) ColumnarExchange -Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] - -(49) ShuffleQueryStage -Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -Arguments: X - -(50) InputAdapter -Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] - -(51) InputIteratorTransformer -Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] - -(52) RegularHashAggregateExecTransformer -Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] -Functions [1]: [sum(l_quantity#X)] -Aggregate Attributes [1]: [sum(l_quantity#X)#X] -Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] - -(53) WholeStageCodegenTransformer (X) -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: false - -(54) TakeOrderedAndProjectExecTransformer -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 - -(55) VeloxColumnarToRow -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] - -(56) Scan parquet -Output [2]: [c_custkey#X, c_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_custkey)] -ReadSchema: struct - -(57) Filter -Input [2]: [c_custkey#X, c_name#X] -Condition : isnotnull(c_custkey#X) - -(58) BroadcastExchange -Input [2]: [c_custkey#X, c_name#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(59) Scan parquet -Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] -ReadSchema: struct - -(60) Filter -Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) - -(61) Scan parquet -Output [2]: [l_orderkey#X, l_quantity#X] -Batched: true -Location: InMemoryFileIndex [*] -ReadSchema: struct - -(62) HashAggregate -Input [2]: [l_orderkey#X, l_quantity#X] -Keys [1]: [l_orderkey#X] -Functions [1]: [partial_sum(l_quantity#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [3]: [l_orderkey#X, sum#X, isEmpty#X] - -(63) Exchange -Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(64) HashAggregate -Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -Keys [1]: [l_orderkey#X] -Functions [1]: [sum(l_quantity#X)] -Aggregate Attributes [1]: [sum(l_quantity#X)#X] -Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] - -(65) Filter -Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) - -(66) Project -Output [1]: [l_orderkey#X] -Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] - -(67) BroadcastExchange -Input [1]: [l_orderkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(68) BroadcastHashJoin -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(69) BroadcastHashJoin -Left keys [1]: [c_custkey#X] -Right keys [1]: [o_custkey#X] -Join condition: None - -(70) Project -Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] - -(71) Scan parquet -Output [2]: [l_orderkey#X, l_quantity#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_orderkey)] -ReadSchema: struct - -(72) Filter -Input [2]: [l_orderkey#X, l_quantity#X] -Condition : isnotnull(l_orderkey#X) - -(73) Scan parquet -Output [2]: [l_orderkey#X, l_quantity#X] -Batched: true -Location: InMemoryFileIndex [*] -ReadSchema: struct - -(74) HashAggregate -Input [2]: [l_orderkey#X, l_quantity#X] -Keys [1]: [l_orderkey#X] -Functions [1]: [partial_sum(l_quantity#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [3]: [l_orderkey#X, sum#X, isEmpty#X] - -(75) Exchange -Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(76) HashAggregate -Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -Keys [1]: [l_orderkey#X] -Functions [1]: [sum(l_quantity#X)] -Aggregate Attributes [1]: [sum(l_quantity#X)#X] -Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] - -(77) Filter -Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) - -(78) Project -Output [1]: [l_orderkey#X] -Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] - -(79) BroadcastExchange -Input [1]: [l_orderkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(80) BroadcastHashJoin -Left keys [1]: [l_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(81) BroadcastExchange -Input [2]: [l_orderkey#X, l_quantity#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(82) BroadcastHashJoin -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(83) Project -Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] -Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] - -(84) HashAggregate -Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] -Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] -Functions [1]: [partial_sum(l_quantity#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] - -(85) Exchange -Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(86) HashAggregate -Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] -Functions [1]: [sum(l_quantity#X)] -Aggregate Attributes [1]: [sum(l_quantity#X)#X] -Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] - -(87) TakeOrderedAndProject -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] - -(88) AdaptiveSparkPlan -Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/19.txt deleted file mode 100644 index 275cf39f533..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/19.txt +++ /dev/null @@ -1,189 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (34) -+- == Final Plan == - VeloxColumnarToRow (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- VeloxResizeBatches (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ FileSourceScanExecTransformer parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ FileSourceScanExecTransformer parquet (4) -+- == Initial Plan == - HashAggregate (33) - +- Exchange (32) - +- HashAggregate (31) - +- Project (30) - +- BroadcastHashJoin Inner BuildRight (29) - :- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- BroadcastExchange (28) - +- Filter (27) - +- Scan parquet (26) - - -(1) FileSourceScanExecTransformer parquet -Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] -ReadSchema: struct - -(2) FilterExecTransformer -Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -Arguments: ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) - -(3) ProjectExecTransformer -Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] - -(4) FileSourceScanExecTransformer parquet -Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] -ReadSchema: struct - -(5) FilterExecTransformer -Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -Arguments: (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) - -(6) WholeStageCodegenTransformer (X) -Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -Arguments: false - -(7) ColumnarBroadcastExchange -Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(8) BroadcastQueryStage -Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -Arguments: X - -(9) InputAdapter -Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] - -(10) InputIteratorTransformer -Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] - -(11) BroadcastHashJoinExecTransformer -Left keys [1]: [l_partkey#X] -Right keys [1]: [p_partkey#X] -Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) - -(12) ProjectExecTransformer -Output [1]: [CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] -Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] - -(13) FlushableHashAggregateExecTransformer -Input [1]: [_pre_X#X] -Keys: [] -Functions [1]: [partial_sum(_pre_X#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [2]: [sum#X, isEmpty#X] - -(14) WholeStageCodegenTransformer (X) -Input [2]: [sum#X, isEmpty#X] -Arguments: false - -(15) VeloxResizeBatches -Input [2]: [sum#X, isEmpty#X] - -(16) ColumnarExchange -Input [2]: [sum#X, isEmpty#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(17) ShuffleQueryStage -Output [2]: [sum#X, isEmpty#X] -Arguments: X - -(18) InputAdapter -Input [2]: [sum#X, isEmpty#X] - -(19) InputIteratorTransformer -Input [2]: [sum#X, isEmpty#X] - -(20) RegularHashAggregateExecTransformer -Input [2]: [sum#X, isEmpty#X] -Keys: [] -Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] - -(21) WholeStageCodegenTransformer (X) -Input [1]: [revenue#X] -Arguments: false - -(22) VeloxColumnarToRow -Input [1]: [revenue#X] - -(23) Scan parquet -Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] -ReadSchema: struct - -(24) Filter -Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) - -(25) Project -Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] - -(26) Scan parquet -Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] -ReadSchema: struct - -(27) Filter -Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) - -(28) BroadcastExchange -Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(29) BroadcastHashJoin -Left keys [1]: [l_partkey#X] -Right keys [1]: [p_partkey#X] -Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) - -(30) Project -Output [2]: [l_extendedprice#X, l_discount#X] -Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] - -(31) HashAggregate -Input [2]: [l_extendedprice#X, l_discount#X] -Keys: [] -Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [2]: [sum#X, isEmpty#X] - -(32) Exchange -Input [2]: [sum#X, isEmpty#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] - -(33) HashAggregate -Input [2]: [sum#X, isEmpty#X] -Keys: [] -Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] - -(34) AdaptiveSparkPlan -Output [1]: [revenue#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt deleted file mode 100644 index 5a1cc18236d..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt +++ /dev/null @@ -1,571 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (109) -+- == Final Plan == - VeloxColumnarToRow (70) - +- AQEShuffleRead (69) - +- ShuffleQueryStage (68), Statistics(X) - +- ColumnarExchange (67) - +- VeloxResizeBatches (66) - +- ^ ProjectExecTransformer (64) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (63) - :- ^ ProjectExecTransformer (54) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (53) - : :- ^ InputIteratorTransformer (10) - : : +- AQEShuffleRead (8) - : : +- ShuffleQueryStage (7), Statistics(X) - : : +- ColumnarExchange (6) - : : +- VeloxResizeBatches (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ FileSourceScanExecTransformer parquet (1) - : +- ^ InputIteratorTransformer (52) - : +- BroadcastQueryStage (50), Statistics(X) - : +- ColumnarBroadcastExchange (49) - : +- ^ ProjectExecTransformer (47) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (46) - : :- ^ InputIteratorTransformer (26) - : : +- BroadcastQueryStage (24), Statistics(X) - : : +- ColumnarBroadcastExchange (23) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (21) - : : :- ^ FilterExecTransformer (12) - : : : +- ^ FileSourceScanExecTransformer parquet (11) - : : +- ^ InputIteratorTransformer (20) - : : +- BroadcastQueryStage (18), Statistics(X) - : : +- ColumnarBroadcastExchange (17) - : : +- ^ ProjectExecTransformer (15) - : : +- ^ FilterExecTransformer (14) - : : +- ^ FileSourceScanExecTransformer parquet (13) - : +- ^ FilterExecTransformer (45) - : +- ^ ProjectExecTransformer (44) - : +- ^ RegularHashAggregateExecTransformer (43) - : +- ^ InputIteratorTransformer (42) - : +- ShuffleQueryStage (40), Statistics(X) - : +- ColumnarExchange (39) - : +- VeloxResizeBatches (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ FlushableHashAggregateExecTransformer (35) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (34) - : :- ^ ProjectExecTransformer (29) - : : +- ^ FilterExecTransformer (28) - : : +- ^ FileSourceScanExecTransformer parquet (27) - : +- ^ InputIteratorTransformer (33) - : +- BroadcastQueryStage (31), Statistics(X) - : +- ReusedExchange (30) - +- ^ InputIteratorTransformer (62) - +- BroadcastQueryStage (60), Statistics(X) - +- ColumnarBroadcastExchange (59) - +- ^ ProjectExecTransformer (57) - +- ^ FilterExecTransformer (56) - +- ^ FileSourceScanExecTransformer parquet (55) -+- == Initial Plan == - Sort (108) - +- Exchange (107) - +- Project (106) - +- BroadcastHashJoin Inner BuildRight (105) - :- Project (100) - : +- SortMergeJoin LeftSemi (99) - : :- Sort (74) - : : +- Exchange (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Sort (98) - : +- Exchange (97) - : +- Project (96) - : +- BroadcastHashJoin Inner BuildLeft (95) - : :- BroadcastExchange (82) - : : +- BroadcastHashJoin LeftSemi BuildRight (81) - : : :- Filter (76) - : : : +- Scan parquet (75) - : : +- BroadcastExchange (80) - : : +- Project (79) - : : +- Filter (78) - : : +- Scan parquet (77) - : +- Filter (94) - : +- HashAggregate (93) - : +- Exchange (92) - : +- HashAggregate (91) - : +- BroadcastHashJoin LeftSemi BuildRight (90) - : :- Project (85) - : : +- Filter (84) - : : +- Scan parquet (83) - : +- BroadcastExchange (89) - : +- Project (88) - : +- Filter (87) - : +- Scan parquet (86) - +- BroadcastExchange (104) - +- Project (103) - +- Filter (102) - +- Scan parquet (101) - - -(1) FileSourceScanExecTransformer parquet -Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_nationkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Arguments: isnotnull(s_nationkey#X) - -(3) ProjectExecTransformer -Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] - -(4) WholeStageCodegenTransformer (X) -Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Arguments: false - -(5) VeloxResizeBatches -Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] - -(6) ColumnarExchange -Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(7) ShuffleQueryStage -Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Arguments: X - -(8) AQEShuffleRead -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Arguments: local - -(9) InputAdapter -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] - -(10) InputIteratorTransformer -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] - -(11) FileSourceScanExecTransformer parquet -Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] -ReadSchema: struct - -(12) FilterExecTransformer -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) - -(13) FileSourceScanExecTransformer parquet -Output [2]: [p_partkey#X, p_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] -ReadSchema: struct - -(14) FilterExecTransformer -Input [2]: [p_partkey#X, p_name#X] -Arguments: (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) - -(15) ProjectExecTransformer -Output [1]: [p_partkey#X] -Input [2]: [p_partkey#X, p_name#X] - -(16) WholeStageCodegenTransformer (X) -Input [1]: [p_partkey#X] -Arguments: false - -(17) ColumnarBroadcastExchange -Input [1]: [p_partkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(18) BroadcastQueryStage -Output [1]: [p_partkey#X] -Arguments: X - -(19) InputAdapter -Input [1]: [p_partkey#X] - -(20) InputIteratorTransformer -Input [1]: [p_partkey#X] - -(21) BroadcastHashJoinExecTransformer -Left keys [1]: [ps_partkey#X] -Right keys [1]: [p_partkey#X] -Join condition: None - -(22) WholeStageCodegenTransformer (X) -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: false - -(23) ColumnarBroadcastExchange -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] - -(24) BroadcastQueryStage -Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: X - -(25) InputAdapter -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] - -(26) InputIteratorTransformer -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] - -(27) FileSourceScanExecTransformer parquet -Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] -ReadSchema: struct - -(28) FilterExecTransformer -Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) - -(29) ProjectExecTransformer -Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] - -(30) ReusedExchange [Reuses operator id: 17] -Output [1]: [p_partkey#X] - -(31) BroadcastQueryStage -Output [1]: [p_partkey#X] -Arguments: X - -(32) InputAdapter -Input [1]: [p_partkey#X] - -(33) InputIteratorTransformer -Input [1]: [p_partkey#X] - -(34) BroadcastHashJoinExecTransformer -Left keys [1]: [l_partkey#X] -Right keys [1]: [p_partkey#X] -Join condition: None - -(35) FlushableHashAggregateExecTransformer -Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -Keys [2]: [l_partkey#X, l_suppkey#X] -Functions [1]: [partial_sum(l_quantity#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] - -(36) ProjectExecTransformer -Output [5]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] - -(37) WholeStageCodegenTransformer (X) -Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -Arguments: false - -(38) VeloxResizeBatches -Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] - -(39) ColumnarExchange -Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] - -(40) ShuffleQueryStage -Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -Arguments: X - -(41) InputAdapter -Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] - -(42) InputIteratorTransformer -Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] - -(43) RegularHashAggregateExecTransformer -Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -Keys [2]: [l_partkey#X, l_suppkey#X] -Functions [1]: [sum(l_quantity#X)] -Aggregate Attributes [1]: [sum(l_quantity#X)#X] -Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] - -(44) ProjectExecTransformer -Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] - -(45) FilterExecTransformer -Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -Arguments: isnotnull((0.5 * sum(l_quantity))#X) - -(46) BroadcastHashJoinExecTransformer -Left keys [2]: [ps_partkey#X, ps_suppkey#X] -Right keys [2]: [l_partkey#X, l_suppkey#X] -Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) - -(47) ProjectExecTransformer -Output [1]: [ps_suppkey#X] -Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] - -(48) WholeStageCodegenTransformer (X) -Input [1]: [ps_suppkey#X] -Arguments: false - -(49) ColumnarBroadcastExchange -Input [1]: [ps_suppkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(50) BroadcastQueryStage -Output [1]: [ps_suppkey#X] -Arguments: X - -(51) InputAdapter -Input [1]: [ps_suppkey#X] - -(52) InputIteratorTransformer -Input [1]: [ps_suppkey#X] - -(53) BroadcastHashJoinExecTransformer -Left keys [1]: [s_suppkey#X] -Right keys [1]: [ps_suppkey#X] -Join condition: None - -(54) ProjectExecTransformer -Output [3]: [s_name#X, s_address#X, s_nationkey#X] -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] - -(55) FileSourceScanExecTransformer parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] -ReadSchema: struct - -(56) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) - -(57) ProjectExecTransformer -Output [1]: [n_nationkey#X] -Input [2]: [n_nationkey#X, n_name#X] - -(58) WholeStageCodegenTransformer (X) -Input [1]: [n_nationkey#X] -Arguments: false - -(59) ColumnarBroadcastExchange -Input [1]: [n_nationkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(60) BroadcastQueryStage -Output [1]: [n_nationkey#X] -Arguments: X - -(61) InputAdapter -Input [1]: [n_nationkey#X] - -(62) InputIteratorTransformer -Input [1]: [n_nationkey#X] - -(63) BroadcastHashJoinExecTransformer -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(64) ProjectExecTransformer -Output [2]: [s_name#X, s_address#X] -Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] - -(65) WholeStageCodegenTransformer (X) -Input [2]: [s_name#X, s_address#X] -Arguments: false - -(66) VeloxResizeBatches -Input [2]: [s_name#X, s_address#X] - -(67) ColumnarExchange -Input [2]: [s_name#X, s_address#X] -Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(68) ShuffleQueryStage -Output [2]: [s_name#X, s_address#X] -Arguments: X - -(69) AQEShuffleRead -Input [2]: [s_name#X, s_address#X] -Arguments: local - -(70) VeloxColumnarToRow -Input [2]: [s_name#X, s_address#X] - -(71) Scan parquet -Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_nationkey)] -ReadSchema: struct - -(72) Filter -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Condition : isnotnull(s_nationkey#X) - -(73) Exchange -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(74) Sort -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(75) Scan parquet -Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] -ReadSchema: struct - -(76) Filter -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) - -(77) Scan parquet -Output [2]: [p_partkey#X, p_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] -ReadSchema: struct - -(78) Filter -Input [2]: [p_partkey#X, p_name#X] -Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) - -(79) Project -Output [1]: [p_partkey#X] -Input [2]: [p_partkey#X, p_name#X] - -(80) BroadcastExchange -Input [1]: [p_partkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(81) BroadcastHashJoin -Left keys [1]: [ps_partkey#X] -Right keys [1]: [p_partkey#X] -Join condition: None - -(82) BroadcastExchange -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] - -(83) Scan parquet -Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] -ReadSchema: struct - -(84) Filter -Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) - -(85) Project -Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] - -(86) Scan parquet -Output [2]: [p_partkey#X, p_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] -ReadSchema: struct - -(87) Filter -Input [2]: [p_partkey#X, p_name#X] -Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) - -(88) Project -Output [1]: [p_partkey#X] -Input [2]: [p_partkey#X, p_name#X] - -(89) BroadcastExchange -Input [1]: [p_partkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(90) BroadcastHashJoin -Left keys [1]: [l_partkey#X] -Right keys [1]: [p_partkey#X] -Join condition: None - -(91) HashAggregate -Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -Keys [2]: [l_partkey#X, l_suppkey#X] -Functions [1]: [partial_sum(l_quantity#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] - -(92) Exchange -Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(93) HashAggregate -Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -Keys [2]: [l_partkey#X, l_suppkey#X] -Functions [1]: [sum(l_quantity#X)] -Aggregate Attributes [1]: [sum(l_quantity#X)#X] -Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] - -(94) Filter -Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -Condition : isnotnull((0.5 * sum(l_quantity))#X) - -(95) BroadcastHashJoin -Left keys [2]: [ps_partkey#X, ps_suppkey#X] -Right keys [2]: [l_partkey#X, l_suppkey#X] -Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) - -(96) Project -Output [1]: [ps_suppkey#X] -Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] - -(97) Exchange -Input [1]: [ps_suppkey#X] -Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(98) Sort -Input [1]: [ps_suppkey#X] -Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 - -(99) SortMergeJoin -Left keys [1]: [s_suppkey#X] -Right keys [1]: [ps_suppkey#X] -Join condition: None - -(100) Project -Output [3]: [s_name#X, s_address#X, s_nationkey#X] -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] - -(101) Scan parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] -ReadSchema: struct - -(102) Filter -Input [2]: [n_nationkey#X, n_name#X] -Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) - -(103) Project -Output [1]: [n_nationkey#X] -Input [2]: [n_nationkey#X, n_name#X] - -(104) BroadcastExchange -Input [1]: [n_nationkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(105) BroadcastHashJoin -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(106) Project -Output [2]: [s_name#X, s_address#X] -Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] - -(107) Exchange -Input [2]: [s_name#X, s_address#X] -Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(108) Sort -Input [2]: [s_name#X, s_address#X] -Arguments: [s_name#X ASC NULLS FIRST], true, 0 - -(109) AdaptiveSparkPlan -Output [2]: [s_name#X, s_address#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/21.txt deleted file mode 100644 index b01c6fb1ecb..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/21.txt +++ /dev/null @@ -1,493 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (92) -+- == Final Plan == - VeloxColumnarToRow (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- VeloxResizeBatches (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- BroadcastQueryStage (5), Statistics(X) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ FileSourceScanExecTransformer parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ FileSourceScanExecTransformer parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- BroadcastQueryStage (14), Statistics(X) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ FileSourceScanExecTransformer parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ FilterExecTransformer (19) - : : +- ^ FileSourceScanExecTransformer parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ FilterExecTransformer (30) - : +- ^ FileSourceScanExecTransformer parquet (29) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ FileSourceScanExecTransformer parquet (39) -+- == Initial Plan == - TakeOrderedAndProject (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (81) - : +- BroadcastHashJoin Inner BuildRight (80) - : :- Project (75) - : : +- BroadcastHashJoin Inner BuildLeft (74) - : : :- BroadcastExchange (62) - : : : +- Filter (61) - : : : +- Scan parquet (60) - : : +- BroadcastHashJoin LeftAnti BuildRight (73) - : : :- BroadcastHashJoin LeftSemi BuildRight (68) - : : : :- Project (65) - : : : : +- Filter (64) - : : : : +- Scan parquet (63) - : : : +- BroadcastExchange (67) - : : : +- Scan parquet (66) - : : +- BroadcastExchange (72) - : : +- Project (71) - : : +- Filter (70) - : : +- Scan parquet (69) - : +- BroadcastExchange (79) - : +- Project (78) - : +- Filter (77) - : +- Scan parquet (76) - +- BroadcastExchange (85) - +- Project (84) - +- Filter (83) - +- Scan parquet (82) - - -(1) FileSourceScanExecTransformer parquet -Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(3) WholeStageCodegenTransformer (X) -Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -Arguments: false - -(4) ColumnarBroadcastExchange -Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(5) BroadcastQueryStage -Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -Arguments: X - -(6) InputAdapter -Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] - -(7) InputIteratorTransformer -Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] - -(8) FileSourceScanExecTransformer parquet -Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] -ReadSchema: struct - -(9) FilterExecTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Arguments: ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(10) ProjectExecTransformer -Output [2]: [l_orderkey#X, l_suppkey#X] -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] - -(11) FileSourceScanExecTransformer parquet -Output [2]: [l_orderkey#X, l_suppkey#X] -Batched: true -Location: InMemoryFileIndex [*] -ReadSchema: struct - -(12) WholeStageCodegenTransformer (X) -Input [2]: [l_orderkey#X, l_suppkey#X] -Arguments: false - -(13) ColumnarBroadcastExchange -Input [2]: [l_orderkey#X, l_suppkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(14) BroadcastQueryStage -Output [2]: [l_orderkey#X, l_suppkey#X] -Arguments: X - -(15) InputAdapter -Input [2]: [l_orderkey#X, l_suppkey#X] - -(16) InputIteratorTransformer -Input [2]: [l_orderkey#X, l_suppkey#X] - -(17) BroadcastHashJoinExecTransformer -Left keys [1]: [l_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: NOT (l_suppkey#X = l_suppkey#X) - -(18) FileSourceScanExecTransformer parquet -Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] -ReadSchema: struct - -(19) FilterExecTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Arguments: ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) - -(20) ProjectExecTransformer -Output [2]: [l_orderkey#X, l_suppkey#X] -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] - -(21) WholeStageCodegenTransformer (X) -Input [2]: [l_orderkey#X, l_suppkey#X] -Arguments: false - -(22) ColumnarBroadcastExchange -Input [2]: [l_orderkey#X, l_suppkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(23) BroadcastQueryStage -Output [2]: [l_orderkey#X, l_suppkey#X] -Arguments: X - -(24) InputAdapter -Input [2]: [l_orderkey#X, l_suppkey#X] - -(25) InputIteratorTransformer -Input [2]: [l_orderkey#X, l_suppkey#X] - -(26) BroadcastHashJoinExecTransformer -Left keys [1]: [l_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: NOT (l_suppkey#X = l_suppkey#X) - -(27) BroadcastHashJoinExecTransformer -Left keys [1]: [s_suppkey#X] -Right keys [1]: [l_suppkey#X] -Join condition: None - -(28) ProjectExecTransformer -Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] - -(29) FileSourceScanExecTransformer parquet -Output [2]: [o_orderkey#X, o_orderstatus#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] -ReadSchema: struct - -(30) FilterExecTransformer -Input [2]: [o_orderkey#X, o_orderstatus#X] -Arguments: ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) - -(31) ProjectExecTransformer -Output [1]: [o_orderkey#X] -Input [2]: [o_orderkey#X, o_orderstatus#X] - -(32) WholeStageCodegenTransformer (X) -Input [1]: [o_orderkey#X] -Arguments: false - -(33) ColumnarBroadcastExchange -Input [1]: [o_orderkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(34) BroadcastQueryStage -Output [1]: [o_orderkey#X] -Arguments: X - -(35) InputAdapter -Input [1]: [o_orderkey#X] - -(36) InputIteratorTransformer -Input [1]: [o_orderkey#X] - -(37) BroadcastHashJoinExecTransformer -Left keys [1]: [l_orderkey#X] -Right keys [1]: [o_orderkey#X] -Join condition: None - -(38) ProjectExecTransformer -Output [2]: [s_name#X, s_nationkey#X] -Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] - -(39) FileSourceScanExecTransformer parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] -ReadSchema: struct - -(40) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) - -(41) ProjectExecTransformer -Output [1]: [n_nationkey#X] -Input [2]: [n_nationkey#X, n_name#X] - -(42) WholeStageCodegenTransformer (X) -Input [1]: [n_nationkey#X] -Arguments: false - -(43) ColumnarBroadcastExchange -Input [1]: [n_nationkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(44) BroadcastQueryStage -Output [1]: [n_nationkey#X] -Arguments: X - -(45) InputAdapter -Input [1]: [n_nationkey#X] - -(46) InputIteratorTransformer -Input [1]: [n_nationkey#X] - -(47) BroadcastHashJoinExecTransformer -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(48) ProjectExecTransformer -Output [1]: [s_name#X] -Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] - -(49) FlushableHashAggregateExecTransformer -Input [1]: [s_name#X] -Keys [1]: [s_name#X] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#X] -Results [2]: [s_name#X, count#X] - -(50) ProjectExecTransformer -Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] -Input [2]: [s_name#X, count#X] - -(51) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, s_name#X, count#X] -Arguments: false - -(52) VeloxResizeBatches -Input [3]: [hash_partition_key#X, s_name#X, count#X] - -(53) ColumnarExchange -Input [3]: [hash_partition_key#X, s_name#X, count#X] -Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [shuffle_writer_type=hash] - -(54) ShuffleQueryStage -Output [2]: [s_name#X, count#X] -Arguments: X - -(55) InputAdapter -Input [2]: [s_name#X, count#X] - -(56) InputIteratorTransformer -Input [2]: [s_name#X, count#X] - -(57) RegularHashAggregateExecTransformer -Input [2]: [s_name#X, count#X] -Keys [1]: [s_name#X] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#X] -Results [2]: [s_name#X, count(1)#X AS numwait#X] - -(58) WholeStageCodegenTransformer (X) -Input [2]: [s_name#X, numwait#X] -Arguments: false - -(59) VeloxColumnarToRow -Input [2]: [s_name#X, numwait#X] - -(60) Scan parquet -Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] -ReadSchema: struct - -(61) Filter -Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(62) BroadcastExchange -Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(63) Scan parquet -Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] -ReadSchema: struct - -(64) Filter -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(65) Project -Output [2]: [l_orderkey#X, l_suppkey#X] -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] - -(66) Scan parquet -Output [2]: [l_orderkey#X, l_suppkey#X] -Batched: true -Location: InMemoryFileIndex [*] -ReadSchema: struct - -(67) BroadcastExchange -Input [2]: [l_orderkey#X, l_suppkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(68) BroadcastHashJoin -Left keys [1]: [l_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: NOT (l_suppkey#X = l_suppkey#X) - -(69) Scan parquet -Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] -ReadSchema: struct - -(70) Filter -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) - -(71) Project -Output [2]: [l_orderkey#X, l_suppkey#X] -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] - -(72) BroadcastExchange -Input [2]: [l_orderkey#X, l_suppkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(73) BroadcastHashJoin -Left keys [1]: [l_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: NOT (l_suppkey#X = l_suppkey#X) - -(74) BroadcastHashJoin -Left keys [1]: [s_suppkey#X] -Right keys [1]: [l_suppkey#X] -Join condition: None - -(75) Project -Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] - -(76) Scan parquet -Output [2]: [o_orderkey#X, o_orderstatus#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] -ReadSchema: struct - -(77) Filter -Input [2]: [o_orderkey#X, o_orderstatus#X] -Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) - -(78) Project -Output [1]: [o_orderkey#X] -Input [2]: [o_orderkey#X, o_orderstatus#X] - -(79) BroadcastExchange -Input [1]: [o_orderkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(80) BroadcastHashJoin -Left keys [1]: [l_orderkey#X] -Right keys [1]: [o_orderkey#X] -Join condition: None - -(81) Project -Output [2]: [s_name#X, s_nationkey#X] -Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] - -(82) Scan parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] -ReadSchema: struct - -(83) Filter -Input [2]: [n_nationkey#X, n_name#X] -Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) - -(84) Project -Output [1]: [n_nationkey#X] -Input [2]: [n_nationkey#X, n_name#X] - -(85) BroadcastExchange -Input [1]: [n_nationkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(86) BroadcastHashJoin -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(87) Project -Output [1]: [s_name#X] -Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] - -(88) HashAggregate -Input [1]: [s_name#X] -Keys [1]: [s_name#X] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#X] -Results [2]: [s_name#X, count#X] - -(89) Exchange -Input [2]: [s_name#X, count#X] -Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(90) HashAggregate -Input [2]: [s_name#X, count#X] -Keys [1]: [s_name#X] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#X] -Results [2]: [s_name#X, count(1)#X AS numwait#X] - -(91) TakeOrderedAndProject -Input [2]: [s_name#X, numwait#X] -Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] - -(92) AdaptiveSparkPlan -Output [2]: [s_name#X, numwait#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/22.txt deleted file mode 100644 index 58844e687d9..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/22.txt +++ /dev/null @@ -1,351 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (40) -+- == Final Plan == - VeloxColumnarToRow (28) - +- ^ SortExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- VeloxResizeBatches (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- VeloxResizeBatches (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ FilterExecTransformer (2) - : +- ^ FileSourceScanExecTransformer parquet (1) - +- ^ InputIteratorTransformer (8) - +- BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ FileSourceScanExecTransformer parquet (3) -+- == Initial Plan == - Sort (39) - +- Exchange (38) - +- HashAggregate (37) - +- Exchange (36) - +- HashAggregate (35) - +- Project (34) - +- BroadcastHashJoin LeftAnti BuildRight (33) - :- Filter (30) - : +- Scan parquet (29) - +- BroadcastExchange (32) - +- Scan parquet (31) - - -(1) FileSourceScanExecTransformer parquet -Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_acctbal)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -Arguments: ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) - -(3) FileSourceScanExecTransformer parquet -Output [1]: [o_custkey#X] -Batched: true -Location: InMemoryFileIndex [*] -ReadSchema: struct - -(4) WholeStageCodegenTransformer (X) -Input [1]: [o_custkey#X] -Arguments: false - -(5) ColumnarBroadcastExchange -Input [1]: [o_custkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(6) BroadcastQueryStage -Output [1]: [o_custkey#X] -Arguments: X - -(7) InputAdapter -Input [1]: [o_custkey#X] - -(8) InputIteratorTransformer -Input [1]: [o_custkey#X] - -(9) BroadcastHashJoinExecTransformer -Left keys [1]: [c_custkey#X] -Right keys [1]: [o_custkey#X] -Join condition: None - -(10) ProjectExecTransformer -Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] -Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] - -(11) FlushableHashAggregateExecTransformer -Input [2]: [cntrycode#X, c_acctbal#X] -Keys [1]: [cntrycode#X] -Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] -Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] -Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] - -(12) ProjectExecTransformer -Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] -Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] - -(13) WholeStageCodegenTransformer (X) -Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] -Arguments: false - -(14) VeloxResizeBatches -Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] - -(15) ColumnarExchange -Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] - -(16) ShuffleQueryStage -Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -Arguments: X - -(17) InputAdapter -Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] - -(18) InputIteratorTransformer -Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] - -(19) RegularHashAggregateExecTransformer -Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -Keys [1]: [cntrycode#X] -Functions [2]: [count(1), sum(c_acctbal#X)] -Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] -Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] - -(20) WholeStageCodegenTransformer (X) -Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -Arguments: false - -(21) VeloxResizeBatches -Input [3]: [cntrycode#X, numcust#X, totacctbal#X] - -(22) ColumnarExchange -Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(23) ShuffleQueryStage -Output [3]: [cntrycode#X, numcust#X, totacctbal#X] -Arguments: X - -(24) InputAdapter -Input [3]: [cntrycode#X, numcust#X, totacctbal#X] - -(25) InputIteratorTransformer -Input [3]: [cntrycode#X, numcust#X, totacctbal#X] - -(26) SortExecTransformer -Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 - -(27) WholeStageCodegenTransformer (X) -Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -Arguments: false - -(28) VeloxColumnarToRow -Input [3]: [cntrycode#X, numcust#X, totacctbal#X] - -(29) Scan parquet -Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_acctbal)] -ReadSchema: struct - -(30) Filter -Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) - -(31) Scan parquet -Output [1]: [o_custkey#X] -Batched: true -Location: InMemoryFileIndex [*] -ReadSchema: struct - -(32) BroadcastExchange -Input [1]: [o_custkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(33) BroadcastHashJoin -Left keys [1]: [c_custkey#X] -Right keys [1]: [o_custkey#X] -Join condition: None - -(34) Project -Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] -Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] - -(35) HashAggregate -Input [2]: [cntrycode#X, c_acctbal#X] -Keys [1]: [cntrycode#X] -Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] -Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] -Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] - -(36) Exchange -Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(37) HashAggregate -Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -Keys [1]: [cntrycode#X] -Functions [2]: [count(1), sum(c_acctbal#X)] -Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] -Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] - -(38) Exchange -Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(39) Sort -Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 - -(40) AdaptiveSparkPlan -Output [3]: [cntrycode#X, numcust#X, totacctbal#X] -Arguments: isFinalPlan=true - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (60) -+- == Final Plan == - VeloxColumnarToRow (53) - +- ^ RegularHashAggregateExecTransformer (51) - +- ^ InputIteratorTransformer (50) - +- ShuffleQueryStage (48), Statistics(X) - +- ColumnarExchange (47) - +- VeloxResizeBatches (46) - +- ^ FlushableHashAggregateExecTransformer (44) - +- ^ ProjectExecTransformer (43) - +- ^ FilterExecTransformer (42) - +- ^ FileSourceScanExecTransformer parquet (41) -+- == Initial Plan == - HashAggregate (59) - +- Exchange (58) - +- HashAggregate (57) - +- Project (56) - +- Filter (55) - +- Scan parquet (54) - - -(41) FileSourceScanExecTransformer parquet -Output [2]: [c_phone#X, c_acctbal#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] -ReadSchema: struct - -(42) FilterExecTransformer -Input [2]: [c_phone#X, c_acctbal#X] -Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) - -(43) ProjectExecTransformer -Output [1]: [c_acctbal#X] -Input [2]: [c_phone#X, c_acctbal#X] - -(44) FlushableHashAggregateExecTransformer -Input [1]: [c_acctbal#X] -Keys: [] -Functions [1]: [partial_avg(c_acctbal#X)] -Aggregate Attributes [2]: [sum#X, count#X] -Results [2]: [sum#X, count#X] - -(45) WholeStageCodegenTransformer (X) -Input [2]: [sum#X, count#X] -Arguments: false - -(46) VeloxResizeBatches -Input [2]: [sum#X, count#X] - -(47) ColumnarExchange -Input [2]: [sum#X, count#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(48) ShuffleQueryStage -Output [2]: [sum#X, count#X] -Arguments: X - -(49) InputAdapter -Input [2]: [sum#X, count#X] - -(50) InputIteratorTransformer -Input [2]: [sum#X, count#X] - -(51) RegularHashAggregateExecTransformer -Input [2]: [sum#X, count#X] -Keys: [] -Functions [1]: [avg(c_acctbal#X)] -Aggregate Attributes [1]: [avg(c_acctbal#X)#X] -Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] - -(52) WholeStageCodegenTransformer (X) -Input [1]: [avg(c_acctbal)#X] -Arguments: false - -(53) VeloxColumnarToRow -Input [1]: [avg(c_acctbal)#X] - -(54) Scan parquet -Output [2]: [c_phone#X, c_acctbal#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] -ReadSchema: struct - -(55) Filter -Input [2]: [c_phone#X, c_acctbal#X] -Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) - -(56) Project -Output [1]: [c_acctbal#X] -Input [2]: [c_phone#X, c_acctbal#X] - -(57) HashAggregate -Input [1]: [c_acctbal#X] -Keys: [] -Functions [1]: [partial_avg(c_acctbal#X)] -Aggregate Attributes [2]: [sum#X, count#X] -Results [2]: [sum#X, count#X] - -(58) Exchange -Input [2]: [sum#X, count#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] - -(59) HashAggregate -Input [2]: [sum#X, count#X] -Keys: [] -Functions [1]: [avg(c_acctbal#X)] -Aggregate Attributes [1]: [avg(c_acctbal#X)#X] -Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] - -(60) AdaptiveSparkPlan -Output [1]: [avg(c_acctbal)#X] -Arguments: isFinalPlan=true - -Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (60) -+- == Final Plan == - VeloxColumnarToRow (53) - +- ^ RegularHashAggregateExecTransformer (51) - +- ^ InputIteratorTransformer (50) - +- ShuffleQueryStage (48), Statistics(X) - +- ColumnarExchange (47) - +- VeloxResizeBatches (46) - +- ^ FlushableHashAggregateExecTransformer (44) - +- ^ ProjectExecTransformer (43) - +- ^ FilterExecTransformer (42) - +- ^ FileSourceScanExecTransformer parquet (41) -+- == Initial Plan == - HashAggregate (59) - +- Exchange (58) - +- HashAggregate (57) - +- Project (56) - +- Filter (55) - +- Scan parquet (54) \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt deleted file mode 100644 index 7d9ef012d0d..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt +++ /dev/null @@ -1,293 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (54) -+- == Final Plan == - VeloxColumnarToRow (35) - +- TakeOrderedAndProjectExecTransformer (34) - +- ^ ProjectExecTransformer (32) - +- ^ RegularHashAggregateExecTransformer (31) - +- ^ InputIteratorTransformer (30) - +- ShuffleQueryStage (28), Statistics(X) - +- ColumnarExchange (27) - +- VeloxResizeBatches (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : :- ^ InputIteratorTransformer (8) - : : +- BroadcastQueryStage (6), Statistics(X) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ FileSourceScanExecTransformer parquet (1) - : +- ^ FilterExecTransformer (10) - : +- ^ FileSourceScanExecTransformer parquet (9) - +- ^ InputIteratorTransformer (20) - +- BroadcastQueryStage (18), Statistics(X) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FilterExecTransformer (14) - +- ^ FileSourceScanExecTransformer parquet (13) -+- == Initial Plan == - TakeOrderedAndProject (53) - +- HashAggregate (52) - +- Exchange (51) - +- HashAggregate (50) - +- Project (49) - +- BroadcastHashJoin Inner BuildRight (48) - :- Project (43) - : +- BroadcastHashJoin Inner BuildLeft (42) - : :- BroadcastExchange (39) - : : +- Project (38) - : : +- Filter (37) - : : +- Scan parquet (36) - : +- Filter (41) - : +- Scan parquet (40) - +- BroadcastExchange (47) - +- Project (46) - +- Filter (45) - +- Scan parquet (44) - - -(1) FileSourceScanExecTransformer parquet -Output [2]: [c_custkey#X, c_mktsegment#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [2]: [c_custkey#X, c_mktsegment#X] -Arguments: ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) - -(3) ProjectExecTransformer -Output [1]: [c_custkey#X] -Input [2]: [c_custkey#X, c_mktsegment#X] - -(4) WholeStageCodegenTransformer (X) -Input [1]: [c_custkey#X] -Arguments: false - -(5) ColumnarBroadcastExchange -Input [1]: [c_custkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(6) BroadcastQueryStage -Output [1]: [c_custkey#X] -Arguments: X - -(7) InputAdapter -Input [1]: [c_custkey#X] - -(8) InputIteratorTransformer -Input [1]: [c_custkey#X] - -(9) FileSourceScanExecTransformer parquet -Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] -ReadSchema: struct - -(10) FilterExecTransformer -Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -Arguments: (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(11) BroadcastHashJoinExecTransformer -Left keys [1]: [c_custkey#X] -Right keys [1]: [o_custkey#X] -Join condition: None - -(12) ProjectExecTransformer -Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] - -(13) FileSourceScanExecTransformer parquet -Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] -ReadSchema: struct - -(14) FilterExecTransformer -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) - -(15) ProjectExecTransformer -Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(16) WholeStageCodegenTransformer (X) -Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -Arguments: false - -(17) ColumnarBroadcastExchange -Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(18) BroadcastQueryStage -Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -Arguments: X - -(19) InputAdapter -Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] - -(20) InputIteratorTransformer -Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] - -(21) BroadcastHashJoinExecTransformer -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(22) ProjectExecTransformer -Output [4]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] -Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] - -(23) FlushableHashAggregateExecTransformer -Input [4]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, _pre_X#X] -Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] -Functions [1]: [partial_sum(_pre_X#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] - -(24) ProjectExecTransformer -Output [6]: [hash(l_orderkey#X, o_orderdate#X, o_shippriority#X, 42) AS hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] - -(25) WholeStageCodegenTransformer (X) -Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -Arguments: false - -(26) VeloxResizeBatches -Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] - -(27) ColumnarExchange -Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] - -(28) ShuffleQueryStage -Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -Arguments: X - -(29) InputAdapter -Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] - -(30) InputIteratorTransformer -Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] - -(31) RegularHashAggregateExecTransformer -Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] -Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] - -(32) ProjectExecTransformer -Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] - -(33) WholeStageCodegenTransformer (X) -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: false - -(34) TakeOrderedAndProjectExecTransformer -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 - -(35) VeloxColumnarToRow -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] - -(36) Scan parquet -Output [2]: [c_custkey#X, c_mktsegment#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] -ReadSchema: struct - -(37) Filter -Input [2]: [c_custkey#X, c_mktsegment#X] -Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) - -(38) Project -Output [1]: [c_custkey#X] -Input [2]: [c_custkey#X, c_mktsegment#X] - -(39) BroadcastExchange -Input [1]: [c_custkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(40) Scan parquet -Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] -ReadSchema: struct - -(41) Filter -Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(42) BroadcastHashJoin -Left keys [1]: [c_custkey#X] -Right keys [1]: [o_custkey#X] -Join condition: None - -(43) Project -Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] - -(44) Scan parquet -Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] -ReadSchema: struct - -(45) Filter -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) - -(46) Project -Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(47) BroadcastExchange -Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(48) BroadcastHashJoin -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(49) Project -Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] - -(50) HashAggregate -Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] -Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] - -(51) Exchange -Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(52) HashAggregate -Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] -Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] - -(53) TakeOrderedAndProject -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] - -(54) AdaptiveSparkPlan -Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/4.txt deleted file mode 100644 index 0a55ea344d4..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/4.txt +++ /dev/null @@ -1,244 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (46) -+- == Final Plan == - VeloxColumnarToRow (31) - +- ^ SortExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- VeloxResizeBatches (24) - +- ^ RegularHashAggregateExecTransformer (22) - +- ^ InputIteratorTransformer (21) - +- ShuffleQueryStage (19), Statistics(X) - +- ColumnarExchange (18) - +- VeloxResizeBatches (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ FileSourceScanExecTransformer parquet (1) - +- ^ InputIteratorTransformer (11) - +- BroadcastQueryStage (9), Statistics(X) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ FilterExecTransformer (5) - +- ^ FileSourceScanExecTransformer parquet (4) -+- == Initial Plan == - Sort (45) - +- Exchange (44) - +- HashAggregate (43) - +- Exchange (42) - +- HashAggregate (41) - +- Project (40) - +- BroadcastHashJoin LeftSemi BuildRight (39) - :- Project (34) - : +- Filter (33) - : +- Scan parquet (32) - +- BroadcastExchange (38) - +- Project (37) - +- Filter (36) - +- Scan parquet (35) - - -(1) FileSourceScanExecTransformer parquet -Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -Arguments: ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) - -(3) ProjectExecTransformer -Output [2]: [o_orderkey#X, o_orderpriority#X] -Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] - -(4) FileSourceScanExecTransformer parquet -Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] -ReadSchema: struct - -(5) FilterExecTransformer -Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -Arguments: ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) - -(6) ProjectExecTransformer -Output [1]: [l_orderkey#X] -Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] - -(7) WholeStageCodegenTransformer (X) -Input [1]: [l_orderkey#X] -Arguments: false - -(8) ColumnarBroadcastExchange -Input [1]: [l_orderkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(9) BroadcastQueryStage -Output [1]: [l_orderkey#X] -Arguments: X - -(10) InputAdapter -Input [1]: [l_orderkey#X] - -(11) InputIteratorTransformer -Input [1]: [l_orderkey#X] - -(12) BroadcastHashJoinExecTransformer -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(13) ProjectExecTransformer -Output [1]: [o_orderpriority#X] -Input [2]: [o_orderkey#X, o_orderpriority#X] - -(14) FlushableHashAggregateExecTransformer -Input [1]: [o_orderpriority#X] -Keys [1]: [o_orderpriority#X] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#X] -Results [2]: [o_orderpriority#X, count#X] - -(15) ProjectExecTransformer -Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] -Input [2]: [o_orderpriority#X, count#X] - -(16) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] -Arguments: false - -(17) VeloxResizeBatches -Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] - -(18) ColumnarExchange -Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] -Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [shuffle_writer_type=hash] - -(19) ShuffleQueryStage -Output [2]: [o_orderpriority#X, count#X] -Arguments: X - -(20) InputAdapter -Input [2]: [o_orderpriority#X, count#X] - -(21) InputIteratorTransformer -Input [2]: [o_orderpriority#X, count#X] - -(22) RegularHashAggregateExecTransformer -Input [2]: [o_orderpriority#X, count#X] -Keys [1]: [o_orderpriority#X] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#X] -Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] - -(23) WholeStageCodegenTransformer (X) -Input [2]: [o_orderpriority#X, order_count#X] -Arguments: false - -(24) VeloxResizeBatches -Input [2]: [o_orderpriority#X, order_count#X] - -(25) ColumnarExchange -Input [2]: [o_orderpriority#X, order_count#X] -Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(26) ShuffleQueryStage -Output [2]: [o_orderpriority#X, order_count#X] -Arguments: X - -(27) InputAdapter -Input [2]: [o_orderpriority#X, order_count#X] - -(28) InputIteratorTransformer -Input [2]: [o_orderpriority#X, order_count#X] - -(29) SortExecTransformer -Input [2]: [o_orderpriority#X, order_count#X] -Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 - -(30) WholeStageCodegenTransformer (X) -Input [2]: [o_orderpriority#X, order_count#X] -Arguments: false - -(31) VeloxColumnarToRow -Input [2]: [o_orderpriority#X, order_count#X] - -(32) Scan parquet -Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] -ReadSchema: struct - -(33) Filter -Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) - -(34) Project -Output [2]: [o_orderkey#X, o_orderpriority#X] -Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] - -(35) Scan parquet -Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] -ReadSchema: struct - -(36) Filter -Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) - -(37) Project -Output [1]: [l_orderkey#X] -Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] - -(38) BroadcastExchange -Input [1]: [l_orderkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(39) BroadcastHashJoin -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(40) Project -Output [1]: [o_orderpriority#X] -Input [2]: [o_orderkey#X, o_orderpriority#X] - -(41) HashAggregate -Input [1]: [o_orderpriority#X] -Keys [1]: [o_orderpriority#X] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#X] -Results [2]: [o_orderpriority#X, count#X] - -(42) Exchange -Input [2]: [o_orderpriority#X, count#X] -Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(43) HashAggregate -Input [2]: [o_orderpriority#X, count#X] -Keys [1]: [o_orderpriority#X] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#X] -Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] - -(44) Exchange -Input [2]: [o_orderpriority#X, order_count#X] -Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(45) Sort -Input [2]: [o_orderpriority#X, order_count#X] -Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 - -(46) AdaptiveSparkPlan -Output [2]: [o_orderpriority#X, order_count#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/5.txt deleted file mode 100644 index 48e55eb2be0..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/5.txt +++ /dev/null @@ -1,540 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (102) -+- == Final Plan == - VeloxColumnarToRow (67) - +- ^ SortExecTransformer (65) - +- ^ InputIteratorTransformer (64) - +- ShuffleQueryStage (62), Statistics(X) - +- ColumnarExchange (61) - +- VeloxResizeBatches (60) - +- ^ RegularHashAggregateExecTransformer (58) - +- ^ InputIteratorTransformer (57) - +- ShuffleQueryStage (55), Statistics(X) - +- ColumnarExchange (54) - +- VeloxResizeBatches (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ FileSourceScanExecTransformer parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ FileSourceScanExecTransformer parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ FileSourceScanExecTransformer parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ FileSourceScanExecTransformer parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ FileSourceScanExecTransformer parquet (31) - +- ^ InputIteratorTransformer (47) - +- BroadcastQueryStage (45), Statistics(X) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ FilterExecTransformer (41) - +- ^ FileSourceScanExecTransformer parquet (40) -+- == Initial Plan == - Sort (101) - +- Exchange (100) - +- HashAggregate (99) - +- Exchange (98) - +- HashAggregate (97) - +- Project (96) - +- BroadcastHashJoin Inner BuildRight (95) - :- Project (90) - : +- BroadcastHashJoin Inner BuildRight (89) - : :- Project (85) - : : +- BroadcastHashJoin Inner BuildRight (84) - : : :- Project (80) - : : : +- BroadcastHashJoin Inner BuildRight (79) - : : : :- Project (75) - : : : : +- BroadcastHashJoin Inner BuildLeft (74) - : : : : :- BroadcastExchange (70) - : : : : : +- Filter (69) - : : : : : +- Scan parquet (68) - : : : : +- Project (73) - : : : : +- Filter (72) - : : : : +- Scan parquet (71) - : : : +- BroadcastExchange (78) - : : : +- Filter (77) - : : : +- Scan parquet (76) - : : +- BroadcastExchange (83) - : : +- Filter (82) - : : +- Scan parquet (81) - : +- BroadcastExchange (88) - : +- Filter (87) - : +- Scan parquet (86) - +- BroadcastExchange (94) - +- Project (93) - +- Filter (92) - +- Scan parquet (91) - - -(1) FileSourceScanExecTransformer parquet -Output [2]: [c_custkey#X, c_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(3) WholeStageCodegenTransformer (X) -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: false - -(4) ColumnarBroadcastExchange -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(5) BroadcastQueryStage -Output [2]: [c_custkey#X, c_nationkey#X] -Arguments: X - -(6) InputAdapter -Input [2]: [c_custkey#X, c_nationkey#X] - -(7) InputIteratorTransformer -Input [2]: [c_custkey#X, c_nationkey#X] - -(8) FileSourceScanExecTransformer parquet -Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] -ReadSchema: struct - -(9) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(10) ProjectExecTransformer -Output [2]: [o_orderkey#X, o_custkey#X] -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] - -(11) BroadcastHashJoinExecTransformer -Left keys [1]: [c_custkey#X] -Right keys [1]: [o_custkey#X] -Join condition: None - -(12) ProjectExecTransformer -Output [2]: [c_nationkey#X, o_orderkey#X] -Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] - -(13) FileSourceScanExecTransformer parquet -Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] -ReadSchema: struct - -(14) FilterExecTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) - -(15) WholeStageCodegenTransformer (X) -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: false - -(16) ColumnarBroadcastExchange -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(17) BroadcastQueryStage -Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: X - -(18) InputAdapter -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(19) InputIteratorTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(20) BroadcastHashJoinExecTransformer -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(21) ProjectExecTransformer -Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(22) FileSourceScanExecTransformer parquet -Output [2]: [s_suppkey#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] -ReadSchema: struct - -(23) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(24) WholeStageCodegenTransformer (X) -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: false - -(25) ColumnarBroadcastExchange -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] - -(26) BroadcastQueryStage -Output [2]: [s_suppkey#X, s_nationkey#X] -Arguments: X - -(27) InputAdapter -Input [2]: [s_suppkey#X, s_nationkey#X] - -(28) InputIteratorTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] - -(29) BroadcastHashJoinExecTransformer -Left keys [2]: [l_suppkey#X, c_nationkey#X] -Right keys [2]: [s_suppkey#X, s_nationkey#X] -Join condition: None - -(30) ProjectExecTransformer -Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] - -(31) FileSourceScanExecTransformer parquet -Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] -ReadSchema: struct - -(32) FilterExecTransformer -Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) - -(33) WholeStageCodegenTransformer (X) -Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -Arguments: false - -(34) ColumnarBroadcastExchange -Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(35) BroadcastQueryStage -Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -Arguments: X - -(36) InputAdapter -Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] - -(37) InputIteratorTransformer -Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] - -(38) BroadcastHashJoinExecTransformer -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(39) ProjectExecTransformer -Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] - -(40) FileSourceScanExecTransformer parquet -Output [2]: [r_regionkey#X, r_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] -ReadSchema: struct - -(41) FilterExecTransformer -Input [2]: [r_regionkey#X, r_name#X] -Arguments: ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) - -(42) ProjectExecTransformer -Output [1]: [r_regionkey#X] -Input [2]: [r_regionkey#X, r_name#X] - -(43) WholeStageCodegenTransformer (X) -Input [1]: [r_regionkey#X] -Arguments: false - -(44) ColumnarBroadcastExchange -Input [1]: [r_regionkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(45) BroadcastQueryStage -Output [1]: [r_regionkey#X] -Arguments: X - -(46) InputAdapter -Input [1]: [r_regionkey#X] - -(47) InputIteratorTransformer -Input [1]: [r_regionkey#X] - -(48) BroadcastHashJoinExecTransformer -Left keys [1]: [n_regionkey#X] -Right keys [1]: [r_regionkey#X] -Join condition: None - -(49) ProjectExecTransformer -Output [2]: [n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] -Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] - -(50) FlushableHashAggregateExecTransformer -Input [2]: [n_name#X, _pre_X#X] -Keys [1]: [n_name#X] -Functions [1]: [partial_sum(_pre_X#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [3]: [n_name#X, sum#X, isEmpty#X] - -(51) ProjectExecTransformer -Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] -Input [3]: [n_name#X, sum#X, isEmpty#X] - -(52) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] -Arguments: false - -(53) VeloxResizeBatches -Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] - -(54) ColumnarExchange -Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] - -(55) ShuffleQueryStage -Output [3]: [n_name#X, sum#X, isEmpty#X] -Arguments: X - -(56) InputAdapter -Input [3]: [n_name#X, sum#X, isEmpty#X] - -(57) InputIteratorTransformer -Input [3]: [n_name#X, sum#X, isEmpty#X] - -(58) RegularHashAggregateExecTransformer -Input [3]: [n_name#X, sum#X, isEmpty#X] -Keys [1]: [n_name#X] -Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] - -(59) WholeStageCodegenTransformer (X) -Input [2]: [n_name#X, revenue#X] -Arguments: false - -(60) VeloxResizeBatches -Input [2]: [n_name#X, revenue#X] - -(61) ColumnarExchange -Input [2]: [n_name#X, revenue#X] -Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(62) ShuffleQueryStage -Output [2]: [n_name#X, revenue#X] -Arguments: X - -(63) InputAdapter -Input [2]: [n_name#X, revenue#X] - -(64) InputIteratorTransformer -Input [2]: [n_name#X, revenue#X] - -(65) SortExecTransformer -Input [2]: [n_name#X, revenue#X] -Arguments: [revenue#X DESC NULLS LAST], true, 0 - -(66) WholeStageCodegenTransformer (X) -Input [2]: [n_name#X, revenue#X] -Arguments: false - -(67) VeloxColumnarToRow -Input [2]: [n_name#X, revenue#X] - -(68) Scan parquet -Output [2]: [c_custkey#X, c_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] -ReadSchema: struct - -(69) Filter -Input [2]: [c_custkey#X, c_nationkey#X] -Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(70) BroadcastExchange -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(71) Scan parquet -Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] -ReadSchema: struct - -(72) Filter -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(73) Project -Output [2]: [o_orderkey#X, o_custkey#X] -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] - -(74) BroadcastHashJoin -Left keys [1]: [c_custkey#X] -Right keys [1]: [o_custkey#X] -Join condition: None - -(75) Project -Output [2]: [c_nationkey#X, o_orderkey#X] -Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] - -(76) Scan parquet -Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] -ReadSchema: struct - -(77) Filter -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) - -(78) BroadcastExchange -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(79) BroadcastHashJoin -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(80) Project -Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(81) Scan parquet -Output [2]: [s_suppkey#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] -ReadSchema: struct - -(82) Filter -Input [2]: [s_suppkey#X, s_nationkey#X] -Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(83) BroadcastExchange -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] - -(84) BroadcastHashJoin -Left keys [2]: [l_suppkey#X, c_nationkey#X] -Right keys [2]: [s_suppkey#X, s_nationkey#X] -Join condition: None - -(85) Project -Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] - -(86) Scan parquet -Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] -ReadSchema: struct - -(87) Filter -Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) - -(88) BroadcastExchange -Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(89) BroadcastHashJoin -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(90) Project -Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] - -(91) Scan parquet -Output [2]: [r_regionkey#X, r_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] -ReadSchema: struct - -(92) Filter -Input [2]: [r_regionkey#X, r_name#X] -Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) - -(93) Project -Output [1]: [r_regionkey#X] -Input [2]: [r_regionkey#X, r_name#X] - -(94) BroadcastExchange -Input [1]: [r_regionkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(95) BroadcastHashJoin -Left keys [1]: [n_regionkey#X] -Right keys [1]: [r_regionkey#X] -Join condition: None - -(96) Project -Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] -Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] - -(97) HashAggregate -Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] -Keys [1]: [n_name#X] -Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [3]: [n_name#X, sum#X, isEmpty#X] - -(98) Exchange -Input [3]: [n_name#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(99) HashAggregate -Input [3]: [n_name#X, sum#X, isEmpty#X] -Keys [1]: [n_name#X] -Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] - -(100) Exchange -Input [2]: [n_name#X, revenue#X] -Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(101) Sort -Input [2]: [n_name#X, revenue#X] -Arguments: [revenue#X DESC NULLS LAST], true, 0 - -(102) AdaptiveSparkPlan -Output [2]: [n_name#X, revenue#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/6.txt deleted file mode 100644 index 349f8ccdbe8..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/6.txt +++ /dev/null @@ -1,115 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (20) -+- == Final Plan == - VeloxColumnarToRow (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- VeloxResizeBatches (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ FileSourceScanExecTransformer parquet (1) -+- == Initial Plan == - HashAggregate (19) - +- Exchange (18) - +- HashAggregate (17) - +- Project (16) - +- Filter (15) - +- Scan parquet (14) - - -(1) FileSourceScanExecTransformer parquet -Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) - -(3) ProjectExecTransformer -Output [1]: [CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)) AS _pre_X#X] -Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(4) FlushableHashAggregateExecTransformer -Input [1]: [_pre_X#X] -Keys: [] -Functions [1]: [partial_sum(_pre_X#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [2]: [sum#X, isEmpty#X] - -(5) WholeStageCodegenTransformer (X) -Input [2]: [sum#X, isEmpty#X] -Arguments: false - -(6) VeloxResizeBatches -Input [2]: [sum#X, isEmpty#X] - -(7) ColumnarExchange -Input [2]: [sum#X, isEmpty#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(8) ShuffleQueryStage -Output [2]: [sum#X, isEmpty#X] -Arguments: X - -(9) InputAdapter -Input [2]: [sum#X, isEmpty#X] - -(10) InputIteratorTransformer -Input [2]: [sum#X, isEmpty#X] - -(11) RegularHashAggregateExecTransformer -Input [2]: [sum#X, isEmpty#X] -Keys: [] -Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] -Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] - -(12) WholeStageCodegenTransformer (X) -Input [1]: [revenue#X] -Arguments: false - -(13) VeloxColumnarToRow -Input [1]: [revenue#X] - -(14) Scan parquet -Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] -ReadSchema: struct - -(15) Filter -Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) - -(16) Project -Output [2]: [l_extendedprice#X, l_discount#X] -Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(17) HashAggregate -Input [2]: [l_extendedprice#X, l_discount#X] -Keys: [] -Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [2]: [sum#X, isEmpty#X] - -(18) Exchange -Input [2]: [sum#X, isEmpty#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] - -(19) HashAggregate -Input [2]: [sum#X, isEmpty#X] -Keys: [] -Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] -Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] - -(20) AdaptiveSparkPlan -Output [1]: [revenue#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/7.txt deleted file mode 100644 index d593cd1bafa..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/7.txt +++ /dev/null @@ -1,502 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (95) -+- == Final Plan == - VeloxColumnarToRow (62) - +- ^ SortExecTransformer (60) - +- ^ InputIteratorTransformer (59) - +- ShuffleQueryStage (57), Statistics(X) - +- ColumnarExchange (56) - +- VeloxResizeBatches (55) - +- ^ RegularHashAggregateExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- VeloxResizeBatches (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ FileSourceScanExecTransformer parquet (1) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ FileSourceScanExecTransformer parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- BroadcastQueryStage (16), Statistics(X) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ FilterExecTransformer (13) - : : : +- ^ FileSourceScanExecTransformer parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- BroadcastQueryStage (25), Statistics(X) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ FilterExecTransformer (22) - : : +- ^ FileSourceScanExecTransformer parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ FilterExecTransformer (31) - : +- ^ FileSourceScanExecTransformer parquet (30) - +- ^ InputIteratorTransformer (42) - +- BroadcastQueryStage (40), Statistics(X) - +- ReusedExchange (39) -+- == Initial Plan == - Sort (94) - +- Exchange (93) - +- HashAggregate (92) - +- Exchange (91) - +- HashAggregate (90) - +- Project (89) - +- BroadcastHashJoin Inner BuildRight (88) - :- Project (84) - : +- BroadcastHashJoin Inner BuildRight (83) - : :- Project (79) - : : +- BroadcastHashJoin Inner BuildRight (78) - : : :- Project (74) - : : : +- BroadcastHashJoin Inner BuildRight (73) - : : : :- Project (69) - : : : : +- BroadcastHashJoin Inner BuildLeft (68) - : : : : :- BroadcastExchange (65) - : : : : : +- Filter (64) - : : : : : +- Scan parquet (63) - : : : : +- Filter (67) - : : : : +- Scan parquet (66) - : : : +- BroadcastExchange (72) - : : : +- Filter (71) - : : : +- Scan parquet (70) - : : +- BroadcastExchange (77) - : : +- Filter (76) - : : +- Scan parquet (75) - : +- BroadcastExchange (82) - : +- Filter (81) - : +- Scan parquet (80) - +- BroadcastExchange (87) - +- Filter (86) - +- Scan parquet (85) - - -(1) FileSourceScanExecTransformer parquet -Output [2]: [s_suppkey#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(3) WholeStageCodegenTransformer (X) -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: false - -(4) ColumnarBroadcastExchange -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(5) BroadcastQueryStage -Output [2]: [s_suppkey#X, s_nationkey#X] -Arguments: X - -(6) InputAdapter -Input [2]: [s_suppkey#X, s_nationkey#X] - -(7) InputIteratorTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] - -(8) FileSourceScanExecTransformer parquet -Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] -ReadSchema: struct - -(9) FilterExecTransformer -Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(10) BroadcastHashJoinExecTransformer -Left keys [1]: [s_suppkey#X] -Right keys [1]: [l_suppkey#X] -Join condition: None - -(11) ProjectExecTransformer -Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(12) FileSourceScanExecTransformer parquet -Output [2]: [o_orderkey#X, o_custkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] -ReadSchema: struct - -(13) FilterExecTransformer -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) - -(14) WholeStageCodegenTransformer (X) -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: false - -(15) ColumnarBroadcastExchange -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(16) BroadcastQueryStage -Output [2]: [o_orderkey#X, o_custkey#X] -Arguments: X - -(17) InputAdapter -Input [2]: [o_orderkey#X, o_custkey#X] - -(18) InputIteratorTransformer -Input [2]: [o_orderkey#X, o_custkey#X] - -(19) BroadcastHashJoinExecTransformer -Left keys [1]: [l_orderkey#X] -Right keys [1]: [o_orderkey#X] -Join condition: None - -(20) ProjectExecTransformer -Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] - -(21) FileSourceScanExecTransformer parquet -Output [2]: [c_custkey#X, c_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] -ReadSchema: struct - -(22) FilterExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(23) WholeStageCodegenTransformer (X) -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: false - -(24) ColumnarBroadcastExchange -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(25) BroadcastQueryStage -Output [2]: [c_custkey#X, c_nationkey#X] -Arguments: X - -(26) InputAdapter -Input [2]: [c_custkey#X, c_nationkey#X] - -(27) InputIteratorTransformer -Input [2]: [c_custkey#X, c_nationkey#X] - -(28) BroadcastHashJoinExecTransformer -Left keys [1]: [o_custkey#X] -Right keys [1]: [c_custkey#X] -Join condition: None - -(29) ProjectExecTransformer -Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] - -(30) FileSourceScanExecTransformer parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] -ReadSchema: struct - -(31) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) - -(32) WholeStageCodegenTransformer (X) -Input [2]: [n_nationkey#X, n_name#X] -Arguments: false - -(33) ColumnarBroadcastExchange -Input [2]: [n_nationkey#X, n_name#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(34) BroadcastQueryStage -Output [2]: [n_nationkey#X, n_name#X] -Arguments: X - -(35) InputAdapter -Input [2]: [n_nationkey#X, n_name#X] - -(36) InputIteratorTransformer -Input [2]: [n_nationkey#X, n_name#X] - -(37) BroadcastHashJoinExecTransformer -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(38) ProjectExecTransformer -Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] - -(39) ReusedExchange [Reuses operator id: 33] -Output [2]: [n_nationkey#X, n_name#X] - -(40) BroadcastQueryStage -Output [2]: [n_nationkey#X, n_name#X] -Arguments: X - -(41) InputAdapter -Input [2]: [n_nationkey#X, n_name#X] - -(42) InputIteratorTransformer -Input [2]: [n_nationkey#X, n_name#X] - -(43) BroadcastHashJoinExecTransformer -Left keys [1]: [c_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) - -(44) ProjectExecTransformer -Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] -Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] - -(45) FlushableHashAggregateExecTransformer -Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] -Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] -Functions [1]: [partial_sum(volume#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] - -(46) ProjectExecTransformer -Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] - -(47) WholeStageCodegenTransformer (X) -Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -Arguments: false - -(48) VeloxResizeBatches -Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] - -(49) ColumnarExchange -Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] - -(50) ShuffleQueryStage -Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -Arguments: X - -(51) InputAdapter -Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] - -(52) InputIteratorTransformer -Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] - -(53) RegularHashAggregateExecTransformer -Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] -Functions [1]: [sum(volume#X)] -Aggregate Attributes [1]: [sum(volume#X)#X] -Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] - -(54) WholeStageCodegenTransformer (X) -Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -Arguments: false - -(55) VeloxResizeBatches -Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] - -(56) ColumnarExchange -Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(57) ShuffleQueryStage -Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -Arguments: X - -(58) InputAdapter -Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] - -(59) InputIteratorTransformer -Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] - -(60) SortExecTransformer -Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 - -(61) WholeStageCodegenTransformer (X) -Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -Arguments: false - -(62) VeloxColumnarToRow -Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] - -(63) Scan parquet -Output [2]: [s_suppkey#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] -ReadSchema: struct - -(64) Filter -Input [2]: [s_suppkey#X, s_nationkey#X] -Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(65) BroadcastExchange -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(66) Scan parquet -Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] -ReadSchema: struct - -(67) Filter -Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(68) BroadcastHashJoin -Left keys [1]: [s_suppkey#X] -Right keys [1]: [l_suppkey#X] -Join condition: None - -(69) Project -Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(70) Scan parquet -Output [2]: [o_orderkey#X, o_custkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] -ReadSchema: struct - -(71) Filter -Input [2]: [o_orderkey#X, o_custkey#X] -Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) - -(72) BroadcastExchange -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(73) BroadcastHashJoin -Left keys [1]: [l_orderkey#X] -Right keys [1]: [o_orderkey#X] -Join condition: None - -(74) Project -Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] - -(75) Scan parquet -Output [2]: [c_custkey#X, c_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] -ReadSchema: struct - -(76) Filter -Input [2]: [c_custkey#X, c_nationkey#X] -Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(77) BroadcastExchange -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(78) BroadcastHashJoin -Left keys [1]: [o_custkey#X] -Right keys [1]: [c_custkey#X] -Join condition: None - -(79) Project -Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] - -(80) Scan parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] -ReadSchema: struct - -(81) Filter -Input [2]: [n_nationkey#X, n_name#X] -Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) - -(82) BroadcastExchange -Input [2]: [n_nationkey#X, n_name#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(83) BroadcastHashJoin -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(84) Project -Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] - -(85) Scan parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] -ReadSchema: struct - -(86) Filter -Input [2]: [n_nationkey#X, n_name#X] -Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) - -(87) BroadcastExchange -Input [2]: [n_nationkey#X, n_name#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(88) BroadcastHashJoin -Left keys [1]: [c_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) - -(89) Project -Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] -Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] - -(90) HashAggregate -Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] -Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] -Functions [1]: [partial_sum(volume#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] - -(91) Exchange -Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(92) HashAggregate -Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] -Functions [1]: [sum(volume#X)] -Aggregate Attributes [1]: [sum(volume#X)#X] -Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] - -(93) Exchange -Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(94) Sort -Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 - -(95) AdaptiveSparkPlan -Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/8.txt deleted file mode 100644 index f87bbf1246f..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/8.txt +++ /dev/null @@ -1,693 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (131) -+- == Final Plan == - VeloxColumnarToRow (86) - +- ^ SortExecTransformer (84) - +- ^ InputIteratorTransformer (83) - +- ShuffleQueryStage (81), Statistics(X) - +- ColumnarExchange (80) - +- VeloxResizeBatches (79) - +- ^ ProjectExecTransformer (77) - +- ^ RegularHashAggregateExecTransformer (76) - +- ^ InputIteratorTransformer (75) - +- ShuffleQueryStage (73), Statistics(X) - +- ColumnarExchange (72) - +- VeloxResizeBatches (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ FileSourceScanExecTransformer parquet (1) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ FileSourceScanExecTransformer parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- BroadcastQueryStage (17), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ FilterExecTransformer (14) - : : : : : +- ^ FileSourceScanExecTransformer parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- BroadcastQueryStage (26), Statistics(X) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ FilterExecTransformer (23) - : : : : +- ^ FileSourceScanExecTransformer parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- BroadcastQueryStage (35), Statistics(X) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ FilterExecTransformer (32) - : : : +- ^ FileSourceScanExecTransformer parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- BroadcastQueryStage (44), Statistics(X) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ FilterExecTransformer (41) - : : +- ^ FileSourceScanExecTransformer parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- BroadcastQueryStage (53), Statistics(X) - : +- ColumnarBroadcastExchange (52) - : +- ^ FilterExecTransformer (50) - : +- ^ FileSourceScanExecTransformer parquet (49) - +- ^ InputIteratorTransformer (65) - +- BroadcastQueryStage (63), Statistics(X) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ FilterExecTransformer (59) - +- ^ FileSourceScanExecTransformer parquet (58) -+- == Initial Plan == - Sort (130) - +- Exchange (129) - +- HashAggregate (128) - +- Exchange (127) - +- HashAggregate (126) - +- Project (125) - +- BroadcastHashJoin Inner BuildRight (124) - :- Project (119) - : +- BroadcastHashJoin Inner BuildRight (118) - : :- Project (114) - : : +- BroadcastHashJoin Inner BuildRight (113) - : : :- Project (109) - : : : +- BroadcastHashJoin Inner BuildRight (108) - : : : :- Project (104) - : : : : +- BroadcastHashJoin Inner BuildRight (103) - : : : : :- Project (99) - : : : : : +- BroadcastHashJoin Inner BuildRight (98) - : : : : : :- Project (94) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (93) - : : : : : : :- BroadcastExchange (90) - : : : : : : : +- Project (89) - : : : : : : : +- Filter (88) - : : : : : : : +- Scan parquet (87) - : : : : : : +- Filter (92) - : : : : : : +- Scan parquet (91) - : : : : : +- BroadcastExchange (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- BroadcastExchange (102) - : : : : +- Filter (101) - : : : : +- Scan parquet (100) - : : : +- BroadcastExchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- BroadcastExchange (112) - : : +- Filter (111) - : : +- Scan parquet (110) - : +- BroadcastExchange (117) - : +- Filter (116) - : +- Scan parquet (115) - +- BroadcastExchange (123) - +- Project (122) - +- Filter (121) - +- Scan parquet (120) - - -(1) FileSourceScanExecTransformer parquet -Output [2]: [p_partkey#X, p_type#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [2]: [p_partkey#X, p_type#X] -Arguments: ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) - -(3) ProjectExecTransformer -Output [1]: [p_partkey#X] -Input [2]: [p_partkey#X, p_type#X] - -(4) WholeStageCodegenTransformer (X) -Input [1]: [p_partkey#X] -Arguments: false - -(5) ColumnarBroadcastExchange -Input [1]: [p_partkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(6) BroadcastQueryStage -Output [1]: [p_partkey#X] -Arguments: X - -(7) InputAdapter -Input [1]: [p_partkey#X] - -(8) InputIteratorTransformer -Input [1]: [p_partkey#X] - -(9) FileSourceScanExecTransformer parquet -Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] -ReadSchema: struct - -(10) FilterExecTransformer -Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(11) BroadcastHashJoinExecTransformer -Left keys [1]: [p_partkey#X] -Right keys [1]: [l_partkey#X] -Join condition: None - -(12) ProjectExecTransformer -Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(13) FileSourceScanExecTransformer parquet -Output [2]: [s_suppkey#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] -ReadSchema: struct - -(14) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(15) WholeStageCodegenTransformer (X) -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: false - -(16) ColumnarBroadcastExchange -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(17) BroadcastQueryStage -Output [2]: [s_suppkey#X, s_nationkey#X] -Arguments: X - -(18) InputAdapter -Input [2]: [s_suppkey#X, s_nationkey#X] - -(19) InputIteratorTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] - -(20) BroadcastHashJoinExecTransformer -Left keys [1]: [l_suppkey#X] -Right keys [1]: [s_suppkey#X] -Join condition: None - -(21) ProjectExecTransformer -Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] - -(22) FileSourceScanExecTransformer parquet -Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] -ReadSchema: struct - -(23) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) - -(24) WholeStageCodegenTransformer (X) -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: false - -(25) ColumnarBroadcastExchange -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(26) BroadcastQueryStage -Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: X - -(27) InputAdapter -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] - -(28) InputIteratorTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] - -(29) BroadcastHashJoinExecTransformer -Left keys [1]: [l_orderkey#X] -Right keys [1]: [o_orderkey#X] -Join condition: None - -(30) ProjectExecTransformer -Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] - -(31) FileSourceScanExecTransformer parquet -Output [2]: [c_custkey#X, c_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] -ReadSchema: struct - -(32) FilterExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(33) WholeStageCodegenTransformer (X) -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: false - -(34) ColumnarBroadcastExchange -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(35) BroadcastQueryStage -Output [2]: [c_custkey#X, c_nationkey#X] -Arguments: X - -(36) InputAdapter -Input [2]: [c_custkey#X, c_nationkey#X] - -(37) InputIteratorTransformer -Input [2]: [c_custkey#X, c_nationkey#X] - -(38) BroadcastHashJoinExecTransformer -Left keys [1]: [o_custkey#X] -Right keys [1]: [c_custkey#X] -Join condition: None - -(39) ProjectExecTransformer -Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] - -(40) FileSourceScanExecTransformer parquet -Output [2]: [n_nationkey#X, n_regionkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] -ReadSchema: struct - -(41) FilterExecTransformer -Input [2]: [n_nationkey#X, n_regionkey#X] -Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) - -(42) WholeStageCodegenTransformer (X) -Input [2]: [n_nationkey#X, n_regionkey#X] -Arguments: false - -(43) ColumnarBroadcastExchange -Input [2]: [n_nationkey#X, n_regionkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(44) BroadcastQueryStage -Output [2]: [n_nationkey#X, n_regionkey#X] -Arguments: X - -(45) InputAdapter -Input [2]: [n_nationkey#X, n_regionkey#X] - -(46) InputIteratorTransformer -Input [2]: [n_nationkey#X, n_regionkey#X] - -(47) BroadcastHashJoinExecTransformer -Left keys [1]: [c_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(48) ProjectExecTransformer -Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] - -(49) FileSourceScanExecTransformer parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_nationkey)] -ReadSchema: struct - -(50) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: isnotnull(n_nationkey#X) - -(51) WholeStageCodegenTransformer (X) -Input [2]: [n_nationkey#X, n_name#X] -Arguments: false - -(52) ColumnarBroadcastExchange -Input [2]: [n_nationkey#X, n_name#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(53) BroadcastQueryStage -Output [2]: [n_nationkey#X, n_name#X] -Arguments: X - -(54) InputAdapter -Input [2]: [n_nationkey#X, n_name#X] - -(55) InputIteratorTransformer -Input [2]: [n_nationkey#X, n_name#X] - -(56) BroadcastHashJoinExecTransformer -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(57) ProjectExecTransformer -Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] - -(58) FileSourceScanExecTransformer parquet -Output [2]: [r_regionkey#X, r_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] -ReadSchema: struct - -(59) FilterExecTransformer -Input [2]: [r_regionkey#X, r_name#X] -Arguments: ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) - -(60) ProjectExecTransformer -Output [1]: [r_regionkey#X] -Input [2]: [r_regionkey#X, r_name#X] - -(61) WholeStageCodegenTransformer (X) -Input [1]: [r_regionkey#X] -Arguments: false - -(62) ColumnarBroadcastExchange -Input [1]: [r_regionkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(63) BroadcastQueryStage -Output [1]: [r_regionkey#X] -Arguments: X - -(64) InputAdapter -Input [1]: [r_regionkey#X] - -(65) InputIteratorTransformer -Input [1]: [r_regionkey#X] - -(66) BroadcastHashJoinExecTransformer -Left keys [1]: [n_regionkey#X] -Right keys [1]: [r_regionkey#X] -Join condition: None - -(67) ProjectExecTransformer -Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X] -Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] - -(68) FlushableHashAggregateExecTransformer -Input [3]: [o_year#X, volume#X, _pre_X#X] -Keys [1]: [o_year#X] -Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] -Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] - -(69) ProjectExecTransformer -Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] - -(70) WholeStageCodegenTransformer (X) -Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -Arguments: false - -(71) VeloxResizeBatches -Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] - -(72) ColumnarExchange -Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] - -(73) ShuffleQueryStage -Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -Arguments: X - -(74) InputAdapter -Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] - -(75) InputIteratorTransformer -Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] - -(76) RegularHashAggregateExecTransformer -Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -Keys [1]: [o_year#X] -Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] -Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] - -(77) ProjectExecTransformer -Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] -Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] - -(78) WholeStageCodegenTransformer (X) -Input [2]: [o_year#X, mkt_share#X] -Arguments: false - -(79) VeloxResizeBatches -Input [2]: [o_year#X, mkt_share#X] - -(80) ColumnarExchange -Input [2]: [o_year#X, mkt_share#X] -Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(81) ShuffleQueryStage -Output [2]: [o_year#X, mkt_share#X] -Arguments: X - -(82) InputAdapter -Input [2]: [o_year#X, mkt_share#X] - -(83) InputIteratorTransformer -Input [2]: [o_year#X, mkt_share#X] - -(84) SortExecTransformer -Input [2]: [o_year#X, mkt_share#X] -Arguments: [o_year#X ASC NULLS FIRST], true, 0 - -(85) WholeStageCodegenTransformer (X) -Input [2]: [o_year#X, mkt_share#X] -Arguments: false - -(86) VeloxColumnarToRow -Input [2]: [o_year#X, mkt_share#X] - -(87) Scan parquet -Output [2]: [p_partkey#X, p_type#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] -ReadSchema: struct - -(88) Filter -Input [2]: [p_partkey#X, p_type#X] -Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) - -(89) Project -Output [1]: [p_partkey#X] -Input [2]: [p_partkey#X, p_type#X] - -(90) BroadcastExchange -Input [1]: [p_partkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(91) Scan parquet -Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] -ReadSchema: struct - -(92) Filter -Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(93) BroadcastHashJoin -Left keys [1]: [p_partkey#X] -Right keys [1]: [l_partkey#X] -Join condition: None - -(94) Project -Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(95) Scan parquet -Output [2]: [s_suppkey#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] -ReadSchema: struct - -(96) Filter -Input [2]: [s_suppkey#X, s_nationkey#X] -Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(97) BroadcastExchange -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(98) BroadcastHashJoin -Left keys [1]: [l_suppkey#X] -Right keys [1]: [s_suppkey#X] -Join condition: None - -(99) Project -Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] - -(100) Scan parquet -Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] -ReadSchema: struct - -(101) Filter -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) - -(102) BroadcastExchange -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(103) BroadcastHashJoin -Left keys [1]: [l_orderkey#X] -Right keys [1]: [o_orderkey#X] -Join condition: None - -(104) Project -Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] - -(105) Scan parquet -Output [2]: [c_custkey#X, c_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] -ReadSchema: struct - -(106) Filter -Input [2]: [c_custkey#X, c_nationkey#X] -Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(107) BroadcastExchange -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(108) BroadcastHashJoin -Left keys [1]: [o_custkey#X] -Right keys [1]: [c_custkey#X] -Join condition: None - -(109) Project -Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] - -(110) Scan parquet -Output [2]: [n_nationkey#X, n_regionkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] -ReadSchema: struct - -(111) Filter -Input [2]: [n_nationkey#X, n_regionkey#X] -Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) - -(112) BroadcastExchange -Input [2]: [n_nationkey#X, n_regionkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(113) BroadcastHashJoin -Left keys [1]: [c_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(114) Project -Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] - -(115) Scan parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_nationkey)] -ReadSchema: struct - -(116) Filter -Input [2]: [n_nationkey#X, n_name#X] -Condition : isnotnull(n_nationkey#X) - -(117) BroadcastExchange -Input [2]: [n_nationkey#X, n_name#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(118) BroadcastHashJoin -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(119) Project -Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] - -(120) Scan parquet -Output [2]: [r_regionkey#X, r_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] -ReadSchema: struct - -(121) Filter -Input [2]: [r_regionkey#X, r_name#X] -Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) - -(122) Project -Output [1]: [r_regionkey#X] -Input [2]: [r_regionkey#X, r_name#X] - -(123) BroadcastExchange -Input [1]: [r_regionkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(124) BroadcastHashJoin -Left keys [1]: [n_regionkey#X] -Right keys [1]: [r_regionkey#X] -Join condition: None - -(125) Project -Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X] -Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] - -(126) HashAggregate -Input [3]: [o_year#X, volume#X, nation#X] -Keys [1]: [o_year#X] -Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] -Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] - -(127) Exchange -Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(128) HashAggregate -Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -Keys [1]: [o_year#X] -Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] -Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] - -(129) Exchange -Input [2]: [o_year#X, mkt_share#X] -Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(130) Sort -Input [2]: [o_year#X, mkt_share#X] -Arguments: [o_year#X ASC NULLS FIRST], true, 0 - -(131) AdaptiveSparkPlan -Output [2]: [o_year#X, mkt_share#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/9.txt deleted file mode 100644 index dfe4d032b2d..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/9.txt +++ /dev/null @@ -1,530 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (100) -+- == Final Plan == - VeloxColumnarToRow (66) - +- ^ SortExecTransformer (64) - +- ^ InputIteratorTransformer (63) - +- ShuffleQueryStage (61), Statistics(X) - +- ColumnarExchange (60) - +- VeloxResizeBatches (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- VeloxResizeBatches (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ FileSourceScanExecTransformer parquet (1) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ FileSourceScanExecTransformer parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ FileSourceScanExecTransformer parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ FileSourceScanExecTransformer parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ FileSourceScanExecTransformer parquet (31) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ FilterExecTransformer (41) - +- ^ FileSourceScanExecTransformer parquet (40) -+- == Initial Plan == - Sort (99) - +- Exchange (98) - +- HashAggregate (97) - +- Exchange (96) - +- HashAggregate (95) - +- Project (94) - +- BroadcastHashJoin Inner BuildRight (93) - :- Project (89) - : +- BroadcastHashJoin Inner BuildRight (88) - : :- Project (84) - : : +- BroadcastHashJoin Inner BuildRight (83) - : : :- Project (79) - : : : +- BroadcastHashJoin Inner BuildRight (78) - : : : :- Project (74) - : : : : +- BroadcastHashJoin Inner BuildLeft (73) - : : : : :- BroadcastExchange (70) - : : : : : +- Project (69) - : : : : : +- Filter (68) - : : : : : +- Scan parquet (67) - : : : : +- Filter (72) - : : : : +- Scan parquet (71) - : : : +- BroadcastExchange (77) - : : : +- Filter (76) - : : : +- Scan parquet (75) - : : +- BroadcastExchange (82) - : : +- Filter (81) - : : +- Scan parquet (80) - : +- BroadcastExchange (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (92) - +- Filter (91) - +- Scan parquet (90) - - -(1) FileSourceScanExecTransformer parquet -Output [2]: [p_partkey#X, p_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [2]: [p_partkey#X, p_name#X] -Arguments: ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) - -(3) ProjectExecTransformer -Output [1]: [p_partkey#X] -Input [2]: [p_partkey#X, p_name#X] - -(4) WholeStageCodegenTransformer (X) -Input [1]: [p_partkey#X] -Arguments: false - -(5) ColumnarBroadcastExchange -Input [1]: [p_partkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(6) BroadcastQueryStage -Output [1]: [p_partkey#X] -Arguments: X - -(7) InputAdapter -Input [1]: [p_partkey#X] - -(8) InputIteratorTransformer -Input [1]: [p_partkey#X] - -(9) FileSourceScanExecTransformer parquet -Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] -ReadSchema: struct - -(10) FilterExecTransformer -Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(11) BroadcastHashJoinExecTransformer -Left keys [1]: [p_partkey#X] -Right keys [1]: [l_partkey#X] -Join condition: None - -(12) ProjectExecTransformer -Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] - -(13) FileSourceScanExecTransformer parquet -Output [2]: [s_suppkey#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] -ReadSchema: struct - -(14) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(15) WholeStageCodegenTransformer (X) -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: false - -(16) ColumnarBroadcastExchange -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(17) BroadcastQueryStage -Output [2]: [s_suppkey#X, s_nationkey#X] -Arguments: X - -(18) InputAdapter -Input [2]: [s_suppkey#X, s_nationkey#X] - -(19) InputIteratorTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] - -(20) BroadcastHashJoinExecTransformer -Left keys [1]: [l_suppkey#X] -Right keys [1]: [s_suppkey#X] -Join condition: None - -(21) ProjectExecTransformer -Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] - -(22) FileSourceScanExecTransformer parquet -Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] -ReadSchema: struct - -(23) FilterExecTransformer -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -Arguments: (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) - -(24) WholeStageCodegenTransformer (X) -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -Arguments: false - -(25) ColumnarBroadcastExchange -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] - -(26) BroadcastQueryStage -Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -Arguments: X - -(27) InputAdapter -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] - -(28) InputIteratorTransformer -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] - -(29) BroadcastHashJoinExecTransformer -Left keys [2]: [l_suppkey#X, l_partkey#X] -Right keys [2]: [ps_suppkey#X, ps_partkey#X] -Join condition: None - -(30) ProjectExecTransformer -Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] - -(31) FileSourceScanExecTransformer parquet -Output [2]: [o_orderkey#X, o_orderdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderkey)] -ReadSchema: struct - -(32) FilterExecTransformer -Input [2]: [o_orderkey#X, o_orderdate#X] -Arguments: isnotnull(o_orderkey#X) - -(33) WholeStageCodegenTransformer (X) -Input [2]: [o_orderkey#X, o_orderdate#X] -Arguments: false - -(34) ColumnarBroadcastExchange -Input [2]: [o_orderkey#X, o_orderdate#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(35) BroadcastQueryStage -Output [2]: [o_orderkey#X, o_orderdate#X] -Arguments: X - -(36) InputAdapter -Input [2]: [o_orderkey#X, o_orderdate#X] - -(37) InputIteratorTransformer -Input [2]: [o_orderkey#X, o_orderdate#X] - -(38) BroadcastHashJoinExecTransformer -Left keys [1]: [l_orderkey#X] -Right keys [1]: [o_orderkey#X] -Join condition: None - -(39) ProjectExecTransformer -Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] - -(40) FileSourceScanExecTransformer parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_nationkey)] -ReadSchema: struct - -(41) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: isnotnull(n_nationkey#X) - -(42) WholeStageCodegenTransformer (X) -Input [2]: [n_nationkey#X, n_name#X] -Arguments: false - -(43) ColumnarBroadcastExchange -Input [2]: [n_nationkey#X, n_name#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(44) BroadcastQueryStage -Output [2]: [n_nationkey#X, n_name#X] -Arguments: X - -(45) InputAdapter -Input [2]: [n_nationkey#X, n_name#X] - -(46) InputIteratorTransformer -Input [2]: [n_nationkey#X, n_name#X] - -(47) BroadcastHashJoinExecTransformer -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(48) ProjectExecTransformer -Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] -Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] - -(49) FlushableHashAggregateExecTransformer -Input [3]: [nation#X, o_year#X, amount#X] -Keys [2]: [nation#X, o_year#X] -Functions [1]: [partial_sum(amount#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] - -(50) ProjectExecTransformer -Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] -Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] - -(51) WholeStageCodegenTransformer (X) -Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] -Arguments: false - -(52) VeloxResizeBatches -Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] - -(53) ColumnarExchange -Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] - -(54) ShuffleQueryStage -Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -Arguments: X - -(55) InputAdapter -Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] - -(56) InputIteratorTransformer -Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] - -(57) RegularHashAggregateExecTransformer -Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -Keys [2]: [nation#X, o_year#X] -Functions [1]: [sum(amount#X)] -Aggregate Attributes [1]: [sum(amount#X)#X] -Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] - -(58) WholeStageCodegenTransformer (X) -Input [3]: [nation#X, o_year#X, sum_profit#X] -Arguments: false - -(59) VeloxResizeBatches -Input [3]: [nation#X, o_year#X, sum_profit#X] - -(60) ColumnarExchange -Input [3]: [nation#X, o_year#X, sum_profit#X] -Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(61) ShuffleQueryStage -Output [3]: [nation#X, o_year#X, sum_profit#X] -Arguments: X - -(62) InputAdapter -Input [3]: [nation#X, o_year#X, sum_profit#X] - -(63) InputIteratorTransformer -Input [3]: [nation#X, o_year#X, sum_profit#X] - -(64) SortExecTransformer -Input [3]: [nation#X, o_year#X, sum_profit#X] -Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 - -(65) WholeStageCodegenTransformer (X) -Input [3]: [nation#X, o_year#X, sum_profit#X] -Arguments: false - -(66) VeloxColumnarToRow -Input [3]: [nation#X, o_year#X, sum_profit#X] - -(67) Scan parquet -Output [2]: [p_partkey#X, p_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] -ReadSchema: struct - -(68) Filter -Input [2]: [p_partkey#X, p_name#X] -Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) - -(69) Project -Output [1]: [p_partkey#X] -Input [2]: [p_partkey#X, p_name#X] - -(70) BroadcastExchange -Input [1]: [p_partkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] - -(71) Scan parquet -Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] -ReadSchema: struct - -(72) Filter -Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(73) BroadcastHashJoin -Left keys [1]: [p_partkey#X] -Right keys [1]: [l_partkey#X] -Join condition: None - -(74) Project -Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] - -(75) Scan parquet -Output [2]: [s_suppkey#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] -ReadSchema: struct - -(76) Filter -Input [2]: [s_suppkey#X, s_nationkey#X] -Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(77) BroadcastExchange -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(78) BroadcastHashJoin -Left keys [1]: [l_suppkey#X] -Right keys [1]: [s_suppkey#X] -Join condition: None - -(79) Project -Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] - -(80) Scan parquet -Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] -ReadSchema: struct - -(81) Filter -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) - -(82) BroadcastExchange -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] - -(83) BroadcastHashJoin -Left keys [2]: [l_suppkey#X, l_partkey#X] -Right keys [2]: [ps_suppkey#X, ps_partkey#X] -Join condition: None - -(84) Project -Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] - -(85) Scan parquet -Output [2]: [o_orderkey#X, o_orderdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderkey)] -ReadSchema: struct - -(86) Filter -Input [2]: [o_orderkey#X, o_orderdate#X] -Condition : isnotnull(o_orderkey#X) - -(87) BroadcastExchange -Input [2]: [o_orderkey#X, o_orderdate#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(88) BroadcastHashJoin -Left keys [1]: [l_orderkey#X] -Right keys [1]: [o_orderkey#X] -Join condition: None - -(89) Project -Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] - -(90) Scan parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_nationkey)] -ReadSchema: struct - -(91) Filter -Input [2]: [n_nationkey#X, n_name#X] -Condition : isnotnull(n_nationkey#X) - -(92) BroadcastExchange -Input [2]: [n_nationkey#X, n_name#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] - -(93) BroadcastHashJoin -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(94) Project -Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] -Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] - -(95) HashAggregate -Input [3]: [nation#X, o_year#X, amount#X] -Keys [2]: [nation#X, o_year#X] -Functions [1]: [partial_sum(amount#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] - -(96) Exchange -Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(97) HashAggregate -Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -Keys [2]: [nation#X, o_year#X] -Functions [1]: [sum(amount#X)] -Aggregate Attributes [1]: [sum(amount#X)#X] -Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] - -(98) Exchange -Input [3]: [nation#X, o_year#X, sum_profit#X] -Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(99) Sort -Input [3]: [nation#X, o_year#X, sum_profit#X] -Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 - -(100) AdaptiveSparkPlan -Output [3]: [nation#X, o_year#X, sum_profit#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/1.txt deleted file mode 100644 index 874b24f6c2a..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/1.txt +++ /dev/null @@ -1,160 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (30) -+- == Final Plan == - VeloxColumnarToRow (21) - +- ^ SortExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- VeloxResizeBatches (14) - +- ^ RegularHashAggregateExecTransformer (12) - +- ^ InputIteratorTransformer (11) - +- ShuffleQueryStage (9), Statistics(X) - +- ColumnarExchange (8) - +- VeloxResizeBatches (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ FileSourceScanExecTransformer parquet (1) -+- == Initial Plan == - Sort (29) - +- Exchange (28) - +- HashAggregate (27) - +- Exchange (26) - +- HashAggregate (25) - +- Project (24) - +- Filter (23) - +- Scan parquet (22) - - -(1) FileSourceScanExecTransformer parquet -Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -Arguments: (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) - -(3) ProjectExecTransformer -Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_returnflag#X, l_linestatus#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X, CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)) AS _pre_X#X] -Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] - -(4) FlushableHashAggregateExecTransformer -Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_returnflag#X, l_linestatus#X, _pre_X#X, _pre_X#X] -Keys [2]: [l_returnflag#X, l_linestatus#X] -Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(_pre_X#X), partial_sum(_pre_X#X), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] -Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] - -(5) ProjectExecTransformer -Output [18]: [hash(l_returnflag#X, l_linestatus#X, 42) AS hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] - -(6) WholeStageCodegenTransformer (X) -Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -Arguments: false - -(7) VeloxResizeBatches -Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] - -(8) ColumnarExchange -Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [shuffle_writer_type=hash] - -(9) ShuffleQueryStage -Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -Arguments: X - -(10) InputAdapter -Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] - -(11) InputIteratorTransformer -Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] - -(12) RegularHashAggregateExecTransformer -Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -Keys [2]: [l_returnflag#X, l_linestatus#X] -Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] -Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] -Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] - -(13) WholeStageCodegenTransformer (X) -Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -Arguments: false - -(14) VeloxResizeBatches -Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] - -(15) ColumnarExchange -Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(16) ShuffleQueryStage -Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -Arguments: X - -(17) InputAdapter -Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] - -(18) InputIteratorTransformer -Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] - -(19) SortExecTransformer -Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 - -(20) WholeStageCodegenTransformer (X) -Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -Arguments: false - -(21) VeloxColumnarToRow -Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] - -(22) Scan parquet -Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] -ReadSchema: struct - -(23) Filter -Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) - -(24) Project -Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] -Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] - -(25) HashAggregate -Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] -Keys [2]: [l_returnflag#X, l_linestatus#X] -Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] -Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] - -(26) Exchange -Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(27) HashAggregate -Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -Keys [2]: [l_returnflag#X, l_linestatus#X] -Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] -Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] -Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] - -(28) Exchange -Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(29) Sort -Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 - -(30) AdaptiveSparkPlan -Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt deleted file mode 100644 index d58682a0c48..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt +++ /dev/null @@ -1,509 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (100) -+- == Final Plan == - VeloxColumnarToRow (67) - +- TakeOrderedAndProjectExecTransformer (66) - +- ^ ProjectExecTransformer (64) - +- ^ RegularHashAggregateExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ShuffleQueryStage (60), Statistics(X) - +- ColumnarExchange (59) - +- VeloxResizeBatches (58) - +- ^ ProjectExecTransformer (56) - +- ^ FlushableHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - :- ^ InputIteratorTransformer (43) - : +- ShuffleQueryStage (41), Statistics(X) - : +- ColumnarExchange (40) - : +- VeloxResizeBatches (39) - : +- ^ ProjectExecTransformer (37) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) - : :- ^ InputIteratorTransformer (26) - : : +- ShuffleQueryStage (24), Statistics(X) - : : +- ColumnarExchange (23) - : : +- VeloxResizeBatches (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - : : :- ^ InputIteratorTransformer (9) - : : : +- ShuffleQueryStage (7), Statistics(X) - : : : +- ColumnarExchange (6) - : : : +- VeloxResizeBatches (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ FileSourceScanExecTransformer parquet (1) - : : +- ^ InputIteratorTransformer (18) - : : +- ShuffleQueryStage (16), Statistics(X) - : : +- ColumnarExchange (15) - : : +- VeloxResizeBatches (14) - : : +- ^ ProjectExecTransformer (12) - : : +- ^ FilterExecTransformer (11) - : : +- ^ FileSourceScanExecTransformer parquet (10) - : +- ^ InputIteratorTransformer (35) - : +- ShuffleQueryStage (33), Statistics(X) - : +- ColumnarExchange (32) - : +- VeloxResizeBatches (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ FilterExecTransformer (28) - : +- ^ FileSourceScanExecTransformer parquet (27) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- VeloxResizeBatches (48) - +- ^ ProjectExecTransformer (46) - +- ^ FilterExecTransformer (45) - +- ^ FileSourceScanExecTransformer parquet (44) -+- == Initial Plan == - TakeOrderedAndProject (99) - +- HashAggregate (98) - +- Exchange (97) - +- HashAggregate (96) - +- Project (95) - +- SortMergeJoin Inner (94) - :- Sort (89) - : +- Exchange (88) - : +- Project (87) - : +- SortMergeJoin Inner (86) - : :- Sort (80) - : : +- Exchange (79) - : : +- Project (78) - : : +- SortMergeJoin Inner (77) - : : :- Sort (71) - : : : +- Exchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- Sort (76) - : : +- Exchange (75) - : : +- Project (74) - : : +- Filter (73) - : : +- Scan parquet (72) - : +- Sort (85) - : +- Exchange (84) - : +- Project (83) - : +- Filter (82) - : +- Scan parquet (81) - +- Sort (93) - +- Exchange (92) - +- Filter (91) - +- Scan parquet (90) - - -(1) FileSourceScanExecTransformer parquet -Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(3) ProjectExecTransformer -Output [8]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] - -(4) WholeStageCodegenTransformer (X) -Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -Arguments: false - -(5) VeloxResizeBatches -Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] - -(6) ColumnarExchange -Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [shuffle_writer_type=hash] - -(7) ShuffleQueryStage -Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -Arguments: X - -(8) InputAdapter -Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] - -(9) InputIteratorTransformer -Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] - -(10) FileSourceScanExecTransformer parquet -Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] -ReadSchema: struct - -(11) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(12) ProjectExecTransformer -Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] - -(13) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] -Arguments: false - -(14) VeloxResizeBatches -Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] - -(15) ColumnarExchange -Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] -Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(16) ShuffleQueryStage -Output [2]: [o_orderkey#X, o_custkey#X] -Arguments: X - -(17) InputAdapter -Input [2]: [o_orderkey#X, o_custkey#X] - -(18) InputIteratorTransformer -Input [2]: [o_orderkey#X, o_custkey#X] - -(19) ShuffledHashJoinExecTransformer -Left keys [1]: [c_custkey#X] -Right keys [1]: [o_custkey#X] -Join condition: None - -(20) ProjectExecTransformer -Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] - -(21) WholeStageCodegenTransformer (X) -Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -Arguments: false - -(22) VeloxResizeBatches -Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] - -(23) ColumnarExchange -Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(24) ShuffleQueryStage -Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -Arguments: X - -(25) InputAdapter -Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] - -(26) InputIteratorTransformer -Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] - -(27) FileSourceScanExecTransformer parquet -Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] -ReadSchema: struct - -(28) FilterExecTransformer -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -Arguments: ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) - -(29) ProjectExecTransformer -Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] - -(30) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -Arguments: false - -(31) VeloxResizeBatches -Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] - -(32) ColumnarExchange -Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] - -(33) ShuffleQueryStage -Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -Arguments: X - -(34) InputAdapter -Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] - -(35) InputIteratorTransformer -Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] - -(36) ShuffledHashJoinExecTransformer -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(37) ProjectExecTransformer -Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] - -(38) WholeStageCodegenTransformer (X) -Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -Arguments: false - -(39) VeloxResizeBatches -Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] - -(40) ColumnarExchange -Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] - -(41) ShuffleQueryStage -Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -Arguments: X - -(42) InputAdapter -Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] - -(43) InputIteratorTransformer -Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] - -(44) FileSourceScanExecTransformer parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_nationkey)] -ReadSchema: struct - -(45) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: isnotnull(n_nationkey#X) - -(46) ProjectExecTransformer -Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] -Input [2]: [n_nationkey#X, n_name#X] - -(47) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] -Arguments: false - -(48) VeloxResizeBatches -Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] - -(49) ColumnarExchange -Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] -Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [shuffle_writer_type=hash] - -(50) ShuffleQueryStage -Output [2]: [n_nationkey#X, n_name#X] -Arguments: X - -(51) InputAdapter -Input [2]: [n_nationkey#X, n_name#X] - -(52) InputIteratorTransformer -Input [2]: [n_nationkey#X, n_name#X] - -(53) ShuffledHashJoinExecTransformer -Left keys [1]: [c_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(54) ProjectExecTransformer -Output [8]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] -Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] - -(55) FlushableHashAggregateExecTransformer -Input [8]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, n_name#X, _pre_X#X] -Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] -Functions [1]: [partial_sum(_pre_X#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] - -(56) ProjectExecTransformer -Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] - -(57) WholeStageCodegenTransformer (X) -Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -Arguments: false - -(58) VeloxResizeBatches -Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] - -(59) ColumnarExchange -Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] - -(60) ShuffleQueryStage -Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -Arguments: X - -(61) InputAdapter -Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] - -(62) InputIteratorTransformer -Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] - -(63) RegularHashAggregateExecTransformer -Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] -Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] - -(64) ProjectExecTransformer -Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] - -(65) WholeStageCodegenTransformer (X) -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: false - -(66) TakeOrderedAndProjectExecTransformer -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 - -(67) VeloxColumnarToRow -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] - -(68) Scan parquet -Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] -ReadSchema: struct - -(69) Filter -Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(70) Exchange -Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(71) Sort -Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(72) Scan parquet -Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] -ReadSchema: struct - -(73) Filter -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(74) Project -Output [2]: [o_orderkey#X, o_custkey#X] -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] - -(75) Exchange -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(76) Sort -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(77) SortMergeJoin -Left keys [1]: [c_custkey#X] -Right keys [1]: [o_custkey#X] -Join condition: None - -(78) Project -Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] - -(79) Exchange -Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(80) Sort -Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(81) Scan parquet -Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] -ReadSchema: struct - -(82) Filter -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) - -(83) Project -Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] - -(84) Exchange -Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(85) Sort -Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(86) SortMergeJoin -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(87) Project -Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] - -(88) Exchange -Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(89) Sort -Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 - -(90) Scan parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_nationkey)] -ReadSchema: struct - -(91) Filter -Input [2]: [n_nationkey#X, n_name#X] -Condition : isnotnull(n_nationkey#X) - -(92) Exchange -Input [2]: [n_nationkey#X, n_name#X] -Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(93) Sort -Input [2]: [n_nationkey#X, n_name#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 - -(94) SortMergeJoin -Left keys [1]: [c_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(95) Project -Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] -Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] - -(96) HashAggregate -Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] -Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] -Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] - -(97) Exchange -Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(98) HashAggregate -Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] -Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] - -(99) TakeOrderedAndProject -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] - -(100) AdaptiveSparkPlan -Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt deleted file mode 100644 index 76b23b31264..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt +++ /dev/null @@ -1,693 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (82) -+- == Final Plan == - VeloxColumnarToRow (56) - +- ^ SortExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ShuffleQueryStage (51), Statistics(X) - +- ColumnarExchange (50) - +- VeloxResizeBatches (49) - +- ^ FilterExecTransformer (47) - +- ^ RegularHashAggregateExecTransformer (46) - +- ^ InputIteratorTransformer (45) - +- ShuffleQueryStage (43), Statistics(X) - +- ColumnarExchange (42) - +- VeloxResizeBatches (41) - +- ^ ProjectExecTransformer (39) - +- ^ FlushableHashAggregateExecTransformer (38) - +- ^ ProjectExecTransformer (37) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - :- ^ InputIteratorTransformer (26) - : +- ShuffleQueryStage (24), Statistics(X) - : +- ColumnarExchange (23) - : +- VeloxResizeBatches (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - : :- ^ InputIteratorTransformer (9) - : : +- ShuffleQueryStage (7), Statistics(X) - : : +- ColumnarExchange (6) - : : +- VeloxResizeBatches (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ FileSourceScanExecTransformer parquet (1) - : +- ^ InputIteratorTransformer (18) - : +- ShuffleQueryStage (16), Statistics(X) - : +- ColumnarExchange (15) - : +- VeloxResizeBatches (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FilterExecTransformer (11) - : +- ^ FileSourceScanExecTransformer parquet (10) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- VeloxResizeBatches (31) - +- ^ ProjectExecTransformer (29) - +- ^ FilterExecTransformer (28) - +- ^ FileSourceScanExecTransformer parquet (27) -+- == Initial Plan == - Sort (81) - +- Exchange (80) - +- Filter (79) - +- HashAggregate (78) - +- Exchange (77) - +- HashAggregate (76) - +- Project (75) - +- SortMergeJoin Inner (74) - :- Sort (68) - : +- Exchange (67) - : +- Project (66) - : +- SortMergeJoin Inner (65) - : :- Sort (60) - : : +- Exchange (59) - : : +- Filter (58) - : : +- Scan parquet (57) - : +- Sort (64) - : +- Exchange (63) - : +- Filter (62) - : +- Scan parquet (61) - +- Sort (73) - +- Exchange (72) - +- Project (71) - +- Filter (70) - +- Scan parquet (69) - - -(1) FileSourceScanExecTransformer parquet -Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(ps_suppkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: isnotnull(ps_suppkey#X) - -(3) ProjectExecTransformer -Output [5]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] - -(4) WholeStageCodegenTransformer (X) -Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: false - -(5) VeloxResizeBatches -Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] - -(6) ColumnarExchange -Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [shuffle_writer_type=hash] - -(7) ShuffleQueryStage -Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: X - -(8) InputAdapter -Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] - -(9) InputIteratorTransformer -Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] - -(10) FileSourceScanExecTransformer parquet -Output [2]: [s_suppkey#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] -ReadSchema: struct - -(11) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(12) ProjectExecTransformer -Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] -Input [2]: [s_suppkey#X, s_nationkey#X] - -(13) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] -Arguments: false - -(14) VeloxResizeBatches -Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] - -(15) ColumnarExchange -Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] -Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(16) ShuffleQueryStage -Output [2]: [s_suppkey#X, s_nationkey#X] -Arguments: X - -(17) InputAdapter -Input [2]: [s_suppkey#X, s_nationkey#X] - -(18) InputIteratorTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] - -(19) ShuffledHashJoinExecTransformer -Left keys [1]: [ps_suppkey#X] -Right keys [1]: [s_suppkey#X] -Join condition: None - -(20) ProjectExecTransformer -Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] - -(21) WholeStageCodegenTransformer (X) -Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -Arguments: false - -(22) VeloxResizeBatches -Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] - -(23) ColumnarExchange -Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(24) ShuffleQueryStage -Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -Arguments: X - -(25) InputAdapter -Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] - -(26) InputIteratorTransformer -Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] - -(27) FileSourceScanExecTransformer parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] -ReadSchema: struct - -(28) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) - -(29) ProjectExecTransformer -Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] -Input [2]: [n_nationkey#X, n_name#X] - -(30) WholeStageCodegenTransformer (X) -Input [2]: [hash_partition_key#X, n_nationkey#X] -Arguments: false - -(31) VeloxResizeBatches -Input [2]: [hash_partition_key#X, n_nationkey#X] - -(32) ColumnarExchange -Input [2]: [hash_partition_key#X, n_nationkey#X] -Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(33) ShuffleQueryStage -Output [1]: [n_nationkey#X] -Arguments: X - -(34) InputAdapter -Input [1]: [n_nationkey#X] - -(35) InputIteratorTransformer -Input [1]: [n_nationkey#X] - -(36) ShuffledHashJoinExecTransformer -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(37) ProjectExecTransformer -Output [2]: [ps_partkey#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] -Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] - -(38) FlushableHashAggregateExecTransformer -Input [2]: [ps_partkey#X, _pre_X#X] -Keys [1]: [ps_partkey#X] -Functions [1]: [partial_sum(_pre_X#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [3]: [ps_partkey#X, sum#X, isEmpty#X] - -(39) ProjectExecTransformer -Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] -Input [3]: [ps_partkey#X, sum#X, isEmpty#X] - -(40) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] -Arguments: false - -(41) VeloxResizeBatches -Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] - -(42) ColumnarExchange -Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] - -(43) ShuffleQueryStage -Output [3]: [ps_partkey#X, sum#X, isEmpty#X] -Arguments: X - -(44) InputAdapter -Input [3]: [ps_partkey#X, sum#X, isEmpty#X] - -(45) InputIteratorTransformer -Input [3]: [ps_partkey#X, sum#X, isEmpty#X] - -(46) RegularHashAggregateExecTransformer -Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -Keys [1]: [ps_partkey#X] -Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] - -(47) FilterExecTransformer -Input [2]: [ps_partkey#X, value#X] -Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) - -(48) WholeStageCodegenTransformer (X) -Input [2]: [ps_partkey#X, value#X] -Arguments: false - -(49) VeloxResizeBatches -Input [2]: [ps_partkey#X, value#X] - -(50) ColumnarExchange -Input [2]: [ps_partkey#X, value#X] -Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(51) ShuffleQueryStage -Output [2]: [ps_partkey#X, value#X] -Arguments: X - -(52) InputAdapter -Input [2]: [ps_partkey#X, value#X] - -(53) InputIteratorTransformer -Input [2]: [ps_partkey#X, value#X] - -(54) SortExecTransformer -Input [2]: [ps_partkey#X, value#X] -Arguments: [value#X DESC NULLS LAST], true, 0 - -(55) WholeStageCodegenTransformer (X) -Input [2]: [ps_partkey#X, value#X] -Arguments: false - -(56) VeloxColumnarToRow -Input [2]: [ps_partkey#X, value#X] - -(57) Scan parquet -Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(ps_suppkey)] -ReadSchema: struct - -(58) Filter -Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Condition : isnotnull(ps_suppkey#X) - -(59) Exchange -Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(60) Sort -Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 - -(61) Scan parquet -Output [2]: [s_suppkey#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] -ReadSchema: struct - -(62) Filter -Input [2]: [s_suppkey#X, s_nationkey#X] -Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(63) Exchange -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(64) Sort -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(65) SortMergeJoin -Left keys [1]: [ps_suppkey#X] -Right keys [1]: [s_suppkey#X] -Join condition: None - -(66) Project -Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] - -(67) Exchange -Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(68) Sort -Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 - -(69) Scan parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] -ReadSchema: struct - -(70) Filter -Input [2]: [n_nationkey#X, n_name#X] -Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) - -(71) Project -Output [1]: [n_nationkey#X] -Input [2]: [n_nationkey#X, n_name#X] - -(72) Exchange -Input [1]: [n_nationkey#X] -Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(73) Sort -Input [1]: [n_nationkey#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 - -(74) SortMergeJoin -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(75) Project -Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] -Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] - -(76) HashAggregate -Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] -Keys [1]: [ps_partkey#X] -Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [3]: [ps_partkey#X, sum#X, isEmpty#X] - -(77) Exchange -Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(78) HashAggregate -Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -Keys [1]: [ps_partkey#X] -Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] - -(79) Filter -Input [2]: [ps_partkey#X, value#X] -Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) - -(80) Exchange -Input [2]: [ps_partkey#X, value#X] -Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(81) Sort -Input [2]: [ps_partkey#X, value#X] -Arguments: [value#X DESC NULLS LAST], true, 0 - -(82) AdaptiveSparkPlan -Output [2]: [ps_partkey#X, value#X] -Arguments: isFinalPlan=true - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (135) -+- == Final Plan == - VeloxColumnarToRow (113) - +- ^ ProjectExecTransformer (111) - +- ^ RegularHashAggregateExecTransformer (110) - +- ^ ProjectExecTransformer (109) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (108) - :- ^ InputIteratorTransformer (103) - : +- ShuffleQueryStage (101), Statistics(X) - : +- ColumnarExchange (100) - : +- VeloxResizeBatches (99) - : +- ^ ProjectExecTransformer (97) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (96) - : :- ^ InputIteratorTransformer (91) - : : +- ShuffleQueryStage (89), Statistics(X) - : : +- ColumnarExchange (88) - : : +- VeloxResizeBatches (87) - : : +- ^ ProjectExecTransformer (85) - : : +- ^ FilterExecTransformer (84) - : : +- ^ FileSourceScanExecTransformer parquet (83) - : +- ^ InputIteratorTransformer (95) - : +- ShuffleQueryStage (93), Statistics(X) - : +- ReusedExchange (92) - +- ^ InputIteratorTransformer (107) - +- ShuffleQueryStage (105), Statistics(X) - +- ReusedExchange (104) -+- == Initial Plan == - HashAggregate (134) - +- HashAggregate (133) - +- Project (132) - +- SortMergeJoin Inner (131) - :- Sort (125) - : +- Exchange (124) - : +- Project (123) - : +- SortMergeJoin Inner (122) - : :- Sort (117) - : : +- Exchange (116) - : : +- Filter (115) - : : +- Scan parquet (114) - : +- Sort (121) - : +- Exchange (120) - : +- Filter (119) - : +- Scan parquet (118) - +- Sort (130) - +- Exchange (129) - +- Project (128) - +- Filter (127) - +- Scan parquet (126) - - -(83) FileSourceScanExecTransformer parquet -Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(ps_suppkey)] -ReadSchema: struct - -(84) FilterExecTransformer -Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: isnotnull(ps_suppkey#X) - -(85) ProjectExecTransformer -Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] - -(86) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: false - -(87) VeloxResizeBatches -Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] - -(88) ColumnarExchange -Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [shuffle_writer_type=hash] - -(89) ShuffleQueryStage -Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: X - -(90) InputAdapter -Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] - -(91) InputIteratorTransformer -Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] - -(92) ReusedExchange [Reuses operator id: 15] -Output [2]: [s_suppkey#X, s_nationkey#X] - -(93) ShuffleQueryStage -Output [2]: [s_suppkey#X, s_nationkey#X] -Arguments: X - -(94) InputAdapter -Input [2]: [s_suppkey#X, s_nationkey#X] - -(95) InputIteratorTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] - -(96) ShuffledHashJoinExecTransformer -Left keys [1]: [ps_suppkey#X] -Right keys [1]: [s_suppkey#X] -Join condition: None - -(97) ProjectExecTransformer -Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] - -(98) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -Arguments: false - -(99) VeloxResizeBatches -Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] - -(100) ColumnarExchange -Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(101) ShuffleQueryStage -Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -Arguments: X - -(102) InputAdapter -Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] - -(103) InputIteratorTransformer -Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] - -(104) ReusedExchange [Reuses operator id: 32] -Output [1]: [n_nationkey#X] - -(105) ShuffleQueryStage -Output [1]: [n_nationkey#X] -Arguments: X - -(106) InputAdapter -Input [1]: [n_nationkey#X] - -(107) InputIteratorTransformer -Input [1]: [n_nationkey#X] - -(108) ShuffledHashJoinExecTransformer -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(109) ProjectExecTransformer -Output [1]: [CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] -Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] - -(110) RegularHashAggregateExecTransformer -Input [1]: [_pre_X#X] -Keys: [] -Functions [1]: [sum(_pre_X#X)] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -Results [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] - -(111) ProjectExecTransformer -Output [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -Input [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] - -(112) WholeStageCodegenTransformer (X) -Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -Arguments: false - -(113) VeloxColumnarToRow -Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] - -(114) Scan parquet -Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(ps_suppkey)] -ReadSchema: struct - -(115) Filter -Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Condition : isnotnull(ps_suppkey#X) - -(116) Exchange -Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(117) Sort -Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 - -(118) Scan parquet -Output [2]: [s_suppkey#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] -ReadSchema: struct - -(119) Filter -Input [2]: [s_suppkey#X, s_nationkey#X] -Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(120) Exchange -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(121) Sort -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(122) SortMergeJoin -Left keys [1]: [ps_suppkey#X] -Right keys [1]: [s_suppkey#X] -Join condition: None - -(123) Project -Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] - -(124) Exchange -Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(125) Sort -Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 - -(126) Scan parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] -ReadSchema: struct - -(127) Filter -Input [2]: [n_nationkey#X, n_name#X] -Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) - -(128) Project -Output [1]: [n_nationkey#X] -Input [2]: [n_nationkey#X, n_name#X] - -(129) Exchange -Input [1]: [n_nationkey#X] -Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(130) Sort -Input [1]: [n_nationkey#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 - -(131) SortMergeJoin -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(132) Project -Output [2]: [ps_availqty#X, ps_supplycost#X] -Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] - -(133) HashAggregate -Input [2]: [ps_availqty#X, ps_supplycost#X] -Keys: [] -Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [2]: [sum#X, isEmpty#X] - -(134) HashAggregate -Input [2]: [sum#X, isEmpty#X] -Keys: [] -Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -Results [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] - -(135) AdaptiveSparkPlan -Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt deleted file mode 100644 index cfc74b18545..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt +++ /dev/null @@ -1,283 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (55) -+- == Final Plan == - VeloxColumnarToRow (38) - +- ^ SortExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- VeloxResizeBatches (31) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- VeloxResizeBatches (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7), Statistics(X) - : +- ColumnarExchange (6) - : +- VeloxResizeBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ FileSourceScanExecTransformer parquet (1) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- VeloxResizeBatches (14) - +- ^ ProjectExecTransformer (12) - +- ^ FilterExecTransformer (11) - +- ^ FileSourceScanExecTransformer parquet (10) -+- == Initial Plan == - Sort (54) - +- Exchange (53) - +- HashAggregate (52) - +- Exchange (51) - +- HashAggregate (50) - +- Project (49) - +- SortMergeJoin Inner (48) - :- Sort (42) - : +- Exchange (41) - : +- Filter (40) - : +- Scan parquet (39) - +- Sort (47) - +- Exchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) - - -(1) FileSourceScanExecTransformer parquet -Output [2]: [o_orderkey#X, o_orderpriority#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [2]: [o_orderkey#X, o_orderpriority#X] -Arguments: isnotnull(o_orderkey#X) - -(3) ProjectExecTransformer -Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderpriority#X] -Input [2]: [o_orderkey#X, o_orderpriority#X] - -(4) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] -Arguments: false - -(5) VeloxResizeBatches -Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] - -(6) ColumnarExchange -Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] -Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [shuffle_writer_type=hash] - -(7) ShuffleQueryStage -Output [2]: [o_orderkey#X, o_orderpriority#X] -Arguments: X - -(8) InputAdapter -Input [2]: [o_orderkey#X, o_orderpriority#X] - -(9) InputIteratorTransformer -Input [2]: [o_orderkey#X, o_orderpriority#X] - -(10) FileSourceScanExecTransformer parquet -Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] -ReadSchema: struct - -(11) FilterExecTransformer -Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -Arguments: ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) - -(12) ProjectExecTransformer -Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] -Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] - -(13) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] -Arguments: false - -(14) VeloxResizeBatches -Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] - -(15) ColumnarExchange -Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [shuffle_writer_type=hash] - -(16) ShuffleQueryStage -Output [2]: [l_orderkey#X, l_shipmode#X] -Arguments: X - -(17) InputAdapter -Input [2]: [l_orderkey#X, l_shipmode#X] - -(18) InputIteratorTransformer -Input [2]: [l_orderkey#X, l_shipmode#X] - -(19) ShuffledHashJoinExecTransformer -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(20) ProjectExecTransformer -Output [3]: [l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] -Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] - -(21) FlushableHashAggregateExecTransformer -Input [3]: [l_shipmode#X, _pre_X#X, _pre_X#X] -Keys [1]: [l_shipmode#X] -Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] -Aggregate Attributes [2]: [sum#X, sum#X] -Results [3]: [l_shipmode#X, sum#X, sum#X] - -(22) ProjectExecTransformer -Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] -Input [3]: [l_shipmode#X, sum#X, sum#X] - -(23) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] -Arguments: false - -(24) VeloxResizeBatches -Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] - -(25) ColumnarExchange -Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] -Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [shuffle_writer_type=hash] - -(26) ShuffleQueryStage -Output [3]: [l_shipmode#X, sum#X, sum#X] -Arguments: X - -(27) InputAdapter -Input [3]: [l_shipmode#X, sum#X, sum#X] - -(28) InputIteratorTransformer -Input [3]: [l_shipmode#X, sum#X, sum#X] - -(29) RegularHashAggregateExecTransformer -Input [3]: [l_shipmode#X, sum#X, sum#X] -Keys [1]: [l_shipmode#X] -Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] -Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] -Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] - -(30) WholeStageCodegenTransformer (X) -Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -Arguments: false - -(31) VeloxResizeBatches -Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] - -(32) ColumnarExchange -Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(33) ShuffleQueryStage -Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -Arguments: X - -(34) InputAdapter -Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] - -(35) InputIteratorTransformer -Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] - -(36) SortExecTransformer -Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 - -(37) WholeStageCodegenTransformer (X) -Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -Arguments: false - -(38) VeloxColumnarToRow -Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] - -(39) Scan parquet -Output [2]: [o_orderkey#X, o_orderpriority#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderkey)] -ReadSchema: struct - -(40) Filter -Input [2]: [o_orderkey#X, o_orderpriority#X] -Condition : isnotnull(o_orderkey#X) - -(41) Exchange -Input [2]: [o_orderkey#X, o_orderpriority#X] -Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(42) Sort -Input [2]: [o_orderkey#X, o_orderpriority#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(43) Scan parquet -Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] -ReadSchema: struct - -(44) Filter -Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) - -(45) Project -Output [2]: [l_orderkey#X, l_shipmode#X] -Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] - -(46) Exchange -Input [2]: [l_orderkey#X, l_shipmode#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(47) Sort -Input [2]: [l_orderkey#X, l_shipmode#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(48) SortMergeJoin -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(49) Project -Output [2]: [o_orderpriority#X, l_shipmode#X] -Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] - -(50) HashAggregate -Input [2]: [o_orderpriority#X, l_shipmode#X] -Keys [1]: [l_shipmode#X] -Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] -Aggregate Attributes [2]: [sum#X, sum#X] -Results [3]: [l_shipmode#X, sum#X, sum#X] - -(51) Exchange -Input [3]: [l_shipmode#X, sum#X, sum#X] -Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(52) HashAggregate -Input [3]: [l_shipmode#X, sum#X, sum#X] -Keys [1]: [l_shipmode#X] -Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] -Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] -Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] - -(53) Exchange -Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(54) Sort -Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 - -(55) AdaptiveSparkPlan -Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt deleted file mode 100644 index 7797caf10fc..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt +++ /dev/null @@ -1,300 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (57) -+- == Final Plan == - VeloxColumnarToRow (39) - +- ^ SortExecTransformer (37) - +- ^ InputIteratorTransformer (36) - +- ShuffleQueryStage (34), Statistics(X) - +- ColumnarExchange (33) - +- VeloxResizeBatches (32) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- VeloxResizeBatches (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- VeloxResizeBatches (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ FileSourceScanExecTransformer parquet (1) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- VeloxResizeBatches (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ FileSourceScanExecTransformer parquet (9) -+- == Initial Plan == - Sort (56) - +- Exchange (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- Project (49) - +- SortMergeJoin LeftOuter (48) - :- Sort (42) - : +- Exchange (41) - : +- Scan parquet (40) - +- Sort (47) - +- Exchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) - - -(1) FileSourceScanExecTransformer parquet -Output [1]: [c_custkey#X] -Batched: true -Location: InMemoryFileIndex [*] -ReadSchema: struct - -(2) ProjectExecTransformer -Output [2]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X] -Input [1]: [c_custkey#X] - -(3) WholeStageCodegenTransformer (X) -Input [2]: [hash_partition_key#X, c_custkey#X] -Arguments: false - -(4) VeloxResizeBatches -Input [2]: [hash_partition_key#X, c_custkey#X] - -(5) ColumnarExchange -Input [2]: [hash_partition_key#X, c_custkey#X] -Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(6) ShuffleQueryStage -Output [1]: [c_custkey#X] -Arguments: X - -(7) InputAdapter -Input [1]: [c_custkey#X] - -(8) InputIteratorTransformer -Input [1]: [c_custkey#X] - -(9) FileSourceScanExecTransformer parquet -Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] -ReadSchema: struct - -(10) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -Arguments: ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) - -(11) ProjectExecTransformer -Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] -Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] - -(12) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] -Arguments: false - -(13) VeloxResizeBatches -Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] - -(14) ColumnarExchange -Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] -Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(15) ShuffleQueryStage -Output [2]: [o_orderkey#X, o_custkey#X] -Arguments: X - -(16) InputAdapter -Input [2]: [o_orderkey#X, o_custkey#X] - -(17) InputIteratorTransformer -Input [2]: [o_orderkey#X, o_custkey#X] - -(18) ShuffledHashJoinExecTransformer -Left keys [1]: [c_custkey#X] -Right keys [1]: [o_custkey#X] -Join condition: None - -(19) ProjectExecTransformer -Output [2]: [c_custkey#X, o_orderkey#X] -Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] - -(20) RegularHashAggregateExecTransformer -Input [2]: [c_custkey#X, o_orderkey#X] -Keys [1]: [c_custkey#X] -Functions [1]: [count(o_orderkey#X)] -Aggregate Attributes [1]: [count(o_orderkey#X)#X] -Results [2]: [c_custkey#X, count(o_orderkey#X)#X] - -(21) ProjectExecTransformer -Output [1]: [count(o_orderkey#X)#X AS c_count#X] -Input [2]: [c_custkey#X, count(o_orderkey#X)#X] - -(22) FlushableHashAggregateExecTransformer -Input [1]: [c_count#X] -Keys [1]: [c_count#X] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#X] -Results [2]: [c_count#X, count#X] - -(23) ProjectExecTransformer -Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] -Input [2]: [c_count#X, count#X] - -(24) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, c_count#X, count#X] -Arguments: false - -(25) VeloxResizeBatches -Input [3]: [hash_partition_key#X, c_count#X, count#X] - -(26) ColumnarExchange -Input [3]: [hash_partition_key#X, c_count#X, count#X] -Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [shuffle_writer_type=hash] - -(27) ShuffleQueryStage -Output [2]: [c_count#X, count#X] -Arguments: X - -(28) InputAdapter -Input [2]: [c_count#X, count#X] - -(29) InputIteratorTransformer -Input [2]: [c_count#X, count#X] - -(30) RegularHashAggregateExecTransformer -Input [2]: [c_count#X, count#X] -Keys [1]: [c_count#X] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#X] -Results [2]: [c_count#X, count(1)#X AS custdist#X] - -(31) WholeStageCodegenTransformer (X) -Input [2]: [c_count#X, custdist#X] -Arguments: false - -(32) VeloxResizeBatches -Input [2]: [c_count#X, custdist#X] - -(33) ColumnarExchange -Input [2]: [c_count#X, custdist#X] -Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(34) ShuffleQueryStage -Output [2]: [c_count#X, custdist#X] -Arguments: X - -(35) InputAdapter -Input [2]: [c_count#X, custdist#X] - -(36) InputIteratorTransformer -Input [2]: [c_count#X, custdist#X] - -(37) SortExecTransformer -Input [2]: [c_count#X, custdist#X] -Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 - -(38) WholeStageCodegenTransformer (X) -Input [2]: [c_count#X, custdist#X] -Arguments: false - -(39) VeloxColumnarToRow -Input [2]: [c_count#X, custdist#X] - -(40) Scan parquet -Output [1]: [c_custkey#X] -Batched: true -Location: InMemoryFileIndex [*] -ReadSchema: struct - -(41) Exchange -Input [1]: [c_custkey#X] -Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(42) Sort -Input [1]: [c_custkey#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(43) Scan parquet -Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] -ReadSchema: struct - -(44) Filter -Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) - -(45) Project -Output [2]: [o_orderkey#X, o_custkey#X] -Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] - -(46) Exchange -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(47) Sort -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(48) SortMergeJoin -Left keys [1]: [c_custkey#X] -Right keys [1]: [o_custkey#X] -Join condition: None - -(49) Project -Output [2]: [c_custkey#X, o_orderkey#X] -Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] - -(50) HashAggregate -Input [2]: [c_custkey#X, o_orderkey#X] -Keys [1]: [c_custkey#X] -Functions [1]: [partial_count(o_orderkey#X)] -Aggregate Attributes [1]: [count#X] -Results [2]: [c_custkey#X, count#X] - -(51) HashAggregate -Input [2]: [c_custkey#X, count#X] -Keys [1]: [c_custkey#X] -Functions [1]: [count(o_orderkey#X)] -Aggregate Attributes [1]: [count(o_orderkey#X)#X] -Results [1]: [count(o_orderkey#X)#X AS c_count#X] - -(52) HashAggregate -Input [1]: [c_count#X] -Keys [1]: [c_count#X] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#X] -Results [2]: [c_count#X, count#X] - -(53) Exchange -Input [2]: [c_count#X, count#X] -Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(54) HashAggregate -Input [2]: [c_count#X, count#X] -Keys [1]: [c_count#X] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#X] -Results [2]: [c_count#X, count(1)#X AS custdist#X] - -(55) Exchange -Input [2]: [c_count#X, custdist#X] -Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(56) Sort -Input [2]: [c_count#X, custdist#X] -Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 - -(57) AdaptiveSparkPlan -Output [2]: [c_count#X, custdist#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/14.txt deleted file mode 100644 index 36746c0c760..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/14.txt +++ /dev/null @@ -1,205 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (38) -+- == Final Plan == - VeloxColumnarToRow (24) - +- ^ ProjectExecTransformer (22) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7), Statistics(X) - : +- ColumnarExchange (6) - : +- VeloxResizeBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ FileSourceScanExecTransformer parquet (1) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- VeloxResizeBatches (14) - +- ^ ProjectExecTransformer (12) - +- ^ FilterExecTransformer (11) - +- ^ FileSourceScanExecTransformer parquet (10) -+- == Initial Plan == - HashAggregate (37) - +- HashAggregate (36) - +- Project (35) - +- SortMergeJoin Inner (34) - :- Sort (29) - : +- Exchange (28) - : +- Project (27) - : +- Filter (26) - : +- Scan parquet (25) - +- Sort (33) - +- Exchange (32) - +- Filter (31) - +- Scan parquet (30) - - -(1) FileSourceScanExecTransformer parquet -Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) - -(3) ProjectExecTransformer -Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] -Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(4) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] -Arguments: false - -(5) VeloxResizeBatches -Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] - -(6) ColumnarExchange -Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] -Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] - -(7) ShuffleQueryStage -Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -Arguments: X - -(8) InputAdapter -Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] - -(9) InputIteratorTransformer -Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] - -(10) FileSourceScanExecTransformer parquet -Output [2]: [p_partkey#X, p_type#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_partkey)] -ReadSchema: struct - -(11) FilterExecTransformer -Input [2]: [p_partkey#X, p_type#X] -Arguments: isnotnull(p_partkey#X) - -(12) ProjectExecTransformer -Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] -Input [2]: [p_partkey#X, p_type#X] - -(13) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] -Arguments: false - -(14) VeloxResizeBatches -Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] - -(15) ColumnarExchange -Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] -Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [shuffle_writer_type=hash] - -(16) ShuffleQueryStage -Output [2]: [p_partkey#X, p_type#X] -Arguments: X - -(17) InputAdapter -Input [2]: [p_partkey#X, p_type#X] - -(18) InputIteratorTransformer -Input [2]: [p_partkey#X, p_type#X] - -(19) ShuffledHashJoinExecTransformer -Left keys [1]: [l_partkey#X] -Right keys [1]: [p_partkey#X] -Join condition: None - -(20) ProjectExecTransformer -Output [2]: [CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] -Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] - -(21) RegularHashAggregateExecTransformer -Input [2]: [_pre_X#X, _pre_X#X] -Keys: [] -Functions [2]: [sum(_pre_X#X), sum(_pre_X#X)] -Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] - -(22) ProjectExecTransformer -Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] -Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] - -(23) WholeStageCodegenTransformer (X) -Input [1]: [promo_revenue#X] -Arguments: false - -(24) VeloxColumnarToRow -Input [1]: [promo_revenue#X] - -(25) Scan parquet -Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] -ReadSchema: struct - -(26) Filter -Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) - -(27) Project -Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(28) Exchange -Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(29) Sort -Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(30) Scan parquet -Output [2]: [p_partkey#X, p_type#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_partkey)] -ReadSchema: struct - -(31) Filter -Input [2]: [p_partkey#X, p_type#X] -Condition : isnotnull(p_partkey#X) - -(32) Exchange -Input [2]: [p_partkey#X, p_type#X] -Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(33) Sort -Input [2]: [p_partkey#X, p_type#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(34) SortMergeJoin -Left keys [1]: [l_partkey#X] -Right keys [1]: [p_partkey#X] -Join condition: None - -(35) Project -Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] -Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] - -(36) HashAggregate -Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] -Keys: [] -Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] - -(37) HashAggregate -Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -Keys: [] -Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] - -(38) AdaptiveSparkPlan -Output [1]: [promo_revenue#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt deleted file mode 100644 index 4dc2ad2b84e..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt +++ /dev/null @@ -1,404 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (47) -+- == Final Plan == - VeloxColumnarToRow (30) - +- AQEShuffleRead (29) - +- ShuffleQueryStage (28), Statistics(X) - +- ColumnarExchange (27) - +- VeloxResizeBatches (26) - +- ^ ProjectExecTransformer (24) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (23) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7), Statistics(X) - : +- ColumnarExchange (6) - : +- VeloxResizeBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ FileSourceScanExecTransformer parquet (1) - +- ^ FilterExecTransformer (22) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ShuffleQueryStage (18), Statistics(X) - +- ColumnarExchange (17) - +- VeloxResizeBatches (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ FilterExecTransformer (11) - +- ^ FileSourceScanExecTransformer parquet (10) -+- == Initial Plan == - Sort (46) - +- Exchange (45) - +- Project (44) - +- SortMergeJoin Inner (43) - :- Sort (34) - : +- Exchange (33) - : +- Filter (32) - : +- Scan parquet (31) - +- Sort (42) - +- Filter (41) - +- HashAggregate (40) - +- Exchange (39) - +- HashAggregate (38) - +- Project (37) - +- Filter (36) - +- Scan parquet (35) - - -(1) FileSourceScanExecTransformer parquet -Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -Arguments: isnotnull(s_suppkey#X) - -(3) ProjectExecTransformer -Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] - -(4) WholeStageCodegenTransformer (X) -Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] -Arguments: false - -(5) VeloxResizeBatches -Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] - -(6) ColumnarExchange -Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] -Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [shuffle_writer_type=hash] - -(7) ShuffleQueryStage -Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -Arguments: X - -(8) InputAdapter -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] - -(9) InputIteratorTransformer -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] - -(10) FileSourceScanExecTransformer parquet -Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] -ReadSchema: struct - -(11) FilterExecTransformer -Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) - -(12) ProjectExecTransformer -Output [2]: [l_suppkey#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] -Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(13) FlushableHashAggregateExecTransformer -Input [2]: [l_suppkey#X, _pre_X#X] -Keys [1]: [l_suppkey#X] -Functions [1]: [partial_sum(_pre_X#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [3]: [l_suppkey#X, sum#X, isEmpty#X] - -(14) ProjectExecTransformer -Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] -Input [3]: [l_suppkey#X, sum#X, isEmpty#X] - -(15) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] -Arguments: false - -(16) VeloxResizeBatches -Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] - -(17) ColumnarExchange -Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] - -(18) ShuffleQueryStage -Output [3]: [l_suppkey#X, sum#X, isEmpty#X] -Arguments: X - -(19) InputAdapter -Input [3]: [l_suppkey#X, sum#X, isEmpty#X] - -(20) InputIteratorTransformer -Input [3]: [l_suppkey#X, sum#X, isEmpty#X] - -(21) RegularHashAggregateExecTransformer -Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -Keys [1]: [l_suppkey#X] -Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] - -(22) FilterExecTransformer -Input [2]: [supplier_no#X, total_revenue#X] -Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) - -(23) ShuffledHashJoinExecTransformer -Left keys [1]: [s_suppkey#X] -Right keys [1]: [supplier_no#X] -Join condition: None - -(24) ProjectExecTransformer -Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] - -(25) WholeStageCodegenTransformer (X) -Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -Arguments: false - -(26) VeloxResizeBatches -Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] - -(27) ColumnarExchange -Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(28) ShuffleQueryStage -Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -Arguments: X - -(29) AQEShuffleRead -Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -Arguments: local - -(30) VeloxColumnarToRow -Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] - -(31) Scan parquet -Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey)] -ReadSchema: struct - -(32) Filter -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -Condition : isnotnull(s_suppkey#X) - -(33) Exchange -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(34) Sort -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(35) Scan parquet -Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] -ReadSchema: struct - -(36) Filter -Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) - -(37) Project -Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] -Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(38) HashAggregate -Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] -Keys [1]: [l_suppkey#X] -Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [3]: [l_suppkey#X, sum#X, isEmpty#X] - -(39) Exchange -Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(40) HashAggregate -Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -Keys [1]: [l_suppkey#X] -Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] - -(41) Filter -Input [2]: [supplier_no#X, total_revenue#X] -Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) - -(42) Sort -Input [2]: [supplier_no#X, total_revenue#X] -Arguments: [supplier_no#X ASC NULLS FIRST], false, 0 - -(43) SortMergeJoin -Left keys [1]: [s_suppkey#X] -Right keys [1]: [supplier_no#X] -Join condition: None - -(44) Project -Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] - -(45) Exchange -Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(46) Sort -Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 - -(47) AdaptiveSparkPlan -Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -Arguments: isFinalPlan=true - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 22 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (72) -+- == Final Plan == - VeloxColumnarToRow (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ ProjectExecTransformer (60) - +- ^ RegularHashAggregateExecTransformer (59) - +- ^ InputIteratorTransformer (58) - +- ShuffleQueryStage (56), Statistics(X) - +- ColumnarExchange (55) - +- VeloxResizeBatches (54) - +- ^ ProjectExecTransformer (52) - +- ^ FlushableHashAggregateExecTransformer (51) - +- ^ ProjectExecTransformer (50) - +- ^ FilterExecTransformer (49) - +- ^ FileSourceScanExecTransformer parquet (48) -+- == Initial Plan == - HashAggregate (71) - +- HashAggregate (70) - +- HashAggregate (69) - +- Exchange (68) - +- HashAggregate (67) - +- Project (66) - +- Filter (65) - +- Scan parquet (64) - - -(48) FileSourceScanExecTransformer parquet -Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] -ReadSchema: struct - -(49) FilterExecTransformer -Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) - -(50) ProjectExecTransformer -Output [2]: [l_suppkey#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] -Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(51) FlushableHashAggregateExecTransformer -Input [2]: [l_suppkey#X, _pre_X#X] -Keys [1]: [l_suppkey#X] -Functions [1]: [partial_sum(_pre_X#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [3]: [l_suppkey#X, sum#X, isEmpty#X] - -(52) ProjectExecTransformer -Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] -Input [3]: [l_suppkey#X, sum#X, isEmpty#X] - -(53) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] -Arguments: false - -(54) VeloxResizeBatches -Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] - -(55) ColumnarExchange -Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] - -(56) ShuffleQueryStage -Output [3]: [l_suppkey#X, sum#X, isEmpty#X] -Arguments: X - -(57) InputAdapter -Input [3]: [l_suppkey#X, sum#X, isEmpty#X] - -(58) InputIteratorTransformer -Input [3]: [l_suppkey#X, sum#X, isEmpty#X] - -(59) RegularHashAggregateExecTransformer -Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -Keys [1]: [l_suppkey#X] -Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] - -(60) ProjectExecTransformer -Output [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -Input [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] - -(61) RegularHashAggregateExecTransformer -Input [1]: [total_revenue#X] -Keys: [] -Functions [1]: [max(total_revenue#X)] -Aggregate Attributes [1]: [max(total_revenue#X)#X] -Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] - -(62) WholeStageCodegenTransformer (X) -Input [1]: [max(total_revenue)#X] -Arguments: false - -(63) VeloxColumnarToRow -Input [1]: [max(total_revenue)#X] - -(64) Scan parquet -Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] -ReadSchema: struct - -(65) Filter -Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) - -(66) Project -Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] -Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(67) HashAggregate -Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] -Keys [1]: [l_suppkey#X] -Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [3]: [l_suppkey#X, sum#X, isEmpty#X] - -(68) Exchange -Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(69) HashAggregate -Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -Keys [1]: [l_suppkey#X] -Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] - -(70) HashAggregate -Input [1]: [total_revenue#X] -Keys: [] -Functions [1]: [partial_max(total_revenue#X)] -Aggregate Attributes [1]: [max#X] -Results [1]: [max#X] - -(71) HashAggregate -Input [1]: [max#X] -Keys: [] -Functions [1]: [max(total_revenue#X)] -Aggregate Attributes [1]: [max(total_revenue#X)#X] -Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] - -(72) AdaptiveSparkPlan -Output [1]: [max(total_revenue)#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt deleted file mode 100644 index fd15a3d3521..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt +++ /dev/null @@ -1,374 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (71) -+- == Final Plan == - VeloxColumnarToRow (47) - +- ^ SortExecTransformer (45) - +- ^ InputIteratorTransformer (44) - +- ShuffleQueryStage (42), Statistics(X) - +- ColumnarExchange (41) - +- VeloxResizeBatches (40) - +- ^ RegularHashAggregateExecTransformer (38) - +- ^ InputIteratorTransformer (37) - +- ShuffleQueryStage (35), Statistics(X) - +- ColumnarExchange (34) - +- VeloxResizeBatches (33) - +- ^ ProjectExecTransformer (31) - +- ^ FlushableHashAggregateExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- VeloxResizeBatches (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7), Statistics(X) - : +- ColumnarExchange (6) - : +- VeloxResizeBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ FileSourceScanExecTransformer parquet (1) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- VeloxResizeBatches (14) - +- ^ ProjectExecTransformer (12) - +- ^ FilterExecTransformer (11) - +- ^ FileSourceScanExecTransformer parquet (10) -+- == Initial Plan == - Sort (70) - +- Exchange (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- SortMergeJoin Inner (61) - :- Sort (56) - : +- Exchange (55) - : +- BroadcastHashJoin LeftAnti BuildRight (54) - : :- Filter (49) - : : +- Scan parquet (48) - : +- BroadcastExchange (53) - : +- Project (52) - : +- Filter (51) - : +- Scan parquet (50) - +- Sort (60) - +- Exchange (59) - +- Filter (58) - +- Scan parquet (57) - - -(1) FileSourceScanExecTransformer parquet -Output [2]: [ps_partkey#X, ps_suppkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(ps_partkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [2]: [ps_partkey#X, ps_suppkey#X] -Arguments: isnotnull(ps_partkey#X) - -(3) ProjectExecTransformer -Output [3]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X] -Input [2]: [ps_partkey#X, ps_suppkey#X] - -(4) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] -Arguments: false - -(5) VeloxResizeBatches -Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] - -(6) ColumnarExchange -Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] -Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(7) ShuffleQueryStage -Output [2]: [ps_partkey#X, ps_suppkey#X] -Arguments: X - -(8) InputAdapter -Input [2]: [ps_partkey#X, ps_suppkey#X] - -(9) InputIteratorTransformer -Input [2]: [ps_partkey#X, ps_suppkey#X] - -(10) FileSourceScanExecTransformer parquet -Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] -ReadSchema: struct - -(11) FilterExecTransformer -Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -Arguments: (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) - -(12) ProjectExecTransformer -Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] - -(13) WholeStageCodegenTransformer (X) -Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -Arguments: false - -(14) VeloxResizeBatches -Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] - -(15) ColumnarExchange -Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [shuffle_writer_type=hash] - -(16) ShuffleQueryStage -Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -Arguments: X - -(17) InputAdapter -Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] - -(18) InputIteratorTransformer -Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] - -(19) ShuffledHashJoinExecTransformer -Left keys [1]: [ps_partkey#X] -Right keys [1]: [p_partkey#X] -Join condition: None - -(20) ProjectExecTransformer -Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] -Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] - -(21) FlushableHashAggregateExecTransformer -Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] -Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -Functions: [] -Aggregate Attributes: [] -Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] - -(22) ProjectExecTransformer -Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] - -(23) WholeStageCodegenTransformer (X) -Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -Arguments: false - -(24) VeloxResizeBatches -Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] - -(25) ColumnarExchange -Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(26) ShuffleQueryStage -Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -Arguments: X - -(27) InputAdapter -Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] - -(28) InputIteratorTransformer -Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] - -(29) RegularHashAggregateExecTransformer -Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -Functions: [] -Aggregate Attributes: [] -Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] - -(30) FlushableHashAggregateExecTransformer -Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -Keys [3]: [p_brand#X, p_type#X, p_size#X] -Functions [1]: [partial_count(distinct ps_suppkey#X)] -Aggregate Attributes [1]: [count(ps_suppkey#X)#X] -Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] - -(31) ProjectExecTransformer -Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] -Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] - -(32) WholeStageCodegenTransformer (X) -Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] -Arguments: false - -(33) VeloxResizeBatches -Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] - -(34) ColumnarExchange -Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] -Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [shuffle_writer_type=hash] - -(35) ShuffleQueryStage -Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] -Arguments: X - -(36) InputAdapter -Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] - -(37) InputIteratorTransformer -Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] - -(38) RegularHashAggregateExecTransformer -Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -Keys [3]: [p_brand#X, p_type#X, p_size#X] -Functions [1]: [count(distinct ps_suppkey#X)] -Aggregate Attributes [1]: [count(ps_suppkey#X)#X] -Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] - -(39) WholeStageCodegenTransformer (X) -Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -Arguments: false - -(40) VeloxResizeBatches -Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] - -(41) ColumnarExchange -Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(42) ShuffleQueryStage -Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -Arguments: X - -(43) InputAdapter -Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] - -(44) InputIteratorTransformer -Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] - -(45) SortExecTransformer -Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 - -(46) WholeStageCodegenTransformer (X) -Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -Arguments: false - -(47) VeloxColumnarToRow -Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] - -(48) Scan parquet -Output [2]: [ps_partkey#X, ps_suppkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(ps_partkey)] -ReadSchema: struct - -(49) Filter -Input [2]: [ps_partkey#X, ps_suppkey#X] -Condition : isnotnull(ps_partkey#X) - -(50) Scan parquet -Output [2]: [s_suppkey#X, s_comment#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_comment)] -ReadSchema: struct - -(51) Filter -Input [2]: [s_suppkey#X, s_comment#X] -Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) - -(52) Project -Output [1]: [s_suppkey#X] -Input [2]: [s_suppkey#X, s_comment#X] - -(53) BroadcastExchange -Input [1]: [s_suppkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] - -(54) BroadcastHashJoin -Left keys [1]: [ps_suppkey#X] -Right keys [1]: [s_suppkey#X] -Join condition: None - -(55) Exchange -Input [2]: [ps_partkey#X, ps_suppkey#X] -Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(56) Sort -Input [2]: [ps_partkey#X, ps_suppkey#X] -Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 - -(57) Scan parquet -Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] -ReadSchema: struct - -(58) Filter -Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) - -(59) Exchange -Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(60) Sort -Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(61) SortMergeJoin -Left keys [1]: [ps_partkey#X] -Right keys [1]: [p_partkey#X] -Join condition: None - -(62) Project -Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] -Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] - -(63) HashAggregate -Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] -Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -Functions: [] -Aggregate Attributes: [] -Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] - -(64) Exchange -Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(65) HashAggregate -Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -Functions: [] -Aggregate Attributes: [] -Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] - -(66) HashAggregate -Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -Keys [3]: [p_brand#X, p_type#X, p_size#X] -Functions [1]: [partial_count(distinct ps_suppkey#X)] -Aggregate Attributes [1]: [count(ps_suppkey#X)#X] -Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] - -(67) Exchange -Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(68) HashAggregate -Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -Keys [3]: [p_brand#X, p_type#X, p_size#X] -Functions [1]: [count(distinct ps_suppkey#X)] -Aggregate Attributes [1]: [count(ps_suppkey#X)#X] -Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] - -(69) Exchange -Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(70) Sort -Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 - -(71) AdaptiveSparkPlan -Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt deleted file mode 100644 index 8537c6b0e1e..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt +++ /dev/null @@ -1,340 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (62) -+- == Final Plan == - VeloxColumnarToRow (39) - +- ^ ProjectExecTransformer (37) - +- ^ RegularHashAggregateExecTransformer (36) - +- ^ ProjectExecTransformer (35) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (34) - :- ^ ProjectExecTransformer (20) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - : :- ^ InputIteratorTransformer (9) - : : +- ShuffleQueryStage (7), Statistics(X) - : : +- ColumnarExchange (6) - : : +- VeloxResizeBatches (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ FileSourceScanExecTransformer parquet (1) - : +- ^ InputIteratorTransformer (18) - : +- ShuffleQueryStage (16), Statistics(X) - : +- ColumnarExchange (15) - : +- VeloxResizeBatches (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FilterExecTransformer (11) - : +- ^ FileSourceScanExecTransformer parquet (10) - +- ^ FilterExecTransformer (33) - +- ^ ProjectExecTransformer (32) - +- ^ RegularHashAggregateExecTransformer (31) - +- ^ InputIteratorTransformer (30) - +- ShuffleQueryStage (28), Statistics(X) - +- ColumnarExchange (27) - +- VeloxResizeBatches (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ FilterExecTransformer (22) - +- ^ FileSourceScanExecTransformer parquet (21) -+- == Initial Plan == - HashAggregate (61) - +- HashAggregate (60) - +- Project (59) - +- SortMergeJoin Inner (58) - :- Project (50) - : +- SortMergeJoin Inner (49) - : :- Sort (43) - : : +- Exchange (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Sort (48) - : +- Exchange (47) - : +- Project (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Sort (57) - +- Filter (56) - +- HashAggregate (55) - +- Exchange (54) - +- HashAggregate (53) - +- Filter (52) - +- Scan parquet (51) - - -(1) FileSourceScanExecTransformer parquet -Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -Arguments: (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) - -(3) ProjectExecTransformer -Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] -Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] - -(4) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] -Arguments: false - -(5) VeloxResizeBatches -Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] - -(6) ColumnarExchange -Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] -Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [shuffle_writer_type=hash] - -(7) ShuffleQueryStage -Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -Arguments: X - -(8) InputAdapter -Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] - -(9) InputIteratorTransformer -Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] - -(10) FileSourceScanExecTransformer parquet -Output [3]: [p_partkey#X, p_brand#X, p_container#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] -ReadSchema: struct - -(11) FilterExecTransformer -Input [3]: [p_partkey#X, p_brand#X, p_container#X] -Arguments: ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) - -(12) ProjectExecTransformer -Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] -Input [3]: [p_partkey#X, p_brand#X, p_container#X] - -(13) WholeStageCodegenTransformer (X) -Input [2]: [hash_partition_key#X, p_partkey#X] -Arguments: false - -(14) VeloxResizeBatches -Input [2]: [hash_partition_key#X, p_partkey#X] - -(15) ColumnarExchange -Input [2]: [hash_partition_key#X, p_partkey#X] -Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(16) ShuffleQueryStage -Output [1]: [p_partkey#X] -Arguments: X - -(17) InputAdapter -Input [1]: [p_partkey#X] - -(18) InputIteratorTransformer -Input [1]: [p_partkey#X] - -(19) ShuffledHashJoinExecTransformer -Left keys [1]: [l_partkey#X] -Right keys [1]: [p_partkey#X] -Join condition: None - -(20) ProjectExecTransformer -Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] -Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] - -(21) FileSourceScanExecTransformer parquet -Output [2]: [l_partkey#X, l_quantity#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_partkey)] -ReadSchema: struct - -(22) FilterExecTransformer -Input [2]: [l_partkey#X, l_quantity#X] -Arguments: isnotnull(l_partkey#X) - -(23) FlushableHashAggregateExecTransformer -Input [2]: [l_partkey#X, l_quantity#X] -Keys [1]: [l_partkey#X] -Functions [1]: [partial_avg(l_quantity#X)] -Aggregate Attributes [2]: [sum#X, count#X] -Results [3]: [l_partkey#X, sum#X, count#X] - -(24) ProjectExecTransformer -Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] -Input [3]: [l_partkey#X, sum#X, count#X] - -(25) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] -Arguments: false - -(26) VeloxResizeBatches -Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] - -(27) ColumnarExchange -Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] -Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [shuffle_writer_type=hash] - -(28) ShuffleQueryStage -Output [3]: [l_partkey#X, sum#X, count#X] -Arguments: X - -(29) InputAdapter -Input [3]: [l_partkey#X, sum#X, count#X] - -(30) InputIteratorTransformer -Input [3]: [l_partkey#X, sum#X, count#X] - -(31) RegularHashAggregateExecTransformer -Input [3]: [l_partkey#X, sum#X, count#X] -Keys [1]: [l_partkey#X] -Functions [1]: [avg(l_quantity#X)] -Aggregate Attributes [1]: [avg(l_quantity#X)#X] -Results [2]: [l_partkey#X, avg(l_quantity#X)#X] - -(32) ProjectExecTransformer -Output [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -Input [2]: [l_partkey#X, avg(l_quantity#X)#X] - -(33) FilterExecTransformer -Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] -Arguments: isnotnull((0.2 * avg(l_quantity))#X) - -(34) ShuffledHashJoinExecTransformer -Left keys [1]: [p_partkey#X] -Right keys [1]: [l_partkey#X] -Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) - -(35) ProjectExecTransformer -Output [1]: [l_extendedprice#X] -Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] - -(36) RegularHashAggregateExecTransformer -Input [1]: [l_extendedprice#X] -Keys: [] -Functions [1]: [sum(l_extendedprice#X)] -Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] -Results [1]: [sum(l_extendedprice#X)#X] - -(37) ProjectExecTransformer -Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] -Input [1]: [sum(l_extendedprice#X)#X] - -(38) WholeStageCodegenTransformer (X) -Input [1]: [avg_yearly#X] -Arguments: false - -(39) VeloxColumnarToRow -Input [1]: [avg_yearly#X] - -(40) Scan parquet -Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] -ReadSchema: struct - -(41) Filter -Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) - -(42) Exchange -Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(43) Sort -Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(44) Scan parquet -Output [3]: [p_partkey#X, p_brand#X, p_container#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] -ReadSchema: struct - -(45) Filter -Input [3]: [p_partkey#X, p_brand#X, p_container#X] -Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) - -(46) Project -Output [1]: [p_partkey#X] -Input [3]: [p_partkey#X, p_brand#X, p_container#X] - -(47) Exchange -Input [1]: [p_partkey#X] -Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(48) Sort -Input [1]: [p_partkey#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(49) SortMergeJoin -Left keys [1]: [l_partkey#X] -Right keys [1]: [p_partkey#X] -Join condition: None - -(50) Project -Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] -Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] - -(51) Scan parquet -Output [2]: [l_partkey#X, l_quantity#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_partkey)] -ReadSchema: struct - -(52) Filter -Input [2]: [l_partkey#X, l_quantity#X] -Condition : isnotnull(l_partkey#X) - -(53) HashAggregate -Input [2]: [l_partkey#X, l_quantity#X] -Keys [1]: [l_partkey#X] -Functions [1]: [partial_avg(l_quantity#X)] -Aggregate Attributes [2]: [sum#X, count#X] -Results [3]: [l_partkey#X, sum#X, count#X] - -(54) Exchange -Input [3]: [l_partkey#X, sum#X, count#X] -Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(55) HashAggregate -Input [3]: [l_partkey#X, sum#X, count#X] -Keys [1]: [l_partkey#X] -Functions [1]: [avg(l_quantity#X)] -Aggregate Attributes [1]: [avg(l_quantity#X)#X] -Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] - -(56) Filter -Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] -Condition : isnotnull((0.2 * avg(l_quantity))#X) - -(57) Sort -Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(58) SortMergeJoin -Left keys [1]: [p_partkey#X] -Right keys [1]: [l_partkey#X] -Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) - -(59) Project -Output [1]: [l_extendedprice#X] -Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] - -(60) HashAggregate -Input [1]: [l_extendedprice#X] -Keys: [] -Functions [1]: [partial_sum(l_extendedprice#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [2]: [sum#X, isEmpty#X] - -(61) HashAggregate -Input [2]: [sum#X, isEmpty#X] -Keys: [] -Functions [1]: [sum(l_extendedprice#X)] -Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] -Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] - -(62) AdaptiveSparkPlan -Output [1]: [avg_yearly#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt deleted file mode 100644 index ab04e33cce0..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt +++ /dev/null @@ -1,575 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (109) -+- == Final Plan == - VeloxColumnarToRow (69) - +- TakeOrderedAndProjectExecTransformer (68) - +- ^ RegularHashAggregateExecTransformer (66) - +- ^ ProjectExecTransformer (65) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (64) - :- ^ InputIteratorTransformer (46) - : +- ShuffleQueryStage (44), Statistics(X) - : +- ColumnarExchange (43) - : +- VeloxResizeBatches (42) - : +- ^ ProjectExecTransformer (40) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (39) - : :- ^ InputIteratorTransformer (9) - : : +- ShuffleQueryStage (7), Statistics(X) - : : +- ColumnarExchange (6) - : : +- VeloxResizeBatches (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ FileSourceScanExecTransformer parquet (1) - : +- ^ InputIteratorTransformer (38) - : +- ShuffleQueryStage (36), Statistics(X) - : +- ColumnarExchange (35) - : +- VeloxResizeBatches (34) - : +- ^ ProjectExecTransformer (32) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (31) - : :- ^ InputIteratorTransformer (18) - : : +- ShuffleQueryStage (16), Statistics(X) - : : +- ColumnarExchange (15) - : : +- VeloxResizeBatches (14) - : : +- ^ ProjectExecTransformer (12) - : : +- ^ FilterExecTransformer (11) - : : +- ^ FileSourceScanExecTransformer parquet (10) - : +- ^ ProjectExecTransformer (30) - : +- ^ FilterExecTransformer (29) - : +- ^ RegularHashAggregateExecTransformer (28) - : +- ^ InputIteratorTransformer (27) - : +- ShuffleQueryStage (25), Statistics(X) - : +- ColumnarExchange (24) - : +- VeloxResizeBatches (23) - : +- ^ ProjectExecTransformer (21) - : +- ^ FlushableHashAggregateExecTransformer (20) - : +- ^ FileSourceScanExecTransformer parquet (19) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (63) - :- ^ InputIteratorTransformer (55) - : +- ShuffleQueryStage (53), Statistics(X) - : +- ColumnarExchange (52) - : +- VeloxResizeBatches (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ FileSourceScanExecTransformer parquet (47) - +- ^ ProjectExecTransformer (62) - +- ^ FilterExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ InputIteratorTransformer (59) - +- ShuffleQueryStage (57), Statistics(X) - +- ReusedExchange (56) -+- == Initial Plan == - TakeOrderedAndProject (108) - +- HashAggregate (107) - +- HashAggregate (106) - +- Project (105) - +- SortMergeJoin Inner (104) - :- Sort (91) - : +- Exchange (90) - : +- Project (89) - : +- SortMergeJoin Inner (88) - : :- Sort (73) - : : +- Exchange (72) - : : +- Filter (71) - : : +- Scan parquet (70) - : +- Sort (87) - : +- Exchange (86) - : +- SortMergeJoin LeftSemi (85) - : :- Sort (77) - : : +- Exchange (76) - : : +- Filter (75) - : : +- Scan parquet (74) - : +- Sort (84) - : +- Project (83) - : +- Filter (82) - : +- HashAggregate (81) - : +- Exchange (80) - : +- HashAggregate (79) - : +- Scan parquet (78) - +- SortMergeJoin LeftSemi (103) - :- Sort (95) - : +- Exchange (94) - : +- Filter (93) - : +- Scan parquet (92) - +- Sort (102) - +- Project (101) - +- Filter (100) - +- HashAggregate (99) - +- Exchange (98) - +- HashAggregate (97) - +- Scan parquet (96) - - -(1) FileSourceScanExecTransformer parquet -Output [2]: [c_custkey#X, c_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_custkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [2]: [c_custkey#X, c_name#X] -Arguments: isnotnull(c_custkey#X) - -(3) ProjectExecTransformer -Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X] -Input [2]: [c_custkey#X, c_name#X] - -(4) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] -Arguments: false - -(5) VeloxResizeBatches -Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] - -(6) ColumnarExchange -Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] -Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [shuffle_writer_type=hash] - -(7) ShuffleQueryStage -Output [2]: [c_custkey#X, c_name#X] -Arguments: X - -(8) InputAdapter -Input [2]: [c_custkey#X, c_name#X] - -(9) InputIteratorTransformer -Input [2]: [c_custkey#X, c_name#X] - -(10) FileSourceScanExecTransformer parquet -Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] -ReadSchema: struct - -(11) FilterExecTransformer -Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Arguments: (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) - -(12) ProjectExecTransformer -Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] - -(13) WholeStageCodegenTransformer (X) -Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Arguments: false - -(14) VeloxResizeBatches -Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] - -(15) ColumnarExchange -Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [shuffle_writer_type=hash] - -(16) ShuffleQueryStage -Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Arguments: X - -(17) InputAdapter -Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] - -(18) InputIteratorTransformer -Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] - -(19) FileSourceScanExecTransformer parquet -Output [2]: [l_orderkey#X, l_quantity#X] -Batched: true -Location: InMemoryFileIndex [*] -ReadSchema: struct - -(20) FlushableHashAggregateExecTransformer -Input [2]: [l_orderkey#X, l_quantity#X] -Keys [1]: [l_orderkey#X] -Functions [1]: [partial_sum(l_quantity#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [3]: [l_orderkey#X, sum#X, isEmpty#X] - -(21) ProjectExecTransformer -Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] -Input [3]: [l_orderkey#X, sum#X, isEmpty#X] - -(22) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] -Arguments: false - -(23) VeloxResizeBatches -Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] - -(24) ColumnarExchange -Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] - -(25) ShuffleQueryStage -Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -Arguments: X - -(26) InputAdapter -Input [3]: [l_orderkey#X, sum#X, isEmpty#X] - -(27) InputIteratorTransformer -Input [3]: [l_orderkey#X, sum#X, isEmpty#X] - -(28) RegularHashAggregateExecTransformer -Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -Keys [1]: [l_orderkey#X] -Functions [1]: [sum(l_quantity#X)] -Aggregate Attributes [1]: [sum(l_quantity#X)#X] -Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] - -(29) FilterExecTransformer -Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) - -(30) ProjectExecTransformer -Output [1]: [l_orderkey#X] -Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] - -(31) ShuffledHashJoinExecTransformer -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(32) ProjectExecTransformer -Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] - -(33) WholeStageCodegenTransformer (X) -Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Arguments: false - -(34) VeloxResizeBatches -Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] - -(35) ColumnarExchange -Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [shuffle_writer_type=hash] - -(36) ShuffleQueryStage -Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Arguments: X - -(37) InputAdapter -Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] - -(38) InputIteratorTransformer -Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] - -(39) ShuffledHashJoinExecTransformer -Left keys [1]: [c_custkey#X] -Right keys [1]: [o_custkey#X] -Join condition: None - -(40) ProjectExecTransformer -Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] - -(41) WholeStageCodegenTransformer (X) -Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -Arguments: false - -(42) VeloxResizeBatches -Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] - -(43) ColumnarExchange -Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [shuffle_writer_type=hash] - -(44) ShuffleQueryStage -Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -Arguments: X - -(45) InputAdapter -Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] - -(46) InputIteratorTransformer -Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] - -(47) FileSourceScanExecTransformer parquet -Output [2]: [l_orderkey#X, l_quantity#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_orderkey)] -ReadSchema: struct - -(48) FilterExecTransformer -Input [2]: [l_orderkey#X, l_quantity#X] -Arguments: isnotnull(l_orderkey#X) - -(49) ProjectExecTransformer -Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] -Input [2]: [l_orderkey#X, l_quantity#X] - -(50) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] -Arguments: false - -(51) VeloxResizeBatches -Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] - -(52) ColumnarExchange -Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [shuffle_writer_type=hash] - -(53) ShuffleQueryStage -Output [2]: [l_orderkey#X, l_quantity#X] -Arguments: X - -(54) InputAdapter -Input [2]: [l_orderkey#X, l_quantity#X] - -(55) InputIteratorTransformer -Input [2]: [l_orderkey#X, l_quantity#X] - -(56) ReusedExchange [Reuses operator id: 24] -Output [3]: [l_orderkey#X, sum#X, isEmpty#X] - -(57) ShuffleQueryStage -Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -Arguments: X - -(58) InputAdapter -Input [3]: [l_orderkey#X, sum#X, isEmpty#X] - -(59) InputIteratorTransformer -Input [3]: [l_orderkey#X, sum#X, isEmpty#X] - -(60) RegularHashAggregateExecTransformer -Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -Keys [1]: [l_orderkey#X] -Functions [1]: [sum(l_quantity#X)] -Aggregate Attributes [1]: [sum(l_quantity#X)#X] -Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] - -(61) FilterExecTransformer -Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) - -(62) ProjectExecTransformer -Output [1]: [l_orderkey#X] -Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] - -(63) ShuffledHashJoinExecTransformer -Left keys [1]: [l_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(64) ShuffledHashJoinExecTransformer -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(65) ProjectExecTransformer -Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] -Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] - -(66) RegularHashAggregateExecTransformer -Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] -Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] -Functions [1]: [sum(l_quantity#X)] -Aggregate Attributes [1]: [sum(l_quantity#X)#X] -Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] - -(67) WholeStageCodegenTransformer (X) -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: false - -(68) TakeOrderedAndProjectExecTransformer -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 - -(69) VeloxColumnarToRow -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] - -(70) Scan parquet -Output [2]: [c_custkey#X, c_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_custkey)] -ReadSchema: struct - -(71) Filter -Input [2]: [c_custkey#X, c_name#X] -Condition : isnotnull(c_custkey#X) - -(72) Exchange -Input [2]: [c_custkey#X, c_name#X] -Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(73) Sort -Input [2]: [c_custkey#X, c_name#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(74) Scan parquet -Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] -ReadSchema: struct - -(75) Filter -Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) - -(76) Exchange -Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(77) Sort -Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(78) Scan parquet -Output [2]: [l_orderkey#X, l_quantity#X] -Batched: true -Location: InMemoryFileIndex [*] -ReadSchema: struct - -(79) HashAggregate -Input [2]: [l_orderkey#X, l_quantity#X] -Keys [1]: [l_orderkey#X] -Functions [1]: [partial_sum(l_quantity#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [3]: [l_orderkey#X, sum#X, isEmpty#X] - -(80) Exchange -Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(81) HashAggregate -Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -Keys [1]: [l_orderkey#X] -Functions [1]: [sum(l_quantity#X)] -Aggregate Attributes [1]: [sum(l_quantity#X)#X] -Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] - -(82) Filter -Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) - -(83) Project -Output [1]: [l_orderkey#X] -Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] - -(84) Sort -Input [1]: [l_orderkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(85) SortMergeJoin -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(86) Exchange -Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(87) Sort -Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(88) SortMergeJoin -Left keys [1]: [c_custkey#X] -Right keys [1]: [o_custkey#X] -Join condition: None - -(89) Project -Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] - -(90) Exchange -Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(91) Sort -Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(92) Scan parquet -Output [2]: [l_orderkey#X, l_quantity#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_orderkey)] -ReadSchema: struct - -(93) Filter -Input [2]: [l_orderkey#X, l_quantity#X] -Condition : isnotnull(l_orderkey#X) - -(94) Exchange -Input [2]: [l_orderkey#X, l_quantity#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(95) Sort -Input [2]: [l_orderkey#X, l_quantity#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(96) Scan parquet -Output [2]: [l_orderkey#X, l_quantity#X] -Batched: true -Location: InMemoryFileIndex [*] -ReadSchema: struct - -(97) HashAggregate -Input [2]: [l_orderkey#X, l_quantity#X] -Keys [1]: [l_orderkey#X] -Functions [1]: [partial_sum(l_quantity#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [3]: [l_orderkey#X, sum#X, isEmpty#X] - -(98) Exchange -Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(99) HashAggregate -Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -Keys [1]: [l_orderkey#X] -Functions [1]: [sum(l_quantity#X)] -Aggregate Attributes [1]: [sum(l_quantity#X)#X] -Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] - -(100) Filter -Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) - -(101) Project -Output [1]: [l_orderkey#X] -Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] - -(102) Sort -Input [1]: [l_orderkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(103) SortMergeJoin -Left keys [1]: [l_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(104) SortMergeJoin -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(105) Project -Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] -Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] - -(106) HashAggregate -Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] -Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] -Functions [1]: [partial_sum(l_quantity#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] - -(107) HashAggregate -Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] -Functions [1]: [sum(l_quantity#X)] -Aggregate Attributes [1]: [sum(l_quantity#X)#X] -Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] - -(108) TakeOrderedAndProject -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] - -(109) AdaptiveSparkPlan -Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/19.txt deleted file mode 100644 index 5c936160b80..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/19.txt +++ /dev/null @@ -1,200 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (37) -+- == Final Plan == - VeloxColumnarToRow (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7), Statistics(X) - : +- ColumnarExchange (6) - : +- VeloxResizeBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ FileSourceScanExecTransformer parquet (1) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- VeloxResizeBatches (14) - +- ^ ProjectExecTransformer (12) - +- ^ FilterExecTransformer (11) - +- ^ FileSourceScanExecTransformer parquet (10) -+- == Initial Plan == - HashAggregate (36) - +- HashAggregate (35) - +- Project (34) - +- SortMergeJoin Inner (33) - :- Sort (28) - : +- Exchange (27) - : +- Project (26) - : +- Filter (25) - : +- Scan parquet (24) - +- Sort (32) - +- Exchange (31) - +- Filter (30) - +- Scan parquet (29) - - -(1) FileSourceScanExecTransformer parquet -Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] -ReadSchema: struct - -(2) FilterExecTransformer -Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -Arguments: ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) - -(3) ProjectExecTransformer -Output [5]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] - -(4) WholeStageCodegenTransformer (X) -Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: false - -(5) VeloxResizeBatches -Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] - -(6) ColumnarExchange -Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] - -(7) ShuffleQueryStage -Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: X - -(8) InputAdapter -Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] - -(9) InputIteratorTransformer -Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] - -(10) FileSourceScanExecTransformer parquet -Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] -ReadSchema: struct - -(11) FilterExecTransformer -Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -Arguments: (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) - -(12) ProjectExecTransformer -Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] - -(13) WholeStageCodegenTransformer (X) -Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -Arguments: false - -(14) VeloxResizeBatches -Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] - -(15) ColumnarExchange -Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [shuffle_writer_type=hash] - -(16) ShuffleQueryStage -Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -Arguments: X - -(17) InputAdapter -Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] - -(18) InputIteratorTransformer -Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] - -(19) ShuffledHashJoinExecTransformer -Left keys [1]: [l_partkey#X] -Right keys [1]: [p_partkey#X] -Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) - -(20) ProjectExecTransformer -Output [1]: [CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] -Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] - -(21) RegularHashAggregateExecTransformer -Input [1]: [_pre_X#X] -Keys: [] -Functions [1]: [sum(_pre_X#X)] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] - -(22) WholeStageCodegenTransformer (X) -Input [1]: [revenue#X] -Arguments: false - -(23) VeloxColumnarToRow -Input [1]: [revenue#X] - -(24) Scan parquet -Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] -ReadSchema: struct - -(25) Filter -Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) - -(26) Project -Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] - -(27) Exchange -Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(28) Sort -Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(29) Scan parquet -Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] -ReadSchema: struct - -(30) Filter -Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) - -(31) Exchange -Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(32) Sort -Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(33) SortMergeJoin -Left keys [1]: [l_partkey#X] -Right keys [1]: [p_partkey#X] -Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) - -(34) Project -Output [2]: [l_extendedprice#X, l_discount#X] -Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] - -(35) HashAggregate -Input [2]: [l_extendedprice#X, l_discount#X] -Keys: [] -Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [2]: [sum#X, isEmpty#X] - -(36) HashAggregate -Input [2]: [sum#X, isEmpty#X] -Keys: [] -Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] - -(37) AdaptiveSparkPlan -Output [1]: [revenue#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/20.txt deleted file mode 100644 index 35a284f1f12..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/20.txt +++ /dev/null @@ -1,714 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (142) -+- == Final Plan == - VeloxColumnarToRow (92) - +- AQEShuffleRead (91) - +- ShuffleQueryStage (90), Statistics(X) - +- ColumnarExchange (89) - +- VeloxResizeBatches (88) - +- ^ ProjectExecTransformer (86) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (85) - :- ^ InputIteratorTransformer (75) - : +- ShuffleQueryStage (73), Statistics(X) - : +- ColumnarExchange (72) - : +- VeloxResizeBatches (71) - : +- ^ ProjectExecTransformer (69) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (68) - : :- ^ InputIteratorTransformer (9) - : : +- ShuffleQueryStage (7), Statistics(X) - : : +- ColumnarExchange (6) - : : +- VeloxResizeBatches (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ FileSourceScanExecTransformer parquet (1) - : +- ^ InputIteratorTransformer (67) - : +- ShuffleQueryStage (65), Statistics(X) - : +- ColumnarExchange (64) - : +- VeloxResizeBatches (63) - : +- ^ ProjectExecTransformer (61) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (60) - : :- ^ InputIteratorTransformer (35) - : : +- ShuffleQueryStage (33), Statistics(X) - : : +- ColumnarExchange (32) - : : +- VeloxResizeBatches (31) - : : +- ^ ProjectExecTransformer (29) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : : :- ^ InputIteratorTransformer (18) - : : : +- ShuffleQueryStage (16), Statistics(X) - : : : +- ColumnarExchange (15) - : : : +- VeloxResizeBatches (14) - : : : +- ^ ProjectExecTransformer (12) - : : : +- ^ FilterExecTransformer (11) - : : : +- ^ FileSourceScanExecTransformer parquet (10) - : : +- ^ InputIteratorTransformer (27) - : : +- ShuffleQueryStage (25), Statistics(X) - : : +- ColumnarExchange (24) - : : +- VeloxResizeBatches (23) - : : +- ^ ProjectExecTransformer (21) - : : +- ^ FilterExecTransformer (20) - : : +- ^ FileSourceScanExecTransformer parquet (19) - : +- ^ InputIteratorTransformer (59) - : +- ShuffleQueryStage (57), Statistics(X) - : +- ColumnarExchange (56) - : +- VeloxResizeBatches (55) - : +- ^ ProjectExecTransformer (53) - : +- ^ FilterExecTransformer (52) - : +- ^ ProjectExecTransformer (51) - : +- ^ RegularHashAggregateExecTransformer (50) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (49) - : :- ^ InputIteratorTransformer (44) - : : +- ShuffleQueryStage (42), Statistics(X) - : : +- ColumnarExchange (41) - : : +- VeloxResizeBatches (40) - : : +- ^ ProjectExecTransformer (38) - : : +- ^ FilterExecTransformer (37) - : : +- ^ FileSourceScanExecTransformer parquet (36) - : +- ^ InputIteratorTransformer (48) - : +- ShuffleQueryStage (46), Statistics(X) - : +- ReusedExchange (45) - +- ^ InputIteratorTransformer (84) - +- ShuffleQueryStage (82), Statistics(X) - +- ColumnarExchange (81) - +- VeloxResizeBatches (80) - +- ^ ProjectExecTransformer (78) - +- ^ FilterExecTransformer (77) - +- ^ FileSourceScanExecTransformer parquet (76) -+- == Initial Plan == - Sort (141) - +- Exchange (140) - +- Project (139) - +- SortMergeJoin Inner (138) - :- Sort (132) - : +- Exchange (131) - : +- Project (130) - : +- SortMergeJoin LeftSemi (129) - : :- Sort (96) - : : +- Exchange (95) - : : +- Filter (94) - : : +- Scan parquet (93) - : +- Sort (128) - : +- Exchange (127) - : +- Project (126) - : +- SortMergeJoin Inner (125) - : :- Sort (108) - : : +- Exchange (107) - : : +- SortMergeJoin LeftSemi (106) - : : :- Sort (100) - : : : +- Exchange (99) - : : : +- Filter (98) - : : : +- Scan parquet (97) - : : +- Sort (105) - : : +- Exchange (104) - : : +- Project (103) - : : +- Filter (102) - : : +- Scan parquet (101) - : +- Sort (124) - : +- Exchange (123) - : +- Filter (122) - : +- HashAggregate (121) - : +- HashAggregate (120) - : +- SortMergeJoin LeftSemi (119) - : :- Sort (113) - : : +- Exchange (112) - : : +- Project (111) - : : +- Filter (110) - : : +- Scan parquet (109) - : +- Sort (118) - : +- Exchange (117) - : +- Project (116) - : +- Filter (115) - : +- Scan parquet (114) - +- Sort (137) - +- Exchange (136) - +- Project (135) - +- Filter (134) - +- Scan parquet (133) - - -(1) FileSourceScanExecTransformer parquet -Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_nationkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Arguments: isnotnull(s_nationkey#X) - -(3) ProjectExecTransformer -Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] - -(4) WholeStageCodegenTransformer (X) -Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Arguments: false - -(5) VeloxResizeBatches -Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] - -(6) ColumnarExchange -Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(7) ShuffleQueryStage -Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Arguments: X - -(8) InputAdapter -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] - -(9) InputIteratorTransformer -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] - -(10) FileSourceScanExecTransformer parquet -Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] -ReadSchema: struct - -(11) FilterExecTransformer -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) - -(12) ProjectExecTransformer -Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] - -(13) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: false - -(14) VeloxResizeBatches -Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] - -(15) ColumnarExchange -Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [shuffle_writer_type=hash] - -(16) ShuffleQueryStage -Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: X - -(17) InputAdapter -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] - -(18) InputIteratorTransformer -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] - -(19) FileSourceScanExecTransformer parquet -Output [2]: [p_partkey#X, p_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] -ReadSchema: struct - -(20) FilterExecTransformer -Input [2]: [p_partkey#X, p_name#X] -Arguments: (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) - -(21) ProjectExecTransformer -Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] -Input [2]: [p_partkey#X, p_name#X] - -(22) WholeStageCodegenTransformer (X) -Input [2]: [hash_partition_key#X, p_partkey#X] -Arguments: false - -(23) VeloxResizeBatches -Input [2]: [hash_partition_key#X, p_partkey#X] - -(24) ColumnarExchange -Input [2]: [hash_partition_key#X, p_partkey#X] -Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(25) ShuffleQueryStage -Output [1]: [p_partkey#X] -Arguments: X - -(26) InputAdapter -Input [1]: [p_partkey#X] - -(27) InputIteratorTransformer -Input [1]: [p_partkey#X] - -(28) ShuffledHashJoinExecTransformer -Left keys [1]: [ps_partkey#X] -Right keys [1]: [p_partkey#X] -Join condition: None - -(29) ProjectExecTransformer -Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] - -(30) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: false - -(31) VeloxResizeBatches -Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] - -(32) ColumnarExchange -Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [shuffle_writer_type=hash] - -(33) ShuffleQueryStage -Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: X - -(34) InputAdapter -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] - -(35) InputIteratorTransformer -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] - -(36) FileSourceScanExecTransformer parquet -Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] -ReadSchema: struct - -(37) FilterExecTransformer -Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) - -(38) ProjectExecTransformer -Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] -Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] - -(39) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] -Arguments: false - -(40) VeloxResizeBatches -Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] - -(41) ColumnarExchange -Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] -Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [shuffle_writer_type=hash] - -(42) ShuffleQueryStage -Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -Arguments: X - -(43) InputAdapter -Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] - -(44) InputIteratorTransformer -Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] - -(45) ReusedExchange [Reuses operator id: 24] -Output [1]: [p_partkey#X] - -(46) ShuffleQueryStage -Output [1]: [p_partkey#X] -Arguments: X - -(47) InputAdapter -Input [1]: [p_partkey#X] - -(48) InputIteratorTransformer -Input [1]: [p_partkey#X] - -(49) ShuffledHashJoinExecTransformer -Left keys [1]: [l_partkey#X] -Right keys [1]: [p_partkey#X] -Join condition: None - -(50) RegularHashAggregateExecTransformer -Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -Keys [2]: [l_partkey#X, l_suppkey#X] -Functions [1]: [sum(l_quantity#X)] -Aggregate Attributes [1]: [sum(l_quantity#X)#X] -Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] - -(51) ProjectExecTransformer -Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] - -(52) FilterExecTransformer -Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -Arguments: isnotnull((0.5 * sum(l_quantity))#X) - -(53) ProjectExecTransformer -Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] - -(54) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -Arguments: false - -(55) VeloxResizeBatches -Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] - -(56) ColumnarExchange -Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(57) ShuffleQueryStage -Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -Arguments: X - -(58) InputAdapter -Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] - -(59) InputIteratorTransformer -Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] - -(60) ShuffledHashJoinExecTransformer -Left keys [2]: [ps_partkey#X, ps_suppkey#X] -Right keys [2]: [l_partkey#X, l_suppkey#X] -Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) - -(61) ProjectExecTransformer -Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] -Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] - -(62) WholeStageCodegenTransformer (X) -Input [2]: [hash_partition_key#X, ps_suppkey#X] -Arguments: false - -(63) VeloxResizeBatches -Input [2]: [hash_partition_key#X, ps_suppkey#X] - -(64) ColumnarExchange -Input [2]: [hash_partition_key#X, ps_suppkey#X] -Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(65) ShuffleQueryStage -Output [1]: [ps_suppkey#X] -Arguments: X - -(66) InputAdapter -Input [1]: [ps_suppkey#X] - -(67) InputIteratorTransformer -Input [1]: [ps_suppkey#X] - -(68) ShuffledHashJoinExecTransformer -Left keys [1]: [s_suppkey#X] -Right keys [1]: [ps_suppkey#X] -Join condition: None - -(69) ProjectExecTransformer -Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] - -(70) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] -Arguments: false - -(71) VeloxResizeBatches -Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] - -(72) ColumnarExchange -Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] -Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(73) ShuffleQueryStage -Output [3]: [s_name#X, s_address#X, s_nationkey#X] -Arguments: X - -(74) InputAdapter -Input [3]: [s_name#X, s_address#X, s_nationkey#X] - -(75) InputIteratorTransformer -Input [3]: [s_name#X, s_address#X, s_nationkey#X] - -(76) FileSourceScanExecTransformer parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] -ReadSchema: struct - -(77) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) - -(78) ProjectExecTransformer -Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] -Input [2]: [n_nationkey#X, n_name#X] - -(79) WholeStageCodegenTransformer (X) -Input [2]: [hash_partition_key#X, n_nationkey#X] -Arguments: false - -(80) VeloxResizeBatches -Input [2]: [hash_partition_key#X, n_nationkey#X] - -(81) ColumnarExchange -Input [2]: [hash_partition_key#X, n_nationkey#X] -Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(82) ShuffleQueryStage -Output [1]: [n_nationkey#X] -Arguments: X - -(83) InputAdapter -Input [1]: [n_nationkey#X] - -(84) InputIteratorTransformer -Input [1]: [n_nationkey#X] - -(85) ShuffledHashJoinExecTransformer -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(86) ProjectExecTransformer -Output [2]: [s_name#X, s_address#X] -Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] - -(87) WholeStageCodegenTransformer (X) -Input [2]: [s_name#X, s_address#X] -Arguments: false - -(88) VeloxResizeBatches -Input [2]: [s_name#X, s_address#X] - -(89) ColumnarExchange -Input [2]: [s_name#X, s_address#X] -Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(90) ShuffleQueryStage -Output [2]: [s_name#X, s_address#X] -Arguments: X - -(91) AQEShuffleRead -Input [2]: [s_name#X, s_address#X] -Arguments: local - -(92) VeloxColumnarToRow -Input [2]: [s_name#X, s_address#X] - -(93) Scan parquet -Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_nationkey)] -ReadSchema: struct - -(94) Filter -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Condition : isnotnull(s_nationkey#X) - -(95) Exchange -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(96) Sort -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(97) Scan parquet -Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] -ReadSchema: struct - -(98) Filter -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) - -(99) Exchange -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(100) Sort -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 - -(101) Scan parquet -Output [2]: [p_partkey#X, p_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] -ReadSchema: struct - -(102) Filter -Input [2]: [p_partkey#X, p_name#X] -Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) - -(103) Project -Output [1]: [p_partkey#X] -Input [2]: [p_partkey#X, p_name#X] - -(104) Exchange -Input [1]: [p_partkey#X] -Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(105) Sort -Input [1]: [p_partkey#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(106) SortMergeJoin -Left keys [1]: [ps_partkey#X] -Right keys [1]: [p_partkey#X] -Join condition: None - -(107) Exchange -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(108) Sort -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: [ps_partkey#X ASC NULLS FIRST, ps_suppkey#X ASC NULLS FIRST], false, 0 - -(109) Scan parquet -Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] -ReadSchema: struct - -(110) Filter -Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) - -(111) Project -Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] - -(112) Exchange -Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(113) Sort -Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(114) Scan parquet -Output [2]: [p_partkey#X, p_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] -ReadSchema: struct - -(115) Filter -Input [2]: [p_partkey#X, p_name#X] -Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) - -(116) Project -Output [1]: [p_partkey#X] -Input [2]: [p_partkey#X, p_name#X] - -(117) Exchange -Input [1]: [p_partkey#X] -Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(118) Sort -Input [1]: [p_partkey#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(119) SortMergeJoin -Left keys [1]: [l_partkey#X] -Right keys [1]: [p_partkey#X] -Join condition: None - -(120) HashAggregate -Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -Keys [2]: [l_partkey#X, l_suppkey#X] -Functions [1]: [partial_sum(l_quantity#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] - -(121) HashAggregate -Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -Keys [2]: [l_partkey#X, l_suppkey#X] -Functions [1]: [sum(l_quantity#X)] -Aggregate Attributes [1]: [sum(l_quantity#X)#X] -Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] - -(122) Filter -Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -Condition : isnotnull((0.5 * sum(l_quantity))#X) - -(123) Exchange -Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(124) Sort -Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -Arguments: [l_partkey#X ASC NULLS FIRST, l_suppkey#X ASC NULLS FIRST], false, 0 - -(125) SortMergeJoin -Left keys [2]: [ps_partkey#X, ps_suppkey#X] -Right keys [2]: [l_partkey#X, l_suppkey#X] -Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) - -(126) Project -Output [1]: [ps_suppkey#X] -Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] - -(127) Exchange -Input [1]: [ps_suppkey#X] -Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(128) Sort -Input [1]: [ps_suppkey#X] -Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 - -(129) SortMergeJoin -Left keys [1]: [s_suppkey#X] -Right keys [1]: [ps_suppkey#X] -Join condition: None - -(130) Project -Output [3]: [s_name#X, s_address#X, s_nationkey#X] -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] - -(131) Exchange -Input [3]: [s_name#X, s_address#X, s_nationkey#X] -Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(132) Sort -Input [3]: [s_name#X, s_address#X, s_nationkey#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 - -(133) Scan parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] -ReadSchema: struct - -(134) Filter -Input [2]: [n_nationkey#X, n_name#X] -Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) - -(135) Project -Output [1]: [n_nationkey#X] -Input [2]: [n_nationkey#X, n_name#X] - -(136) Exchange -Input [1]: [n_nationkey#X] -Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(137) Sort -Input [1]: [n_nationkey#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 - -(138) SortMergeJoin -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(139) Project -Output [2]: [s_name#X, s_address#X] -Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] - -(140) Exchange -Input [2]: [s_name#X, s_address#X] -Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(141) Sort -Input [2]: [s_name#X, s_address#X] -Arguments: [s_name#X ASC NULLS FIRST], true, 0 - -(142) AdaptiveSparkPlan -Output [2]: [s_name#X, s_address#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt deleted file mode 100644 index fbe2f5acc84..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt +++ /dev/null @@ -1,693 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (137) -+- == Final Plan == - VeloxColumnarToRow (91) - +- ^ RegularHashAggregateExecTransformer (89) - +- ^ InputIteratorTransformer (88) - +- ShuffleQueryStage (86), Statistics(X) - +- ColumnarExchange (85) - +- VeloxResizeBatches (84) - +- ^ ProjectExecTransformer (82) - +- ^ FlushableHashAggregateExecTransformer (81) - +- ^ ProjectExecTransformer (80) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) - :- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67), Statistics(X) - : +- ColumnarExchange (66) - : +- VeloxResizeBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50), Statistics(X) - : : +- ColumnarExchange (49) - : : +- VeloxResizeBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) - : : :- ^ InputIteratorTransformer (9) - : : : +- ShuffleQueryStage (7), Statistics(X) - : : : +- ColumnarExchange (6) - : : : +- VeloxResizeBatches (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ FileSourceScanExecTransformer parquet (1) - : : +- ^ InputIteratorTransformer (44) - : : +- ShuffleQueryStage (42), Statistics(X) - : : +- ColumnarExchange (41) - : : +- VeloxResizeBatches (40) - : : +- ^ ProjectExecTransformer (38) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) - : : : :- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16), Statistics(X) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxResizeBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ FilterExecTransformer (11) - : : : : +- ^ FileSourceScanExecTransformer parquet (10) - : : : +- ^ InputIteratorTransformer (26) - : : : +- ShuffleQueryStage (24), Statistics(X) - : : : +- ColumnarExchange (23) - : : : +- VeloxResizeBatches (22) - : : : +- ^ ProjectExecTransformer (20) - : : : +- ^ FileSourceScanExecTransformer parquet (19) - : : +- ^ InputIteratorTransformer (36) - : : +- ShuffleQueryStage (34), Statistics(X) - : : +- ColumnarExchange (33) - : : +- VeloxResizeBatches (32) - : : +- ^ ProjectExecTransformer (30) - : : +- ^ FilterExecTransformer (29) - : : +- ^ FileSourceScanExecTransformer parquet (28) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- VeloxResizeBatches (57) - : +- ^ ProjectExecTransformer (55) - : +- ^ FilterExecTransformer (54) - : +- ^ FileSourceScanExecTransformer parquet (53) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- VeloxResizeBatches (74) - +- ^ ProjectExecTransformer (72) - +- ^ FilterExecTransformer (71) - +- ^ FileSourceScanExecTransformer parquet (70) -+- == Initial Plan == - TakeOrderedAndProject (136) - +- HashAggregate (135) - +- Exchange (134) - +- HashAggregate (133) - +- Project (132) - +- SortMergeJoin Inner (131) - :- Sort (125) - : +- Exchange (124) - : +- Project (123) - : +- SortMergeJoin Inner (122) - : :- Sort (116) - : : +- Exchange (115) - : : +- Project (114) - : : +- SortMergeJoin Inner (113) - : : :- Sort (95) - : : : +- Exchange (94) - : : : +- Filter (93) - : : : +- Scan parquet (92) - : : +- Sort (112) - : : +- Exchange (111) - : : +- SortMergeJoin LeftAnti (110) - : : :- SortMergeJoin LeftSemi (104) - : : : :- Sort (100) - : : : : +- Exchange (99) - : : : : +- Project (98) - : : : : +- Filter (97) - : : : : +- Scan parquet (96) - : : : +- Sort (103) - : : : +- Exchange (102) - : : : +- Scan parquet (101) - : : +- Sort (109) - : : +- Exchange (108) - : : +- Project (107) - : : +- Filter (106) - : : +- Scan parquet (105) - : +- Sort (121) - : +- Exchange (120) - : +- Project (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Sort (130) - +- Exchange (129) - +- Project (128) - +- Filter (127) - +- Scan parquet (126) - - -(1) FileSourceScanExecTransformer parquet -Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(3) ProjectExecTransformer -Output [4]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] -Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] - -(4) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] -Arguments: false - -(5) VeloxResizeBatches -Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] - -(6) ColumnarExchange -Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] -Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(7) ShuffleQueryStage -Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -Arguments: X - -(8) InputAdapter -Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] - -(9) InputIteratorTransformer -Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] - -(10) FileSourceScanExecTransformer parquet -Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] -ReadSchema: struct - -(11) FilterExecTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Arguments: ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(12) ProjectExecTransformer -Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] - -(13) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] -Arguments: false - -(14) VeloxResizeBatches -Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] - -(15) ColumnarExchange -Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(16) ShuffleQueryStage -Output [2]: [l_orderkey#X, l_suppkey#X] -Arguments: X - -(17) InputAdapter -Input [2]: [l_orderkey#X, l_suppkey#X] - -(18) InputIteratorTransformer -Input [2]: [l_orderkey#X, l_suppkey#X] - -(19) FileSourceScanExecTransformer parquet -Output [2]: [l_orderkey#X, l_suppkey#X] -Batched: true -Location: InMemoryFileIndex [*] -ReadSchema: struct - -(20) ProjectExecTransformer -Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] -Input [2]: [l_orderkey#X, l_suppkey#X] - -(21) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] -Arguments: false - -(22) VeloxResizeBatches -Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] - -(23) ColumnarExchange -Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(24) ShuffleQueryStage -Output [2]: [l_orderkey#X, l_suppkey#X] -Arguments: X - -(25) InputAdapter -Input [2]: [l_orderkey#X, l_suppkey#X] - -(26) InputIteratorTransformer -Input [2]: [l_orderkey#X, l_suppkey#X] - -(27) ShuffledHashJoinExecTransformer -Left keys [1]: [l_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: NOT (l_suppkey#X = l_suppkey#X) - -(28) FileSourceScanExecTransformer parquet -Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] -ReadSchema: struct - -(29) FilterExecTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Arguments: ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) - -(30) ProjectExecTransformer -Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] - -(31) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] -Arguments: false - -(32) VeloxResizeBatches -Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] - -(33) ColumnarExchange -Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(34) ShuffleQueryStage -Output [2]: [l_orderkey#X, l_suppkey#X] -Arguments: X - -(35) InputAdapter -Input [2]: [l_orderkey#X, l_suppkey#X] - -(36) InputIteratorTransformer -Input [2]: [l_orderkey#X, l_suppkey#X] - -(37) ShuffledHashJoinExecTransformer -Left keys [1]: [l_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: NOT (l_suppkey#X = l_suppkey#X) - -(38) ProjectExecTransformer -Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] -Input [2]: [l_orderkey#X, l_suppkey#X] - -(39) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] -Arguments: false - -(40) VeloxResizeBatches -Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] - -(41) ColumnarExchange -Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] -Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(42) ShuffleQueryStage -Output [2]: [l_orderkey#X, l_suppkey#X] -Arguments: X - -(43) InputAdapter -Input [2]: [l_orderkey#X, l_suppkey#X] - -(44) InputIteratorTransformer -Input [2]: [l_orderkey#X, l_suppkey#X] - -(45) ShuffledHashJoinExecTransformer -Left keys [1]: [s_suppkey#X] -Right keys [1]: [l_suppkey#X] -Join condition: None - -(46) ProjectExecTransformer -Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] -Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] - -(47) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] -Arguments: false - -(48) VeloxResizeBatches -Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] - -(49) ColumnarExchange -Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(50) ShuffleQueryStage -Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -Arguments: X - -(51) InputAdapter -Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] - -(52) InputIteratorTransformer -Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] - -(53) FileSourceScanExecTransformer parquet -Output [2]: [o_orderkey#X, o_orderstatus#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] -ReadSchema: struct - -(54) FilterExecTransformer -Input [2]: [o_orderkey#X, o_orderstatus#X] -Arguments: ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) - -(55) ProjectExecTransformer -Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] -Input [2]: [o_orderkey#X, o_orderstatus#X] - -(56) WholeStageCodegenTransformer (X) -Input [2]: [hash_partition_key#X, o_orderkey#X] -Arguments: false - -(57) VeloxResizeBatches -Input [2]: [hash_partition_key#X, o_orderkey#X] - -(58) ColumnarExchange -Input [2]: [hash_partition_key#X, o_orderkey#X] -Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(59) ShuffleQueryStage -Output [1]: [o_orderkey#X] -Arguments: X - -(60) InputAdapter -Input [1]: [o_orderkey#X] - -(61) InputIteratorTransformer -Input [1]: [o_orderkey#X] - -(62) ShuffledHashJoinExecTransformer -Left keys [1]: [l_orderkey#X] -Right keys [1]: [o_orderkey#X] -Join condition: None - -(63) ProjectExecTransformer -Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] -Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] - -(64) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] -Arguments: false - -(65) VeloxResizeBatches -Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] - -(66) ColumnarExchange -Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] -Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(67) ShuffleQueryStage -Output [2]: [s_name#X, s_nationkey#X] -Arguments: X - -(68) InputAdapter -Input [2]: [s_name#X, s_nationkey#X] - -(69) InputIteratorTransformer -Input [2]: [s_name#X, s_nationkey#X] - -(70) FileSourceScanExecTransformer parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] -ReadSchema: struct - -(71) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) - -(72) ProjectExecTransformer -Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] -Input [2]: [n_nationkey#X, n_name#X] - -(73) WholeStageCodegenTransformer (X) -Input [2]: [hash_partition_key#X, n_nationkey#X] -Arguments: false - -(74) VeloxResizeBatches -Input [2]: [hash_partition_key#X, n_nationkey#X] - -(75) ColumnarExchange -Input [2]: [hash_partition_key#X, n_nationkey#X] -Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(76) ShuffleQueryStage -Output [1]: [n_nationkey#X] -Arguments: X - -(77) InputAdapter -Input [1]: [n_nationkey#X] - -(78) InputIteratorTransformer -Input [1]: [n_nationkey#X] - -(79) ShuffledHashJoinExecTransformer -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(80) ProjectExecTransformer -Output [1]: [s_name#X] -Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] - -(81) FlushableHashAggregateExecTransformer -Input [1]: [s_name#X] -Keys [1]: [s_name#X] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#X] -Results [2]: [s_name#X, count#X] - -(82) ProjectExecTransformer -Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] -Input [2]: [s_name#X, count#X] - -(83) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, s_name#X, count#X] -Arguments: false - -(84) VeloxResizeBatches -Input [3]: [hash_partition_key#X, s_name#X, count#X] - -(85) ColumnarExchange -Input [3]: [hash_partition_key#X, s_name#X, count#X] -Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [shuffle_writer_type=hash] - -(86) ShuffleQueryStage -Output [2]: [s_name#X, count#X] -Arguments: X - -(87) InputAdapter -Input [2]: [s_name#X, count#X] - -(88) InputIteratorTransformer -Input [2]: [s_name#X, count#X] - -(89) RegularHashAggregateExecTransformer -Input [2]: [s_name#X, count#X] -Keys [1]: [s_name#X] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#X] -Results [2]: [s_name#X, count(1)#X AS numwait#X] - -(90) WholeStageCodegenTransformer (X) -Input [2]: [s_name#X, numwait#X] -Arguments: false - -(91) VeloxColumnarToRow -Input [2]: [s_name#X, numwait#X] - -(92) Scan parquet -Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] -ReadSchema: struct - -(93) Filter -Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(94) Exchange -Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(95) Sort -Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(96) Scan parquet -Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] -ReadSchema: struct - -(97) Filter -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(98) Project -Output [2]: [l_orderkey#X, l_suppkey#X] -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] - -(99) Exchange -Input [2]: [l_orderkey#X, l_suppkey#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(100) Sort -Input [2]: [l_orderkey#X, l_suppkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(101) Scan parquet -Output [2]: [l_orderkey#X, l_suppkey#X] -Batched: true -Location: InMemoryFileIndex [*] -ReadSchema: struct - -(102) Exchange -Input [2]: [l_orderkey#X, l_suppkey#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(103) Sort -Input [2]: [l_orderkey#X, l_suppkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(104) SortMergeJoin -Left keys [1]: [l_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: NOT (l_suppkey#X = l_suppkey#X) - -(105) Scan parquet -Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] -ReadSchema: struct - -(106) Filter -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) - -(107) Project -Output [2]: [l_orderkey#X, l_suppkey#X] -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] - -(108) Exchange -Input [2]: [l_orderkey#X, l_suppkey#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(109) Sort -Input [2]: [l_orderkey#X, l_suppkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(110) SortMergeJoin -Left keys [1]: [l_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: NOT (l_suppkey#X = l_suppkey#X) - -(111) Exchange -Input [2]: [l_orderkey#X, l_suppkey#X] -Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(112) Sort -Input [2]: [l_orderkey#X, l_suppkey#X] -Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 - -(113) SortMergeJoin -Left keys [1]: [s_suppkey#X] -Right keys [1]: [l_suppkey#X] -Join condition: None - -(114) Project -Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] - -(115) Exchange -Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(116) Sort -Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(117) Scan parquet -Output [2]: [o_orderkey#X, o_orderstatus#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] -ReadSchema: struct - -(118) Filter -Input [2]: [o_orderkey#X, o_orderstatus#X] -Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) - -(119) Project -Output [1]: [o_orderkey#X] -Input [2]: [o_orderkey#X, o_orderstatus#X] - -(120) Exchange -Input [1]: [o_orderkey#X] -Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(121) Sort -Input [1]: [o_orderkey#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(122) SortMergeJoin -Left keys [1]: [l_orderkey#X] -Right keys [1]: [o_orderkey#X] -Join condition: None - -(123) Project -Output [2]: [s_name#X, s_nationkey#X] -Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] - -(124) Exchange -Input [2]: [s_name#X, s_nationkey#X] -Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(125) Sort -Input [2]: [s_name#X, s_nationkey#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 - -(126) Scan parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] -ReadSchema: struct - -(127) Filter -Input [2]: [n_nationkey#X, n_name#X] -Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) - -(128) Project -Output [1]: [n_nationkey#X] -Input [2]: [n_nationkey#X, n_name#X] - -(129) Exchange -Input [1]: [n_nationkey#X] -Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(130) Sort -Input [1]: [n_nationkey#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 - -(131) SortMergeJoin -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(132) Project -Output [1]: [s_name#X] -Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] - -(133) HashAggregate -Input [1]: [s_name#X] -Keys [1]: [s_name#X] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#X] -Results [2]: [s_name#X, count#X] - -(134) Exchange -Input [2]: [s_name#X, count#X] -Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(135) HashAggregate -Input [2]: [s_name#X, count#X] -Keys [1]: [s_name#X] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#X] -Results [2]: [s_name#X, count(1)#X AS numwait#X] - -(136) TakeOrderedAndProject -Input [2]: [s_name#X, numwait#X] -Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] - -(137) AdaptiveSparkPlan -Output [2]: [s_name#X, numwait#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt deleted file mode 100644 index a9d590440b6..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt +++ /dev/null @@ -1,405 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (52) -+- == Final Plan == - VeloxColumnarToRow (37) - +- ^ SortExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ShuffleQueryStage (32), Statistics(X) - +- ColumnarExchange (31) - +- VeloxResizeBatches (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- VeloxResizeBatches (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7), Statistics(X) - : +- ColumnarExchange (6) - : +- VeloxResizeBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ FileSourceScanExecTransformer parquet (1) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- VeloxResizeBatches (13) - +- ^ ProjectExecTransformer (11) - +- ^ FileSourceScanExecTransformer parquet (10) -+- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- Project (46) - +- SortMergeJoin LeftAnti (45) - :- Sort (41) - : +- Exchange (40) - : +- Filter (39) - : +- Scan parquet (38) - +- Sort (44) - +- Exchange (43) - +- Scan parquet (42) - - -(1) FileSourceScanExecTransformer parquet -Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_acctbal)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -Arguments: ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) - -(3) ProjectExecTransformer -Output [4]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] -Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] - -(4) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] -Arguments: false - -(5) VeloxResizeBatches -Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] - -(6) ColumnarExchange -Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] -Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [shuffle_writer_type=hash] - -(7) ShuffleQueryStage -Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -Arguments: X - -(8) InputAdapter -Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] - -(9) InputIteratorTransformer -Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] - -(10) FileSourceScanExecTransformer parquet -Output [1]: [o_custkey#X] -Batched: true -Location: InMemoryFileIndex [*] -ReadSchema: struct - -(11) ProjectExecTransformer -Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] -Input [1]: [o_custkey#X] - -(12) WholeStageCodegenTransformer (X) -Input [2]: [hash_partition_key#X, o_custkey#X] -Arguments: false - -(13) VeloxResizeBatches -Input [2]: [hash_partition_key#X, o_custkey#X] - -(14) ColumnarExchange -Input [2]: [hash_partition_key#X, o_custkey#X] -Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(15) ShuffleQueryStage -Output [1]: [o_custkey#X] -Arguments: X - -(16) InputAdapter -Input [1]: [o_custkey#X] - -(17) InputIteratorTransformer -Input [1]: [o_custkey#X] - -(18) ShuffledHashJoinExecTransformer -Left keys [1]: [c_custkey#X] -Right keys [1]: [o_custkey#X] -Join condition: None - -(19) ProjectExecTransformer -Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] -Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] - -(20) FlushableHashAggregateExecTransformer -Input [2]: [cntrycode#X, c_acctbal#X] -Keys [1]: [cntrycode#X] -Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] -Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] -Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] - -(21) ProjectExecTransformer -Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] -Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] - -(22) WholeStageCodegenTransformer (X) -Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] -Arguments: false - -(23) VeloxResizeBatches -Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] - -(24) ColumnarExchange -Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] - -(25) ShuffleQueryStage -Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -Arguments: X - -(26) InputAdapter -Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] - -(27) InputIteratorTransformer -Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] - -(28) RegularHashAggregateExecTransformer -Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -Keys [1]: [cntrycode#X] -Functions [2]: [count(1), sum(c_acctbal#X)] -Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] -Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] - -(29) WholeStageCodegenTransformer (X) -Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -Arguments: false - -(30) VeloxResizeBatches -Input [3]: [cntrycode#X, numcust#X, totacctbal#X] - -(31) ColumnarExchange -Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(32) ShuffleQueryStage -Output [3]: [cntrycode#X, numcust#X, totacctbal#X] -Arguments: X - -(33) InputAdapter -Input [3]: [cntrycode#X, numcust#X, totacctbal#X] - -(34) InputIteratorTransformer -Input [3]: [cntrycode#X, numcust#X, totacctbal#X] - -(35) SortExecTransformer -Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 - -(36) WholeStageCodegenTransformer (X) -Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -Arguments: false - -(37) VeloxColumnarToRow -Input [3]: [cntrycode#X, numcust#X, totacctbal#X] - -(38) Scan parquet -Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_acctbal)] -ReadSchema: struct - -(39) Filter -Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) - -(40) Exchange -Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(41) Sort -Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(42) Scan parquet -Output [1]: [o_custkey#X] -Batched: true -Location: InMemoryFileIndex [*] -ReadSchema: struct - -(43) Exchange -Input [1]: [o_custkey#X] -Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(44) Sort -Input [1]: [o_custkey#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(45) SortMergeJoin -Left keys [1]: [c_custkey#X] -Right keys [1]: [o_custkey#X] -Join condition: None - -(46) Project -Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] -Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] - -(47) HashAggregate -Input [2]: [cntrycode#X, c_acctbal#X] -Keys [1]: [cntrycode#X] -Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] -Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] -Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] - -(48) Exchange -Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(49) HashAggregate -Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -Keys [1]: [cntrycode#X] -Functions [2]: [count(1), sum(c_acctbal#X)] -Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] -Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] - -(50) Exchange -Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(51) Sort -Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 - -(52) AdaptiveSparkPlan -Output [3]: [cntrycode#X, numcust#X, totacctbal#X] -Arguments: isFinalPlan=true - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (72) -+- == Final Plan == - VeloxColumnarToRow (65) - +- ^ RegularHashAggregateExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ShuffleQueryStage (60), Statistics(X) - +- ColumnarExchange (59) - +- VeloxResizeBatches (58) - +- ^ FlushableHashAggregateExecTransformer (56) - +- ^ ProjectExecTransformer (55) - +- ^ FilterExecTransformer (54) - +- ^ FileSourceScanExecTransformer parquet (53) -+- == Initial Plan == - HashAggregate (71) - +- Exchange (70) - +- HashAggregate (69) - +- Project (68) - +- Filter (67) - +- Scan parquet (66) - - -(53) FileSourceScanExecTransformer parquet -Output [2]: [c_phone#X, c_acctbal#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] -ReadSchema: struct - -(54) FilterExecTransformer -Input [2]: [c_phone#X, c_acctbal#X] -Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) - -(55) ProjectExecTransformer -Output [1]: [c_acctbal#X] -Input [2]: [c_phone#X, c_acctbal#X] - -(56) FlushableHashAggregateExecTransformer -Input [1]: [c_acctbal#X] -Keys: [] -Functions [1]: [partial_avg(c_acctbal#X)] -Aggregate Attributes [2]: [sum#X, count#X] -Results [2]: [sum#X, count#X] - -(57) WholeStageCodegenTransformer (X) -Input [2]: [sum#X, count#X] -Arguments: false - -(58) VeloxResizeBatches -Input [2]: [sum#X, count#X] - -(59) ColumnarExchange -Input [2]: [sum#X, count#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(60) ShuffleQueryStage -Output [2]: [sum#X, count#X] -Arguments: X - -(61) InputAdapter -Input [2]: [sum#X, count#X] - -(62) InputIteratorTransformer -Input [2]: [sum#X, count#X] - -(63) RegularHashAggregateExecTransformer -Input [2]: [sum#X, count#X] -Keys: [] -Functions [1]: [avg(c_acctbal#X)] -Aggregate Attributes [1]: [avg(c_acctbal#X)#X] -Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] - -(64) WholeStageCodegenTransformer (X) -Input [1]: [avg(c_acctbal)#X] -Arguments: false - -(65) VeloxColumnarToRow -Input [1]: [avg(c_acctbal)#X] - -(66) Scan parquet -Output [2]: [c_phone#X, c_acctbal#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] -ReadSchema: struct - -(67) Filter -Input [2]: [c_phone#X, c_acctbal#X] -Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) - -(68) Project -Output [1]: [c_acctbal#X] -Input [2]: [c_phone#X, c_acctbal#X] - -(69) HashAggregate -Input [1]: [c_acctbal#X] -Keys: [] -Functions [1]: [partial_avg(c_acctbal#X)] -Aggregate Attributes [2]: [sum#X, count#X] -Results [2]: [sum#X, count#X] - -(70) Exchange -Input [2]: [sum#X, count#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] - -(71) HashAggregate -Input [2]: [sum#X, count#X] -Keys: [] -Functions [1]: [avg(c_acctbal#X)] -Aggregate Attributes [1]: [avg(c_acctbal#X)#X] -Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] - -(72) AdaptiveSparkPlan -Output [1]: [avg(c_acctbal)#X] -Arguments: isFinalPlan=true - -Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (72) -+- == Final Plan == - VeloxColumnarToRow (65) - +- ^ RegularHashAggregateExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ShuffleQueryStage (60), Statistics(X) - +- ColumnarExchange (59) - +- VeloxResizeBatches (58) - +- ^ FlushableHashAggregateExecTransformer (56) - +- ^ ProjectExecTransformer (55) - +- ^ FilterExecTransformer (54) - +- ^ FileSourceScanExecTransformer parquet (53) -+- == Initial Plan == - HashAggregate (71) - +- Exchange (70) - +- HashAggregate (69) - +- Project (68) - +- Filter (67) - +- Scan parquet (66) \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt deleted file mode 100644 index 3b400d8678c..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt +++ /dev/null @@ -1,343 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (66) -+- == Final Plan == - VeloxColumnarToRow (42) - +- TakeOrderedAndProjectExecTransformer (41) - +- ^ ProjectExecTransformer (39) - +- ^ RegularHashAggregateExecTransformer (38) - +- ^ ProjectExecTransformer (37) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) - :- ^ InputIteratorTransformer (26) - : +- ShuffleQueryStage (24), Statistics(X) - : +- ColumnarExchange (23) - : +- VeloxResizeBatches (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : :- ^ InputIteratorTransformer (9) - : : +- ShuffleQueryStage (7), Statistics(X) - : : +- ColumnarExchange (6) - : : +- VeloxResizeBatches (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ FileSourceScanExecTransformer parquet (1) - : +- ^ InputIteratorTransformer (18) - : +- ShuffleQueryStage (16), Statistics(X) - : +- ColumnarExchange (15) - : +- VeloxResizeBatches (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FilterExecTransformer (11) - : +- ^ FileSourceScanExecTransformer parquet (10) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- VeloxResizeBatches (31) - +- ^ ProjectExecTransformer (29) - +- ^ FilterExecTransformer (28) - +- ^ FileSourceScanExecTransformer parquet (27) -+- == Initial Plan == - TakeOrderedAndProject (65) - +- HashAggregate (64) - +- HashAggregate (63) - +- Project (62) - +- SortMergeJoin Inner (61) - :- Sort (55) - : +- Exchange (54) - : +- Project (53) - : +- SortMergeJoin Inner (52) - : :- Sort (47) - : : +- Exchange (46) - : : +- Project (45) - : : +- Filter (44) - : : +- Scan parquet (43) - : +- Sort (51) - : +- Exchange (50) - : +- Filter (49) - : +- Scan parquet (48) - +- Sort (60) - +- Exchange (59) - +- Project (58) - +- Filter (57) - +- Scan parquet (56) - - -(1) FileSourceScanExecTransformer parquet -Output [2]: [c_custkey#X, c_mktsegment#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [2]: [c_custkey#X, c_mktsegment#X] -Arguments: ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) - -(3) ProjectExecTransformer -Output [2]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X] -Input [2]: [c_custkey#X, c_mktsegment#X] - -(4) WholeStageCodegenTransformer (X) -Input [2]: [hash_partition_key#X, c_custkey#X] -Arguments: false - -(5) VeloxResizeBatches -Input [2]: [hash_partition_key#X, c_custkey#X] - -(6) ColumnarExchange -Input [2]: [hash_partition_key#X, c_custkey#X] -Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(7) ShuffleQueryStage -Output [1]: [c_custkey#X] -Arguments: X - -(8) InputAdapter -Input [1]: [c_custkey#X] - -(9) InputIteratorTransformer -Input [1]: [c_custkey#X] - -(10) FileSourceScanExecTransformer parquet -Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] -ReadSchema: struct - -(11) FilterExecTransformer -Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -Arguments: (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(12) ProjectExecTransformer -Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] - -(13) WholeStageCodegenTransformer (X) -Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -Arguments: false - -(14) VeloxResizeBatches -Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] - -(15) ColumnarExchange -Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [shuffle_writer_type=hash] - -(16) ShuffleQueryStage -Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -Arguments: X - -(17) InputAdapter -Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] - -(18) InputIteratorTransformer -Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] - -(19) ShuffledHashJoinExecTransformer -Left keys [1]: [c_custkey#X] -Right keys [1]: [o_custkey#X] -Join condition: None - -(20) ProjectExecTransformer -Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] -Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] - -(21) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] -Arguments: false - -(22) VeloxResizeBatches -Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] - -(23) ColumnarExchange -Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] -Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [shuffle_writer_type=hash] - -(24) ShuffleQueryStage -Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -Arguments: X - -(25) InputAdapter -Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] - -(26) InputIteratorTransformer -Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] - -(27) FileSourceScanExecTransformer parquet -Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] -ReadSchema: struct - -(28) FilterExecTransformer -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) - -(29) ProjectExecTransformer -Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(30) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -Arguments: false - -(31) VeloxResizeBatches -Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] - -(32) ColumnarExchange -Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] - -(33) ShuffleQueryStage -Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -Arguments: X - -(34) InputAdapter -Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] - -(35) InputIteratorTransformer -Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] - -(36) ShuffledHashJoinExecTransformer -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(37) ProjectExecTransformer -Output [4]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] -Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] - -(38) RegularHashAggregateExecTransformer -Input [4]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, _pre_X#X] -Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] -Functions [1]: [sum(_pre_X#X)] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] - -(39) ProjectExecTransformer -Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] - -(40) WholeStageCodegenTransformer (X) -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: false - -(41) TakeOrderedAndProjectExecTransformer -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 - -(42) VeloxColumnarToRow -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] - -(43) Scan parquet -Output [2]: [c_custkey#X, c_mktsegment#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] -ReadSchema: struct - -(44) Filter -Input [2]: [c_custkey#X, c_mktsegment#X] -Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) - -(45) Project -Output [1]: [c_custkey#X] -Input [2]: [c_custkey#X, c_mktsegment#X] - -(46) Exchange -Input [1]: [c_custkey#X] -Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(47) Sort -Input [1]: [c_custkey#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(48) Scan parquet -Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] -ReadSchema: struct - -(49) Filter -Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(50) Exchange -Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(51) Sort -Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(52) SortMergeJoin -Left keys [1]: [c_custkey#X] -Right keys [1]: [o_custkey#X] -Join condition: None - -(53) Project -Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] - -(54) Exchange -Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(55) Sort -Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(56) Scan parquet -Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] -ReadSchema: struct - -(57) Filter -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) - -(58) Project -Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(59) Exchange -Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(60) Sort -Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(61) SortMergeJoin -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(62) Project -Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] - -(63) HashAggregate -Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] -Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] - -(64) HashAggregate -Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] -Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] - -(65) TakeOrderedAndProject -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] - -(66) AdaptiveSparkPlan -Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt deleted file mode 100644 index 31370029a35..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt +++ /dev/null @@ -1,288 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (56) -+- == Final Plan == - VeloxColumnarToRow (38) - +- ^ SortExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- VeloxResizeBatches (31) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- VeloxResizeBatches (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) - :- ^ InputIteratorTransformer (9) - : +- ShuffleQueryStage (7), Statistics(X) - : +- ColumnarExchange (6) - : +- VeloxResizeBatches (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ FileSourceScanExecTransformer parquet (1) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- VeloxResizeBatches (14) - +- ^ ProjectExecTransformer (12) - +- ^ FilterExecTransformer (11) - +- ^ FileSourceScanExecTransformer parquet (10) -+- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- Project (50) - +- SortMergeJoin LeftSemi (49) - :- Sort (43) - : +- Exchange (42) - : +- Project (41) - : +- Filter (40) - : +- Scan parquet (39) - +- Sort (48) - +- Exchange (47) - +- Project (46) - +- Filter (45) - +- Scan parquet (44) - - -(1) FileSourceScanExecTransformer parquet -Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -Arguments: ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) - -(3) ProjectExecTransformer -Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderpriority#X] -Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] - -(4) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] -Arguments: false - -(5) VeloxResizeBatches -Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] - -(6) ColumnarExchange -Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] -Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [shuffle_writer_type=hash] - -(7) ShuffleQueryStage -Output [2]: [o_orderkey#X, o_orderpriority#X] -Arguments: X - -(8) InputAdapter -Input [2]: [o_orderkey#X, o_orderpriority#X] - -(9) InputIteratorTransformer -Input [2]: [o_orderkey#X, o_orderpriority#X] - -(10) FileSourceScanExecTransformer parquet -Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] -ReadSchema: struct - -(11) FilterExecTransformer -Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -Arguments: ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) - -(12) ProjectExecTransformer -Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] -Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] - -(13) WholeStageCodegenTransformer (X) -Input [2]: [hash_partition_key#X, l_orderkey#X] -Arguments: false - -(14) VeloxResizeBatches -Input [2]: [hash_partition_key#X, l_orderkey#X] - -(15) ColumnarExchange -Input [2]: [hash_partition_key#X, l_orderkey#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(16) ShuffleQueryStage -Output [1]: [l_orderkey#X] -Arguments: X - -(17) InputAdapter -Input [1]: [l_orderkey#X] - -(18) InputIteratorTransformer -Input [1]: [l_orderkey#X] - -(19) ShuffledHashJoinExecTransformer -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(20) ProjectExecTransformer -Output [1]: [o_orderpriority#X] -Input [2]: [o_orderkey#X, o_orderpriority#X] - -(21) FlushableHashAggregateExecTransformer -Input [1]: [o_orderpriority#X] -Keys [1]: [o_orderpriority#X] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#X] -Results [2]: [o_orderpriority#X, count#X] - -(22) ProjectExecTransformer -Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] -Input [2]: [o_orderpriority#X, count#X] - -(23) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] -Arguments: false - -(24) VeloxResizeBatches -Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] - -(25) ColumnarExchange -Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] -Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [shuffle_writer_type=hash] - -(26) ShuffleQueryStage -Output [2]: [o_orderpriority#X, count#X] -Arguments: X - -(27) InputAdapter -Input [2]: [o_orderpriority#X, count#X] - -(28) InputIteratorTransformer -Input [2]: [o_orderpriority#X, count#X] - -(29) RegularHashAggregateExecTransformer -Input [2]: [o_orderpriority#X, count#X] -Keys [1]: [o_orderpriority#X] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#X] -Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] - -(30) WholeStageCodegenTransformer (X) -Input [2]: [o_orderpriority#X, order_count#X] -Arguments: false - -(31) VeloxResizeBatches -Input [2]: [o_orderpriority#X, order_count#X] - -(32) ColumnarExchange -Input [2]: [o_orderpriority#X, order_count#X] -Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(33) ShuffleQueryStage -Output [2]: [o_orderpriority#X, order_count#X] -Arguments: X - -(34) InputAdapter -Input [2]: [o_orderpriority#X, order_count#X] - -(35) InputIteratorTransformer -Input [2]: [o_orderpriority#X, order_count#X] - -(36) SortExecTransformer -Input [2]: [o_orderpriority#X, order_count#X] -Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 - -(37) WholeStageCodegenTransformer (X) -Input [2]: [o_orderpriority#X, order_count#X] -Arguments: false - -(38) VeloxColumnarToRow -Input [2]: [o_orderpriority#X, order_count#X] - -(39) Scan parquet -Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] -ReadSchema: struct - -(40) Filter -Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) - -(41) Project -Output [2]: [o_orderkey#X, o_orderpriority#X] -Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] - -(42) Exchange -Input [2]: [o_orderkey#X, o_orderpriority#X] -Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(43) Sort -Input [2]: [o_orderkey#X, o_orderpriority#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(44) Scan parquet -Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] -ReadSchema: struct - -(45) Filter -Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) - -(46) Project -Output [1]: [l_orderkey#X] -Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] - -(47) Exchange -Input [1]: [l_orderkey#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(48) Sort -Input [1]: [l_orderkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(49) SortMergeJoin -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(50) Project -Output [1]: [o_orderpriority#X] -Input [2]: [o_orderkey#X, o_orderpriority#X] - -(51) HashAggregate -Input [1]: [o_orderpriority#X] -Keys [1]: [o_orderpriority#X] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#X] -Results [2]: [o_orderpriority#X, count#X] - -(52) Exchange -Input [2]: [o_orderpriority#X, count#X] -Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(53) HashAggregate -Input [2]: [o_orderpriority#X, count#X] -Keys [1]: [o_orderpriority#X] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#X] -Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] - -(54) Exchange -Input [2]: [o_orderpriority#X, order_count#X] -Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(55) Sort -Input [2]: [o_orderpriority#X, order_count#X] -Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 - -(56) AdaptiveSparkPlan -Output [2]: [o_orderpriority#X, order_count#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt deleted file mode 100644 index 112cbb89cec..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt +++ /dev/null @@ -1,780 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (156) -+- == Final Plan == - VeloxColumnarToRow (106) - +- ^ SortExecTransformer (104) - +- ^ InputIteratorTransformer (103) - +- ShuffleQueryStage (101), Statistics(X) - +- ColumnarExchange (100) - +- VeloxResizeBatches (99) - +- ^ RegularHashAggregateExecTransformer (97) - +- ^ InputIteratorTransformer (96) - +- ShuffleQueryStage (94), Statistics(X) - +- ColumnarExchange (93) - +- VeloxResizeBatches (92) - +- ^ ProjectExecTransformer (90) - +- ^ FlushableHashAggregateExecTransformer (89) - +- ^ ProjectExecTransformer (88) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - :- ^ InputIteratorTransformer (77) - : +- ShuffleQueryStage (75), Statistics(X) - : +- ColumnarExchange (74) - : +- VeloxResizeBatches (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - : :- ^ InputIteratorTransformer (60) - : : +- ShuffleQueryStage (58), Statistics(X) - : : +- ColumnarExchange (57) - : : +- VeloxResizeBatches (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - : : :- ^ InputIteratorTransformer (43) - : : : +- ShuffleQueryStage (41), Statistics(X) - : : : +- ColumnarExchange (40) - : : : +- VeloxResizeBatches (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) - : : : :- ^ InputIteratorTransformer (26) - : : : : +- ShuffleQueryStage (24), Statistics(X) - : : : : +- ColumnarExchange (23) - : : : : +- VeloxResizeBatches (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ InputIteratorTransformer (9) - : : : : : +- ShuffleQueryStage (7), Statistics(X) - : : : : : +- ColumnarExchange (6) - : : : : : +- VeloxResizeBatches (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ FileSourceScanExecTransformer parquet (1) - : : : : +- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16), Statistics(X) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxResizeBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ FilterExecTransformer (11) - : : : : +- ^ FileSourceScanExecTransformer parquet (10) - : : : +- ^ InputIteratorTransformer (35) - : : : +- ShuffleQueryStage (33), Statistics(X) - : : : +- ColumnarExchange (32) - : : : +- VeloxResizeBatches (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ FilterExecTransformer (28) - : : : +- ^ FileSourceScanExecTransformer parquet (27) - : : +- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50), Statistics(X) - : : +- ColumnarExchange (49) - : : +- VeloxResizeBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ FilterExecTransformer (45) - : : +- ^ FileSourceScanExecTransformer parquet (44) - : +- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67), Statistics(X) - : +- ColumnarExchange (66) - : +- VeloxResizeBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ FilterExecTransformer (62) - : +- ^ FileSourceScanExecTransformer parquet (61) - +- ^ InputIteratorTransformer (86) - +- ShuffleQueryStage (84), Statistics(X) - +- ColumnarExchange (83) - +- VeloxResizeBatches (82) - +- ^ ProjectExecTransformer (80) - +- ^ FilterExecTransformer (79) - +- ^ FileSourceScanExecTransformer parquet (78) -+- == Initial Plan == - Sort (155) - +- Exchange (154) - +- HashAggregate (153) - +- Exchange (152) - +- HashAggregate (151) - +- Project (150) - +- SortMergeJoin Inner (149) - :- Sort (143) - : +- Exchange (142) - : +- Project (141) - : +- SortMergeJoin Inner (140) - : :- Sort (135) - : : +- Exchange (134) - : : +- Project (133) - : : +- SortMergeJoin Inner (132) - : : :- Sort (127) - : : : +- Exchange (126) - : : : +- Project (125) - : : : +- SortMergeJoin Inner (124) - : : : :- Sort (119) - : : : : +- Exchange (118) - : : : : +- Project (117) - : : : : +- SortMergeJoin Inner (116) - : : : : :- Sort (110) - : : : : : +- Exchange (109) - : : : : : +- Filter (108) - : : : : : +- Scan parquet (107) - : : : : +- Sort (115) - : : : : +- Exchange (114) - : : : : +- Project (113) - : : : : +- Filter (112) - : : : : +- Scan parquet (111) - : : : +- Sort (123) - : : : +- Exchange (122) - : : : +- Filter (121) - : : : +- Scan parquet (120) - : : +- Sort (131) - : : +- Exchange (130) - : : +- Filter (129) - : : +- Scan parquet (128) - : +- Sort (139) - : +- Exchange (138) - : +- Filter (137) - : +- Scan parquet (136) - +- Sort (148) - +- Exchange (147) - +- Project (146) - +- Filter (145) - +- Scan parquet (144) - - -(1) FileSourceScanExecTransformer parquet -Output [2]: [c_custkey#X, c_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(3) ProjectExecTransformer -Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] -Input [2]: [c_custkey#X, c_nationkey#X] - -(4) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] -Arguments: false - -(5) VeloxResizeBatches -Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] - -(6) ColumnarExchange -Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] -Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(7) ShuffleQueryStage -Output [2]: [c_custkey#X, c_nationkey#X] -Arguments: X - -(8) InputAdapter -Input [2]: [c_custkey#X, c_nationkey#X] - -(9) InputIteratorTransformer -Input [2]: [c_custkey#X, c_nationkey#X] - -(10) FileSourceScanExecTransformer parquet -Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] -ReadSchema: struct - -(11) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(12) ProjectExecTransformer -Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] - -(13) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] -Arguments: false - -(14) VeloxResizeBatches -Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] - -(15) ColumnarExchange -Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] -Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(16) ShuffleQueryStage -Output [2]: [o_orderkey#X, o_custkey#X] -Arguments: X - -(17) InputAdapter -Input [2]: [o_orderkey#X, o_custkey#X] - -(18) InputIteratorTransformer -Input [2]: [o_orderkey#X, o_custkey#X] - -(19) ShuffledHashJoinExecTransformer -Left keys [1]: [c_custkey#X] -Right keys [1]: [o_custkey#X] -Join condition: None - -(20) ProjectExecTransformer -Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] -Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] - -(21) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] -Arguments: false - -(22) VeloxResizeBatches -Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] - -(23) ColumnarExchange -Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] -Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(24) ShuffleQueryStage -Output [2]: [c_nationkey#X, o_orderkey#X] -Arguments: X - -(25) InputAdapter -Input [2]: [c_nationkey#X, o_orderkey#X] - -(26) InputIteratorTransformer -Input [2]: [c_nationkey#X, o_orderkey#X] - -(27) FileSourceScanExecTransformer parquet -Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] -ReadSchema: struct - -(28) FilterExecTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) - -(29) ProjectExecTransformer -Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(30) WholeStageCodegenTransformer (X) -Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: false - -(31) VeloxResizeBatches -Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(32) ColumnarExchange -Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] - -(33) ShuffleQueryStage -Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: X - -(34) InputAdapter -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(35) InputIteratorTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(36) ShuffledHashJoinExecTransformer -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(37) ProjectExecTransformer -Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(38) WholeStageCodegenTransformer (X) -Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: false - -(39) VeloxResizeBatches -Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(40) ColumnarExchange -Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] - -(41) ShuffleQueryStage -Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: X - -(42) InputAdapter -Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(43) InputIteratorTransformer -Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(44) FileSourceScanExecTransformer parquet -Output [2]: [s_suppkey#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] -ReadSchema: struct - -(45) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(46) ProjectExecTransformer -Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] -Input [2]: [s_suppkey#X, s_nationkey#X] - -(47) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] -Arguments: false - -(48) VeloxResizeBatches -Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] - -(49) ColumnarExchange -Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] -Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(50) ShuffleQueryStage -Output [2]: [s_suppkey#X, s_nationkey#X] -Arguments: X - -(51) InputAdapter -Input [2]: [s_suppkey#X, s_nationkey#X] - -(52) InputIteratorTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] - -(53) ShuffledHashJoinExecTransformer -Left keys [2]: [l_suppkey#X, c_nationkey#X] -Right keys [2]: [s_suppkey#X, s_nationkey#X] -Join condition: None - -(54) ProjectExecTransformer -Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] - -(55) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -Arguments: false - -(56) VeloxResizeBatches -Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] - -(57) ColumnarExchange -Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(58) ShuffleQueryStage -Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -Arguments: X - -(59) InputAdapter -Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] - -(60) InputIteratorTransformer -Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] - -(61) FileSourceScanExecTransformer parquet -Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] -ReadSchema: struct - -(62) FilterExecTransformer -Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) - -(63) ProjectExecTransformer -Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] -Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] - -(64) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] -Arguments: false - -(65) VeloxResizeBatches -Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] - -(66) ColumnarExchange -Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] -Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(67) ShuffleQueryStage -Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -Arguments: X - -(68) InputAdapter -Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] - -(69) InputIteratorTransformer -Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] - -(70) ShuffledHashJoinExecTransformer -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(71) ProjectExecTransformer -Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] - -(72) WholeStageCodegenTransformer (X) -Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -Arguments: false - -(73) VeloxResizeBatches -Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] - -(74) ColumnarExchange -Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(75) ShuffleQueryStage -Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -Arguments: X - -(76) InputAdapter -Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] - -(77) InputIteratorTransformer -Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] - -(78) FileSourceScanExecTransformer parquet -Output [2]: [r_regionkey#X, r_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] -ReadSchema: struct - -(79) FilterExecTransformer -Input [2]: [r_regionkey#X, r_name#X] -Arguments: ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) - -(80) ProjectExecTransformer -Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] -Input [2]: [r_regionkey#X, r_name#X] - -(81) WholeStageCodegenTransformer (X) -Input [2]: [hash_partition_key#X, r_regionkey#X] -Arguments: false - -(82) VeloxResizeBatches -Input [2]: [hash_partition_key#X, r_regionkey#X] - -(83) ColumnarExchange -Input [2]: [hash_partition_key#X, r_regionkey#X] -Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(84) ShuffleQueryStage -Output [1]: [r_regionkey#X] -Arguments: X - -(85) InputAdapter -Input [1]: [r_regionkey#X] - -(86) InputIteratorTransformer -Input [1]: [r_regionkey#X] - -(87) ShuffledHashJoinExecTransformer -Left keys [1]: [n_regionkey#X] -Right keys [1]: [r_regionkey#X] -Join condition: None - -(88) ProjectExecTransformer -Output [2]: [n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] -Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] - -(89) FlushableHashAggregateExecTransformer -Input [2]: [n_name#X, _pre_X#X] -Keys [1]: [n_name#X] -Functions [1]: [partial_sum(_pre_X#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [3]: [n_name#X, sum#X, isEmpty#X] - -(90) ProjectExecTransformer -Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] -Input [3]: [n_name#X, sum#X, isEmpty#X] - -(91) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] -Arguments: false - -(92) VeloxResizeBatches -Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] - -(93) ColumnarExchange -Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] - -(94) ShuffleQueryStage -Output [3]: [n_name#X, sum#X, isEmpty#X] -Arguments: X - -(95) InputAdapter -Input [3]: [n_name#X, sum#X, isEmpty#X] - -(96) InputIteratorTransformer -Input [3]: [n_name#X, sum#X, isEmpty#X] - -(97) RegularHashAggregateExecTransformer -Input [3]: [n_name#X, sum#X, isEmpty#X] -Keys [1]: [n_name#X] -Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] - -(98) WholeStageCodegenTransformer (X) -Input [2]: [n_name#X, revenue#X] -Arguments: false - -(99) VeloxResizeBatches -Input [2]: [n_name#X, revenue#X] - -(100) ColumnarExchange -Input [2]: [n_name#X, revenue#X] -Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(101) ShuffleQueryStage -Output [2]: [n_name#X, revenue#X] -Arguments: X - -(102) InputAdapter -Input [2]: [n_name#X, revenue#X] - -(103) InputIteratorTransformer -Input [2]: [n_name#X, revenue#X] - -(104) SortExecTransformer -Input [2]: [n_name#X, revenue#X] -Arguments: [revenue#X DESC NULLS LAST], true, 0 - -(105) WholeStageCodegenTransformer (X) -Input [2]: [n_name#X, revenue#X] -Arguments: false - -(106) VeloxColumnarToRow -Input [2]: [n_name#X, revenue#X] - -(107) Scan parquet -Output [2]: [c_custkey#X, c_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] -ReadSchema: struct - -(108) Filter -Input [2]: [c_custkey#X, c_nationkey#X] -Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(109) Exchange -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(110) Sort -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(111) Scan parquet -Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] -ReadSchema: struct - -(112) Filter -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(113) Project -Output [2]: [o_orderkey#X, o_custkey#X] -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] - -(114) Exchange -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(115) Sort -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(116) SortMergeJoin -Left keys [1]: [c_custkey#X] -Right keys [1]: [o_custkey#X] -Join condition: None - -(117) Project -Output [2]: [c_nationkey#X, o_orderkey#X] -Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] - -(118) Exchange -Input [2]: [c_nationkey#X, o_orderkey#X] -Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(119) Sort -Input [2]: [c_nationkey#X, o_orderkey#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(120) Scan parquet -Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] -ReadSchema: struct - -(121) Filter -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) - -(122) Exchange -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(123) Sort -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(124) SortMergeJoin -Left keys [1]: [o_orderkey#X] -Right keys [1]: [l_orderkey#X] -Join condition: None - -(125) Project -Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(126) Exchange -Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(127) Sort -Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_suppkey#X ASC NULLS FIRST, c_nationkey#X ASC NULLS FIRST], false, 0 - -(128) Scan parquet -Output [2]: [s_suppkey#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] -ReadSchema: struct - -(129) Filter -Input [2]: [s_suppkey#X, s_nationkey#X] -Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(130) Exchange -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(131) Sort -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST, s_nationkey#X ASC NULLS FIRST], false, 0 - -(132) SortMergeJoin -Left keys [2]: [l_suppkey#X, c_nationkey#X] -Right keys [2]: [s_suppkey#X, s_nationkey#X] -Join condition: None - -(133) Project -Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] - -(134) Exchange -Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(135) Sort -Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 - -(136) Scan parquet -Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] -ReadSchema: struct - -(137) Filter -Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) - -(138) Exchange -Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(139) Sort -Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 - -(140) SortMergeJoin -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(141) Project -Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] - -(142) Exchange -Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(143) Sort -Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 - -(144) Scan parquet -Output [2]: [r_regionkey#X, r_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] -ReadSchema: struct - -(145) Filter -Input [2]: [r_regionkey#X, r_name#X] -Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) - -(146) Project -Output [1]: [r_regionkey#X] -Input [2]: [r_regionkey#X, r_name#X] - -(147) Exchange -Input [1]: [r_regionkey#X] -Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(148) Sort -Input [1]: [r_regionkey#X] -Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 - -(149) SortMergeJoin -Left keys [1]: [n_regionkey#X] -Right keys [1]: [r_regionkey#X] -Join condition: None - -(150) Project -Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] -Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] - -(151) HashAggregate -Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] -Keys [1]: [n_name#X] -Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [3]: [n_name#X, sum#X, isEmpty#X] - -(152) Exchange -Input [3]: [n_name#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(153) HashAggregate -Input [3]: [n_name#X, sum#X, isEmpty#X] -Keys [1]: [n_name#X] -Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] - -(154) Exchange -Input [2]: [n_name#X, revenue#X] -Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(155) Sort -Input [2]: [n_name#X, revenue#X] -Arguments: [revenue#X DESC NULLS LAST], true, 0 - -(156) AdaptiveSparkPlan -Output [2]: [n_name#X, revenue#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/6.txt deleted file mode 100644 index 349f8ccdbe8..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/6.txt +++ /dev/null @@ -1,115 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (20) -+- == Final Plan == - VeloxColumnarToRow (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- VeloxResizeBatches (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ FileSourceScanExecTransformer parquet (1) -+- == Initial Plan == - HashAggregate (19) - +- Exchange (18) - +- HashAggregate (17) - +- Project (16) - +- Filter (15) - +- Scan parquet (14) - - -(1) FileSourceScanExecTransformer parquet -Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) - -(3) ProjectExecTransformer -Output [1]: [CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)) AS _pre_X#X] -Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(4) FlushableHashAggregateExecTransformer -Input [1]: [_pre_X#X] -Keys: [] -Functions [1]: [partial_sum(_pre_X#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [2]: [sum#X, isEmpty#X] - -(5) WholeStageCodegenTransformer (X) -Input [2]: [sum#X, isEmpty#X] -Arguments: false - -(6) VeloxResizeBatches -Input [2]: [sum#X, isEmpty#X] - -(7) ColumnarExchange -Input [2]: [sum#X, isEmpty#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(8) ShuffleQueryStage -Output [2]: [sum#X, isEmpty#X] -Arguments: X - -(9) InputAdapter -Input [2]: [sum#X, isEmpty#X] - -(10) InputIteratorTransformer -Input [2]: [sum#X, isEmpty#X] - -(11) RegularHashAggregateExecTransformer -Input [2]: [sum#X, isEmpty#X] -Keys: [] -Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] -Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] - -(12) WholeStageCodegenTransformer (X) -Input [1]: [revenue#X] -Arguments: false - -(13) VeloxColumnarToRow -Input [1]: [revenue#X] - -(14) Scan parquet -Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] -ReadSchema: struct - -(15) Filter -Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) - -(16) Project -Output [2]: [l_extendedprice#X, l_discount#X] -Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(17) HashAggregate -Input [2]: [l_extendedprice#X, l_discount#X] -Keys: [] -Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [2]: [sum#X, isEmpty#X] - -(18) Exchange -Input [2]: [sum#X, isEmpty#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] - -(19) HashAggregate -Input [2]: [sum#X, isEmpty#X] -Keys: [] -Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] -Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] -Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] - -(20) AdaptiveSparkPlan -Output [1]: [revenue#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt deleted file mode 100644 index e12a31fc9de..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt +++ /dev/null @@ -1,743 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (149) -+- == Final Plan == - VeloxColumnarToRow (101) - +- ^ SortExecTransformer (99) - +- ^ InputIteratorTransformer (98) - +- ShuffleQueryStage (96), Statistics(X) - +- ColumnarExchange (95) - +- VeloxResizeBatches (94) - +- ^ RegularHashAggregateExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- VeloxResizeBatches (87) - +- ^ ProjectExecTransformer (85) - +- ^ FlushableHashAggregateExecTransformer (84) - +- ^ ProjectExecTransformer (83) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) - :- ^ InputIteratorTransformer (77) - : +- ShuffleQueryStage (75), Statistics(X) - : +- ColumnarExchange (74) - : +- VeloxResizeBatches (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - : :- ^ InputIteratorTransformer (60) - : : +- ShuffleQueryStage (58), Statistics(X) - : : +- ColumnarExchange (57) - : : +- VeloxResizeBatches (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - : : :- ^ InputIteratorTransformer (43) - : : : +- ShuffleQueryStage (41), Statistics(X) - : : : +- ColumnarExchange (40) - : : : +- VeloxResizeBatches (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : :- ^ InputIteratorTransformer (26) - : : : : +- ShuffleQueryStage (24), Statistics(X) - : : : : +- ColumnarExchange (23) - : : : : +- VeloxResizeBatches (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ InputIteratorTransformer (9) - : : : : : +- ShuffleQueryStage (7), Statistics(X) - : : : : : +- ColumnarExchange (6) - : : : : : +- VeloxResizeBatches (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ FileSourceScanExecTransformer parquet (1) - : : : : +- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16), Statistics(X) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxResizeBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ FilterExecTransformer (11) - : : : : +- ^ FileSourceScanExecTransformer parquet (10) - : : : +- ^ InputIteratorTransformer (35) - : : : +- ShuffleQueryStage (33), Statistics(X) - : : : +- ColumnarExchange (32) - : : : +- VeloxResizeBatches (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ FilterExecTransformer (28) - : : : +- ^ FileSourceScanExecTransformer parquet (27) - : : +- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50), Statistics(X) - : : +- ColumnarExchange (49) - : : +- VeloxResizeBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ FilterExecTransformer (45) - : : +- ^ FileSourceScanExecTransformer parquet (44) - : +- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67), Statistics(X) - : +- ColumnarExchange (66) - : +- VeloxResizeBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ FilterExecTransformer (62) - : +- ^ FileSourceScanExecTransformer parquet (61) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ReusedExchange (78) -+- == Initial Plan == - Sort (148) - +- Exchange (147) - +- HashAggregate (146) - +- Exchange (145) - +- HashAggregate (144) - +- Project (143) - +- SortMergeJoin Inner (142) - :- Sort (137) - : +- Exchange (136) - : +- Project (135) - : +- SortMergeJoin Inner (134) - : :- Sort (129) - : : +- Exchange (128) - : : +- Project (127) - : : +- SortMergeJoin Inner (126) - : : :- Sort (121) - : : : +- Exchange (120) - : : : +- Project (119) - : : : +- SortMergeJoin Inner (118) - : : : :- Sort (113) - : : : : +- Exchange (112) - : : : : +- Project (111) - : : : : +- SortMergeJoin Inner (110) - : : : : :- Sort (105) - : : : : : +- Exchange (104) - : : : : : +- Filter (103) - : : : : : +- Scan parquet (102) - : : : : +- Sort (109) - : : : : +- Exchange (108) - : : : : +- Filter (107) - : : : : +- Scan parquet (106) - : : : +- Sort (117) - : : : +- Exchange (116) - : : : +- Filter (115) - : : : +- Scan parquet (114) - : : +- Sort (125) - : : +- Exchange (124) - : : +- Filter (123) - : : +- Scan parquet (122) - : +- Sort (133) - : +- Exchange (132) - : +- Filter (131) - : +- Scan parquet (130) - +- Sort (141) - +- Exchange (140) - +- Filter (139) - +- Scan parquet (138) - - -(1) FileSourceScanExecTransformer parquet -Output [2]: [s_suppkey#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(3) ProjectExecTransformer -Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] -Input [2]: [s_suppkey#X, s_nationkey#X] - -(4) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] -Arguments: false - -(5) VeloxResizeBatches -Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] - -(6) ColumnarExchange -Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] -Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(7) ShuffleQueryStage -Output [2]: [s_suppkey#X, s_nationkey#X] -Arguments: X - -(8) InputAdapter -Input [2]: [s_suppkey#X, s_nationkey#X] - -(9) InputIteratorTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] - -(10) FileSourceScanExecTransformer parquet -Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] -ReadSchema: struct - -(11) FilterExecTransformer -Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(12) ProjectExecTransformer -Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(13) WholeStageCodegenTransformer (X) -Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: false - -(14) VeloxResizeBatches -Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(15) ColumnarExchange -Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [shuffle_writer_type=hash] - -(16) ShuffleQueryStage -Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: X - -(17) InputAdapter -Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(18) InputIteratorTransformer -Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(19) ShuffledHashJoinExecTransformer -Left keys [1]: [s_suppkey#X] -Right keys [1]: [l_suppkey#X] -Join condition: None - -(20) ProjectExecTransformer -Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(21) WholeStageCodegenTransformer (X) -Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: false - -(22) VeloxResizeBatches -Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(23) ColumnarExchange -Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [shuffle_writer_type=hash] - -(24) ShuffleQueryStage -Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: X - -(25) InputAdapter -Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(26) InputIteratorTransformer -Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(27) FileSourceScanExecTransformer parquet -Output [2]: [o_orderkey#X, o_custkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] -ReadSchema: struct - -(28) FilterExecTransformer -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) - -(29) ProjectExecTransformer -Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] -Input [2]: [o_orderkey#X, o_custkey#X] - -(30) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] -Arguments: false - -(31) VeloxResizeBatches -Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] - -(32) ColumnarExchange -Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] -Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(33) ShuffleQueryStage -Output [2]: [o_orderkey#X, o_custkey#X] -Arguments: X - -(34) InputAdapter -Input [2]: [o_orderkey#X, o_custkey#X] - -(35) InputIteratorTransformer -Input [2]: [o_orderkey#X, o_custkey#X] - -(36) ShuffledHashJoinExecTransformer -Left keys [1]: [l_orderkey#X] -Right keys [1]: [o_orderkey#X] -Join condition: None - -(37) ProjectExecTransformer -Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] - -(38) WholeStageCodegenTransformer (X) -Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -Arguments: false - -(39) VeloxResizeBatches -Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] - -(40) ColumnarExchange -Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(41) ShuffleQueryStage -Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -Arguments: X - -(42) InputAdapter -Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] - -(43) InputIteratorTransformer -Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] - -(44) FileSourceScanExecTransformer parquet -Output [2]: [c_custkey#X, c_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] -ReadSchema: struct - -(45) FilterExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(46) ProjectExecTransformer -Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] -Input [2]: [c_custkey#X, c_nationkey#X] - -(47) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] -Arguments: false - -(48) VeloxResizeBatches -Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] - -(49) ColumnarExchange -Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] -Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(50) ShuffleQueryStage -Output [2]: [c_custkey#X, c_nationkey#X] -Arguments: X - -(51) InputAdapter -Input [2]: [c_custkey#X, c_nationkey#X] - -(52) InputIteratorTransformer -Input [2]: [c_custkey#X, c_nationkey#X] - -(53) ShuffledHashJoinExecTransformer -Left keys [1]: [o_custkey#X] -Right keys [1]: [c_custkey#X] -Join condition: None - -(54) ProjectExecTransformer -Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] - -(55) WholeStageCodegenTransformer (X) -Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -Arguments: false - -(56) VeloxResizeBatches -Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] - -(57) ColumnarExchange -Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(58) ShuffleQueryStage -Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -Arguments: X - -(59) InputAdapter -Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] - -(60) InputIteratorTransformer -Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] - -(61) FileSourceScanExecTransformer parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] -ReadSchema: struct - -(62) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) - -(63) ProjectExecTransformer -Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] -Input [2]: [n_nationkey#X, n_name#X] - -(64) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] -Arguments: false - -(65) VeloxResizeBatches -Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] - -(66) ColumnarExchange -Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] -Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [shuffle_writer_type=hash] - -(67) ShuffleQueryStage -Output [2]: [n_nationkey#X, n_name#X] -Arguments: X - -(68) InputAdapter -Input [2]: [n_nationkey#X, n_name#X] - -(69) InputIteratorTransformer -Input [2]: [n_nationkey#X, n_name#X] - -(70) ShuffledHashJoinExecTransformer -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(71) ProjectExecTransformer -Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] - -(72) WholeStageCodegenTransformer (X) -Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -Arguments: false - -(73) VeloxResizeBatches -Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] - -(74) ColumnarExchange -Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [shuffle_writer_type=hash] - -(75) ShuffleQueryStage -Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -Arguments: X - -(76) InputAdapter -Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] - -(77) InputIteratorTransformer -Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] - -(78) ReusedExchange [Reuses operator id: 66] -Output [2]: [n_nationkey#X, n_name#X] - -(79) ShuffleQueryStage -Output [2]: [n_nationkey#X, n_name#X] -Arguments: X - -(80) InputAdapter -Input [2]: [n_nationkey#X, n_name#X] - -(81) InputIteratorTransformer -Input [2]: [n_nationkey#X, n_name#X] - -(82) ShuffledHashJoinExecTransformer -Left keys [1]: [c_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) - -(83) ProjectExecTransformer -Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] -Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] - -(84) FlushableHashAggregateExecTransformer -Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] -Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] -Functions [1]: [partial_sum(volume#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] - -(85) ProjectExecTransformer -Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] - -(86) WholeStageCodegenTransformer (X) -Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -Arguments: false - -(87) VeloxResizeBatches -Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] - -(88) ColumnarExchange -Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] - -(89) ShuffleQueryStage -Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -Arguments: X - -(90) InputAdapter -Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] - -(91) InputIteratorTransformer -Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] - -(92) RegularHashAggregateExecTransformer -Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] -Functions [1]: [sum(volume#X)] -Aggregate Attributes [1]: [sum(volume#X)#X] -Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] - -(93) WholeStageCodegenTransformer (X) -Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -Arguments: false - -(94) VeloxResizeBatches -Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] - -(95) ColumnarExchange -Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(96) ShuffleQueryStage -Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -Arguments: X - -(97) InputAdapter -Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] - -(98) InputIteratorTransformer -Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] - -(99) SortExecTransformer -Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 - -(100) WholeStageCodegenTransformer (X) -Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -Arguments: false - -(101) VeloxColumnarToRow -Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] - -(102) Scan parquet -Output [2]: [s_suppkey#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] -ReadSchema: struct - -(103) Filter -Input [2]: [s_suppkey#X, s_nationkey#X] -Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(104) Exchange -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(105) Sort -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(106) Scan parquet -Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] -ReadSchema: struct - -(107) Filter -Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(108) Exchange -Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(109) Sort -Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 - -(110) SortMergeJoin -Left keys [1]: [s_suppkey#X] -Right keys [1]: [l_suppkey#X] -Join condition: None - -(111) Project -Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] - -(112) Exchange -Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(113) Sort -Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(114) Scan parquet -Output [2]: [o_orderkey#X, o_custkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] -ReadSchema: struct - -(115) Filter -Input [2]: [o_orderkey#X, o_custkey#X] -Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) - -(116) Exchange -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(117) Sort -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(118) SortMergeJoin -Left keys [1]: [l_orderkey#X] -Right keys [1]: [o_orderkey#X] -Join condition: None - -(119) Project -Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] - -(120) Exchange -Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(121) Sort -Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(122) Scan parquet -Output [2]: [c_custkey#X, c_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] -ReadSchema: struct - -(123) Filter -Input [2]: [c_custkey#X, c_nationkey#X] -Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(124) Exchange -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(125) Sort -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(126) SortMergeJoin -Left keys [1]: [o_custkey#X] -Right keys [1]: [c_custkey#X] -Join condition: None - -(127) Project -Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] - -(128) Exchange -Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(129) Sort -Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 - -(130) Scan parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] -ReadSchema: struct - -(131) Filter -Input [2]: [n_nationkey#X, n_name#X] -Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) - -(132) Exchange -Input [2]: [n_nationkey#X, n_name#X] -Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(133) Sort -Input [2]: [n_nationkey#X, n_name#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 - -(134) SortMergeJoin -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(135) Project -Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] - -(136) Exchange -Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(137) Sort -Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 - -(138) Scan parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] -ReadSchema: struct - -(139) Filter -Input [2]: [n_nationkey#X, n_name#X] -Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) - -(140) Exchange -Input [2]: [n_nationkey#X, n_name#X] -Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(141) Sort -Input [2]: [n_nationkey#X, n_name#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 - -(142) SortMergeJoin -Left keys [1]: [c_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) - -(143) Project -Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] -Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] - -(144) HashAggregate -Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] -Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] -Functions [1]: [partial_sum(volume#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] - -(145) Exchange -Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(146) HashAggregate -Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] -Functions [1]: [sum(volume#X)] -Aggregate Attributes [1]: [sum(volume#X)#X] -Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] - -(147) Exchange -Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(148) Sort -Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 - -(149) AdaptiveSparkPlan -Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt deleted file mode 100644 index 5135615ca7a..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt +++ /dev/null @@ -1,1031 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (207) -+- == Final Plan == - VeloxColumnarToRow (141) - +- ^ SortExecTransformer (139) - +- ^ InputIteratorTransformer (138) - +- ShuffleQueryStage (136), Statistics(X) - +- ColumnarExchange (135) - +- VeloxResizeBatches (134) - +- ^ ProjectExecTransformer (132) - +- ^ RegularHashAggregateExecTransformer (131) - +- ^ InputIteratorTransformer (130) - +- ShuffleQueryStage (128), Statistics(X) - +- ColumnarExchange (127) - +- VeloxResizeBatches (126) - +- ^ ProjectExecTransformer (124) - +- ^ FlushableHashAggregateExecTransformer (123) - +- ^ ProjectExecTransformer (122) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) - :- ^ InputIteratorTransformer (111) - : +- ShuffleQueryStage (109), Statistics(X) - : +- ColumnarExchange (108) - : +- VeloxResizeBatches (107) - : +- ^ ProjectExecTransformer (105) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) - : :- ^ InputIteratorTransformer (94) - : : +- ShuffleQueryStage (92), Statistics(X) - : : +- ColumnarExchange (91) - : : +- VeloxResizeBatches (90) - : : +- ^ ProjectExecTransformer (88) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - : : :- ^ InputIteratorTransformer (77) - : : : +- ShuffleQueryStage (75), Statistics(X) - : : : +- ColumnarExchange (74) - : : : +- VeloxResizeBatches (73) - : : : +- ^ ProjectExecTransformer (71) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) - : : : :- ^ InputIteratorTransformer (60) - : : : : +- ShuffleQueryStage (58), Statistics(X) - : : : : +- ColumnarExchange (57) - : : : : +- VeloxResizeBatches (56) - : : : : +- ^ ProjectExecTransformer (54) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) - : : : : :- ^ InputIteratorTransformer (43) - : : : : : +- ShuffleQueryStage (41), Statistics(X) - : : : : : +- ColumnarExchange (40) - : : : : : +- VeloxResizeBatches (39) - : : : : : +- ^ ProjectExecTransformer (37) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : : : :- ^ InputIteratorTransformer (26) - : : : : : : +- ShuffleQueryStage (24), Statistics(X) - : : : : : : +- ColumnarExchange (23) - : : : : : : +- VeloxResizeBatches (22) - : : : : : : +- ^ ProjectExecTransformer (20) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : : : :- ^ InputIteratorTransformer (9) - : : : : : : : +- ShuffleQueryStage (7), Statistics(X) - : : : : : : : +- ColumnarExchange (6) - : : : : : : : +- VeloxResizeBatches (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ FileSourceScanExecTransformer parquet (1) - : : : : : : +- ^ InputIteratorTransformer (18) - : : : : : : +- ShuffleQueryStage (16), Statistics(X) - : : : : : : +- ColumnarExchange (15) - : : : : : : +- VeloxResizeBatches (14) - : : : : : : +- ^ ProjectExecTransformer (12) - : : : : : : +- ^ FilterExecTransformer (11) - : : : : : : +- ^ FileSourceScanExecTransformer parquet (10) - : : : : : +- ^ InputIteratorTransformer (35) - : : : : : +- ShuffleQueryStage (33), Statistics(X) - : : : : : +- ColumnarExchange (32) - : : : : : +- VeloxResizeBatches (31) - : : : : : +- ^ ProjectExecTransformer (29) - : : : : : +- ^ FilterExecTransformer (28) - : : : : : +- ^ FileSourceScanExecTransformer parquet (27) - : : : : +- ^ InputIteratorTransformer (52) - : : : : +- ShuffleQueryStage (50), Statistics(X) - : : : : +- ColumnarExchange (49) - : : : : +- VeloxResizeBatches (48) - : : : : +- ^ ProjectExecTransformer (46) - : : : : +- ^ FilterExecTransformer (45) - : : : : +- ^ FileSourceScanExecTransformer parquet (44) - : : : +- ^ InputIteratorTransformer (69) - : : : +- ShuffleQueryStage (67), Statistics(X) - : : : +- ColumnarExchange (66) - : : : +- VeloxResizeBatches (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ FilterExecTransformer (62) - : : : +- ^ FileSourceScanExecTransformer parquet (61) - : : +- ^ InputIteratorTransformer (86) - : : +- ShuffleQueryStage (84), Statistics(X) - : : +- ColumnarExchange (83) - : : +- VeloxResizeBatches (82) - : : +- ^ ProjectExecTransformer (80) - : : +- ^ FilterExecTransformer (79) - : : +- ^ FileSourceScanExecTransformer parquet (78) - : +- ^ InputIteratorTransformer (103) - : +- ShuffleQueryStage (101), Statistics(X) - : +- ColumnarExchange (100) - : +- VeloxResizeBatches (99) - : +- ^ ProjectExecTransformer (97) - : +- ^ FilterExecTransformer (96) - : +- ^ FileSourceScanExecTransformer parquet (95) - +- ^ InputIteratorTransformer (120) - +- ShuffleQueryStage (118), Statistics(X) - +- ColumnarExchange (117) - +- VeloxResizeBatches (116) - +- ^ ProjectExecTransformer (114) - +- ^ FilterExecTransformer (113) - +- ^ FileSourceScanExecTransformer parquet (112) -+- == Initial Plan == - Sort (206) - +- Exchange (205) - +- HashAggregate (204) - +- Exchange (203) - +- HashAggregate (202) - +- Project (201) - +- SortMergeJoin Inner (200) - :- Sort (194) - : +- Exchange (193) - : +- Project (192) - : +- SortMergeJoin Inner (191) - : :- Sort (186) - : : +- Exchange (185) - : : +- Project (184) - : : +- SortMergeJoin Inner (183) - : : :- Sort (178) - : : : +- Exchange (177) - : : : +- Project (176) - : : : +- SortMergeJoin Inner (175) - : : : :- Sort (170) - : : : : +- Exchange (169) - : : : : +- Project (168) - : : : : +- SortMergeJoin Inner (167) - : : : : :- Sort (162) - : : : : : +- Exchange (161) - : : : : : +- Project (160) - : : : : : +- SortMergeJoin Inner (159) - : : : : : :- Sort (154) - : : : : : : +- Exchange (153) - : : : : : : +- Project (152) - : : : : : : +- SortMergeJoin Inner (151) - : : : : : : :- Sort (146) - : : : : : : : +- Exchange (145) - : : : : : : : +- Project (144) - : : : : : : : +- Filter (143) - : : : : : : : +- Scan parquet (142) - : : : : : : +- Sort (150) - : : : : : : +- Exchange (149) - : : : : : : +- Filter (148) - : : : : : : +- Scan parquet (147) - : : : : : +- Sort (158) - : : : : : +- Exchange (157) - : : : : : +- Filter (156) - : : : : : +- Scan parquet (155) - : : : : +- Sort (166) - : : : : +- Exchange (165) - : : : : +- Filter (164) - : : : : +- Scan parquet (163) - : : : +- Sort (174) - : : : +- Exchange (173) - : : : +- Filter (172) - : : : +- Scan parquet (171) - : : +- Sort (182) - : : +- Exchange (181) - : : +- Filter (180) - : : +- Scan parquet (179) - : +- Sort (190) - : +- Exchange (189) - : +- Filter (188) - : +- Scan parquet (187) - +- Sort (199) - +- Exchange (198) - +- Project (197) - +- Filter (196) - +- Scan parquet (195) - - -(1) FileSourceScanExecTransformer parquet -Output [2]: [p_partkey#X, p_type#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [2]: [p_partkey#X, p_type#X] -Arguments: ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) - -(3) ProjectExecTransformer -Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] -Input [2]: [p_partkey#X, p_type#X] - -(4) WholeStageCodegenTransformer (X) -Input [2]: [hash_partition_key#X, p_partkey#X] -Arguments: false - -(5) VeloxResizeBatches -Input [2]: [hash_partition_key#X, p_partkey#X] - -(6) ColumnarExchange -Input [2]: [hash_partition_key#X, p_partkey#X] -Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(7) ShuffleQueryStage -Output [1]: [p_partkey#X] -Arguments: X - -(8) InputAdapter -Input [1]: [p_partkey#X] - -(9) InputIteratorTransformer -Input [1]: [p_partkey#X] - -(10) FileSourceScanExecTransformer parquet -Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] -ReadSchema: struct - -(11) FilterExecTransformer -Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(12) ProjectExecTransformer -Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(13) WholeStageCodegenTransformer (X) -Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: false - -(14) VeloxResizeBatches -Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(15) ColumnarExchange -Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] - -(16) ShuffleQueryStage -Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: X - -(17) InputAdapter -Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(18) InputIteratorTransformer -Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(19) ShuffledHashJoinExecTransformer -Left keys [1]: [p_partkey#X] -Right keys [1]: [l_partkey#X] -Join condition: None - -(20) ProjectExecTransformer -Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(21) WholeStageCodegenTransformer (X) -Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: false - -(22) VeloxResizeBatches -Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(23) ColumnarExchange -Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] - -(24) ShuffleQueryStage -Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: X - -(25) InputAdapter -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(26) InputIteratorTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(27) FileSourceScanExecTransformer parquet -Output [2]: [s_suppkey#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] -ReadSchema: struct - -(28) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(29) ProjectExecTransformer -Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] -Input [2]: [s_suppkey#X, s_nationkey#X] - -(30) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] -Arguments: false - -(31) VeloxResizeBatches -Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] - -(32) ColumnarExchange -Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] -Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(33) ShuffleQueryStage -Output [2]: [s_suppkey#X, s_nationkey#X] -Arguments: X - -(34) InputAdapter -Input [2]: [s_suppkey#X, s_nationkey#X] - -(35) InputIteratorTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] - -(36) ShuffledHashJoinExecTransformer -Left keys [1]: [l_suppkey#X] -Right keys [1]: [s_suppkey#X] -Join condition: None - -(37) ProjectExecTransformer -Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] - -(38) WholeStageCodegenTransformer (X) -Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -Arguments: false - -(39) VeloxResizeBatches -Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] - -(40) ColumnarExchange -Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(41) ShuffleQueryStage -Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -Arguments: X - -(42) InputAdapter -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] - -(43) InputIteratorTransformer -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] - -(44) FileSourceScanExecTransformer parquet -Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] -ReadSchema: struct - -(45) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) - -(46) ProjectExecTransformer -Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] - -(47) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: false - -(48) VeloxResizeBatches -Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] - -(49) ColumnarExchange -Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [shuffle_writer_type=hash] - -(50) ShuffleQueryStage -Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: X - -(51) InputAdapter -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] - -(52) InputIteratorTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] - -(53) ShuffledHashJoinExecTransformer -Left keys [1]: [l_orderkey#X] -Right keys [1]: [o_orderkey#X] -Join condition: None - -(54) ProjectExecTransformer -Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] - -(55) WholeStageCodegenTransformer (X) -Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -Arguments: false - -(56) VeloxResizeBatches -Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] - -(57) ColumnarExchange -Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [shuffle_writer_type=hash] - -(58) ShuffleQueryStage -Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -Arguments: X - -(59) InputAdapter -Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] - -(60) InputIteratorTransformer -Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] - -(61) FileSourceScanExecTransformer parquet -Output [2]: [c_custkey#X, c_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] -ReadSchema: struct - -(62) FilterExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(63) ProjectExecTransformer -Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] -Input [2]: [c_custkey#X, c_nationkey#X] - -(64) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] -Arguments: false - -(65) VeloxResizeBatches -Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] - -(66) ColumnarExchange -Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] -Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(67) ShuffleQueryStage -Output [2]: [c_custkey#X, c_nationkey#X] -Arguments: X - -(68) InputAdapter -Input [2]: [c_custkey#X, c_nationkey#X] - -(69) InputIteratorTransformer -Input [2]: [c_custkey#X, c_nationkey#X] - -(70) ShuffledHashJoinExecTransformer -Left keys [1]: [o_custkey#X] -Right keys [1]: [c_custkey#X] -Join condition: None - -(71) ProjectExecTransformer -Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] - -(72) WholeStageCodegenTransformer (X) -Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -Arguments: false - -(73) VeloxResizeBatches -Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] - -(74) ColumnarExchange -Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(75) ShuffleQueryStage -Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -Arguments: X - -(76) InputAdapter -Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] - -(77) InputIteratorTransformer -Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] - -(78) FileSourceScanExecTransformer parquet -Output [2]: [n_nationkey#X, n_regionkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] -ReadSchema: struct - -(79) FilterExecTransformer -Input [2]: [n_nationkey#X, n_regionkey#X] -Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) - -(80) ProjectExecTransformer -Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] -Input [2]: [n_nationkey#X, n_regionkey#X] - -(81) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] -Arguments: false - -(82) VeloxResizeBatches -Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] - -(83) ColumnarExchange -Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] -Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(84) ShuffleQueryStage -Output [2]: [n_nationkey#X, n_regionkey#X] -Arguments: X - -(85) InputAdapter -Input [2]: [n_nationkey#X, n_regionkey#X] - -(86) InputIteratorTransformer -Input [2]: [n_nationkey#X, n_regionkey#X] - -(87) ShuffledHashJoinExecTransformer -Left keys [1]: [c_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(88) ProjectExecTransformer -Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] - -(89) WholeStageCodegenTransformer (X) -Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -Arguments: false - -(90) VeloxResizeBatches -Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] - -(91) ColumnarExchange -Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(92) ShuffleQueryStage -Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -Arguments: X - -(93) InputAdapter -Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] - -(94) InputIteratorTransformer -Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] - -(95) FileSourceScanExecTransformer parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_nationkey)] -ReadSchema: struct - -(96) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: isnotnull(n_nationkey#X) - -(97) ProjectExecTransformer -Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] -Input [2]: [n_nationkey#X, n_name#X] - -(98) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] -Arguments: false - -(99) VeloxResizeBatches -Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] - -(100) ColumnarExchange -Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] -Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [shuffle_writer_type=hash] - -(101) ShuffleQueryStage -Output [2]: [n_nationkey#X, n_name#X] -Arguments: X - -(102) InputAdapter -Input [2]: [n_nationkey#X, n_name#X] - -(103) InputIteratorTransformer -Input [2]: [n_nationkey#X, n_name#X] - -(104) ShuffledHashJoinExecTransformer -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(105) ProjectExecTransformer -Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] - -(106) WholeStageCodegenTransformer (X) -Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -Arguments: false - -(107) VeloxResizeBatches -Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] - -(108) ColumnarExchange -Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [shuffle_writer_type=hash] - -(109) ShuffleQueryStage -Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -Arguments: X - -(110) InputAdapter -Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] - -(111) InputIteratorTransformer -Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] - -(112) FileSourceScanExecTransformer parquet -Output [2]: [r_regionkey#X, r_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] -ReadSchema: struct - -(113) FilterExecTransformer -Input [2]: [r_regionkey#X, r_name#X] -Arguments: ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) - -(114) ProjectExecTransformer -Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] -Input [2]: [r_regionkey#X, r_name#X] - -(115) WholeStageCodegenTransformer (X) -Input [2]: [hash_partition_key#X, r_regionkey#X] -Arguments: false - -(116) VeloxResizeBatches -Input [2]: [hash_partition_key#X, r_regionkey#X] - -(117) ColumnarExchange -Input [2]: [hash_partition_key#X, r_regionkey#X] -Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(118) ShuffleQueryStage -Output [1]: [r_regionkey#X] -Arguments: X - -(119) InputAdapter -Input [1]: [r_regionkey#X] - -(120) InputIteratorTransformer -Input [1]: [r_regionkey#X] - -(121) ShuffledHashJoinExecTransformer -Left keys [1]: [n_regionkey#X] -Right keys [1]: [r_regionkey#X] -Join condition: None - -(122) ProjectExecTransformer -Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X] -Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] - -(123) FlushableHashAggregateExecTransformer -Input [3]: [o_year#X, volume#X, _pre_X#X] -Keys [1]: [o_year#X] -Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] -Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] - -(124) ProjectExecTransformer -Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] - -(125) WholeStageCodegenTransformer (X) -Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -Arguments: false - -(126) VeloxResizeBatches -Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] - -(127) ColumnarExchange -Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] - -(128) ShuffleQueryStage -Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -Arguments: X - -(129) InputAdapter -Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] - -(130) InputIteratorTransformer -Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] - -(131) RegularHashAggregateExecTransformer -Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -Keys [1]: [o_year#X] -Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] -Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] - -(132) ProjectExecTransformer -Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] -Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] - -(133) WholeStageCodegenTransformer (X) -Input [2]: [o_year#X, mkt_share#X] -Arguments: false - -(134) VeloxResizeBatches -Input [2]: [o_year#X, mkt_share#X] - -(135) ColumnarExchange -Input [2]: [o_year#X, mkt_share#X] -Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(136) ShuffleQueryStage -Output [2]: [o_year#X, mkt_share#X] -Arguments: X - -(137) InputAdapter -Input [2]: [o_year#X, mkt_share#X] - -(138) InputIteratorTransformer -Input [2]: [o_year#X, mkt_share#X] - -(139) SortExecTransformer -Input [2]: [o_year#X, mkt_share#X] -Arguments: [o_year#X ASC NULLS FIRST], true, 0 - -(140) WholeStageCodegenTransformer (X) -Input [2]: [o_year#X, mkt_share#X] -Arguments: false - -(141) VeloxColumnarToRow -Input [2]: [o_year#X, mkt_share#X] - -(142) Scan parquet -Output [2]: [p_partkey#X, p_type#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] -ReadSchema: struct - -(143) Filter -Input [2]: [p_partkey#X, p_type#X] -Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) - -(144) Project -Output [1]: [p_partkey#X] -Input [2]: [p_partkey#X, p_type#X] - -(145) Exchange -Input [1]: [p_partkey#X] -Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(146) Sort -Input [1]: [p_partkey#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(147) Scan parquet -Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] -ReadSchema: struct - -(148) Filter -Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(149) Exchange -Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(150) Sort -Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(151) SortMergeJoin -Left keys [1]: [p_partkey#X] -Right keys [1]: [l_partkey#X] -Join condition: None - -(152) Project -Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(153) Exchange -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(154) Sort -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 - -(155) Scan parquet -Output [2]: [s_suppkey#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] -ReadSchema: struct - -(156) Filter -Input [2]: [s_suppkey#X, s_nationkey#X] -Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(157) Exchange -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(158) Sort -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(159) SortMergeJoin -Left keys [1]: [l_suppkey#X] -Right keys [1]: [s_suppkey#X] -Join condition: None - -(160) Project -Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] - -(161) Exchange -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(162) Sort -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(163) Scan parquet -Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] -ReadSchema: struct - -(164) Filter -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) - -(165) Exchange -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(166) Sort -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(167) SortMergeJoin -Left keys [1]: [l_orderkey#X] -Right keys [1]: [o_orderkey#X] -Join condition: None - -(168) Project -Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] - -(169) Exchange -Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(170) Sort -Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(171) Scan parquet -Output [2]: [c_custkey#X, c_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] -ReadSchema: struct - -(172) Filter -Input [2]: [c_custkey#X, c_nationkey#X] -Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(173) Exchange -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(174) Sort -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(175) SortMergeJoin -Left keys [1]: [o_custkey#X] -Right keys [1]: [c_custkey#X] -Join condition: None - -(176) Project -Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] - -(177) Exchange -Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(178) Sort -Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 - -(179) Scan parquet -Output [2]: [n_nationkey#X, n_regionkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] -ReadSchema: struct - -(180) Filter -Input [2]: [n_nationkey#X, n_regionkey#X] -Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) - -(181) Exchange -Input [2]: [n_nationkey#X, n_regionkey#X] -Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(182) Sort -Input [2]: [n_nationkey#X, n_regionkey#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 - -(183) SortMergeJoin -Left keys [1]: [c_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(184) Project -Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] - -(185) Exchange -Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(186) Sort -Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 - -(187) Scan parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_nationkey)] -ReadSchema: struct - -(188) Filter -Input [2]: [n_nationkey#X, n_name#X] -Condition : isnotnull(n_nationkey#X) - -(189) Exchange -Input [2]: [n_nationkey#X, n_name#X] -Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(190) Sort -Input [2]: [n_nationkey#X, n_name#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 - -(191) SortMergeJoin -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(192) Project -Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] - -(193) Exchange -Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(194) Sort -Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 - -(195) Scan parquet -Output [2]: [r_regionkey#X, r_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] -ReadSchema: struct - -(196) Filter -Input [2]: [r_regionkey#X, r_name#X] -Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) - -(197) Project -Output [1]: [r_regionkey#X] -Input [2]: [r_regionkey#X, r_name#X] - -(198) Exchange -Input [1]: [r_regionkey#X] -Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(199) Sort -Input [1]: [r_regionkey#X] -Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 - -(200) SortMergeJoin -Left keys [1]: [n_regionkey#X] -Right keys [1]: [r_regionkey#X] -Join condition: None - -(201) Project -Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X] -Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] - -(202) HashAggregate -Input [3]: [o_year#X, volume#X, nation#X] -Keys [1]: [o_year#X] -Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] -Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] - -(203) Exchange -Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(204) HashAggregate -Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -Keys [1]: [o_year#X] -Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] -Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] - -(205) Exchange -Input [2]: [o_year#X, mkt_share#X] -Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(206) Sort -Input [2]: [o_year#X, mkt_share#X] -Arguments: [o_year#X ASC NULLS FIRST], true, 0 - -(207) AdaptiveSparkPlan -Output [2]: [o_year#X, mkt_share#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt deleted file mode 100644 index 5d5281e0ee3..00000000000 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt +++ /dev/null @@ -1,775 +0,0 @@ -== Physical Plan == -AdaptiveSparkPlan (155) -+- == Final Plan == - VeloxColumnarToRow (106) - +- ^ SortExecTransformer (104) - +- ^ InputIteratorTransformer (103) - +- ShuffleQueryStage (101), Statistics(X) - +- ColumnarExchange (100) - +- VeloxResizeBatches (99) - +- ^ RegularHashAggregateExecTransformer (97) - +- ^ InputIteratorTransformer (96) - +- ShuffleQueryStage (94), Statistics(X) - +- ColumnarExchange (93) - +- VeloxResizeBatches (92) - +- ^ ProjectExecTransformer (90) - +- ^ FlushableHashAggregateExecTransformer (89) - +- ^ ProjectExecTransformer (88) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - :- ^ InputIteratorTransformer (77) - : +- ShuffleQueryStage (75), Statistics(X) - : +- ColumnarExchange (74) - : +- VeloxResizeBatches (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) - : :- ^ InputIteratorTransformer (60) - : : +- ShuffleQueryStage (58), Statistics(X) - : : +- ColumnarExchange (57) - : : +- VeloxResizeBatches (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) - : : :- ^ InputIteratorTransformer (43) - : : : +- ShuffleQueryStage (41), Statistics(X) - : : : +- ColumnarExchange (40) - : : : +- VeloxResizeBatches (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : :- ^ InputIteratorTransformer (26) - : : : : +- ShuffleQueryStage (24), Statistics(X) - : : : : +- ColumnarExchange (23) - : : : : +- VeloxResizeBatches (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ InputIteratorTransformer (9) - : : : : : +- ShuffleQueryStage (7), Statistics(X) - : : : : : +- ColumnarExchange (6) - : : : : : +- VeloxResizeBatches (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ FileSourceScanExecTransformer parquet (1) - : : : : +- ^ InputIteratorTransformer (18) - : : : : +- ShuffleQueryStage (16), Statistics(X) - : : : : +- ColumnarExchange (15) - : : : : +- VeloxResizeBatches (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ FilterExecTransformer (11) - : : : : +- ^ FileSourceScanExecTransformer parquet (10) - : : : +- ^ InputIteratorTransformer (35) - : : : +- ShuffleQueryStage (33), Statistics(X) - : : : +- ColumnarExchange (32) - : : : +- VeloxResizeBatches (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ FilterExecTransformer (28) - : : : +- ^ FileSourceScanExecTransformer parquet (27) - : : +- ^ InputIteratorTransformer (52) - : : +- ShuffleQueryStage (50), Statistics(X) - : : +- ColumnarExchange (49) - : : +- VeloxResizeBatches (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ FilterExecTransformer (45) - : : +- ^ FileSourceScanExecTransformer parquet (44) - : +- ^ InputIteratorTransformer (69) - : +- ShuffleQueryStage (67), Statistics(X) - : +- ColumnarExchange (66) - : +- VeloxResizeBatches (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ FilterExecTransformer (62) - : +- ^ FileSourceScanExecTransformer parquet (61) - +- ^ InputIteratorTransformer (86) - +- ShuffleQueryStage (84), Statistics(X) - +- ColumnarExchange (83) - +- VeloxResizeBatches (82) - +- ^ ProjectExecTransformer (80) - +- ^ FilterExecTransformer (79) - +- ^ FileSourceScanExecTransformer parquet (78) -+- == Initial Plan == - Sort (154) - +- Exchange (153) - +- HashAggregate (152) - +- Exchange (151) - +- HashAggregate (150) - +- Project (149) - +- SortMergeJoin Inner (148) - :- Sort (143) - : +- Exchange (142) - : +- Project (141) - : +- SortMergeJoin Inner (140) - : :- Sort (135) - : : +- Exchange (134) - : : +- Project (133) - : : +- SortMergeJoin Inner (132) - : : :- Sort (127) - : : : +- Exchange (126) - : : : +- Project (125) - : : : +- SortMergeJoin Inner (124) - : : : :- Sort (119) - : : : : +- Exchange (118) - : : : : +- Project (117) - : : : : +- SortMergeJoin Inner (116) - : : : : :- Sort (111) - : : : : : +- Exchange (110) - : : : : : +- Project (109) - : : : : : +- Filter (108) - : : : : : +- Scan parquet (107) - : : : : +- Sort (115) - : : : : +- Exchange (114) - : : : : +- Filter (113) - : : : : +- Scan parquet (112) - : : : +- Sort (123) - : : : +- Exchange (122) - : : : +- Filter (121) - : : : +- Scan parquet (120) - : : +- Sort (131) - : : +- Exchange (130) - : : +- Filter (129) - : : +- Scan parquet (128) - : +- Sort (139) - : +- Exchange (138) - : +- Filter (137) - : +- Scan parquet (136) - +- Sort (147) - +- Exchange (146) - +- Filter (145) - +- Scan parquet (144) - - -(1) FileSourceScanExecTransformer parquet -Output [2]: [p_partkey#X, p_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] -ReadSchema: struct - -(2) FilterExecTransformer -Input [2]: [p_partkey#X, p_name#X] -Arguments: ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) - -(3) ProjectExecTransformer -Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] -Input [2]: [p_partkey#X, p_name#X] - -(4) WholeStageCodegenTransformer (X) -Input [2]: [hash_partition_key#X, p_partkey#X] -Arguments: false - -(5) VeloxResizeBatches -Input [2]: [hash_partition_key#X, p_partkey#X] - -(6) ColumnarExchange -Input [2]: [hash_partition_key#X, p_partkey#X] -Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(7) ShuffleQueryStage -Output [1]: [p_partkey#X] -Arguments: X - -(8) InputAdapter -Input [1]: [p_partkey#X] - -(9) InputIteratorTransformer -Input [1]: [p_partkey#X] - -(10) FileSourceScanExecTransformer parquet -Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] -ReadSchema: struct - -(11) FilterExecTransformer -Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(12) ProjectExecTransformer -Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] - -(13) WholeStageCodegenTransformer (X) -Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: false - -(14) VeloxResizeBatches -Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] - -(15) ColumnarExchange -Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] - -(16) ShuffleQueryStage -Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: X - -(17) InputAdapter -Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] - -(18) InputIteratorTransformer -Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] - -(19) ShuffledHashJoinExecTransformer -Left keys [1]: [p_partkey#X] -Right keys [1]: [l_partkey#X] -Join condition: None - -(20) ProjectExecTransformer -Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] - -(21) WholeStageCodegenTransformer (X) -Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: false - -(22) VeloxResizeBatches -Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] - -(23) ColumnarExchange -Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [shuffle_writer_type=hash] - -(24) ShuffleQueryStage -Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: X - -(25) InputAdapter -Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] - -(26) InputIteratorTransformer -Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] - -(27) FileSourceScanExecTransformer parquet -Output [2]: [s_suppkey#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] -ReadSchema: struct - -(28) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(29) ProjectExecTransformer -Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] -Input [2]: [s_suppkey#X, s_nationkey#X] - -(30) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] -Arguments: false - -(31) VeloxResizeBatches -Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] - -(32) ColumnarExchange -Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] -Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(33) ShuffleQueryStage -Output [2]: [s_suppkey#X, s_nationkey#X] -Arguments: X - -(34) InputAdapter -Input [2]: [s_suppkey#X, s_nationkey#X] - -(35) InputIteratorTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] - -(36) ShuffledHashJoinExecTransformer -Left keys [1]: [l_suppkey#X] -Right keys [1]: [s_suppkey#X] -Join condition: None - -(37) ProjectExecTransformer -Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] - -(38) WholeStageCodegenTransformer (X) -Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -Arguments: false - -(39) VeloxResizeBatches -Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] - -(40) ColumnarExchange -Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [shuffle_writer_type=hash] - -(41) ShuffleQueryStage -Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -Arguments: X - -(42) InputAdapter -Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] - -(43) InputIteratorTransformer -Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] - -(44) FileSourceScanExecTransformer parquet -Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] -ReadSchema: struct - -(45) FilterExecTransformer -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -Arguments: (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) - -(46) ProjectExecTransformer -Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] - -(47) WholeStageCodegenTransformer (X) -Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -Arguments: false - -(48) VeloxResizeBatches -Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] - -(49) ColumnarExchange -Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [shuffle_writer_type=hash] - -(50) ShuffleQueryStage -Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -Arguments: X - -(51) InputAdapter -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] - -(52) InputIteratorTransformer -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] - -(53) ShuffledHashJoinExecTransformer -Left keys [2]: [l_suppkey#X, l_partkey#X] -Right keys [2]: [ps_suppkey#X, ps_partkey#X] -Join condition: None - -(54) ProjectExecTransformer -Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] - -(55) WholeStageCodegenTransformer (X) -Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -Arguments: false - -(56) VeloxResizeBatches -Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] - -(57) ColumnarExchange -Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [shuffle_writer_type=hash] - -(58) ShuffleQueryStage -Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -Arguments: X - -(59) InputAdapter -Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] - -(60) InputIteratorTransformer -Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] - -(61) FileSourceScanExecTransformer parquet -Output [2]: [o_orderkey#X, o_orderdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderkey)] -ReadSchema: struct - -(62) FilterExecTransformer -Input [2]: [o_orderkey#X, o_orderdate#X] -Arguments: isnotnull(o_orderkey#X) - -(63) ProjectExecTransformer -Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] -Input [2]: [o_orderkey#X, o_orderdate#X] - -(64) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] -Arguments: false - -(65) VeloxResizeBatches -Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] - -(66) ColumnarExchange -Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] -Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [shuffle_writer_type=hash] - -(67) ShuffleQueryStage -Output [2]: [o_orderkey#X, o_orderdate#X] -Arguments: X - -(68) InputAdapter -Input [2]: [o_orderkey#X, o_orderdate#X] - -(69) InputIteratorTransformer -Input [2]: [o_orderkey#X, o_orderdate#X] - -(70) ShuffledHashJoinExecTransformer -Left keys [1]: [l_orderkey#X] -Right keys [1]: [o_orderkey#X] -Join condition: None - -(71) ProjectExecTransformer -Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] - -(72) WholeStageCodegenTransformer (X) -Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -Arguments: false - -(73) VeloxResizeBatches -Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] - -(74) ColumnarExchange -Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [shuffle_writer_type=hash] - -(75) ShuffleQueryStage -Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -Arguments: X - -(76) InputAdapter -Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] - -(77) InputIteratorTransformer -Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] - -(78) FileSourceScanExecTransformer parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_nationkey)] -ReadSchema: struct - -(79) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: isnotnull(n_nationkey#X) - -(80) ProjectExecTransformer -Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] -Input [2]: [n_nationkey#X, n_name#X] - -(81) WholeStageCodegenTransformer (X) -Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] -Arguments: false - -(82) VeloxResizeBatches -Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] - -(83) ColumnarExchange -Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] -Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [shuffle_writer_type=hash] - -(84) ShuffleQueryStage -Output [2]: [n_nationkey#X, n_name#X] -Arguments: X - -(85) InputAdapter -Input [2]: [n_nationkey#X, n_name#X] - -(86) InputIteratorTransformer -Input [2]: [n_nationkey#X, n_name#X] - -(87) ShuffledHashJoinExecTransformer -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(88) ProjectExecTransformer -Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] -Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] - -(89) FlushableHashAggregateExecTransformer -Input [3]: [nation#X, o_year#X, amount#X] -Keys [2]: [nation#X, o_year#X] -Functions [1]: [partial_sum(amount#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] - -(90) ProjectExecTransformer -Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] -Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] - -(91) WholeStageCodegenTransformer (X) -Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] -Arguments: false - -(92) VeloxResizeBatches -Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] - -(93) ColumnarExchange -Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [shuffle_writer_type=hash] - -(94) ShuffleQueryStage -Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -Arguments: X - -(95) InputAdapter -Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] - -(96) InputIteratorTransformer -Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] - -(97) RegularHashAggregateExecTransformer -Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -Keys [2]: [nation#X, o_year#X] -Functions [1]: [sum(amount#X)] -Aggregate Attributes [1]: [sum(amount#X)#X] -Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] - -(98) WholeStageCodegenTransformer (X) -Input [3]: [nation#X, o_year#X, sum_profit#X] -Arguments: false - -(99) VeloxResizeBatches -Input [3]: [nation#X, o_year#X, sum_profit#X] - -(100) ColumnarExchange -Input [3]: [nation#X, o_year#X, sum_profit#X] -Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [shuffle_writer_type=hash] - -(101) ShuffleQueryStage -Output [3]: [nation#X, o_year#X, sum_profit#X] -Arguments: X - -(102) InputAdapter -Input [3]: [nation#X, o_year#X, sum_profit#X] - -(103) InputIteratorTransformer -Input [3]: [nation#X, o_year#X, sum_profit#X] - -(104) SortExecTransformer -Input [3]: [nation#X, o_year#X, sum_profit#X] -Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 - -(105) WholeStageCodegenTransformer (X) -Input [3]: [nation#X, o_year#X, sum_profit#X] -Arguments: false - -(106) VeloxColumnarToRow -Input [3]: [nation#X, o_year#X, sum_profit#X] - -(107) Scan parquet -Output [2]: [p_partkey#X, p_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] -ReadSchema: struct - -(108) Filter -Input [2]: [p_partkey#X, p_name#X] -Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) - -(109) Project -Output [1]: [p_partkey#X] -Input [2]: [p_partkey#X, p_name#X] - -(110) Exchange -Input [1]: [p_partkey#X] -Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(111) Sort -Input [1]: [p_partkey#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(112) Scan parquet -Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] -ReadSchema: struct - -(113) Filter -Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(114) Exchange -Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(115) Sort -Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(116) SortMergeJoin -Left keys [1]: [p_partkey#X] -Right keys [1]: [l_partkey#X] -Join condition: None - -(117) Project -Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] - -(118) Exchange -Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(119) Sort -Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 - -(120) Scan parquet -Output [2]: [s_suppkey#X, s_nationkey#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] -ReadSchema: struct - -(121) Filter -Input [2]: [s_suppkey#X, s_nationkey#X] -Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(122) Exchange -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(123) Sort -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(124) SortMergeJoin -Left keys [1]: [l_suppkey#X] -Right keys [1]: [s_suppkey#X] -Join condition: None - -(125) Project -Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] - -(126) Exchange -Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(127) Sort -Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -Arguments: [l_suppkey#X ASC NULLS FIRST, l_partkey#X ASC NULLS FIRST], false, 0 - -(128) Scan parquet -Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] -ReadSchema: struct - -(129) Filter -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) - -(130) Exchange -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(131) Sort -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -Arguments: [ps_suppkey#X ASC NULLS FIRST, ps_partkey#X ASC NULLS FIRST], false, 0 - -(132) SortMergeJoin -Left keys [2]: [l_suppkey#X, l_partkey#X] -Right keys [2]: [ps_suppkey#X, ps_partkey#X] -Join condition: None - -(133) Project -Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] - -(134) Exchange -Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(135) Sort -Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(136) Scan parquet -Output [2]: [o_orderkey#X, o_orderdate#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(o_orderkey)] -ReadSchema: struct - -(137) Filter -Input [2]: [o_orderkey#X, o_orderdate#X] -Condition : isnotnull(o_orderkey#X) - -(138) Exchange -Input [2]: [o_orderkey#X, o_orderdate#X] -Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(139) Sort -Input [2]: [o_orderkey#X, o_orderdate#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(140) SortMergeJoin -Left keys [1]: [l_orderkey#X] -Right keys [1]: [o_orderkey#X] -Join condition: None - -(141) Project -Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] - -(142) Exchange -Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(143) Sort -Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 - -(144) Scan parquet -Output [2]: [n_nationkey#X, n_name#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(n_nationkey)] -ReadSchema: struct - -(145) Filter -Input [2]: [n_nationkey#X, n_name#X] -Condition : isnotnull(n_nationkey#X) - -(146) Exchange -Input [2]: [n_nationkey#X, n_name#X] -Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(147) Sort -Input [2]: [n_nationkey#X, n_name#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 - -(148) SortMergeJoin -Left keys [1]: [s_nationkey#X] -Right keys [1]: [n_nationkey#X] -Join condition: None - -(149) Project -Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] -Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] - -(150) HashAggregate -Input [3]: [nation#X, o_year#X, amount#X] -Keys [2]: [nation#X, o_year#X] -Functions [1]: [partial_sum(amount#X)] -Aggregate Attributes [2]: [sum#X, isEmpty#X] -Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] - -(151) Exchange -Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(152) HashAggregate -Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -Keys [2]: [nation#X, o_year#X] -Functions [1]: [sum(amount#X)] -Aggregate Attributes [1]: [sum(amount#X)#X] -Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] - -(153) Exchange -Input [3]: [nation#X, o_year#X, sum_profit#X] -Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] - -(154) Sort -Input [3]: [nation#X, o_year#X, sum_profit#X] -Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 - -(155) AdaptiveSparkPlan -Output [3]: [nation#X, o_year#X, sum_profit#X] -Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala index 7341b179f57..64e50b44105 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala @@ -90,7 +90,6 @@ abstract class VeloxTPCHSuite extends VeloxTPCHTableSupport { def shouldCheckGoldenFiles(): Boolean = { Seq("v1", "v1-bhj").contains(subType()) && ( formatSparkVersion match { - case "33" => true case "34" => true case "35" => true case _ => false diff --git a/dev/bloop-test.sh b/dev/bloop-test.sh index 5e3c9a9f2be..1d4730e1181 100755 --- a/dev/bloop-test.sh +++ b/dev/bloop-test.sh @@ -73,7 +73,6 @@ declare -A MODULE_MAP=( # Shims modules ["shims/common"]="spark-sql-columnar-shims-common" - ["shims/spark33"]="spark-sql-columnar-shims-spark33" ["shims/spark34"]="spark-sql-columnar-shims-spark34" ["shims/spark35"]="spark-sql-columnar-shims-spark35" ["shims/spark40"]="spark-sql-columnar-shims-spark40" @@ -82,7 +81,6 @@ declare -A MODULE_MAP=( # Unit test modules ["gluten-ut/common"]="gluten-ut-common" ["gluten-ut/test"]="gluten-ut-test" - ["gluten-ut/spark33"]="gluten-ut-spark33" ["gluten-ut/spark34"]="gluten-ut-spark34" ["gluten-ut/spark35"]="gluten-ut-spark35" ["gluten-ut/spark40"]="gluten-ut-spark40" diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh index 563f06e0b52..91ecc343bc3 100755 --- a/dev/builddeps-veloxbe.sh +++ b/dev/builddeps-veloxbe.sh @@ -210,7 +210,7 @@ if [ "$ENABLE_VCPKG" = "ON" ]; then fi # Supported Spark versions -SUPPORTED_SPARK_VERSIONS=("3.3" "3.4" "3.5" "4.0" "4.1" "ALL") +SUPPORTED_SPARK_VERSIONS=("3.4" "3.5" "4.0" "4.1" "ALL") # Check if SPARK_VERSION is in the supported list pattern=" $SPARK_VERSION " @@ -218,7 +218,7 @@ if [[ " ${SUPPORTED_SPARK_VERSIONS[*]} " =~ $pattern ]]; then echo "Building for Spark $SPARK_VERSION" else echo "Invalid Spark version: $SPARK_VERSION" - echo "Supported versions: 3.3 3.4 3.5 4.0 4.1 ALL" + echo "Supported versions: 3.4 3.5 4.0 4.1 ALL" exit 1 fi diff --git a/dev/docker/Dockerfile.centos8-dynamic-build b/dev/docker/Dockerfile.centos8-dynamic-build index 10ff0d823cb..bf960e61a7a 100644 --- a/dev/docker/Dockerfile.centos8-dynamic-build +++ b/dev/docker/Dockerfile.centos8-dynamic-build @@ -49,7 +49,6 @@ RUN set -ex; \ wget -nv ${mirror_host}/hadoop/common/hadoop-2.10.2/hadoop-2.10.2.tar.gz?action=download -O /opt/hadoop-2.10.2.tar.gz; \ git clone --depth=1 --branch "${GLUTEN_BRANCH}" "${GLUTEN_REPO}" /opt/gluten; \ cd /opt/gluten/.github/workflows/util/; \ - ./install-spark-resources.sh 3.3; \ ./install-spark-resources.sh 3.4; \ ./install-spark-resources.sh 3.5; \ ./install-spark-resources.sh 3.5-scala2.13; \ diff --git a/dev/docker/Dockerfile.centos9-dynamic-build b/dev/docker/Dockerfile.centos9-dynamic-build index 7879395657c..92bac1d692d 100644 --- a/dev/docker/Dockerfile.centos9-dynamic-build +++ b/dev/docker/Dockerfile.centos9-dynamic-build @@ -41,7 +41,6 @@ RUN set -ex; \ wget -nv ${mirror_host}/celeborn/celeborn-0.6.3/apache-celeborn-0.6.3-bin.tgz?action=download -O /opt/apache-celeborn-0.6.3-bin.tgz; \ git clone --depth=1 --branch "${GLUTEN_BRANCH}" "${GLUTEN_REPO}" /opt/gluten; \ cd /opt/gluten/.github/workflows/util/; \ - ./install-spark-resources.sh 3.3; \ ./install-spark-resources.sh 3.4; \ ./install-spark-resources.sh 3.5; \ ./install-spark-resources.sh 3.5-scala2.13; \ diff --git a/dev/docker/ubuntu/Dockerfile.ubuntu22-m2-cache b/dev/docker/ubuntu/Dockerfile.ubuntu22-m2-cache index 237bdd5acec..bc563f43dc7 100644 --- a/dev/docker/ubuntu/Dockerfile.ubuntu22-m2-cache +++ b/dev/docker/ubuntu/Dockerfile.ubuntu22-m2-cache @@ -54,11 +54,6 @@ RUN set -ex; \ # Spark 3.4 $MVN_CMD -Pjava-17,spark-3.4,backends-velox,hadoop-3.3,spark-ut \ -Piceberg,iceberg-test,delta,paimon \ - dependency:go-offline -DskipTests || true; \ - - # Spark 3.3 - $MVN_CMD -Pjava-17,spark-3.3,backends-velox,hadoop-3.3,spark-ut \ - -Piceberg,delta,paimon \ dependency:go-offline -DskipTests || true WORKDIR /work diff --git a/dev/format-scala-code.sh b/dev/format-scala-code.sh index bec09380a30..d3ec8d48954 100755 --- a/dev/format-scala-code.sh +++ b/dev/format-scala-code.sh @@ -22,7 +22,7 @@ MVN_CMD="${BASEDIR}/../build/mvn" # If a new profile is introduced for new modules, please add it here to ensure # the new modules are covered. PROFILES="-Pbackends-velox,backends-clickhouse -Pceleborn,uniffle -Piceberg,delta,hudi,paimon \ - -Pspark-3.3,spark-3.4,spark-3.5,spark-4.0,spark-4.1 -Pspark-ut -Pkafka" + -Pspark-3.4,spark-3.5,spark-4.0,spark-4.1 -Pspark-ut -Pkafka" COMMAND=$1 diff --git a/dev/release/build-release.sh b/dev/release/build-release.sh index 5ffcd561344..0f7a23ee404 100755 --- a/dev/release/build-release.sh +++ b/dev/release/build-release.sh @@ -25,22 +25,6 @@ cd ${GLUTEN_HOME} ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF --build_tests=OFF --build_benchmarks=OFF \ --build_examples=OFF --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON -JAVA_VERSION=$("java" -version 2>&1 | awk -F '"' '/version/ {print $2}') - -if [[ $JAVA_VERSION == 1.8* ]]; then - echo "Java 8 is being used." -else - echo "Error: Java 8 is required. Current version is $JAVA_VERSION." - exit 1 -fi - -# Build Gluten for Spark 3.3 with Java 8. All feature modules are enabled. -for spark_version in 3.3 -do - ${GLUTEN_HOME}/build/mvn clean install -Pbackends-velox -Pspark-${spark_version} -Pceleborn,uniffle \ - -Piceberg,delta,hudi,paimon -DskipTests -done - sudo curl -Lo /etc/yum.repos.d/corretto.repo https://yum.corretto.aws/corretto.repo sudo yum install -y java-17-amazon-corretto-devel export JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto diff --git a/dev/release/package-release.sh b/dev/release/package-release.sh index daf91e22a9a..21a772bd060 100755 --- a/dev/release/package-release.sh +++ b/dev/release/package-release.sh @@ -51,7 +51,7 @@ done pushd $GLUTEN_HOME/release/ -SPARK_VERSIONS="3.3 3.4 3.5 4.0 4.1" +SPARK_VERSIONS="3.4 3.5 4.0 4.1" for v in $SPARK_VERSIONS; do # Spark 4.x requires Scala 2.13; the spark-4.x Maven profiles enforce it. diff --git a/dev/run-scala-test.sh b/dev/run-scala-test.sh index df87b09d7db..8e98bca1977 100755 --- a/dev/run-scala-test.sh +++ b/dev/run-scala-test.sh @@ -160,7 +160,6 @@ declare -A MODULE_MAP=( # Shims modules (Java only, no scala subdirectory) ["spark-sql-columnar-shims-common"]="shims/common:java" - ["spark-sql-columnar-shims-spark33"]="shims/spark33:java" ["spark-sql-columnar-shims-spark34"]="shims/spark34:java" ["spark-sql-columnar-shims-spark35"]="shims/spark35:java" ["spark-sql-columnar-shims-spark40"]="shims/spark40:java" @@ -169,7 +168,6 @@ declare -A MODULE_MAP=( # Unit test modules ["gluten-ut-common"]="gluten-ut/common:scala" ["gluten-ut-test"]="gluten-ut/test:scala" - ["gluten-ut-spark33"]="gluten-ut/spark33:scala" ["gluten-ut-spark34"]="gluten-ut/spark34:scala" ["gluten-ut-spark35"]="gluten-ut/spark35:scala" ["gluten-ut-spark40"]="gluten-ut/spark40:scala" diff --git a/docs/developers/HowTo.md b/docs/developers/HowTo.md index 147a64bc426..19eb84544eb 100644 --- a/docs/developers/HowTo.md +++ b/docs/developers/HowTo.md @@ -61,7 +61,7 @@ mvn test -Pspark-3.5 -Pbackends-velox -pl backends-velox \ - After the above operations, the example files are generated under `${GLUTEN_HOME}/backends-velox` - You can check it by the command `tree ${GLUTEN_HOME}/backends-velox/generated-native-benchmark/` -- You may replace `-Pspark-3.5` with `-Pspark-3.3` or `-Pspark-3.4` for earlier Spark versions +- You may replace `-Pspark-3.5` with `-Pspark-3.4` for an earlier Spark version ```shell $ tree ${GLUTEN_HOME}/backends-velox/generated-native-benchmark/ diff --git a/docs/developers/HowToRelease.md b/docs/developers/HowToRelease.md index 8025f3e45f0..897147e5267 100644 --- a/docs/developers/HowToRelease.md +++ b/docs/developers/HowToRelease.md @@ -132,11 +132,10 @@ mkdir -p release cp -R package/target/* release/ ``` -The directory should end up holding these five JARs, matching the Spark and Scala combinations +The directory should end up holding these four JARs, matching the Spark and Scala combinations `package-release.sh` expects: ``` -gluten-velox-bundle-spark3.3_2.12-linux_amd64-1.7.0.jar gluten-velox-bundle-spark3.4_2.12-linux_amd64-1.7.0.jar gluten-velox-bundle-spark3.5_2.12-linux_amd64-1.7.0.jar gluten-velox-bundle-spark4.0_2.13-linux_amd64-1.7.0.jar @@ -168,7 +167,6 @@ ls -1 release/*.tar.gz ``` release/apache-gluten-1.7.0-src.tar.gz -release/apache-gluten-1.7.0-bin-spark-3.3.tar.gz release/apache-gluten-1.7.0-bin-spark-3.4.tar.gz release/apache-gluten-1.7.0-bin-spark-3.5.tar.gz release/apache-gluten-1.7.0-bin-spark-4.0.tar.gz @@ -349,7 +347,6 @@ one binary archive per supported Spark version, each with its `.asc` and `.sha51 ``` apache-gluten-1.7.0-src.tar.gz{,.asc,.sha512} -apache-gluten-1.7.0-bin-spark-3.3.tar.gz{,.asc,.sha512} apache-gluten-1.7.0-bin-spark-3.4.tar.gz{,.asc,.sha512} apache-gluten-1.7.0-bin-spark-3.5.tar.gz{,.asc,.sha512} apache-gluten-1.7.0-bin-spark-4.0.tar.gz{,.asc,.sha512} diff --git a/docs/developers/MicroBenchmarks.md b/docs/developers/MicroBenchmarks.md index ce0996a6cc2..65aa6873fba 100644 --- a/docs/developers/MicroBenchmarks.md +++ b/docs/developers/MicroBenchmarks.md @@ -36,7 +36,7 @@ generate example input files: cd /path/to/gluten/ ./dev/buildbundle-veloxbe.sh --build_tests=ON --build_benchmarks=ON -# Run test to generate input data files. If you are using spark 3.3, replace -Pspark-3.5 with -Pspark-3.3. +# Run test to generate input data files. If you are using spark 3.4, replace -Pspark-3.5 with -Pspark-3.4. mvn test -Pspark-3.5 -Pbackends-velox -pl backends-velox -am \ -DtagsToInclude="org.apache.gluten.tags.GenerateExample" -Dtest=none -DfailIfNoTests=false -Dexec.skip ``` diff --git a/docs/developers/NewToGluten.md b/docs/developers/NewToGluten.md index 16f0463247d..1a6c9d0f6eb 100644 --- a/docs/developers/NewToGluten.md +++ b/docs/developers/NewToGluten.md @@ -13,8 +13,8 @@ Gluten supports Ubuntu 20.04/22.04, CentOS 7/8, and MacOS. ### JDK -Gluten supports JDK 8 for Spark 3.3, 3.4, and 3.5. For Spark 3.3 and later versions, Gluten -also supports JDK 11 and 17. +Gluten supports JDK 8 for Spark 3.4 and 3.5, and also supports JDK 11 and 17 for those +versions. Note: Starting with Spark 4.0, the minimum required JDK version is 17. JDK 21 and 25 are also supported for Spark 4.0 and later versions. diff --git a/docs/developers/clickhouse-backend-debug.md b/docs/developers/clickhouse-backend-debug.md index c2e273583d4..d5c518ad65e 100644 --- a/docs/developers/clickhouse-backend-debug.md +++ b/docs/developers/clickhouse-backend-debug.md @@ -18,7 +18,7 @@ parent: /developer-overview/ 2. Maven Build Gluten ClickHouse with Profile ``` - mvn clean install -DskipTests -P delta -Pbackends-clickhouse -Pspark-3.3 -Pspark-ut + mvn clean install -DskipTests -P delta -Pbackends-clickhouse -Pspark-3.5 -Pspark-ut ``` 3. Set Maven Profiles in IntelliJ IDEA @@ -31,9 +31,9 @@ parent: /developer-overview/ ![gluten-debug-idea-config.png](../image/ClickHouse/gluten-debug-idea-config.png) VM Options: - `-Dgluten.test.data.path=/data -Dspark.gluten.sql.columnar.libpath=/path/to/gluten/cpp-ch/build/utils/extern-local-engine/libch.so -Dspark.test.home=/path/to/spark33` + `-Dgluten.test.data.path=/data -Dspark.gluten.sql.columnar.libpath=/path/to/gluten/cpp-ch/build/utils/extern-local-engine/libch.so -Dspark.test.home=/tmp/spark35` > Download tpcds-data in https://gluten-nginx.kyligence.com/dataset/ - > Download spark33 using `git clone --depth 1 --branch v3.3.1 https://github.com/apache/spark.git /tmp/spark33` + > Download spark35 using `git clone --depth 1 --branch v3.5.5 https://github.com/apache/spark.git /tmp/spark35` Environment Variables: `LD_PRELOAD=/path/to/gluten/cpp-ch/build/utils/extern-local-engine/libch.so:/usr/lib/jvm/java-1.8.0-openjdk-amd64/jre/lib/amd64/libjsig.so` diff --git a/docs/developers/dev-container.md b/docs/developers/dev-container.md index bc8c0974d2e..6cdf7d8f233 100644 --- a/docs/developers/dev-container.md +++ b/docs/developers/dev-container.md @@ -57,7 +57,7 @@ Build the native backend and the Spark 3.5 jars: | `--run_setup_script=OFF` | Velox's third-party libraries are already installed; `ON` rebuilds them from source. | | `--build_arrow=OFF` | Arrow is already installed under `/usr/local` and its jars are in `~/.m2`. | | `--build_tests=ON` | Also builds the C++ unit tests. Drop it if you only need the jars. | -| `--spark_version=3.5` | The default, `ALL`, runs five Maven builds (Spark 3.3 to 4.1). | +| `--spark_version=3.5` | The default, `ALL`, runs four Maven builds (Spark 3.4 to 4.1). | ### Build for Spark 4.1 diff --git a/docs/get-started/ClickHouse.md b/docs/get-started/ClickHouse.md index c0dd4002fc3..dba6570bc12 100644 --- a/docs/get-started/ClickHouse.md +++ b/docs/get-started/ClickHouse.md @@ -48,7 +48,7 @@ You can also refer to [How-to-Build-ClickHouse-on-Linux](https://clickhouse.com/ You need to install the following software manually: - Java 8 - Maven 3.6.3 or higher version -- Spark 3.3.1 or higher version +- Spark 3.5.5 Then, get Gluten code: ```shell @@ -173,14 +173,14 @@ The result is in `$clickhouse_root/build/utils/extern-local-engine/libch.so`. The prerequisites are the same as the one mentioned above. Compile Gluten with ClickHouse backend through maven: -- for Spark 3.3.1 +- for Spark 3.5.5 ``` git clone https://github.com/apache/gluten.git cd gluten/ export MAVEN_OPTS="-Xmx8g -XX:ReservedCodeCacheSize=2g" - mvn clean install -Pbackends-clickhouse -Phadoop-2.7.4 -Pspark-3.3 -Dhadoop.version=2.8.5 -DskipTests -Dcheckstyle.skip - ls -al backends-clickhouse/target/gluten-XXXXX-spark-3.3-jar-with-dependencies.jar + mvn clean install -Pbackends-clickhouse -Pdelta -Phadoop-2.7.4 -Pspark-3.5 -Dhadoop.version=2.8.5 -DskipTests -Dcheckstyle.skip + ls -al backends-clickhouse/target/gluten-XXXXX-spark-3.5-jar-with-dependencies.jar ``` ### Gluten in local Spark Thrift Server @@ -188,22 +188,22 @@ The prerequisites are the same as the one mentioned above. Compile Gluten with C #### Prepare working directory -- for Spark 3.3.1 +- for Spark 3.5.5 ``` -tar zxf spark-3.3.1-bin-hadoop2.7.tgz -cd spark-3.3.1-bin-hadoop2.7 -#download delta-core_2.12-2.2.0.jar and delta-storage-2.2.0.jar -wget https://repo1.maven.org/maven2/io/delta/delta-core_2.12/2.2.0/delta-core_2.12-2.2.0.jar -P ./jars -wget https://repo1.maven.org/maven2/io/delta/delta-storage/2.2.0/delta-storage-2.2.0.jar -P ./jars -cp gluten-XXXXX-spark-3.3-jar-with-dependencies.jar jars/ +tar zxf spark-3.5.5-bin-hadoop3.tgz +cd spark-3.5.5-bin-hadoop3 +#download delta-spark_2.12-3.3.2.jar and delta-storage-3.3.2.jar +wget https://repo1.maven.org/maven2/io/delta/delta-spark_2.12/3.3.2/delta-spark_2.12-3.3.2.jar -P ./jars +wget https://repo1.maven.org/maven2/io/delta/delta-storage/3.3.2/delta-storage-3.3.2.jar -P ./jars +cp gluten-XXXXX-spark-3.5-jar-with-dependencies.jar jars/ ``` #### Query local data ##### Start Spark Thriftserver on local ``` -cd spark-3.5.0-bin-hadoop3 +cd spark-3.5.5-bin-hadoop3 ./sbin/start-thriftserver.sh \ --master local[3] \ --driver-memory 10g \ @@ -382,7 +382,7 @@ export HADOOP_CONF_DIR=/path_to_spark/conf ```bash hdfs_conf_file=/your_local_path/hdfs-site.xml -cd spark-3.5.0-bin-hadoop3 +cd spark-3.5.5-bin-hadoop3 # add a new option: spark.gluten.sql.columnar.backend.ch.runtime_config.hdfs.libhdfs3_conf ./sbin/start-thriftserver.sh \ --master local[3] \ @@ -567,7 +567,7 @@ Please refer to [Data-preparation](#data-preparation) to generate MergeTree part #### Run Spark Thriftserver ``` -cd spark-3.5.0-bin-hadoop3 +cd spark-3.5.5-bin-hadoop3 ./sbin/start-thriftserver.sh \ --master spark://master-ip:7070 --deploy-mode client \ --driver-memory 16g --driver-cores 4 \ @@ -661,7 +661,7 @@ First refer to this URL(https://github.com/apache/celeborn) to setup a celeborn When compiling the Gluten Java module, it's required to enable `celeborn` profile, as follows: ``` -mvn clean package -Pbackends-clickhouse -Pspark-3.3 -Pceleborn -DskipTests +mvn clean package -Pbackends-clickhouse -Pdelta -Pspark-3.5 -Pceleborn -DskipTests ``` Then add the Spark Celeborn Client packages to your Spark application's classpath(usually add them into `$SPARK_HOME/jars`). diff --git a/docs/get-started/Velox.md b/docs/get-started/Velox.md index 03aa02ef007..c66d420542b 100644 --- a/docs/get-started/Velox.md +++ b/docs/get-started/Velox.md @@ -9,18 +9,18 @@ parent: Getting-Started | Type | Version | |-------|-------------------------------------| -| Spark | 3.3.1, 3.4.4, 3.5.5, 4.0.2, 4.1.1 | +| Spark | 3.4.4, 3.5.5, 4.0.2, 4.1.1 | | OS | Ubuntu20.04/22.04, Centos7/8 | | jdk | openjdk8/jdk17 | | scala | 2.12 | -Note: Spark 4.0 and 4.1 require JDK 17+ and Scala 2.13 (build with `-Pspark-4.0` or `-Pspark-4.1` plus `-Pjava-17 -Pscala-2.13`). Spark 3.3 to 3.5 can be built with JDK 8/17 and Scala 2.12. +Note: Spark 4.0 and 4.1 require JDK 17+ and Scala 2.13 (build with `-Pspark-4.0` or `-Pspark-4.1` plus `-Pjava-17 -Pscala-2.13`). Spark 3.4 and 3.5 can be built with JDK 8/17 and Scala 2.12. # Prerequisite Currently, the statically built Gluten+Velox backend supports all Linux OSes but is only tested on **Ubuntu 20.04/22.04/CentOS 7/8**. The dynamically built backend supports **Ubuntu 20.04/22.04/CentOS 7/8** and their variants. -Currently, the officially supported Spark versions are 3.3.1, 3.4.4, 3.5.5, 4.0.2 and 4.1.1. +Currently, the officially supported Spark versions are 3.4.4, 3.5.5, 4.0.2 and 4.1.1. We need to set up the `JAVA_HOME` env. Currently, Gluten supports **java 8** and **java 17**. @@ -94,8 +94,6 @@ Currently, Gluten uses an [IBM Velox fork](https://github.com/IBM/velox), which ## compile Gluten java module and create package jar cd /path/to/gluten -# For spark3.3.x -mvn clean package -Pbackends-velox -Pspark-3.3 -DskipTests # For spark3.4.x mvn clean package -Pbackends-velox -Pspark-3.4 -DskipTests # For spark3.5.x (default) @@ -272,7 +270,7 @@ First refer to this URL(https://github.com/apache/celeborn) to setup a celeborn When compiling the Gluten Java module, it's required to enable `celeborn` profile, as follows: ``` -mvn clean package -Pbackends-velox -Pspark-3.3 -Pceleborn -DskipTests +mvn clean package -Pbackends-velox -Pspark-3.5 -Pceleborn -DskipTests ``` Then add the Gluten and Spark Celeborn Client packages to your Spark application's classpath (usually add them into `$SPARK_HOME/jars`). @@ -329,7 +327,7 @@ First refer to this URL(https://uniffle.apache.org/docs/intro) to get start with When compiling the Gluten Java module, it's required to enable `uniffle` profile, as follows: ``` -mvn clean package -Pbackends-velox -Pspark-3.3 -Puniffle -DskipTests +mvn clean package -Pbackends-velox -Pspark-3.5 -Puniffle -DskipTests ``` Then add the Uniffle and Spark Celeborn Client packages to your Spark application's classpath (usually add them into `$SPARK_HOME/jars`). @@ -369,7 +367,7 @@ Gluten with velox backend supports [DeltaLake](https://delta.io/) table. First of all, compile gluten-delta module by a `delta` profile, as follows: ``` -mvn clean package -Pbackends-velox -Pspark-3.3 -Pdelta -DskipTests +mvn clean package -Pbackends-velox -Pspark-3.5 -Pdelta -DskipTests ``` Once built successfully, delta features will be included in gluten-velox-bundle-X jar. Then you can query delta table by gluten/velox without scan's fallback. @@ -386,7 +384,8 @@ Gluten with velox backend supports [Iceberg](https://iceberg.apache.org/) table. First, compile the gluten-iceberg module with the `iceberg` profile, as follows: ``` -mvn clean package -Pbackends-velox -Pspark-3.3 -Piceberg -DskipTests +# For spark3.5.x with Iceberg (requires JDK 11+; CI and the release build use JDK 17) +mvn clean package -Pbackends-velox -Pspark-3.5 -Piceberg -DskipTests ``` Once built successfully, iceberg features will be included in the gluten-velox-bundle-X jar. You can then query iceberg tables via Gluten/Velox without falling back on scan. @@ -397,7 +396,7 @@ Gluten with velox backend supports [Hudi](https://hudi.apache.org/) table. Curre ## Paimon Support -Gluten with velox backend supports [Paimon](https://paimon.apache.org/) table. Currently, only non-pk table is supported, and the Spark version needs to be >= 3.3. +Gluten with velox backend supports [Paimon](https://paimon.apache.org/) table. Currently, only non-pk table is supported. ### How to use @@ -414,7 +413,7 @@ Once built successfully, paimon features will be included in the gluten-velox-bu First, compile the gluten-hudi module with the `hudi` profile, as follows: ``` -mvn clean package -Pbackends-velox -Pspark-3.3 -Phudi -DskipTests +mvn clean package -Pbackends-velox -Pspark-3.5 -Phudi -DskipTests ``` Once built successfully, hudi features will be included in the gluten-velox-bundle-X jar. You can then query hudi **COW** tables via Gluten/Velox without falling back on scan. diff --git a/docs/get-started/build-guide.md b/docs/get-started/build-guide.md index fc6d5012bf2..a5244fd4e89 100644 --- a/docs/get-started/build-guide.md +++ b/docs/get-started/build-guide.md @@ -28,7 +28,7 @@ Please set them via `--`, e.g. `--build_type=Release`. | build_velox_tests | Build Velox tests. | OFF | | build_velox_benchmarks | Build Velox benchmarks (velox_tests and connectors will be disabled if ON) | OFF | | build_arrow | Build arrow java/cpp and install the libs in local. Can turn it OFF after first build. | ON | -| spark_version | Build for specified version of Spark(3.3, 3.4, 3.5, 4.0, 4.1, ALL). `ALL` means build for all versions. | ALL | +| spark_version | Build for specified version of Spark(3.4, 3.5, 4.0, 4.1, ALL). `ALL` means build for all versions. | ALL | ### Environment variables for build These environment variables can be set before running build scripts to control build behavior. @@ -66,7 +66,6 @@ The below parameters can be set via `-P` for mvn. | delta | Build Gluten with Delta Lake support. | disabled | | iceberg | Build Gluten with Iceberg support. | disabled | | hudi | Build Gluten with Hudi support. | disabled | -| spark-3.3 | Build Gluten for Spark 3.3. | disabled | | spark-3.4 | Build Gluten for Spark 3.4. | disabled | | spark-3.5 | Build Gluten for Spark 3.5. | enabled | | spark-4.0 | Build Gluten for Spark 4.0. Requires JDK 17+ and Scala 2.13. | disabled | @@ -78,7 +77,6 @@ It's name pattern is `gluten--bundle-spark_< | Spark Version | spark.bundle.version | scala.binary.version | |---------------|----------------------|----------------------| -| 3.3.1 | 3.3 | 2.12 | | 3.4.4 | 3.4 | 2.12 | | 3.5.5 | 3.5 | 2.12 | | 4.0.2 | 4.0 | 2.13 | diff --git a/docs/get-started/getting-started.md b/docs/get-started/getting-started.md index 0842dc0472d..b85c64708e3 100644 --- a/docs/get-started/getting-started.md +++ b/docs/get-started/getting-started.md @@ -23,9 +23,9 @@ Gluten supports two native backends: ### 1. Prerequisites - **OS**: Ubuntu 20.04/22.04 or CentOS 7/8 (other Linux distros may work with static build but are not officially tested) -- **JDK**: OpenJDK 8 or 17 (Spark 4.0 requires JDK 17+) -- **Spark**: 3.3.1, 3.4.4, 3.5.5, 4.0.2, or 4.1.1 -- **Scala**: 2.12 (Spark 4.0 requires Scala 2.13) +- **JDK**: OpenJDK 8 or 17 (Spark 4.x requires JDK 17+) +- **Spark**: 3.4.4, 3.5.5, 4.0.2, or 4.1.1 +- **Scala**: 2.12 (Spark 4.x requires Scala 2.13) ### 2. Build diff --git a/docs/index.md b/docs/index.md index e5b769d1a49..ad1c1ebb736 100644 --- a/docs/index.md +++ b/docs/index.md @@ -61,4 +61,4 @@ There are several key components in Gluten: * **Columnar Shuffle**: shuffles Gluten columnar data. The shuffle service still reuses the one in Spark core. A kind of columnar exchange operator is implemented to support Gluten columnar data format. * **Fallback Mechanism**: supports falling back to Vanilla spark for unsupported operators. Gluten ColumnarToRow (C2R) and RowToColumnar (R2C) will convert Gluten columnar data and Spark's internal row data if needed. Both C2R and R2C are implemented in native code as well * **Metrics**: collected from Gluten native engine to help identify bugs, performance bottlenecks, etc. The metrics are displayed in Spark UI. -* **Shim Layer**: supports multiple Spark versions. We plan to only support Spark's latest 3-4 releases. Currently, Spark 3.3, 3.4, 3.5, 4.0, and 4.1 are supported. +* **Shim Layer**: supports multiple Spark versions. We plan to only support Spark's latest 3-4 releases. Currently, Spark 3.4, 3.5, 4.0, and 4.1 are supported. diff --git a/docs/velox-backend-limitations.md b/docs/velox-backend-limitations.md index a39f2a40fb3..02fbfa152c8 100644 --- a/docs/velox-backend-limitations.md +++ b/docs/velox-backend-limitations.md @@ -57,25 +57,7 @@ Spark has `spark.sql.parquet.datetimeRebaseModeInWrite` config to decide whether or Proleptic Gregorian calendar should be used during parquet writing for dates/timestamps. If the parquet to read is written by Spark with this config as true, Velox's TableScan will output different result when reading it back. -#### Partition write (For Spark3.3) - -Gluten only supports static partition writes and does not support dynamic partition writes. - -```scala -spark.sql("CREATE TABLE t (c int, d long, e long) STORED AS PARQUET partitioned by (c, d)") -spark.sql("INSERT OVERWRITE TABLE t partition(c=1, d=2) SELECT 3 as e") -``` -Gluten does not support dynamic partition write and bucket write, Exception may be raised if you use. e.g., - -```scala -spark.range(100).selectExpr("id as c1", "id % 7 as p") - .write - .format("parquet") - .partitionBy("p") - .save(f.getCanonicalPath) -``` - -#### Partition write (For Spark3.4 and later) +#### Partition write Gluten supports static partition writes and dynamic partition writes. @@ -94,18 +76,7 @@ spark.range(100).selectExpr("id as c1", "id % 7 as p") .save(f.getCanonicalPath) ``` -#### CTAS write (For Spark3.3) - -Gluten does not create table as select. It may raise exception. e.g., - -```scala -spark.range(100).toDF("id") - .write - .format("parquet") - .saveAsTable("velox_ctas") -``` - -#### CTAS write (For Spark3.4 and later) +#### CTAS write Gluten supports create table as select with parquet file format. diff --git a/ep/build-clickhouse/src/package.sh b/ep/build-clickhouse/src/package.sh index 91d03e3920d..a796a1074ea 100755 --- a/ep/build-clickhouse/src/package.sh +++ b/ep/build-clickhouse/src/package.sh @@ -33,7 +33,7 @@ function detect_os_version() { } detect_os_version -DEFAULT_SPARK_PROFILE="spark-3.3" +DEFAULT_SPARK_PROFILE="spark-3.5" function get_project_version() { cd "${GLUTEN_SOURCE}" # use mvn command to get project version @@ -48,9 +48,7 @@ OS_ARCH=$(uname -m) PACKAGE_NAME=gluten-${BUILD_VERSION}-${OS_VERSION}-${OS_ARCH} PACKAGE_DIR_PATH="${GLUTEN_SOURCE}"/dist/"${PACKAGE_NAME}" -# spark_scala_versions=("3.3_2.12" "3.5_2.13") -# TODO: support spark 3.5 later -spark_scala_versions=("3.3_2.12") +spark_scala_versions=("3.5_2.12") # cleanup working directory [[ -d "${GLUTEN_SOURCE}"/dist/"${PACKAGE_NAME}" ]] && rm -rf "${GLUTEN_SOURCE}"/dist/"${PACKAGE_NAME}" diff --git a/ep/build-clickhouse/src/resources/bin/check-env.sh b/ep/build-clickhouse/src/resources/bin/check-env.sh index 7b771119504..e8436852872 100755 --- a/ep/build-clickhouse/src/resources/bin/check-env.sh +++ b/ep/build-clickhouse/src/resources/bin/check-env.sh @@ -81,8 +81,8 @@ function check_spark_version() { echo "SPARK_HOME=${SPARK_HOME}" SPARK_VERSION=$(cat ${SPARK_HOME}/RELEASE | grep "^Spark" | cut -d " " -f 2) SPARK_MAJOR_MINOR_VERSION=$(echo ${SPARK_VERSION} | cut -d '.' -f 1-2) - if [[ "${SPARK_MAJOR_MINOR_VERSION}" != "3.3" ]] && [[ "${SPARK_MAJOR_MINOR_VERSION}" != "3.4" ]] && [[ "${SPARK_MAJOR_MINOR_VERSION}" != "3.5" ]] && [[ "${SPARK_MAJOR_MINOR_VERSION}" != "4.0" ]]; then - echo "[ERROR] SPARK_VERSION ${SPARK_VERSION} which defined in $SPARK_HOME/RELEASE, is not supported. Please use spark 3.3, 3.4, 3.5, or 4.0." + if [[ "${SPARK_MAJOR_MINOR_VERSION}" != "3.5" ]]; then + echo "[ERROR] SPARK_VERSION ${SPARK_VERSION} which defined in $SPARK_HOME/RELEASE, is not supported. Please use spark 3.5." exit 1 fi export SPARK_MAJOR_MINOR_VERSION=${SPARK_MAJOR_MINOR_VERSION} diff --git a/ep/build-clickhouse/src/resources/bin/gluten.sh b/ep/build-clickhouse/src/resources/bin/gluten.sh index 3ae65b115c0..4270cd7b472 100755 --- a/ep/build-clickhouse/src/resources/bin/gluten.sh +++ b/ep/build-clickhouse/src/resources/bin/gluten.sh @@ -36,8 +36,8 @@ function start() { DRIVER_OPTIONS="${DRIVER_OPTIONS} $(cat ${GLUTEN_HOME}/conf/gluten.properties | grep "^spark.driver.extraJavaOptions" | cut -d "=" -f 2)" GLUTEN_JARS= - if [ "${SPARK_MAJOR_MINOR_VERSION}" == "3.3" ]; then - GLUTEN_JARS=${GLUTEN_HOME}/jars/spark33/* + if [ "${SPARK_MAJOR_MINOR_VERSION}" == "3.5" ]; then + GLUTEN_JARS=${GLUTEN_HOME}/jars/spark35/* else echo "Unsupported spark version: ${SPARK_MAJOR_MINOR_VERSION}" exit 1 diff --git a/gluten-delta/src-delta23/main/scala/org/apache/gluten/delta/DeltaDeletionVectorScanInfo.scala b/gluten-delta/src-delta23/main/scala/org/apache/gluten/delta/DeltaDeletionVectorScanInfo.scala deleted file mode 100644 index 084f3d3319f..00000000000 --- a/gluten-delta/src-delta23/main/scala/org/apache/gluten/delta/DeltaDeletionVectorScanInfo.scala +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.delta - -import org.apache.gluten.substrait.rel.DeltaLocalFilesNode.DeltaFileReadOptions - -import org.apache.spark.sql.execution.datasources.PartitionedFile - -import org.apache.hadoop.fs.Path - -import java.util.{Map => JMap} - -/** Reading deletion vectors natively requires Delta 3.3+, so there is nothing to materialize. */ -object DeltaDeletionVectorScanInfo { - def normalize( - partitionFiles: Seq[PartitionedFile], - tablePath: Path) - : Option[(Seq[JMap[String, Object]], Seq[DeltaFileReadOptions])] = { - normalize(partitionFiles, tablePath, None) - } - - def normalize( - partitionFiles: Seq[PartitionedFile], - tablePath: Path, - readMetrics: Option[DeletionVectorReadMetrics]) - : Option[(Seq[JMap[String, Object]], Seq[DeltaFileReadOptions])] = None -} diff --git a/gluten-delta/src-delta23/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala b/gluten-delta/src-delta23/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala deleted file mode 100644 index ca4665c0d0c..00000000000 --- a/gluten-delta/src-delta23/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.execution - -import org.apache.spark.sql.catalyst.expressions.Expression -import org.apache.spark.sql.execution.SparkPlan - -case class DeltaFilterExecTransformer(condition: Expression, child: SparkPlan) - extends FilterExecTransformerBase(condition, child) { - - override protected def withNewChildInternal(newChild: SparkPlan): DeltaFilterExecTransformer = - copy(child = newChild) -} diff --git a/gluten-delta/src-delta23/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala b/gluten-delta/src-delta23/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala deleted file mode 100644 index 9b720b19c5b..00000000000 --- a/gluten-delta/src-delta23/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.execution - -import org.apache.spark.sql.catalyst.expressions.NamedExpression -import org.apache.spark.sql.execution.SparkPlan - -case class DeltaProjectExecTransformer(projectList: Seq[NamedExpression], child: SparkPlan) - extends ProjectExecTransformerBase(projectList, child) { - - override protected def withNewChildInternal(newChild: SparkPlan): DeltaProjectExecTransformer = - copy(child = newChild) -} diff --git a/gluten-delta/src-delta23/main/scala/org/apache/gluten/extension/DeltaCDFRelationHelper.scala b/gluten-delta/src-delta23/main/scala/org/apache/gluten/extension/DeltaCDFRelationHelper.scala deleted file mode 100644 index 30bdb75e99a..00000000000 --- a/gluten-delta/src-delta23/main/scala/org/apache/gluten/extension/DeltaCDFRelationHelper.scala +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.extension - -import org.apache.spark.sql.{DataFrame, SparkSession} -import org.apache.spark.sql.delta.BatchCDFSchemaEndVersion -import org.apache.spark.sql.delta.commands.cdc.CDCReader - -object DeltaCDFRelationHelper { - def changesToBatchDF( - relation: CDCReader.DeltaCDFRelation, - spark: SparkSession): DataFrame = { - val deltaLog = relation.snapshotWithSchemaMode.snapshot.deltaLog - val latestVersion = deltaLog.update().version - val endingVersionForBatchSchema = - relation.endingVersion.map(v => latestVersion.min(v)).getOrElse(latestVersion) - val snapshotForBatchSchema = relation.snapshotWithSchemaMode.schemaMode match { - case BatchCDFSchemaEndVersion => deltaLog.getSnapshotAt(endingVersionForBatchSchema) - case _ => relation.snapshotWithSchemaMode.snapshot - } - val endVersion = relation.endingVersion.getOrElse(latestVersion) - - CDCReader.changesToBatchDF( - deltaLog, - relation.startingVersion.get, - endVersion, - spark, - readSchemaSnapshot = Some(snapshotForBatchSchema)) - } -} diff --git a/gluten-iceberg/src-iceberg3/main/java/org/apache/gluten/ContentFileUtil.java b/gluten-iceberg/src-iceberg3/main/java/org/apache/gluten/ContentFileUtil.java deleted file mode 100644 index 52c168aa79b..00000000000 --- a/gluten-iceberg/src-iceberg3/main/java/org/apache/gluten/ContentFileUtil.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten; - -import org.apache.iceberg.ContentFile; - -public class ContentFileUtil { - public static String getFilePath(ContentFile file) { - return file.path().toString(); - } -} diff --git a/gluten-iceberg/src-iceberg3/main/java/org/apache/gluten/IcebergDefaultValueUtil.java b/gluten-iceberg/src-iceberg3/main/java/org/apache/gluten/IcebergDefaultValueUtil.java deleted file mode 100644 index fff8cb444f1..00000000000 --- a/gluten-iceberg/src-iceberg3/main/java/org/apache/gluten/IcebergDefaultValueUtil.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten; - -import org.apache.iceberg.types.Types; - -public final class IcebergDefaultValueUtil { - private IcebergDefaultValueUtil() {} - - public static Object getInitialDefault(Types.NestedField field) { - return null; - } -} diff --git a/gluten-iceberg/src-iceberg5/main/java/org/apache/gluten/ContentFileUtil.java b/gluten-iceberg/src-iceberg5/main/java/org/apache/gluten/ContentFileUtil.java deleted file mode 100644 index 52c168aa79b..00000000000 --- a/gluten-iceberg/src-iceberg5/main/java/org/apache/gluten/ContentFileUtil.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten; - -import org.apache.iceberg.ContentFile; - -public class ContentFileUtil { - public static String getFilePath(ContentFile file) { - return file.path().toString(); - } -} diff --git a/gluten-iceberg/src-iceberg5/main/java/org/apache/gluten/IcebergDefaultValueUtil.java b/gluten-iceberg/src-iceberg5/main/java/org/apache/gluten/IcebergDefaultValueUtil.java deleted file mode 100644 index fff8cb444f1..00000000000 --- a/gluten-iceberg/src-iceberg5/main/java/org/apache/gluten/IcebergDefaultValueUtil.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten; - -import org.apache.iceberg.types.Types; - -public final class IcebergDefaultValueUtil { - private IcebergDefaultValueUtil() {} - - public static Object getInitialDefault(Types.NestedField field) { - return null; - } -} diff --git a/gluten-ut/pom.xml b/gluten-ut/pom.xml index 1316d878c78..996ee8203e5 100644 --- a/gluten-ut/pom.xml +++ b/gluten-ut/pom.xml @@ -200,12 +200,6 @@ - - spark-3.3 - - spark33 - - spark-3.4 diff --git a/gluten-ut/spark33/pom.xml b/gluten-ut/spark33/pom.xml deleted file mode 100644 index 9c3926ea9e0..00000000000 --- a/gluten-ut/spark33/pom.xml +++ /dev/null @@ -1,164 +0,0 @@ - - - - 4.0.0 - - org.apache.gluten - gluten-ut - 1.8.0-SNAPSHOT - ../pom.xml - - - gluten-ut-spark33 - jar - Gluten Unit Test Spark33 - - - - org.apache.gluten - gluten-ut-common - ${project.version} - test-jar - compile - - - org.apache.parquet - parquet-column - 1.12.3 - tests - test - - - - - - - org.apache.maven.plugins - maven-resources-plugin - - - net.alchim31.maven - scala-maven-plugin - - - org.apache.maven.plugins - maven-compiler-plugin - - - org.scalastyle - scalastyle-maven-plugin - - - com.diffplug.spotless - spotless-maven-plugin - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - org.scalatest - scalatest-maven-plugin - - . - - - - test - - test - - - - - - org.apache.maven.plugins - maven-jar-plugin - - - prepare-test-jar - - test-jar - - - - - - target/scala-${scala.binary.version}/classes - target/scala-${scala.binary.version}/test-classes - - - - - backends-clickhouse - - false - - - - org.apache.gluten - backends-clickhouse - ${project.version} - test - - - org.apache.celeborn - celeborn-client-spark-${spark.major.version}-shaded_${scala.binary.version} - ${celeborn.version} - test - - - - - - org.codehaus.mojo - build-helper-maven-plugin - - - add-sources - - add-test-source - - generate-sources - - - src/test/backends-clickhouse - - - - - - - - - - backends-velox - - false - - - - org.apache.gluten - backends-velox - ${project.version} - test - - - - - diff --git a/gluten-ut/spark33/src/test/backends-clickhouse/org/apache/gluten/execution/parquet/GlutenParquetV1FilterSuite2.scala b/gluten-ut/spark33/src/test/backends-clickhouse/org/apache/gluten/execution/parquet/GlutenParquetV1FilterSuite2.scala deleted file mode 100644 index 32c7784cff9..00000000000 --- a/gluten-ut/spark33/src/test/backends-clickhouse/org/apache/gluten/execution/parquet/GlutenParquetV1FilterSuite2.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.execution.parquet - -import org.apache.spark.SparkConf -import org.apache.spark.sql.execution.datasources.parquet.GlutenParquetV1FilterSuite - -/** testing use_local_format parquet reader. */ -class GlutenParquetV1FilterSuite2 extends GlutenParquetV1FilterSuite { - override def sparkConf: SparkConf = - super.sparkConf - .set("spark.gluten.sql.columnar.backend.ch.runtime_config.use_local_format", "true") -} diff --git a/gluten-ut/spark33/src/test/resources/log4j2.properties b/gluten-ut/spark33/src/test/resources/log4j2.properties deleted file mode 100644 index fb1cadec5f5..00000000000 --- a/gluten-ut/spark33/src/test/resources/log4j2.properties +++ /dev/null @@ -1,39 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -rootLogger.level = info -rootLogger.appenderRef.stdout.ref = STDOUT -rootLogger.appenderRef.file.ref = File - -#Console Appender -appender.console.type = Console -appender.console.name = STDOUT -appender.console.target = SYSTEM_OUT -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %maxLen{%m}{512}%n%ex{8}%n -appender.console.filter.threshold.type = ThresholdFilter -appender.console.filter.threshold.level = warn - -#File Appender -appender.file.type = File -appender.file.name = File -appender.file.fileName = target/unit-tests.log -appender.file.layout.type = PatternLayout -appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n%ex - -appender.file.filter.threshold.type = ThresholdFilter -appender.file.filter.threshold.level = info diff --git a/gluten-ut/spark33/src/test/resources/sql-tests/inputs/group-by-ordinal.sql b/gluten-ut/spark33/src/test/resources/sql-tests/inputs/group-by-ordinal.sql deleted file mode 100644 index b773396c050..00000000000 --- a/gluten-ut/spark33/src/test/resources/sql-tests/inputs/group-by-ordinal.sql +++ /dev/null @@ -1,96 +0,0 @@ --- group by ordinal positions - -create temporary view data as select * from values - (1, 1), - (1, 2), - (2, 1), - (2, 2), - (3, 1), - (3, 2) - as data(a, b); - --- basic case -select a, sum(b) from data group by 1; - --- constant case -select 1, 2, sum(b) from data group by 1, 2; - --- duplicate group by column -select a, 1, sum(b) from data group by a, 1; -select a, 1, sum(b) from data group by 1, 2; - --- group by a non-aggregate expression's ordinal -select a, b + 2, count(2) from data group by a, 2; - --- with alias -select a as aa, b + 2 as bb, count(2) from data group by 1, 2; - --- foldable non-literal: this should be the same as no grouping. -select sum(b) from data group by 1 + 0; - --- negative cases: ordinal out of range -select a, b from data group by -1; -select a, b from data group by 0; -select a, b from data group by 3; - --- negative case: position is an aggregate expression -select a, b, sum(b) from data group by 3; -select a, b, sum(b) + 2 from data group by 3; - --- negative case: nondeterministic expression -select a, rand(0), sum(b) -from -(select /*+ REPARTITION(1) */ a, b from data) group by a, 2; - --- negative case: star -select * from data group by a, b, 1; - --- group by ordinal followed by order by -select a, count(a) from (select 1 as a) tmp group by 1 order by 1; - --- group by ordinal followed by having -select count(a), a from (select 1 as a) tmp group by 2 having a > 0; - --- mixed cases: group-by ordinals and aliases -select a, a AS k, count(b) from data group by k, 1; - --- can use ordinal in CUBE -select a, b, count(1) from data group by cube(1, 2); - --- mixed cases: can use ordinal in CUBE -select a, b, count(1) from data group by cube(1, b); - --- can use ordinal with cube -select a, b, count(1) from data group by 1, 2 with cube; - --- can use ordinal in ROLLUP -select a, b, count(1) from data group by rollup(1, 2); - --- mixed cases: can use ordinal in ROLLUP -select a, b, count(1) from data group by rollup(1, b); - --- can use ordinal with rollup -select a, b, count(1) from data group by 1, 2 with rollup; - --- can use ordinal in GROUPING SETS -select a, b, count(1) from data group by grouping sets((1), (2), (1, 2)); - --- mixed cases: can use ordinal in GROUPING SETS -select a, b, count(1) from data group by grouping sets((1), (b), (a, 2)); - -select a, b, count(1) from data group by a, 2 grouping sets((1), (b), (a, 2)); - --- range error -select a, b, count(1) from data group by a, -1; - -select a, b, count(1) from data group by a, 3; - -select a, b, count(1) from data group by cube(-1, 2); - -select a, b, count(1) from data group by cube(1, 3); - --- turn off group by ordinal -set spark.sql.groupByOrdinal=false; - --- can now group by negative literal -select sum(b) from data group by -1; diff --git a/gluten-ut/spark33/src/test/resources/sql-tests/inputs/group-by.sql b/gluten-ut/spark33/src/test/resources/sql-tests/inputs/group-by.sql deleted file mode 100644 index 291a8478c7a..00000000000 --- a/gluten-ut/spark33/src/test/resources/sql-tests/inputs/group-by.sql +++ /dev/null @@ -1,297 +0,0 @@ --- Test aggregate operator with codegen on and off. ---CONFIG_DIM1 spark.sql.codegen.wholeStage=true ---CONFIG_DIM1 spark.sql.codegen.wholeStage=false,spark.sql.codegen.factoryMode=CODEGEN_ONLY ---CONFIG_DIM1 spark.sql.codegen.wholeStage=false,spark.sql.codegen.factoryMode=NO_CODEGEN - --- Test data. -CREATE OR REPLACE TEMPORARY VIEW testData AS SELECT * FROM VALUES -(1, 1), (1, 2), (2, 1), (2, 2), (3, 1), (3, 2), (null, 1), (3, null), (null, null) -AS testData(a, b); -CREATE OR REPLACE TEMPORARY VIEW testRegression AS SELECT * FROM VALUES -(1, 10, null), (2, 10, 11), (2, 20, 22), (2, 25, null), (2, 30, 35) -AS testRegression(k, y, x); -CREATE OR REPLACE TEMPORARY VIEW aggr AS SELECT * FROM VALUES -(0, 0), (0, 10), (0, 20), (0, 30), (0, 40), (1, 10), (1, 20), (2, 10), (2, 20), (2, 25), (2, 30), (3, 60), (4, null) -AS aggr(k, v); - --- Aggregate with empty GroupBy expressions. -SELECT a, COUNT(b) FROM testData; -SELECT COUNT(a), COUNT(b) FROM testData; - --- Aggregate with non-empty GroupBy expressions. -SELECT a, COUNT(b) FROM testData GROUP BY a; -SELECT a, COUNT(b) FROM testData GROUP BY b; -SELECT COUNT(a), COUNT(b) FROM testData GROUP BY a; - --- Aggregate grouped by literals. -SELECT 'foo', COUNT(a) FROM testData GROUP BY 1; - --- Aggregate grouped by literals (whole stage code generation). -SELECT 'foo' FROM testData WHERE a = 0 GROUP BY 1; - --- Aggregate grouped by literals (hash aggregate). -SELECT 'foo', APPROX_COUNT_DISTINCT(a) FROM testData WHERE a = 0 GROUP BY 1; - --- Aggregate grouped by literals (sort aggregate). -SELECT 'foo', MAX(STRUCT(a)) FROM testData WHERE a = 0 GROUP BY 1; - --- Aggregate with complex GroupBy expressions. -SELECT a + b, COUNT(b) FROM testData GROUP BY a + b; -SELECT a + 2, COUNT(b) FROM testData GROUP BY a + 1; -SELECT a + 1 + 1, COUNT(b) FROM testData GROUP BY a + 1; - --- Aggregate with nulls. -SELECT SKEWNESS(a), KURTOSIS(a), MIN(a), MAX(a), AVG(a), VARIANCE(a), STDDEV(a), SUM(a), COUNT(a) -FROM testData; - --- Aggregate with foldable input and multiple distinct groups. -SELECT COUNT(DISTINCT b), COUNT(DISTINCT b, c) FROM (SELECT 1 AS a, 2 AS b, 3 AS c) GROUP BY a; - --- Aliases in SELECT could be used in GROUP BY -SELECT a AS k, COUNT(b) FROM testData GROUP BY k; -SELECT a AS k, COUNT(b) FROM testData GROUP BY k HAVING k > 1; - --- GROUP BY alias with invalid col in SELECT list -SELECT a AS k, COUNT(non_existing) FROM testData GROUP BY k; - --- Aggregate functions cannot be used in GROUP BY -SELECT COUNT(b) AS k FROM testData GROUP BY k; - --- Test data. -CREATE OR REPLACE TEMPORARY VIEW testDataHasSameNameWithAlias AS SELECT * FROM VALUES -(1, 1, 3), (1, 2, 1) AS testDataHasSameNameWithAlias(k, a, v); -SELECT k AS a, COUNT(v) FROM testDataHasSameNameWithAlias GROUP BY a; - --- turn off group by aliases -set spark.sql.groupByAliases=false; - --- Check analysis exceptions -SELECT a AS k, COUNT(b) FROM testData GROUP BY k; - --- Aggregate with empty input and non-empty GroupBy expressions. -SELECT a, COUNT(1) FROM testData WHERE false GROUP BY a; - --- Aggregate with empty input and empty GroupBy expressions. -SELECT COUNT(1) FROM testData WHERE false; -SELECT 1 FROM (SELECT COUNT(1) FROM testData WHERE false) t; - --- Aggregate with empty GroupBy expressions and filter on top -SELECT 1 from ( - SELECT 1 AS z, - MIN(a.x) - FROM (select 1 as x) a - WHERE false -) b -where b.z != b.z; - --- SPARK-24369 multiple distinct aggregations having the same argument set -SELECT corr(DISTINCT x, y), corr(DISTINCT y, x), count(*) - FROM (VALUES (1, 1), (2, 2), (2, 2)) t(x, y); - --- SPARK-25708 HAVING without GROUP BY means global aggregate -SELECT 1 FROM range(10) HAVING true; - -SELECT 1 FROM range(10) HAVING MAX(id) > 0; - -SELECT id FROM range(10) HAVING id > 0; - -SET spark.sql.legacy.parser.havingWithoutGroupByAsWhere=true; - -SELECT 1 FROM range(10) HAVING true; - -SELECT 1 FROM range(10) HAVING MAX(id) > 0; - -SELECT id FROM range(10) HAVING id > 0; - -SET spark.sql.legacy.parser.havingWithoutGroupByAsWhere=false; - --- Test data -CREATE OR REPLACE TEMPORARY VIEW test_agg AS SELECT * FROM VALUES - (1, true), (1, false), - (2, true), - (3, false), (3, null), - (4, null), (4, null), - (5, null), (5, true), (5, false) AS test_agg(k, v); - --- empty table -SELECT every(v), some(v), any(v), bool_and(v), bool_or(v) FROM test_agg WHERE 1 = 0; - --- all null values -SELECT every(v), some(v), any(v), bool_and(v), bool_or(v) FROM test_agg WHERE k = 4; - --- aggregates are null Filtering -SELECT every(v), some(v), any(v), bool_and(v), bool_or(v) FROM test_agg WHERE k = 5; - --- group by -SELECT k, every(v), some(v), any(v), bool_and(v), bool_or(v) FROM test_agg GROUP BY k; - --- having -SELECT k, every(v) FROM test_agg GROUP BY k HAVING every(v) = false; -SELECT k, every(v) FROM test_agg GROUP BY k HAVING every(v) IS NULL; - --- basic subquery path to make sure rewrite happens in both parent and child plans. -SELECT k, - Every(v) AS every -FROM test_agg -WHERE k = 2 - AND v IN (SELECT Any(v) - FROM test_agg - WHERE k = 1) -GROUP BY k; - --- basic subquery path to make sure rewrite happens in both parent and child plans. -SELECT k, - Every(v) AS every -FROM test_agg -WHERE k = 2 - AND v IN (SELECT Every(v) - FROM test_agg - WHERE k = 1) -GROUP BY k; - --- input type checking Int -SELECT every(1); - --- input type checking Short -SELECT some(1S); - --- input type checking Long -SELECT any(1L); - --- input type checking String -SELECT every("true"); - --- input type checking Decimal -SELECT bool_and(1.0); - --- input type checking double -SELECT bool_or(1.0D); - --- every/some/any aggregates/bool_and/bool_or are supported as windows expression. -SELECT k, v, every(v) OVER (PARTITION BY k ORDER BY v) FROM test_agg; -SELECT k, v, some(v) OVER (PARTITION BY k ORDER BY v) FROM test_agg; -SELECT k, v, any(v) OVER (PARTITION BY k ORDER BY v) FROM test_agg; -SELECT k, v, bool_and(v) OVER (PARTITION BY k ORDER BY v) FROM test_agg; -SELECT k, v, bool_or(v) OVER (PARTITION BY k ORDER BY v) FROM test_agg; - --- Having referencing aggregate expressions is ok. -SELECT count(*) FROM test_agg HAVING count(*) > 1L; -SELECT k, max(v) FROM test_agg GROUP BY k HAVING max(v) = true; - --- Aggrgate expressions can be referenced through an alias -SELECT * FROM (SELECT COUNT(*) AS cnt FROM test_agg) WHERE cnt > 1L; - --- Error when aggregate expressions are in where clause directly -SELECT count(*) FROM test_agg WHERE count(*) > 1L; -SELECT count(*) FROM test_agg WHERE count(*) + 1L > 1L; -SELECT count(*) FROM test_agg WHERE k = 1 or k = 2 or count(*) + 1L > 1L or max(k) > 1; - --- Aggregate with multiple distinct decimal columns -SELECT AVG(DISTINCT decimal_col), SUM(DISTINCT decimal_col) FROM VALUES (CAST(1 AS DECIMAL(9, 0))) t(decimal_col); - --- SPARK-34581: Don't optimize out grouping expressions from aggregate expressions without aggregate function -SELECT not(a IS NULL), count(*) AS c -FROM testData -GROUP BY a IS NULL; - -SELECT if(not(a IS NULL), rand(0), 1), count(*) AS c -FROM testData -GROUP BY a IS NULL; - - --- Histogram aggregates with different numeric input types -SELECT - histogram_numeric(col, 2) as histogram_2, - histogram_numeric(col, 3) as histogram_3, - histogram_numeric(col, 5) as histogram_5, - histogram_numeric(col, 10) as histogram_10 -FROM VALUES - (1), (2), (3), (4), (5), (6), (7), (8), (9), (10), - (11), (12), (13), (14), (15), (16), (17), (18), (19), (20), - (21), (22), (23), (24), (25), (26), (27), (28), (29), (30), - (31), (32), (33), (34), (35), (3), (37), (38), (39), (40), - (41), (42), (43), (44), (45), (46), (47), (48), (49), (50) AS tab(col); -SELECT histogram_numeric(col, 3) FROM VALUES (1), (2), (3) AS tab(col); -SELECT histogram_numeric(col, 3) FROM VALUES (1L), (2L), (3L) AS tab(col); -SELECT histogram_numeric(col, 3) FROM VALUES (1F), (2F), (3F) AS tab(col); -SELECT histogram_numeric(col, 3) FROM VALUES (1D), (2D), (3D) AS tab(col); -SELECT histogram_numeric(col, 3) FROM VALUES (1S), (2S), (3S) AS tab(col); -SELECT histogram_numeric(col, 3) FROM VALUES - (CAST(1 AS BYTE)), (CAST(2 AS BYTE)), (CAST(3 AS BYTE)) AS tab(col); -SELECT histogram_numeric(col, 3) FROM VALUES - (CAST(1 AS TINYINT)), (CAST(2 AS TINYINT)), (CAST(3 AS TINYINT)) AS tab(col); -SELECT histogram_numeric(col, 3) FROM VALUES - (CAST(1 AS SMALLINT)), (CAST(2 AS SMALLINT)), (CAST(3 AS SMALLINT)) AS tab(col); -SELECT histogram_numeric(col, 3) FROM VALUES - (CAST(1 AS BIGINT)), (CAST(2 AS BIGINT)), (CAST(3 AS BIGINT)) AS tab(col); -SELECT histogram_numeric(col, 3) FROM VALUES (TIMESTAMP '2017-03-01 00:00:00'), - (TIMESTAMP '2017-04-01 00:00:00'), (TIMESTAMP '2017-05-01 00:00:00') AS tab(col); -SELECT histogram_numeric(col, 3) FROM VALUES (INTERVAL '100-00' YEAR TO MONTH), - (INTERVAL '110-00' YEAR TO MONTH), (INTERVAL '120-00' YEAR TO MONTH) AS tab(col); -SELECT histogram_numeric(col, 3) FROM VALUES (INTERVAL '12 20:4:0' DAY TO SECOND), - (INTERVAL '12 21:4:0' DAY TO SECOND), (INTERVAL '12 22:4:0' DAY TO SECOND) AS tab(col); -SELECT histogram_numeric(col, 3) -FROM VALUES (NULL), (NULL), (NULL) AS tab(col); -SELECT histogram_numeric(col, 3) -FROM VALUES (CAST(NULL AS DOUBLE)), (CAST(NULL AS DOUBLE)), (CAST(NULL AS DOUBLE)) AS tab(col); -SELECT histogram_numeric(col, 3) -FROM VALUES (CAST(NULL AS INT)), (CAST(NULL AS INT)), (CAST(NULL AS INT)) AS tab(col); - - --- SPARK-37613: Support ANSI Aggregate Function: regr_count -SELECT regr_count(y, x) FROM testRegression; -SELECT regr_count(y, x) FROM testRegression WHERE x IS NOT NULL; -SELECT k, count(*), regr_count(y, x) FROM testRegression GROUP BY k; -SELECT k, count(*) FILTER (WHERE x IS NOT NULL), regr_count(y, x) FROM testRegression GROUP BY k; - --- SPARK-37613: Support ANSI Aggregate Function: regr_r2 -SELECT regr_r2(y, x) FROM testRegression; -SELECT regr_r2(y, x) FROM testRegression WHERE x IS NOT NULL; -SELECT k, corr(y, x), regr_r2(y, x) FROM testRegression GROUP BY k; -SELECT k, corr(y, x) FILTER (WHERE x IS NOT NULL), regr_r2(y, x) FROM testRegression GROUP BY k; - --- SPARK-27974: Support ANSI Aggregate Function: array_agg -SELECT - collect_list(col), - array_agg(col) -FROM VALUES - (1), (2), (1) AS tab(col); -SELECT - a, - collect_list(b), - array_agg(b) -FROM VALUES - (1,4),(2,3),(1,4),(2,4) AS v(a,b) -GROUP BY a; - --- SPARK-37614: Support ANSI Aggregate Function: regr_avgx & regr_avgy -SELECT regr_avgx(y, x), regr_avgy(y, x) FROM testRegression; -SELECT regr_avgx(y, x), regr_avgy(y, x) FROM testRegression WHERE x IS NOT NULL AND y IS NOT NULL; -SELECT k, avg(x), avg(y), regr_avgx(y, x), regr_avgy(y, x) FROM testRegression GROUP BY k; -SELECT k, avg(x) FILTER (WHERE x IS NOT NULL AND y IS NOT NULL), avg(y) FILTER (WHERE x IS NOT NULL AND y IS NOT NULL), regr_avgx(y, x), regr_avgy(y, x) FROM testRegression GROUP BY k; - --- SPARK-37676: Support ANSI Aggregation Function: percentile_cont -SELECT - percentile_cont(0.25) WITHIN GROUP (ORDER BY v), - percentile_cont(0.25) WITHIN GROUP (ORDER BY v DESC) -FROM aggr; -SELECT - k, - percentile_cont(0.25) WITHIN GROUP (ORDER BY v), - percentile_cont(0.25) WITHIN GROUP (ORDER BY v DESC) -FROM aggr -GROUP BY k -ORDER BY k; - --- SPARK-37691: Support ANSI Aggregation Function: percentile_disc -SELECT - percentile_disc(0.25) WITHIN GROUP (ORDER BY v), - percentile_disc(0.25) WITHIN GROUP (ORDER BY v DESC) -FROM aggr; -SELECT - k, - percentile_disc(0.25) WITHIN GROUP (ORDER BY v), - percentile_disc(0.25) WITHIN GROUP (ORDER BY v DESC) -FROM aggr -GROUP BY k -ORDER BY k; diff --git a/gluten-ut/spark33/src/test/resources/sql-tests/inputs/misc-functions.sql b/gluten-ut/spark33/src/test/resources/sql-tests/inputs/misc-functions.sql deleted file mode 100644 index 907ff33000d..00000000000 --- a/gluten-ut/spark33/src/test/resources/sql-tests/inputs/misc-functions.sql +++ /dev/null @@ -1,22 +0,0 @@ --- test for misc functions - --- typeof -select typeof(null); -select typeof(true); -select typeof(1Y), typeof(1S), typeof(1), typeof(1L); -select typeof(cast(1.0 as float)), typeof(1.0D), typeof(1.2); -select typeof(date '1986-05-23'), typeof(timestamp '1986-05-23'), typeof(interval '23 days'); -select typeof(x'ABCD'), typeof('SPARK'); -select typeof(array(1, 2)), typeof(map(1, 2)), typeof(named_struct('a', 1, 'b', 'spark')); - --- Spark-32793: Rewrite AssertTrue with RaiseError -SELECT assert_true(true), assert_true(boolean(1)); -SELECT assert_true(false); -SELECT assert_true(boolean(0)); -SELECT assert_true(null); -SELECT assert_true(boolean(null)); -SELECT assert_true(false, 'custom error message'); - -CREATE TEMPORARY VIEW tbl_misc AS SELECT * FROM (VALUES (1), (8), (2)) AS T(v); -SELECT raise_error('error message'); -SELECT if(v > 5, raise_error('too big: ' || v), v + 1) FROM tbl_misc; diff --git a/gluten-ut/spark33/src/test/resources/sql-tests/inputs/random.sql b/gluten-ut/spark33/src/test/resources/sql-tests/inputs/random.sql deleted file mode 100644 index a1aae7b8759..00000000000 --- a/gluten-ut/spark33/src/test/resources/sql-tests/inputs/random.sql +++ /dev/null @@ -1,17 +0,0 @@ --- rand with the seed 0 -SELECT rand(0); -SELECT rand(cast(3 / 7 AS int)); -SELECT rand(NULL); -SELECT rand(cast(NULL AS int)); - --- rand unsupported data type -SELECT rand(1.0); - --- randn with the seed 0 -SELECT randn(0L); -SELECT randn(cast(3 / 7 AS long)); -SELECT randn(NULL); -SELECT randn(cast(NULL AS long)); - --- randn unsupported data type -SELECT rand('1') diff --git a/gluten-ut/spark33/src/test/resources/sql-tests/inputs/udf/udf-group-by.sql b/gluten-ut/spark33/src/test/resources/sql-tests/inputs/udf/udf-group-by.sql deleted file mode 100644 index 0cc57c97b02..00000000000 --- a/gluten-ut/spark33/src/test/resources/sql-tests/inputs/udf/udf-group-by.sql +++ /dev/null @@ -1,156 +0,0 @@ --- This test file was converted from group-by.sql. --- Test data. -CREATE OR REPLACE TEMPORARY VIEW testData AS SELECT * FROM VALUES -(1, 1), (1, 2), (2, 1), (2, 2), (3, 1), (3, 2), (null, 1), (3, null), (null, null) -AS testData(a, b); - --- Aggregate with empty GroupBy expressions. -SELECT udf(a), udf(COUNT(b)) FROM testData; -SELECT COUNT(udf(a)), udf(COUNT(b)) FROM testData; - --- Aggregate with non-empty GroupBy expressions. -SELECT udf(a), COUNT(udf(b)) FROM testData GROUP BY a; -SELECT udf(a), udf(COUNT(udf(b))) FROM testData GROUP BY b; -SELECT COUNT(udf(a)), COUNT(udf(b)) FROM testData GROUP BY udf(a); - --- Aggregate grouped by literals. -SELECT 'foo', COUNT(udf(a)) FROM testData GROUP BY 1; - --- Aggregate grouped by literals (whole stage code generation). -SELECT 'foo' FROM testData WHERE a = 0 GROUP BY udf(1); - --- Aggregate grouped by literals (hash aggregate). -SELECT 'foo', udf(APPROX_COUNT_DISTINCT(udf(a))) FROM testData WHERE a = 0 GROUP BY udf(1); - --- Aggregate grouped by literals (sort aggregate). -SELECT 'foo', MAX(STRUCT(udf(a))) FROM testData WHERE a = 0 GROUP BY udf(1); - --- Aggregate with complex GroupBy expressions. -SELECT udf(a + b), udf(COUNT(b)) FROM testData GROUP BY a + b; -SELECT udf(a + 2), udf(COUNT(b)) FROM testData GROUP BY a + 1; -SELECT udf(a + 1) + 1, udf(COUNT(b)) FROM testData GROUP BY udf(a + 1); - --- Aggregate with nulls. -SELECT SKEWNESS(udf(a)), udf(KURTOSIS(a)), udf(MIN(a)), MAX(udf(a)), udf(AVG(udf(a))), udf(VARIANCE(a)), STDDEV(udf(a)), udf(SUM(a)), udf(COUNT(a)) -FROM testData; - --- Aggregate with foldable input and multiple distinct groups. -SELECT COUNT(DISTINCT udf(b)), udf(COUNT(DISTINCT b, c)) FROM (SELECT 1 AS a, 2 AS b, 3 AS c) GROUP BY udf(a); - --- Aliases in SELECT could be used in GROUP BY -SELECT udf(a) AS k, COUNT(udf(b)) FROM testData GROUP BY k; -SELECT a AS k, udf(COUNT(b)) FROM testData GROUP BY k HAVING k > 1; - --- Aggregate functions cannot be used in GROUP BY -SELECT udf(COUNT(b)) AS k FROM testData GROUP BY k; - --- Test data. -CREATE OR REPLACE TEMPORARY VIEW testDataHasSameNameWithAlias AS SELECT * FROM VALUES -(1, 1, 3), (1, 2, 1) AS testDataHasSameNameWithAlias(k, a, v); -SELECT k AS a, udf(COUNT(udf(v))) FROM testDataHasSameNameWithAlias GROUP BY udf(a); - --- turn off group by aliases -set spark.sql.groupByAliases=false; - --- Check analysis exceptions -SELECT a AS k, udf(COUNT(udf(b))) FROM testData GROUP BY k; - --- Aggregate with empty input and non-empty GroupBy expressions. -SELECT udf(a), COUNT(udf(1)) FROM testData WHERE false GROUP BY udf(a); - --- Aggregate with empty input and empty GroupBy expressions. -SELECT udf(COUNT(1)) FROM testData WHERE false; -SELECT 1 FROM (SELECT udf(COUNT(1)) FROM testData WHERE false) t; - --- Aggregate with empty GroupBy expressions and filter on top -SELECT 1 from ( - SELECT 1 AS z, - udf(MIN(a.x)) - FROM (select 1 as x) a - WHERE false -) b -where b.z != b.z; - --- SPARK-24369 multiple distinct aggregations having the same argument set -SELECT corr(DISTINCT x, y), udf(corr(DISTINCT y, x)), count(*) - FROM (VALUES (1, 1), (2, 2), (2, 2)) t(x, y); - --- SPARK-25708 HAVING without GROUP BY means global aggregate -SELECT udf(1) FROM range(10) HAVING true; - -SELECT udf(udf(1)) FROM range(10) HAVING MAX(id) > 0; - -SELECT udf(id) FROM range(10) HAVING id > 0; - --- Test data -CREATE OR REPLACE TEMPORARY VIEW test_agg AS SELECT * FROM VALUES - (1, true), (1, false), - (2, true), - (3, false), (3, null), - (4, null), (4, null), - (5, null), (5, true), (5, false) AS test_agg(k, v); - --- empty table -SELECT udf(every(v)), udf(some(v)), any(v) FROM test_agg WHERE 1 = 0; - --- all null values -SELECT udf(every(udf(v))), some(v), any(v) FROM test_agg WHERE k = 4; - --- aggregates are null Filtering -SELECT every(v), udf(some(v)), any(v) FROM test_agg WHERE k = 5; - --- group by -SELECT udf(k), every(v), udf(some(v)), any(v) FROM test_agg GROUP BY udf(k); - --- having -SELECT udf(k), every(v) FROM test_agg GROUP BY k HAVING every(v) = false; -SELECT udf(k), udf(every(v)) FROM test_agg GROUP BY udf(k) HAVING every(v) IS NULL; - --- basic subquery path to make sure rewrite happens in both parent and child plans. -SELECT udf(k), - udf(Every(v)) AS every -FROM test_agg -WHERE k = 2 - AND v IN (SELECT Any(v) - FROM test_agg - WHERE k = 1) -GROUP BY udf(k); - --- basic subquery path to make sure rewrite happens in both parent and child plans. -SELECT udf(udf(k)), - Every(v) AS every -FROM test_agg -WHERE k = 2 - AND v IN (SELECT Every(v) - FROM test_agg - WHERE k = 1) -GROUP BY udf(udf(k)); - --- input type checking Int -SELECT every(udf(1)); - --- input type checking Short -SELECT some(udf(1S)); - --- input type checking Long -SELECT any(udf(1L)); - --- input type checking String -SELECT udf(every("true")); - --- every/some/any aggregates are supported as windows expression. -SELECT k, v, every(v) OVER (PARTITION BY k ORDER BY v) FROM test_agg; -SELECT k, udf(udf(v)), some(v) OVER (PARTITION BY k ORDER BY v) FROM test_agg; -SELECT udf(udf(k)), v, any(v) OVER (PARTITION BY k ORDER BY v) FROM test_agg; - --- Having referencing aggregate expressions is ok. -SELECT udf(count(*)) FROM test_agg HAVING count(*) > 1L; -SELECT k, udf(max(v)) FROM test_agg GROUP BY k HAVING max(v) = true; - --- Aggrgate expressions can be referenced through an alias -SELECT * FROM (SELECT udf(COUNT(*)) AS cnt FROM test_agg) WHERE cnt > 1L; - --- Error when aggregate expressions are in where clause directly -SELECT udf(count(*)) FROM test_agg WHERE count(*) > 1L; -SELECT udf(count(*)) FROM test_agg WHERE count(*) + 1L > 1L; -SELECT udf(count(*)) FROM test_agg WHERE k = 1 or k = 2 or count(*) + 1L > 1L or max(k) > 1; diff --git a/gluten-ut/spark33/src/test/resources/sql-tests/results/group-by-ordinal.sql.out b/gluten-ut/spark33/src/test/resources/sql-tests/results/group-by-ordinal.sql.out deleted file mode 100644 index 92e4a861fa1..00000000000 --- a/gluten-ut/spark33/src/test/resources/sql-tests/results/group-by-ordinal.sql.out +++ /dev/null @@ -1,398 +0,0 @@ --- Automatically generated by SQLQueryTestSuite --- Number of queries: 33 - - --- !query -create temporary view data as select * from values - (1, 1), - (1, 2), - (2, 1), - (2, 2), - (3, 1), - (3, 2) - as data(a, b) --- !query schema -struct<> --- !query output - - - --- !query -select a, sum(b) from data group by 1 --- !query schema -struct --- !query output -1 3 -2 3 -3 3 - - --- !query -select 1, 2, sum(b) from data group by 1, 2 --- !query schema -struct<1:int,2:int,sum(b):bigint> --- !query output -1 2 9 - - --- !query -select a, 1, sum(b) from data group by a, 1 --- !query schema -struct --- !query output -1 1 3 -2 1 3 -3 1 3 - - --- !query -select a, 1, sum(b) from data group by 1, 2 --- !query schema -struct --- !query output -1 1 3 -2 1 3 -3 1 3 - - --- !query -select a, b + 2, count(2) from data group by a, 2 --- !query schema -struct --- !query output -1 3 1 -1 4 1 -2 3 1 -2 4 1 -3 3 1 -3 4 1 - - --- !query -select a as aa, b + 2 as bb, count(2) from data group by 1, 2 --- !query schema -struct --- !query output -1 3 1 -1 4 1 -2 3 1 -2 4 1 -3 3 1 -3 4 1 - - --- !query -select sum(b) from data group by 1 + 0 --- !query schema -struct --- !query output -9 - - --- !query -select a, b from data group by -1 --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -GROUP BY position -1 is not in select list (valid range is [1, 2]); line 1 pos 31 - - --- !query -select a, b from data group by 0 --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -GROUP BY position 0 is not in select list (valid range is [1, 2]); line 1 pos 31 - - --- !query -select a, b from data group by 3 --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -GROUP BY position 3 is not in select list (valid range is [1, 2]); line 1 pos 31 - - --- !query -select a, b, sum(b) from data group by 3 --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -GROUP BY 3 refers to an expression that is or contains an aggregate function. Aggregate functions are not allowed in GROUP BY, but got sum(data.b) AS `sum(b)`; line 1 pos 39 - - --- !query -select a, b, sum(b) + 2 from data group by 3 --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -GROUP BY 3 refers to an expression that is or contains an aggregate function. Aggregate functions are not allowed in GROUP BY, but got (sum(data.b) + CAST(2 AS BIGINT)) AS `(sum(b) + 2)`; line 1 pos 43 - - --- !query -select a, rand(0), sum(b) -from -(select /*+ REPARTITION(1) */ a, b from data) group by a, 2 --- !query schema -struct --- !query output -1 0.5234194256885571 2 -1 0.7604953758285915 1 -2 0.0953472826424725 1 -2 0.3163249920547614 2 -3 0.2710259815484829 2 -3 0.7141011170991605 1 - - --- !query -select * from data group by a, b, 1 --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -Star (*) is not allowed in select list when GROUP BY ordinal position is used - - --- !query -select a, count(a) from (select 1 as a) tmp group by 1 order by 1 --- !query schema -struct --- !query output -1 1 - - --- !query -select count(a), a from (select 1 as a) tmp group by 2 having a > 0 --- !query schema -struct --- !query output -1 1 - - --- !query -select a, a AS k, count(b) from data group by k, 1 --- !query schema -struct --- !query output -1 1 2 -2 2 2 -3 3 2 - - --- !query -select a, b, count(1) from data group by cube(1, 2) --- !query schema -struct --- !query output -1 1 1 -1 2 1 -1 NULL 2 -2 1 1 -2 2 1 -2 NULL 2 -3 1 1 -3 2 1 -3 NULL 2 -NULL 1 3 -NULL 2 3 -NULL NULL 6 - - --- !query -select a, b, count(1) from data group by cube(1, b) --- !query schema -struct --- !query output -1 1 1 -1 2 1 -1 NULL 2 -2 1 1 -2 2 1 -2 NULL 2 -3 1 1 -3 2 1 -3 NULL 2 -NULL 1 3 -NULL 2 3 -NULL NULL 6 - - --- !query -select a, b, count(1) from data group by 1, 2 with cube --- !query schema -struct --- !query output -1 1 1 -1 2 1 -1 NULL 2 -2 1 1 -2 2 1 -2 NULL 2 -3 1 1 -3 2 1 -3 NULL 2 -NULL 1 3 -NULL 2 3 -NULL NULL 6 - - --- !query -select a, b, count(1) from data group by rollup(1, 2) --- !query schema -struct --- !query output -1 1 1 -1 2 1 -1 NULL 2 -2 1 1 -2 2 1 -2 NULL 2 -3 1 1 -3 2 1 -3 NULL 2 -NULL NULL 6 - - --- !query -select a, b, count(1) from data group by rollup(1, b) --- !query schema -struct --- !query output -1 1 1 -1 2 1 -1 NULL 2 -2 1 1 -2 2 1 -2 NULL 2 -3 1 1 -3 2 1 -3 NULL 2 -NULL NULL 6 - - --- !query -select a, b, count(1) from data group by 1, 2 with rollup --- !query schema -struct --- !query output -1 1 1 -1 2 1 -1 NULL 2 -2 1 1 -2 2 1 -2 NULL 2 -3 1 1 -3 2 1 -3 NULL 2 -NULL NULL 6 - - --- !query -select a, b, count(1) from data group by grouping sets((1), (2), (1, 2)) --- !query schema -struct --- !query output -1 1 1 -1 2 1 -1 NULL 2 -2 1 1 -2 2 1 -2 NULL 2 -3 1 1 -3 2 1 -3 NULL 2 -NULL 1 3 -NULL 2 3 - - --- !query -select a, b, count(1) from data group by grouping sets((1), (b), (a, 2)) --- !query schema -struct --- !query output -1 1 1 -1 2 1 -1 NULL 2 -2 1 1 -2 2 1 -2 NULL 2 -3 1 1 -3 2 1 -3 NULL 2 -NULL 1 3 -NULL 2 3 - - --- !query -select a, b, count(1) from data group by a, 2 grouping sets((1), (b), (a, 2)) --- !query schema -struct --- !query output -1 1 1 -1 2 1 -1 NULL 2 -2 1 1 -2 2 1 -2 NULL 2 -3 1 1 -3 2 1 -3 NULL 2 -NULL 1 3 -NULL 2 3 - - --- !query -select a, b, count(1) from data group by a, -1 --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -GROUP BY position -1 is not in select list (valid range is [1, 3]); line 1 pos 44 - - --- !query -select a, b, count(1) from data group by a, 3 --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -GROUP BY 3 refers to an expression that is or contains an aggregate function. Aggregate functions are not allowed in GROUP BY, but got count(1) AS `count(1)`; line 1 pos 44 - - --- !query -select a, b, count(1) from data group by cube(-1, 2) --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -GROUP BY position -1 is not in select list (valid range is [1, 3]); line 1 pos 46 - - --- !query -select a, b, count(1) from data group by cube(1, 3) --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -GROUP BY 3 refers to an expression that is or contains an aggregate function. Aggregate functions are not allowed in GROUP BY, but got count(1) AS `count(1)`; line 1 pos 49 - - --- !query -set spark.sql.groupByOrdinal=false --- !query schema -struct --- !query output -spark.sql.groupByOrdinal false - - --- !query -select sum(b) from data group by -1 --- !query schema -struct --- !query output -9 diff --git a/gluten-ut/spark33/src/test/resources/sql-tests/results/group-by.sql.out b/gluten-ut/spark33/src/test/resources/sql-tests/results/group-by.sql.out deleted file mode 100644 index 48b35bf1e0d..00000000000 --- a/gluten-ut/spark33/src/test/resources/sql-tests/results/group-by.sql.out +++ /dev/null @@ -1,1030 +0,0 @@ --- Automatically generated by SQLQueryTestSuite --- Number of queries: 101 - - --- !query -CREATE OR REPLACE TEMPORARY VIEW testData AS SELECT * FROM VALUES -(1, 1), (1, 2), (2, 1), (2, 2), (3, 1), (3, 2), (null, 1), (3, null), (null, null) -AS testData(a, b) --- !query schema -struct<> --- !query output - - - --- !query -CREATE OR REPLACE TEMPORARY VIEW testRegression AS SELECT * FROM VALUES -(1, 10, null), (2, 10, 11), (2, 20, 22), (2, 25, null), (2, 30, 35) -AS testRegression(k, y, x) --- !query schema -struct<> --- !query output - - - --- !query -CREATE OR REPLACE TEMPORARY VIEW aggr AS SELECT * FROM VALUES -(0, 0), (0, 10), (0, 20), (0, 30), (0, 40), (1, 10), (1, 20), (2, 10), (2, 20), (2, 25), (2, 30), (3, 60), (4, null) -AS aggr(k, v) --- !query schema -struct<> --- !query output - - - --- !query -SELECT a, COUNT(b) FROM testData --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -grouping expressions sequence is empty, and 'testdata.a' is not an aggregate function. Wrap '(count(testdata.b) AS `count(b)`)' in windowing function(s) or wrap 'testdata.a' in first() (or first_value) if you don't care which value you get. - - --- !query -SELECT COUNT(a), COUNT(b) FROM testData --- !query schema -struct --- !query output -7 7 - - --- !query -SELECT a, COUNT(b) FROM testData GROUP BY a --- !query schema -struct --- !query output -1 2 -2 2 -3 2 -NULL 1 - - --- !query -SELECT a, COUNT(b) FROM testData GROUP BY b --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -expression 'testdata.a' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get. - - --- !query -SELECT COUNT(a), COUNT(b) FROM testData GROUP BY a --- !query schema -struct --- !query output -0 1 -2 2 -2 2 -3 2 - - --- !query -SELECT 'foo', COUNT(a) FROM testData GROUP BY 1 --- !query schema -struct --- !query output -foo 7 - - --- !query -SELECT 'foo' FROM testData WHERE a = 0 GROUP BY 1 --- !query schema -struct --- !query output - - - --- !query -SELECT 'foo', APPROX_COUNT_DISTINCT(a) FROM testData WHERE a = 0 GROUP BY 1 --- !query schema -struct --- !query output - - - --- !query -SELECT 'foo', MAX(STRUCT(a)) FROM testData WHERE a = 0 GROUP BY 1 --- !query schema -struct> --- !query output - - - --- !query -SELECT a + b, COUNT(b) FROM testData GROUP BY a + b --- !query schema -struct<(a + b):int,count(b):bigint> --- !query output -2 1 -3 2 -4 2 -5 1 -NULL 1 - - --- !query -SELECT a + 2, COUNT(b) FROM testData GROUP BY a + 1 --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -expression 'testdata.a' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get. - - --- !query -SELECT a + 1 + 1, COUNT(b) FROM testData GROUP BY a + 1 --- !query schema -struct<((a + 1) + 1):int,count(b):bigint> --- !query output -3 2 -4 2 -5 2 -NULL 1 - - --- !query -SELECT SKEWNESS(a), KURTOSIS(a), MIN(a), MAX(a), AVG(a), VARIANCE(a), STDDEV(a), SUM(a), COUNT(a) -FROM testData --- !query schema -struct --- !query output --0.27238010581457284 -1.5069204152249138 1 3 2.142857142857143 0.8095238095238096 0.8997354108424375 15 7 - - --- !query -SELECT COUNT(DISTINCT b), COUNT(DISTINCT b, c) FROM (SELECT 1 AS a, 2 AS b, 3 AS c) GROUP BY a --- !query schema -struct --- !query output -1 1 - - --- !query -SELECT a AS k, COUNT(b) FROM testData GROUP BY k --- !query schema -struct --- !query output -1 2 -2 2 -3 2 -NULL 1 - - --- !query -SELECT a AS k, COUNT(b) FROM testData GROUP BY k HAVING k > 1 --- !query schema -struct --- !query output -2 2 -3 2 - - --- !query -SELECT a AS k, COUNT(non_existing) FROM testData GROUP BY k --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -Column 'non_existing' does not exist. Did you mean one of the following? [testdata.a, testdata.b]; line 1 pos 21 - - --- !query -SELECT COUNT(b) AS k FROM testData GROUP BY k --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -aggregate functions are not allowed in GROUP BY, but found count(testdata.b) - - --- !query -CREATE OR REPLACE TEMPORARY VIEW testDataHasSameNameWithAlias AS SELECT * FROM VALUES -(1, 1, 3), (1, 2, 1) AS testDataHasSameNameWithAlias(k, a, v) --- !query schema -struct<> --- !query output - - - --- !query -SELECT k AS a, COUNT(v) FROM testDataHasSameNameWithAlias GROUP BY a --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -expression 'testdatahassamenamewithalias.k' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get. - - --- !query -set spark.sql.groupByAliases=false --- !query schema -struct --- !query output -spark.sql.groupByAliases false - - --- !query -SELECT a AS k, COUNT(b) FROM testData GROUP BY k --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -Column 'k' does not exist. Did you mean one of the following? [testdata.a, testdata.b]; line 1 pos 47 - - --- !query -SELECT a, COUNT(1) FROM testData WHERE false GROUP BY a --- !query schema -struct --- !query output - - - --- !query -SELECT COUNT(1) FROM testData WHERE false --- !query schema -struct --- !query output -0 - - --- !query -SELECT 1 FROM (SELECT COUNT(1) FROM testData WHERE false) t --- !query schema -struct<1:int> --- !query output -1 - - --- !query -SELECT 1 from ( - SELECT 1 AS z, - MIN(a.x) - FROM (select 1 as x) a - WHERE false -) b -where b.z != b.z --- !query schema -struct<1:int> --- !query output - - - --- !query -SELECT corr(DISTINCT x, y), corr(DISTINCT y, x), count(*) - FROM (VALUES (1, 1), (2, 2), (2, 2)) t(x, y) --- !query schema -struct --- !query output -0.9999999999999999 0.9999999999999999 3 - - --- !query -SELECT 1 FROM range(10) HAVING true --- !query schema -struct<1:int> --- !query output -1 - - --- !query -SELECT 1 FROM range(10) HAVING MAX(id) > 0 --- !query schema -struct<1:int> --- !query output -1 - - --- !query -SELECT id FROM range(10) HAVING id > 0 --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -grouping expressions sequence is empty, and 'id' is not an aggregate function. Wrap '()' in windowing function(s) or wrap 'id' in first() (or first_value) if you don't care which value you get. - - --- !query -SET spark.sql.legacy.parser.havingWithoutGroupByAsWhere=true --- !query schema -struct --- !query output -spark.sql.legacy.parser.havingWithoutGroupByAsWhere true - - --- !query -SELECT 1 FROM range(10) HAVING true --- !query schema -struct<1:int> --- !query output -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 - - --- !query -SELECT 1 FROM range(10) HAVING MAX(id) > 0 --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException - -Aggregate/Window/Generate expressions are not valid in where clause of the query. -Expression in where clause: [(max(id) > CAST(0 AS BIGINT))] -Invalid expressions: [max(id)] - - --- !query -SELECT id FROM range(10) HAVING id > 0 --- !query schema -struct --- !query output -1 -2 -3 -4 -5 -6 -7 -8 -9 - - --- !query -SET spark.sql.legacy.parser.havingWithoutGroupByAsWhere=false --- !query schema -struct --- !query output -spark.sql.legacy.parser.havingWithoutGroupByAsWhere false - - --- !query -CREATE OR REPLACE TEMPORARY VIEW test_agg AS SELECT * FROM VALUES - (1, true), (1, false), - (2, true), - (3, false), (3, null), - (4, null), (4, null), - (5, null), (5, true), (5, false) AS test_agg(k, v) --- !query schema -struct<> --- !query output - - - --- !query -SELECT every(v), some(v), any(v), bool_and(v), bool_or(v) FROM test_agg WHERE 1 = 0 --- !query schema -struct --- !query output -NULL NULL NULL NULL NULL - - --- !query -SELECT every(v), some(v), any(v), bool_and(v), bool_or(v) FROM test_agg WHERE k = 4 --- !query schema -struct --- !query output -NULL NULL NULL NULL NULL - - --- !query -SELECT every(v), some(v), any(v), bool_and(v), bool_or(v) FROM test_agg WHERE k = 5 --- !query schema -struct --- !query output -false true true false true - - --- !query -SELECT k, every(v), some(v), any(v), bool_and(v), bool_or(v) FROM test_agg GROUP BY k --- !query schema -struct --- !query output -1 false true true false true -2 true true true true true -3 false false false false false -4 NULL NULL NULL NULL NULL -5 false true true false true - - --- !query -SELECT k, every(v) FROM test_agg GROUP BY k HAVING every(v) = false --- !query schema -struct --- !query output -1 false -3 false -5 false - - --- !query -SELECT k, every(v) FROM test_agg GROUP BY k HAVING every(v) IS NULL --- !query schema -struct --- !query output -4 NULL - - --- !query -SELECT k, - Every(v) AS every -FROM test_agg -WHERE k = 2 - AND v IN (SELECT Any(v) - FROM test_agg - WHERE k = 1) -GROUP BY k --- !query schema -struct --- !query output -2 true - - --- !query -SELECT k, - Every(v) AS every -FROM test_agg -WHERE k = 2 - AND v IN (SELECT Every(v) - FROM test_agg - WHERE k = 1) -GROUP BY k --- !query schema -struct --- !query output - - - --- !query -SELECT every(1) --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -cannot resolve 'every(1)' due to data type mismatch: argument 1 requires boolean type, however, '1' is of int type.; line 1 pos 7 - - --- !query -SELECT some(1S) --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -cannot resolve 'some(1S)' due to data type mismatch: argument 1 requires boolean type, however, '1S' is of smallint type.; line 1 pos 7 - - --- !query -SELECT any(1L) --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -cannot resolve 'any(1L)' due to data type mismatch: argument 1 requires boolean type, however, '1L' is of bigint type.; line 1 pos 7 - - --- !query -SELECT every("true") --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -cannot resolve 'every('true')' due to data type mismatch: argument 1 requires boolean type, however, ''true'' is of string type.; line 1 pos 7 - - --- !query -SELECT bool_and(1.0) --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -cannot resolve 'bool_and(1.0BD)' due to data type mismatch: argument 1 requires boolean type, however, '1.0BD' is of decimal(2,1) type.; line 1 pos 7 - - --- !query -SELECT bool_or(1.0D) --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -cannot resolve 'bool_or(1.0D)' due to data type mismatch: argument 1 requires boolean type, however, '1.0D' is of double type.; line 1 pos 7 - - --- !query -SELECT k, v, every(v) OVER (PARTITION BY k ORDER BY v) FROM test_agg --- !query schema -struct --- !query output -1 false false -1 true false -2 true true -3 NULL NULL -3 false false -4 NULL NULL -4 NULL NULL -5 NULL NULL -5 false false -5 true false - - --- !query -SELECT k, v, some(v) OVER (PARTITION BY k ORDER BY v) FROM test_agg --- !query schema -struct --- !query output -1 false false -1 true true -2 true true -3 NULL NULL -3 false false -4 NULL NULL -4 NULL NULL -5 NULL NULL -5 false false -5 true true - - --- !query -SELECT k, v, any(v) OVER (PARTITION BY k ORDER BY v) FROM test_agg --- !query schema -struct --- !query output -1 false false -1 true true -2 true true -3 NULL NULL -3 false false -4 NULL NULL -4 NULL NULL -5 NULL NULL -5 false false -5 true true - - --- !query -SELECT k, v, bool_and(v) OVER (PARTITION BY k ORDER BY v) FROM test_agg --- !query schema -struct --- !query output -1 false false -1 true false -2 true true -3 NULL NULL -3 false false -4 NULL NULL -4 NULL NULL -5 NULL NULL -5 false false -5 true false - - --- !query -SELECT k, v, bool_or(v) OVER (PARTITION BY k ORDER BY v) FROM test_agg --- !query schema -struct --- !query output -1 false false -1 true true -2 true true -3 NULL NULL -3 false false -4 NULL NULL -4 NULL NULL -5 NULL NULL -5 false false -5 true true - - --- !query -SELECT count(*) FROM test_agg HAVING count(*) > 1L --- !query schema -struct --- !query output -10 - - --- !query -SELECT k, max(v) FROM test_agg GROUP BY k HAVING max(v) = true --- !query schema -struct --- !query output -1 true -2 true -5 true - - --- !query -SELECT * FROM (SELECT COUNT(*) AS cnt FROM test_agg) WHERE cnt > 1L --- !query schema -struct --- !query output -10 - - --- !query -SELECT count(*) FROM test_agg WHERE count(*) > 1L --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException - -Aggregate/Window/Generate expressions are not valid in where clause of the query. -Expression in where clause: [(count(1) > 1L)] -Invalid expressions: [count(1)] - - --- !query -SELECT count(*) FROM test_agg WHERE count(*) + 1L > 1L --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException - -Aggregate/Window/Generate expressions are not valid in where clause of the query. -Expression in where clause: [((count(1) + 1L) > 1L)] -Invalid expressions: [count(1)] - - --- !query -SELECT count(*) FROM test_agg WHERE k = 1 or k = 2 or count(*) + 1L > 1L or max(k) > 1 --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException - -Aggregate/Window/Generate expressions are not valid in where clause of the query. -Expression in where clause: [(((test_agg.k = 1) OR (test_agg.k = 2)) OR (((count(1) + 1L) > 1L) OR (max(test_agg.k) > 1)))] -Invalid expressions: [count(1), max(test_agg.k)] - - --- !query -SELECT AVG(DISTINCT decimal_col), SUM(DISTINCT decimal_col) FROM VALUES (CAST(1 AS DECIMAL(9, 0))) t(decimal_col) --- !query schema -struct --- !query output -1.0000 1 - - --- !query -SELECT not(a IS NULL), count(*) AS c -FROM testData -GROUP BY a IS NULL --- !query schema -struct<(NOT (a IS NULL)):boolean,c:bigint> --- !query output -false 2 -true 7 - - --- !query -SELECT if(not(a IS NULL), rand(0), 1), count(*) AS c -FROM testData -GROUP BY a IS NULL --- !query schema -struct<(IF((NOT (a IS NULL)), rand(0), 1)):double,c:bigint> --- !query output -0.7604953758285915 7 -1.0 2 - - --- !query -SELECT - histogram_numeric(col, 2) as histogram_2, - histogram_numeric(col, 3) as histogram_3, - histogram_numeric(col, 5) as histogram_5, - histogram_numeric(col, 10) as histogram_10 -FROM VALUES - (1), (2), (3), (4), (5), (6), (7), (8), (9), (10), - (11), (12), (13), (14), (15), (16), (17), (18), (19), (20), - (21), (22), (23), (24), (25), (26), (27), (28), (29), (30), - (31), (32), (33), (34), (35), (3), (37), (38), (39), (40), - (41), (42), (43), (44), (45), (46), (47), (48), (49), (50) AS tab(col) --- !query schema -struct>,histogram_3:array>,histogram_5:array>,histogram_10:array>> --- !query output -[{"x":12,"y":26.0},{"x":38,"y":24.0}] [{"x":9,"y":20.0},{"x":25,"y":11.0},{"x":40,"y":19.0}] [{"x":5,"y":11.0},{"x":14,"y":8.0},{"x":22,"y":7.0},{"x":30,"y":10.0},{"x":43,"y":14.0}] [{"x":3,"y":6.0},{"x":8,"y":6.0},{"x":13,"y":4.0},{"x":17,"y":3.0},{"x":20,"y":4.0},{"x":25,"y":6.0},{"x":31,"y":7.0},{"x":39,"y":5.0},{"x":43,"y":4.0},{"x":48,"y":5.0}] - - --- !query -SELECT histogram_numeric(col, 3) FROM VALUES (1), (2), (3) AS tab(col) --- !query schema -struct>> --- !query output -[{"x":1,"y":1.0},{"x":2,"y":1.0},{"x":3,"y":1.0}] - - --- !query -SELECT histogram_numeric(col, 3) FROM VALUES (1L), (2L), (3L) AS tab(col) --- !query schema -struct>> --- !query output -[{"x":1,"y":1.0},{"x":2,"y":1.0},{"x":3,"y":1.0}] - - --- !query -SELECT histogram_numeric(col, 3) FROM VALUES (1F), (2F), (3F) AS tab(col) --- !query schema -struct>> --- !query output -[{"x":1.0,"y":1.0},{"x":2.0,"y":1.0},{"x":3.0,"y":1.0}] - - --- !query -SELECT histogram_numeric(col, 3) FROM VALUES (1D), (2D), (3D) AS tab(col) --- !query schema -struct>> --- !query output -[{"x":1.0,"y":1.0},{"x":2.0,"y":1.0},{"x":3.0,"y":1.0}] - - --- !query -SELECT histogram_numeric(col, 3) FROM VALUES (1S), (2S), (3S) AS tab(col) --- !query schema -struct>> --- !query output -[{"x":1,"y":1.0},{"x":2,"y":1.0},{"x":3,"y":1.0}] - - --- !query -SELECT histogram_numeric(col, 3) FROM VALUES - (CAST(1 AS BYTE)), (CAST(2 AS BYTE)), (CAST(3 AS BYTE)) AS tab(col) --- !query schema -struct>> --- !query output -[{"x":1,"y":1.0},{"x":2,"y":1.0},{"x":3,"y":1.0}] - - --- !query -SELECT histogram_numeric(col, 3) FROM VALUES - (CAST(1 AS TINYINT)), (CAST(2 AS TINYINT)), (CAST(3 AS TINYINT)) AS tab(col) --- !query schema -struct>> --- !query output -[{"x":1,"y":1.0},{"x":2,"y":1.0},{"x":3,"y":1.0}] - - --- !query -SELECT histogram_numeric(col, 3) FROM VALUES - (CAST(1 AS SMALLINT)), (CAST(2 AS SMALLINT)), (CAST(3 AS SMALLINT)) AS tab(col) --- !query schema -struct>> --- !query output -[{"x":1,"y":1.0},{"x":2,"y":1.0},{"x":3,"y":1.0}] - - --- !query -SELECT histogram_numeric(col, 3) FROM VALUES - (CAST(1 AS BIGINT)), (CAST(2 AS BIGINT)), (CAST(3 AS BIGINT)) AS tab(col) --- !query schema -struct>> --- !query output -[{"x":1,"y":1.0},{"x":2,"y":1.0},{"x":3,"y":1.0}] - - --- !query -SELECT histogram_numeric(col, 3) FROM VALUES (TIMESTAMP '2017-03-01 00:00:00'), - (TIMESTAMP '2017-04-01 00:00:00'), (TIMESTAMP '2017-05-01 00:00:00') AS tab(col) --- !query schema -struct>> --- !query output -[{"x":2017-03-01 00:00:00,"y":1.0},{"x":2017-04-01 00:00:00,"y":1.0},{"x":2017-05-01 00:00:00,"y":1.0}] - - --- !query -SELECT histogram_numeric(col, 3) FROM VALUES (INTERVAL '100-00' YEAR TO MONTH), - (INTERVAL '110-00' YEAR TO MONTH), (INTERVAL '120-00' YEAR TO MONTH) AS tab(col) --- !query schema -struct>> --- !query output -[{"x":100-0,"y":1.0},{"x":110-0,"y":1.0},{"x":120-0,"y":1.0}] - - --- !query -SELECT histogram_numeric(col, 3) FROM VALUES (INTERVAL '12 20:4:0' DAY TO SECOND), - (INTERVAL '12 21:4:0' DAY TO SECOND), (INTERVAL '12 22:4:0' DAY TO SECOND) AS tab(col) --- !query schema -struct>> --- !query output -[{"x":12 20:04:00.000000000,"y":1.0},{"x":12 21:04:00.000000000,"y":1.0},{"x":12 22:04:00.000000000,"y":1.0}] - - --- !query -SELECT histogram_numeric(col, 3) -FROM VALUES (NULL), (NULL), (NULL) AS tab(col) --- !query schema -struct>> --- !query output -NULL - - --- !query -SELECT histogram_numeric(col, 3) -FROM VALUES (CAST(NULL AS DOUBLE)), (CAST(NULL AS DOUBLE)), (CAST(NULL AS DOUBLE)) AS tab(col) --- !query schema -struct>> --- !query output -NULL - - --- !query -SELECT histogram_numeric(col, 3) -FROM VALUES (CAST(NULL AS INT)), (CAST(NULL AS INT)), (CAST(NULL AS INT)) AS tab(col) --- !query schema -struct>> --- !query output -NULL - - --- !query -SELECT regr_count(y, x) FROM testRegression --- !query schema -struct --- !query output -3 - - --- !query -SELECT regr_count(y, x) FROM testRegression WHERE x IS NOT NULL --- !query schema -struct --- !query output -3 - - --- !query -SELECT k, count(*), regr_count(y, x) FROM testRegression GROUP BY k --- !query schema -struct --- !query output -1 1 0 -2 4 3 - - --- !query -SELECT k, count(*) FILTER (WHERE x IS NOT NULL), regr_count(y, x) FROM testRegression GROUP BY k --- !query schema -struct --- !query output -1 0 0 -2 3 3 - - --- !query -SELECT regr_r2(y, x) FROM testRegression --- !query schema -struct --- !query output -0.9976905311778291 - - --- !query -SELECT regr_r2(y, x) FROM testRegression WHERE x IS NOT NULL --- !query schema -struct --- !query output -0.9976905311778291 - - --- !query -SELECT k, corr(y, x), regr_r2(y, x) FROM testRegression GROUP BY k --- !query schema -struct --- !query output -1 NULL NULL -2 0.9988445981121532 0.9976905311778291 - - --- !query -SELECT k, corr(y, x) FILTER (WHERE x IS NOT NULL), regr_r2(y, x) FROM testRegression GROUP BY k --- !query schema -struct --- !query output -1 NULL NULL -2 0.9988445981121532 0.9976905311778291 - - --- !query -SELECT - collect_list(col), - array_agg(col) -FROM VALUES - (1), (2), (1) AS tab(col) --- !query schema -struct,collect_list(col):array> --- !query output -[1,2,1] [1,2,1] - - --- !query -SELECT - a, - collect_list(b), - array_agg(b) -FROM VALUES - (1,4),(2,3),(1,4),(2,4) AS v(a,b) -GROUP BY a --- !query schema -struct,collect_list(b):array> --- !query output -1 [4,4] [4,4] -2 [3,4] [3,4] - - --- !query -SELECT regr_avgx(y, x), regr_avgy(y, x) FROM testRegression --- !query schema -struct --- !query output -22.666666666666668 20.0 - - --- !query -SELECT regr_avgx(y, x), regr_avgy(y, x) FROM testRegression WHERE x IS NOT NULL AND y IS NOT NULL --- !query schema -struct --- !query output -22.666666666666668 20.0 - - --- !query -SELECT k, avg(x), avg(y), regr_avgx(y, x), regr_avgy(y, x) FROM testRegression GROUP BY k --- !query schema -struct --- !query output -1 NULL 10.0 NULL NULL -2 22.666666666666668 21.25 22.666666666666668 20.0 - - --- !query -SELECT k, avg(x) FILTER (WHERE x IS NOT NULL AND y IS NOT NULL), avg(y) FILTER (WHERE x IS NOT NULL AND y IS NOT NULL), regr_avgx(y, x), regr_avgy(y, x) FROM testRegression GROUP BY k --- !query schema -struct --- !query output -1 NULL NULL NULL NULL -2 22.666666666666668 20.0 22.666666666666668 20.0 - - --- !query -SELECT - percentile_cont(0.25) WITHIN GROUP (ORDER BY v), - percentile_cont(0.25) WITHIN GROUP (ORDER BY v DESC) -FROM aggr --- !query schema -struct --- !query output -10.0 30.0 - - --- !query -SELECT - k, - percentile_cont(0.25) WITHIN GROUP (ORDER BY v), - percentile_cont(0.25) WITHIN GROUP (ORDER BY v DESC) -FROM aggr -GROUP BY k -ORDER BY k --- !query schema -struct --- !query output -0 10.0 30.0 -1 12.5 17.5 -2 17.5 26.25 -3 60.0 60.0 -4 NULL NULL - - --- !query -SELECT - percentile_disc(0.25) WITHIN GROUP (ORDER BY v), - percentile_disc(0.25) WITHIN GROUP (ORDER BY v DESC) -FROM aggr --- !query schema -struct --- !query output -10.0 30.0 - - --- !query -SELECT - k, - percentile_disc(0.25) WITHIN GROUP (ORDER BY v), - percentile_disc(0.25) WITHIN GROUP (ORDER BY v DESC) -FROM aggr -GROUP BY k -ORDER BY k --- !query schema -struct --- !query output -0 10.0 30.0 -1 10.0 20.0 -2 10.0 30.0 -3 60.0 60.0 -4 NULL NULL diff --git a/gluten-ut/spark33/src/test/resources/sql-tests/results/misc-functions.sql.out b/gluten-ut/spark33/src/test/resources/sql-tests/results/misc-functions.sql.out deleted file mode 100644 index 6985233c331..00000000000 --- a/gluten-ut/spark33/src/test/resources/sql-tests/results/misc-functions.sql.out +++ /dev/null @@ -1,137 +0,0 @@ --- Automatically generated by SQLQueryTestSuite --- Number of queries: 16 - - --- !query -select typeof(null) --- !query schema -struct --- !query output -void - - --- !query -select typeof(true) --- !query schema -struct --- !query output -boolean - - --- !query -select typeof(1Y), typeof(1S), typeof(1), typeof(1L) --- !query schema -struct --- !query output -tinyint smallint int bigint - - --- !query -select typeof(cast(1.0 as float)), typeof(1.0D), typeof(1.2) --- !query schema -struct --- !query output -float double decimal(2,1) - - --- !query -select typeof(date '1986-05-23'), typeof(timestamp '1986-05-23'), typeof(interval '23 days') --- !query schema -struct --- !query output -date timestamp interval day - - --- !query -select typeof(x'ABCD'), typeof('SPARK') --- !query schema -struct --- !query output -binary string - - --- !query -select typeof(array(1, 2)), typeof(map(1, 2)), typeof(named_struct('a', 1, 'b', 'spark')) --- !query schema -struct --- !query output -array map struct - - --- !query -SELECT assert_true(true), assert_true(boolean(1)) --- !query schema -struct --- !query output -NULL NULL - - --- !query -SELECT assert_true(false) --- !query schema -struct<> --- !query output -org.apache.gluten.exception.GlutenException -'false' is not true! - - --- !query -SELECT assert_true(boolean(0)) --- !query schema -struct<> --- !query output -org.apache.gluten.exception.GlutenException -'cast(0 as boolean)' is not true! - - --- !query -SELECT assert_true(null) --- !query schema -struct<> --- !query output -org.apache.gluten.exception.GlutenException -'null' is not true! - - --- !query -SELECT assert_true(boolean(null)) --- !query schema -struct<> --- !query output -org.apache.gluten.exception.GlutenException -'cast(null as boolean)' is not true! - - --- !query -SELECT assert_true(false, 'custom error message') --- !query schema -struct<> --- !query output -org.apache.gluten.exception.GlutenException -custom error message - - --- !query -CREATE TEMPORARY VIEW tbl_misc AS SELECT * FROM (VALUES (1), (8), (2)) AS T(v) --- !query schema -struct<> --- !query output - - - --- !query -SELECT raise_error('error message') --- !query schema -struct<> --- !query output -org.apache.gluten.exception.GlutenException -error message - - --- !query -SELECT if(v > 5, raise_error('too big: ' || v), v + 1) FROM tbl_misc --- !query schema -struct<> --- !query output -org.apache.gluten.exception.GlutenException -too big: 8 diff --git a/gluten-ut/spark33/src/test/resources/sql-tests/results/random.sql.out b/gluten-ut/spark33/src/test/resources/sql-tests/results/random.sql.out deleted file mode 100644 index b269d40c356..00000000000 --- a/gluten-ut/spark33/src/test/resources/sql-tests/results/random.sql.out +++ /dev/null @@ -1,84 +0,0 @@ --- Automatically generated by SQLQueryTestSuite --- Number of queries: 10 - - --- !query -SELECT rand(0) --- !query schema -struct --- !query output -0.7604953758285915 - - --- !query -SELECT rand(cast(3 / 7 AS int)) --- !query schema -struct --- !query output -0.7604953758285915 - - --- !query -SELECT rand(NULL) --- !query schema -struct --- !query output -0.7604953758285915 - - --- !query -SELECT rand(cast(NULL AS int)) --- !query schema -struct --- !query output -0.7604953758285915 - - --- !query -SELECT rand(1.0) --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -cannot resolve 'rand(1.0BD)' due to data type mismatch: argument 1 requires (int or bigint) type, however, '1.0BD' is of decimal(2,1) type.; line 1 pos 7 - - --- !query -SELECT randn(0L) --- !query schema -struct --- !query output -1.6034991609278433 - - --- !query -SELECT randn(cast(3 / 7 AS long)) --- !query schema -struct --- !query output -1.6034991609278433 - - --- !query -SELECT randn(NULL) --- !query schema -struct --- !query output -1.6034991609278433 - - --- !query -SELECT randn(cast(NULL AS long)) --- !query schema -struct --- !query output -1.6034991609278433 - - --- !query -SELECT rand('1') --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -cannot resolve 'rand('1')' due to data type mismatch: argument 1 requires (int or bigint) type, however, ''1'' is of string type.; line 1 pos 7 diff --git a/gluten-ut/spark33/src/test/resources/sql-tests/results/udf/udf-group-by.sql.out b/gluten-ut/spark33/src/test/resources/sql-tests/results/udf/udf-group-by.sql.out deleted file mode 100644 index ea088f8e8f4..00000000000 --- a/gluten-ut/spark33/src/test/resources/sql-tests/results/udf/udf-group-by.sql.out +++ /dev/null @@ -1,523 +0,0 @@ --- Automatically generated by SQLQueryTestSuite --- Number of queries: 52 - - --- !query -CREATE OR REPLACE TEMPORARY VIEW testData AS SELECT * FROM VALUES -(1, 1), (1, 2), (2, 1), (2, 2), (3, 1), (3, 2), (null, 1), (3, null), (null, null) -AS testData(a, b) --- !query schema -struct<> --- !query output - - - --- !query -SELECT udf(a), udf(COUNT(b)) FROM testData --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -grouping expressions sequence is empty, and 'testdata.a' is not an aggregate function. Wrap '(CAST(udf(cast(count(b) as string)) AS BIGINT) AS `udf(count(b))`)' in windowing function(s) or wrap 'testdata.a' in first() (or first_value) if you don't care which value you get. - - --- !query -SELECT COUNT(udf(a)), udf(COUNT(b)) FROM testData --- !query schema -struct --- !query output -7 7 - - --- !query -SELECT udf(a), COUNT(udf(b)) FROM testData GROUP BY a --- !query schema -struct --- !query output -1 2 -2 2 -3 2 -NULL 1 - - --- !query -SELECT udf(a), udf(COUNT(udf(b))) FROM testData GROUP BY b --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -expression 'testdata.a' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get. - - --- !query -SELECT COUNT(udf(a)), COUNT(udf(b)) FROM testData GROUP BY udf(a) --- !query schema -struct --- !query output -0 1 -2 2 -2 2 -3 2 - - --- !query -SELECT 'foo', COUNT(udf(a)) FROM testData GROUP BY 1 --- !query schema -struct --- !query output -foo 7 - - --- !query -SELECT 'foo' FROM testData WHERE a = 0 GROUP BY udf(1) --- !query schema -struct --- !query output - - - --- !query -SELECT 'foo', udf(APPROX_COUNT_DISTINCT(udf(a))) FROM testData WHERE a = 0 GROUP BY udf(1) --- !query schema -struct --- !query output - - - --- !query -SELECT 'foo', MAX(STRUCT(udf(a))) FROM testData WHERE a = 0 GROUP BY udf(1) --- !query schema -struct> --- !query output - - - --- !query -SELECT udf(a + b), udf(COUNT(b)) FROM testData GROUP BY a + b --- !query schema -struct --- !query output -2 1 -3 2 -4 2 -5 1 -NULL 1 - - --- !query -SELECT udf(a + 2), udf(COUNT(b)) FROM testData GROUP BY a + 1 --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -expression 'testdata.a' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get. - - --- !query -SELECT udf(a + 1) + 1, udf(COUNT(b)) FROM testData GROUP BY udf(a + 1) --- !query schema -struct<(udf((a + 1)) + 1):int,udf(count(b)):bigint> --- !query output -3 2 -4 2 -5 2 -NULL 1 - - --- !query -SELECT SKEWNESS(udf(a)), udf(KURTOSIS(a)), udf(MIN(a)), MAX(udf(a)), udf(AVG(udf(a))), udf(VARIANCE(a)), STDDEV(udf(a)), udf(SUM(a)), udf(COUNT(a)) -FROM testData --- !query schema -struct --- !query output --0.27238010581457284 -1.5069204152249138 1 3 2.142857142857143 0.8095238095238096 0.8997354108424375 15 7 - - --- !query -SELECT COUNT(DISTINCT udf(b)), udf(COUNT(DISTINCT b, c)) FROM (SELECT 1 AS a, 2 AS b, 3 AS c) GROUP BY udf(a) --- !query schema -struct --- !query output -1 1 - - --- !query -SELECT udf(a) AS k, COUNT(udf(b)) FROM testData GROUP BY k --- !query schema -struct --- !query output -1 2 -2 2 -3 2 -NULL 1 - - --- !query -SELECT a AS k, udf(COUNT(b)) FROM testData GROUP BY k HAVING k > 1 --- !query schema -struct --- !query output -2 2 -3 2 - - --- !query -SELECT udf(COUNT(b)) AS k FROM testData GROUP BY k --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -aggregate functions are not allowed in GROUP BY, but found CAST(udf(cast(count(b) as string)) AS BIGINT) - - --- !query -CREATE OR REPLACE TEMPORARY VIEW testDataHasSameNameWithAlias AS SELECT * FROM VALUES -(1, 1, 3), (1, 2, 1) AS testDataHasSameNameWithAlias(k, a, v) --- !query schema -struct<> --- !query output - - - --- !query -SELECT k AS a, udf(COUNT(udf(v))) FROM testDataHasSameNameWithAlias GROUP BY udf(a) --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -expression 'testdatahassamenamewithalias.k' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get. - - --- !query -set spark.sql.groupByAliases=false --- !query schema -struct --- !query output -spark.sql.groupByAliases false - - --- !query -SELECT a AS k, udf(COUNT(udf(b))) FROM testData GROUP BY k --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -Column 'k' does not exist. Did you mean one of the following? [testdata.a, testdata.b]; line 1 pos 57 - - --- !query -SELECT udf(a), COUNT(udf(1)) FROM testData WHERE false GROUP BY udf(a) --- !query schema -struct --- !query output - - - --- !query -SELECT udf(COUNT(1)) FROM testData WHERE false --- !query schema -struct --- !query output -0 - - --- !query -SELECT 1 FROM (SELECT udf(COUNT(1)) FROM testData WHERE false) t --- !query schema -struct<1:int> --- !query output -1 - - --- !query -SELECT 1 from ( - SELECT 1 AS z, - udf(MIN(a.x)) - FROM (select 1 as x) a - WHERE false -) b -where b.z != b.z --- !query schema -struct<1:int> --- !query output - - - --- !query -SELECT corr(DISTINCT x, y), udf(corr(DISTINCT y, x)), count(*) - FROM (VALUES (1, 1), (2, 2), (2, 2)) t(x, y) --- !query schema -struct --- !query output -0.9999999999999999 0.9999999999999999 3 - - --- !query -SELECT udf(1) FROM range(10) HAVING true --- !query schema -struct --- !query output -1 - - --- !query -SELECT udf(udf(1)) FROM range(10) HAVING MAX(id) > 0 --- !query schema -struct --- !query output -1 - - --- !query -SELECT udf(id) FROM range(10) HAVING id > 0 --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -grouping expressions sequence is empty, and 'id' is not an aggregate function. Wrap '()' in windowing function(s) or wrap 'id' in first() (or first_value) if you don't care which value you get. - - --- !query -CREATE OR REPLACE TEMPORARY VIEW test_agg AS SELECT * FROM VALUES - (1, true), (1, false), - (2, true), - (3, false), (3, null), - (4, null), (4, null), - (5, null), (5, true), (5, false) AS test_agg(k, v) --- !query schema -struct<> --- !query output - - - --- !query -SELECT udf(every(v)), udf(some(v)), any(v) FROM test_agg WHERE 1 = 0 --- !query schema -struct --- !query output -NULL NULL NULL - - --- !query -SELECT udf(every(udf(v))), some(v), any(v) FROM test_agg WHERE k = 4 --- !query schema -struct --- !query output -NULL NULL NULL - - --- !query -SELECT every(v), udf(some(v)), any(v) FROM test_agg WHERE k = 5 --- !query schema -struct --- !query output -false true true - - --- !query -SELECT udf(k), every(v), udf(some(v)), any(v) FROM test_agg GROUP BY udf(k) --- !query schema -struct --- !query output -1 false true true -2 true true true -3 false false false -4 NULL NULL NULL -5 false true true - - --- !query -SELECT udf(k), every(v) FROM test_agg GROUP BY k HAVING every(v) = false --- !query schema -struct --- !query output -1 false -3 false -5 false - - --- !query -SELECT udf(k), udf(every(v)) FROM test_agg GROUP BY udf(k) HAVING every(v) IS NULL --- !query schema -struct --- !query output -4 NULL - - --- !query -SELECT udf(k), - udf(Every(v)) AS every -FROM test_agg -WHERE k = 2 - AND v IN (SELECT Any(v) - FROM test_agg - WHERE k = 1) -GROUP BY udf(k) --- !query schema -struct --- !query output -2 true - - --- !query -SELECT udf(udf(k)), - Every(v) AS every -FROM test_agg -WHERE k = 2 - AND v IN (SELECT Every(v) - FROM test_agg - WHERE k = 1) -GROUP BY udf(udf(k)) --- !query schema -struct --- !query output - - - --- !query -SELECT every(udf(1)) --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -cannot resolve 'every(CAST(udf(cast(1 as string)) AS INT))' due to data type mismatch: argument 1 requires boolean type, however, 'CAST(udf(cast(1 as string)) AS INT)' is of int type.; line 1 pos 7 - - --- !query -SELECT some(udf(1S)) --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -cannot resolve 'some(CAST(udf(cast(1 as string)) AS SMALLINT))' due to data type mismatch: argument 1 requires boolean type, however, 'CAST(udf(cast(1 as string)) AS SMALLINT)' is of smallint type.; line 1 pos 7 - - --- !query -SELECT any(udf(1L)) --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -cannot resolve 'any(CAST(udf(cast(1 as string)) AS BIGINT))' due to data type mismatch: argument 1 requires boolean type, however, 'CAST(udf(cast(1 as string)) AS BIGINT)' is of bigint type.; line 1 pos 7 - - --- !query -SELECT udf(every("true")) --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -cannot resolve 'every('true')' due to data type mismatch: argument 1 requires boolean type, however, ''true'' is of string type.; line 1 pos 11 - - --- !query -SELECT k, v, every(v) OVER (PARTITION BY k ORDER BY v) FROM test_agg --- !query schema -struct --- !query output -1 false false -1 true false -2 true true -3 NULL NULL -3 false false -4 NULL NULL -4 NULL NULL -5 NULL NULL -5 false false -5 true false - - --- !query -SELECT k, udf(udf(v)), some(v) OVER (PARTITION BY k ORDER BY v) FROM test_agg --- !query schema -struct --- !query output -1 false false -1 true true -2 true true -3 NULL NULL -3 false false -4 NULL NULL -4 NULL NULL -5 NULL NULL -5 false false -5 true true - - --- !query -SELECT udf(udf(k)), v, any(v) OVER (PARTITION BY k ORDER BY v) FROM test_agg --- !query schema -struct --- !query output -1 false false -1 true true -2 true true -3 NULL NULL -3 false false -4 NULL NULL -4 NULL NULL -5 NULL NULL -5 false false -5 true true - - --- !query -SELECT udf(count(*)) FROM test_agg HAVING count(*) > 1L --- !query schema -struct --- !query output -10 - - --- !query -SELECT k, udf(max(v)) FROM test_agg GROUP BY k HAVING max(v) = true --- !query schema -struct --- !query output -1 true -2 true -5 true - - --- !query -SELECT * FROM (SELECT udf(COUNT(*)) AS cnt FROM test_agg) WHERE cnt > 1L --- !query schema -struct --- !query output -10 - - --- !query -SELECT udf(count(*)) FROM test_agg WHERE count(*) > 1L --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException - -Aggregate/Window/Generate expressions are not valid in where clause of the query. -Expression in where clause: [(count(1) > 1L)] -Invalid expressions: [count(1)] - - --- !query -SELECT udf(count(*)) FROM test_agg WHERE count(*) + 1L > 1L --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException - -Aggregate/Window/Generate expressions are not valid in where clause of the query. -Expression in where clause: [((count(1) + 1L) > 1L)] -Invalid expressions: [count(1)] - - --- !query -SELECT udf(count(*)) FROM test_agg WHERE k = 1 or k = 2 or count(*) + 1L > 1L or max(k) > 1 --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException - -Aggregate/Window/Generate expressions are not valid in where clause of the query. -Expression in where clause: [(((test_agg.k = 1) OR (test_agg.k = 2)) OR (((count(1) + 1L) > 1L) OR (max(test_agg.k) > 1)))] -Invalid expressions: [count(1), max(test_agg.k)] diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseSQLQueryTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseSQLQueryTestSettings.scala deleted file mode 100644 index d3c65f94803..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseSQLQueryTestSettings.scala +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.utils.clickhouse - -import org.apache.gluten.utils.SQLQueryTestSettings - -object ClickHouseSQLQueryTestSettings extends SQLQueryTestSettings { - override def getResourceFilePath: String = - getClass.getResource("/").getPath + "../../../src/test/resources/sql-tests" - - override def getSupportedSQLQueryTests: Set[String] = Set() - - override def getOverwriteSQLQueryTests: Set[String] = Set() -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala deleted file mode 100644 index d9dcdde5b7f..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ /dev/null @@ -1,1885 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.utils.clickhouse - -import org.apache.gluten.utils.{BackendTestSettings, SQLQueryTestSettings} - -import org.apache.spark.sql._ -import org.apache.spark.sql.GlutenTestConstants.GLUTEN_TEST -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.expressions.aggregate._ -import org.apache.spark.sql.connector._ -import org.apache.spark.sql.errors._ -import org.apache.spark.sql.execution._ -import org.apache.spark.sql.execution.adaptive.clickhouse.ClickHouseAdaptiveQueryExecSuite -import org.apache.spark.sql.execution.datasources._ -import org.apache.spark.sql.execution.datasources.binaryfile.GlutenBinaryFileFormatSuite -import org.apache.spark.sql.execution.datasources.exchange._ -import org.apache.spark.sql.execution.datasources.json._ -import org.apache.spark.sql.execution.datasources.orc._ -import org.apache.spark.sql.execution.datasources.parquet._ -import org.apache.spark.sql.execution.datasources.text.{GlutenTextV1Suite, GlutenTextV2Suite} -import org.apache.spark.sql.execution.datasources.v2.{GlutenDataSourceV2StrategySuite, GlutenFileTableSuite, GlutenV2PredicateSuite} -import org.apache.spark.sql.execution.exchange.GlutenEnsureRequirementsSuite -import org.apache.spark.sql.execution.joins.{GlutenBroadcastJoinSuite, GlutenExistenceJoinSuite, GlutenInnerJoinSuite, GlutenOuterJoinSuite} -import org.apache.spark.sql.extension.GlutenSessionExtensionSuite -import org.apache.spark.sql.gluten.GlutenFallbackSuite -import org.apache.spark.sql.hive.execution.GlutenHiveSQLQueryCHSuite -import org.apache.spark.sql.sources._ - -// Some settings' line length exceeds 100 -// scalastyle:off line.size.limit - -class ClickHouseTestSettings extends BackendTestSettings { - import SuiteSettings._ - - // disable tests that will break the whole UT - override def shouldRun(suiteName: String, testName: String): Boolean = { - val preCheck = suiteName.split("[.]").last match { - case "GlutenCSVSuite" => !csvCoreDumpCases.contains(testName) - case "GlutenCSVv1Suite" => !csvCoreDumpCases.contains(testName) - case "GlutenCSVv2Suite" => !csvCoreDumpCases.contains(testName) - case "GlutenCSVLegacyTimeParserSuite" => !csvCoreDumpCases.contains(testName) - case "GlutenDataFrameSuite" => !dfCoreDumpCases.contains(testName) - case "GlutenDatasetSuite" => !dsSlowCases.contains(testName) - case "GlutenSQLQuerySuite" => !sqlQuerySlowCases.contains(testName) - // Below 2 suites temporarily ignored because of gluten domain name change - case "GlutenClickHouseMergeTreeWriteOnHDFSSuite" => - false - case "GlutenClickHouseMergeTreeWriteOnS3Suite" => - false - case "GlutenDataFrameWriterV2Suite" => - false // nativeDoValidate failed due to spark conf cleanup - case "GlutenDataSourceV2DataFrameSuite" => - false // nativeDoValidate failed due to spark conf cleanup - case "GlutenDataSourceV2FunctionSuite" => - false // nativeDoValidate failed due to spark conf cleanup - case "GlutenDataSourceV2SQLSuite" => - false // nativeDoValidate failed due to spark conf cleanup - case "GlutenMetadataColumnSuite" => false // nativeDoValidate failed due to spark conf cleanup - case "GlutenQueryCompilationErrorsDSv2Suite" => - false // nativeDoValidate failed due to spark conf cleanup - case "GlutenBloomFilterAggregateQuerySuite" => - !bloomFilterCases.contains(testName) - case "GlutenSortShuffleSuite" => false - case _ => true - } - preCheck && super.shouldRun(suiteName, testName) - } - - private val csvCoreDumpCases: Seq[String] = Seq( - "test with alternative delimiter and quote", - "SPARK-24540: test with multiple character delimiter (comma space)", - "DDL test with tab separated file", - "test with null quote character", - "SPARK-24540: test with multiple (crazy) character delimiter", - "nullable fields with user defined null value of \"null\"", - "SPARK-15585 turn off quotations", - "SPARK-29101 test count with DROPMALFORMED mode" - ) - - private val dfCoreDumpCases: Seq[String] = Seq( - "repartitionByRange", - GLUTEN_TEST + "repartitionByRange" - ) - - private val dsSlowCases: Seq[String] = Seq( - "SPARK-16995: flat mapping on Dataset containing a column created with lit/expr" - ) - - private val sqlQuerySlowCases: Seq[String] = Seq( - "SPARK-33084: Add jar support Ivy URI in SQL" - ) - - private val bloomFilterCases: Seq[String] = Seq( - // Currently return a empty set(same reason as sum(empty set), - // both behaviors are acceptable. - "Test that bloom_filter_agg produces a NULL with empty input" - ) - - enableSuite[GlutenApproxCountDistinctForIntervalsQuerySuite].exclude( - "test ApproxCountDistinctForIntervals with large number of endpoints") - enableSuite[GlutenApproximatePercentileQuerySuite].exclude( - "SPARK-32908: maximum target error in percentile_approx") - enableSuite[GlutenBloomFilterAggregateQuerySuite] - .exclude("Test bloom_filter_agg and might_contain") - .excludeGlutenTest("Test bloom_filter_agg with big RUNTIME_BLOOM_FILTER_MAX_NUM_ITEMS") - enableSuite[GlutenCTEHintSuite] - enableSuite[GlutenCTEInlineSuiteAEOff] - enableSuite[GlutenCTEInlineSuiteAEOn] - enableSuite[GlutenCachedTableSuite] - .exclude("SPARK-37369: Avoid redundant ColumnarToRow transition on InMemoryTableScan") - .exclude("analyzes column statistics in cached query") - .excludeGlutenTest("InMemoryRelation statistics") - // Needs to rewrite TimestampNTZType. - .excludeGlutenTest("SPARK-36120: Support cache/uncache table with TimestampNTZ type") - enableSuite[GlutenColumnExpressionSuite] - .exclude("input_file_name, input_file_block_start, input_file_block_length - FileScanRDD") - .exclude("withField should add field with no name") - .exclude("withField should add field to nullable struct") - .exclude("withField should add field to nested nullable struct") - .exclude("withField should add multiple fields to nullable struct") - .exclude("withField should add multiple fields to nested nullable struct") - .exclude("withField should replace field in nullable struct") - .exclude("withField should replace field in nested nullable struct") - .exclude("withField should replace multiple fields in nullable struct") - .exclude("withField should replace multiple fields in nested nullable struct") - .exclude("withField should replace all fields with given name in struct") - .exclude("withField user-facing examples") - .exclude("dropFields should drop field in nullable struct") - .exclude("dropFields should drop field with no name in struct") - .exclude("dropFields should drop field in nested nullable struct") - .exclude("dropFields should drop multiple fields in nested nullable struct") - .exclude("dropFields should drop all fields with given name in struct") - .exclude("dropFields user-facing examples") - .exclude("should move field up one level of nesting") - .exclude("SPARK-36778: add ilike API for scala") - enableSuite[GlutenComplexTypesSuite] - enableSuite[GlutenConfigBehaviorSuite].exclude( - "SPARK-22160 spark.sql.execution.rangeExchange.sampleSizePerPartition") - enableSuite[GlutenCountMinSketchAggQuerySuite] - enableSuite[GlutenCsvFunctionsSuite] - enableSuite[GlutenDSV2CharVarcharTestSuite] - // Excluded. The Gluten tests for char/varchar validation were rewritten for Velox. - // ClickHouse backend doesn't support this feature and falls back to vanilla Spark, - // causing mismatches in error messages. - .excludeGlutenTest("length check for input string values: nested in struct of array") - enableSuite[GlutenDSV2SQLInsertTestSuite] - enableSuite[GlutenDataFrameAggregateSuite] - .exclude("average") - .exclude("zero average") - .exclude("zero stddev") - .exclude("collect functions") - .exclude("collect functions structs") - .exclude("SPARK-17641: collect functions should not collect null values") - .exclude("collect functions should be able to cast to array type with no null values") - .exclude("SPARK-17616: distinct aggregate combined with a non-partial aggregate") - .exclude("SPARK-19471: AggregationIterator does not initialize the generated result projection before using it") - .excludeGlutenTest("SPARK-19471: AggregationIterator does not initialize the generated" + - " result projection before using it") - .exclude("SPARK-26021: NaN and -0.0 in grouping expressions") - .exclude("SPARK-32038: NormalizeFloatingNumbers should work on distinct aggregate") - .exclude("SPARK-32136: NormalizeFloatingNumbers should work on null struct") - .exclude("SPARK-34713: group by CreateStruct with ExtractValue") - .exclude("SPARK-34716: Support ANSI SQL intervals by the aggregate function `sum`") - .exclude("SPARK-34837: Support ANSI SQL intervals by the aggregate function `avg`") - .exclude("SPARK-35412: groupBy of year-month/day-time intervals should work") - .exclude("SPARK-36926: decimal average mistakenly overflow") - .excludeGlutenTest("use gluten hash agg to replace vanilla spark sort agg") - enableSuite[GlutenDataFrameAsOfJoinSuite] - enableSuite[GlutenDataFrameComplexTypeSuite] - enableSuite[GlutenDataFrameFunctionsSuite] - // Expected exception org.apache.spark.SparkException to be thrown, but no exception was thrown - .exclude("map_concat function") - .exclude("map with arrays") - .exclude("flatten function") - .exclude("aggregate function - array for primitive type not containing null") - .exclude("aggregate function - array for primitive type containing null") - .exclude("aggregate function - array for non-primitive type") - .exclude("SPARK-14393: values generated by non-deterministic functions shouldn't change after coalesce or union") - .exclude("SPARK-24734: Fix containsNull of Concat for array type") - .exclude("transform keys function - primitive data types") - .exclude("transform keys function - Invalid lambda functions and exceptions") - .exclude("transform values function - test primitive data types") - .exclude("transform values function - test empty") - enableSuite[GlutenDataFrameHintSuite] - enableSuite[GlutenDataFrameImplicitsSuite] - enableSuite[GlutenDataFrameJoinSuite].exclude( - "SPARK-32693: Compare two dataframes with same schema except nullable property") - enableSuite[GlutenDataFrameNaFunctionsSuite] - .exclude("replace nan with float") - .exclude("replace nan with double") - enableSuite[GlutenDataFramePivotSuite] - .exclude("pivot with column definition in groupby") - .exclude("pivot with timestamp and count should not print internal representation") - .exclude("SPARK-38133: Grouping by TIMESTAMP_NTZ should not corrupt results") - enableSuite[GlutenDataFrameRangeSuite] - enableSuite[GlutenDataFrameSelfJoinSuite] - enableSuite[GlutenDataFrameSessionWindowingSuite] - .exclude("simple session window with record at window start") - .exclude("session window groupBy statement") - .exclude("SPARK-36465: filter out events with negative/zero gap duration") - .exclude("session window groupBy with multiple keys statement") - .exclude("session window groupBy with multiple keys statement - one distinct") - .exclude("session window groupBy with multiple keys statement - two distinct") - .exclude("session window groupBy with multiple keys statement - keys overlapped with sessions") - .exclude("session window with multi-column projection") - .exclude("SPARK-36724: Support timestamp_ntz as a type of time column for SessionWindow") - enableSuite[GlutenDataFrameSetOperationsSuite] - .exclude("SPARK-10740: handle nondeterministic expressions correctly for set operations") - .exclude( - "SPARK-34283: SQL-style union using Dataset, keep necessary deduplicate in multiple unions") - .exclude("union should union DataFrames with UDTs (SPARK-13410)") - .exclude( - "SPARK-32376: Make unionByName null-filling behavior work with struct columns - simple") - .exclude( - "SPARK-32376: Make unionByName null-filling behavior work with struct columns - nested") - .exclude("SPARK-32376: Make unionByName null-filling behavior work with struct columns - case-sensitive cases") - .exclude( - "SPARK-32376: Make unionByName null-filling behavior work with struct columns - edge case") - .exclude("SPARK-35290: Make unionByName null-filling behavior work with struct columns - sorting edge case") - .exclude( - "SPARK-32376: Make unionByName null-filling behavior work with struct columns - deep expr") - .exclude("SPARK-35756: unionByName support struct having same col names but different sequence") - .exclude("SPARK-36797: Union should resolve nested columns as top-level columns") - .exclude("SPARK-37371: UnionExec should support columnar if all children support columnar") - .exclude( - "SPARK-36673: Only merge nullability for Unions of struct" - ) // disabled due to case-insensitive not supported in CH tuple - .exclude("except all") - .exclude("exceptAll - nullability") - .exclude("intersectAll") - .exclude("intersectAll - nullability") - enableSuite[GlutenDataFrameStatSuite] - enableSuite[GlutenDataFrameSuite] - .exclude("SPARK-27439: Explain result should match collected result after view change") - .exclude("Uuid expressions should produce same results at retries in the same DataFrame") - .exclude("SPARK-28224: Aggregate sum big decimal overflow") - .exclude("SPARK-28067: Aggregate sum should not return wrong results for decimal overflow") - .exclude("SPARK-35955: Aggregate avg should not return wrong results for decimal overflow") - .exclude("describe") - .exclude("getRows: array") - .exclude("showString: array") - .exclude("showString: array, vertical = true") - .exclude("SPARK-23023 Cast rows to strings in showString") - .exclude("SPARK-18350 show with session local timezone") - .exclude("SPARK-18350 show with session local timezone, vertical = true") - .exclude("SPARK-6899: type should match when using codegen") - .exclude("SPARK-7324 dropDuplicates") - .exclude( - "SPARK-8608: call `show` on local DataFrame with random columns should return same value") - .exclude("SPARK-8609: local DataFrame with random columns should return same value after sort") - .exclude("SPARK-9083: sort with non-deterministic expressions") - .exclude("SPARK-10316: respect non-deterministic expressions in PhysicalOperation") - .exclude("distributeBy and localSort") - .exclude("reuse exchange") - .exclude("SPARK-22271: mean overflows and returns null for some decimal variables") - .exclude("SPARK-22520: support code generation for large CaseWhen") - .exclude("SPARK-24165: CaseWhen/If - nullability of nested types") - .exclude("SPARK-27671: Fix analysis exception when casting null in nested field in struct") - .exclude("summary") - .excludeGlutenTest( - "SPARK-27439: Explain result should match collected result after view change") - .excludeGlutenTest("distributeBy and localSort") - .excludeGlutenTest("describe") - .excludeGlutenTest("Allow leading/trailing whitespace in string before casting") - enableSuite[GlutenDataFrameTimeWindowingSuite] - .exclude("simple tumbling window with record at window start") - .exclude("SPARK-21590: tumbling window using negative start time") - .exclude("tumbling window groupBy statement") - .exclude("tumbling window groupBy statement with startTime") - .exclude("SPARK-21590: tumbling window groupBy statement with negative startTime") - .exclude("tumbling window with multi-column projection") - .exclude("sliding window grouping") - .exclude("time window joins") - .exclude("negative timestamps") - .exclude("millisecond precision sliding windows") - enableSuite[GlutenDataFrameTungstenSuite].excludeGlutenTest("Map type with struct type as key") - enableSuite[GlutenDataFrameWindowFramesSuite] - .exclude("rows between should accept int/long values as boundary") - .exclude("range between should accept int/long values as boundary") - .exclude("reverse preceding/following range between with aggregation") - enableSuite[GlutenDataFrameWindowFunctionsSuite] - .exclude("corr, covar_pop, stddev_pop functions in specific window") - .exclude( - "SPARK-13860: corr, covar_pop, stddev_pop functions in specific window LEGACY_STATISTICAL_AGGREGATE off") - .exclude("covar_samp, var_samp (variance), stddev_samp (stddev) functions in specific window") - .exclude("SPARK-13860: covar_samp, var_samp (variance), stddev_samp (stddev) functions in specific window LEGACY_STATISTICAL_AGGREGATE off") - .exclude("lead/lag with ignoreNulls") - .exclude("Window spill with more than the inMemoryThreshold and spillThreshold") - .exclude("SPARK-21258: complex object in combination with spilling") - .exclude( - "SPARK-38237: require all cluster keys for child required distribution for window query") - .excludeGlutenTest("corr, covar_pop, stddev_pop functions in specific window") - enableSuite[GlutenDatasetAggregatorSuite] - enableSuite[GlutenDatasetCacheSuite] - enableSuite[GlutenDatasetOptimizationSuite] - .exclude("Pruned nested serializers: map of map key") - .exclude("Pruned nested serializers: map of complex key") - enableSuite[GlutenDatasetPrimitiveSuite] - enableSuite[GlutenDatasetSerializerRegistratorSuite] - enableSuite[GlutenDatasetSuite] - .exclude("SPARK-16853: select, case class and tuple") - .exclude("select 2, primitive and tuple") - .exclude("SPARK-15550 Dataset.show() should show inner nested products as rows") - .exclude("dropDuplicates") - .exclude("dropDuplicates: columns with same column name") - .exclude("SPARK-24762: select Option[Product] field") - .exclude("SPARK-24762: typed agg on Option[Product] type") - .exclude("SPARK-26233: serializer should enforce decimal precision and scale") - .exclude("groupBy.as") - .exclude("SPARK-40407: repartition should not result in severe data skew") - .exclude("SPARK-40660: Switch to XORShiftRandom to distribute elements") - enableSuite[GlutenDateFunctionsSuite] - .exclude("function to_date") - .exclude("unix_timestamp") - .exclude("to_unix_timestamp") - .exclude("to_timestamp with microseconds precision") - .exclude("SPARK-30668: use legacy timestamp parser in to_timestamp") - .exclude("SPARK-30766: date_trunc of old timestamps to hours and days") - .exclude("SPARK-30793: truncate timestamps before the epoch to seconds and minutes") - .excludeGlutenTest("unix_timestamp") - .excludeGlutenTest("to_unix_timestamp") - .exclude("to_utc_timestamp with column zone") - .exclude("from_utc_timestamp with column zone") - enableSuite[GlutenDeprecatedAPISuite] - enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOff].excludeGlutenTest( - "SPARK-32659: Fix the data issue when pruning DPP on non-atomic type") - enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOn].excludeGlutenTest( - "SPARK-32659: Fix the data issue when pruning DPP on non-atomic type") - enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOff].excludeGlutenTest( - "SPARK-32659: Fix the data issue when pruning DPP on non-atomic type") - enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOn].excludeGlutenTest( - "SPARK-32659: Fix the data issue when pruning DPP on non-atomic type") - enableSuite[GlutenExpressionsSchemaSuite] - enableSuite[GlutenExtraStrategiesSuite] - enableSuite[GlutenFileBasedDataSourceSuite] - .exclude("SPARK-23072 Write and read back unicode column names - csv") - .excludeByPrefix("Enabling/disabling ignoreMissingFiles using") - .excludeGlutenTestsByPrefix("Enabling/disabling ignoreMissingFiles using") - .exclude("Spark native readers should respect spark.sql.caseSensitive - parquet") - .exclude("Spark native readers should respect spark.sql.caseSensitive - orc") - .exclude("SPARK-25237 compute correct input metrics in FileScanRDD") - .exclude("SPARK-30362: test input metrics for DSV2") - .exclude("SPARK-37585: test input metrics for DSV2 with output limits") - .exclude("UDF input_file_name()") - .exclude("Option recursiveFileLookup: disable partition inferring") - .exclude("SPARK-31116: Select nested schema with case insensitive mode") - .exclude("SPARK-35669: special char in CSV header with filter pushdown") - // DISABLED: GLUTEN-4893 Vanilla UT checks scan operator by exactly matching the class type - .exclude("File source v2: support passing data filters to FileScan without partitionFilters") - // DISABLED: GLUTEN-4893 Vanilla UT checks scan operator by exactly matching the class type - .exclude("File source v2: support partition pruning") - .excludeGlutenTest("Spark native readers should respect spark.sql.caseSensitive - parquet") - .excludeGlutenTest("SPARK-25237 compute correct input metrics in FileScanRDD") - .excludeGlutenTest("Option recursiveFileLookup: disable partition inferring") - enableSuite[GlutenFileScanSuite] - enableSuite[GlutenFileSourceCharVarcharTestSuite] - .exclude("char type values should be padded or trimmed: partitioned columns") - .exclude("varchar type values length check and trim: partitioned columns") - .exclude("char/varchar type values length check: partitioned columns of other types") - .exclude("char type comparison: partitioned columns") - // Excluded. The Gluten tests for char/varchar validation were rewritten for Velox. - // ClickHouse backend doesn't support this feature and falls back to vanilla Spark, - // causing mismatches in error messages. - .excludeGlutenTest("length check for input string values: nested in struct of array") - enableSuite[GlutenFileSourceSQLInsertTestSuite] - .exclude("SPARK-33474: Support typed literals as partition spec values") - .exclude( - "SPARK-34556: checking duplicate static partition columns should respect case sensitive conf") - enableSuite[GlutenGeneratorFunctionSuite] - .exclude("single explode_outer") - .exclude("single posexplode") - .exclude("single posexplode_outer") - .exclude("explode_outer and other columns") - .exclude("aliased explode_outer") - .exclude("explode_outer on map") - .exclude("explode_outer on map with aliases") - .exclude("inline_outer") - .exclude("SPARK-14986: Outer lateral view with empty generate expression") - .exclude("outer explode()") - .exclude("generator in aggregate expression") - .exclude("SPARK-37947: lateral view _outer()") - enableSuite[GlutenInjectRuntimeFilterSuite].exclude("Merge runtime bloom filters") - enableSuite[GlutenIntervalFunctionsSuite] - enableSuite[GlutenJoinSuite].exclude( - "SPARK-36794: Ignore duplicated key when building relation for semi/anti hash join") - enableSuite[GlutenJsonExpressionsSuite] - .exclude( - "$.store.basket[0][*].b" - ) // issue: https://github.com/apache/gluten/issues/8529 - .exclude("from_json - invalid data") - .exclude("from_json - input=object, schema=array, output=array of single row") - .exclude("from_json - input=empty object, schema=array, output=array of single row with null") - .exclude("from_json - input=array of single object, schema=struct, output=single row") - .exclude("from_json - input=array, schema=struct, output=single row") - .exclude("from_json - input=empty array, schema=struct, output=single row with null") - .exclude("from_json - input=empty object, schema=struct, output=single row with null") - .exclude("SPARK-20549: from_json bad UTF-8") - .exclude("from_json with timestamp") - .exclude("to_json - struct") - .exclude("to_json - array") - .exclude("to_json - array with single empty row") - .exclude("to_json with timestamp") - .exclude("SPARK-21513: to_json support map[string, struct] to json") - .exclude("SPARK-21513: to_json support map[struct, struct] to json") - .exclude("parse date with locale") - .exclude("parse decimals using locale") - // NOT use gluten - .exclude("$..no_recursive") - .exclude("non foldable literal") - .exclude("some big value") - .exclude("from/to json - interval support") - .exclude("SPARK-24709: infer schema of json strings") - .exclude("infer schema of JSON strings by using options") - .exclude("inferring the decimal type using locale") - .exclude("json_object_keys") - enableSuite[GlutenJsonFunctionsSuite] - .exclude("from_json with option") - .exclude("from_json missing columns") - .exclude("from_json invalid json") - .exclude("from_json array support") - .exclude("to_json with option") - .exclude("roundtrip in to_json and from_json - array") - .exclude("SPARK-19637 Support to_json in SQL") - .exclude("SPARK-19967 Support from_json in SQL") - .exclude("pretty print - roundtrip from_json -> to_json") - .exclude("from_json invalid json - check modes") - .exclude("SPARK-36069: from_json invalid json schema - check field name and field value") - .exclude("corrupt record column in the middle") - .exclude("parse timestamps with locale") - .exclude("from_json - timestamp in micros") - .exclude("SPARK-33134: return partial results only for root JSON objects") - .exclude("SPARK-33907: bad json input with json pruning optimization: GetStructField") - .exclude("SPARK-33907: json pruning optimization with corrupt record field") - .exclude("SPARK-33907: bad json input with json pruning optimization: GetArrayStructFields") - enableSuite[GlutenMathFunctionsSuite] - enableSuite[GlutenMetadataCacheSuite].exclude( - "SPARK-16336,SPARK-27961 Suggest fixing FileNotFoundException") - enableSuite[GlutenMiscFunctionsSuite] - enableSuite[GlutenNestedDataSourceV1Suite] - enableSuite[GlutenNestedDataSourceV2Suite] - enableSuite[GlutenProcessingTimeSuite] - enableSuite[GlutenProductAggSuite] - enableSuite[GlutenReplaceNullWithFalseInPredicateEndToEndSuite] - enableSuite[GlutenSQLQuerySuite] - .exclude("self join with alias in agg") - .exclude("SPARK-3176 Added Parser of SQL LAST()") - .exclude("SPARK-3173 Timestamp support in the parser") - .exclude("SPARK-11111 null-safe join should not use cartesian product") - .exclude("SPARK-3349 partitioning after limit") - .exclude("aggregation with codegen updates peak execution memory") - .exclude("SPARK-10215 Div of Decimal returns null") - .exclude("precision smaller than scale") - .exclude("external sorting updates peak execution memory") - .exclude("run sql directly on files") - .exclude("Struct Star Expansion") - .exclude("Common subexpression elimination") - .exclude( - "SPARK-27619: When spark.sql.legacy.allowHashOnMapType is true, hash can be used on Maptype") - .exclude("SPARK-24940: coalesce and repartition hint") - .exclude("SPARK-25144 'distinct' causes memory leak") - .exclude("SPARK-29239: Subquery should not cause NPE when eliminating subexpression") - .exclude("normalize special floating numbers in subquery") - .exclude("SPARK-33677: LikeSimplification should be skipped if pattern contains any escapeChar") - .exclude("SPARK-33593: Vector reader got incorrect data with binary partition value") - .exclude("SPARK-33084: Add jar support Ivy URI in SQL -- jar contains udf class") - .exclude("SPARK-27442: Spark support read/write parquet file with invalid char in field name") - .exclude("SPARK-37965: Spark support read/write orc file with invalid char in field name") - .exclude("SPARK-38548: try_sum should return null if overflow happens before merging") - .exclude("SPARK-38589: try_avg should return null if overflow happens before merging") - .exclude("SPARK-39548: CreateView will make queries go into inline CTE code path thustrigger a mis-clarified `window definition not found` issue") - .excludeGlutenTest("SPARK-33593: Vector reader got incorrect data with binary partition value") - .excludeGlutenTest( - "SPARK-33677: LikeSimplification should be skipped if pattern contains any escapeChar") - enableSuite[GlutenSQLQueryTestSuite] - enableSuite[GlutenScalaReflectionRelationSuite] - enableSuite[GlutenSerializationSuite] - enableSuite[GlutenStatisticsCollectionSuite] - .exclude("analyze empty table") - .exclude("analyze column command - result verification") - .exclude("column stats collection for null columns") - .exclude("store and retrieve column stats in different time zones") - .excludeGlutenTest("store and retrieve column stats in different time zones") - .excludeCH("statistics collection of a table with zero column") - enableSuite[GlutenStringFunctionsSuite] - .exclude("string regex_replace / regex_extract") - .exclude("string overlay function") - .exclude("binary overlay function") - .exclude("string / binary length function") - .exclude("SPARK-36751: add octet length api for scala") - .exclude("SPARK-36751: add bit length api for scala") - enableSuite[GlutenSubquerySuite] - .exclude("SPARK-15370: COUNT bug in subquery in subquery in subquery") - .exclude("SPARK-26893: Allow pushdown of partition pruning subquery filters to file source") - .exclude("SPARK-28441: COUNT bug in nested subquery with non-foldable expr") - .exclude("SPARK-28441: COUNT bug with non-foldable expression in Filter condition") - .exclude("SPARK-36280: Remove redundant aliases after RewritePredicateSubquery") - .exclude("SPARK-36656: Do not collapse projects with correlate scalar subqueries") - .exclude("Merge non-correlated scalar subqueries from different parent plans") - .exclude("Merge non-correlated scalar subqueries with conflicting names") - enableSuite[GlutenTypedImperativeAggregateSuite] - enableSuite[GlutenUnwrapCastInComparisonEndToEndSuite].exclude("cases when literal is max") - enableSuite[GlutenXPathFunctionsSuite] - enableSuite[QueryTestSuite] - enableSuite[GlutenAnsiCastSuiteWithAnsiModeOff] - .exclude("cast string to date") - .exclude("SPARK-35711: cast timestamp without time zone to timestamp with local time zone") - .exclude("SPARK-35819: Support cast YearMonthIntervalType in different fields") - enableSuite[GlutenAnsiCastSuiteWithAnsiModeOn] - .exclude("null cast") - .exclude("cast string to date") - .exclude("cast string to timestamp") - .exclude("cast from boolean") - .exclude("cast from int") - .exclude("cast from long") - .exclude("cast from float") - .exclude("cast from double") - .exclude("cast from timestamp") - .exclude("data type casting") - .exclude("cast and add") - .exclude("from decimal") - .exclude("cast from array") - .exclude("cast from map") - .exclude("cast from struct") - .exclude("cast struct with a timestamp field") - .exclude("cast between string and interval") - .exclude("cast string to boolean") - .exclude("SPARK-20302 cast with same structure") - .exclude("SPARK-22500: cast for struct should not generate codes beyond 64KB") - .exclude("SPARK-27671: cast from nested null type in struct") - .exclude("Process Infinity, -Infinity, NaN in case insensitive manner") - .exclude("SPARK-22825 Cast array to string") - .exclude("SPARK-33291: Cast array with null elements to string") - .exclude("SPARK-22973 Cast map to string") - .exclude("SPARK-22981 Cast struct to string") - .exclude("SPARK-33291: Cast struct with null elements to string") - .exclude("SPARK-34667: cast year-month interval to string") - .exclude("SPARK-34668: cast day-time interval to string") - .exclude("SPARK-35698: cast timestamp without time zone to string") - .exclude("SPARK-35711: cast timestamp without time zone to timestamp with local time zone") - .exclude("SPARK-35716: cast timestamp without time zone to date type") - .exclude("SPARK-35718: cast date type to timestamp without timezone") - .exclude("SPARK-35719: cast timestamp with local time zone to timestamp without timezone") - .exclude("SPARK-35720: cast string to timestamp without timezone") - .exclude("SPARK-35112: Cast string to day-time interval") - .exclude("SPARK-35111: Cast string to year-month interval") - .exclude("SPARK-35820: Support cast DayTimeIntervalType in different fields") - .exclude("SPARK-35819: Support cast YearMonthIntervalType in different fields") - .exclude("SPARK-35768: Take into account year-month interval fields in cast") - .exclude("SPARK-35735: Take into account day-time interval fields in cast") - .exclude("ANSI mode: Throw exception on casting out-of-range value to byte type") - .exclude("ANSI mode: Throw exception on casting out-of-range value to short type") - .exclude("ANSI mode: Throw exception on casting out-of-range value to int type") - .exclude("ANSI mode: Throw exception on casting out-of-range value to long type") - .exclude("Fast fail for cast string type to decimal type in ansi mode") - .exclude("cast a timestamp before the epoch 1970-01-01 00:00:00Z") - .exclude("cast from array III") - .exclude("cast from map II") - .exclude("cast from map III") - .exclude("cast from struct II") - .exclude("cast from struct III") - enableSuite[GlutenArithmeticExpressionSuite] - .exclude("- (UnaryMinus)") - .exclude("/ (Divide) basic") - .exclude("/ (Divide) for Long and Decimal type") - .exclude("% (Remainder)") - .exclude("SPARK-17617: % (Remainder) double % double on super big double") - .exclude("Abs") - .exclude("pmod") - .exclude("SPARK-28322: IntegralDivide supports decimal type") - .exclude("SPARK-33008: division by zero on divide-like operations returns incorrect result") - .exclude("SPARK-34920: error class") - .exclude("SPARK-36920: Support year-month intervals by ABS") - .exclude("SPARK-36920: Support day-time intervals by ABS") - .exclude("SPARK-36921: Support YearMonthIntervalType by div") - .exclude("SPARK-36921: Support DayTimeIntervalType by div") - enableSuite[GlutenBitwiseExpressionsSuite] - enableSuite[GlutenCastSuite] - .exclude("null cast") - .exclude("cast string to date") - .exclude("cast string to timestamp") - .excludeGlutenTest("cast string to timestamp") - .exclude("cast from boolean") - .exclude("data type casting") - .excludeGlutenTest("data type casting") - .exclude("cast between string and interval") - .exclude("SPARK-27671: cast from nested null type in struct") - .exclude("Process Infinity, -Infinity, NaN in case insensitive manner") - .exclude("SPARK-22825 Cast array to string") - .exclude("SPARK-33291: Cast array with null elements to string") - .exclude("SPARK-22973 Cast map to string") - .exclude("SPARK-22981 Cast struct to string") - .exclude("SPARK-33291: Cast struct with null elements to string") - .exclude("SPARK-34667: cast year-month interval to string") - .exclude("SPARK-34668: cast day-time interval to string") - .exclude("SPARK-35698: cast timestamp without time zone to string") - .exclude("SPARK-35711: cast timestamp without time zone to timestamp with local time zone") - .exclude("SPARK-35716: cast timestamp without time zone to date type") - .exclude("SPARK-35718: cast date type to timestamp without timezone") - .exclude("SPARK-35719: cast timestamp with local time zone to timestamp without timezone") - .exclude("SPARK-35720: cast string to timestamp without timezone") - .exclude("SPARK-35112: Cast string to day-time interval") - .exclude("SPARK-35111: Cast string to year-month interval") - .exclude("SPARK-35820: Support cast DayTimeIntervalType in different fields") - .exclude("SPARK-35819: Support cast YearMonthIntervalType in different fields") - .exclude("SPARK-35768: Take into account year-month interval fields in cast") - .exclude("SPARK-35735: Take into account day-time interval fields in cast") - .exclude("null cast #2") - .exclude("cast string to date #2") - .exclude("casting to fixed-precision decimals") - .exclude("SPARK-28470: Cast should honor nullOnOverflow property") - .exclude("cast string to boolean II") - .exclude("cast from array II") - .exclude("cast from map II") - .exclude("cast from struct II") - .exclude("cast from date") - .exclude("cast from timestamp II") - .exclude("cast a timestamp before the epoch 1970-01-01 00:00:00Z") - .exclude("SPARK-32828: cast from a derived user-defined type to a base type") - .exclude("SPARK-34727: cast from float II") - .exclude("SPARK-35720: cast invalid string input to timestamp without time zone") - .exclude("SPARK-36924: Cast DayTimeIntervalType to IntegralType") - .exclude("SPARK-36924: Cast IntegralType to DayTimeIntervalType") - .exclude("SPARK-36924: Cast YearMonthIntervalType to IntegralType") - .exclude("SPARK-36924: Cast IntegralType to YearMonthIntervalType") - .exclude("Cast should output null for invalid strings when ANSI is not enabled.") - .exclude("cast timestamp to Int64 with floor division") - .exclude("cast from boolean to timestamp") - enableSuite[GlutenCastSuiteWithAnsiModeOn] - .exclude("null cast") - .exclude("cast string to date") - .exclude("cast string to timestamp") - .exclude("cast from boolean") - .exclude("cast from int") - .exclude("cast from long") - .exclude("cast from float") - .exclude("cast from double") - .exclude("cast from timestamp") - .exclude("data type casting") - .exclude("cast and add") - .exclude("from decimal") - .exclude("cast from array") - .exclude("cast from map") - .exclude("cast from struct") - .exclude("cast struct with a timestamp field") - .exclude("cast between string and interval") - .exclude("cast string to boolean") - .exclude("SPARK-20302 cast with same structure") - .exclude("SPARK-22500: cast for struct should not generate codes beyond 64KB") - .exclude("SPARK-27671: cast from nested null type in struct") - .exclude("Process Infinity, -Infinity, NaN in case insensitive manner") - .exclude("SPARK-22825 Cast array to string") - .exclude("SPARK-33291: Cast array with null elements to string") - .exclude("SPARK-22973 Cast map to string") - .exclude("SPARK-22981 Cast struct to string") - .exclude("SPARK-33291: Cast struct with null elements to string") - .exclude("SPARK-34667: cast year-month interval to string") - .exclude("SPARK-34668: cast day-time interval to string") - .exclude("SPARK-35698: cast timestamp without time zone to string") - .exclude("SPARK-35711: cast timestamp without time zone to timestamp with local time zone") - .exclude("SPARK-35716: cast timestamp without time zone to date type") - .exclude("SPARK-35718: cast date type to timestamp without timezone") - .exclude("SPARK-35719: cast timestamp with local time zone to timestamp without timezone") - .exclude("SPARK-35720: cast string to timestamp without timezone") - .exclude("SPARK-35112: Cast string to day-time interval") - .exclude("SPARK-35111: Cast string to year-month interval") - .exclude("SPARK-35820: Support cast DayTimeIntervalType in different fields") - .exclude("SPARK-35819: Support cast YearMonthIntervalType in different fields") - .exclude("SPARK-35768: Take into account year-month interval fields in cast") - .exclude("SPARK-35735: Take into account day-time interval fields in cast") - .exclude("ANSI mode: Throw exception on casting out-of-range value to byte type") - .exclude("ANSI mode: Throw exception on casting out-of-range value to short type") - .exclude("ANSI mode: Throw exception on casting out-of-range value to int type") - .exclude("ANSI mode: Throw exception on casting out-of-range value to long type") - .exclude("Fast fail for cast string type to decimal type in ansi mode") - .exclude("cast a timestamp before the epoch 1970-01-01 00:00:00Z") - .exclude("cast from array III") - .exclude("cast from map II") - .exclude("cast from map III") - .exclude("cast from struct II") - .exclude("cast from struct III") - enableSuite[GlutenCollectionExpressionsSuite] - .exclude("ArraysZip") // wait for https://github.com/ClickHouse/ClickHouse/pull/69576 - .exclude("Sequence of numbers") - .exclude("Shuffle") - .exclude("SPARK-33386: element_at ArrayIndexOutOfBoundsException") - .exclude("SPARK-33460: element_at NoSuchElementException") - .exclude("SPARK-36753: ArrayExcept should handle duplicated Double.NaN and Float.Nan") - .exclude( - "SPARK-36740: ArrayMin/ArrayMax/SortArray should handle NaN greater then non-NaN value") - .excludeGlutenTest("Shuffle") - enableSuite[GlutenComplexTypeSuite] - enableSuite[GlutenConditionalExpressionSuite] - .exclude("case when") - .exclude("if/case when - null flags of non-primitive types") - enableSuite[GlutenDateExpressionsSuite] - .exclude("DayOfYear") - .exclude("Year") - .exclude("Quarter") - .exclude("Month") - .exclude("Day / DayOfMonth") - .exclude("Seconds") - .exclude("DayOfWeek") - .exclude("WeekDay") - .exclude("WeekOfYear") - .exclude("DateFormat") - .excludeGlutenTest("DateFormat") - .exclude("Hour") - .exclude("Minute") - .exclude("date add interval") - .exclude("time_add") - .exclude("time_sub") - .exclude("add_months") - .exclude("SPARK-34721: add a year-month interval to a date") - .exclude("months_between") - .excludeGlutenTest("months_between") - .exclude("next_day") - .exclude("TruncDate") - .exclude("TruncTimestamp") - .exclude("unsupported fmt fields for trunc/date_trunc results null") - .exclude("from_unixtime") - .excludeGlutenTest("from_unixtime") - .exclude("unix_timestamp") - .exclude("to_unix_timestamp") - .exclude("to_utc_timestamp") - .exclude("from_utc_timestamp") - .exclude("creating values of DateType via make_date") - .exclude("creating values of Timestamp/TimestampNTZ via make_timestamp") - .exclude("ISO 8601 week-numbering year") - .exclude("extract the seconds part with fraction from timestamps") - .exclude("SPARK-34903: timestamps difference") - .exclude("SPARK-35916: timestamps without time zone difference") - .exclude("SPARK-34896: subtract dates") - .exclude("to_timestamp_ntz") - .exclude("to_timestamp exception mode") - .exclude("SPARK-31896: Handle am-pm timestamp parsing when hour is missing") - .exclude("UNIX_SECONDS") - .exclude("TIMESTAMP_SECONDS") // refer to https://github.com/ClickHouse/ClickHouse/issues/69280 - .exclude("TIMESTAMP_MICROS") // refer to https://github.com/apache/gluten/issues/7127 - .exclude("SPARK-33498: GetTimestamp,UnixTimestamp,ToUnixTimestamp with parseError") - .exclude("SPARK-34739,SPARK-35889: add a year-month interval to a timestamp") - .exclude("SPARK-34761,SPARK-35889: add a day-time interval to a timestamp") - .exclude("SPARK-37552: convert a timestamp_ntz to another time zone") - .exclude("SPARK-38195: add a quantity of interval units to a timestamp") - .exclude("SPARK-38284: difference between two timestamps in units") - .excludeGlutenTest("unix_timestamp") - .excludeGlutenTest("to_unix_timestamp") - .excludeGlutenTest("Hour") - enableSuite[GlutenDecimalExpressionSuite] - enableSuite[GlutenDecimalPrecisionSuite] - enableSuite[GlutenHashExpressionsSuite] - .exclude("sha2") - .exclude("murmur3/xxHash64/hive hash: struct") - .exclude("SPARK-30633: xxHash64 with long seed: struct") - .exclude("murmur3/xxHash64/hive hash: struct,arrayOfString:array,arrayOfArrayOfString:array>,arrayOfArrayOfInt:array>,arrayOfStruct:array>,arrayOfUDT:array>") - .exclude("SPARK-30633: xxHash64 with long seed: struct,arrayOfString:array,arrayOfArrayOfString:array>,arrayOfArrayOfInt:array>,arrayOfStruct:array>,arrayOfUDT:array>") - .exclude("murmur3/xxHash64/hive hash: struct,structOfStructOfString:struct>,structOfArray:struct>,structOfUDT:struct>") - .exclude("SPARK-30633: xxHash64 with long seed: struct,structOfStructOfString:struct>,structOfArray:struct>,structOfUDT:struct>") - .exclude("SPARK-30633: xxHash with different type seeds") - .exclude("SPARK-35113: HashExpression support DayTimeIntervalType/YearMonthIntervalType") - .exclude("SPARK-35207: Compute hash consistent between -0.0 and 0.0") - enableSuite[GlutenIntervalExpressionsSuite] - .exclude("years") - .exclude("months") - .exclude("days") - .exclude("hours") - .exclude("minutes") - .exclude("seconds") - .exclude("multiply") - .exclude("divide") - .exclude("make interval") - .exclude("ANSI mode: make interval") - .exclude("SPARK-35130: make day time interval") - .exclude("SPARK-34824: multiply year-month interval by numeric") - .exclude("SPARK-34850: multiply day-time interval by numeric") - .exclude("SPARK-34868: divide year-month interval by numeric") - .exclude("SPARK-34875: divide day-time interval by numeric") - .exclude("ANSI: extract years and months") - .exclude("ANSI: extract days, hours, minutes and seconds") - .exclude("SPARK-35129: make_ym_interval") - .exclude("SPARK-35728: Check multiply/divide of day-time intervals of any fields by numeric") - .exclude("SPARK-35778: Check multiply/divide of year-month intervals of any fields by numeric") - enableSuite[GlutenLiteralExpressionSuite] - .exclude("default") - .exclude("SPARK-37967: Literal.create support ObjectType") - enableSuite[GlutenMathExpressionsSuite] - .exclude("unhex") // https://github.com/apache/gluten/issues/7232 - .exclude("round/bround/floor/ceil") // https://github.com/apache/gluten/issues/7233 - .exclude("atan2") // https://github.com/apache/gluten/issues/7233 - enableSuite[GlutenMiscExpressionsSuite] - enableSuite[GlutenNondeterministicSuite] - .exclude("MonotonicallyIncreasingID") - .exclude("SparkPartitionID") - .exclude("InputFileName") - enableSuite[GlutenNullExpressionsSuite] - .exclude("AtLeastNNonNulls") - .exclude("AtLeastNNonNulls should not throw 64KiB exception") - enableSuite[GlutenPredicateSuite] - .exclude("3VL Not") - .exclude("3VL AND") - .exclude("3VL OR") - .exclude("3VL =") - .exclude("basic IN/INSET predicate test") - .exclude("IN with different types") - .exclude("IN/INSET: binary") - .exclude("IN/INSET: struct") - .exclude("IN/INSET: array") - .exclude("BinaryComparison: lessThan") - .exclude("BinaryComparison: LessThanOrEqual") - .exclude("BinaryComparison: GreaterThan") - .exclude("BinaryComparison: GreaterThanOrEqual") - .exclude("BinaryComparison: EqualTo") - .exclude("BinaryComparison: EqualNullSafe") - .exclude("BinaryComparison: null test") - .exclude("EqualTo on complex type") - .exclude("isunknown and isnotunknown") - .exclude("SPARK-32764: compare special double/float values") - .exclude("SPARK-32110: compare special double/float values in array") - .exclude("SPARK-32110: compare special double/float values in struct") - enableSuite[GlutenRandomSuite].exclude("random").exclude("SPARK-9127 codegen with long seed") - enableSuite[GlutenRegexpExpressionsSuite] - .exclude("LIKE Pattern") - .exclude("LIKE Pattern ESCAPE '/'") - .exclude("LIKE Pattern ESCAPE '#'") - .exclude("LIKE Pattern ESCAPE '\"'") - .exclude("RLIKE Regular Expression") - .exclude("RegexReplace") - .exclude("RegexExtract") - .exclude("RegexExtractAll") - enableSuite[GlutenSortOrderExpressionsSuite].exclude("SortPrefix") - enableSuite[GlutenStringExpressionsSuite] - .exclude("StringComparison") - .exclude("Substring") - .exclude("string substring_index function") - .exclude("SPARK-40213: ascii for Latin-1 Supplement characters") - .exclude("ascii for string") - .exclude("base64/unbase64 for string") - .exclude("encode/decode for string") - .exclude("overlay for string") - .exclude("overlay for byte array") - .exclude("translate") - .exclude("LOCATE") - .exclude("REPEAT") - .exclude("ParseUrl") - .exclude("SPARK-33468: ParseUrl in ANSI mode should fail if input string is not a valid url") - .exclude("FORMAT") // refer https://github.com/apache/gluten/issues/6765 - .exclude( - "soundex unit test" - ) // CH and spark returns different results when input non-ASCII characters - enableSuite[GlutenTryCastSuite] - .exclude("null cast") - .exclude("cast string to date") - .exclude("cast string to timestamp") - .excludeGlutenTest("cast string to timestamp") - .exclude("cast from boolean") - .exclude("cast from int") - .exclude("cast from long") - .exclude("cast from float") - .exclude("cast from double") - .exclude("cast from timestamp") - .exclude("data type casting") - .exclude("cast and add") - .exclude("from decimal") - .exclude("cast from array") - .exclude("cast from map") - .exclude("cast from struct") - .exclude("cast struct with a timestamp field") - .exclude("cast between string and interval") - .exclude("cast string to boolean") - .exclude("SPARK-20302 cast with same structure") - .exclude("SPARK-22500: cast for struct should not generate codes beyond 64KB") - .exclude("SPARK-27671: cast from nested null type in struct") - .exclude("Process Infinity, -Infinity, NaN in case insensitive manner") - .exclude("SPARK-22825 Cast array to string") - .exclude("SPARK-33291: Cast array with null elements to string") - .exclude("SPARK-22973 Cast map to string") - .exclude("SPARK-22981 Cast struct to string") - .exclude("SPARK-33291: Cast struct with null elements to string") - .exclude("SPARK-34667: cast year-month interval to string") - .exclude("SPARK-34668: cast day-time interval to string") - .exclude("SPARK-35698: cast timestamp without time zone to string") - .exclude("SPARK-35711: cast timestamp without time zone to timestamp with local time zone") - .exclude("SPARK-35716: cast timestamp without time zone to date type") - .exclude("SPARK-35718: cast date type to timestamp without timezone") - .exclude("SPARK-35719: cast timestamp with local time zone to timestamp without timezone") - .exclude("SPARK-35720: cast string to timestamp without timezone") - .exclude("SPARK-35112: Cast string to day-time interval") - .exclude("SPARK-35111: Cast string to year-month interval") - .exclude("SPARK-35820: Support cast DayTimeIntervalType in different fields") - .exclude("SPARK-35819: Support cast YearMonthIntervalType in different fields") - .exclude("SPARK-35768: Take into account year-month interval fields in cast") - .exclude("SPARK-35735: Take into account day-time interval fields in cast") - .exclude("ANSI mode: Throw exception on casting out-of-range value to byte type") - .exclude("ANSI mode: Throw exception on casting out-of-range value to short type") - .exclude("ANSI mode: Throw exception on casting out-of-range value to int type") - .exclude("ANSI mode: Throw exception on casting out-of-range value to long type") - .exclude("ANSI mode: Throw exception on casting out-of-range value to decimal type") - .exclude("cast from invalid string to numeric should throw NumberFormatException") - .exclude("Fast fail for cast string type to decimal type in ansi mode") - .exclude("ANSI mode: cast string to boolean with parse error") - .exclude("cast from timestamp II") - .exclude("cast a timestamp before the epoch 1970-01-01 00:00:00Z II") - .exclude("cast a timestamp before the epoch 1970-01-01 00:00:00Z") - .exclude("cast from map II") - .exclude("cast from struct II") - .exclude("ANSI mode: cast string to timestamp with parse error") - .exclude("ANSI mode: cast string to date with parse error") - .exclude("SPARK-26218: Fix the corner case of codegen when casting float to Integer") - .exclude("SPARK-35720: cast invalid string input to timestamp without time zone") - .excludeGlutenTest("SPARK-35698: cast timestamp without time zone to string") - enableSuite[GlutenDataSourceV2DataFrameSessionCatalogSuite] - enableSuite[GlutenDataSourceV2SQLSessionCatalogSuite] - enableSuite[GlutenDataSourceV2SQLSuite] - enableSuite[GlutenDataSourceV2Suite] - .exclude("partitioning reporting") - .exclude("SPARK-33267: push down with condition 'in (..., null)' should not throw NPE") - enableSuite[GlutenDeleteFromTableSuite] - enableSuite[GlutenFileDataSourceV2FallBackSuite] - // Rewritten - .exclude("Fallback Parquet V2 to V1") - enableSuite[GlutenKeyGroupedPartitioningSuite] - .exclude("partitioned join: number of buckets mismatch should trigger shuffle") - .exclude("partitioned join: only one side reports partitioning") - .exclude("partitioned join: join with two partition keys and different # of partition keys") - enableSuite[GlutenLocalScanSuite] - enableSuite[GlutenSupportsCatalogOptionsSuite] - enableSuite[GlutenTableCapabilityCheckSuite] - enableSuite[GlutenWriteDistributionAndOrderingSuite] - enableSuite[GlutenQueryCompilationErrorsSuite] - .exclude("CANNOT_USE_MIXTURE: Using aggregate function with grouped aggregate pandas UDF") - .exclude("UNSUPPORTED_FEATURE: Using pandas UDF aggregate expression with pivot") - enableSuite[GlutenQueryExecutionErrorsSuite] - .exclude( - "INCONSISTENT_BEHAVIOR_CROSS_VERSION: compatibility with Spark 2.4/3.2 in reading/writing dates") - .exclude("UNSUPPORTED_OPERATION - SPARK-38504: can't read TimestampNTZ as TimestampLTZ") - enableSuite[GlutenQueryParsingErrorsSuite] - enableSuite[FallbackStrategiesSuite] - enableSuite[GlutenBroadcastExchangeSuite] - enableSuite[GlutenCoalesceShufflePartitionsSuite] - .exclude( - "determining the number of reducers: aggregate operator(minNumPostShufflePartitions: 5)") - .exclude("determining the number of reducers: join operator(minNumPostShufflePartitions: 5)") - .exclude("determining the number of reducers: complex query 1(minNumPostShufflePartitions: 5)") - .exclude("determining the number of reducers: complex query 2(minNumPostShufflePartitions: 5)") - .exclude( - "determining the number of reducers: plan already partitioned(minNumPostShufflePartitions: 5)") - .exclude("determining the number of reducers: aggregate operator") - .exclude("determining the number of reducers: join operator") - .exclude("determining the number of reducers: complex query 1") - .exclude("determining the number of reducers: complex query 2") - .exclude("determining the number of reducers: plan already partitioned") - .exclude("SPARK-24705 adaptive query execution works correctly when exchange reuse enabled") - .exclude("Do not reduce the number of shuffle partition for repartition") - .exclude("Union two datasets with different pre-shuffle partition number") - .exclude("SPARK-34790: enable IO encryption in AQE partition coalescing") - .excludeGlutenTest( - "SPARK-24705 adaptive query execution works correctly when exchange reuse enabled") - .excludeGlutenTest("SPARK-34790: enable IO encryption in AQE partition coalescing") - .excludeGlutenTest( - "determining the number of reducers: aggregate operator(minNumPostShufflePartitions: 5)") - .excludeGlutenTest( - "determining the number of reducers: join operator(minNumPostShufflePartitions: 5)") - .excludeGlutenTest( - "determining the number of reducers: complex query 1(minNumPostShufflePartitions: 5)") - .excludeGlutenTest( - "determining the number of reducers: complex query 2(minNumPostShufflePartitions: 5)") - .excludeGlutenTest( - "determining the number of reducers: plan already partitioned(minNumPostShufflePartitions: 5)") - .excludeGlutenTest("determining the number of reducers: aggregate operator") - .excludeGlutenTest("determining the number of reducers: join operator") - .excludeGlutenTest("determining the number of reducers: complex query 1") - .excludeGlutenTest("determining the number of reducers: complex query 2") - .excludeGlutenTest("determining the number of reducers: plan already partitioned") - enableSuite[GlutenExchangeSuite] - .exclude("shuffling UnsafeRows in exchange") - .exclude("SPARK-23207: Make repartition() generate consistent output") - .exclude("Exchange reuse across the whole plan") - enableSuite[GlutenReplaceHashWithSortAggSuite] - .exclude("replace partial hash aggregate with sort aggregate") - .exclude("replace partial and final hash aggregate together with sort aggregate") - .exclude("do not replace hash aggregate if child does not have sort order") - .exclude("do not replace hash aggregate if there is no group-by column") - .excludeGlutenTest("replace partial hash aggregate with sort aggregate") - enableSuite[GlutenReuseExchangeAndSubquerySuite] - enableSuite[GlutenSQLAggregateFunctionSuite] - .excludeGlutenTest("Return NaN or null when dividing by zero") - enableSuite[GlutenSQLWindowFunctionSuite] - .exclude("window function: partition and order expressions") - .exclude("window function: expressions in arguments of a window functions") - .exclude( - "window function: multiple window expressions specified by range in a single expression") - .exclude("SPARK-7595: Window will cause resolve failed with self join") - .exclude( - "SPARK-16633: lead/lag should return the default value if the offset row does not exist") - .exclude("lead/lag should respect null values") - .exclude("test with low buffer spill threshold") - enableSuite[GlutenSameResultSuite] - enableSuite[GlutenSortSuite] - .exclude("basic sorting using ExternalSort") - .exclude("sort followed by limit") - .exclude("sorting does not crash for large inputs") - .exclude("sorting updates peak execution memory") - .exclude("SPARK-33260: sort order is a Stream") - .exclude("sorting on StringType with nullable=true, sortOrder=List('a ASC NULLS FIRST)") - .exclude("sorting on StringType with nullable=true, sortOrder=List('a ASC NULLS LAST)") - .exclude("sorting on StringType with nullable=true, sortOrder=List('a DESC NULLS LAST)") - .exclude("sorting on StringType with nullable=true, sortOrder=List('a DESC NULLS FIRST)") - .exclude("sorting on StringType with nullable=false, sortOrder=List('a ASC NULLS FIRST)") - .exclude("sorting on StringType with nullable=false, sortOrder=List('a ASC NULLS LAST)") - .exclude("sorting on StringType with nullable=false, sortOrder=List('a DESC NULLS LAST)") - .exclude("sorting on StringType with nullable=false, sortOrder=List('a DESC NULLS FIRST)") - .exclude("sorting on LongType with nullable=true, sortOrder=List('a ASC NULLS FIRST)") - .exclude("sorting on LongType with nullable=true, sortOrder=List('a ASC NULLS LAST)") - .exclude("sorting on LongType with nullable=true, sortOrder=List('a DESC NULLS LAST)") - .exclude("sorting on LongType with nullable=true, sortOrder=List('a DESC NULLS FIRST)") - .exclude("sorting on LongType with nullable=false, sortOrder=List('a ASC NULLS FIRST)") - .exclude("sorting on LongType with nullable=false, sortOrder=List('a ASC NULLS LAST)") - .exclude("sorting on LongType with nullable=false, sortOrder=List('a DESC NULLS LAST)") - .exclude("sorting on LongType with nullable=false, sortOrder=List('a DESC NULLS FIRST)") - .exclude("sorting on IntegerType with nullable=true, sortOrder=List('a ASC NULLS FIRST)") - .exclude("sorting on IntegerType with nullable=true, sortOrder=List('a ASC NULLS LAST)") - .exclude("sorting on IntegerType with nullable=true, sortOrder=List('a DESC NULLS LAST)") - .exclude("sorting on IntegerType with nullable=true, sortOrder=List('a DESC NULLS FIRST)") - .exclude("sorting on IntegerType with nullable=false, sortOrder=List('a ASC NULLS FIRST)") - .exclude("sorting on IntegerType with nullable=false, sortOrder=List('a ASC NULLS LAST)") - .exclude("sorting on IntegerType with nullable=false, sortOrder=List('a DESC NULLS LAST)") - .exclude("sorting on IntegerType with nullable=false, sortOrder=List('a DESC NULLS FIRST)") - .exclude("sorting on DecimalType(20,5) with nullable=true, sortOrder=List('a ASC NULLS FIRST)") - .exclude("sorting on DecimalType(20,5) with nullable=true, sortOrder=List('a ASC NULLS LAST)") - .exclude("sorting on DecimalType(20,5) with nullable=true, sortOrder=List('a DESC NULLS LAST)") - .exclude("sorting on DecimalType(20,5) with nullable=true, sortOrder=List('a DESC NULLS FIRST)") - .exclude("sorting on DecimalType(20,5) with nullable=false, sortOrder=List('a ASC NULLS FIRST)") - .exclude("sorting on DecimalType(20,5) with nullable=false, sortOrder=List('a ASC NULLS LAST)") - .exclude("sorting on DecimalType(20,5) with nullable=false, sortOrder=List('a DESC NULLS LAST)") - .exclude( - "sorting on DecimalType(20,5) with nullable=false, sortOrder=List('a DESC NULLS FIRST)") - .exclude("sorting on DoubleType with nullable=true, sortOrder=List('a ASC NULLS FIRST)") - .exclude("sorting on DoubleType with nullable=true, sortOrder=List('a ASC NULLS LAST)") - .exclude("sorting on DoubleType with nullable=true, sortOrder=List('a DESC NULLS LAST)") - .exclude("sorting on DoubleType with nullable=true, sortOrder=List('a DESC NULLS FIRST)") - .exclude("sorting on DoubleType with nullable=false, sortOrder=List('a ASC NULLS FIRST)") - .exclude("sorting on DoubleType with nullable=false, sortOrder=List('a ASC NULLS LAST)") - .exclude("sorting on DoubleType with nullable=false, sortOrder=List('a DESC NULLS LAST)") - .exclude("sorting on DoubleType with nullable=false, sortOrder=List('a DESC NULLS FIRST)") - .exclude("sorting on DateType with nullable=true, sortOrder=List('a ASC NULLS FIRST)") - .exclude("sorting on DateType with nullable=true, sortOrder=List('a ASC NULLS LAST)") - .exclude("sorting on DateType with nullable=true, sortOrder=List('a DESC NULLS LAST)") - .exclude("sorting on DateType with nullable=true, sortOrder=List('a DESC NULLS FIRST)") - .exclude("sorting on DateType with nullable=false, sortOrder=List('a ASC NULLS FIRST)") - .exclude("sorting on DateType with nullable=false, sortOrder=List('a ASC NULLS LAST)") - .exclude("sorting on DateType with nullable=false, sortOrder=List('a DESC NULLS LAST)") - .exclude("sorting on DateType with nullable=false, sortOrder=List('a DESC NULLS FIRST)") - .exclude("sorting on BooleanType with nullable=true, sortOrder=List('a ASC NULLS FIRST)") - .exclude("sorting on BooleanType with nullable=true, sortOrder=List('a ASC NULLS LAST)") - .exclude("sorting on BooleanType with nullable=true, sortOrder=List('a DESC NULLS LAST)") - .exclude("sorting on BooleanType with nullable=true, sortOrder=List('a DESC NULLS FIRST)") - .exclude("sorting on BooleanType with nullable=false, sortOrder=List('a ASC NULLS FIRST)") - .exclude("sorting on BooleanType with nullable=false, sortOrder=List('a ASC NULLS LAST)") - .exclude("sorting on BooleanType with nullable=false, sortOrder=List('a DESC NULLS LAST)") - .exclude("sorting on BooleanType with nullable=false, sortOrder=List('a DESC NULLS FIRST)") - .exclude("sorting on DecimalType(38,18) with nullable=true, sortOrder=List('a ASC NULLS FIRST)") - .exclude("sorting on DecimalType(38,18) with nullable=true, sortOrder=List('a ASC NULLS LAST)") - .exclude("sorting on DecimalType(38,18) with nullable=true, sortOrder=List('a DESC NULLS LAST)") - .exclude( - "sorting on DecimalType(38,18) with nullable=true, sortOrder=List('a DESC NULLS FIRST)") - .exclude( - "sorting on DecimalType(38,18) with nullable=false, sortOrder=List('a ASC NULLS FIRST)") - .exclude("sorting on DecimalType(38,18) with nullable=false, sortOrder=List('a ASC NULLS LAST)") - .exclude( - "sorting on DecimalType(38,18) with nullable=false, sortOrder=List('a DESC NULLS LAST)") - .exclude( - "sorting on DecimalType(38,18) with nullable=false, sortOrder=List('a DESC NULLS FIRST)") - .exclude("sorting on ByteType with nullable=true, sortOrder=List('a ASC NULLS FIRST)") - .exclude("sorting on ByteType with nullable=true, sortOrder=List('a ASC NULLS LAST)") - .exclude("sorting on ByteType with nullable=true, sortOrder=List('a DESC NULLS LAST)") - .exclude("sorting on ByteType with nullable=true, sortOrder=List('a DESC NULLS FIRST)") - .exclude("sorting on ByteType with nullable=false, sortOrder=List('a ASC NULLS FIRST)") - .exclude("sorting on ByteType with nullable=false, sortOrder=List('a ASC NULLS LAST)") - .exclude("sorting on ByteType with nullable=false, sortOrder=List('a DESC NULLS LAST)") - .exclude("sorting on ByteType with nullable=false, sortOrder=List('a DESC NULLS FIRST)") - .exclude("sorting on FloatType with nullable=true, sortOrder=List('a ASC NULLS FIRST)") - .exclude("sorting on FloatType with nullable=true, sortOrder=List('a ASC NULLS LAST)") - .exclude("sorting on FloatType with nullable=true, sortOrder=List('a DESC NULLS LAST)") - .exclude("sorting on FloatType with nullable=true, sortOrder=List('a DESC NULLS FIRST)") - .exclude("sorting on FloatType with nullable=false, sortOrder=List('a ASC NULLS FIRST)") - .exclude("sorting on FloatType with nullable=false, sortOrder=List('a ASC NULLS LAST)") - .exclude("sorting on FloatType with nullable=false, sortOrder=List('a DESC NULLS LAST)") - .exclude("sorting on FloatType with nullable=false, sortOrder=List('a DESC NULLS FIRST)") - .exclude("sorting on ShortType with nullable=true, sortOrder=List('a ASC NULLS FIRST)") - .exclude("sorting on ShortType with nullable=true, sortOrder=List('a ASC NULLS LAST)") - .exclude("sorting on ShortType with nullable=true, sortOrder=List('a DESC NULLS LAST)") - .exclude("sorting on ShortType with nullable=true, sortOrder=List('a DESC NULLS FIRST)") - .exclude("sorting on ShortType with nullable=false, sortOrder=List('a ASC NULLS FIRST)") - .exclude("sorting on ShortType with nullable=false, sortOrder=List('a ASC NULLS LAST)") - .exclude("sorting on ShortType with nullable=false, sortOrder=List('a DESC NULLS LAST)") - .exclude("sorting on ShortType with nullable=false, sortOrder=List('a DESC NULLS FIRST)") - .exclude("SPARK-40089: decimal values sort correctly") - .excludeByPrefix("sorting on YearMonthIntervalType(0,1) with") - enableSuite[GlutenTakeOrderedAndProjectSuite] - .exclude("TakeOrderedAndProject.doExecute without project") - .exclude("TakeOrderedAndProject.doExecute with project") - enableSuite[ClickHouseAdaptiveQueryExecSuite] - .exclude("Change merge join to broadcast join") - .exclude("Reuse the parallelism of coalesced shuffle in local shuffle read") - .exclude("Reuse the default parallelism in local shuffle read") - .exclude("Empty stage coalesced to 1-partition RDD") - .exclude("Scalar subquery") - .exclude("Scalar subquery in later stages") - .exclude("multiple joins") - .exclude("multiple joins with aggregate") - .exclude("multiple joins with aggregate 2") - .exclude("Exchange reuse") - .exclude("Exchange reuse with subqueries") - .exclude("Exchange reuse across subqueries") - .exclude("Subquery reuse") - .exclude("Broadcast exchange reuse across subqueries") - .exclude("Change merge join to broadcast join without local shuffle read") - .exclude( - "Avoid changing merge join to broadcast join if too many empty partitions on build plan") - .exclude("SPARK-32932: Do not use local shuffle read at final stage on write command") - .exclude( - "SPARK-30953: InsertAdaptiveSparkPlan should apply AQE on child plan of v2 write commands") - .exclude("SPARK-37753: Allow changing outer join to broadcast join even if too many empty partitions on broadcast side") - .exclude("SPARK-29544: adaptive skew join with different join types") - .exclude("SPARK-34682: AQEShuffleReadExec operating on canonicalized plan") - .exclude("SPARK-32717: AQEOptimizer should respect excludedRules configuration") - .exclude("metrics of the shuffle read") - .exclude("SPARK-31220, SPARK-32056: repartition by expression with AQE") - .exclude("SPARK-31220, SPARK-32056: repartition by range with AQE") - .exclude("SPARK-31220, SPARK-32056: repartition using sql and hint with AQE") - .exclude("SPARK-32753: Only copy tags to node with no tags") - .exclude("Logging plan changes for AQE") - .exclude("SPARK-33551: Do not use AQE shuffle read for repartition") - .exclude("SPARK-34091: Batch shuffle fetch in AQE partition coalescing") - .exclude("SPARK-34899: Use origin plan if we can not coalesce shuffle partition") - .exclude("SPARK-34980: Support coalesce partition through union") - .exclude("SPARK-35239: Coalesce shuffle partition should handle empty input RDD") - .exclude("SPARK-35264: Support AQE side broadcastJoin threshold") - .exclude("SPARK-35264: Support AQE side shuffled hash join formula") - .exclude("SPARK-35650: Coalesce number of partitions by AEQ") - .exclude("SPARK-35650: Use local shuffle read if can not coalesce number of partitions") - .exclude("SPARK-35725: Support optimize skewed partitions in RebalancePartitions") - .exclude("SPARK-35888: join with a 0-partition table") - .exclude("SPARK-33832: Support optimize skew join even if introduce extra shuffle") - .exclude("SPARK-35968: AQE coalescing should not produce too small partitions by default") - .exclude("SPARK-35794: Allow custom plugin for cost evaluator") - .exclude("SPARK-36020: Check logical link in remove redundant projects") - .exclude("SPARK-36032: Use inputPlan instead of currentPhysicalPlan to initialize logical link") - .exclude("SPARK-37063: OptimizeSkewInRebalancePartitions support optimize non-root node") - .exclude("SPARK-37357: Add small partition factor for rebalance partitions") - .exclude("SPARK-37742: AQE reads invalid InMemoryRelation stats and mistakenly plans BHJ") - .exclude("SPARK-37328: skew join with 3 tables") - .exclude("SPARK-39915: Dataset.repartition(N) may not create N partitions") - .exclude("Change broadcast join to merge join") - .exclude("Avoid plan change if cost is greater") - .exclude("SPARK-37652: optimize skewed join through union") - .exclude("SPARK-35455: Unify empty relation optimization between normal and AQE optimizer " + - "- single join") - .exclude("SPARK-35455: Unify empty relation optimization between normal and AQE optimizer " + - "- multi join") - // Gluten columnar operator will have different number of shuffle - .exclude("SPARK-29906: AQE should not introduce extra shuffle for outermost limit") - .excludeGlutenTest("Empty stage coalesced to 1-partition RDD") - .excludeGlutenTest( - "Avoid changing merge join to broadcast join if too many empty partitions on build plan") - .exclude("SPARK-30524: Do not optimize skew join if introduce additional shuffle") - .excludeGlutenTest("SPARK-33551: Do not use AQE shuffle read for repartition") - .excludeGlutenTest("SPARK-35264: Support AQE side broadcastJoin threshold") - .excludeGlutenTest("SPARK-35264: Support AQE side shuffled hash join formula") - .excludeGlutenTest("SPARK-35725: Support optimize skewed partitions in RebalancePartitions") - .excludeGlutenTest( - "SPARK-35968: AQE coalescing should not produce too small partitions by default") - .excludeGlutenTest( - "SPARK-37742: AQE reads invalid InMemoryRelation stats and mistakenly plans BHJ") - enableSuite[GlutenBucketingUtilsSuite] - enableSuite[GlutenCSVReadSchemaSuite] - enableSuite[GlutenDataSourceStrategySuite] - enableSuite[GlutenDataSourceSuite] - enableSuite[GlutenFileFormatWriterSuite].excludeByPrefix( - "empty file should be skipped while write to file") - enableSuite[GlutenFileIndexSuite] - enableSuite[GlutenFileMetadataStructSuite] - .exclude("metadata struct (json): file metadata in streaming") - .exclude("metadata struct (parquet): file metadata in streaming") - enableSuite[GlutenFileSourceStrategySuite] - .exclude("unpartitioned table, single partition") - .exclude("partitioned table - after scan filters") - .exclude("SPARK-32019: Add spark.sql.files.minPartitionNum config") - .exclude( - "SPARK-32352: Partially push down support data filter if it mixed in partition filters") - enableSuite[GlutenHadoopFileLinesReaderSuite] - enableSuite[GlutenHeaderCSVReadSchemaSuite] - .exclude("append column at the end") - .exclude("hide column at the end") - .exclude("change column type from byte to short/int/long") - .exclude("change column type from short to int/long") - .exclude("change column type from int to long") - .exclude("read byte, int, short, long together") - .exclude("change column type from float to double") - .exclude("read float and double together") - .exclude("change column type from float to decimal") - .exclude("change column type from double to decimal") - .exclude("read float, double, decimal together") - .exclude("read as string") - enableSuite[GlutenJsonReadSchemaSuite] - enableSuite[GlutenMergedOrcReadSchemaSuite] - enableSuite[GlutenMergedParquetReadSchemaSuite] - enableSuite[GlutenOrcCodecSuite] - enableSuite[GlutenOrcReadSchemaSuite] - enableSuite[GlutenOrcV1AggregatePushDownSuite].exclude( - "aggregate push down - different data types") - enableSuite[GlutenOrcV2AggregatePushDownSuite].exclude( - "aggregate push down - different data types") - enableSuite[GlutenParquetCodecSuite] - enableSuite[GlutenParquetReadSchemaSuite] - enableSuite[GlutenParquetV1AggregatePushDownSuite] - enableSuite[GlutenParquetV2AggregatePushDownSuite] - enableSuite[GlutenPathFilterStrategySuite] - enableSuite[GlutenPathFilterSuite] - enableSuite[GlutenPruneFileSourcePartitionsSuite] - enableSuite[GlutenVectorizedOrcReadSchemaSuite] - enableSuite[GlutenVectorizedParquetReadSchemaSuite] - enableSuite[GlutenBinaryFileFormatSuite] - .exclude("column pruning - non-readable file") - enableSuite[GlutenValidateRequirementsSuite] - enableSuite[GlutenJsonLegacyTimeParserSuite] - .exclude("Complex field and type inferring") - .exclude("Loading a JSON dataset primitivesAsString returns complex fields as strings") - .exclude("SPARK-4228 DataFrame to JSON") - .exclude("SPARK-18352: Handle multi-line corrupt documents (PERMISSIVE)") - .exclude("SPARK-37360: Write and infer TIMESTAMP_NTZ values with a non-default pattern") - .exclude("SPARK-37360: Timestamp type inference for a column with TIMESTAMP_NTZ values") - .exclude("SPARK-36830: Support reading and writing ANSI intervals") - enableSuite[GlutenJsonSuite] - .exclude("Complex field and type inferring") - .exclude("Loading a JSON dataset primitivesAsString returns complex fields as strings") - .exclude("SPARK-4228 DataFrame to JSON") - .exclude("SPARK-18352: Handle multi-line corrupt documents (PERMISSIVE)") - .exclude("SPARK-37360: Write and infer TIMESTAMP_NTZ values with a non-default pattern") - .exclude("SPARK-37360: Timestamp type inference for a column with TIMESTAMP_NTZ values") - .exclude("SPARK-36830: Support reading and writing ANSI intervals") - enableSuite[GlutenJsonV1Suite] - .exclude("Complex field and type inferring") - .exclude("Loading a JSON dataset primitivesAsString returns complex fields as strings") - .exclude("SPARK-4228 DataFrame to JSON") - .exclude("SPARK-18352: Handle multi-line corrupt documents (PERMISSIVE)") - .exclude("SPARK-37360: Write and infer TIMESTAMP_NTZ values with a non-default pattern") - .exclude("SPARK-37360: Timestamp type inference for a column with TIMESTAMP_NTZ values") - .exclude("SPARK-36830: Support reading and writing ANSI intervals") - enableSuite[GlutenJsonV2Suite] - .exclude("Complex field and type inferring") - .exclude("Loading a JSON dataset primitivesAsString returns complex fields as strings") - .exclude("SPARK-4228 DataFrame to JSON") - .exclude("SPARK-18352: Handle multi-line corrupt documents (PERMISSIVE)") - .exclude("SPARK-37360: Write and infer TIMESTAMP_NTZ values with a non-default pattern") - .exclude("SPARK-37360: Timestamp type inference for a column with TIMESTAMP_NTZ values") - .exclude("SPARK-36830: Support reading and writing ANSI intervals") - enableSuite[GlutenOrcColumnarBatchReaderSuite] - enableSuite[GlutenOrcFilterSuite].exclude("SPARK-32622: case sensitivity in predicate pushdown") - enableSuite[GlutenOrcPartitionDiscoverySuite] - enableSuite[GlutenOrcQuerySuite] - .exclude("Enabling/disabling ignoreCorruptFiles") - .exclude("SPARK-27160 Predicate pushdown correctness on DecimalType for ORC") - .exclude("SPARK-20728 Make ORCFileFormat configurable between sql/hive and sql/core") - .exclude("SPARK-36594: ORC vectorized reader should properly check maximal number of fields") - // DISABLED: GLUTEN-4893 Vanilla UT checks scan operator by exactly matching the class type - .exclude( - "SPARK-37728: Reading nested columns with ORC vectorized reader should not cause ArrayIndexOutOfBoundsException") - // DISABLED: GLUTEN-4893 Vanilla UT checks scan operator by exactly matching the class type - .exclude("SPARK-34862: Support ORC vectorized reader for nested column") - enableSuite[GlutenOrcSourceSuite] - .exclude("SPARK-24322 Fix incorrect workaround for bug in java.sql.Timestamp") - .exclude("SPARK-31238: compatibility with Spark 2.4 in reading dates") - .exclude("SPARK-31238, SPARK-31423: rebasing dates in write") - .exclude("SPARK-31284: compatibility with Spark 2.4 in reading timestamps") - .exclude("SPARK-31284, SPARK-31423: rebasing timestamps in write") - .exclude("SPARK-36663: OrcUtils.toCatalystSchema should correctly handle a column name which consists of only numbers") - .exclude("SPARK-37812: Reuse result row when deserializing a struct") - // DISABLED: GLUTEN-4893 Vanilla UT checks scan operator by exactly matching the class type - .exclude("SPARK-34862: Support ORC vectorized reader for nested column") - .excludeByPrefix( - "SPARK-36931: Support reading and writing ANSI intervals (spark.sql.orc.enableVectorizedReader=false,") - .excludeGlutenTest("SPARK-31238: compatibility with Spark 2.4 in reading dates") - .excludeGlutenTest("SPARK-31238, SPARK-31423: rebasing dates in write") - .excludeGlutenTest("SPARK-31284: compatibility with Spark 2.4 in reading timestamps") - .excludeGlutenTest("SPARK-31284, SPARK-31423: rebasing timestamps in write") - .excludeGlutenTest("SPARK-34862: Support ORC vectorized reader for nested column") - .excludeGlutenTest( - "SPARK-36931: Support reading and writing ANSI intervals (spark.sql.orc.enableVectorizedReader=false, spark.sql.orc.enableNestedColumnVectorizedReader=false)") - enableSuite[GlutenOrcV1FilterSuite].exclude("SPARK-32622: case sensitivity in predicate pushdown") - enableSuite[GlutenOrcV1PartitionDiscoverySuite] - enableSuite[GlutenOrcV1QuerySuite] - .exclude("Enabling/disabling ignoreCorruptFiles") - .exclude("SPARK-27160 Predicate pushdown correctness on DecimalType for ORC") - .exclude("SPARK-20728 Make ORCFileFormat configurable between sql/hive and sql/core") - .exclude("SPARK-36594: ORC vectorized reader should properly check maximal number of fields") - // DISABLED: GLUTEN-4893 Vanilla UT checks scan operator by exactly matching the class type - .exclude( - "SPARK-37728: Reading nested columns with ORC vectorized reader should not cause ArrayIndexOutOfBoundsException") - // DISABLED: GLUTEN-4893 Vanilla UT checks scan operator by exactly matching the class type - .exclude("SPARK-34862: Support ORC vectorized reader for nested column") - enableSuite[GlutenOrcV1SchemaPruningSuite] - .exclude( - "Spark vectorized reader - without partition data column - select a single complex field from a map entry and its parent map entry") - .exclude("Spark vectorized reader - with partition data column - select a single complex field from a map entry and its parent map entry") - .exclude("Non-vectorized reader - without partition data column - select a single complex field from a map entry and its parent map entry") - .exclude("Non-vectorized reader - with partition data column - select a single complex field from a map entry and its parent map entry") - .exclude("Case-insensitive parser - mixed-case schema - select with exact column names") - .exclude("Case-insensitive parser - mixed-case schema - select with lowercase column names") - .exclude( - "Case-insensitive parser - mixed-case schema - select with different-case column names") - .exclude( - "Case-insensitive parser - mixed-case schema - filter with different-case column names") - .exclude("Case-insensitive parser - mixed-case schema - subquery filter with different-case column names") - .exclude("SPARK-36352: Spark should check result plan's output schema name") - enableSuite[GlutenOrcV2QuerySuite] - .exclude("Enabling/disabling ignoreCorruptFiles") - .exclude("SPARK-27160 Predicate pushdown correctness on DecimalType for ORC") - .exclude("SPARK-20728 Make ORCFileFormat configurable between sql/hive and sql/core") - .exclude("SPARK-36594: ORC vectorized reader should properly check maximal number of fields") - // DISABLED: GLUTEN-4893 Vanilla UT checks scan operator by exactly matching the class type - .exclude( - "SPARK-37728: Reading nested columns with ORC vectorized reader should not cause ArrayIndexOutOfBoundsException") - // DISABLED: GLUTEN-4893 Vanilla UT checks scan operator by exactly matching the class type - .exclude("SPARK-34862: Support ORC vectorized reader for nested column") - enableSuite[GlutenOrcV2SchemaPruningSuite] - .exclude("Spark vectorized reader - without partition data column - select a single complex field from a map entry and its parent map entry") - .exclude("Spark vectorized reader - with partition data column - select a single complex field from a map entry and its parent map entry") - .exclude("Non-vectorized reader - without partition data column - select a single complex field from a map entry and its parent map entry") - .exclude("Non-vectorized reader - with partition data column - select a single complex field from a map entry and its parent map entry") - .exclude("Spark vectorized reader - without partition data column - select a single complex field and in where clause") - .exclude("Spark vectorized reader - with partition data column - select a single complex field and in where clause") - .exclude("Non-vectorized reader - without partition data column - select a single complex field and in where clause") - .exclude("Non-vectorized reader - with partition data column - select a single complex field and in where clause") - .exclude("Spark vectorized reader - without partition data column - select one complex field and having is null predicate on another complex field") - .exclude("Spark vectorized reader - with partition data column - select one complex field and having is null predicate on another complex field") - .exclude("Non-vectorized reader - without partition data column - select one complex field and having is null predicate on another complex field") - .exclude("Non-vectorized reader - with partition data column - select one complex field and having is null predicate on another complex field") - .exclude("Spark vectorized reader - without partition data column - select one deep nested complex field after repartition by expression") - .exclude("Spark vectorized reader - with partition data column - select one deep nested complex field after repartition by expression") - .exclude("Non-vectorized reader - without partition data column - select one deep nested complex field after repartition by expression") - .exclude("Non-vectorized reader - with partition data column - select one deep nested complex field after repartition by expression") - .exclude("Case-insensitive parser - mixed-case schema - select with exact column names") - .exclude("Case-insensitive parser - mixed-case schema - select with lowercase column names") - .exclude( - "Case-insensitive parser - mixed-case schema - select with different-case column names") - .exclude( - "Case-insensitive parser - mixed-case schema - filter with different-case column names") - .exclude("Case-insensitive parser - mixed-case schema - subquery filter with different-case column names") - .exclude("SPARK-36352: Spark should check result plan's output schema name") - .exclude("Spark vectorized reader - without partition data column - SPARK-34638: nested column prune on generator output - case-sensitivity") - .exclude("Spark vectorized reader - with partition data column - SPARK-34638: nested column prune on generator output - case-sensitivity") - .exclude("Non-vectorized reader - without partition data column - SPARK-34638: nested column prune on generator output - case-sensitivity") - .exclude("Non-vectorized reader - with partition data column - SPARK-34638: nested column prune on generator output - case-sensitivity") - .exclude("SPARK-37450: Prunes unnecessary fields from Explode for count aggregation") - enableSuite[GlutenParquetColumnIndexSuite] - .exclude("test reading unaligned pages - test all types") - .exclude("test reading unaligned pages - test all types (dict encode)") - enableSuite[GlutenParquetCompressionCodecPrecedenceSuite] - enableSuite[GlutenParquetDeltaByteArrayEncodingSuite] - enableSuite[GlutenParquetDeltaEncodingInteger] - enableSuite[GlutenParquetDeltaEncodingLong] - enableSuite[GlutenParquetDeltaLengthByteArrayEncodingSuite] - enableSuite[GlutenParquetEncodingSuite].exclude("All Types Dictionary").exclude("All Types Null") - enableSuite[GlutenParquetFieldIdIOSuite] - enableSuite[GlutenParquetFileFormatV1Suite] - .exclude( - "SPARK-36825, SPARK-36854: year-month/day-time intervals written and read as INT32/INT64") - enableSuite[GlutenParquetFileFormatV2Suite] - .exclude( - "SPARK-36825, SPARK-36854: year-month/day-time intervals written and read as INT32/INT64") - enableSuite[GlutenParquetIOSuite] - .exclude("Standard mode - nested map with struct as key type") - .exclude("Legacy mode - nested map with struct as key type") - .exclude("vectorized reader: missing all struct fields") - .exclude("SPARK-35640: read binary as timestamp should throw schema incompatible error") - .exclude("SPARK-35640: int as long should throw schema incompatible error") - .exclude("SPARK-36726: test incorrect Parquet row group file offset") - // TODO: after rebase-25.12, failed, fix later - .exclude("SPARK-34167: read LongDecimals with precision < 10, VectorizedReader true") - .exclude("SPARK-34167: read LongDecimals with precision < 10, VectorizedReader false") - enableSuite[GlutenParquetInteroperabilitySuite].exclude("parquet timestamp conversion") - enableSuite[GlutenParquetProtobufCompatibilitySuite].exclude("struct with unannotated array") - enableSuite[GlutenParquetRebaseDatetimeV1Suite] - .exclude( - "SPARK-31159, SPARK-37705: compatibility with Spark 2.4/3.2 in reading dates/timestamps") - .exclude("SPARK-31159, SPARK-37705: rebasing timestamps in write") - .exclude("SPARK-31159: rebasing dates in write") - .exclude("SPARK-35427: datetime rebasing in the EXCEPTION mode") - .excludeGlutenTest("SPARK-31159: rebasing dates in write") - enableSuite[GlutenParquetRebaseDatetimeV2Suite] - .exclude( - "SPARK-31159, SPARK-37705: compatibility with Spark 2.4/3.2 in reading dates/timestamps") - .exclude("SPARK-31159, SPARK-37705: rebasing timestamps in write") - .exclude("SPARK-31159: rebasing dates in write") - .exclude("SPARK-35427: datetime rebasing in the EXCEPTION mode") - enableSuite[GlutenParquetSchemaInferenceSuite] - enableSuite[GlutenParquetSchemaSuite] - .exclude("schema mismatch failure error message for parquet reader") - .exclude("schema mismatch failure error message for parquet vectorized reader") - enableSuite[GlutenParquetThriftCompatibilitySuite] - .exclude("Read Parquet file generated by parquet-thrift") - .exclude("SPARK-10136 list of primitive list") - enableSuite[GlutenParquetV1FilterSuite] - .exclude("filter pushdown - date") - .exclude("filter pushdown - timestamp") - .exclude("Filters should be pushed down for vectorized Parquet reader at row group level") - .exclude("SPARK-31026: Parquet predicate pushdown for fields having dots in the names") - .exclude("Filters should be pushed down for Parquet readers at row group level") - .exclude("filter pushdown - StringStartsWith") - .exclude("SPARK-17091: Convert IN predicate to Parquet filter push-down") - .exclude("SPARK-25207: exception when duplicate fields in case-insensitive mode") - .exclude("Support Parquet column index") - .exclude("SPARK-34562: Bloom filter push down") - .exclude("SPARK-38825: in and notIn filters") - .exclude("SPARK-36866: filter pushdown - year-month interval") - .excludeGlutenTest("SPARK-25207: exception when duplicate fields in case-insensitive mode") - enableSuite("org.apache.gluten.execution.parquet.GlutenParquetV1FilterSuite2") - .exclude("filter pushdown - date") - .exclude("filter pushdown - timestamp") - .exclude("Filters should be pushed down for vectorized Parquet reader at row group level") - .exclude("SPARK-31026: Parquet predicate pushdown for fields having dots in the names") - .exclude("Filters should be pushed down for Parquet readers at row group level") - .exclude("filter pushdown - StringStartsWith") - .exclude("SPARK-17091: Convert IN predicate to Parquet filter push-down") - .exclude("SPARK-25207: exception when duplicate fields in case-insensitive mode") - .exclude("Support Parquet column index") - .exclude("SPARK-34562: Bloom filter push down") - .exclude("SPARK-38825: in and notIn filters") - .exclude("SPARK-36866: filter pushdown - year-month interval") - .excludeGlutenTest("SPARK-25207: exception when duplicate fields in case-insensitive mode") - enableSuite[GlutenParquetV1PartitionDiscoverySuite] - .exclude("SPARK-7847: Dynamic partition directory path escaping and unescaping") - .exclude("Various partition value types") - .exclude("Various inferred partition value types") - .exclude( - "SPARK-22109: Resolve type conflicts between strings and timestamps in partition column") - .exclude("Resolve type conflicts - decimals, dates and timestamps in partition column") - enableSuite[GlutenParquetV1QuerySuite] - .exclude("Enabling/disabling ignoreCorruptFiles") - .exclude( - "SPARK-26677: negated null-safe equality comparison should not filter matched row groups") - .exclude("SPARK-34212 Parquet should read decimals correctly") - enableSuite[GlutenParquetV1SchemaPruningSuite] - .exclude( - "Spark vectorized reader - without partition data column - select only top-level fields") - .exclude("Spark vectorized reader - with partition data column - select only top-level fields") - .exclude("Non-vectorized reader - without partition data column - select only top-level fields") - .exclude("Non-vectorized reader - with partition data column - select only top-level fields") - .exclude("Spark vectorized reader - without partition data column - select a single complex field with disabled nested schema pruning") - .exclude("Spark vectorized reader - with partition data column - select a single complex field with disabled nested schema pruning") - .exclude("Non-vectorized reader - without partition data column - select a single complex field with disabled nested schema pruning") - .exclude("Non-vectorized reader - with partition data column - select a single complex field with disabled nested schema pruning") - .exclude( - "Spark vectorized reader - without partition data column - select only input_file_name()") - .exclude("Spark vectorized reader - with partition data column - select only input_file_name()") - .exclude( - "Non-vectorized reader - without partition data column - select only input_file_name()") - .exclude("Non-vectorized reader - with partition data column - select only input_file_name()") - .exclude("Spark vectorized reader - without partition data column - select only expressions without references") - .exclude("Spark vectorized reader - with partition data column - select only expressions without references") - .exclude("Non-vectorized reader - without partition data column - select only expressions without references") - .exclude("Non-vectorized reader - with partition data column - select only expressions without references") - .exclude( - "Spark vectorized reader - without partition data column - select a single complex field") - .exclude("Spark vectorized reader - with partition data column - select a single complex field") - .exclude( - "Non-vectorized reader - without partition data column - select a single complex field") - .exclude("Non-vectorized reader - with partition data column - select a single complex field") - .exclude("Spark vectorized reader - without partition data column - select a single complex field and its parent struct") - .exclude("Spark vectorized reader - with partition data column - select a single complex field and its parent struct") - .exclude("Non-vectorized reader - without partition data column - select a single complex field and its parent struct") - .exclude("Non-vectorized reader - with partition data column - select a single complex field and its parent struct") - .exclude("Spark vectorized reader - without partition data column - select a single complex field array and its parent struct array") - .exclude("Spark vectorized reader - with partition data column - select a single complex field array and its parent struct array") - .exclude("Non-vectorized reader - without partition data column - select a single complex field array and its parent struct array") - .exclude("Non-vectorized reader - with partition data column - select a single complex field array and its parent struct array") - .exclude("Spark vectorized reader - without partition data column - select a single complex field from a map entry and its parent map entry") - .exclude("Spark vectorized reader - with partition data column - select a single complex field from a map entry and its parent map entry") - .exclude("Non-vectorized reader - without partition data column - select a single complex field from a map entry and its parent map entry") - .exclude("Non-vectorized reader - with partition data column - select a single complex field from a map entry and its parent map entry") - .exclude("Spark vectorized reader - without partition data column - select a single complex field and the partition column") - .exclude("Spark vectorized reader - with partition data column - select a single complex field and the partition column") - .exclude("Non-vectorized reader - without partition data column - select a single complex field and the partition column") - .exclude("Non-vectorized reader - with partition data column - select a single complex field and the partition column") - .exclude("Spark vectorized reader - without partition data column - partial schema intersection - select missing subfield") - .exclude("Spark vectorized reader - with partition data column - partial schema intersection - select missing subfield") - .exclude("Non-vectorized reader - without partition data column - partial schema intersection - select missing subfield") - .exclude("Non-vectorized reader - with partition data column - partial schema intersection - select missing subfield") - .exclude( - "Spark vectorized reader - without partition data column - no unnecessary schema pruning") - .exclude("Spark vectorized reader - with partition data column - no unnecessary schema pruning") - .exclude( - "Non-vectorized reader - without partition data column - no unnecessary schema pruning") - .exclude("Non-vectorized reader - with partition data column - no unnecessary schema pruning") - .exclude("Spark vectorized reader - without partition data column - empty schema intersection") - .exclude("Spark vectorized reader - with partition data column - empty schema intersection") - .exclude("Non-vectorized reader - without partition data column - empty schema intersection") - .exclude("Non-vectorized reader - with partition data column - empty schema intersection") - .exclude("Spark vectorized reader - without partition data column - select a single complex field and in where clause") - .exclude("Spark vectorized reader - with partition data column - select a single complex field and in where clause") - .exclude("Non-vectorized reader - without partition data column - select a single complex field and in where clause") - .exclude("Non-vectorized reader - with partition data column - select a single complex field and in where clause") - .exclude("Spark vectorized reader - without partition data column - select nullable complex field and having is not null predicate") - .exclude("Spark vectorized reader - with partition data column - select nullable complex field and having is not null predicate") - .exclude("Non-vectorized reader - without partition data column - select nullable complex field and having is not null predicate") - .exclude("Non-vectorized reader - with partition data column - select nullable complex field and having is not null predicate") - .exclude("Spark vectorized reader - without partition data column - select a single complex field and is null expression in project") - .exclude("Spark vectorized reader - with partition data column - select a single complex field and is null expression in project") - .exclude("Non-vectorized reader - without partition data column - select a single complex field and is null expression in project") - .exclude("Non-vectorized reader - with partition data column - select a single complex field and is null expression in project") - .exclude("Spark vectorized reader - without partition data column - select a single complex field from a map entry and in clause") - .exclude("Spark vectorized reader - with partition data column - select a single complex field from a map entry and in clause") - .exclude("Non-vectorized reader - without partition data column - select a single complex field from a map entry and in clause") - .exclude("Non-vectorized reader - with partition data column - select a single complex field from a map entry and in clause") - .exclude("Spark vectorized reader - without partition data column - select one complex field and having is null predicate on another complex field") - .exclude("Spark vectorized reader - with partition data column - select one complex field and having is null predicate on another complex field") - .exclude("Non-vectorized reader - without partition data column - select one complex field and having is null predicate on another complex field") - .exclude("Non-vectorized reader - with partition data column - select one complex field and having is null predicate on another complex field") - .exclude("Spark vectorized reader - without partition data column - select one deep nested complex field and having is null predicate on another deep nested complex field") - .exclude("Spark vectorized reader - with partition data column - select one deep nested complex field and having is null predicate on another deep nested complex field") - .exclude("Non-vectorized reader - without partition data column - select one deep nested complex field and having is null predicate on another deep nested complex field") - .exclude("Non-vectorized reader - with partition data column - select one deep nested complex field and having is null predicate on another deep nested complex field") - .exclude("Spark vectorized reader - without partition data column - select nested field from a complex map value using map_values") - .exclude("Spark vectorized reader - with partition data column - select nested field from a complex map value using map_values") - .exclude("Non-vectorized reader - without partition data column - select nested field from a complex map value using map_values") - .exclude("Non-vectorized reader - with partition data column - select nested field from a complex map value using map_values") - .exclude("Spark vectorized reader - without partition data column - select explode of nested field of array of struct") - .exclude("Spark vectorized reader - with partition data column - select explode of nested field of array of struct") - .exclude("Non-vectorized reader - without partition data column - select explode of nested field of array of struct") - .exclude("Non-vectorized reader - with partition data column - select explode of nested field of array of struct") - .exclude("Spark vectorized reader - without partition data column - SPARK-34638: nested column prune on generator output") - .exclude("Spark vectorized reader - with partition data column - SPARK-34638: nested column prune on generator output") - .exclude("Non-vectorized reader - without partition data column - SPARK-34638: nested column prune on generator output") - .exclude("Non-vectorized reader - with partition data column - SPARK-34638: nested column prune on generator output") - .exclude("Spark vectorized reader - without partition data column - select one deep nested complex field after repartition") - .exclude("Spark vectorized reader - with partition data column - select one deep nested complex field after repartition") - .exclude("Non-vectorized reader - without partition data column - select one deep nested complex field after repartition") - .exclude("Non-vectorized reader - with partition data column - select one deep nested complex field after repartition") - .exclude("Spark vectorized reader - without partition data column - select one deep nested complex field after repartition by expression") - .exclude("Spark vectorized reader - with partition data column - select one deep nested complex field after repartition by expression") - .exclude("Non-vectorized reader - without partition data column - select one deep nested complex field after repartition by expression") - .exclude("Non-vectorized reader - with partition data column - select one deep nested complex field after repartition by expression") - .exclude("Spark vectorized reader - without partition data column - select one deep nested complex field after join") - .exclude("Spark vectorized reader - with partition data column - select one deep nested complex field after join") - .exclude("Non-vectorized reader - without partition data column - select one deep nested complex field after join") - .exclude("Non-vectorized reader - with partition data column - select one deep nested complex field after join") - .exclude("Spark vectorized reader - without partition data column - select one deep nested complex field after outer join") - .exclude("Spark vectorized reader - with partition data column - select one deep nested complex field after outer join") - .exclude("Non-vectorized reader - without partition data column - select one deep nested complex field after outer join") - .exclude("Non-vectorized reader - with partition data column - select one deep nested complex field after outer join") - .exclude("Spark vectorized reader - without partition data column - select nested field in aggregation function of Aggregate") - .exclude("Spark vectorized reader - with partition data column - select nested field in aggregation function of Aggregate") - .exclude("Non-vectorized reader - without partition data column - select nested field in aggregation function of Aggregate") - .exclude("Non-vectorized reader - with partition data column - select nested field in aggregation function of Aggregate") - .exclude("Spark vectorized reader - without partition data column - select nested field in window function") - .exclude("Spark vectorized reader - with partition data column - select nested field in window function") - .exclude("Non-vectorized reader - without partition data column - select nested field in window function") - .exclude( - "Non-vectorized reader - with partition data column - select nested field in window function") - .exclude("Spark vectorized reader - without partition data column - select nested field in window function and then order by") - .exclude("Spark vectorized reader - with partition data column - select nested field in window function and then order by") - .exclude("Non-vectorized reader - without partition data column - select nested field in window function and then order by") - .exclude("Non-vectorized reader - with partition data column - select nested field in window function and then order by") - .exclude( - "Spark vectorized reader - without partition data column - select nested field in Sort") - .exclude("Spark vectorized reader - with partition data column - select nested field in Sort") - .exclude("Non-vectorized reader - without partition data column - select nested field in Sort") - .exclude("Non-vectorized reader - with partition data column - select nested field in Sort") - .exclude( - "Spark vectorized reader - without partition data column - select nested field in Expand") - .exclude("Spark vectorized reader - with partition data column - select nested field in Expand") - .exclude( - "Non-vectorized reader - without partition data column - select nested field in Expand") - .exclude("Non-vectorized reader - with partition data column - select nested field in Expand") - .exclude("Spark vectorized reader - without partition data column - SPARK-32163: nested pruning should work even with cosmetic variations") - .exclude("Spark vectorized reader - with partition data column - SPARK-32163: nested pruning should work even with cosmetic variations") - .exclude("Non-vectorized reader - without partition data column - SPARK-32163: nested pruning should work even with cosmetic variations") - .exclude("Non-vectorized reader - with partition data column - SPARK-32163: nested pruning should work even with cosmetic variations") - .exclude("Spark vectorized reader - without partition data column - SPARK-38918: nested schema pruning with correlated subqueries") - .exclude("Spark vectorized reader - with partition data column - SPARK-38918: nested schema pruning with correlated subqueries") - .exclude("Non-vectorized reader - without partition data column - SPARK-38918: nested schema pruning with correlated subqueries") - .exclude("Non-vectorized reader - with partition data column - SPARK-38918: nested schema pruning with correlated subqueries") - .exclude("Case-insensitive parser - mixed-case schema - select with exact column names") - .exclude("Case-insensitive parser - mixed-case schema - select with lowercase column names") - .exclude( - "Case-insensitive parser - mixed-case schema - select with different-case column names") - .exclude( - "Case-insensitive parser - mixed-case schema - filter with different-case column names") - .exclude("Case-insensitive parser - mixed-case schema - subquery filter with different-case column names") - .exclude("Spark vectorized reader - without partition data column - SPARK-34963: extract case-insensitive struct field from array") - .exclude("Spark vectorized reader - with partition data column - SPARK-34963: extract case-insensitive struct field from array") - .exclude("Non-vectorized reader - without partition data column - SPARK-34963: extract case-insensitive struct field from array") - .exclude("Non-vectorized reader - with partition data column - SPARK-34963: extract case-insensitive struct field from array") - .exclude("Spark vectorized reader - without partition data column - SPARK-34963: extract case-insensitive struct field from struct") - .exclude("Spark vectorized reader - with partition data column - SPARK-34963: extract case-insensitive struct field from struct") - .exclude("Non-vectorized reader - without partition data column - SPARK-34963: extract case-insensitive struct field from struct") - .exclude("Non-vectorized reader - with partition data column - SPARK-34963: extract case-insensitive struct field from struct") - .exclude("SPARK-36352: Spark should check result plan's output schema name") - .exclude("Spark vectorized reader - without partition data column - SPARK-38977: schema pruning with correlated EXISTS subquery") - .exclude("Spark vectorized reader - with partition data column - SPARK-38977: schema pruning with correlated EXISTS subquery") - .exclude("Non-vectorized reader - without partition data column - SPARK-38977: schema pruning with correlated EXISTS subquery") - .exclude("Non-vectorized reader - with partition data column - SPARK-38977: schema pruning with correlated EXISTS subquery") - .exclude("Spark vectorized reader - without partition data column - SPARK-38977: schema pruning with correlated NOT EXISTS subquery") - .exclude("Spark vectorized reader - with partition data column - SPARK-38977: schema pruning with correlated NOT EXISTS subquery") - .exclude("Non-vectorized reader - without partition data column - SPARK-38977: schema pruning with correlated NOT EXISTS subquery") - .exclude("Non-vectorized reader - with partition data column - SPARK-38977: schema pruning with correlated NOT EXISTS subquery") - .exclude("Spark vectorized reader - without partition data column - SPARK-38977: schema pruning with correlated IN subquery") - .exclude("Spark vectorized reader - with partition data column - SPARK-38977: schema pruning with correlated IN subquery") - .exclude("Non-vectorized reader - without partition data column - SPARK-38977: schema pruning with correlated IN subquery") - .exclude("Non-vectorized reader - with partition data column - SPARK-38977: schema pruning with correlated IN subquery") - .exclude("Spark vectorized reader - without partition data column - SPARK-38977: schema pruning with correlated NOT IN subquery") - .exclude("Spark vectorized reader - with partition data column - SPARK-38977: schema pruning with correlated NOT IN subquery") - .exclude("Non-vectorized reader - without partition data column - SPARK-38977: schema pruning with correlated NOT IN subquery") - .exclude("Non-vectorized reader - with partition data column - SPARK-38977: schema pruning with correlated NOT IN subquery") - .exclude("Spark vectorized reader - without partition data column - SPARK-34638: nested column prune on generator output - case-sensitivity") - .exclude("Spark vectorized reader - with partition data column - SPARK-34638: nested column prune on generator output - case-sensitivity") - .exclude("Non-vectorized reader - without partition data column - SPARK-34638: nested column prune on generator output - case-sensitivity") - .exclude("Non-vectorized reader - with partition data column - SPARK-34638: nested column prune on generator output - case-sensitivity") - .exclude("SPARK-37450: Prunes unnecessary fields from Explode for count aggregation") - enableSuite[GlutenParquetV2FilterSuite] - .exclude("filter pushdown - date") - .exclude("filter pushdown - timestamp") - .exclude("Filters should be pushed down for vectorized Parquet reader at row group level") - .exclude("SPARK-31026: Parquet predicate pushdown for fields having dots in the names") - .exclude("Filters should be pushed down for Parquet readers at row group level") - .exclude("filter pushdown - StringStartsWith") - .exclude("SPARK-17091: Convert IN predicate to Parquet filter push-down") - .exclude("SPARK-25207: exception when duplicate fields in case-insensitive mode") - .exclude("Support Parquet column index") - .exclude("SPARK-34562: Bloom filter push down") - .exclude("SPARK-38825: in and notIn filters") - .exclude("SPARK-36866: filter pushdown - year-month interval") - .excludeGlutenTest("SPARK-25207: exception when duplicate fields in case-insensitive mode") - .excludeGlutenTest("filter pushdown - date") - enableSuite[GlutenParquetV2PartitionDiscoverySuite] - .exclude("SPARK-7847: Dynamic partition directory path escaping and unescaping") - .exclude("Various partition value types") - .exclude("Various inferred partition value types") - .exclude( - "SPARK-22109: Resolve type conflicts between strings and timestamps in partition column") - .exclude("Resolve type conflicts - decimals, dates and timestamps in partition column") - enableSuite[GlutenParquetV2QuerySuite] - .exclude("Enabling/disabling ignoreCorruptFiles") - .exclude( - "SPARK-26677: negated null-safe equality comparison should not filter matched row groups") - .exclude("SPARK-34212 Parquet should read decimals correctly") - enableSuite[GlutenParquetV2SchemaPruningSuite] - .exclude("Spark vectorized reader - without partition data column - select a single complex field from a map entry and its parent map entry") - .exclude("Spark vectorized reader - with partition data column - select a single complex field from a map entry and its parent map entry") - .exclude("Non-vectorized reader - without partition data column - select a single complex field from a map entry and its parent map entry") - .exclude("Non-vectorized reader - with partition data column - select a single complex field from a map entry and its parent map entry") - .exclude("Spark vectorized reader - without partition data column - select a single complex field and in where clause") - .exclude("Spark vectorized reader - with partition data column - select a single complex field and in where clause") - .exclude("Non-vectorized reader - without partition data column - select a single complex field and in where clause") - .exclude("Non-vectorized reader - with partition data column - select a single complex field and in where clause") - .exclude("Spark vectorized reader - without partition data column - select one complex field and having is null predicate on another complex field") - .exclude("Spark vectorized reader - with partition data column - select one complex field and having is null predicate on another complex field") - .exclude("Non-vectorized reader - without partition data column - select one complex field and having is null predicate on another complex field") - .exclude("Non-vectorized reader - with partition data column - select one complex field and having is null predicate on another complex field") - .exclude("Spark vectorized reader - without partition data column - select one deep nested complex field after repartition by expression") - .exclude("Spark vectorized reader - with partition data column - select one deep nested complex field after repartition by expression") - .exclude("Non-vectorized reader - without partition data column - select one deep nested complex field after repartition by expression") - .exclude("Non-vectorized reader - with partition data column - select one deep nested complex field after repartition by expression") - .exclude("Case-insensitive parser - mixed-case schema - select with exact column names") - .exclude("Case-insensitive parser - mixed-case schema - select with lowercase column names") - .exclude( - "Case-insensitive parser - mixed-case schema - select with different-case column names") - .exclude( - "Case-insensitive parser - mixed-case schema - filter with different-case column names") - .exclude("Case-insensitive parser - mixed-case schema - subquery filter with different-case column names") - .exclude("SPARK-36352: Spark should check result plan's output schema name") - .exclude("Spark vectorized reader - without partition data column - SPARK-34638: nested column prune on generator output - case-sensitivity") - .exclude("Spark vectorized reader - with partition data column - SPARK-34638: nested column prune on generator output - case-sensitivity") - .exclude("Non-vectorized reader - without partition data column - SPARK-34638: nested column prune on generator output - case-sensitivity") - .exclude("Non-vectorized reader - with partition data column - SPARK-34638: nested column prune on generator output - case-sensitivity") - .exclude("SPARK-37450: Prunes unnecessary fields from Explode for count aggregation") - enableSuite[GlutenParquetVectorizedSuite] - enableSuite[GlutenTextV1Suite] - enableSuite[GlutenTextV2Suite] - enableSuite[GlutenDataSourceV2StrategySuite] - enableSuite[GlutenFileTableSuite] - enableSuite[GlutenV2PredicateSuite] - enableSuite[GlutenEnsureRequirementsSuite] - .exclude("reorder should handle PartitioningCollection") - .exclude("SPARK-35675: EnsureRequirements remove shuffle should respect PartitioningCollection") - enableSuite[GlutenBroadcastJoinSuite] - .exclude("unsafe broadcast hash join updates peak execution memory") - .exclude("unsafe broadcast hash outer join updates peak execution memory") - .exclude("unsafe broadcast left semi join updates peak execution memory") - .exclude("SPARK-23192: broadcast hint should be retained after using the cached data") - .exclude("SPARK-23214: cached data should not carry extra hint info") - .exclude("broadcast hint in SQL") - .exclude("Broadcast timeout") - .exclude("broadcast join where streamed side's output partitioning is HashPartitioning") - .exclude("broadcast join where streamed side's output partitioning is PartitioningCollection") - .exclude("BroadcastHashJoinExec output partitioning size should be limited with a config") - .exclude("SPARK-37742: join planning shouldn't read invalid InMemoryRelation stats") - enableSuite[GlutenExistenceJoinSuite] - .exclude("test single condition (equal) for left semi join using ShuffledHashJoin (whole-stage-codegen off)") - .exclude("test single condition (equal) for left semi join using ShuffledHashJoin (whole-stage-codegen on)") - .exclude("test single condition (equal) for left semi join using SortMergeJoin (whole-stage-codegen off)") - .exclude("test single condition (equal) for left semi join using SortMergeJoin (whole-stage-codegen on)") - .exclude("test single unique condition (equal) for left semi join using ShuffledHashJoin (whole-stage-codegen off)") - .exclude("test single unique condition (equal) for left semi join using ShuffledHashJoin (whole-stage-codegen on)") - .exclude("test single unique condition (equal) for left semi join using BroadcastHashJoin (whole-stage-codegen off)") - .exclude("test single unique condition (equal) for left semi join using BroadcastHashJoin (whole-stage-codegen on)") - .exclude("test single unique condition (equal) for left semi join using SortMergeJoin (whole-stage-codegen off)") - .exclude("test single unique condition (equal) for left semi join using SortMergeJoin (whole-stage-codegen on)") - .exclude("test single unique condition (equal) for left semi join using BroadcastNestedLoopJoin build left") - .exclude("test single unique condition (equal) for left semi join using BroadcastNestedLoopJoin build right (whole-stage-codegen off)") - .exclude("test single unique condition (equal) for left semi join using BroadcastNestedLoopJoin build right (whole-stage-codegen on)") - .exclude("test composed condition (equal & non-equal) for left semi join using ShuffledHashJoin (whole-stage-codegen off)") - .exclude("test composed condition (equal & non-equal) for left semi join using ShuffledHashJoin (whole-stage-codegen on)") - .exclude("test composed condition (equal & non-equal) for left semi join using SortMergeJoin (whole-stage-codegen off)") - .exclude("test composed condition (equal & non-equal) for left semi join using SortMergeJoin (whole-stage-codegen on)") - .exclude("test single condition (equal) for left anti join using ShuffledHashJoin (whole-stage-codegen off)") - .exclude("test single condition (equal) for left anti join using ShuffledHashJoin (whole-stage-codegen on)") - .exclude("test single condition (equal) for left anti join using SortMergeJoin (whole-stage-codegen off)") - .exclude("test single condition (equal) for left anti join using SortMergeJoin (whole-stage-codegen on)") - .exclude("test single unique condition (equal) for left anti join using ShuffledHashJoin (whole-stage-codegen off)") - .exclude("test single unique condition (equal) for left anti join using ShuffledHashJoin (whole-stage-codegen on)") - .exclude("test single unique condition (equal) for left anti join using BroadcastHashJoin (whole-stage-codegen off)") - .exclude("test single unique condition (equal) for left anti join using BroadcastHashJoin (whole-stage-codegen on)") - .exclude("test single unique condition (equal) for left anti join using SortMergeJoin (whole-stage-codegen off)") - .exclude("test single unique condition (equal) for left anti join using SortMergeJoin (whole-stage-codegen on)") - .exclude("test single unique condition (equal) for left anti join using BroadcastNestedLoopJoin build left") - .exclude("test single unique condition (equal) for left anti join using BroadcastNestedLoopJoin build right (whole-stage-codegen off)") - .exclude("test single unique condition (equal) for left anti join using BroadcastNestedLoopJoin build right (whole-stage-codegen on)") - .exclude("test composed condition (equal & non-equal) test for left anti join using ShuffledHashJoin (whole-stage-codegen off)") - .exclude("test composed condition (equal & non-equal) test for left anti join using ShuffledHashJoin (whole-stage-codegen on)") - .exclude("test composed condition (equal & non-equal) test for left anti join using SortMergeJoin (whole-stage-codegen off)") - .exclude("test composed condition (equal & non-equal) test for left anti join using SortMergeJoin (whole-stage-codegen on)") - .exclude("test composed unique condition (both non-equal) for left anti join using ShuffledHashJoin (whole-stage-codegen off)") - .exclude("test composed unique condition (both non-equal) for left anti join using ShuffledHashJoin (whole-stage-codegen on)") - .exclude("test composed unique condition (both non-equal) for left anti join using SortMergeJoin (whole-stage-codegen off)") - .exclude("test composed unique condition (both non-equal) for left anti join using SortMergeJoin (whole-stage-codegen on)") - enableSuite[GlutenInnerJoinSuite] - .exclude( - "inner join, one match per row using ShuffledHashJoin (build=left) (whole-stage-codegen off)") - .exclude( - "inner join, one match per row using ShuffledHashJoin (build=left) (whole-stage-codegen on)") - .exclude( - "inner join, one match per row using ShuffledHashJoin (build=right) (whole-stage-codegen off)") - .exclude( - "inner join, one match per row using ShuffledHashJoin (build=right) (whole-stage-codegen on)") - .exclude("inner join, one match per row using SortMergeJoin (whole-stage-codegen off)") - .exclude("inner join, one match per row using SortMergeJoin (whole-stage-codegen on)") - .exclude( - "inner join, multiple matches using ShuffledHashJoin (build=left) (whole-stage-codegen off)") - .exclude( - "inner join, multiple matches using ShuffledHashJoin (build=left) (whole-stage-codegen on)") - .exclude( - "inner join, multiple matches using ShuffledHashJoin (build=right) (whole-stage-codegen off)") - .exclude( - "inner join, multiple matches using ShuffledHashJoin (build=right) (whole-stage-codegen on)") - .exclude("inner join, multiple matches using SortMergeJoin (whole-stage-codegen off)") - .exclude("inner join, multiple matches using SortMergeJoin (whole-stage-codegen on)") - .exclude("inner join, no matches using ShuffledHashJoin (build=left) (whole-stage-codegen off)") - .exclude("inner join, no matches using ShuffledHashJoin (build=left) (whole-stage-codegen on)") - .exclude( - "inner join, no matches using ShuffledHashJoin (build=right) (whole-stage-codegen off)") - .exclude("inner join, no matches using ShuffledHashJoin (build=right) (whole-stage-codegen on)") - .exclude("inner join, no matches using SortMergeJoin (whole-stage-codegen off)") - .exclude("inner join, no matches using SortMergeJoin (whole-stage-codegen on)") - .exclude("inner join, null safe using ShuffledHashJoin (build=left) (whole-stage-codegen off)") - .exclude("inner join, null safe using ShuffledHashJoin (build=left) (whole-stage-codegen on)") - .exclude("inner join, null safe using ShuffledHashJoin (build=right) (whole-stage-codegen off)") - .exclude("inner join, null safe using ShuffledHashJoin (build=right) (whole-stage-codegen on)") - .exclude("inner join, null safe using SortMergeJoin (whole-stage-codegen off)") - .exclude("inner join, null safe using SortMergeJoin (whole-stage-codegen on)") - .exclude("SPARK-15822 - test structs as keys using BroadcastHashJoin (build=left) (whole-stage-codegen off)") - .exclude("SPARK-15822 - test structs as keys using BroadcastHashJoin (build=left) (whole-stage-codegen on)") - .exclude("SPARK-15822 - test structs as keys using BroadcastHashJoin (build=right) (whole-stage-codegen off)") - .exclude("SPARK-15822 - test structs as keys using BroadcastHashJoin (build=right) (whole-stage-codegen on)") - .exclude("SPARK-15822 - test structs as keys using ShuffledHashJoin (build=left) (whole-stage-codegen off)") - .exclude("SPARK-15822 - test structs as keys using ShuffledHashJoin (build=left) (whole-stage-codegen on)") - .exclude("SPARK-15822 - test structs as keys using ShuffledHashJoin (build=right) (whole-stage-codegen off)") - .exclude("SPARK-15822 - test structs as keys using ShuffledHashJoin (build=right) (whole-stage-codegen on)") - .exclude("SPARK-15822 - test structs as keys using SortMergeJoin (whole-stage-codegen off)") - .exclude("SPARK-15822 - test structs as keys using SortMergeJoin (whole-stage-codegen on)") - .exclude("SPARK-15822 - test structs as keys using CartesianProduct") - .exclude("SPARK-15822 - test structs as keys using BroadcastNestedLoopJoin build left (whole-stage-codegen off)") - .exclude("SPARK-15822 - test structs as keys using BroadcastNestedLoopJoin build left (whole-stage-codegen on)") - .exclude("SPARK-15822 - test structs as keys using BroadcastNestedLoopJoin build right (whole-stage-codegen off)") - .exclude("SPARK-15822 - test structs as keys using BroadcastNestedLoopJoin build right (whole-stage-codegen on)") - enableSuite[GlutenOuterJoinSuite] - .exclude("basic left outer join using ShuffledHashJoin (whole-stage-codegen off)") - .exclude("basic left outer join using ShuffledHashJoin (whole-stage-codegen on)") - .exclude("basic left outer join using SortMergeJoin (whole-stage-codegen off)") - .exclude("basic left outer join using SortMergeJoin (whole-stage-codegen on)") - .exclude("basic right outer join using ShuffledHashJoin (whole-stage-codegen off)") - .exclude("basic right outer join using ShuffledHashJoin (whole-stage-codegen on)") - .exclude("basic right outer join using SortMergeJoin (whole-stage-codegen off)") - .exclude("basic right outer join using SortMergeJoin (whole-stage-codegen on)") - .exclude("basic full outer join using ShuffledHashJoin (whole-stage-codegen off)") - .exclude("basic full outer join using ShuffledHashJoin (whole-stage-codegen on)") - .exclude("basic full outer join using SortMergeJoin (whole-stage-codegen off)") - .exclude("basic full outer join using SortMergeJoin (whole-stage-codegen on)") - .exclude("left outer join with unique keys using ShuffledHashJoin (whole-stage-codegen off)") - .exclude("left outer join with unique keys using ShuffledHashJoin (whole-stage-codegen on)") - .exclude("left outer join with unique keys using SortMergeJoin (whole-stage-codegen off)") - .exclude("left outer join with unique keys using SortMergeJoin (whole-stage-codegen on)") - .exclude("right outer join with unique keys using ShuffledHashJoin (whole-stage-codegen off)") - .exclude("right outer join with unique keys using ShuffledHashJoin (whole-stage-codegen on)") - .exclude("right outer join with unique keys using SortMergeJoin (whole-stage-codegen off)") - .exclude("right outer join with unique keys using SortMergeJoin (whole-stage-codegen on)") - .exclude("full outer join with unique keys using ShuffledHashJoin (whole-stage-codegen off)") - .exclude("full outer join with unique keys using ShuffledHashJoin (whole-stage-codegen on)") - .exclude("full outer join with unique keys using SortMergeJoin (whole-stage-codegen off)") - .exclude("full outer join with unique keys using SortMergeJoin (whole-stage-codegen on)") - enableSuite[GlutenSessionExtensionSuite] - enableSuite[GlutenFallbackSuite] - enableSuite[GlutenBucketedReadWithoutHiveSupportSuite] - .exclude("avoid shuffle when join 2 bucketed tables") - .exclude("only shuffle one side when join bucketed table and non-bucketed table") - .exclude("only shuffle one side when 2 bucketed tables have different bucket number") - .exclude("only shuffle one side when 2 bucketed tables have different bucket keys") - .exclude("shuffle when join keys are not equal to bucket keys") - .exclude("shuffle when join 2 bucketed tables with bucketing disabled") - .exclude("check sort and shuffle when bucket and sort columns are join keys") - .exclude("avoid shuffle and sort when sort columns are a super set of join keys") - .exclude("only sort one side when sort columns are different") - .exclude("only sort one side when sort columns are same but their ordering is different") - .exclude("SPARK-17698 Join predicates should not contain filter clauses") - .exclude( - "SPARK-19122 Re-order join predicates if they match with the child's output partitioning") - .exclude("SPARK-19122 No re-ordering should happen if set of join columns != set of child's partitioning columns") - .exclude("SPARK-29655 Read bucketed tables obeys spark.sql.shuffle.partitions") - .exclude("SPARK-32767 Bucket join should work if SHUFFLE_PARTITIONS larger than bucket number") - .exclude("bucket coalescing eliminates shuffle") - .exclude("bucket coalescing is not satisfied") - // DISABLED: GLUTEN-4893 Vanilla UT checks scan operator by exactly matching the class type - .exclude("disable bucketing when the output doesn't contain all bucketing columns") - .exclude( - "bucket coalescing is applied when join expressions match with partitioning expressions") - enableSuite[GlutenBucketedWriteWithoutHiveSupportSuite] - enableSuite[GlutenCreateTableAsSelectSuite] - .exclude("CREATE TABLE USING AS SELECT based on the file without write permission") - .exclude("create a table, drop it and create another one with the same name") - enableSuite[GlutenDDLSourceLoadSuite] - disableSuite[GlutenDisableUnnecessaryBucketedScanWithoutHiveSupportSuite]( - "GLUTEN-4893: Vanilla UT checks scan operator by exactly matching the class type") - enableSuite[GlutenDisableUnnecessaryBucketedScanWithoutHiveSupportSuiteAE] - enableSuite[GlutenExternalCommandRunnerSuite] - enableSuite[GlutenFilteredScanSuite] - enableSuite[GlutenFiltersSuite] - enableSuite[GlutenInsertSuite] - enableSuite[GlutenPartitionedWriteSuite] - .exclude("SPARK-37231, SPARK-37240: Dynamic writes/reads of ANSI interval partitions") - enableSuite[GlutenPathOptionSuite] - enableSuite[GlutenPrunedScanSuite] - enableSuite[GlutenResolvedDataSourceSuite] - enableSuite[GlutenSaveLoadSuite] - enableSuite[GlutenTableScanSuite] - .exclude("Schema and all fields") - .exclude("SELECT count(*) FROM tableWithSchema") - .exclude("SELECT `string$%Field` FROM tableWithSchema") - .exclude("SELECT int_Field FROM tableWithSchema WHERE int_Field < 5") - .exclude("SELECT `longField_:,<>=+/~^` * 2 FROM tableWithSchema") - .exclude( - "SELECT structFieldSimple.key, arrayFieldSimple[1] FROM tableWithSchema a where int_Field=1") - .exclude("SELECT structFieldComplex.Value.`value_(2)` FROM tableWithSchema") - enableSuite[GlutenSparkSessionExtensionSuite] - .includeGlutenTest("customColumnarOp") - enableSuite[GlutenHiveSQLQueryCHSuite] - enableSuite[GlutenPercentileSuite] - - override def getSQLQueryTestSettings: SQLQueryTestSettings = ClickHouseSQLQueryTestSettings -} -// scalastyle:on line.size.limit diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxSQLQueryTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxSQLQueryTestSettings.scala deleted file mode 100644 index 182005437df..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxSQLQueryTestSettings.scala +++ /dev/null @@ -1,250 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.utils.velox - -import org.apache.gluten.utils.SQLQueryTestSettings - -object VeloxSQLQueryTestSettings extends SQLQueryTestSettings { - override def getResourceFilePath: String = - getClass.getResource("/").getPath + "../../../src/test/resources/sql-tests" - - override def getSupportedSQLQueryTests: Set[String] = SUPPORTED_SQL_QUERY_LIST - - override def getOverwriteSQLQueryTests: Set[String] = OVERWRITE_SQL_QUERY_LIST - - // Put relative path to "/path/to/spark/sql/core/src/test/resources/sql-tests/inputs" in this list - private val SUPPORTED_SQL_QUERY_LIST: Set[String] = Set( - "array.sql", - "bitwise.sql", - "cast.sql", - "change-column.sql", - "charvarchar.sql", - "columnresolution-negative.sql", - "columnresolution-views.sql", - "columnresolution.sql", - "comments.sql", - "comparator.sql", - "count.sql", - "cross-join.sql", - "csv-functions.sql", - "cte-legacy.sql", - "cte-nested.sql", - "cte-nonlegacy.sql", - "cte.sql", - "current_database_catalog.sql", - "date.sql", - "datetime-formatting-invalid.sql", - // Velox had different handling for some illegal cases. - // "datetime-formatting-legacy.sql", - // "datetime-formatting.sql", - "datetime-legacy.sql", - "datetime-parsing-invalid.sql", - "datetime-parsing-legacy.sql", - "datetime-parsing.sql", - "datetime-special.sql", - "decimalArithmeticOperations.sql", - "describe-part-after-analyze.sql", - "describe-query.sql", - "describe-table-after-alter-table.sql", - "describe-table-column.sql", - "describe.sql", - "except-all.sql", - "except.sql", - "extract.sql", - "group-analytics.sql", - "group-by-filter.sql", - "group-by-ordinal.sql", - "grouping_set.sql", - "having.sql", - "higher-order-functions.sql", - "ignored.sql", - "ilike-all.sql", - "ilike-any.sql", - "inline-table.sql", - "inner-join.sql", - "intersect-all.sql", - "interval.sql", - "join-empty-relation.sql", - "join-lateral.sql", - "json-functions.sql", - "like-all.sql", - "like-any.sql", - "limit.sql", - "literals.sql", - "map.sql", - "misc-functions.sql", - "natural-join.sql", - "null-handling.sql", - "null-propagation.sql", - "operators.sql", - "order-by-nulls-ordering.sql", - "order-by-ordinal.sql", - "outer-join.sql", - "parse-schema-string.sql", - "pivot.sql", - "pred-pushdown.sql", - "predicate-functions.sql", - "query_regex_column.sql", - "random.sql", - "regexp-functions.sql", - "show-create-table.sql", - "show-tables.sql", - "show-tblproperties.sql", - "show-views.sql", - "show_columns.sql", - "sql-compatibility-functions.sql", - "string-functions.sql", - "struct.sql", - "subexp-elimination.sql", - "table-aliases.sql", - "table-valued-functions.sql", - "tablesample-negative.sql", - "timestamp-ltz.sql", - "timestamp-ntz.sql", - "timestamp.sql", - "timezone.sql", - "transform.sql", - "try-string-functions.sql", - "try_arithmetic.sql", - "try_cast.sql", - "udaf.sql", - "union.sql", - "using-join.sql", - "window.sql", - "ansi/cast.sql", - "ansi/date.sql", - "ansi/datetime-parsing-invalid.sql", - "ansi/datetime-special.sql", - "ansi/decimalArithmeticOperations.sql", - "ansi/interval.sql", - "ansi/literals.sql", - "ansi/map.sql", - "ansi/parse-schema-string.sql", - "ansi/string-functions.sql", - "ansi/timestamp.sql", - "ansi/try_arithmetic.sql", - "postgreSQL/aggregates_part1.sql", - "postgreSQL/aggregates_part2.sql", - "postgreSQL/aggregates_part3.sql", - "postgreSQL/aggregates_part4.sql", - "postgreSQL/boolean.sql", - "postgreSQL/case.sql", - "postgreSQL/comments.sql", - "postgreSQL/create_view.sql", - "postgreSQL/date.sql", - "postgreSQL/float4.sql", - "postgreSQL/insert.sql", - "postgreSQL/int2.sql", - "postgreSQL/int4.sql", - "postgreSQL/int8.sql", - "postgreSQL/interval.sql", - "postgreSQL/join.sql", - "postgreSQL/limit.sql", - "postgreSQL/numeric.sql", - "postgreSQL/select.sql", - "postgreSQL/select_distinct.sql", - "postgreSQL/select_having.sql", - "postgreSQL/select_implicit.sql", - "postgreSQL/strings.sql", - "postgreSQL/text.sql", - "postgreSQL/timestamp.sql", - "postgreSQL/union.sql", - "postgreSQL/window_part1.sql", - "postgreSQL/window_part2.sql", - "postgreSQL/window_part3.sql", - "postgreSQL/window_part4.sql", - "postgreSQL/with.sql", - "subquery/subquery-in-from.sql", - "timestampNTZ/datetime-special.sql", - "timestampNTZ/timestamp-ansi.sql", - "timestampNTZ/timestamp.sql", - "udf/udf-count.sql", - "udf/udf-cross-join.sql", - "udf/udf-except-all.sql", - "udf/udf-except.sql", - "udf/udf-having.sql", - "udf/udf-inline-table.sql", - "udf/udf-inner-join.sql", - "udf/udf-intersect-all.sql", - "udf/udf-join-empty-relation.sql", - "udf/udf-natural-join.sql", - "udf/udf-outer-join.sql", - "udf/udf-pivot.sql", - "udf/udf-udaf.sql", - "udf/udf-union.sql", - "udf/udf-window.sql", - "udf/postgreSQL/udf-select_having.sql", - "subquery/exists-subquery/exists-aggregate.sql", - "subquery/exists-subquery/exists-basic.sql", - "subquery/exists-subquery/exists-cte.sql", - "subquery/exists-subquery/exists-having.sql", - "subquery/exists-subquery/exists-joins-and-set-ops.sql", - "subquery/exists-subquery/exists-orderby-limit.sql", - "subquery/exists-subquery/exists-within-and-or.sql", - "subquery/in-subquery/in-basic.sql", - "subquery/in-subquery/in-group-by.sql", - "subquery/in-subquery/in-having.sql", - "subquery/in-subquery/in-joins.sql", - "subquery/in-subquery/in-limit.sql", - "subquery/in-subquery/in-multiple-columns.sql", - "subquery/in-subquery/in-order-by.sql", - "subquery/in-subquery/in-set-operations.sql", - "subquery/in-subquery/in-with-cte.sql", - "subquery/in-subquery/nested-not-in.sql", - "subquery/in-subquery/not-in-group-by.sql", - "subquery/in-subquery/not-in-joins.sql", - "subquery/in-subquery/not-in-unit-tests-multi-column-literal.sql", - "subquery/in-subquery/not-in-unit-tests-multi-column.sql", - "subquery/in-subquery/not-in-unit-tests-single-column-literal.sql", - "subquery/in-subquery/not-in-unit-tests-single-column.sql", - "subquery/in-subquery/simple-in.sql", - "subquery/negative-cases/invalid-correlation.sql", - "subquery/negative-cases/subq-input-typecheck.sql", - "subquery/scalar-subquery/scalar-subquery-predicate.sql", - "subquery/scalar-subquery/scalar-subquery-select.sql", - "typeCoercion/native/arrayJoin.sql", - "typeCoercion/native/binaryComparison.sql", - "typeCoercion/native/booleanEquality.sql", - "typeCoercion/native/caseWhenCoercion.sql", - "typeCoercion/native/concat.sql", - "typeCoercion/native/dateTimeOperations.sql", - "typeCoercion/native/decimalPrecision.sql", - "typeCoercion/native/division.sql", - "typeCoercion/native/elt.sql", - "typeCoercion/native/ifCoercion.sql", - "typeCoercion/native/implicitTypeCasts.sql", - "typeCoercion/native/inConversion.sql", - "typeCoercion/native/mapZipWith.sql", - "typeCoercion/native/mapconcat.sql", - "typeCoercion/native/mapconcat.sql", - "typeCoercion/native/promoteStrings.sql", - "typeCoercion/native/stringCastAndExpressions.sql", - "typeCoercion/native/widenSetOperationTypes.sql", - "typeCoercion/native/windowFrameCoercion.sql" - ) - - private val OVERWRITE_SQL_QUERY_LIST: Set[String] = Set( - // The calculation formulas for corr, skewness, kurtosis, variance, and stddev in Velox differ - // slightly from those in Spark, resulting in some differences in the final results. - // Overwrite below test cases. - // -- SPARK-24369 multiple distinct aggregations having the same argument set - // -- Aggregate with nulls. - // -- SPARK-37613: Support ANSI Aggregate Function: regr_r2 - "group-by.sql", - "udf/udf-group-by.sql" - ) -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala deleted file mode 100644 index fb42198e2da..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ /dev/null @@ -1,936 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.utils.velox - -import org.apache.gluten.utils.{BackendTestSettings, SQLQueryTestSettings} - -import org.apache.spark.GlutenSortShuffleSuite -import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.connector._ -import org.apache.spark.sql.errors.{GlutenQueryCompilationErrorsDSv2Suite, GlutenQueryExecutionErrorsSuite, GlutenQueryParsingErrorsSuite} -import org.apache.spark.sql.execution._ -import org.apache.spark.sql.execution.adaptive.velox.VeloxAdaptiveQueryExecSuite -import org.apache.spark.sql.execution.datasources._ -import org.apache.spark.sql.execution.datasources.binaryfile.GlutenBinaryFileFormatSuite -import org.apache.spark.sql.execution.datasources.csv.{GlutenCSVLegacyTimeParserSuite, GlutenCSVv1Suite, GlutenCSVv2Suite} -import org.apache.spark.sql.execution.datasources.exchange.GlutenValidateRequirementsSuite -import org.apache.spark.sql.execution.datasources.json.{GlutenJsonLegacyTimeParserSuite, GlutenJsonV1Suite, GlutenJsonV2Suite} -import org.apache.spark.sql.execution.datasources.orc._ -import org.apache.spark.sql.execution.datasources.parquet._ -import org.apache.spark.sql.execution.datasources.text.{GlutenTextV1Suite, GlutenTextV2Suite} -import org.apache.spark.sql.execution.datasources.v2.{GlutenDataSourceV2StrategySuite, GlutenFileTableSuite, GlutenV2PredicateSuite} -import org.apache.spark.sql.execution.exchange.GlutenEnsureRequirementsSuite -import org.apache.spark.sql.execution.joins.{GlutenBroadcastJoinSuite, GlutenExistenceJoinSuite, GlutenInnerJoinSuite, GlutenOuterJoinSuite} -import org.apache.spark.sql.execution.python._ -import org.apache.spark.sql.extension.{GlutenCollapseProjectExecTransformerSuite, GlutenSessionExtensionSuite} -import org.apache.spark.sql.gluten.GlutenFallbackSuite -import org.apache.spark.sql.hive.execution._ -import org.apache.spark.sql.sources._ - -// Some settings' line length exceeds 100 -// scalastyle:off line.size.limit - -class VeloxTestSettings extends BackendTestSettings { - import SuiteSettings._ - enableSuite[GlutenStringFunctionsSuite] - enableSuite[GlutenBloomFilterAggregateQuerySuite] - enableSuite[GlutenBloomFilterAggregateQuerySuiteCGOff] - enableSuite[GlutenDataSourceV2DataFrameSessionCatalogSuite] - enableSuite[GlutenDataSourceV2DataFrameSuite] - enableSuite[GlutenDataSourceV2FunctionSuite] - enableSuite[GlutenDataSourceV2SQLSessionCatalogSuite] - enableSuite[GlutenDataSourceV2SQLSuite] - enableSuite[GlutenDataSourceV2Suite] - // Rewrite the following test in GlutenDataSourceV2Suite. - .exclude("partitioning reporting") - enableSuite[GlutenDeleteFromTableSuite] - enableSuite[GlutenFileDataSourceV2FallBackSuite] - // Rewritten - .exclude("Fallback Parquet V2 to V1") - enableSuite[GlutenKeyGroupedPartitioningSuite] - // NEW SUITE: disable as they check vanilla spark plan - .exclude("partitioned join: number of buckets mismatch should trigger shuffle") - .exclude("partitioned join: only one side reports partitioning") - .exclude("partitioned join: join with two partition keys and different # of partition keys") - enableSuite[GlutenLocalScanSuite] - enableSuite[GlutenMetadataColumnSuite] - enableSuite[GlutenSupportsCatalogOptionsSuite] - enableSuite[GlutenTableCapabilityCheckSuite] - enableSuite[GlutenWriteDistributionAndOrderingSuite] - - enableSuite[GlutenQueryCompilationErrorsDSv2Suite] - - enableSuite[GlutenQueryExecutionErrorsSuite] - // NEW SUITE: disable as it expects exception which doesn't happen when offloaded to gluten - .exclude( - "INCONSISTENT_BEHAVIOR_CROSS_VERSION: compatibility with Spark 2.4/3.2 in reading/writing dates") - // Different exceptions when reading Timestamp from ORC. - .exclude("UNSUPPORTED_OPERATION - SPARK-36346: can't read Timestamp as TimestampNTZ") - enableSuite[GlutenQueryParsingErrorsSuite] - enableSuite[GlutenAnsiCastSuiteWithAnsiModeOff] - .exclude( - "Process Infinity, -Infinity, NaN in case insensitive manner" // +inf not supported in folly. - ) - .exclude("SPARK-35711: cast timestamp without time zone to timestamp with local time zone") - .exclude("SPARK-35719: cast timestamp with local time zone to timestamp without timezone") - - enableSuite[GlutenAnsiCastSuiteWithAnsiModeOn] - .exclude( - "Process Infinity, -Infinity, NaN in case insensitive manner" // +inf not supported in folly. - ) - .exclude("SPARK-35711: cast timestamp without time zone to timestamp with local time zone") - .exclude("SPARK-35719: cast timestamp with local time zone to timestamp without timezone") - - enableSuite[GlutenCastSuiteWithAnsiModeOn] - .exclude( - "Process Infinity, -Infinity, NaN in case insensitive manner" // +inf not supported in folly. - ) - .exclude("SPARK-35711: cast timestamp without time zone to timestamp with local time zone") - .exclude("SPARK-35719: cast timestamp with local time zone to timestamp without timezone") - enableSuite[GlutenTryCastSuite] - .exclude("SPARK-35711: cast timestamp without time zone to timestamp with local time zone") - .exclude("SPARK-35719: cast timestamp with local time zone to timestamp without timezone") - // Revised by setting timezone through config and commented unsupported cases. - .exclude("cast string to timestamp") - enableSuite[GlutenArithmeticExpressionSuite] - enableSuite[GlutenBitwiseExpressionsSuite] - enableSuite[GlutenCastSuite] - .exclude( - "Process Infinity, -Infinity, NaN in case insensitive manner" // +inf not supported in folly. - ) - // Timezone. - .exclude("SPARK-35711: cast timestamp without time zone to timestamp with local time zone") - // Timezone. - .exclude("SPARK-35719: cast timestamp with local time zone to timestamp without timezone") - // Set timezone through config. - .exclude("data type casting") - // Revised by setting timezone through config and commented unsupported cases. - .exclude("cast string to timestamp") - .exclude("SPARK-36286: invalid string cast to timestamp") - enableSuite[GlutenCollectionExpressionsSuite] - // Rewrite in Gluten to replace Seq with Array - .exclude("Shuffle") - .excludeGlutenTest("Shuffle") - // Rewrite. - .exclude("MapFromEntries") - enableSuite[GlutenConditionalExpressionSuite] - enableSuite[GlutenDateExpressionsSuite] - // Has exception in fallback execution when we use resultDF.collect in evaluation. - .exclude("TIMESTAMP_MICROS") - // Replaced by a gluten test to pass timezone through config. - .exclude("unix_timestamp") - // Replaced by a gluten test to pass timezone through config. - .exclude("to_unix_timestamp") - // Replaced by a gluten test to pass timezone through config. - .exclude("Hour") - // Unsupported format: yyyy-MM-dd HH:mm:ss.SSS - .exclude("SPARK-33498: GetTimestamp,UnixTimestamp,ToUnixTimestamp with parseError") - // Replaced by a gluten test to pass timezone through config. - .exclude("DateFormat") - // Legacy mode is not supported, assuming this mode is not commonly used. - .exclude("to_timestamp exception mode") - // Replaced by a gluten test to pass timezone through config. - .exclude("from_unixtime") - // Replaced by a gluten test to pass timezone through config. - .exclude("months_between") - .exclude("test timestamp add") - // https://github.com/facebookincubator/velox/pull/10563/files#diff-140dc50e6dac735f72d29014da44b045509df0dd1737f458de1fe8cfd33d8145 - .excludeGlutenTest("from_unixtime") - enableSuite[GlutenDecimalExpressionSuite] - enableSuite[GlutenDecimalPrecisionSuite] - enableSuite[GlutenHashExpressionsSuite] - enableSuite[GlutenHigherOrderFunctionsSuite] - enableSuite[GlutenGeneratorExpressionSuite] - enableSuite[GlutenIntervalExpressionsSuite] - enableSuite[GlutenJsonExpressionsSuite] - // https://github.com/apache/gluten/issues/8102 - .exclude("$.store.book") - .exclude("$") - .exclude("$.store.book[0]") - .exclude("$.store.book[*]") - .exclude("$.store.book[*].category") - .exclude("$.store.book[*].isbn") - .exclude("$.store.book[*].reader") - .exclude("$.store.basket[*]") - .exclude("$.store.basket[*][0]") - .exclude("$.store.basket[0][*]") - .exclude("$.store.basket[*][*]") - .exclude("$.store.basket[0][*].b") - // Exception class different. - .exclude("from_json - invalid data") - enableSuite[GlutenJsonFunctionsSuite] - // Velox does not support single quotes in get_json_object function. - .exclude("function get_json_object - support single quotes") - enableSuite[GlutenLiteralExpressionSuite] - .exclude("default") - // FIXME(yma11): ObjectType is not covered in RowEncoder/Serializer in vanilla spark - .exclude("SPARK-37967: Literal.create support ObjectType") - enableSuite[GlutenMathExpressionsSuite] - // Spark round UT for round(3.1415,3) is not correct. - .exclude("round/bround/floor/ceil") - enableSuite[GlutenMiscExpressionsSuite] - enableSuite[GlutenNondeterministicSuite] - .exclude("MonotonicallyIncreasingID") - .exclude("SparkPartitionID") - enableSuite[GlutenNullExpressionsSuite] - enableSuite[GlutenPredicateSuite] - enableSuite[GlutenRandomSuite] - .exclude("random") - .exclude("SPARK-9127 codegen with long seed") - enableSuite[GlutenRegexpExpressionsSuite] - enableSuite[GlutenSortShuffleSuite] - enableSuite[GlutenSortOrderExpressionsSuite] - enableSuite[GlutenStringExpressionsSuite] - enableSuite[VeloxAdaptiveQueryExecSuite] - .includeAllGlutenTests() - .includeByPrefix( - "SPARK-30291", - "SPARK-30403", - "SPARK-30719", - "SPARK-31384", - "SPARK-31658", - "SPARK-32717", - "SPARK-32649", - "SPARK-34533", - "SPARK-34781", - "SPARK-35585", - "SPARK-33494", - "SPARK-33933", - "SPARK-31220", - "SPARK-35874", - "SPARK-39551" - ) - .include( - "Union/Except/Intersect queries", - "Subquery de-correlation in Union queries", - "force apply AQE", - "tree string output", - "control a plan explain mode in listener vis SQLConf", - "AQE should set active session during execution", - "No deadlock in UI update", - "SPARK-35455: Unify empty relation optimization between normal and AQE optimizer - multi join" - ) - enableSuite[GlutenBinaryFileFormatSuite] - // Exception. - .exclude("column pruning - non-readable file") - enableSuite[GlutenCSVv1Suite] - // file cars.csv include null string, Arrow not support to read - .exclude("DDL test with schema") - .exclude("save csv") - .exclude("save csv with compression codec option") - .exclude("save csv with quote") - .exclude("SPARK-13543 Write the output as uncompressed via option()") - .exclude("DDL test with tab separated file") - .exclude("DDL test parsing decimal type") - .exclude("test with tab delimiter and double quote") - // Arrow not support corrupt record - .exclude("SPARK-27873: disabling enforceSchema should not fail columnNameOfCorruptRecord") - enableSuite[GlutenCSVv2Suite] - .exclude("Gluten - test for FAILFAST parsing mode") - // file cars.csv include null string, Arrow not support to read - .exclude("DDL test with schema") - .exclude("save csv") - .exclude("save csv with compression codec option") - .exclude("save csv with quote") - .exclude("SPARK-13543 Write the output as uncompressed via option()") - .exclude("DDL test with tab separated file") - .exclude("DDL test parsing decimal type") - .exclude("test with tab delimiter and double quote") - // Rule org.apache.spark.sql.execution.datasources.v2.V2ScanRelationPushDown in batch - // Early Filter and Projection Push-Down generated an invalid plan - .exclude("SPARK-26208: write and read empty data to csv file with headers") - enableSuite[GlutenCSVLegacyTimeParserSuite] - // file cars.csv include null string, Arrow not support to read - .exclude("DDL test with schema") - .exclude("save csv") - .exclude("save csv with compression codec option") - .exclude("save csv with quote") - .exclude("SPARK-13543 Write the output as uncompressed via option()") - .exclude("DDL test with tab separated file") - .exclude("DDL test parsing decimal type") - .exclude("test with tab delimiter and double quote") - // Arrow not support corrupt record - .exclude("SPARK-27873: disabling enforceSchema should not fail columnNameOfCorruptRecord") - enableSuite[GlutenJsonV1Suite] - enableSuite[GlutenJsonV2Suite] - enableSuite[GlutenJsonLegacyTimeParserSuite] - enableSuite[GlutenValidateRequirementsSuite] - enableSuite[GlutenOrcColumnarBatchReaderSuite] - enableSuite[GlutenOrcFilterSuite] - .exclude("SPARK-32622: case sensitivity in predicate pushdown") - enableSuite[GlutenOrcPartitionDiscoverySuite] - .exclude("read partitioned table - normal case") - .exclude("read partitioned table - with nulls") - enableSuite[GlutenOrcV1PartitionDiscoverySuite] - .exclude("read partitioned table - normal case") - .exclude("read partitioned table - with nulls") - .exclude("read partitioned table - partition key included in orc file") - .exclude("read partitioned table - with nulls and partition keys are included in Orc file") - enableSuite[GlutenOrcV1QuerySuite] - // Rewrite to disable Spark's columnar reader. - .exclude("Simple selection form ORC table") - .exclude("simple select queries") - .exclude("overwriting") - .exclude("self-join") - .exclude("columns only referenced by pushed down filters should remain") - .exclude("SPARK-5309 strings stored using dictionary compression in orc") - // For exception test. - .exclude("SPARK-20728 Make ORCFileFormat configurable between sql/hive and sql/core") - .exclude("Read/write binary data") - .exclude("Read/write all types with non-primitive type") - .exclude("Creating case class RDD table") - .exclude("save and load case class RDD with `None`s as orc") - .exclude("SPARK-16610: Respect orc.compress (i.e., OrcConf.COMPRESS) when" + - " compression is unset") - .exclude("Compression options for writing to an ORC file (SNAPPY, ZLIB and NONE)") - .exclude("appending") - .exclude("nested data - struct with array field") - .exclude("nested data - array of struct") - .exclude("SPARK-9170: Don't implicitly lowercase of user-provided columns") - .exclude("SPARK-10623 Enable ORC PPD") - .exclude("SPARK-14962 Produce correct results on array type with isnotnull") - .exclude("SPARK-15198 Support for pushing down filters for boolean types") - .exclude("Support for pushing down filters for decimal types") - .exclude("Support for pushing down filters for timestamp types") - .exclude("column nullability and comment - write and then read") - .exclude("Empty schema does not read data from ORC file") - .exclude("read from multiple orc input paths") - .exclude("Enabling/disabling ignoreCorruptFiles") - .exclude("SPARK-27160 Predicate pushdown correctness on DecimalType for ORC") - .exclude("LZO compression options for writing to an ORC file") - .exclude("Schema discovery on empty ORC files") - .exclude("SPARK-21791 ORC should support column names with dot") - .exclude("SPARK-25579 ORC PPD should support column names with dot") - .exclude("SPARK-34862: Support ORC vectorized reader for nested column") - .exclude("SPARK-37728: Reading nested columns with ORC vectorized reader should not") - .exclude("SPARK-36594: ORC vectorized reader should properly check maximal number of fields") - .exclude("Read/write all timestamp types") - .exclude("SPARK-37463: read/write Timestamp ntz to Orc with different time zone") - .exclude("SPARK-39381: Make vectorized orc columar writer batch size configurable") - .exclude("SPARK-39830: Reading ORC table that requires type promotion may throw AIOOBE") - enableSuite[GlutenOrcV2QuerySuite] - .exclude("Read/write binary data") - .exclude("Read/write all types with non-primitive type") - // Rewrite to disable Spark's columnar reader. - .exclude("Simple selection form ORC table") - .exclude("Creating case class RDD table") - .exclude("save and load case class RDD with `None`s as orc") - .exclude("SPARK-16610: Respect orc.compress (i.e., OrcConf.COMPRESS) when compression is unset") - .exclude("Compression options for writing to an ORC file (SNAPPY, ZLIB and NONE)") - .exclude("appending") - .exclude("nested data - struct with array field") - .exclude("nested data - array of struct") - .exclude("SPARK-9170: Don't implicitly lowercase of user-provided columns") - .exclude("SPARK-10623 Enable ORC PPD") - .exclude("SPARK-14962 Produce correct results on array type with isnotnull") - .exclude("SPARK-15198 Support for pushing down filters for boolean types") - .exclude("Support for pushing down filters for decimal types") - .exclude("Support for pushing down filters for timestamp types") - .exclude("column nullability and comment - write and then read") - .exclude("Empty schema does not read data from ORC file") - .exclude("read from multiple orc input paths") - .exclude("Enabling/disabling ignoreCorruptFiles") - .exclude("SPARK-27160 Predicate pushdown correctness on DecimalType for ORC") - .exclude("LZO compression options for writing to an ORC file") - .exclude("Schema discovery on empty ORC files") - .exclude("SPARK-21791 ORC should support column names with dot") - .exclude("SPARK-25579 ORC PPD should support column names with dot") - .exclude("SPARK-34862: Support ORC vectorized reader for nested column") - .exclude("SPARK-37728: Reading nested columns with ORC vectorized reader should not") - .exclude("SPARK-36594: ORC vectorized reader should properly check maximal number of fields") - .exclude("Read/write all timestamp types") - .exclude("SPARK-37463: read/write Timestamp ntz to Orc with different time zone") - .exclude("SPARK-39381: Make vectorized orc columar writer batch size configurable") - .exclude("SPARK-39830: Reading ORC table that requires type promotion may throw AIOOBE") - .exclude("simple select queries") - .exclude("overwriting") - .exclude("self-join") - .exclude("columns only referenced by pushed down filters should remain") - .exclude("SPARK-5309 strings stored using dictionary compression in orc") - // For exception test. - .exclude("SPARK-20728 Make ORCFileFormat configurable between sql/hive and sql/core") - enableSuite[GlutenOrcSourceSuite] - // Rewrite to disable Spark's columnar reader. - .exclude("SPARK-31238: compatibility with Spark 2.4 in reading dates") - .exclude("SPARK-31238, SPARK-31423: rebasing dates in write") - .exclude("SPARK-31284: compatibility with Spark 2.4 in reading timestamps") - .exclude("SPARK-31284, SPARK-31423: rebasing timestamps in write") - .exclude("SPARK-34862: Support ORC vectorized reader for nested column") - // Ignored to disable vectorized reading check. - .exclude("SPARK-36594: ORC vectorized reader should properly check maximal number of fields") - .exclude("create temporary orc table") - .exclude("create temporary orc table as") - .exclude("appending insert") - .exclude("overwrite insert") - .exclude("SPARK-34897: Support reconcile schemas based on index after nested column pruning") - .excludeGlutenTest("SPARK-31238: compatibility with Spark 2.4 in reading dates") - .excludeGlutenTest("SPARK-31238, SPARK-31423: rebasing dates in write") - .excludeGlutenTest("SPARK-34862: Support ORC vectorized reader for nested column") - // exclude as struct not supported - .exclude("SPARK-36663: OrcUtils.toCatalystSchema should correctly handle a column name which consists of only numbers") - .exclude("SPARK-37812: Reuse result row when deserializing a struct") - // rewrite - .exclude("SPARK-36931: Support reading and writing ANSI intervals (spark.sql.orc.enableVectorizedReader=true, spark.sql.orc.enableNestedColumnVectorizedReader=true)") - .exclude("SPARK-36931: Support reading and writing ANSI intervals (spark.sql.orc.enableVectorizedReader=true, spark.sql.orc.enableNestedColumnVectorizedReader=false)") - enableSuite[GlutenOrcV1FilterSuite] - .exclude("SPARK-32622: case sensitivity in predicate pushdown") - enableSuite[GlutenOrcV1SchemaPruningSuite] - enableSuite[GlutenOrcV2SchemaPruningSuite] - enableSuite[GlutenParquetColumnIndexSuite] - // Rewrite by just removing test timestamp. - .exclude("test reading unaligned pages - test all types") - // Rewrite by converting smaller integral value to timestamp. - .exclude("test reading unaligned pages - test all types (dict encode)") - enableSuite[GlutenParquetCompressionCodecPrecedenceSuite] - enableSuite[GlutenParquetDeltaByteArrayEncodingSuite] - enableSuite[GlutenParquetDeltaEncodingInteger] - enableSuite[GlutenParquetDeltaEncodingLong] - enableSuite[GlutenParquetDeltaLengthByteArrayEncodingSuite] - enableSuite[GlutenParquetEncodingSuite] - // Velox does not support rle encoding. - .exclude("parquet v2 pages - rle encoding for boolean value columns") - enableSuite[GlutenParquetFieldIdIOSuite] - enableSuite[GlutenParquetFileFormatV1Suite] - enableSuite[GlutenParquetFileFormatV2Suite] - enableSuite[GlutenParquetV1FilterSuite] - // Rewrite. - .exclude("SPARK-23852: Broken Parquet push-down for partially-written stats") - // Rewrite for supported INT96 - timestamp. - .exclude("filter pushdown - timestamp") - .exclude("filter pushdown - date") - // Exception bebaviour. - .exclude("SPARK-25207: exception when duplicate fields in case-insensitive mode") - // Ignore Spark's filter pushdown check. - .exclude("Filters should be pushed down for vectorized Parquet reader at row group level") - .exclude("SPARK-31026: Parquet predicate pushdown for fields having dots in the names") - .exclude("Filters should be pushed down for Parquet readers at row group level") - .exclude("filter pushdown - StringStartsWith") - .exclude("SPARK-17091: Convert IN predicate to Parquet filter push-down") - .exclude("Support Parquet column index") - .exclude("SPARK-34562: Bloom filter push down") - .exclude("SPARK-16371 Do not push down filters when inner name and outer name are the same") - .exclude("SPARK-38825: in and notIn filters") - enableSuite[GlutenParquetV2FilterSuite] - // Rewrite. - .exclude("SPARK-23852: Broken Parquet push-down for partially-written stats") - // Rewrite for supported INT96 - timestamp. - .exclude("filter pushdown - timestamp") - .exclude("filter pushdown - date") - // Exception bebaviour. - .exclude("SPARK-25207: exception when duplicate fields in case-insensitive mode") - // Ignore Spark's filter pushdown check. - .exclude("Filters should be pushed down for vectorized Parquet reader at row group level") - .exclude("SPARK-31026: Parquet predicate pushdown for fields having dots in the names") - .exclude("Filters should be pushed down for Parquet readers at row group level") - .exclude("filter pushdown - StringStartsWith") - .exclude("SPARK-17091: Convert IN predicate to Parquet filter push-down") - .exclude("Support Parquet column index") - .exclude("SPARK-34562: Bloom filter push down") - .exclude("SPARK-16371 Do not push down filters when inner name and outer name are the same") - .exclude("SPARK-38825: in and notIn filters") - enableSuite[GlutenParquetInteroperabilitySuite] - .exclude("parquet timestamp conversion") - // TODO: https://github.com/apache/gluten/issues/11865 - .exclude("SPARK-36803: parquet files with legacy mode and schema evolution") - enableSuite[GlutenParquetIOSuite] - // Exception. - .exclude("SPARK-35640: read binary as timestamp should throw schema incompatible error") - // Exception msg. - .exclude("SPARK-35640: int as long should throw schema incompatible error") - // Velox parquet reader not allow offset zero. - .exclude("SPARK-40128 read DELTA_LENGTH_BYTE_ARRAY encoded strings") - enableSuite[GlutenParquetV1PartitionDiscoverySuite] - enableSuite[GlutenParquetV2PartitionDiscoverySuite] - enableSuite[GlutenParquetProtobufCompatibilitySuite] - enableSuite[GlutenParquetV1QuerySuite] - // Unsupport spark.sql.files.ignoreCorruptFiles. - .exclude("Enabling/disabling ignoreCorruptFiles") - // decimal failed ut - .exclude("SPARK-34212 Parquet should read decimals correctly") - // new added in spark-3.3 and need fix later, random failure may caused by memory free - .exclude("SPARK-39833: pushed filters with project without filter columns") - .exclude("SPARK-39833: pushed filters with count()") - // Rewrite because the filter after datasource is not needed. - .exclude( - "SPARK-26677: negated null-safe equality comparison should not filter matched row groups") - // Velox currently does not distinguish `isAdjustedToUTC` in Parquet. - .exclude("SPARK-36182: can't read TimestampLTZ as TimestampNTZ") - enableSuite[GlutenParquetV2QuerySuite] - // Unsupport spark.sql.files.ignoreCorruptFiles. - .exclude("Enabling/disabling ignoreCorruptFiles") - // decimal failed ut - .exclude("SPARK-34212 Parquet should read decimals correctly") - // Rewrite because the filter after datasource is not needed. - .exclude( - "SPARK-26677: negated null-safe equality comparison should not filter matched row groups") - // Velox currently does not distinguish `isAdjustedToUTC` in Parquet. - .exclude("SPARK-36182: can't read TimestampLTZ as TimestampNTZ") - enableSuite[GlutenParquetV1SchemaPruningSuite] - enableSuite[GlutenParquetV2SchemaPruningSuite] - enableSuite[GlutenParquetRebaseDatetimeV1Suite] - // jar path and ignore PARQUET_REBASE_MODE_IN_READ, rewrite some - .excludeByPrefix("SPARK-31159") - .excludeByPrefix("SPARK-35427") - enableSuite[GlutenParquetRebaseDatetimeV2Suite] - // jar path and ignore PARQUET_REBASE_MODE_IN_READ - .excludeByPrefix("SPARK-31159") - .excludeByPrefix("SPARK-35427") - enableSuite[GlutenParquetSchemaInferenceSuite] - enableSuite[GlutenParquetSchemaSuite] - // error message mismatch is accepted - .exclude("schema mismatch failure error message for parquet reader") - .exclude("schema mismatch failure error message for parquet vectorized reader") - enableSuite[GlutenParquetThriftCompatibilitySuite] - // Rewrite for file locating. - .exclude("Read Parquet file generated by parquet-thrift") - enableSuite[GlutenParquetVectorizedSuite] - enableSuite[GlutenTextV1Suite] - enableSuite[GlutenTextV2Suite] - enableSuite[GlutenDataSourceV2StrategySuite] - enableSuite[GlutenFileTableSuite] - enableSuite[GlutenV2PredicateSuite] - enableSuite[GlutenBucketingUtilsSuite] - enableSuite[GlutenDataSourceStrategySuite] - enableSuite[GlutenDataSourceSuite] - enableSuite[GlutenFileFormatWriterSuite] - enableSuite[GlutenFileIndexSuite] - enableSuite[GlutenFileMetadataStructSuite] - enableSuite[GlutenParquetV1AggregatePushDownSuite] - enableSuite[GlutenParquetV2AggregatePushDownSuite] - enableSuite[GlutenOrcV1AggregatePushDownSuite] - .exclude("nested column: Count(nested sub-field) not push down") - enableSuite[GlutenOrcV2AggregatePushDownSuite] - .exclude("nested column: Max(top level column) not push down") - .exclude("nested column: Count(nested sub-field) not push down") - enableSuite[GlutenParquetCodecSuite] - // Unsupported compression codec. - .exclude("write and read - file source parquet - codec: lz4") - enableSuite[GlutenOrcCodecSuite] - enableSuite[GlutenFileSourceStrategySuite] - // Plan comparison. - .exclude("partitioned table - after scan filters") - enableSuite[GlutenHadoopFileLinesReaderSuite] - enableSuite[GlutenPathFilterStrategySuite] - enableSuite[GlutenPathFilterSuite] - enableSuite[GlutenPruneFileSourcePartitionsSuite] - enableSuite[GlutenCSVReadSchemaSuite] - enableSuite[GlutenHeaderCSVReadSchemaSuite] - enableSuite[GlutenJsonReadSchemaSuite] - enableSuite[GlutenOrcReadSchemaSuite] - enableSuite[GlutenVectorizedOrcReadSchemaSuite] - enableSuite[GlutenMergedOrcReadSchemaSuite] - enableSuite[GlutenParquetReadSchemaSuite] - enableSuite[GlutenVectorizedParquetReadSchemaSuite] - enableSuite[GlutenMergedParquetReadSchemaSuite] - enableSuite[GlutenEnsureRequirementsSuite] - // Rewrite to change the shuffle partitions for optimizing repartition - .excludeByPrefix("SPARK-35675") - - enableSuite[GlutenBroadcastJoinSuite] - .exclude("Shouldn't change broadcast join buildSide if user clearly specified") - .exclude("Shouldn't bias towards build right if user didn't specify") - .exclude("SPARK-23192: broadcast hint should be retained after using the cached data") - .exclude("broadcast join where streamed side's output partitioning is HashPartitioning") - - enableSuite[GlutenExistenceJoinSuite] - enableSuite[GlutenInnerJoinSuite] - enableSuite[GlutenOuterJoinSuite] - enableSuite[FallbackStrategiesSuite] - enableSuite[GlutenBroadcastExchangeSuite] - enableSuite[GlutenCoalesceShufflePartitionsSuite] - // Rewrite for Gluten. Change details are in the inline comments in individual tests. - .excludeByPrefix("determining the number of reducers") - enableSuite[GlutenExchangeSuite] - // ColumnarShuffleExchangeExec does not support doExecute() method - .exclude("shuffling UnsafeRows in exchange") - // This test will re-run in GlutenExchangeSuite with shuffle partitions > 1 - .exclude("Exchange reuse across the whole plan") - enableSuite[GlutenReplaceHashWithSortAggSuite] - .exclude("replace partial hash aggregate with sort aggregate") - .exclude("replace partial and final hash aggregate together with sort aggregate") - .exclude("do not replace hash aggregate if child does not have sort order") - .exclude("do not replace hash aggregate if there is no group-by column") - enableSuite[GlutenReuseExchangeAndSubquerySuite] - enableSuite[GlutenSameResultSuite] - enableSuite[GlutenSortSuite] - enableSuite[GlutenSQLAggregateFunctionSuite] - // spill not supported yet. - enableSuite[GlutenSQLWindowFunctionSuite] - .exclude("test with low buffer spill threshold") - enableSuite[GlutenTakeOrderedAndProjectSuite] - enableSuite[GlutenSessionExtensionSuite] - enableSuite[GlutenBucketedReadWithoutHiveSupportSuite] - // Exclude the following suite for plan changed from SMJ to SHJ. - .exclude("avoid shuffle when join 2 bucketed tables") - .exclude("avoid shuffle and sort when sort columns are a super set of join keys") - .exclude("only shuffle one side when join bucketed table and non-bucketed table") - .exclude("only shuffle one side when 2 bucketed tables have different bucket number") - .exclude("only shuffle one side when 2 bucketed tables have different bucket keys") - .exclude("shuffle when join keys are not equal to bucket keys") - .exclude("shuffle when join 2 bucketed tables with bucketing disabled") - .exclude("check sort and shuffle when bucket and sort columns are join keys") - .exclude("only sort one side when sort columns are different") - .exclude("only sort one side when sort columns are same but their ordering is different") - .exclude("SPARK-17698 Join predicates should not contain filter clauses") - .exclude("SPARK-19122 Re-order join predicates if they match with the child's" + - " output partitioning") - .exclude("SPARK-19122 No re-ordering should happen if set of join columns != set of child's " + - "partitioning columns") - .exclude("SPARK-29655 Read bucketed tables obeys spark.sql.shuffle.partitions") - .exclude("SPARK-32767 Bucket join should work if SHUFFLE_PARTITIONS larger than bucket number") - .exclude("bucket coalescing eliminates shuffle") - .exclude("bucket coalescing is not satisfied") - // DISABLED: GLUTEN-4893 Vanilla UT checks scan operator by exactly matching the class type - .exclude("disable bucketing when the output doesn't contain all bucketing columns") - .excludeByPrefix("bucket coalescing is applied when join expressions match") - enableSuite[GlutenBucketedWriteWithoutHiveSupportSuite] - enableSuite[GlutenCreateTableAsSelectSuite] - // TODO Gluten can not catch the spark exception in Driver side. - .exclude("CREATE TABLE USING AS SELECT based on the file without write permission") - .exclude("create a table, drop it and create another one with the same name") - enableSuite[GlutenDDLSourceLoadSuite] - disableSuite[GlutenDisableUnnecessaryBucketedScanWithoutHiveSupportSuite]( - "GLUTEN-4893: Vanilla UT checks scan operator by exactly matching the class type") - enableSuite[GlutenDisableUnnecessaryBucketedScanWithoutHiveSupportSuiteAE] - enableSuite[GlutenExternalCommandRunnerSuite] - enableSuite[GlutenFilteredScanSuite] - enableSuite[GlutenFiltersSuite] - enableSuite[GlutenInsertSuite] - enableSuite[GlutenPartitionedWriteSuite] - enableSuite[GlutenPathOptionSuite] - enableSuite[GlutenPrunedScanSuite] - enableSuite[GlutenResolvedDataSourceSuite] - enableSuite[GlutenSaveLoadSuite] - enableSuite[GlutenTableScanSuite] - enableSuite[GlutenApproxCountDistinctForIntervalsQuerySuite] - enableSuite[GlutenApproximatePercentileQuerySuite] - // requires resource files from Vanilla spark jar - .exclude("SPARK-32908: maximum target error in percentile_approx") - enableSuite[GlutenCachedTableSuite] - .exclude("InMemoryRelation statistics") - // Extra ColumnarToRow is needed to transform vanilla columnar data to gluten columnar data. - .exclude("SPARK-37369: Avoid redundant ColumnarToRow transition on InMemoryTableScan") - // Rewrite for different cache size. - .exclude("SPARK-36120: Support cache/uncache table with TimestampNTZ type") - enableSuite[GlutenFileSourceCharVarcharTestSuite] - // Following test is excluded as it is overridden in Gluten test suite.. - // The overridden tests assert against Velox-specific error messages for char/varchar - // length validation, which differ from the original vanilla Spark tests. - .exclude("length check for input string values: nested in struct of array") - enableSuite[GlutenDSV2CharVarcharTestSuite] - // Following test is excluded as it is overridden in Gluten test suite.. - // The overridden tests assert against Velox-specific error messages for char/varchar - // length validation, which differ from the original vanilla Spark tests. - .exclude("length check for input string values: nested in struct of array") - enableSuite[GlutenColumnExpressionSuite] - // Velox raise_error('errMsg') throws a velox_user_error exception with the message 'errMsg'. - // The final caught Spark exception's getCause().getMessage() contains 'errMsg' but does not - // equal 'errMsg' exactly. The following two tests will be skipped and overridden in Gluten. - .exclude("raise_error") - .exclude("assert_true") - enableSuite[GlutenComplexTypeSuite] - enableSuite[GlutenConfigBehaviorSuite] - // Will be fixed by cleaning up ColumnarShuffleExchangeExec. - .exclude("SPARK-22160 spark.sql.execution.rangeExchange.sampleSizePerPartition") - // Gluten columnar operator will have different number of jobs - .exclude("SPARK-40211: customize initialNumPartitions for take") - enableSuite[GlutenCountMinSketchAggQuerySuite] - enableSuite[GlutenCsvFunctionsSuite] - enableSuite[GlutenCTEHintSuite] - enableSuite[GlutenCTEInlineSuiteAEOff] - enableSuite[GlutenCTEInlineSuiteAEOn] - enableSuite[GlutenDataFrameAggregateSuite] - .exclude( - "zero moments", // [velox does not return NaN] - "SPARK-26021: NaN and -0.0 in grouping expressions", // NaN case - // incorrect result, distinct NaN case - "SPARK-32038: NormalizeFloatingNumbers should work on distinct aggregate", - // Replaced with another test. - "SPARK-19471: AggregationIterator does not initialize the generated result projection" + - " before using it", - // Velox's collect_list / collect_set are by design declarative aggregate so plan check - // for ObjectHashAggregateExec will fail. - "SPARK-22223: ObjectHashAggregate should not introduce unnecessary shuffle", - "SPARK-31620: agg with subquery (whole-stage-codegen = true)", - "SPARK-31620: agg with subquery (whole-stage-codegen = false)", - // The below test just verifies Spark's scala code. The involved toString - // implementation has different result on Java 17. - "SPARK-24788: RelationalGroupedDataset.toString with unresolved exprs should not fail" - ) - enableSuite[GlutenDataFrameAsOfJoinSuite] - enableSuite[GlutenDataFrameComplexTypeSuite] - enableSuite[GlutenDataFrameFunctionsSuite] - // blocked by Velox-5768 - .exclude("aggregate function - array for primitive type containing null") - .exclude("aggregate function - array for non-primitive type") - // Rewrite this test because Velox sorts rows by key for primitive data types, which disrupts the original row sequence. - .exclude("map_zip_with function - map of primitive types") - enableSuite[GlutenDataFrameHintSuite] - enableSuite[GlutenDataFrameImplicitsSuite] - enableSuite[GlutenDataFrameJoinSuite] - enableSuite[GlutenDataFrameNaFunctionsSuite] - .exclude( - // NaN case - "replace nan with float", - "replace nan with double" - ) - enableSuite[GlutenDataFramePivotSuite] - // substring issue - .exclude("pivot with column definition in groupby") - // array comparison not supported for values that contain nulls - .exclude( - "pivot with null and aggregate type not supported by PivotFirst returns correct result") - enableSuite[GlutenDataFrameRangeSuite] - .exclude("SPARK-20430 Initialize Range parameters in a driver side") - .excludeByPrefix("Cancelling stage in a query with Range") - enableSuite[GlutenDataFrameSelfJoinSuite] - enableSuite[GlutenDataFrameSessionWindowingSuite] - enableSuite[GlutenDataFrameSetOperationsSuite] - .exclude("SPARK-37371: UnionExec should support columnar if all children support columnar") - // Result depends on the implementation for nondeterministic expression rand. - // Not really an issue. - .exclude("SPARK-10740: handle nondeterministic expressions correctly for set operations") - enableSuite[GlutenDataFrameStatSuite] - enableSuite[GlutenDataFrameSuite] - // Rewrite these tests because it checks Spark's physical operators. - .excludeByPrefix("SPARK-22520", "reuse exchange") - .exclude( - /** - * Rewrite these tests because the rdd partition is equal to the configuration - * "spark.sql.shuffle.partitions". - */ - "repartitionByRange", - "distributeBy and localSort", - // Mismatch when max NaN and infinite value - "NaN is greater than all other non-NaN numeric values", - // Rewrite this test because the describe functions creates unmatched plan. - "describe", - // decimal failed ut. - "SPARK-22271: mean overflows and returns null for some decimal variables", - // Result depends on the implementation for nondeterministic expression rand. - // Not really an issue. - "SPARK-9083: sort with non-deterministic expressions" - ) - // The describe issue is just fixed by https://github.com/apache/spark/pull/40914. - // We can enable the below test for spark 3.4 and higher versions. - .excludeGlutenTest("describe") - // Rewrite this test since it checks the physical operator which is changed in Gluten - .exclude("SPARK-27439: Explain result should match collected result after view change") - enableSuite[GlutenDataFrameTimeWindowingSuite] - enableSuite[GlutenDataFrameTungstenSuite] - enableSuite[GlutenDataFrameWindowFunctionsSuite] - // does not support `spark.sql.legacy.statisticalAggregate=true` (null -> NAN) - .exclude("corr, covar_pop, stddev_pop functions in specific window") - .exclude("covar_samp, var_samp (variance), stddev_samp (stddev) functions in specific window") - // does not support spill - .exclude("Window spill with more than the inMemoryThreshold and spillThreshold") - .exclude("SPARK-21258: complex object in combination with spilling") - // rewrite `WindowExec -> WindowExecTransformer` - .exclude( - "SPARK-38237: require all cluster keys for child required distribution for window query") - enableSuite[GlutenDataFrameWindowFramesSuite] - // Local window fixes are not added. - .exclude("range between should accept int/long values as boundary") - .exclude("unbounded preceding/following range between with aggregation") - .exclude("sliding range between with aggregation") - .exclude("store and retrieve column stats in different time zones") - enableSuite[GlutenDataFrameWriterV2Suite] - enableSuite[GlutenDatasetAggregatorSuite] - enableSuite[GlutenDatasetCacheSuite] - enableSuite[GlutenDatasetOptimizationSuite] - enableSuite[GlutenDatasetPrimitiveSuite] - enableSuite[GlutenDatasetSerializerRegistratorSuite] - enableSuite[GlutenDatasetSuite] - // Rewrite the following two tests in GlutenDatasetSuite. - .exclude("dropDuplicates: columns with same column name") - .exclude("groupBy.as") - // The below two tests just verify Spark's scala code. The involved toString - // implementation has different result on Java 17. - .exclude("Check RelationalGroupedDataset toString: Single data") - .exclude("Check RelationalGroupedDataset toString: over length schema ") - enableSuite[GlutenDateFunctionsSuite] - // The below two are replaced by two modified versions. - .exclude("unix_timestamp") - .exclude("to_unix_timestamp") - // Unsupported datetime format: specifier X is not supported by velox. - .exclude("to_timestamp with microseconds precision") - // Legacy mode is not supported, assuming this mode is not commonly used. - .exclude("SPARK-30668: use legacy timestamp parser in to_timestamp") - // Legacy mode is not supported and velox getTimestamp function does not throw - // exception when format is "yyyy-dd-aa". - .exclude("function to_date") - enableSuite[GlutenDeprecatedAPISuite] - enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOff] - enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOn] - enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOnDisableScan] - enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOffDisableScan] - enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOffWSCGOnDisableProject] - enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOffWSCGOffDisableProject] - enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOff] - enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOn] - enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOnDisableScan] - enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOffDisableScan] - enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOffWSCGOnDisableProject] - enableSuite[GlutenDynamicPartitionPruningV2SuiteAEOffWSCGOffDisableProject] - enableSuite[GlutenExpressionsSchemaSuite] - enableSuite[GlutenExtraStrategiesSuite] - enableSuite[GlutenFileBasedDataSourceSuite] - // test data path is jar path, rewrite - .exclude("Option recursiveFileLookup: disable partition inferring") - // gluten executor exception cannot get in driver, rewrite - .exclude("Spark native readers should respect spark.sql.caseSensitive - parquet") - // shuffle_partitions config is different, rewrite - .excludeByPrefix("SPARK-22790") - // plan is different cause metric is different, rewrite - .excludeByPrefix("SPARK-25237") - // ignoreMissingFiles mode: error msg from velox is different, rewrite - .exclude("Enabling/disabling ignoreMissingFiles using parquet") - .exclude("Enabling/disabling ignoreMissingFiles using orc") - .exclude("Spark native readers should respect spark.sql.caseSensitive - orc") - .exclude("Return correct results when data columns overlap with partition columns") - .exclude("Return correct results when data columns overlap with partition " + - "columns (nested data)") - .exclude("SPARK-31116: Select nested schema with case insensitive mode") - // exclude as original metric not correct when task offloaded to velox - .exclude("SPARK-37585: test input metrics for DSV2 with output limits") - // DISABLED: GLUTEN-4893 Vanilla UT checks scan operator by exactly matching the class type - .exclude("File source v2: support passing data filters to FileScan without partitionFilters") - // DISABLED: GLUTEN-4893 Vanilla UT checks scan operator by exactly matching the class type - .exclude("File source v2: support partition pruning") - // https://github.com/apache/gluten/pull/9145. - .excludeGlutenTest("SPARK-25237 compute correct input metrics in FileScanRDD") - enableSuite[GlutenFileScanSuite] - enableSuite[GlutenGeneratorFunctionSuite] - enableSuite[GlutenInjectRuntimeFilterSuite] - .exclude("Merge runtime bloom filters") - enableSuite[GlutenIntervalFunctionsSuite] - enableSuite[GlutenJoinSuite] - // exclude as it check spark plan - .exclude("SPARK-36794: Ignore duplicated key when building relation for semi/anti hash join") - enableSuite[GlutenMathFunctionsSuite] - enableSuite[GlutenMetadataCacheSuite] - .exclude("SPARK-16336,SPARK-27961 Suggest fixing FileNotFoundException") - enableSuite[GlutenMiscFunctionsSuite] - enableSuite[GlutenNestedDataSourceV1Suite] - enableSuite[GlutenNestedDataSourceV2Suite] - enableSuite[GlutenProcessingTimeSuite] - enableSuite[GlutenProductAggSuite] - enableSuite[GlutenReplaceNullWithFalseInPredicateEndToEndSuite] - enableSuite[GlutenScalaReflectionRelationSuite] - enableSuite[GlutenSerializationSuite] - // following UT is removed in spark3.3.1 - // enableSuite[GlutenSimpleShowCreateTableSuite] - enableSuite[GlutenFileSourceSQLInsertTestSuite] - enableSuite[GlutenDSV2SQLInsertTestSuite] - enableSuite[org.apache.spark.sql.GlutenSQLQuerySuite] - // Decimal precision exceeds. - .exclude("should be able to resolve a persistent view") - // Unstable. Needs to be fixed. - .exclude("SPARK-36093: RemoveRedundantAliases should not change expression's name") - // Rewrite from ORC scan to Parquet scan because ORC is not well supported. - .exclude("SPARK-28156: self-join should not miss cached view") - .exclude("SPARK-33338: GROUP BY using literal map should not fail") - // Rewrite to disable plan check for SMJ because SHJ is preferred in Gluten. - .exclude("SPARK-11111 null-safe join should not use cartesian product") - // Rewrite to change the information of a caught exception. - .exclude("SPARK-33677: LikeSimplification should be skipped if pattern contains any escapeChar") - // Different exception. - .exclude("run sql directly on files") - // Not useful and time consuming. - .exclude("SPARK-33084: Add jar support Ivy URI in SQL") - .exclude("SPARK-33084: Add jar support Ivy URI in SQL -- jar contains udf class") - // https://github.com/apache/gluten/pull/9145. - .exclude("SPARK-17515: CollectLimit.execute() should perform per-partition limits") - // https://github.com/apache/gluten/pull/9145. - .exclude("SPARK-19650: An action on a Command should not trigger a Spark job") - enableSuite[GlutenSQLQueryTestSuite] - enableSuite[GlutenStatisticsCollectionSuite] - .exclude("SPARK-33687: analyze all tables in a specific database") - .exclude("column stats collection for null columns") - .exclude("analyze column command - result verification") - enableSuite[GlutenSubquerySuite] - .excludeByPrefix( - "SPARK-26893" // Rewrite this test because it checks Spark's physical operators. - ) - // exclude as it checks spark plan - .exclude("SPARK-36280: Remove redundant aliases after RewritePredicateSubquery") - enableSuite[GlutenTypedImperativeAggregateSuite] - enableSuite[GlutenUnwrapCastInComparisonEndToEndSuite] - // Rewrite with NaN test cases excluded. - .exclude("cases when literal is max") - enableSuite[GlutenXPathFunctionsSuite] - enableSuite[GlutenFallbackSuite] - enableSuite[GlutenHashAggregationQuerySuite] - // TODO: fix on https://github.com/apache/gluten/issues/11919 - .exclude("udaf with all data types") - enableSuite[GlutenHashAggregationQueryWithControlledFallbackSuite] - // TODO: fix on https://github.com/apache/gluten/issues/11919 - .exclude("udaf with all data types") - enableSuite[GlutenHiveCommandSuite] - enableSuite[GlutenHiveDDLSuite] - enableSuite[GlutenHiveExplainSuite] - .exclude("explain output of physical plan should contain proper codegen stage ID") - .exclude("EXPLAIN CODEGEN command") - enableSuite[GlutenHivePlanTest] - enableSuite[GlutenHiveQuerySuite] - enableSuite[GlutenHiveResolutionSuite] - enableSuite[GlutenHiveSQLQuerySuite] - enableSuite[GlutenHiveSQLViewSuite] - enableSuite[GlutenHiveScriptTransformationSuite] - enableSuite[GlutenHiveSerDeReadWriteSuite] - enableSuite[GlutenHiveSerDeSuite] - enableSuite[GlutenHiveTableScanSuite] - enableSuite[GlutenHiveTypeCoercionSuite] - enableSuite[GlutenHiveUDAFSuite] - enableSuite[GlutenHiveUDFSuite] - enableSuite[GlutenObjectHashAggregateSuite] - enableSuite[GlutenPruneHiveTablePartitionsSuite] - enableSuite[GlutenPruningSuite] - enableSuite[GlutenSQLMetricsSuite] - enableSuite[org.apache.spark.sql.hive.execution.GlutenSQLQuerySuite] - enableSuite[GlutenHashUDAQuerySuite] - enableSuite[GlutenHashUDAQueryWithControlledFallbackSuite] - enableSuite[GlutenSQLQuerySuiteAE] - enableSuite[GlutenWindowQuerySuite] - enableSuite[GlutenCollapseProjectExecTransformerSuite] - enableSuite[GlutenSparkSessionExtensionSuite] - .includeGlutenTest("customColumnarOp") - enableSuite[GlutenSQLCollectLimitExecSuite] - enableSuite[GlutenBatchEvalPythonExecSuite] - // Replaced with other tests that check for native operations - .exclude("Python UDF: push down deterministic FilterExec predicates") - .exclude("Nested Python UDF: push down deterministic FilterExec predicates") - .exclude("Python UDF: no push down on non-deterministic") - .exclude("Python UDF: push down on deterministic predicates after the first non-deterministic") - enableSuite[GlutenExtractPythonUDFsSuite] - // Replaced with test that check for native operations - .exclude("Python UDF should not break column pruning/filter pushdown -- Parquet V1") - .exclude("Chained Scalar Pandas UDFs should be combined to a single physical node") - .exclude("Mixed Batched Python UDFs and Pandas UDF should be separate physical node") - .exclude("Independent Batched Python UDFs and Scalar Pandas UDFs should be combined separately") - .exclude("Dependent Batched Python UDFs and Scalar Pandas UDFs should not be combined") - .exclude("Python UDF should not break column pruning/filter pushdown -- Parquet V2") - enableSuite[GlutenQueryExecutionSuite] - // Rewritten to set root logger level to INFO so that logs can be parsed - .exclude("Logging plan changes for execution") - // Rewrite for transformed plan - .exclude("dumping query execution info to a file - explainMode=formatted") - - override def getSQLQueryTestSettings: SQLQueryTestSettings = VeloxSQLQueryTestSettings -} -// scalastyle:on line.size.limit diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala deleted file mode 100644 index 70579c88624..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark - -import org.apache.spark.sql.GlutenTestsBaseTrait - -class GlutenSortShuffleSuite extends SortShuffleSuite with GlutenTestsBaseTrait { - override def beforeAll(): Unit = { - super.beforeAll() - conf.set("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager") - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenApproxCountDistinctForIntervalsQuerySuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenApproxCountDistinctForIntervalsQuerySuite.scala deleted file mode 100644 index 86ef1238965..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenApproxCountDistinctForIntervalsQuerySuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenApproxCountDistinctForIntervalsQuerySuite - extends ApproxCountDistinctForIntervalsQuerySuite - with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenApproximatePercentileQuerySuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenApproximatePercentileQuerySuite.scala deleted file mode 100644 index eb82baa78da..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenApproximatePercentileQuerySuite.scala +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenApproximatePercentileQuerySuite - extends ApproximatePercentileQuerySuite - with GlutenSQLTestsTrait { - - override def testFile(fileName: String): String = { - Thread.currentThread().getContextClassLoader.getResource(fileName).toString - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenBloomFilterAggregateQuerySuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenBloomFilterAggregateQuerySuite.scala deleted file mode 100644 index b24b81ed90a..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenBloomFilterAggregateQuerySuite.scala +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -import org.apache.gluten.backendsapi.BackendsApiManager -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.execution.HashAggregateExecBaseTransformer - -import org.apache.spark.SparkConf -import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper -import org.apache.spark.sql.internal.SQLConf - -class GlutenBloomFilterAggregateQuerySuite - extends BloomFilterAggregateQuerySuite - with GlutenSQLTestsTrait - with AdaptiveSparkPlanHelper { - import testImplicits._ - - val veloxBloomFilterMaxNumBits = 4194304L - - testGluten("Test bloom_filter_agg with big RUNTIME_BLOOM_FILTER_MAX_NUM_ITEMS") { - val table = "bloom_filter_test" - withSQLConf( - SQLConf.RUNTIME_BLOOM_FILTER_MAX_NUM_ITEMS.key -> "5000000" - ) { - val numEstimatedItems = 5000000L - val sqlString = s""" - |SELECT every(might_contain( - | (SELECT bloom_filter_agg(col, - | cast($numEstimatedItems as long), - | cast($veloxBloomFilterMaxNumBits as long)) - | FROM $table), - | col)) positive_membership_test - |FROM $table - """.stripMargin - withTempView(table) { - (Seq(Long.MinValue, 0, Long.MaxValue) ++ (1L to 200000L)) - .toDF("col") - .createOrReplaceTempView(table) - checkAnswer(spark.sql(sqlString), Row(true)) - } - } - } - - testGluten("Test that might_contain on bloom_filter_agg with empty input") { - checkAnswer( - spark.sql("""SELECT might_contain((select bloom_filter_agg(cast(id as long)) - | from range(1, 1)), cast(123 as long))""".stripMargin), - Row(null) - ) - - checkAnswer( - spark.sql("""SELECT might_contain((select bloom_filter_agg(cast(id as long)) - | from range(1, 1)), null)""".stripMargin), - Row(null)) - } - - testGluten("Test bloom_filter_agg filter fallback") { - val table = "bloom_filter_test" - val numEstimatedItems = 5000000L - val sqlString = s""" - |SELECT col positive_membership_test - |FROM $table - |WHERE might_contain( - | (SELECT bloom_filter_agg(col, - | cast($numEstimatedItems as long), - | cast($veloxBloomFilterMaxNumBits as long)) - | FROM $table), col) - """.stripMargin - withTempView(table) { - (Seq(Long.MinValue, 0, Long.MaxValue) ++ (1L to 200000L)) - .toDF("col") - .createOrReplaceTempView(table) - withSQLConf( - GlutenConfig.COLUMNAR_PROJECT_ENABLED.key -> "false" - ) { - val df = spark.sql(sqlString) - df.collect - assert( - collectWithSubqueries(df.queryExecution.executedPlan) { - case h if h.isInstanceOf[HashAggregateExecBaseTransformer] => h - }.size == 2, - df.queryExecution.executedPlan - ) - } - if (BackendsApiManager.getSettings.requireBloomFilterAggMightContainJointFallback()) { - withSQLConf( - GlutenConfig.COLUMNAR_FILTER_ENABLED.key -> "false" - ) { - val df = spark.sql(sqlString) - df.collect - assert( - collectWithSubqueries(df.queryExecution.executedPlan) { - case h if h.isInstanceOf[HashAggregateExecBaseTransformer] => h - }.size == 2, - df.queryExecution.executedPlan - ) - } - } - } - } - - testGluten("Test bloom_filter_agg agg fallback") { - val table = "bloom_filter_test" - val numEstimatedItems = 5000000L - val sqlString = s""" - |SELECT col positive_membership_test - |FROM $table - |WHERE might_contain( - | (SELECT bloom_filter_agg(col, - | cast($numEstimatedItems as long), - | cast($veloxBloomFilterMaxNumBits as long)) - | FROM $table), col) - """.stripMargin - - withTempView(table) { - (Seq(Long.MinValue, 0, Long.MaxValue) ++ (1L to 200000L)) - .toDF("col") - .createOrReplaceTempView(table) - withSQLConf( - GlutenConfig.COLUMNAR_HASHAGG_ENABLED.key -> "false" - ) { - val df = spark.sql(sqlString) - df.collect - assert( - collectWithSubqueries(df.queryExecution.executedPlan) { - case h if h.isInstanceOf[HashAggregateExecBaseTransformer] => h - }.isEmpty, - df.queryExecution.executedPlan - ) - } - } - } -} - -class GlutenBloomFilterAggregateQuerySuiteCGOff extends GlutenBloomFilterAggregateQuerySuite { - override def sparkConf: SparkConf = { - super.sparkConf - .set("spark.sql.codegen.wholeStage", "false") - .set("spark.sql.codegen.factoryMode", "NO_CODEGEN") - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCTEHintSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCTEHintSuite.scala deleted file mode 100644 index 8005bffc310..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCTEHintSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenCTEHintSuite extends CTEHintSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCTEInlineSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCTEInlineSuite.scala deleted file mode 100644 index 3a05eda7119..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCTEInlineSuite.scala +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -import org.apache.spark.SparkConf - -class GlutenCTEInlineSuiteAEOff extends CTEInlineSuiteAEOff with GlutenSQLTestsTrait { - override def sparkConf: SparkConf = - super.sparkConf - .set("spark.gluten.sql.columnar.backend.ch.enable.coalesce.project.union", "false") - -} - -class GlutenCTEInlineSuiteAEOn extends CTEInlineSuiteAEOn with GlutenSQLTestsTrait { - override def sparkConf: SparkConf = - super.sparkConf - .set("spark.gluten.sql.columnar.backend.ch.enable.coalesce.project.union", "false") - -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala deleted file mode 100644 index 3042670deab..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -import org.apache.gluten.config.GlutenConfig - -import org.apache.spark.SparkConf -import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper -import org.apache.spark.sql.execution.columnar.InMemoryRelation - -import java.time.LocalDateTime - -class GlutenCachedTableSuite - extends CachedTableSuite - with GlutenSQLTestsTrait - with AdaptiveSparkPlanHelper { - // for temporarily disable the columnar table cache globally. - sys.props.put(GlutenConfig.COLUMNAR_TABLE_CACHE_ENABLED.key, "true") - override def sparkConf: SparkConf = { - super.sparkConf.set("spark.sql.shuffle.partitions", "5") - super.sparkConf.set(GlutenConfig.COLUMNAR_TABLE_CACHE_ENABLED.key, "true") - } - - testGluten("InMemoryRelation statistics") { - sql("CACHE TABLE testData") - spark.table("testData").queryExecution.withCachedData.collect { - case cached: InMemoryRelation => - assert(cached.stats.sizeInBytes === 1130) - } - } - - testGluten("SPARK-36120: Support cache/uncache table with TimestampNTZ type") { - val tableName = "ntzCache" - withTable(tableName) { - sql(s"CACHE TABLE $tableName AS SELECT TIMESTAMP_NTZ'2021-01-01 00:00:00'") - checkAnswer(spark.table(tableName), Row(LocalDateTime.parse("2021-01-01T00:00:00"))) - spark.table(tableName).queryExecution.withCachedData.collect { - case cached: InMemoryRelation => - assert(cached.stats.sizeInBytes === 55) - } - sql(s"UNCACHE TABLE $tableName") - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCharVarcharTestSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCharVarcharTestSuite.scala deleted file mode 100644 index 8c59c323ee2..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCharVarcharTestSuite.scala +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -import org.apache.spark.SparkException - -class GlutenFileSourceCharVarcharTestSuite - extends FileSourceCharVarcharTestSuite - with GlutenSQLTestsTrait { - - private val VELOX_ERROR_MESSAGE = "Exceeds allowed length limitation: 5" - - private def testTableWrite(f: String => Unit): Unit = { - withTable("t")(f("char")) - withTable("t")(f("varchar")) - } - - testGluten("length check for input string values: nested in struct of array") { - testTableWrite { - typeName => - sql(s"CREATE TABLE t(c STRUCT>) USING $format") - sql("INSERT INTO t SELECT struct(array(null))") - checkAnswer(spark.table("t"), Row(Row(Seq(null)))) - val e = intercept[SparkException](sql("INSERT INTO t SELECT struct(array('123456'))")) - assert(e.getCause.getMessage.contains(VELOX_ERROR_MESSAGE)) - } - } -} - -class GlutenDSV2CharVarcharTestSuite extends DSV2CharVarcharTestSuite with GlutenSQLTestsTrait { - - private val VELOX_ERROR_MESSAGE = "Exceeds allowed length limitation: 5" - - private def testTableWrite(f: String => Unit): Unit = { - withTable("t")(f("char")) - withTable("t")(f("varchar")) - } - - testGluten("length check for input string values: nested in struct of array") { - testTableWrite { - typeName => - sql(s"CREATE TABLE t(c STRUCT>) USING $format") - sql("INSERT INTO t SELECT struct(array(null))") - checkAnswer(spark.table("t"), Row(Row(Seq(null)))) - val e = intercept[SparkException](sql("INSERT INTO t SELECT struct(array('123456'))")) - assert(e.getCause.getMessage.contains(VELOX_ERROR_MESSAGE)) - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenColumnExpressionSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenColumnExpressionSuite.scala deleted file mode 100644 index f525bab61a7..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenColumnExpressionSuite.scala +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -import org.apache.gluten.config.GlutenConfig - -import org.apache.spark.SparkException -import org.apache.spark.sql.execution.ProjectExec -import org.apache.spark.sql.functions.{assert_true, expr, input_file_name, lit, raise_error} - -class GlutenColumnExpressionSuite extends ColumnExpressionSuite with GlutenSQLTestsTrait { - import testImplicits._ - testGluten("raise_error") { - val strDf = Seq(("hello")).toDF("a") - - val e1 = intercept[SparkException] { - strDf.select(raise_error(lit(null.asInstanceOf[String]))).collect() - } - assert(e1.getCause.isInstanceOf[RuntimeException]) - - val e2 = intercept[SparkException] { - strDf.select(raise_error($"a")).collect() - } - assert(e2.getCause.isInstanceOf[RuntimeException]) - assert(e2.getCause.getMessage.contains("hello")) - } - - testGluten("assert_true") { - // assert_true(condition, errMsgCol) - val booleanDf = Seq((true), (false)).toDF("cond") - checkAnswer( - booleanDf.filter("cond = true").select(assert_true($"cond")), - Row(null) :: Nil - ) - val e1 = intercept[SparkException] { - booleanDf.select(assert_true($"cond", lit(null.asInstanceOf[String]))).collect() - } - assert(e1.getCause.isInstanceOf[RuntimeException]) - - val nullDf = Seq(("first row", None), ("second row", Some(true))).toDF("n", "cond") - checkAnswer( - nullDf.filter("cond = true").select(assert_true($"cond", $"cond")), - Row(null) :: Nil - ) - val e2 = intercept[SparkException] { - nullDf.select(assert_true($"cond", $"n")).collect() - } - assert(e2.getCause.isInstanceOf[RuntimeException]) - assert(e2.getCause.getMessage.contains("first row")) - - // assert_true(condition) - val intDf = Seq((0, 1)).toDF("a", "b") - checkAnswer(intDf.select(assert_true($"a" < $"b")), Row(null) :: Nil) - val e3 = intercept[SparkException] { - intDf.select(assert_true($"a" > $"b")).collect() - } - assert(e3.getCause.isInstanceOf[RuntimeException]) - assert(e3.getCause.getMessage.contains("'('a > 'b)' is not true!")) - } - - testGluten( - "input_file_name, input_file_block_start and input_file_block_length " + - "should fall back if scan falls back") { - withSQLConf((GlutenConfig.COLUMNAR_FILESCAN_ENABLED.key, "false")) { - withTempPath { - dir => - val data = sparkContext.parallelize(0 to 10).toDF("id") - data.write.parquet(dir.getCanonicalPath) - - val q = - spark.read - .parquet(dir.getCanonicalPath) - .select( - input_file_name(), - expr("input_file_block_start()"), - expr("input_file_block_length()")) - val firstRow = q.head() - assert(firstRow.getString(0).contains(dir.toURI.getPath)) - assert(firstRow.getLong(1) == 0) - assert(firstRow.getLong(2) > 0) - val project = q.queryExecution.executedPlan.collect { case p: ProjectExec => p } - assert(project.size == 1) - } - } - } - - test("scan filter references input_file_name but project does not select it") { - withTempPath { - dir => - val data = sparkContext.parallelize(0 to 10).toDF("id") - data.write.parquet(dir.getCanonicalPath) - - // Filter references input_file_name(), but the project only selects `id`. - val q = spark.read - .parquet(dir.getCanonicalPath) - .filter(input_file_name().contains("parquet")) - .select($"id") - checkAnswer(q, (0 to 10).map(Row(_))) - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenComplexTypesSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenComplexTypesSuite.scala deleted file mode 100644 index 835b7ecfd88..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenComplexTypesSuite.scala +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenComplexTypesSuite extends ComplexTypesSuite with GlutenSQLTestsTrait { - - override def beforeAll(): Unit = { - super.beforeAll() - spark - .range(10) - .selectExpr( - "(id % 2 = 0) as bool", - "cast(id as BYTE) as i8", - "cast(id as SHORT) as i16", - "cast(id as FLOAT) as fp32", - "cast(id as DOUBLE) as fp64", - "cast(id as DECIMAL(4, 2)) as dec", - "cast(cast(id as BYTE) as BINARY) as vbin", - "binary(id) as vbin1", - "map_from_arrays(array(id),array(id+2)) as map", - "array(id, id+1, id+2) as list", - "struct(cast(id as LONG) as a, cast(id+1 as STRING) as b) as struct" - ) - .write - .saveAsTable("tab_types") - } - - override def afterAll(): Unit = { - try { - spark.sql("DROP TABLE IF EXISTS tab_types") - } finally { - super.afterAll() - } - } - - testGluten("types bool/byte/short/float/double/decimal/binary/map/array/struct") { - val df = spark - .table("tab_types") - .selectExpr( - "bool", - "i8", - "i16", - "fp32", - "fp64", - "dec", - "vbin", - "length(vbin)", - "vbin1", - "length(vbin1)", - "struct", - "struct.a", - "list", - "map" - ) - .sort("i8") - .limit(1) - - checkAnswer( - df, - Seq( - Row( - true, - 0.toByte, - 0.toShort, - 0.toFloat, - 0.toDouble, - BigDecimal(0), - Array.fill[Byte](1)(0.toByte), - 1.toInt, - Array.fill[Byte](8)(0.toByte), - 8.toInt, - Row(0.toLong, "1"), - 0.toLong, - Array(0, 1, 2), - Map(0 -> 2) - )) - ) - - checkNamedStruct(df.queryExecution.optimizedPlan, expectedCount = 0) - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenConfigBehaviorSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenConfigBehaviorSuite.scala deleted file mode 100644 index c1984a5e22d..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenConfigBehaviorSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenConfigBehaviorSuite extends ConfigBehaviorSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCountMinSketchAggQuerySuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCountMinSketchAggQuerySuite.scala deleted file mode 100644 index 182464c0a5e..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCountMinSketchAggQuerySuite.scala +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -/** End-to-end test suite for count_min_sketch. */ -class GlutenCountMinSketchAggQuerySuite - extends CountMinSketchAggQuerySuite - with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCsvFunctionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCsvFunctionsSuite.scala deleted file mode 100644 index 0550fef442f..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCsvFunctionsSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenCsvFunctionsSuite extends CsvFunctionsSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameAggregateSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameAggregateSuite.scala deleted file mode 100644 index 2f3777caa17..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameAggregateSuite.scala +++ /dev/null @@ -1,283 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.execution.HashAggregateExecBaseTransformer - -import org.apache.spark.sql.execution.WholeStageCodegenExec -import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, SortAggregateExec} -import org.apache.spark.sql.expressions.Aggregator -import org.apache.spark.sql.functions._ -import org.apache.spark.sql.internal.SQLConf - -import java.lang.{Long => JLong} - -import scala.util.Random - -class GlutenDataFrameAggregateSuite extends DataFrameAggregateSuite with GlutenSQLTestsTrait { - - import testImplicits._ - - // blackTestNameList is defined in ClickHouseNotSupport - - testGluten("count") { - // agg with no input col - assert(testData2.count() === testData2.rdd.map(_ => 1).count()) - - checkAnswer( - testData2.agg(count($"a"), sum_distinct($"a")), // non-partial - Row(6, 6.0)) - } - - testGluten("null count") { - checkAnswer(testData3.groupBy($"a").agg(count($"b")), Seq(Row(1, 0), Row(2, 1))) - - checkAnswer(testData3.groupBy($"a").agg(count($"a" + $"b")), Seq(Row(1, 0), Row(2, 1))) - - checkAnswer( - testData3 - .agg(count($"a"), count($"b"), count(lit(1)), count_distinct($"a"), count_distinct($"b")), - Row(2, 1, 2, 2, 1)) - - // [wishlist] does not support sum distinct -// checkAnswer( -// testData3.agg(count($"b"), count_distinct($"b"), sum_distinct($"b")), // non-partial -// Row(1, 1, 2) -// ) - } - - testGluten("groupBy") { - checkAnswer(testData2.groupBy("a").agg(sum($"b")), Seq(Row(1, 3), Row(2, 3), Row(3, 3))) - checkAnswer(testData2.groupBy("a").agg(sum($"b").as("totB")).agg(sum($"totB")), Row(9)) - checkAnswer(testData2.groupBy("a").agg(count("*")), Row(1, 2) :: Row(2, 2) :: Row(3, 2) :: Nil) - checkAnswer( - testData2.groupBy("a").agg(Map("*" -> "count")), - Row(1, 2) :: Row(2, 2) :: Row(3, 2) :: Nil) - checkAnswer( - testData2.groupBy("a").agg(Map("b" -> "sum")), - Row(1, 3) :: Row(2, 3) :: Row(3, 3) :: Nil) - - val df1 = Seq(("a", 1, 0, "b"), ("b", 2, 4, "c"), ("a", 2, 3, "d")) - .toDF("key", "value1", "value2", "rest") - - checkAnswer(df1.groupBy("key").min(), df1.groupBy("key").min("value1", "value2").collect()) - checkAnswer(df1.groupBy("key").min("value2"), Seq(Row("a", 0), Row("b", 4))) - - // [wishlist] does not support decimal -// checkAnswer( -// decimalData.groupBy("a").agg(sum("b")), -// Seq(Row(new java.math.BigDecimal(1), new java.math.BigDecimal(3)), -// Row(new java.math.BigDecimal(2), new java.math.BigDecimal(3)), -// Row(new java.math.BigDecimal(3), new java.math.BigDecimal(3))) -// ) -// -// val decimalDataWithNulls = spark.sparkContext.parallelize( -// DecimalData(1, 1) :: -// DecimalData(1, null) :: -// DecimalData(2, 1) :: -// DecimalData(2, null) :: -// DecimalData(3, 1) :: -// DecimalData(3, 2) :: -// DecimalData(null, 2) :: Nil).toDF() -// checkAnswer( -// decimalDataWithNulls.groupBy("a").agg(sum("b")), -// Seq(Row(new java.math.BigDecimal(1), new java.math.BigDecimal(1)), -// Row(new java.math.BigDecimal(2), new java.math.BigDecimal(1)), -// Row(new java.math.BigDecimal(3), new java.math.BigDecimal(3)), -// Row(null, new java.math.BigDecimal(2))) -// ) - } - - testGluten("average") { - - checkAnswer(testData2.agg(avg($"a"), mean($"a")), Row(2.0, 2.0)) - - checkAnswer( - testData2.agg(avg($"a"), sum_distinct($"a")), // non-partial and test deprecated version - Row(2.0, 6.0) :: Nil) - - // [wishlist] does not support decimal -// checkAnswer( -// decimalData.agg(avg($"a")), -// Row(new java.math.BigDecimal(2))) -// -// checkAnswer( -// decimalData.agg(avg($"a"), sum_distinct($"a")), // non-partial -// Row(new java.math.BigDecimal(2), new java.math.BigDecimal(6)) :: Nil) -// -// checkAnswer( -// decimalData.agg(avg($"a" cast DecimalType(10, 2))), -// Row(new java.math.BigDecimal(2))) -// // non-partial -// checkAnswer( -// decimalData.agg( -// avg($"a" cast DecimalType(10, 2)), sum_distinct($"a" cast DecimalType(10, 2))), -// Row(new java.math.BigDecimal(2), new java.math.BigDecimal(6)) :: Nil) - } - - ignoreGluten("SPARK-32038: NormalizeFloatingNumbers should work on distinct aggregate") { - withTempView("view") { - Seq( - ("mithunr", Float.NaN), - ("mithunr", Float.NaN), - ("mithunr", Float.NaN), - ("abellina", 1.0f), - ("abellina", 2.0f)).toDF("uid", "score").createOrReplaceTempView("view") - - val df = spark.sql("select uid, count(distinct score) from view group by 1 order by 1 asc") - checkAnswer(df, Row("abellina", 2) :: Row("mithunr", 1) :: Nil) - } - } - - testGluten("variance") { - checkAnswer( - testData2.agg(var_samp($"a"), var_pop($"a"), variance($"a")), - Row(0.8, 2.0 / 3.0, 0.8)) - checkAnswer(testData2.agg(var_samp("a"), var_pop("a"), variance("a")), Row(0.8, 2.0 / 3.0, 0.8)) - } - - testGluten("aggregation with filter") { - Seq( - ("mithunr", 12.3f, 5.0f, true, 9.4f), - ("mithunr", 15.5f, 4.0f, false, 19.9f), - ("mithunr", 19.8f, 3.0f, false, 35.6f), - ("abellina", 20.1f, 2.0f, true, 98.0f), - ("abellina", 20.1f, 1.0f, true, 0.5f), - ("abellina", 23.6f, 2.0f, true, 3.9f) - ) - .toDF("uid", "time", "score", "pass", "rate") - .createOrReplaceTempView("view") - var df = spark.sql("select count(score) filter (where pass) from view group by time") - checkAnswer(df, Row(1) :: Row(0) :: Row(0) :: Row(2) :: Row(1) :: Nil) - - df = spark.sql("select count(score) filter (where pass) from view") - checkAnswer(df, Row(4) :: Nil) - - df = spark.sql("select count(score) filter (where rate > 20) from view group by time") - checkAnswer(df, Row(0) :: Row(0) :: Row(1) :: Row(1) :: Row(0) :: Nil) - - df = spark.sql("select count(score) filter (where rate > 20) from view") - checkAnswer(df, Row(2) :: Nil) - } - - testGluten("extend with cast expression") { - checkAnswer( - decimalData.agg( - sum($"a".cast("double")), - avg($"b".cast("double")), - count_distinct($"a"), - count_distinct($"b")), - Row(12.0, 1.5, 3, 2)) - } - - // This test is applicable to velox backend. For CH backend, the replacement is disabled. - testGluten("use gluten hash agg to replace vanilla spark sort agg") { - - withSQLConf((GlutenConfig.COLUMNAR_FORCE_HASHAGG_ENABLED.key, "false")) { - Seq("A", "B", "C", "D").toDF("col1").createOrReplaceTempView("t1") - // SortAggregateExec is expected to be used for string type input. - val df = spark.sql("select max(col1) from t1") - checkAnswer(df, Row("D") :: Nil) - assert(find(df.queryExecution.executedPlan)(_.isInstanceOf[SortAggregateExec]).isDefined) - } - - withSQLConf((GlutenConfig.COLUMNAR_FORCE_HASHAGG_ENABLED.key, "true")) { - Seq("A", "B", "C", "D").toDF("col1").createOrReplaceTempView("t1") - val df = spark.sql("select max(col1) from t1") - checkAnswer(df, Row("D") :: Nil) - // Sort agg is expected to be replaced by gluten's hash agg. - assert( - find(df.queryExecution.executedPlan)( - _.isInstanceOf[HashAggregateExecBaseTransformer]).isDefined) - } - } - - testGluten("mixed supported and unsupported aggregate functions") { - withUserDefinedFunction(("udaf_sum", true)) { - spark.udf.register( - "udaf_sum", - udaf(new Aggregator[JLong, JLong, JLong] { - override def zero: JLong = 0 - override def reduce(b: JLong, a: JLong): JLong = a + b - override def merge(b1: JLong, b2: JLong): JLong = b1 + b2 - override def finish(reduction: JLong): JLong = reduction - override def bufferEncoder: Encoder[JLong] = Encoders.LONG - override def outputEncoder: Encoder[JLong] = Encoders.LONG - }) - ) - - val df = spark.sql("SELECT a, udaf_sum(b), max(b) FROM testData2 group by a") - checkAnswer(df, Row(1, 3, 2) :: Row(2, 3, 2) :: Row(3, 3, 2) :: Nil) - } - } - - // Ported from spark DataFrameAggregateSuite only with plan check changed. - private def assertNoExceptions(c: Column): Unit = { - for ( - (wholeStage, useObjectHashAgg) <- - Seq((true, true), (true, false), (false, true), (false, false)) - ) { - withSQLConf( - (SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, wholeStage.toString), - (SQLConf.USE_OBJECT_HASH_AGG.key, useObjectHashAgg.toString)) { - - val df = Seq(("1", 1), ("1", 2), ("2", 3), ("2", 4)).toDF("x", "y") - - // test case for HashAggregate - val hashAggDF = df.groupBy("x").agg(c, sum("y")) - hashAggDF.collect() - val hashAggPlan = hashAggDF.queryExecution.executedPlan - if (wholeStage) { - assert(find(hashAggPlan) { - case WholeStageCodegenExec(_: HashAggregateExec) => true - // If offloaded, spark whole stage codegen takes no effect and a gluten hash agg is - // expected to be used. - case _: HashAggregateExecBaseTransformer => true - case _ => false - }.isDefined) - } else { - assert( - stripAQEPlan(hashAggPlan).isInstanceOf[HashAggregateExec] || - stripAQEPlan(hashAggPlan).find { - case _: HashAggregateExecBaseTransformer => true - case _ => false - }.isDefined) - } - - // test case for ObjectHashAggregate and SortAggregate - val objHashAggOrSortAggDF = df.groupBy("x").agg(c, collect_list("y")) - objHashAggOrSortAggDF.collect() - assert(stripAQEPlan(objHashAggOrSortAggDF.queryExecution.executedPlan).find { - case _: HashAggregateExecBaseTransformer => true - case _ => false - }.isDefined) - } - } - } - - testGluten( - "SPARK-19471: AggregationIterator does not initialize the generated" + - " result projection before using it") { - Seq( - monotonically_increasing_id(), - spark_partition_id(), - rand(Random.nextLong()), - randn(Random.nextLong()) - ).foreach(assertNoExceptions) - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameAsOfJoinSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameAsOfJoinSuite.scala deleted file mode 100644 index 9367fab17f2..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameAsOfJoinSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenDataFrameAsOfJoinSuite extends DataFrameAsOfJoinSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameComplexTypeSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameComplexTypeSuite.scala deleted file mode 100644 index 7464968cba5..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameComplexTypeSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenDataFrameComplexTypeSuite extends DataFrameComplexTypeSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala deleted file mode 100644 index e64f760ab55..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -import org.apache.spark.sql.functions._ - -class GlutenDataFrameFunctionsSuite extends DataFrameFunctionsSuite with GlutenSQLTestsTrait { - import testImplicits._ - - testGluten("map_zip_with function - map of primitive types") { - val df = Seq( - (Map(8 -> 6L, 3 -> 5L, 6 -> 2L), Map[Integer, Integer]((6, 4), (8, 2), (3, 2))), - (Map(10 -> 6L, 8 -> 3L), Map[Integer, Integer]((8, 4), (4, null))), - (Map.empty[Int, Long], Map[Integer, Integer]((5, 1))), - (Map(5 -> 1L), null) - ).toDF("m1", "m2") - - GlutenQueryTestUtil.sameRows( - df.selectExpr("map_zip_with(m1, m2, (k, v1, v2) -> k == v1 + v2)").collect.toSeq, - Seq( - Row(Map(8 -> true, 3 -> false, 6 -> true)), - Row(Map(10 -> null, 8 -> false, 4 -> null)), - Row(Map(5 -> null)), - Row(null)), - false - ) - - GlutenQueryTestUtil.sameRows( - df.select(map_zip_with(df("m1"), df("m2"), (k, v1, v2) => k === v1 + v2)).collect.toSeq, - Seq( - Row(Map(8 -> true, 3 -> false, 6 -> true)), - Row(Map(10 -> null, 8 -> false, 4 -> null)), - Row(Map(5 -> null)), - Row(null)), - false - ) - } - - testGluten("flatten function") { - // Test cases with a primitive type - val intDF = Seq( - (Seq(Seq(1, 2, 3), Seq(4, 5), Seq(6))), - (Seq(Seq(1, 2))), - (Seq(Seq(1), Seq.empty)), - (Seq(Seq.empty, Seq(1))) - ).toDF("i") - - val intDFResult = Seq(Row(Seq(1, 2, 3, 4, 5, 6)), Row(Seq(1, 2)), Row(Seq(1)), Row(Seq(1))) - - def testInt(): Unit = { - checkAnswer(intDF.select(flatten($"i")), intDFResult) - checkAnswer(intDF.selectExpr("flatten(i)"), intDFResult) - } - - // Test with local relation, the Project will be evaluated without codegen - testInt() - // Test with cached relation, the Project will be evaluated with codegen - intDF.cache() - testInt() - - // Test cases with non-primitive types - val strDF = Seq( - (Seq(Seq("a", "b"), Seq("c"), Seq("d", "e", "f"))), - (Seq(Seq("a", "b"))), - (Seq(Seq("a", null), Seq(null, "b"), Seq(null, null))), - (Seq(Seq("a"), Seq.empty)), - (Seq(Seq.empty, Seq("a"))) - ).toDF("s") - - val strDFResult = Seq( - Row(Seq("a", "b", "c", "d", "e", "f")), - Row(Seq("a", "b")), - Row(Seq("a", null, null, "b", null, null)), - Row(Seq("a")), - Row(Seq("a"))) - - def testString(): Unit = { - checkAnswer(strDF.select(flatten($"s")), strDFResult) - checkAnswer(strDF.selectExpr("flatten(s)"), strDFResult) - } - - // Test with local relation, the Project will be evaluated without codegen - testString() - // Test with cached relation, the Project will be evaluated with codegen - strDF.cache() - testString() - - val arrDF = Seq((1, "a", Seq(1, 2, 3))).toDF("i", "s", "arr") - - def testArray(): Unit = { - checkAnswer( - arrDF.selectExpr("flatten(array(arr, array(null, 5), array(6, null)))"), - Seq(Row(Seq(1, 2, 3, null, 5, 6, null)))) - checkAnswer( - arrDF.selectExpr("flatten(array(array(arr, arr), array(arr)))"), - Seq(Row(Seq(Seq(1, 2, 3), Seq(1, 2, 3), Seq(1, 2, 3))))) - } - - // Test with local relation, the Project will be evaluated without codegen - testArray() - // Test with cached relation, the Project will be evaluated with codegen - arrDF.cache() - testArray() - - // Error test cases - val oneRowDF = Seq((1, "a", Seq(1, 2, 3))).toDF("i", "s", "arr") - intercept[AnalysisException] { - oneRowDF.select(flatten($"arr")) - } - intercept[AnalysisException] { - oneRowDF.select(flatten($"i")) - } - intercept[AnalysisException] { - oneRowDF.select(flatten($"s")) - } - intercept[AnalysisException] { - oneRowDF.selectExpr("flatten(null)") - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameHintSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameHintSuite.scala deleted file mode 100644 index 663a6111b0d..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameHintSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenDataFrameHintSuite extends DataFrameHintSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameImplicitsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameImplicitsSuite.scala deleted file mode 100644 index 2a6e367bc08..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameImplicitsSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenDataFrameImplicitsSuite extends DataFrameImplicitsSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameJoinSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameJoinSuite.scala deleted file mode 100644 index 6581d7f2d88..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameJoinSuite.scala +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenDataFrameJoinSuite extends DataFrameJoinSuite with GlutenSQLTestsTrait { - - override def testNameBlackList: Seq[String] = Seq( - "Supports multi-part names for broadcast hint resolution" - ) -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameNaFunctionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameNaFunctionsSuite.scala deleted file mode 100644 index 424087c8de8..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameNaFunctionsSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenDataFrameNaFunctionsSuite extends DataFrameNaFunctionsSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFramePivotSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFramePivotSuite.scala deleted file mode 100644 index e1b91d71997..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFramePivotSuite.scala +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -import org.apache.spark.sql.functions._ - -class GlutenDataFramePivotSuite extends DataFramePivotSuite with GlutenSQLTestsTrait { - - // This test is ported from vanilla spark with pos value (1-based) changed from 0 to 1 for - // substring. In vanilla spark, pos=0 has same effectiveness as pos=1. But in velox, pos=0 - // will return an empty string as substring result. - testGluten("pivot with column definition in groupby - using pos=1") { - val df = courseSales - .groupBy(substring(col("course"), 1, 1).as("foo")) - .pivot("year", Seq(2012, 2013)) - .sum("earnings") - .queryExecution - .executedPlan - - checkAnswer( - courseSales - .groupBy(substring(col("course"), 1, 1).as("foo")) - .pivot("year", Seq(2012, 2013)) - .sum("earnings"), - Row("d", 15000.0, 48000.0) :: Row("J", 20000.0, 30000.0) :: Nil - ) - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameRangeSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameRangeSuite.scala deleted file mode 100644 index e8a424de5be..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameRangeSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenDataFrameRangeSuite extends DataFrameRangeSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameSelfJoinSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameSelfJoinSuite.scala deleted file mode 100644 index 61cc4bc4c08..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameSelfJoinSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenDataFrameSelfJoinSuite extends DataFrameSelfJoinSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameSessionWindowingSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameSessionWindowingSuite.scala deleted file mode 100644 index d76d8b21cdc..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameSessionWindowingSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenDataFrameSessionWindowingSuite - extends DataFrameSessionWindowingSuite - with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameSetOperationsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameSetOperationsSuite.scala deleted file mode 100644 index fe7958b6777..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameSetOperationsSuite.scala +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -import org.apache.spark.SparkConf - -class GlutenDataFrameSetOperationsSuite - extends DataFrameSetOperationsSuite - with GlutenSQLTestsTrait { - override def sparkConf: SparkConf = - super.sparkConf - .set("spark.gluten.sql.columnar.backend.ch.enable.coalesce.project.union", "false") - .set("spark.gluten.sql.columnar.backend.ch.enable.coalesce.aggregation.union", "false") - -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameStatSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameStatSuite.scala deleted file mode 100644 index bab8e9b83cb..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameStatSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenDataFrameStatSuite extends DataFrameStatSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameSuite.scala deleted file mode 100644 index b7cdea9fb8d..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameSuite.scala +++ /dev/null @@ -1,424 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -import org.apache.gluten.execution.{ProjectExecTransformer, WholeStageTransformer} - -import org.apache.spark.SparkException -import org.apache.spark.sql.catalyst.expressions.{EqualTo, Expression} -import org.apache.spark.sql.execution.ColumnarShuffleExchangeExec -import org.apache.spark.sql.execution.aggregate.HashAggregateExec -import org.apache.spark.sql.execution.exchange.{ReusedExchangeExec, ShuffleExchangeExec} -import org.apache.spark.sql.functions._ -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.test.SQLTestData.TestData2 -import org.apache.spark.sql.types.StringType - -import java.io.ByteArrayOutputStream -import java.nio.charset.StandardCharsets - -import scala.util.Random - -class GlutenDataFrameSuite extends DataFrameSuite with GlutenSQLTestsTrait { - - testGluten("repartitionByRange") { - val partitionNum = 10 - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", - SQLConf.SHUFFLE_PARTITIONS.key -> partitionNum.toString) { - import testImplicits._ - val data1d = Random.shuffle(0.to(partitionNum - 1)) - val data2d = data1d.map(i => (i, data1d.size - i)) - - checkAnswer( - data1d - .toDF("val") - .repartitionByRange(data1d.size, $"val".asc) - .select(spark_partition_id().as("id"), $"val"), - data1d.map(i => Row(i, i))) - - checkAnswer( - data1d - .toDF("val") - .repartitionByRange(data1d.size, $"val".desc) - .select(spark_partition_id().as("id"), $"val"), - data1d.map(i => Row(i, data1d.size - 1 - i))) - - checkAnswer( - data1d - .toDF("val") - .repartitionByRange(data1d.size, lit(42)) - .select(spark_partition_id().as("id"), $"val"), - data1d.map(i => Row(0, i))) - - checkAnswer( - data1d - .toDF("val") - .repartitionByRange(data1d.size, lit(null), $"val".asc, rand()) - .select(spark_partition_id().as("id"), $"val"), - data1d.map(i => Row(i, i))) - - // .repartitionByRange() assumes .asc by default if no explicit sort order is specified - checkAnswer( - data2d - .toDF("a", "b") - .repartitionByRange(data2d.size, $"a".desc, $"b") - .select(spark_partition_id().as("id"), $"a", $"b"), - data2d - .toDF("a", "b") - .repartitionByRange(data2d.size, $"a".desc, $"b".asc) - .select(spark_partition_id().as("id"), $"a", $"b") - ) - - // at least one partition-by expression must be specified - intercept[IllegalArgumentException] { - data1d.toDF("val").repartitionByRange(data1d.size) - } - intercept[IllegalArgumentException] { - data1d.toDF("val").repartitionByRange(data1d.size, Seq.empty: _*) - } - } - } - - testGluten("distributeBy and localSort") { - import testImplicits._ - val data = spark.sparkContext.parallelize((1 to 100).map(i => TestData2(i % 10, i))).toDF() - - /** partitionNum = 1 */ - var partitionNum = 1 - val original = testData.repartition(partitionNum) - assert(original.rdd.partitions.length == partitionNum) - - // Distribute into one partition and order by. This partition should contain all the values. - val df6 = data.repartition(partitionNum, $"a").sortWithinPartitions("b") - // Walk each partition and verify that it is sorted ascending and not globally sorted. - df6.rdd.foreachPartition { - p => - var previousValue: Int = -1 - var allSequential: Boolean = true - p.foreach { - r => - val v: Int = r.getInt(1) - if (previousValue != -1) { - if (previousValue > v) throw new SparkException("Partition is not ordered.") - if (v - 1 != previousValue) allSequential = false - } - previousValue = v - } - if (!allSequential) { - throw new SparkException("Partition should contain all sequential values") - } - } - - /** partitionNum = 5 */ - partitionNum = 5 - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", - SQLConf.SHUFFLE_PARTITIONS.key -> partitionNum.toString) { - val df = original.repartition(partitionNum, $"key") - assert(df.rdd.partitions.length == partitionNum) - checkAnswer(original.select(), df.select()) - - // Distribute and order by. - val df4 = data.repartition(partitionNum, $"a").sortWithinPartitions($"b".desc) - // Walk each partition and verify that it is sorted descending and does not contain all - // the values. - df4.rdd.foreachPartition { - p => - // Skip empty partition - if (p.hasNext) { - var previousValue: Int = -1 - var allSequential: Boolean = true - p.foreach { - r => - val v: Int = r.getInt(1) - if (previousValue != -1) { - if (previousValue < v) throw new SparkException("Partition is not ordered.") - if (v + 1 != previousValue) allSequential = false - } - previousValue = v - } - if (allSequential) throw new SparkException("Partition should not be globally ordered") - } - } - } - - /** partitionNum = 10 */ - partitionNum = 10 - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", - SQLConf.SHUFFLE_PARTITIONS.key -> partitionNum.toString) { - val df2 = original.repartition(partitionNum, $"key") - assert(df2.rdd.partitions.length == partitionNum) - checkAnswer(original.select(), df2.select()) - } - - // Group by the column we are distributed by. This should generate a plan with no exchange - // between the aggregates - val df3 = testData.repartition($"key").groupBy("key").count() - verifyNonExchangingAgg(df3) - verifyNonExchangingAgg( - testData - .repartition($"key", $"value") - .groupBy("key", "value") - .count()) - - // Grouping by just the first distributeBy expr, need to exchange. - verifyExchangingAgg( - testData - .repartition($"key", $"value") - .groupBy("key") - .count()) - - /** partitionNum = 2 */ - partitionNum = 2 - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", - SQLConf.SHUFFLE_PARTITIONS.key -> partitionNum.toString) { - // Distribute and order by with multiple order bys - val df5 = data.repartition(partitionNum, $"a").sortWithinPartitions($"b".asc, $"a".asc) - // Walk each partition and verify that it is sorted ascending - df5.rdd.foreachPartition { - p => - var previousValue: Int = -1 - var allSequential: Boolean = true - p.foreach { - r => - val v: Int = r.getInt(1) - if (previousValue != -1) { - if (previousValue > v) throw new SparkException("Partition is not ordered.") - if (v - 1 != previousValue) allSequential = false - } - previousValue = v - } - if (allSequential) throw new SparkException("Partition should not be all sequential") - } - } - } - - testGluten("reuse exchange") { - withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "2") { - val df = spark.range(100).toDF() - val join = df.join(df, "id") - val plan = join.queryExecution.executedPlan - checkAnswer(join, df) - assert(collect(join.queryExecution.executedPlan) { - // replace ShuffleExchangeExec - case e: ColumnarShuffleExchangeExec => true - }.size === 1) - assert(collect(join.queryExecution.executedPlan) { - case e: ReusedExchangeExec => true - }.size === 1) - val broadcasted = broadcast(join) - val join2 = join.join(broadcasted, "id").join(broadcasted, "id") - checkAnswer(join2, df) - assert(collect(join2.queryExecution.executedPlan) { - // replace ShuffleExchangeExec - case e: ColumnarShuffleExchangeExec => true - }.size == 1) - assert(collect(join2.queryExecution.executedPlan) { - case e: ReusedExchangeExec => true - }.size == 4) - } - } - - /** Failed to check WholeStageCodegenExec, so we rewrite the UT. */ - testGluten("SPARK-22520: support code generation for large CaseWhen") { - import org.apache.spark.sql.catalyst.dsl.expressions.StringToAttributeConversionHelper - val N = 30 - var expr1 = when(equalizer($"id", lit(0)), 0) - var expr2 = when(equalizer($"id", lit(0)), 10) - (1 to N).foreach { - i => - expr1 = expr1.when(equalizer($"id", lit(i)), -i) - expr2 = expr2.when(equalizer($"id", lit(i + 10)), i) - } - val df = spark.range(1).select(expr1, expr2.otherwise(0)) - checkAnswer(df, Row(0, 10) :: Nil) - // We check WholeStageTransformer instead of WholeStageCodegenExec - assert(df.queryExecution.executedPlan.find(_.isInstanceOf[WholeStageTransformer]).isDefined) - } - - import testImplicits._ - - private lazy val person2: DataFrame = Seq( - ("Bob", 16, 176), - ("Alice", 32, 164), - ("David", 60, 192), - ("Amy", 24, 180)).toDF("name", "age", "height") - - testGluten("describe") { - val describeResult = Seq( - Row("count", "4", "4", "4"), - Row("mean", null, "33.0", "178.0"), - Row("stddev", null, "19.148542155126762", "11.547005383792516"), - Row("min", "Alice", "16", "164"), - Row("max", "David", "60", "192") - ) - - val emptyDescribeResult = Seq( - Row("count", "0", "0", "0"), - Row("mean", null, null, null), - Row("stddev", null, null, null), - Row("min", null, null, null), - Row("max", null, null, null)) - - val aggResult = Seq( - Row("4", "33.0", "19.148542155126762", "16", "60") - ) - - def getSchemaAsSeq(df: DataFrame): Seq[String] = df.schema.map(_.name) - - Seq("true", "false").foreach { - ansiEnabled => - withSQLConf(SQLConf.ANSI_ENABLED.key -> ansiEnabled) { - val describeAllCols = person2.describe() - assert(getSchemaAsSeq(describeAllCols) === Seq("summary", "name", "age", "height")) - checkAnswer(describeAllCols, describeResult) - // All aggregate value should have been cast to string - describeAllCols.collect().foreach { - row => - row.toSeq.foreach { - value => - if (value != null) { - assert( - value.isInstanceOf[String], - "expected string but found " + value.getClass) - } - } - } - - val describeOneCol = person2.describe("age") - assert(getSchemaAsSeq(describeOneCol) === Seq("summary", "age")) - val aggOneCol = person2.agg( - count("age").cast(StringType), - avg("age").cast(StringType), - stddev_samp("age").cast(StringType), - min("age").cast(StringType), - max("age").cast(StringType) - ) - checkAnswer(aggOneCol, aggResult) - - val describeNoCol = person2.select().describe() - assert(getSchemaAsSeq(describeNoCol) === Seq("summary")) - checkAnswer(describeNoCol, describeResult.map { case Row(s, _, _, _) => Row(s) }) - - val emptyDescription = person2.limit(0).describe() - assert(getSchemaAsSeq(emptyDescription) === Seq("summary", "name", "age", "height")) - checkAnswer(emptyDescription, emptyDescribeResult) - } - } - } - - testGluten("Allow leading/trailing whitespace in string before casting") { - def checkResult(df: DataFrame, expectedResult: Seq[Row]): Unit = { - checkAnswer(df, expectedResult) - assert(find(df.queryExecution.executedPlan)(_.isInstanceOf[ProjectExecTransformer]).isDefined) - } - - // scalastyle:off nonascii - Seq(" 123", "123 ", " 123 ", "\u2000123\n\n\n", "123\r\r\r", "123\f\f\f", "123\u000C") - .toDF("col1") - .createOrReplaceTempView("t1") - // scalastyle:on nonascii - val expectedIntResult = Row(123) :: Row(123) :: - Row(123) :: Row(123) :: Row(123) :: Row(123) :: Row(123) :: Nil - var df = spark.sql("select cast(col1 as int) from t1") - checkResult(df, expectedIntResult) - df = spark.sql("select cast(col1 as long) from t1") - checkResult(df, expectedIntResult) - - Seq(" 123.5", "123.5 ", " 123.5 ", "123.5\n\n\n", "123.5\r\r\r", "123.5\f\f\f", "123.5\u000C") - .toDF("col1") - .createOrReplaceTempView("t1") - val expectedFloatResult = Row(123.5) :: Row(123.5) :: - Row(123.5) :: Row(123.5) :: Row(123.5) :: Row(123.5) :: Row(123.5) :: Nil - df = spark.sql("select cast(col1 as float) from t1") - checkResult(df, expectedFloatResult) - df = spark.sql("select cast(col1 as double) from t1") - checkResult(df, expectedFloatResult) - - // scalastyle:off nonascii - val rawData = - Seq(" abc", "abc ", " abc ", "\u2000abc\n\n\n", "abc\r\r\r", "abc\f\f\f", "abc\u000C") - // scalastyle:on nonascii - rawData.toDF("col1").createOrReplaceTempView("t1") - val expectedBinaryResult = rawData.map(d => Row(d.getBytes(StandardCharsets.UTF_8))).seq - df = spark.sql("select cast(col1 as binary) from t1") - checkResult(df, expectedBinaryResult) - } - - testGluten("SPARK-27439: Explain result should match collected result after view change") { - withTempView("test", "test2", "tmp") { - spark.range(10).createOrReplaceTempView("test") - spark.range(5).createOrReplaceTempView("test2") - spark.sql("select * from test").createOrReplaceTempView("tmp") - val df = spark.sql("select * from tmp") - spark.sql("select * from test2").createOrReplaceTempView("tmp") - - val captured = new ByteArrayOutputStream() - Console.withOut(captured) { - df.explain(extended = true) - } - checkAnswer(df, spark.range(10).toDF) - val output = captured.toString - assert(output.contains("""== Parsed Logical Plan == - |'Project [*] - |+- 'UnresolvedRelation [tmp]""".stripMargin)) - assert(output.contains("""== Physical Plan == - |*(1) ColumnarToRow - |+- ColumnarRange 0, 10, 1, 2, 10""".stripMargin)) - } - } - - private def withExpr(newExpr: Expression): Column = new Column(newExpr) - - def equalizer(expr: Expression, other: Any): Column = withExpr { - val right = lit(other).expr - if (expr == right) { - logWarning( - s"Constructing trivially true equals predicate, '$expr = $right'. " + - "Perhaps you need to use aliases.") - } - EqualTo(expr, right) - } - - private def verifyNonExchangingAgg(df: DataFrame): Unit = { - var atFirstAgg: Boolean = false - df.queryExecution.executedPlan.foreach { - case agg: HashAggregateExec => - atFirstAgg = !atFirstAgg - case _ => - if (atFirstAgg) { - fail("Should not have operators between the two aggregations") - } - } - } - - private def verifyExchangingAgg(df: DataFrame): Unit = { - var atFirstAgg: Boolean = false - df.queryExecution.executedPlan.foreach { - case _: HashAggregateExec => - if (atFirstAgg) { - fail("Should not have back to back Aggregates") - } - atFirstAgg = true - case _: ShuffleExchangeExec => atFirstAgg = false - case _ => - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameTimeWindowingSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameTimeWindowingSuite.scala deleted file mode 100644 index f2833a357cd..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameTimeWindowingSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenDataFrameTimeWindowingSuite - extends DataFrameTimeWindowingSuite - with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameTungstenSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameTungstenSuite.scala deleted file mode 100644 index 0e555c8eac6..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameTungstenSuite.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -import org.apache.spark.sql.types._ - -class GlutenDataFrameTungstenSuite extends DataFrameTungstenSuite with GlutenSQLTestsTrait { - - testGluten("Map type with struct type as key") { - val kv = Map(Row(1, 2L) -> Seq("v")) - val data = sparkContext.parallelize(Seq(Row(1, kv))) - val schema = new StructType() - .add("a", IntegerType) - .add( - "b", - MapType(new StructType().add("k1", IntegerType).add("k2", LongType), ArrayType(StringType))) - val df = spark.createDataFrame(data, schema) - assert(df.select("b").first() === Row(kv)) - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameWindowFramesSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameWindowFramesSuite.scala deleted file mode 100644 index 3ba990d2eea..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameWindowFramesSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenDataFrameWindowFramesSuite - extends DataFrameWindowFramesSuite - with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameWindowFunctionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameWindowFunctionsSuite.scala deleted file mode 100644 index 213c4c1aec3..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameWindowFunctionsSuite.scala +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -import org.apache.gluten.execution.WindowExecTransformer - -import org.apache.spark.SparkConf -import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression} -import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning -import org.apache.spark.sql.execution.ColumnarShuffleExchangeExec -import org.apache.spark.sql.execution.exchange.ENSURE_REQUIREMENTS -import org.apache.spark.sql.expressions.Window -import org.apache.spark.sql.functions._ -import org.apache.spark.sql.internal.SQLConf - -class GlutenDataFrameWindowFunctionsSuite - extends DataFrameWindowFunctionsSuite - with GlutenSQLTestsTrait { - - import testImplicits._ - - override def sparkConf: SparkConf = { - super.sparkConf - // avoid single partition - .set("spark.sql.shuffle.partitions", "2") - } - - testGluten("covar_samp, var_samp (variance), stddev_samp (stddev) functions in specific window") { - withSQLConf(SQLConf.LEGACY_STATISTICAL_AGGREGATE.key -> "true") { - val df = Seq( - ("a", "p1", 10.0, 20.0), - ("b", "p1", 20.0, 10.0), - ("c", "p2", 20.0, 20.0), - ("d", "p2", 20.0, 20.0), - ("e", "p3", 0.0, 0.0), - ("f", "p3", 6.0, 12.0), - ("g", "p3", 6.0, 12.0), - ("h", "p3", 8.0, 16.0) - ).toDF("key", "partitionId", "value1", "value2") - checkAnswer( - df.select( - $"key", - covar_samp("value1", "value2").over( - Window - .partitionBy("partitionId") - .orderBy("key") - .rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)), - var_samp("value1").over( - Window - .partitionBy("partitionId") - .orderBy("key") - .rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)), - variance("value1").over( - Window - .partitionBy("partitionId") - .orderBy("key") - .rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)), - stddev_samp("value1").over( - Window - .partitionBy("partitionId") - .orderBy("key") - .rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)), - stddev("value1").over( - Window - .partitionBy("partitionId") - .orderBy("key") - .rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)) - ), - Seq( - Row("a", -50.0, 50.0, 50.0, 7.0710678118654755, 7.0710678118654755), - Row("b", -50.0, 50.0, 50.0, 7.0710678118654755, 7.0710678118654755), - Row("c", 0.0, 0.0, 0.0, 0.0, 0.0), - Row("d", 0.0, 0.0, 0.0, 0.0, 0.0), - Row("e", 24.0, 12.0, 12.0, 3.4641016151377544, 3.4641016151377544), - Row("f", 24.0, 12.0, 12.0, 3.4641016151377544, 3.4641016151377544), - Row("g", 24.0, 12.0, 12.0, 3.4641016151377544, 3.4641016151377544), - Row("h", 24.0, 12.0, 12.0, 3.4641016151377544, 3.4641016151377544) - ) - ) - } - } - - testGluten("corr, covar_pop, stddev_pop functions in specific window") { - withSQLConf(SQLConf.LEGACY_STATISTICAL_AGGREGATE.key -> "true") { - val df = Seq( - ("a", "p1", 10.0, 20.0), - ("b", "p1", 20.0, 10.0), - ("c", "p2", 20.0, 20.0), - ("d", "p2", 20.0, 20.0), - ("e", "p3", 0.0, 0.0), - ("f", "p3", 6.0, 12.0), - ("g", "p3", 6.0, 12.0), - ("h", "p3", 8.0, 16.0) - ).toDF("key", "partitionId", "value1", "value2") - checkAnswer( - df.select( - $"key", - corr("value1", "value2").over( - Window - .partitionBy("partitionId") - .orderBy("key") - .rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)), - covar_pop("value1", "value2") - .over( - Window - .partitionBy("partitionId") - .orderBy("key") - .rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)), - var_pop("value1") - .over( - Window - .partitionBy("partitionId") - .orderBy("key") - .rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)), - stddev_pop("value1") - .over( - Window - .partitionBy("partitionId") - .orderBy("key") - .rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)), - var_pop("value2") - .over( - Window - .partitionBy("partitionId") - .orderBy("key") - .rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)), - stddev_pop("value2") - .over( - Window - .partitionBy("partitionId") - .orderBy("key") - .rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)) - ), - - // As stddev_pop(expr) = sqrt(var_pop(expr)) - // the "stddev_pop" column can be calculated from the "var_pop" column. - // - // As corr(expr1, expr2) = covar_pop(expr1, expr2) / (stddev_pop(expr1) * stddev_pop(expr2)) - // the "corr" column can be calculated from the "covar_pop" and the two "stddev_pop" columns - Seq( - Row("a", -1.0, -25.0, 25.0, 5.0, 25.0, 5.0), - Row("b", -1.0, -25.0, 25.0, 5.0, 25.0, 5.0), - Row("c", null, 0.0, 0.0, 0.0, 0.0, 0.0), - Row("d", null, 0.0, 0.0, 0.0, 0.0, 0.0), - Row("e", 1.0, 18.0, 9.0, 3.0, 36.0, 6.0), - Row("f", 1.0, 18.0, 9.0, 3.0, 36.0, 6.0), - Row("g", 1.0, 18.0, 9.0, 3.0, 36.0, 6.0), - Row("h", 1.0, 18.0, 9.0, 3.0, 36.0, 6.0) - ) - ) - } - } - - testGluten( - "SPARK-38237: require all cluster keys for child required distribution for window query") { - def partitionExpressionsColumns(expressions: Seq[Expression]): Seq[String] = { - expressions.flatMap { case ref: AttributeReference => Some(ref.name) } - } - - def isShuffleExecByRequirement( - plan: ColumnarShuffleExchangeExec, - desiredClusterColumns: Seq[String]): Boolean = plan match { - case ColumnarShuffleExchangeExec(op: HashPartitioning, _, ENSURE_REQUIREMENTS, _, _, _) => - partitionExpressionsColumns(op.expressions) === desiredClusterColumns - case _ => false - } - - val df = Seq(("a", 1, 1), ("a", 2, 2), ("b", 1, 3), ("b", 1, 4)).toDF("key1", "key2", "value") - val windowSpec = Window.partitionBy("key1", "key2").orderBy("value") - - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", - SQLConf.REQUIRE_ALL_CLUSTER_KEYS_FOR_DISTRIBUTION.key -> "true") { - - val windowed = df - // repartition by subset of window partitionBy keys which satisfies ClusteredDistribution - .repartition($"key1") - .select(lead($"key1", 1).over(windowSpec), lead($"value", 1).over(windowSpec)) - - checkAnswer(windowed, Seq(Row("b", 4), Row(null, null), Row(null, null), Row(null, null))) - - val shuffleByRequirement = windowed.queryExecution.executedPlan.exists { - case w: WindowExecTransformer => - w.child.exists { - case s: ColumnarShuffleExchangeExec => - isShuffleExecByRequirement(s, Seq("key1", "key2")) - case _ => false - } - case _ => false - } - - assert(shuffleByRequirement, "Can't find desired shuffle node from the query plan") - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameWriterV2Suite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameWriterV2Suite.scala deleted file mode 100644 index ddae3139d06..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameWriterV2Suite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenDataFrameWriterV2Suite extends DataFrameWriterV2Suite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDatasetAggregatorSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDatasetAggregatorSuite.scala deleted file mode 100644 index 8a9a6b5756e..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDatasetAggregatorSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenDatasetAggregatorSuite extends DatasetAggregatorSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDatasetCacheSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDatasetCacheSuite.scala deleted file mode 100644 index 84856019272..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDatasetCacheSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenDatasetCacheSuite extends DatasetCacheSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDatasetOptimizationSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDatasetOptimizationSuite.scala deleted file mode 100644 index a9d1bd29cea..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDatasetOptimizationSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenDatasetOptimizationSuite extends DatasetOptimizationSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDatasetPrimitiveSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDatasetPrimitiveSuite.scala deleted file mode 100644 index c7463dcef75..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDatasetPrimitiveSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenDatasetPrimitiveSuite extends DatasetPrimitiveSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDatasetSerializerRegistratorSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDatasetSerializerRegistratorSuite.scala deleted file mode 100644 index 6749227ed79..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDatasetSerializerRegistratorSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenDatasetSerializerRegistratorSuite - extends DatasetSerializerRegistratorSuite - with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDatasetSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDatasetSuite.scala deleted file mode 100644 index a8e73cee5a8..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDatasetSuite.scala +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -import org.apache.spark.sql.execution.ColumnarShuffleExchangeExec - -class GlutenDatasetSuite extends DatasetSuite with GlutenSQLTestsTrait { - import testImplicits._ - - testGluten("dropDuplicates: columns with same column name") { - val ds1 = Seq(("a", 1), ("a", 2), ("b", 1), ("a", 1)).toDS() - val ds2 = Seq(("a", 1), ("a", 2), ("b", 1), ("a", 1)).toDS() - // The dataset joined has two columns of the same name "_2". - val joined = ds1.join(ds2, "_1").select(ds1("_2").as[Int], ds2("_2").as[Int]) - // Using the checkDatasetUnorderly method to sort the result in Gluten. - checkDatasetUnorderly(joined.dropDuplicates(), (1, 2), (1, 1), (2, 1), (2, 2)) - } - - testGluten("groupBy.as") { - val df1 = Seq(DoubleData(1, "one"), DoubleData(2, "two"), DoubleData(3, "three")) - .toDS() - .repartition($"id") - .sortWithinPartitions("id") - val df2 = Seq(DoubleData(5, "one"), DoubleData(1, "two"), DoubleData(3, "three")) - .toDS() - .repartition($"id") - .sortWithinPartitions("id") - - val df3 = df1 - .groupBy("id") - .as[Int, DoubleData] - .cogroup(df2.groupBy("id").as[Int, DoubleData]) { - case (key, data1, data2) => - if (key == 1) { - Iterator(DoubleData(key, (data1 ++ data2).foldLeft("")((cur, next) => cur + next.val1))) - } else Iterator.empty - } - checkDataset(df3, DoubleData(1, "onetwo")) - - // Assert that no extra shuffle introduced by cogroup. - val exchanges = collect(df3.queryExecution.executedPlan) { - case h: ColumnarShuffleExchangeExec => h - } - // Assert the number of ColumnarShuffleExchangeExec - // instead of ShuffleExchangeExec in Gluten. - assert(exchanges.size == 2) - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala deleted file mode 100644 index f40bc9e20a3..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala +++ /dev/null @@ -1,301 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -import org.apache.spark.sql.catalyst.util.DateTimeUtils -import org.apache.spark.sql.functions._ -import org.apache.spark.sql.internal.SQLConf - -import java.sql.{Date, Timestamp} -import java.time.{LocalDateTime, ZoneId} -import java.util.concurrent.TimeUnit - -class GlutenDateFunctionsSuite extends DateFunctionsSuite with GlutenSQLTestsTrait { - import testImplicits._ - - private def secs(millis: Long): Long = TimeUnit.MILLISECONDS.toSeconds(millis) - - testGluten("unix_timestamp") { - Seq("corrected", "legacy").foreach { - legacyParserPolicy => - withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, - SQLConf.SESSION_LOCAL_TIMEZONE.key -> ZoneId.systemDefault().toString) { - val date1 = Date.valueOf("2015-07-24") - val date2 = Date.valueOf("2015-07-25") - val ts1 = Timestamp.valueOf("2015-07-24 10:00:00.3") - val ts2 = Timestamp.valueOf("2015-07-25 02:02:02.2") - val ntzTs1 = LocalDateTime.parse("2015-07-24T10:00:00.3") - val ntzTs2 = LocalDateTime.parse("2015-07-25T02:02:02.2") - val s1 = "2015/07/24 10:00:00.5" - val s2 = "2015/07/25 02:02:02.6" - val ss1 = "2015-07-24 10:00:00" - val ss2 = "2015-07-25 02:02:02" - val fmt = "yyyy/MM/dd HH:mm:ss.S" - val df = Seq((date1, ts1, ntzTs1, s1, ss1), (date2, ts2, ntzTs2, s2, ss2)).toDF( - "d", - "ts", - "ntzTs", - "s", - "ss") - checkAnswer( - df.select(unix_timestamp(col("ts"))), - Seq(Row(secs(ts1.getTime)), Row(secs(ts2.getTime)))) - checkAnswer( - df.select(unix_timestamp(col("ss"))), - Seq(Row(secs(ts1.getTime)), Row(secs(ts2.getTime)))) - checkAnswer( - df.select(unix_timestamp(col("ntzTs"))), - Seq( - Row(secs(DateTimeUtils.microsToMillis(DateTimeUtils.localDateTimeToMicros(ntzTs1)))), - Row(secs(DateTimeUtils.microsToMillis(DateTimeUtils.localDateTimeToMicros(ntzTs2)))) - ) - ) - checkAnswer( - df.select(unix_timestamp(col("d"), fmt)), - Seq(Row(secs(date1.getTime)), Row(secs(date2.getTime)))) - checkAnswer( - df.select(unix_timestamp(col("s"), fmt)), - Seq(Row(secs(ts1.getTime)), Row(secs(ts2.getTime)))) - checkAnswer( - df.selectExpr("unix_timestamp(ts)"), - Seq(Row(secs(ts1.getTime)), Row(secs(ts2.getTime)))) - checkAnswer( - df.selectExpr("unix_timestamp(ss)"), - Seq(Row(secs(ts1.getTime)), Row(secs(ts2.getTime)))) - checkAnswer( - df.selectExpr("unix_timestamp(ntzTs)"), - Seq( - Row(secs(DateTimeUtils.microsToMillis(DateTimeUtils.localDateTimeToMicros(ntzTs1)))), - Row(secs(DateTimeUtils.microsToMillis(DateTimeUtils.localDateTimeToMicros(ntzTs2)))) - ) - ) - checkAnswer( - df.selectExpr(s"unix_timestamp(d, '$fmt')"), - Seq(Row(secs(date1.getTime)), Row(secs(date2.getTime)))) - checkAnswer( - df.selectExpr(s"unix_timestamp(s, '$fmt')"), - Seq(Row(secs(ts1.getTime)), Row(secs(ts2.getTime)))) - - val x1 = "2015-07-24 10:00:00" - val x2 = "2015-25-07 02:02:02" - val x3 = "2015-07-24 25:02:02" - val x4 = "2015-24-07 26:02:02" - val ts3 = Timestamp.valueOf("2015-07-24 02:25:02") - val ts4 = Timestamp.valueOf("2015-07-24 00:10:00") - - val df1 = Seq(x1, x2, x3, x4).toDF("x") - checkAnswer( - df1.select(unix_timestamp(col("x"))), - Seq(Row(secs(ts1.getTime)), Row(null), Row(null), Row(null))) - checkAnswer( - df1.selectExpr("unix_timestamp(x)"), - Seq(Row(secs(ts1.getTime)), Row(null), Row(null), Row(null))) - checkAnswer( - df1.select(unix_timestamp(col("x"), "yyyy-dd-MM HH:mm:ss")), - Seq(Row(null), Row(secs(ts2.getTime)), Row(null), Row(null))) - checkAnswer( - df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd mm:HH:ss')"), - Seq(Row(secs(ts4.getTime)), Row(null), Row(secs(ts3.getTime)), Row(null))) - - // invalid format - val invalid = df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd aa:HH:ss')") - checkAnswer(invalid, Seq(Row(null), Row(null), Row(null), Row(null))) - - // February - val y1 = "2016-02-29" - val y2 = "2017-02-29" - val ts5 = Timestamp.valueOf("2016-02-29 00:00:00") - val df2 = Seq(y1, y2).toDF("y") - checkAnswer( - df2.select(unix_timestamp(col("y"), "yyyy-MM-dd")), - Seq(Row(secs(ts5.getTime)), Row(null))) - - val now = sql("select unix_timestamp()").collect().head.getLong(0) - checkAnswer( - sql(s"select timestamp_seconds($now)"), - Row(new java.util.Date(TimeUnit.SECONDS.toMillis(now)))) - } - } - } - - testGluten("to_unix_timestamp") { - Seq("corrected", "legacy").foreach { - legacyParserPolicy => - withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, - SQLConf.SESSION_LOCAL_TIMEZONE.key -> ZoneId.systemDefault().toString - ) { - val date1 = Date.valueOf("2015-07-24") - val date2 = Date.valueOf("2015-07-25") - val ts1 = Timestamp.valueOf("2015-07-24 10:00:00.3") - val ts2 = Timestamp.valueOf("2015-07-25 02:02:02.2") - val s1 = "2015/07/24 10:00:00.5" - val s2 = "2015/07/25 02:02:02.6" - val ss1 = "2015-07-24 10:00:00" - val ss2 = "2015-07-25 02:02:02" - val fmt = "yyyy/MM/dd HH:mm:ss.S" - val df = Seq((date1, ts1, s1, ss1), (date2, ts2, s2, ss2)).toDF("d", "ts", "s", "ss") - checkAnswer( - df.selectExpr("to_unix_timestamp(ts)"), - Seq(Row(secs(ts1.getTime)), Row(secs(ts2.getTime)))) - checkAnswer( - df.selectExpr("to_unix_timestamp(ss)"), - Seq(Row(secs(ts1.getTime)), Row(secs(ts2.getTime)))) - checkAnswer( - df.selectExpr(s"to_unix_timestamp(d, '$fmt')"), - Seq(Row(secs(date1.getTime)), Row(secs(date2.getTime)))) - checkAnswer( - df.selectExpr(s"to_unix_timestamp(s, '$fmt')"), - Seq(Row(secs(ts1.getTime)), Row(secs(ts2.getTime)))) - - val x1 = "2015-07-24 10:00:00" - val x2 = "2015-25-07 02:02:02" - val x3 = "2015-07-24 25:02:02" - val x4 = "2015-24-07 26:02:02" - val ts3 = Timestamp.valueOf("2015-07-24 02:25:02") - val ts4 = Timestamp.valueOf("2015-07-24 00:10:00") - - val df1 = Seq(x1, x2, x3, x4).toDF("x") - checkAnswer( - df1.selectExpr("to_unix_timestamp(x)"), - Seq(Row(secs(ts1.getTime)), Row(null), Row(null), Row(null))) - checkAnswer( - df1.selectExpr(s"to_unix_timestamp(x, 'yyyy-MM-dd mm:HH:ss')"), - Seq(Row(secs(ts4.getTime)), Row(null), Row(secs(ts3.getTime)), Row(null))) - - // February - val y1 = "2016-02-29" - val y2 = "2017-02-29" - val ts5 = Timestamp.valueOf("2016-02-29 00:00:00") - val df2 = Seq(y1, y2).toDF("y") - checkAnswer( - df2.select(unix_timestamp(col("y"), "yyyy-MM-dd")), - Seq(Row(secs(ts5.getTime)), Row(null))) - - val invalid = df1.selectExpr(s"to_unix_timestamp(x, 'yyyy-MM-dd bb:HH:ss')") - checkAnswer(invalid, Seq(Row(null), Row(null), Row(null), Row(null))) - } - } - } - - testGluten("function to_date") { - val d1 = Date.valueOf("2015-07-22") - val d2 = Date.valueOf("2015-07-01") - val d3 = Date.valueOf("2014-12-31") - val t1 = Timestamp.valueOf("2015-07-22 10:00:00") - val t2 = Timestamp.valueOf("2014-12-31 23:59:59") - val t3 = Timestamp.valueOf("2014-12-31 23:59:59") - val s1 = "2015-07-22 10:00:00" - val s2 = "2014-12-31" - val s3 = "2014-31-12" - val df = Seq((d1, t1, s1), (d2, t2, s2), (d3, t3, s3)).toDF("d", "t", "s") - - checkAnswer( - df.select(to_date(col("t"))), - Seq( - Row(Date.valueOf("2015-07-22")), - Row(Date.valueOf("2014-12-31")), - Row(Date.valueOf("2014-12-31")))) - checkAnswer( - df.select(to_date(col("d"))), - Seq( - Row(Date.valueOf("2015-07-22")), - Row(Date.valueOf("2015-07-01")), - Row(Date.valueOf("2014-12-31")))) - checkAnswer( - df.select(to_date(col("s"))), - Seq(Row(Date.valueOf("2015-07-22")), Row(Date.valueOf("2014-12-31")), Row(null))) - - checkAnswer( - df.selectExpr("to_date(t)"), - Seq( - Row(Date.valueOf("2015-07-22")), - Row(Date.valueOf("2014-12-31")), - Row(Date.valueOf("2014-12-31")))) - checkAnswer( - df.selectExpr("to_date(d)"), - Seq( - Row(Date.valueOf("2015-07-22")), - Row(Date.valueOf("2015-07-01")), - Row(Date.valueOf("2014-12-31")))) - checkAnswer( - df.selectExpr("to_date(s)"), - Seq(Row(Date.valueOf("2015-07-22")), Row(Date.valueOf("2014-12-31")), Row(null))) - - // now with format - checkAnswer( - df.select(to_date(col("t"), "yyyy-MM-dd")), - Seq( - Row(Date.valueOf("2015-07-22")), - Row(Date.valueOf("2014-12-31")), - Row(Date.valueOf("2014-12-31")))) - checkAnswer( - df.select(to_date(col("d"), "yyyy-MM-dd")), - Seq( - Row(Date.valueOf("2015-07-22")), - Row(Date.valueOf("2015-07-01")), - Row(Date.valueOf("2014-12-31")))) - val confKey = SQLConf.LEGACY_TIME_PARSER_POLICY.key - withSQLConf(confKey -> "corrected") { - checkAnswer( - df.select(to_date(col("s"), "yyyy-MM-dd")), - Seq(Row(null), Row(Date.valueOf("2014-12-31")), Row(null))) - } - // legacyParserPolicy is not respected by Gluten. - // withSQLConf(confKey -> "exception") { - // checkExceptionMessage(df.select(to_date(col("s"), "yyyy-MM-dd"))) - // } - - // now switch format - checkAnswer( - df.select(to_date(col("s"), "yyyy-dd-MM")), - Seq(Row(null), Row(null), Row(Date.valueOf("2014-12-31")))) - - // invalid format - checkAnswer(df.select(to_date(col("s"), "yyyy-hh-MM")), Seq(Row(null), Row(null), Row(null))) - // velox getTimestamp function does not throw exception when format is "yyyy-dd-aa". - // val e = - // intercept[SparkUpgradeException](df.select(to_date(col("s"), "yyyy-dd-aa")).collect()) - // assert(e.getCause.isInstanceOf[IllegalArgumentException]) - // assert( - // e.getMessage.contains("You may get a different result due to the upgrading to Spark")) - - // February - val x1 = "2016-02-29" - val x2 = "2017-02-29" - val df1 = Seq(x1, x2).toDF("x") - checkAnswer(df1.select(to_date(col("x"))), Row(Date.valueOf("2016-02-29")) :: Row(null) :: Nil) - } - testGluten("date_from_unix_date") { - // -100000 and 200000 are outside ClickHouse's native Date32 range - // [1900-01-01, 2299-12-31]. They guard against implementations that clamp to that - // range (e.g. mapping to CH toDate32), which would silently diverge from Spark. - val df = Seq(Some(0), Some(1000), Some(-100000), Some(200000), None).toDF("unix_date") - val expected = Seq( - Row(Date.valueOf("1970-01-01")), - Row(Date.valueOf("1972-09-27")), - Row(Date.valueOf("1696-03-17")), - Row(Date.valueOf("2517-08-01")), - Row(null)) - - // date_from_unix_date is only available in the Scala functions API since Spark 3.5, - // so go through expr() here. - checkAnswer(df.select(expr("date_from_unix_date(unix_date)")), expected) - checkAnswer(df.selectExpr("date_from_unix_date(unix_date)"), expected) - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDeprecatedAPISuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDeprecatedAPISuite.scala deleted file mode 100644 index b6428773f1d..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDeprecatedAPISuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenDeprecatedAPISuite extends DeprecatedAPISuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala deleted file mode 100644 index e22b9b95824..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala +++ /dev/null @@ -1,788 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.execution.{BatchScanExecTransformer, FileSourceScanExecTransformer, FilterExecTransformerBase} -import org.apache.gluten.utils.BackendTestUtils - -import org.apache.spark.SparkConf -import org.apache.spark.sql.catalyst.expressions.{DynamicPruningExpression, Expression} -import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode.{CODEGEN_ONLY, NO_CODEGEN} -import org.apache.spark.sql.catalyst.plans.ExistenceJoin -import org.apache.spark.sql.connector.catalog.InMemoryTableCatalog -import org.apache.spark.sql.execution._ -import org.apache.spark.sql.execution.adaptive._ -import org.apache.spark.sql.execution.datasources.v2.BatchScanExec -import org.apache.spark.sql.execution.exchange.{BroadcastExchangeLike, ReusedExchangeExec} -import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec -import org.apache.spark.sql.execution.streaming.{MemoryStream, StreamingQueryWrapper} -import org.apache.spark.sql.functions.col -import org.apache.spark.sql.internal.SQLConf - -abstract class GlutenDynamicPartitionPruningSuiteBase - extends DynamicPartitionPruningSuiteBase - with GlutenSQLTestsTrait { - - import testImplicits._ - - override def beforeAll(): Unit = { - prepareWorkDir() - super.beforeAll() - spark.sparkContext.setLogLevel("WARN") - } - - override def testNameBlackList: Seq[String] = Seq( - // overwritten with different plan - "Make sure dynamic pruning works on uncorrelated queries", - "Subquery reuse across the whole plan", - // struct join key not supported, fell-back to Vanilla join - "SPARK-32659: Fix the data issue when pruning DPP on non-atomic type" - ) - - // === Following cases override super class's cases === - - ignoreGluten("DPP should not be rewritten as an existential join") { - // ignored: BroadcastHashJoinExec is from Vanilla Spark - withSQLConf( - SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true", - SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "1.5", - SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false", - SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false" - ) { - val df = sql(s""" - |SELECT * FROM product p WHERE p.store_id NOT IN - | (SELECT f.store_id FROM fact_sk f JOIN dim_store d ON - | f.store_id = d.store_id - | WHERE d.state_province = 'NL' - | ) - """.stripMargin) - - val found = df.queryExecution.executedPlan.find { - case _ @BroadcastHashJoinExec(_, _, _: ExistenceJoin, _, _, _, _, _) => true - case _ => false - } - - assert(found.isEmpty) - } - } - - testGluten("no partition pruning when the build side is a stream") { - withTable("fact") { - val input = MemoryStream[Int] - val stream = input.toDF.select($"value".as("one"), ($"value" * 3).as("code")) - spark - .range(100) - .select($"id", ($"id" + 1).as("one"), ($"id" + 2).as("two"), ($"id" + 3).as("three")) - .write - .partitionBy("one") - .format(tableFormat) - .mode("overwrite") - .saveAsTable("fact") - val table = sql("SELECT * from fact f") - - // join a partitioned table with a stream - val joined = table.join(stream, Seq("one")).where("code > 40") - val query = joined.writeStream.format("memory").queryName("test").start() - input.addData(1, 10, 20, 40, 50) - try { - query.processAllAvailable() - } finally { - query.stop() - } - // search dynamic pruning predicates on the executed plan - val plan = query.asInstanceOf[StreamingQueryWrapper].streamingQuery.lastExecution.executedPlan - val ret = plan.find { - case s: FileSourceScanExecTransformer => - s.partitionFilters.exists { - case _: DynamicPruningExpression => true - case _ => false - } - case s: FileSourceScanExec => - s.partitionFilters.exists { - case _: DynamicPruningExpression => true - case _ => false - } - case _ => false - } - assert(ret.isDefined == false) - } - } - - testGluten("Make sure dynamic pruning works on uncorrelated queries") { - withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") { - val df = sql(""" - |SELECT d.store_id, - | SUM(f.units_sold), - | (SELECT SUM(f.units_sold) - | FROM fact_stats f JOIN dim_stats d ON d.store_id = f.store_id - | WHERE d.country = 'US') AS total_prod - |FROM fact_stats f JOIN dim_stats d ON d.store_id = f.store_id - |WHERE d.country = 'US' - |GROUP BY 1 - """.stripMargin) - checkAnswer(df, Row(4, 50, 70) :: Row(5, 10, 70) :: Row(6, 10, 70) :: Nil) - - val plan = df.queryExecution.executedPlan - val countSubqueryBroadcasts = - collectWithSubqueries(plan) { - case _: SubqueryBroadcastExec => 1 - case _: ColumnarSubqueryBroadcastExec => 1 - }.sum - - val countReusedSubqueryBroadcasts = - collectWithSubqueries(plan) { - case ReusedSubqueryExec(_: SubqueryBroadcastExec) => 1 - case ReusedSubqueryExec(_: ColumnarSubqueryBroadcastExec) => 1 - }.sum - - assert(countSubqueryBroadcasts == 1) - assert(countReusedSubqueryBroadcasts == 1) - } - } - - testGluten( - "SPARK-32509: Unused Dynamic Pruning filter shouldn't affect " + - "canonicalization and exchange reuse") { - withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") { - withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") { - val df = sql(""" WITH view1 as ( - | SELECT f.store_id FROM fact_stats f WHERE f.units_sold = 70 - | ) - | - | SELECT * FROM view1 v1 join view1 v2 WHERE v1.store_id = v2.store_id - """.stripMargin) - - checkPartitionPruningPredicate(df, false, false) - val reuseExchangeNodes = collect(df.queryExecution.executedPlan) { - case se: ReusedExchangeExec => se - } - assert( - reuseExchangeNodes.size == 1, - "Expected plan to contain 1 ReusedExchangeExec " + - s"nodes. Found ${reuseExchangeNodes.size}") - - checkAnswer(df, Row(15, 15) :: Nil) - } - } - } - - testGluten("SPARK-32659: Fix the data issue when pruning DPP on non-atomic type") { - Seq(NO_CODEGEN, CODEGEN_ONLY).foreach { - mode => - Seq(true, false).foreach { - pruning => - withSQLConf( - SQLConf.CODEGEN_FACTORY_MODE.key -> mode.toString, - SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> s"$pruning") { - Seq("struct", "array").foreach { - dataType => - val df = sql( - s""" - |SELECT f.date_id, f.product_id, f.units_sold, f.store_id FROM fact_stats f - |JOIN dim_stats s - |ON $dataType(f.store_id) = $dataType(s.store_id) WHERE s.country = 'DE' - """.stripMargin) - - if (pruning) { - df.collect() - - val plan = df.queryExecution.executedPlan - val dpExprs = collectDynamicPruningExpressions(plan) - val hasSubquery = dpExprs.exists { - case InSubqueryExec(_, _: SubqueryExec, _, _, _, _) => true - case _ => false - } - val subqueryBroadcast = dpExprs.collect { - case InSubqueryExec(_, b: SubqueryBroadcastExec, _, _, _, _) => b - case InSubqueryExec(_, b: ColumnarSubqueryBroadcastExec, _, _, _, _) => b - } - - val hasFilter = if (false) "Should" else "Shouldn't" - assert( - !hasSubquery, - s"$hasFilter trigger DPP with a subquery duplicate:\n${df.queryExecution}") - val hasBroadcast = if (true) "Should" else "Shouldn't" - assert( - subqueryBroadcast.nonEmpty, - s"$hasBroadcast trigger DPP " + - s"with a reused broadcast exchange:\n${df.queryExecution}") - - subqueryBroadcast.foreach { - s => - s.child match { - case _: ReusedExchangeExec => // reuse check ok. - case BroadcastQueryStageExec( - _, - _: ReusedExchangeExec, - _ - ) => // reuse check ok. - case b: BroadcastExchangeLike => - val hasReuse = plan.find { - case ReusedExchangeExec(_, e) => e eq b - case _ => false - }.isDefined - // assert(hasReuse, s"$s\nshould have been reused in\n$plan") - case a: AdaptiveSparkPlanExec => - val broadcastQueryStage = collectFirst(a) { - case b: BroadcastQueryStageExec => b - } - val broadcastPlan = broadcastQueryStage.get.broadcast - val hasReuse = find(plan) { - case ReusedExchangeExec(_, e) => e eq broadcastPlan - case b: BroadcastExchangeLike => b eq broadcastPlan - case _ => false - }.isDefined - // assert(hasReuse, s"$s\nshould have been reused in\n$plan") - case _ => - fail(s"Invalid child node found in\n$s") - } - } - - val isMainQueryAdaptive = plan.isInstanceOf[AdaptiveSparkPlanExec] - subqueriesAll(plan).filterNot(subqueryBroadcast.contains).foreach { - s => - val subquery = s match { - case r: ReusedSubqueryExec => r.child - case o => o - } - assert( - subquery - .find(_.isInstanceOf[AdaptiveSparkPlanExec]) - .isDefined == isMainQueryAdaptive) - } - } else { - checkPartitionPruningPredicate(df, false, false) - } - - checkAnswer( - df, - Row(1030, 2, 10, 3) :: - Row(1040, 2, 50, 3) :: - Row(1050, 2, 50, 3) :: - Row(1060, 2, 50, 3) :: Nil) - } - } - } - } - } - - // === Following methods override super class's methods === - - override protected def collectDynamicPruningExpressions(plan: SparkPlan): Seq[Expression] = { - flatMap(plan) { - case s: FileSourceScanExecTransformer => - s.partitionFilters.collect { case d: DynamicPruningExpression => d.child } - case s: FileSourceScanExec => - s.partitionFilters.collect { case d: DynamicPruningExpression => d.child } - case s: BatchScanExecTransformer => - s.runtimeFilters.collect { case d: DynamicPruningExpression => d.child } - case s: BatchScanExec => - s.runtimeFilters.collect { case d: DynamicPruningExpression => d.child } - case _ => Nil - } - } - - override def checkPartitionPruningPredicate( - df: DataFrame, - withSubquery: Boolean, - withBroadcast: Boolean): Unit = { - df.collect() - - val plan = df.queryExecution.executedPlan - val dpExprs = collectDynamicPruningExpressions(plan) - val hasSubquery = dpExprs.exists { - case InSubqueryExec(_, _: SubqueryExec, _, _, _, _) => true - case _ => false - } - val subqueryBroadcast = dpExprs.collect { - case InSubqueryExec(_, b: SubqueryBroadcastExec, _, _, _, _) => b - case InSubqueryExec(_, b: ColumnarSubqueryBroadcastExec, _, _, _, _) => b - } - - val hasFilter = if (withSubquery) "Should" else "Shouldn't" - assert( - hasSubquery == withSubquery, - s"$hasFilter trigger DPP with a subquery duplicate:\n${df.queryExecution}") - val hasBroadcast = if (withBroadcast) "Should" else "Shouldn't" - assert( - subqueryBroadcast.nonEmpty == withBroadcast, - s"$hasBroadcast trigger DPP with a reused broadcast exchange:\n${df.queryExecution}") - - subqueryBroadcast.foreach { - s => - s.child match { - case _: ReusedExchangeExec => // reuse check ok. - case BroadcastQueryStageExec(_, _: ReusedExchangeExec, _) => // reuse check ok. - case b: BroadcastExchangeLike => - val hasReuse = plan.find { - case ReusedExchangeExec(_, e) => e eq b - case _ => false - }.isDefined - assert(hasReuse, s"$s\nshould have been reused in\n$plan") - case a: AdaptiveSparkPlanExec => - val broadcastQueryStage = collectFirst(a) { case b: BroadcastQueryStageExec => b } - val broadcastPlan = broadcastQueryStage.get.broadcast - val hasReuse = find(plan) { - case ReusedExchangeExec(_, e) => e eq broadcastPlan - case b: BroadcastExchangeLike => b eq broadcastPlan - case _ => false - }.isDefined - assert(hasReuse, s"$s\nshould have been reused in\n$plan") - case _ => - fail(s"Invalid child node found in\n$s") - } - } - - val isMainQueryAdaptive = plan.isInstanceOf[AdaptiveSparkPlanExec] - subqueriesAll(plan).filterNot(subqueryBroadcast.contains).foreach { - s => - val subquery = s match { - case r: ReusedSubqueryExec => r.child - case o => o - } - assert( - subquery.find(_.isInstanceOf[AdaptiveSparkPlanExec]).isDefined == isMainQueryAdaptive) - } - } - - override def checkDistinctSubqueries(df: DataFrame, n: Int): Unit = { - df.collect() - - val buf = collectDynamicPruningExpressions(df.queryExecution.executedPlan).collect { - case InSubqueryExec(_, b: SubqueryBroadcastExec, _, _, _, _) => - b.index - case InSubqueryExec(_, b: ColumnarSubqueryBroadcastExec, _, _, _, _) => - b.indices - } - assert(buf.distinct.size == n) - } - - override def checkUnpushedFilters(df: DataFrame): Boolean = { - find(df.queryExecution.executedPlan) { - case FilterExec(condition, _) => - splitConjunctivePredicates(condition).exists { - case _: DynamicPruningExpression => true - case _ => false - } - case transformer: FilterExecTransformerBase => - splitConjunctivePredicates(transformer.cond).exists { - case _: DynamicPruningExpression => true - case _ => false - } - case FilterTransformer(condition, _) => - splitConjunctivePredicates(condition).exists { - case _: DynamicPruningExpression => true - case _ => false - } - case _ => false - }.isDefined - } - - object FilterTransformer { - def unapply(plan: SparkPlan): Option[(Expression, SparkPlan)] = { - plan match { - case transformer: FilterExecTransformerBase => - Some((transformer.cond, transformer.input)) - case _ => None - } - } - } -} - -abstract class GlutenDynamicPartitionPruningV1Suite extends GlutenDynamicPartitionPruningSuiteBase { - - import testImplicits._ - - /** Check the static scan metrics with and without DPP */ - testGluten("static scan metrics", DisableAdaptiveExecution("DPP in AQE must reuse broadcast")) { - withSQLConf( - SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true", - SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false", - SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false" - ) { - withTable("fact", "dim") { - val numPartitions = 10 - - spark - .range(10) - .map(x => Tuple3(x, x + 1, 0)) - .toDF("did", "d1", "d2") - .write - .format(tableFormat) - .mode("overwrite") - .saveAsTable("dim") - - spark - .range(100) - .map(x => Tuple2(x, x % numPartitions)) - .toDF("f1", "fid") - .write - .partitionBy("fid") - .format(tableFormat) - .mode("overwrite") - .saveAsTable("fact") - - def getFactScan(plan: SparkPlan): SparkPlan = { - val scanOption = - find(plan) { - case s: FileSourceScanExec => - s.output.exists(_.find(_.argString(maxFields = 100).contains("fid")).isDefined) - case s: FileSourceScanExecTransformer => - s.output.exists(_.find(_.argString(maxFields = 100).contains("fid")).isDefined) - case s: BatchScanExec => - // we use f1 col for v2 tables due to schema pruning - s.output.exists(_.find(_.argString(maxFields = 100).contains("f1")).isDefined) - case s: BatchScanExecTransformer => - // we use f1 col for v2 tables due to schema pruning - s.output.exists(_.find(_.argString(maxFields = 100).contains("f1")).isDefined) - case _ => false - } - assert(scanOption.isDefined) - scanOption.get - } - - // No dynamic partition pruning, so no static metrics - // All files in fact table are scanned - val df1 = sql("SELECT sum(f1) FROM fact") - df1.collect() - val scan1 = getFactScan(df1.queryExecution.executedPlan) - assert(!scan1.metrics.contains("staticFilesNum")) - assert(!scan1.metrics.contains("staticFilesSize")) - val allFilesNum = scan1.metrics("numFiles").value - val allFilesSize = scan1.metrics("filesSize").value - assert(scan1.metrics("numPartitions").value === numPartitions) - assert(scan1.metrics("pruningTime").value === -1) - - // No dynamic partition pruning, so no static metrics - // Only files from fid = 5 partition are scanned - val df2 = sql("SELECT sum(f1) FROM fact WHERE fid = 5") - df2.collect() - val scan2 = getFactScan(df2.queryExecution.executedPlan) - assert(!scan2.metrics.contains("staticFilesNum")) - assert(!scan2.metrics.contains("staticFilesSize")) - val partFilesNum = scan2.metrics("numFiles").value - val partFilesSize = scan2.metrics("filesSize").value - assert(0 < partFilesNum && partFilesNum < allFilesNum) - assert(0 < partFilesSize && partFilesSize < allFilesSize) - assert(scan2.metrics("numPartitions").value === 1) - assert(scan2.metrics("pruningTime").value === -1) - - // Dynamic partition pruning is used - // Static metrics are as-if reading the whole fact table - // "Regular" metrics are as-if reading only the "fid = 5" partition - val df3 = sql("SELECT sum(f1) FROM fact, dim WHERE fid = did AND d1 = 6") - df3.collect() - val scan3 = getFactScan(df3.queryExecution.executedPlan) - assert(scan3.metrics("staticFilesNum").value == allFilesNum) - assert(scan3.metrics("staticFilesSize").value == allFilesSize) - assert(scan3.metrics("numFiles").value == partFilesNum) - assert(scan3.metrics("filesSize").value == partFilesSize) - assert(scan3.metrics("numPartitions").value === 1) - assert(scan3.metrics("pruningTime").value !== -1) - } - } - } -} - -class GlutenDynamicPartitionPruningV1SuiteAEOff - extends GlutenDynamicPartitionPruningV1Suite - with DisableAdaptiveExecutionSuite { - - import testImplicits._ - - testGluten( - "override static scan metrics", - DisableAdaptiveExecution("DPP in AQE must reuse broadcast")) { - withSQLConf( - SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true", - SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false", - // "spark.gluten.enabled" -> "false", - SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false" - ) { - withTable("fact", "dim") { - val numPartitions = 10 - - spark - .range(10) - .map(x => Tuple3(x, x + 1, 0)) - .toDF("did", "d1", "d2") - .write - .format(tableFormat) - .mode("overwrite") - .saveAsTable("dim") - - spark - .range(100) - .map(x => Tuple2(x, x % numPartitions)) - .toDF("f1", "fid") - .write - .partitionBy("fid") - .format(tableFormat) - .mode("overwrite") - .saveAsTable("fact") - - def getFactScan(plan: SparkPlan): SparkPlan = { - val scanOption = - find(plan) { - case s: FileSourceScanExecTransformer => - s.output.exists(_.find(_.argString(maxFields = 100).contains("fid")).isDefined) - case s: FileSourceScanExec => - s.output.exists(_.find(_.argString(maxFields = 100).contains("fid")).isDefined) - case s: BatchScanExecTransformer => - // we use f1 col for v2 tables due to schema pruning - s.output.exists(_.find(_.argString(maxFields = 100).contains("f1")).isDefined) - case s: BatchScanExec => - // we use f1 col for v2 tables due to schema pruning - s.output.exists(_.find(_.argString(maxFields = 100).contains("f1")).isDefined) - case _ => false - } - assert(scanOption.isDefined) - scanOption.get - } - - // No dynamic partition pruning, so no static metrics - // All files in fact table are scanned - val df1 = sql("SELECT sum(f1) FROM fact") - df1.collect() - val scan1 = getFactScan(df1.queryExecution.executedPlan) - assert(!scan1.metrics.contains("staticFilesNum")) - assert(!scan1.metrics.contains("staticFilesSize")) - val allFilesNum = scan1.metrics("numFiles").value - val allFilesSize = scan1.metrics("filesSize").value - assert(scan1.metrics("numPartitions").value === numPartitions) - assert(scan1.metrics("pruningTime").value === -1) - - // No dynamic partition pruning, so no static metrics - // Only files from fid = 5 partition are scanned - val df2 = sql("SELECT sum(f1) FROM fact WHERE fid = 5") - df2.collect() - val scan2 = getFactScan(df2.queryExecution.executedPlan) - assert(!scan2.metrics.contains("staticFilesNum")) - assert(!scan2.metrics.contains("staticFilesSize")) - val partFilesNum = scan2.metrics("numFiles").value - val partFilesSize = scan2.metrics("filesSize").value - assert(0 < partFilesNum && partFilesNum < allFilesNum) - assert(0 < partFilesSize && partFilesSize < allFilesSize) - assert(scan2.metrics("numPartitions").value === 1) - assert(scan2.metrics("pruningTime").value === -1) - - // Dynamic partition pruning is used - // Static metrics are as-if reading the whole fact table - // "Regular" metrics are as-if reading only the "fid = 5" partition - val df3 = sql("SELECT sum(f1) FROM fact, dim WHERE fid = did AND d1 = 6") - df3.collect() - val scan3 = getFactScan(df3.queryExecution.executedPlan) - assert(scan3.metrics("staticFilesNum").value == allFilesNum) - assert(scan3.metrics("staticFilesSize").value == allFilesSize) - assert(scan3.metrics("numFiles").value == partFilesNum) - assert(scan3.metrics("filesSize").value == partFilesSize) - assert(scan3.metrics("numPartitions").value === 1) - assert(scan3.metrics("pruningTime").value !== -1) - } - } - } - - testGluten( - "Subquery reuse across the whole plan", - DisableAdaptiveExecution("DPP in AQE must reuse broadcast")) { - withSQLConf( - SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true", - SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false", - SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false" - ) { - withTable("df1", "df2") { - spark - .range(100) - .select(col("id"), col("id").as("k")) - .write - .partitionBy("k") - .format(tableFormat) - .mode("overwrite") - .saveAsTable("df1") - - spark - .range(10) - .select(col("id"), col("id").as("k")) - .write - .partitionBy("k") - .format(tableFormat) - .mode("overwrite") - .saveAsTable("df2") - - val df = sql(""" - |SELECT df1.id, df2.k - |FROM df1 JOIN df2 ON df1.k = df2.k - |WHERE df2.id < (SELECT max(id) FROM df2 WHERE id <= 2) - |""".stripMargin) - - checkPartitionPruningPredicate(df, true, false) - - checkAnswer(df, Row(0, 0) :: Row(1, 1) :: Nil) - - val plan = df.queryExecution.executedPlan - - val subqueryIds = plan.collectWithSubqueries { case s: SubqueryExec => s.id } - val reusedSubqueryIds = plan.collectWithSubqueries { - case rs: ReusedSubqueryExec => rs.child.id - } - - // By default Gluten pushes more filters than vanilla Spark. - // - // See also org.apache.gluten.execution.FilterHandler#applyFilterPushdownToScan - // See also DynamicPartitionPruningSuite.scala:1362 - if (BackendTestUtils.isCHBackendLoaded()) { - assert(subqueryIds.size == 2, "Whole plan subquery reusing not working correctly") - assert(reusedSubqueryIds.size == 1, "Whole plan subquery reusing not working correctly") - } else if (BackendTestUtils.isVeloxBackendLoaded()) { - assert(subqueryIds.size == 3, "Whole plan subquery reusing not working correctly") - assert(reusedSubqueryIds.size == 2, "Whole plan subquery reusing not working correctly") - } else { - assert(false, "Unknown backend") - } - assert( - reusedSubqueryIds.forall(subqueryIds.contains(_)), - "ReusedSubqueryExec should reuse an existing subquery") - } - } - } -} - -class GlutenDynamicPartitionPruningV1SuiteAEOn - extends GlutenDynamicPartitionPruningV1Suite - with EnableAdaptiveExecutionSuite { - - testGluten("SPARK-39447: Avoid AssertionError in AdaptiveSparkPlanExec.doExecuteBroadcast") { - val df = sql(""" - |WITH empty_result AS ( - | SELECT * FROM fact_stats WHERE product_id < 0 - |) - |SELECT * - |FROM (SELECT /*+ SHUFFLE_MERGE(fact_sk) */ empty_result.store_id - | FROM fact_sk - | JOIN empty_result - | ON fact_sk.product_id = empty_result.product_id) t2 - | JOIN empty_result - | ON t2.store_id = empty_result.store_id - """.stripMargin) - - checkPartitionPruningPredicate(df, false, false) - checkAnswer(df, Nil) - } - - testGluten( - "SPARK-37995: PlanAdaptiveDynamicPruningFilters should use prepareExecutedPlan " + - "rather than createSparkPlan to re-plan subquery") { - withSQLConf( - SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true", - SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false", - SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false" - ) { - val df = sql(""" - |SELECT f.date_id, f.store_id FROM fact_sk f - |JOIN dim_store s ON f.store_id = s.store_id AND s.country = 'NL' - |WHERE s.state_province != (SELECT max(state_province) FROM dim_stats) - """.stripMargin) - - checkPartitionPruningPredicate(df, true, false) - checkAnswer(df, Row(1000, 1) :: Row(1010, 2) :: Row(1020, 2) :: Nil) - } - } -} - -abstract class GlutenDynamicPartitionPruningV2Suite extends GlutenDynamicPartitionPruningSuiteBase { - override protected def runAnalyzeColumnCommands: Boolean = false - - override protected def initState(): Unit = { - spark.conf.set("spark.sql.catalog.testcat", classOf[InMemoryTableCatalog].getName) - spark.conf.set("spark.sql.defaultCatalog", "testcat") - } -} - -class GlutenDynamicPartitionPruningV2SuiteAEOff - extends GlutenDynamicPartitionPruningV2Suite - with DisableAdaptiveExecutionSuite - -class GlutenDynamicPartitionPruningV2SuiteAEOn - extends GlutenDynamicPartitionPruningV2Suite - with EnableAdaptiveExecutionSuite - -// Test DPP with file scan disabled by user for some reason, which can also mock the situation -// that scan is not transformable. -class GlutenDynamicPartitionPruningV1SuiteAEOnDisableScan - extends GlutenDynamicPartitionPruningV1SuiteAEOn { - override def sparkConf: SparkConf = { - super.sparkConf.set(GlutenConfig.COLUMNAR_FILESCAN_ENABLED.key, "false") - } -} - -// Same as above except AQE is off. -class GlutenDynamicPartitionPruningV1SuiteAEOffDisableScan - extends GlutenDynamicPartitionPruningV2SuiteAEOff { - override def sparkConf: SparkConf = { - super.sparkConf.set(GlutenConfig.COLUMNAR_FILESCAN_ENABLED.key, "false") - } -} - -class GlutenDynamicPartitionPruningV1SuiteAEOffWSCGOnDisableProject - extends GlutenDynamicPartitionPruningV2SuiteAEOff { - override def sparkConf: SparkConf = { - super.sparkConf.set(GlutenConfig.COLUMNAR_PROJECT_ENABLED.key, "false") - } -} - -class GlutenDynamicPartitionPruningV1SuiteAEOffWSCGOffDisableProject - extends GlutenDynamicPartitionPruningV2SuiteAEOff { - override def sparkConf: SparkConf = { - super.sparkConf - .set(GlutenConfig.COLUMNAR_PROJECT_ENABLED.key, "false") - .set(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, "false") - } -} - -// Test DPP with batch scan disabled by user for some reason, which can also mock the situation -// that scan is not transformable. -class GlutenDynamicPartitionPruningV2SuiteAEOnDisableScan - extends GlutenDynamicPartitionPruningV2SuiteAEOn { - override def sparkConf: SparkConf = { - super.sparkConf.set(GlutenConfig.COLUMNAR_BATCHSCAN_ENABLED.key, "false") - } -} - -// Same as above except AQE is off. -class GlutenDynamicPartitionPruningV2SuiteAEOffDisableScan - extends GlutenDynamicPartitionPruningV2SuiteAEOff { - override def sparkConf: SparkConf = { - super.sparkConf.set(GlutenConfig.COLUMNAR_BATCHSCAN_ENABLED.key, "false") - } -} - -class GlutenDynamicPartitionPruningV2SuiteAEOffWSCGOnDisableProject - extends GlutenDynamicPartitionPruningV2SuiteAEOff { - override def sparkConf: SparkConf = { - super.sparkConf.set(GlutenConfig.COLUMNAR_PROJECT_ENABLED.key, "false") - } -} - -class GlutenDynamicPartitionPruningV2SuiteAEOffWSCGOffDisableProject - extends GlutenDynamicPartitionPruningV2SuiteAEOff { - override def sparkConf: SparkConf = { - super.sparkConf - .set(GlutenConfig.COLUMNAR_PROJECT_ENABLED.key, "false") - .set(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, "false") - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenExpressionsSchemaSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenExpressionsSchemaSuite.scala deleted file mode 100644 index 0dd285c7426..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenExpressionsSchemaSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenExpressionsSchemaSuite extends ExpressionsSchemaSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenExtraStrategiesSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenExtraStrategiesSuite.scala deleted file mode 100644 index 3c3b438f3cf..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenExtraStrategiesSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenExtraStrategiesSuite extends ExtraStrategiesSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenFileBasedDataSourceSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenFileBasedDataSourceSuite.scala deleted file mode 100644 index e8264556d59..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenFileBasedDataSourceSuite.scala +++ /dev/null @@ -1,248 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -import org.apache.gluten.config.GlutenConfig - -import org.apache.spark.{SparkConf, SparkException} -import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd} -import org.apache.spark.sql.GlutenTestConstants.GLUTEN_TEST -import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, SortMergeJoinExec} -import org.apache.spark.sql.internal.SQLConf - -import org.apache.hadoop.fs.Path - -import java.io.FileNotFoundException - -import scala.collection.mutable - -class GlutenFileBasedDataSourceSuite extends FileBasedDataSourceSuite with GlutenSQLTestsTrait { - import testImplicits._ - - override def sparkConf: SparkConf = { - super.sparkConf - .set(GlutenConfig.COLUMNAR_FORCE_SHUFFLED_HASH_JOIN_ENABLED.key, "false") - .set(SQLConf.SHUFFLE_PARTITIONS.key, "5") - } - - // test data path is jar path, so failed, test code is same with spark - testGluten("Option recursiveFileLookup: disable partition inferring") { - val dataPath = getWorkspaceFilePath( - "sql", - "core", - "src", - "test", - "resources").toString + "/" + "test-data/text-partitioned" - - val df = spark.read - .format("binaryFile") - .option("recursiveFileLookup", true) - .load(dataPath) - - assert(!df.columns.contains("year"), "Expect partition inferring disabled") - val fileList = df.select("path").collect().map(_.getString(0)) - - val expectedFileList = Array( - dataPath + "/year=2014/data.txt", - dataPath + "/year=2015/data.txt" - ).map(path => "file:" + new Path(path).toString) - - assert(fileList.toSet === expectedFileList.toSet) - } - - testGluten("Spark native readers should respect spark.sql.caseSensitive - parquet") { - withTempDir { - dir => - val format = "parquet" - val tableName = s"spark_25132_${format}_native" - val tableDir = dir.getCanonicalPath + s"/$tableName" - withTable(tableName) { - val end = 5 - val data = spark.range(end).selectExpr("id as A", "id * 2 as b", "id * 3 as B") - withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") { - data.write.format(format).mode("overwrite").save(tableDir) - } - sql(s"CREATE TABLE $tableName (a LONG, b LONG) USING $format LOCATION '$tableDir'") - - withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") { - checkAnswer(sql(s"select a from $tableName"), data.select("A")) - checkAnswer(sql(s"select A from $tableName"), data.select("A")) - - // TODO: gluten can catch exception in executor side, but cannot catch SparkException - // in Driver side - // RuntimeException is triggered at executor side, which is then wrapped as - // SparkException at driver side - // val e1 = intercept[SparkException] { - // sql(s"select b from $tableName").collect() - // } - // - // assert( - // e1.getCause.isInstanceOf[RuntimeException] && - // e1.getMessage.contains( - // """Found duplicate field(s) b in case-insensitive mode """)) - // val e2 = intercept[SparkException] { - // sql(s"select B from $tableName").collect() - // } - // assert( - // e2.getCause.isInstanceOf[RuntimeException] && - // e2.getMessage.contains( - // """Found duplicate field(s) b in case-insensitive mode""")) - } - - withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") { - checkAnswer(sql(s"select a from $tableName"), (0 until end).map(_ => Row(null))) - checkAnswer(sql(s"select b from $tableName"), data.select("b")) - } - } - } - } - - testGluten("SPARK-22790,SPARK-27668: spark.sql.sources.compressionFactor takes effect") { - Seq(1.0, 0.5).foreach { - compressionFactor => - withSQLConf( - SQLConf.FILE_COMPRESSION_FACTOR.key -> compressionFactor.toString, - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "350") { - withTempPath { - workDir => - // the file size is 504 bytes - val workDirPath = workDir.getAbsolutePath - val data1 = Seq(100, 200, 300, 400).toDF("count") - data1.write.orc(workDirPath + "/data1") - val df1FromFile = spark.read.orc(workDirPath + "/data1") - val data2 = Seq(100, 200, 300, 400).toDF("count") - data2.write.orc(workDirPath + "/data2") - val df2FromFile = spark.read.orc(workDirPath + "/data2") - val joinedDF = df1FromFile.join(df2FromFile, Seq("count")) - if (compressionFactor == 0.5) { - val bJoinExec = collect(joinedDF.queryExecution.executedPlan) { - case bJoin: BroadcastHashJoinExec => bJoin - } - assert(bJoinExec.nonEmpty) - val smJoinExec = collect(joinedDF.queryExecution.executedPlan) { - case smJoin: SortMergeJoinExec => smJoin - } - assert(smJoinExec.isEmpty) - } else { - // compressionFactor is 1.0 - val bJoinExec = collect(joinedDF.queryExecution.executedPlan) { - case bJoin: BroadcastHashJoinExec => bJoin - } - assert(bJoinExec.isEmpty) - val smJoinExec = collect(joinedDF.queryExecution.executedPlan) { - case smJoin: SortMergeJoinExec => smJoin - } - assert(smJoinExec.nonEmpty) - } - } - } - } - } - - testGluten("SPARK-25237 compute correct input metrics in FileScanRDD") { - // TODO: Test CSV V2 as well after it implements [[SupportsReportStatistics]]. - withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "csv") { - withTempPath { - p => - val path = p.getAbsolutePath - spark.range(1000).repartition(1).write.csv(path) - val bytesReads = new mutable.ArrayBuffer[Long]() - val bytesReadListener = new SparkListener() { - override def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit = { - bytesReads += taskEnd.taskMetrics.inputMetrics.bytesRead - } - } - sparkContext.addSparkListener(bytesReadListener) - try { - spark.read.csv(path).limit(1).collect() - sparkContext.listenerBus.waitUntilEmpty() - // plan is different, so metric is different - assert(bytesReads.sum === 7864) - } finally { - sparkContext.removeSparkListener(bytesReadListener) - } - } - } - } - - Seq("orc", "parquet").foreach { - format => - testQuietly(GLUTEN_TEST + s"Enabling/disabling ignoreMissingFiles using $format") { - def testIgnoreMissingFiles(options: Map[String, String]): Unit = { - withTempDir { - dir => - val basePath = dir.getCanonicalPath - - Seq("0").toDF("a").write.format(format).save(new Path(basePath, "second").toString) - Seq("1").toDF("a").write.format(format).save(new Path(basePath, "fourth").toString) - - val firstPath = new Path(basePath, "first") - val thirdPath = new Path(basePath, "third") - val fs = thirdPath.getFileSystem(spark.sessionState.newHadoopConf()) - Seq("2").toDF("a").write.format(format).save(firstPath.toString) - Seq("3").toDF("a").write.format(format).save(thirdPath.toString) - val files = Seq(firstPath, thirdPath).flatMap { - p => fs.listStatus(p).filter(_.isFile).map(_.getPath) - } - - val df = spark.read - .options(options) - .format(format) - .load( - new Path(basePath, "first").toString, - new Path(basePath, "second").toString, - new Path(basePath, "third").toString, - new Path(basePath, "fourth").toString) - - // Make sure all data files are deleted and can't be opened. - files.foreach(f => fs.delete(f, false)) - assert(fs.delete(thirdPath, true)) - for (f <- files) { - intercept[FileNotFoundException](fs.open(f)) - } - - checkAnswer(df, Seq(Row("0"), Row("1"))) - } - } - - // Test set ignoreMissingFiles via SQL Conf - // Rewrite this test as error msg is different from velox - for { - (ignore, options, sqlConf) <- Seq( - // Set via SQL Conf: leave options empty - ("true", Map.empty[String, String], "true"), - ("false", Map.empty[String, String], "false") - ) - sources <- Seq("", format) - } { - withSQLConf( - SQLConf.USE_V1_SOURCE_LIST.key -> sources, - SQLConf.IGNORE_MISSING_FILES.key -> sqlConf) { - if (ignore.toBoolean) { - testIgnoreMissingFiles(options) - } else { - val exception = intercept[SparkException] { - testIgnoreMissingFiles(options) - } - assert(exception.getMessage().contains("No such file or directory")) - } - } - } - } - } - -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenFileScanSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenFileScanSuite.scala deleted file mode 100644 index d5885afaee9..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenFileScanSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenFileScanSuite extends FileScanSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenGeneratorFunctionSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenGeneratorFunctionSuite.scala deleted file mode 100644 index ea0330f9c26..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenGeneratorFunctionSuite.scala +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -import org.apache.gluten.execution.GenerateExecTransformerBase - -class GlutenGeneratorFunctionSuite extends GeneratorFunctionSuite with GlutenSQLTestsTrait { - testGluten("stack is offloaded") { - val df = spark.range(2).selectExpr("stack(2, id, id + 1, id + 2)") - checkAnswer(df, Seq(Row(0L, 1L), Row(2L, null), Row(1L, 2L), Row(3L, null))) - assert( - df.queryExecution.executedPlan - .find(_.isInstanceOf[GenerateExecTransformerBase]) - .isDefined) - } - - testGluten("stack without null padding is offloaded") { - val df = spark.range(2).selectExpr("stack(2, id, id + 1, id + 2, id + 3)") - checkAnswer(df, Seq(Row(0L, 1L), Row(2L, 3L), Row(1L, 2L), Row(3L, 4L))) - assert( - df.queryExecution.executedPlan - .find(_.isInstanceOf[GenerateExecTransformerBase]) - .isDefined) - } - - testGluten("single-column stack with null padding is offloaded") { - val df = spark.range(2).selectExpr("stack(3, id, id + 1)") - checkAnswer(df, Seq(Row(0L), Row(1L), Row(null), Row(1L), Row(2L), Row(null))) - assert( - df.queryExecution.executedPlan - .find(_.isInstanceOf[GenerateExecTransformerBase]) - .isDefined) - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenInjectRuntimeFilterSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenInjectRuntimeFilterSuite.scala deleted file mode 100644 index 11b6d99828c..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenInjectRuntimeFilterSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenInjectRuntimeFilterSuite - extends InjectRuntimeFilterSuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenIntervalFunctionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenIntervalFunctionsSuite.scala deleted file mode 100644 index 0a354a1fc39..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenIntervalFunctionsSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenIntervalFunctionsSuite extends IntervalFunctionsSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenJoinSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenJoinSuite.scala deleted file mode 100644 index 8a5a5923f72..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenJoinSuite.scala +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenJoinSuite extends JoinSuite with GlutenSQLTestsTrait { - - override def testNameBlackList: Seq[String] = Seq( - // Below tests are to verify operators, just skip. - "join operator selection", - "broadcasted hash join operator selection", - "broadcasted hash outer join operator selection", - "broadcasted existence join operator selection", - "SPARK-28323: PythonUDF should be able to use in join condition", - "SPARK-28345: PythonUDF predicate should be able to pushdown to join", - "cross join with broadcast", - "test SortMergeJoin output ordering", - "SPARK-22445 Respect stream-side child's needCopyResult in BroadcastHashJoin", - "SPARK-32330: Preserve shuffled hash join build side partitioning", - "SPARK-32383: Preserve hash join (BHJ and SHJ) stream side ordering", - "SPARK-32399: Full outer shuffled hash join", - "SPARK-32649: Optimize BHJ/SHJ inner/semi join with empty hashed relation", - "SPARK-34593: Preserve broadcast nested loop join partitioning and ordering", - "SPARK-35984: Config to force applying shuffled hash join", - "test SortMergeJoin (with spill)", - // NaN is not supported currently, just skip. - "NaN and -0.0 in join keys" - ) - - testGluten("test case sensitive for BHJ") { - spark.sql("create table t_bhj(a int, b int, C int) using parquet") - spark.sql("insert overwrite t_bhj select id as a, (id+1) as b, (id+2) as c from range(3)") - val sql = - """ - |select /*+ BROADCAST(t1) */ t0.a, t0.b - |from t_bhj as t0 join t_bhj as t1 on t0.a = t1.a and t0.b = t1.b and t0.c = t1.c - |group by t0.a, t0.b - |order by t0.a, t0.b - |""".stripMargin - checkAnswer(spark.sql(sql), Seq(Row(0, 1), Row(1, 2), Row(2, 3))) - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenJsonFunctionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenJsonFunctionsSuite.scala deleted file mode 100644 index cba4e7a3755..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenJsonFunctionsSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenJsonFunctionsSuite extends JsonFunctionsSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenMathFunctionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenMathFunctionsSuite.scala deleted file mode 100644 index ee39f013850..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenMathFunctionsSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenMathFunctionsSuite extends MathFunctionsSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenMetadataCacheSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenMetadataCacheSuite.scala deleted file mode 100644 index d9fc6fd05e1..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenMetadataCacheSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenMetadataCacheSuite extends MetadataCacheSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenMiscFunctionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenMiscFunctionsSuite.scala deleted file mode 100644 index a95d8a2b2e5..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenMiscFunctionsSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenMiscFunctionsSuite extends MiscFunctionsSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenNestedDataSourceSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenNestedDataSourceSuite.scala deleted file mode 100644 index d139221f631..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenNestedDataSourceSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenNestedDataSourceV1Suite extends NestedDataSourceV1Suite with GlutenSQLTestsTrait {} - -class GlutenNestedDataSourceV2Suite extends NestedDataSourceV2Suite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenProcessingTimeSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenProcessingTimeSuite.scala deleted file mode 100644 index f8ab9b16adf..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenProcessingTimeSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenProcessingTimeSuite extends ProcessingTimeSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenProductAggSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenProductAggSuite.scala deleted file mode 100644 index 9cb35efbfbd..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenProductAggSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenProductAggSuite extends ProductAggSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenReplaceNullWithFalseInPredicateEndToEndSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenReplaceNullWithFalseInPredicateEndToEndSuite.scala deleted file mode 100644 index e345309ab11..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenReplaceNullWithFalseInPredicateEndToEndSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenReplaceNullWithFalseInPredicateEndToEndSuite - extends ReplaceNullWithFalseInPredicateEndToEndSuite - with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSQLInsertTestSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSQLInsertTestSuite.scala deleted file mode 100644 index b99c2aef6cb..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSQLInsertTestSuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenFileSourceSQLInsertTestSuite - extends FileSourceSQLInsertTestSuite - with GlutenSQLTestsTrait {} - -class GlutenDSV2SQLInsertTestSuite extends DSV2SQLInsertTestSuite {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSQLQuerySuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSQLQuerySuite.scala deleted file mode 100644 index 424fd1b3ec8..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSQLQuerySuite.scala +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -import org.apache.spark.SparkException -import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec -import org.apache.spark.sql.internal.SQLConf - -class GlutenSQLQuerySuite extends SQLQuerySuite with GlutenSQLTestsTrait { - import testImplicits._ - - testGluten("SPARK-28156: self-join should not miss cached view") { - withTable("table1") { - withView("table1_vw") { - withTempView("cachedview") { - val df = Seq.tabulate(5)(x => (x, x + 1, x + 2, x + 3)).toDF("a", "b", "c", "d") - df.write.mode("overwrite").format("parquet").saveAsTable("table1") - sql("drop view if exists table1_vw") - sql("create view table1_vw as select * from table1") - - val cachedView = sql("select a, b, c, d from table1_vw") - - cachedView.createOrReplaceTempView("cachedview") - cachedView.persist() - - val queryDf = sql(s"""select leftside.a, leftside.b - |from cachedview leftside - |join cachedview rightside - |on leftside.a = rightside.a - """.stripMargin) - - val inMemoryTableScan = collect(queryDf.queryExecution.executedPlan) { - case i: InMemoryTableScanExec => i - } - assert(inMemoryTableScan.size == 2) - checkAnswer(queryDf, Row(0, 1) :: Row(1, 2) :: Row(2, 3) :: Row(3, 4) :: Row(4, 5) :: Nil) - } - } - } - - } - - testGluten("SPARK-33338: GROUP BY using literal map should not fail") { - withTable("t") { - withTempDir { - dir => - sql( - s"CREATE TABLE t USING PARQUET LOCATION '${dir.toURI}' AS SELECT map('k1', 'v1') m," + - s" 'k1' k") - Seq( - "SELECT map('k1', 'v1')[k] FROM t GROUP BY 1", - "SELECT map('k1', 'v1')[k] FROM t GROUP BY map('k1', 'v1')[k]", - "SELECT map('k1', 'v1')[k] a FROM t GROUP BY a" - ).foreach(statement => checkAnswer(sql(statement), Row("v1"))) - } - } - } - - testGluten("SPARK-33593: Vector reader got incorrect data with binary partition value") { - Seq("false").foreach( - value => { - withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> value) { - withTable("t1") { - sql("""CREATE TABLE t1(name STRING, id BINARY, part BINARY) - |USING PARQUET PARTITIONED BY (part)""".stripMargin) - sql("INSERT INTO t1 PARTITION(part = 'Spark SQL') VALUES('a', X'537061726B2053514C')") - checkAnswer( - sql("SELECT name, cast(id as string), cast(part as string) FROM t1"), - Row("a", "Spark SQL", "Spark SQL")) - } - } - - withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> value) { - withTable("t2") { - sql("""CREATE TABLE t2(name STRING, id BINARY, part BINARY) - |USING PARQUET PARTITIONED BY (part)""".stripMargin) - sql("INSERT INTO t2 PARTITION(part = 'Spark SQL') VALUES('a', X'537061726B2053514C')") - checkAnswer( - sql("SELECT name, cast(id as string), cast(part as string) FROM t2"), - Row("a", "Spark SQL", "Spark SQL")) - } - } - }) - } - - testGluten( - "SPARK-33677: LikeSimplification should be skipped if pattern contains any escapeChar") { - withTempView("df") { - Seq("m@ca").toDF("s").createOrReplaceTempView("df") - - val e = intercept[SparkException] { - sql("SELECT s LIKE 'm%@ca' ESCAPE '%' FROM df").collect() - } - assert( - e.getMessage.contains( - "Escape character must be followed by '%', '_' or the escape character itself")) - - checkAnswer(sql("SELECT s LIKE 'm@@ca' ESCAPE '@' FROM df"), Row(true)) - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala deleted file mode 100644 index 5039d4ccf44..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala +++ /dev/null @@ -1,806 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.exception.GlutenException -import org.apache.gluten.utils.{BackendTestSettings, BackendTestUtils} - -import org.apache.spark.{SparkConf, SparkException} -import org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator -import org.apache.spark.sql.catalyst.plans.SQLHelper -import org.apache.spark.sql.catalyst.rules.RuleExecutor -import org.apache.spark.sql.catalyst.util.{fileToString, stringToFile} -import org.apache.spark.sql.catalyst.util.DateTimeConstants.NANOS_PER_SECOND -import org.apache.spark.sql.execution.WholeStageCodegenExec -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.internal.SQLConf.TimestampTypes -import org.apache.spark.sql.test.SharedSparkSession -import org.apache.spark.tags.ExtendedSQLTest -import org.apache.spark.util.Utils - -import java.io.{File, FileNotFoundException} -import java.net.URI -import java.util.Locale - -import scala.collection.mutable.ArrayBuffer -import scala.sys.process.{Process, ProcessLogger} -import scala.util.Try -import scala.util.control.NonFatal - -/** - * End-to-end test cases for SQL queries. - * - * Each case is loaded from a file in "spark/sql/core/src/test/resources/sql-tests/inputs". Each - * case has a golden result file in "spark/sql/core/src/test/resources/sql-tests/results". - * - * To run the entire test suite: - * {{{ - * build/sbt "sql/testOnly *SQLQueryTestSuite" - * }}} - * - * To run a single test file upon change: - * {{{ - * build/sbt "~sql/testOnly *SQLQueryTestSuite -- -z inline-table.sql" - * }}} - * - * To re-generate golden files for entire suite, run: - * {{{ - * SPARK_GENERATE_GOLDEN_FILES=1 build/sbt "sql/testOnly *SQLQueryTestSuite" - * }}} - * - * To re-generate golden file for a single test, run: - * {{{ - * SPARK_GENERATE_GOLDEN_FILES=1 build/sbt "sql/testOnly *SQLQueryTestSuite -- -z describe.sql" - * }}} - * - * The format for input files is simple: - * 1. A list of SQL queries separated by semicolons by default. If the semicolon cannot - * effectively separate the SQL queries in the test file(e.g. bracketed comments), please use - * --QUERY-DELIMITER-START and --QUERY-DELIMITER-END. Lines starting with --QUERY-DELIMITER-START - * and --QUERY-DELIMITER-END represent the beginning and end of a query, respectively. Code that is - * not surrounded by lines that begin with --QUERY-DELIMITER-START and --QUERY-DELIMITER-END is - * still separated by semicolons. 2. Lines starting with -- are treated as comments and ignored. 3. - * Lines starting with --SET are used to specify the configs when running this testing file. You can - * set multiple configs in one --SET, using comma to separate them. Or you can use multiple --SET - * statements. 4. Lines starting with --IMPORT are used to load queries from another test file. 5. - * Lines starting with --CONFIG_DIM are used to specify config dimensions of this testing file. The - * dimension name is decided by the string after --CONFIG_DIM. For example, --CONFIG_DIM1 belongs to - * dimension 1. One dimension can have multiple lines, each line representing one config set (one or - * more configs, separated by comma). Spark will run this testing file many times, each time picks - * one config set from each dimension, until all the combinations are tried. For example, if - * dimension 1 has 2 lines, dimension 2 has 3 lines, this testing file will be run 6 times - * (cartesian product). - * - * For example: - * {{{ - * -- this is a comment - * select 1, -1; - * select current_date; - * }}} - * - * The format for golden result files look roughly like: - * {{{ - * -- some header information - * - * -- !query - * select 1, -1 - * -- !query schema - * struct<...schema...> - * -- !query output - * ... data row 1 ... - * ... data row 2 ... - * ... - * - * -- !query - * ... - * }}} - * - * Note that UDF tests work differently. After the test files under 'inputs/udf' directory are - * detected, it creates three test cases: - * - * - Scala UDF test case with a Scalar UDF registered as the name 'udf'. - * - Python UDF test case with a Python UDF registered as the name 'udf' iff Python executable and - * pyspark are available. - * - Scalar Pandas UDF test case with a Scalar Pandas UDF registered as the name 'udf' iff Python - * executable, pyspark, pandas and pyarrow are available. - * - * Therefore, UDF test cases should have single input and output files but executed by three - * different types of UDFs. See 'udf/udf-inner-join.sql' as an example. - */ -@ExtendedSQLTest -class GlutenSQLQueryTestSuite - extends QueryTest - with SharedSparkSession - with SQLHelper - with SQLQueryTestHelper { - - import IntegratedUDFTestUtils._ - - private val regenerateGoldenFiles: Boolean = System.getenv("SPARK_GENERATE_GOLDEN_FILES") == "1" - - // FIXME it's not needed to install Spark in testing since the following code only fetchs - // some resource files from source folder - - protected val baseResourcePath = { - // We use a path based on Spark home for 2 reasons: - // 1. Maven can't get correct resource directory when resources in other jars. - // 2. We test subclasses in the hive-thriftserver module. - getWorkspaceFilePath("sql", "core", "src", "test", "resources", "sql-tests").toFile - } - - protected val resourcesPath = { - // We use a path based on Spark home for 2 reasons: - // 1. Maven can't get correct resource directory when resources in other jars. - // 2. We test subclasses in the hive-thriftserver module. - getWorkspaceFilePath("sql", "core", "src", "test", "resources").toFile - } - - protected val inputFilePath = new File(baseResourcePath, "inputs").getAbsolutePath - protected val goldenFilePath = new File(baseResourcePath, "results").getAbsolutePath - protected val testDataPath = new File(resourcesPath, "test-data").getAbsolutePath - - protected val overwriteResourcePath = - getClass.getResource("/").getPath + "../../../src/test/resources/sql-tests" - protected val overwriteInputFilePath = new File(overwriteResourcePath, "inputs").getAbsolutePath - protected val overwriteGoldenFilePath = new File(overwriteResourcePath, "results").getAbsolutePath - - protected val validFileExtensions = ".sql" - - /** Test if a command is available. */ - def testCommandAvailable(command: String): Boolean = { - val attempt = if (Utils.isWindows) { - Try(Process(Seq("cmd.exe", "/C", s"where $command")).run(ProcessLogger(_ => ())).exitValue()) - } else { - Try(Process(Seq("sh", "-c", s"command -v $command")).run(ProcessLogger(_ => ())).exitValue()) - } - attempt.isSuccess && attempt.get == 0 - } - - private val isCHBackend = BackendTestUtils.isCHBackendLoaded() - - override protected def sparkConf: SparkConf = { - val conf = super.sparkConf - // Fewer shuffle partitions to speed up testing. - .set(SQLConf.SHUFFLE_PARTITIONS, 4) - // use Java 8 time API to handle negative years properly - .set(SQLConf.DATETIME_JAVA8API_ENABLED, true) - .setAppName("Gluten-UT") - .set("spark.driver.memory", "1G") - .set("spark.sql.adaptive.enabled", "true") - .set("spark.sql.files.maxPartitionBytes", "134217728") - .set("spark.memory.offHeap.enabled", "true") - .set("spark.memory.offHeap.size", "1024MB") - .set("spark.plugins", "org.apache.gluten.GlutenPlugin") - .set("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager") - .set(GlutenConfig.SMALL_FILE_THRESHOLD.key, "0") - - if (isCHBackend) { - conf - .set("spark.io.compression.codec", "LZ4") - .set("spark.gluten.sql.columnar.backend.ch.worker.id", "1") - .set(GlutenConfig.NATIVE_VALIDATION_ENABLED.key, "false") - .set("spark.sql.files.openCostInBytes", "134217728") - .set("spark.unsafe.exceptionOnMemoryLeak", "true") - } else { - conf.set("spark.unsafe.exceptionOnMemoryLeak", "true") - } - conf - } - - // SPARK-32106 Since we add SQL test 'transform.sql' will use `cat` command, - // here we need to ignore it. - private val otherIgnoreList = - if (testCommandAvailable("/bin/bash")) Nil else Set("transform.sql") - - /** List of test cases to ignore, in lower cases. */ - protected def ignoreList: Set[String] = Set( - "ignored.sql", // Do NOT remove this one. It is here to test the ignore functionality. - "explain-aqe.sql", // explain plan is different - "explain-cbo.sql", // explain - "explain.sql", // explain - "udf/udf-window.sql", // Local window fixes are not added. - "window.sql" // Local window fixes are not added. - ) ++ otherIgnoreList - - // List of supported cases to run with a certain backend, in lower case. - private val supportedList: Set[String] = - BackendTestSettings.instance.getSQLQueryTestSettings.getSupportedSQLQueryTests ++ - BackendTestSettings.instance.getSQLQueryTestSettings.getOverwriteSQLQueryTests - // Create all the test cases. - listTestCases.foreach(createScalaTestCase) - - /** A single SQL query's output. */ - protected case class QueryOutput(sql: String, schema: String, output: String) { - override def toString: String = { - // We are explicitly not using multi-line string due to stripMargin removing "|" in output. - s"-- !query\n" + - sql + "\n" + - s"-- !query schema\n" + - schema + "\n" + - s"-- !query output\n" + - output - } - } - - /** A test case. */ - protected trait TestCase { - val name: String - val inputFile: String - val resultFile: String - } - - /** - * traits that indicate UDF or PgSQL to trigger the code path specific to each. For instance, - * PgSQL tests require to register some UDF functions. - */ - protected trait PgSQLTest - - /** traits that indicate ANSI-related tests with the ANSI mode enabled. */ - protected trait AnsiTest - - /** traits that indicate the default timestamp type is TimestampNTZType. */ - protected trait TimestampNTZTest - - protected trait UDFTest { - val udf: TestUDF - } - - /** A regular test case. */ - protected case class RegularTestCase(name: String, inputFile: String, resultFile: String) - extends TestCase - - /** A PostgreSQL test case. */ - protected case class PgSQLTestCase(name: String, inputFile: String, resultFile: String) - extends TestCase - with PgSQLTest - - /** A UDF test case. */ - protected case class UDFTestCase( - name: String, - inputFile: String, - resultFile: String, - udf: TestUDF) - extends TestCase - with UDFTest - - /** A UDF PostgreSQL test case. */ - protected case class UDFPgSQLTestCase( - name: String, - inputFile: String, - resultFile: String, - udf: TestUDF) - extends TestCase - with UDFTest - with PgSQLTest - - /** An ANSI-related test case. */ - protected case class AnsiTestCase(name: String, inputFile: String, resultFile: String) - extends TestCase - with AnsiTest - - /** An date time test case with default timestamp as TimestampNTZType */ - protected case class TimestampNTZTestCase(name: String, inputFile: String, resultFile: String) - extends TestCase - with TimestampNTZTest - - protected def createScalaTestCase(testCase: TestCase): Unit = { - // If a test case is not in the test list, or it is in the ignore list, ignore this test case. - if ( - !supportedList.exists( - t => testCase.name.toLowerCase(Locale.ROOT).contains(t.toLowerCase(Locale.ROOT))) || - ignoreList.exists( - t => testCase.name.toLowerCase(Locale.ROOT).contains(t.toLowerCase(Locale.ROOT))) - ) { - // Create a test case to ignore this case. - ignore(testCase.name) { /* Do nothing */ } - } else { - testCase match { - case udfTestCase: UDFTest - if udfTestCase.udf.isInstanceOf[TestPythonUDF] && !shouldTestPythonUDFs => - ignore( - s"${testCase.name} is skipped because " + - s"[$pythonExec] and/or pyspark were not available.") { - /* Do nothing */ - } - case udfTestCase: UDFTest - if udfTestCase.udf.isInstanceOf[TestScalarPandasUDF] && !shouldTestScalarPandasUDFs => - ignore( - s"${testCase.name} is skipped because pyspark," + - s"pandas and/or pyarrow were not available in [$pythonExec].") { - /* Do nothing */ - } - case _ => - // Create a test case to run this case. - test(testCase.name) { - runTest(testCase) - } - } - } - } - - /** Run a test case. */ - protected def runTest(testCase: TestCase): Unit = { - def splitWithSemicolon(seq: Seq[String]) = { - seq.mkString("\n").split("(?<=[^\\\\]);") - } - - def splitCommentsAndCodes(input: String) = input.split("\n").partition { - line => - val newLine = line.trim - newLine.startsWith("--") && !newLine.startsWith("--QUERY-DELIMITER") - } - - val input = fileToString(new File(testCase.inputFile)) - - val (comments, code) = splitCommentsAndCodes(input) - - // If `--IMPORT` found, load code from another test case file, then insert them - // into the head in this test. - val importedTestCaseName = comments.filter(_.startsWith("--IMPORT ")).map(_.substring(9)) - val importedCode = importedTestCaseName.flatMap { - testCaseName => - listTestCases.find(_.name == testCaseName).map { - testCase => - val input = fileToString(new File(testCase.inputFile)) - val (_, code) = splitCommentsAndCodes(input) - code - } - }.flatten - - val allCode = importedCode ++ code - val tempQueries = if (allCode.exists(_.trim.startsWith("--QUERY-DELIMITER"))) { - // Although the loop is heavy, only used for bracketed comments test. - val queries = new ArrayBuffer[String] - val otherCodes = new ArrayBuffer[String] - var tempStr = "" - var start = false - for (c <- allCode) { - if (c.trim.startsWith("--QUERY-DELIMITER-START")) { - start = true - queries ++= splitWithSemicolon(otherCodes.toSeq) - otherCodes.clear() - } else if (c.trim.startsWith("--QUERY-DELIMITER-END")) { - start = false - queries += s"\n${tempStr.stripSuffix(";")}" - tempStr = "" - } else if (start) { - tempStr += s"\n$c" - } else { - otherCodes += c - } - } - if (otherCodes.nonEmpty) { - queries ++= splitWithSemicolon(otherCodes.toSeq) - } - queries.toSeq - } else { - splitWithSemicolon(allCode).toSeq - } - - // List of SQL queries to run - val queries = tempQueries - .map(_.trim) - .filter(_ != "") - .toSeq - // Fix misplacement when comment is at the end of the query. - .map(_.split("\n").filterNot(_.startsWith("--")).mkString("\n")) - .map(_.trim) - .filter(_ != "") - - val settingLines = comments.filter(_.startsWith("--SET ")).map(_.substring(6)) - val settings = settingLines.flatMap(_.split(",").map { - kv => - val (conf, value) = kv.span(_ != '=') - conf.trim -> value.substring(1).trim - }) - - if (regenerateGoldenFiles) { - runQueries(queries, testCase, settings) - } else { - // A config dimension has multiple config sets, and a config set has multiple configs. - // - config dim: Seq[Seq[(String, String)]] - // - config set: Seq[(String, String)] - // - config: (String, String)) - // We need to do cartesian product for all the config dimensions, to get a list of - // config sets, and run the query once for each config set. - val configDimLines = comments.filter(_.startsWith("--CONFIG_DIM")).map(_.substring(12)) - val configDims = configDimLines.groupBy(_.takeWhile(_ != ' ')).mapValues { - lines => - lines - .map(_.dropWhile(_ != ' ').substring(1)) - .map(_.split(",") - .map { - kv => - val (conf, value) = kv.span(_ != '=') - conf.trim -> value.substring(1).trim - } - .toSeq) - .toSeq - } - - val configSets = configDims.values.foldLeft(Seq(Seq[(String, String)]())) { - (res, dim) => dim.flatMap(configSet => res.map(_ ++ configSet)) - } - - configSets.foreach { - configSet => - try { - runQueries(queries, testCase, settings ++ configSet) - } catch { - case e: Throwable => - val configs = configSet.map { case (k, v) => s"$k=$v" } - logError(s"Error using configs: ${configs.mkString(",")}") - throw e - } - } - } - } - - protected def runQueries( - queries: Seq[String], - testCase: TestCase, - configSet: Seq[(String, String)]): Unit = { - // Create a local SparkSession to have stronger isolation between different test cases. - // This does not isolate catalog changes. - val localSparkSession = spark.newSession() - - testCase match { - case udfTestCase: UDFTest => - registerTestUDF(udfTestCase.udf, localSparkSession) - case _ => - } - - testCase match { - case _: PgSQLTest => - // booleq/boolne used by boolean.sql - localSparkSession.udf.register("booleq", (b1: Boolean, b2: Boolean) => b1 == b2) - localSparkSession.udf.register("boolne", (b1: Boolean, b2: Boolean) => b1 != b2) - // vol used by boolean.sql and case.sql. - localSparkSession.udf.register("vol", (s: String) => s) - localSparkSession.conf.set(SQLConf.ANSI_ENABLED.key, true) - localSparkSession.conf.set(SQLConf.LEGACY_INTERVAL_ENABLED.key, true) - case _: AnsiTest => - localSparkSession.conf.set(SQLConf.ANSI_ENABLED.key, true) - case _: TimestampNTZTest => - localSparkSession.conf.set( - SQLConf.TIMESTAMP_TYPE.key, - TimestampTypes.TIMESTAMP_NTZ.toString) - case _ => - } - - if (configSet.nonEmpty) { - // Execute the list of set operation in order to add the desired configs - val setOperations = configSet.map { case (key, value) => s"set $key=$value" } - logInfo(s"Setting configs: ${setOperations.mkString(", ")}") - setOperations.foreach(localSparkSession.sql) - } - - // Run the SQL queries preparing them for comparison. - val outputs: Seq[QueryOutput] = queries.map { - sql => - val (schema, output) = handleExceptions(getNormalizedResult(localSparkSession, sql)) - // We might need to do some query canonicalization in the future. - QueryOutput( - sql = sql, - schema = schema, - output = output.mkString("\n").replaceAll("\\s+$", "")) - } - - if (regenerateGoldenFiles) { - // Again, we are explicitly not using multi-line string due to stripMargin removing "|". - val goldenOutput = { - s"-- Automatically generated by ${getClass.getSimpleName}\n" + - s"-- Number of queries: ${outputs.size}\n\n\n" + - outputs.mkString("\n\n\n") + "\n" - } - val resultFile = new File(testCase.resultFile) - val parent = resultFile.getParentFile - if (!parent.exists()) { - assert(parent.mkdirs(), "Could not create directory: " + parent) - } - stringToFile(resultFile, goldenOutput) - } - - // This is a temporary workaround for SPARK-28894. The test names are truncated after - // the last dot due to a bug in SBT. This makes easier to debug via Jenkins test result - // report. See SPARK-28894. - // See also SPARK-29127. It is difficult to see the version information in the failed test - // cases so the version information related to Python was also added. - val clue = testCase match { - case udfTestCase: UDFTest - if udfTestCase.udf.isInstanceOf[TestPythonUDF] && shouldTestPythonUDFs => - s"${testCase.name}${System.lineSeparator()}Python: $pythonVer${System.lineSeparator()}" - case udfTestCase: UDFTest - if udfTestCase.udf.isInstanceOf[TestScalarPandasUDF] && shouldTestScalarPandasUDFs => - s"${testCase.name}${System.lineSeparator()}" + - s"Python: $pythonVer Pandas: $pandasVer PyArrow: $pyarrowVer${System.lineSeparator()}" - case _ => - s"${testCase.name}${System.lineSeparator()}" - } - - withClue(clue) { - // Read back the golden file. - val expectedOutputs: Seq[QueryOutput] = { - val goldenOutput = fileToString(new File(testCase.resultFile)) - val segments = goldenOutput.split("-- !query.*\n") - - // each query has 3 segments, plus the header - assert( - segments.size == outputs.size * 3 + 1, - s"Expected ${outputs.size * 3 + 1} blocks in result file but got ${segments.size}. " + - s"Try regenerate the result files." - ) - Seq.tabulate(outputs.size) { - i => - QueryOutput( - sql = segments(i * 3 + 1).trim, - schema = segments(i * 3 + 2).trim, - output = segments(i * 3 + 3).replaceAll("\\s+$", "") - ) - } - } - - // Compare results. - assertResult(expectedOutputs.size, s"Number of queries should be ${expectedOutputs.size}") { - outputs.size - } - - outputs.zip(expectedOutputs).zipWithIndex.foreach { - case ((output, expected), i) => - assertResult(expected.sql, s"SQL query did not match for query #$i\n${expected.sql}") { - output.sql - } - assertResult( - expected.schema, - s"Schema did not match for query #$i\n${expected.sql}: $output") { - output.schema - } - assertResult( - expected.output, - s"Result did not match" + - s" for query #$i\n${expected.sql}")(output.output) - } - } - } - - protected lazy val listTestCases: Seq[TestCase] = { - val createTestCase = (file: File, parentDir: String, resultPath: String) => { - val resultFile = file.getAbsolutePath.replace(parentDir, resultPath) + ".out" - val absPath = file.getAbsolutePath - val testCaseName = absPath.stripPrefix(parentDir).stripPrefix(File.separator) - - if ( - file.getAbsolutePath.startsWith( - s"$parentDir${File.separator}udf${File.separator}postgreSQL") - ) { - Seq(TestScalaUDF("udf"), TestPythonUDF("udf"), TestScalarPandasUDF("udf")).map { - udf => UDFPgSQLTestCase(s"$testCaseName - ${udf.prettyName}", absPath, resultFile, udf) - } - } else if (file.getAbsolutePath.startsWith(s"$parentDir${File.separator}udf")) { - Seq(TestScalaUDF("udf"), TestPythonUDF("udf"), TestScalarPandasUDF("udf")).map { - udf => UDFTestCase(s"$testCaseName - ${udf.prettyName}", absPath, resultFile, udf) - } - } else if (file.getAbsolutePath.startsWith(s"$parentDir${File.separator}postgreSQL")) { - PgSQLTestCase(testCaseName, absPath, resultFile) :: Nil - } else if (file.getAbsolutePath.startsWith(s"$parentDir${File.separator}ansi")) { - AnsiTestCase(testCaseName, absPath, resultFile) :: Nil - } else if (file.getAbsolutePath.startsWith(s"$parentDir${File.separator}timestampNTZ")) { - TimestampNTZTestCase(testCaseName, absPath, resultFile) :: Nil - } else { - RegularTestCase(testCaseName, absPath, resultFile) :: Nil - } - } - val overwriteTestCases = listFilesRecursively(new File(overwriteInputFilePath)) - .flatMap(createTestCase(_, overwriteInputFilePath, overwriteGoldenFilePath)) - val overwriteTestCaseNames = overwriteTestCases.map(_.name) - listFilesRecursively(new File(inputFilePath)) - .flatMap(createTestCase(_, inputFilePath, goldenFilePath)) - .filterNot(testCase => overwriteTestCaseNames.contains(testCase.name)) ++ overwriteTestCases - } - - /** Returns all the files (not directories) in a directory, recursively. */ - protected def listFilesRecursively(path: File): Seq[File] = { - if (path.exists) { - val (dirs, files) = path.listFiles().partition(_.isDirectory) - // Filter out test files with invalid extensions such as temp files created - // by vi (.swp), Mac (.DS_Store) etc. - val filteredFiles = files.filter(_.getName.endsWith(validFileExtensions)) - filteredFiles ++ dirs.flatMap(listFilesRecursively) - } else { - throw new FileNotFoundException(s"Directory does not exist: ${path.getAbsolutePath}") - } - } - - /** Load built-in test tables into the SparkSession. */ - private def createTestTables(session: SparkSession): Unit = { - import session.implicits._ - - // Before creating test tables, deletes orphan directories in warehouse dir - Seq("testdata", "arraydata", "mapdata", "aggtest", "onek", "tenk1").foreach { - dirName => - val f = new File(new URI(s"${conf.warehousePath}/$dirName")) - if (f.exists()) { - Utils.deleteRecursively(f) - } - } - - (1 to 100) - .map(i => (i, i.toString)) - .toDF("key", "value") - .repartition(1) - .write - .format("parquet") - .saveAsTable("testdata") - - ((Seq(1, 2, 3), Seq(Seq(1, 2, 3))) :: (Seq(2, 3, 4), Seq(Seq(2, 3, 4))) :: Nil) - .toDF("arraycol", "nestedarraycol") - .write - .format("parquet") - .saveAsTable("arraydata") - - (Tuple1(Map(1 -> "a1", 2 -> "b1", 3 -> "c1", 4 -> "d1", 5 -> "e1")) :: - Tuple1(Map(1 -> "a2", 2 -> "b2", 3 -> "c2", 4 -> "d2")) :: - Tuple1(Map(1 -> "a3", 2 -> "b3", 3 -> "c3")) :: - Tuple1(Map(1 -> "a4", 2 -> "b4")) :: - Tuple1(Map(1 -> "a5")) :: Nil) - .toDF("mapcol") - .write - .format("parquet") - .saveAsTable("mapdata") - - session.read - .format("csv") - .options(Map("delimiter" -> "\t", "header" -> "false")) - .schema("a int, b float") - .load(testDataPath + "/postgresql/agg.data") - .write - .format("parquet") - .saveAsTable("aggtest") - - session.read - .format("csv") - .options(Map("delimiter" -> "\t", "header" -> "false")) - .schema(""" - |unique1 int, - |unique2 int, - |two int, - |four int, - |ten int, - |twenty int, - |hundred int, - |thousand int, - |twothousand int, - |fivethous int, - |tenthous int, - |odd int, - |even int, - |stringu1 string, - |stringu2 string, - |string4 string - """.stripMargin) - .load(testDataPath + "/postgresql/onek.data") - .write - .format("parquet") - .saveAsTable("onek") - - session.read - .format("csv") - .options(Map("delimiter" -> "\t", "header" -> "false")) - .schema(""" - |unique1 int, - |unique2 int, - |two int, - |four int, - |ten int, - |twenty int, - |hundred int, - |thousand int, - |twothousand int, - |fivethous int, - |tenthous int, - |odd int, - |even int, - |stringu1 string, - |stringu2 string, - |string4 string - """.stripMargin) - .load(testDataPath + "/postgresql/tenk.data") - .write - .format("parquet") - .saveAsTable("tenk1") - } - - private def removeTestTables(session: SparkSession): Unit = { - session.sql("DROP TABLE IF EXISTS testdata") - session.sql("DROP TABLE IF EXISTS arraydata") - session.sql("DROP TABLE IF EXISTS mapdata") - session.sql("DROP TABLE IF EXISTS aggtest") - session.sql("DROP TABLE IF EXISTS onek") - session.sql("DROP TABLE IF EXISTS tenk1") - } - - override def beforeAll(): Unit = { - super.beforeAll() - createTestTables(spark) - RuleExecutor.resetMetrics() - CodeGenerator.resetCompileTime() - WholeStageCodegenExec.resetCodeGenTime() - } - - override def afterAll(): Unit = { - try { - removeTestTables(spark) - - // For debugging dump some statistics about how much time was spent in various optimizer rules - logWarning(RuleExecutor.dumpTimeSpent()) - - val codeGenTime = WholeStageCodegenExec.codeGenTime.toDouble / NANOS_PER_SECOND - val compileTime = CodeGenerator.compileTime.toDouble / NANOS_PER_SECOND - val codegenInfo = - s""" - |=== Metrics of Whole-stage Codegen === - |Total code generation time: $codeGenTime seconds - |Total compile time: $compileTime seconds - """.stripMargin - logWarning(codegenInfo) - } finally { - super.afterAll() - } - } - - /** - * This method handles exceptions occurred during query execution as they may need special care to - * become comparable to the expected output. - * - * @param result - * a function that returns a pair of schema and output - */ - override protected def handleExceptions( - result: => (String, Seq[String])): (String, Seq[String]) = { - try { - result - } catch { - case a: AnalysisException => - // Do not output the logical plan tree which contains expression IDs. - // Also implement a crude way of masking expression IDs in the error message - // with a generic pattern "###". - val msg = if (a.plan.nonEmpty) a.getSimpleMessage else a.getMessage - (emptySchema, Seq(a.getClass.getName, msg.replaceAll("#\\d+", "#x"))) - case s: SparkException if s.getCause != null => - // For a runtime exception, it is hard to match because its message contains - // information of stage, task ID, etc. - // To make result matching simpler, here we match the cause of the exception if it exists. - val cause = s.getCause - cause match { - case e: GlutenException => - val reasonPattern = "Reason: (.*)".r - val reason = reasonPattern.findFirstMatchIn(e.getMessage).map(_.group(1)) - - reason match { - case Some(r) => - (emptySchema, Seq(e.getClass.getName, r)) - case None => (emptySchema, Seq()) - } - case _ => (emptySchema, Seq(cause.getClass.getName, cause.getMessage)) - } - case NonFatal(e) => - // If there is an exception, put the exception class followed by the message. - (emptySchema, Seq(e.getClass.getName, e.getMessage)) - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenScalaReflectionRelationSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenScalaReflectionRelationSuite.scala deleted file mode 100644 index 75bc845b5c8..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenScalaReflectionRelationSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenScalaReflectionRelationSuite - extends ScalaReflectionRelationSuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSerializationSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSerializationSuite.scala deleted file mode 100644 index 569de43a75c..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSerializationSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenSerializationSuite extends SerializationSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSparkSessionExtensionSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSparkSessionExtensionSuite.scala deleted file mode 100644 index ae9b3901afb..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSparkSessionExtensionSuite.scala +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -import org.apache.gluten.config.GlutenConfig - -class GlutenSparkSessionExtensionSuite - extends SparkSessionExtensionSuite - with GlutenTestsCommonTrait { - - testGluten("customColumnarOp") { - val extensions = DummyFilterColmnarHelper.create { - extensions => extensions.injectPlannerStrategy(_ => DummyFilterColumnarStrategy) - } - DummyFilterColmnarHelper.withSession(extensions) { - session => - try { - session.range(2).write.format("parquet").mode("overwrite").saveAsTable("a") - def testWithFallbackSettings(scanFallback: Boolean, aggFallback: Boolean): Unit = { - session.sessionState.conf - .setConfString(GlutenConfig.COLUMNAR_FILESCAN_ENABLED.key, scanFallback.toString) - session.sessionState.conf - .setConfString(GlutenConfig.COLUMNAR_HASHAGG_ENABLED.key, aggFallback.toString) - val df = session.sql("SELECT max(id) FROM a") - val newDf = DummyFilterColmnarHelper.dfWithDummyFilterColumnar( - session, - df.queryExecution.optimizedPlan) - val result = newDf.collect - newDf.explain(true) - assert(result(0).getLong(0) == 1) - } - testWithFallbackSettings(true, true) - testWithFallbackSettings(true, false) - testWithFallbackSettings(false, true) - testWithFallbackSettings(false, false) - } finally { - session.sql(s"DROP TABLE IF EXISTS a") - } - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenStatisticsCollectionSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenStatisticsCollectionSuite.scala deleted file mode 100644 index fab70638048..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenStatisticsCollectionSuite.scala +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -import org.apache.spark.sql.catalyst.plans.logical.ColumnStat -import org.apache.spark.sql.catalyst.util.DateTimeTestUtils -import org.apache.spark.sql.catalyst.util.DateTimeUtils.TimeZoneUTC -import org.apache.spark.sql.functions.timestamp_seconds -import org.apache.spark.sql.types.{DataType, DateType, TimestampType} - -import java.util.TimeZone -import java.util.concurrent.TimeUnit - -class GlutenStatisticsCollectionSuite extends StatisticsCollectionSuite with GlutenSQLTestsTrait { - - import testImplicits._ - - testGluten("store and retrieve column stats in different time zones") { - // TODO: bug fix on TableScan. - // val (start, end) = (0, TimeUnit.DAYS.toSeconds(2)) - val (start, end) = (0, 200) - - def checkTimestampStats(t: DataType, srcTimeZone: TimeZone, dstTimeZone: TimeZone)( - checker: ColumnStat => Unit): Unit = { - val table = "time_table" - val column = "T" - val original = TimeZone.getDefault - try { - withTable(table) { - TimeZone.setDefault(srcTimeZone) - spark - .range(start, end) - .select(timestamp_seconds($"id").cast(t).as(column)) - .write - .saveAsTable(table) - sql(s"ANALYZE TABLE $table COMPUTE STATISTICS FOR COLUMNS $column") - - TimeZone.setDefault(dstTimeZone) - val stats = getCatalogTable(table).stats.get.colStats(column).toPlanStat(column, t) - checker(stats) - } - } finally { - TimeZone.setDefault(original) - } - } - - DateTimeTestUtils.outstandingZoneIds.foreach { - zid => - val timeZone = TimeZone.getTimeZone(zid) - checkTimestampStats(DateType, TimeZoneUTC, timeZone) { - stats => - assert(stats.min.get.asInstanceOf[Int] == TimeUnit.SECONDS.toDays(start)) - assert(stats.max.get.asInstanceOf[Int] == TimeUnit.SECONDS.toDays(end - 1)) - } - checkTimestampStats(TimestampType, TimeZoneUTC, timeZone) { - stats => - assert(stats.min.get.asInstanceOf[Long] == TimeUnit.SECONDS.toMicros(start)) - assert(stats.max.get.asInstanceOf[Long] == TimeUnit.SECONDS.toMicros(end - 1)) - } - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala deleted file mode 100644 index 3d82e214f03..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -import org.apache.gluten.test.FallbackUtil - -import org.apache.spark.sql.catalyst.expressions.ExpressionEvalHelper -import org.apache.spark.sql.functions._ - -import org.junit.Assert - -class GlutenStringFunctionsSuite - extends StringFunctionsSuite - with GlutenSQLTestsTrait - with ExpressionEvalHelper { - - import testImplicits._ - - testGluten("string split function with no limit and regex pattern") { - val df1 = Seq(("aaAbbAcc4")).toDF("a").select(split($"a", "A")) - checkAnswer(df1, Row(Seq("aa", "bb", "cc4"))) - Assert.assertFalse(FallbackUtil.hasFallback(df1.queryExecution.executedPlan)) - - // scalastyle:off nonascii - val df2 = Seq(("test_gluten单测_")).toDF("a").select(split($"a", "_")) - checkAnswer(df2, Row(Seq("test", "gluten单测", ""))) - // scalastyle:on nonascii - Assert.assertFalse(FallbackUtil.hasFallback(df2.queryExecution.executedPlan)) - } - - testGluten("string split function with limit explicitly set to 0") { - val df1 = Seq(("aaAbbAcc4")).toDF("a").select(split($"a", "A", 0)) - checkAnswer(df1, Row(Seq("aa", "bb", "cc4"))) - Assert.assertFalse(FallbackUtil.hasFallback(df1.queryExecution.executedPlan)) - - // scalastyle:off nonascii - val df2 = Seq(("test_gluten单测_")).toDF("a").select(split($"a", "_", 0)) - checkAnswer(df2, Row(Seq("test", "gluten单测", ""))) - // scalastyle:on nonascii - Assert.assertFalse(FallbackUtil.hasFallback(df2.queryExecution.executedPlan)) - } - - testGluten("string split function with negative limit") { - val df1 = Seq(("aaAbbAcc4")).toDF("a").select(split($"a", "A", -1)) - checkAnswer(df1, Row(Seq("aa", "bb", "cc4"))) - Assert.assertFalse(FallbackUtil.hasFallback(df1.queryExecution.executedPlan)) - - // scalastyle:off nonascii - val df2 = Seq(("test_gluten单测_")).toDF("a").select(split($"a", "_", -2)) - checkAnswer(df2, Row(Seq("test", "gluten单测", ""))) - // scalastyle:on nonascii - Assert.assertFalse(FallbackUtil.hasFallback(df2.queryExecution.executedPlan)) - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSubquerySuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSubquerySuite.scala deleted file mode 100644 index 8365d9bb62b..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSubquerySuite.scala +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -import org.apache.gluten.execution.{FileSourceScanExecTransformer, WholeStageTransformer} - -class GlutenSubquerySuite extends SubquerySuite with GlutenSQLTestsTrait { - - // Test Canceled: IntegratedUDFTestUtils.shouldTestPythonUDFs was false - override def testNameBlackList: Seq[String] = Seq( - "SPARK-28441: COUNT bug in WHERE clause (Filter) with PythonUDF", - "SPARK-28441: COUNT bug in SELECT clause (Project) with PythonUDF", - "SPARK-28441: COUNT bug in Aggregate with PythonUDF", - "SPARK-28441: COUNT bug negative examples with PythonUDF", - "SPARK-28441: COUNT bug in nested subquery with PythonUDF", - "SPARK-28441: COUNT bug with nasty predicate expr with PythonUDF", - "SPARK-28441: COUNT bug in HAVING clause (Filter) with PythonUDF", - "SPARK-28441: COUNT bug with attribute ref in subquery input and output with PythonUDF" - ) - - // === Following cases override super class's cases === - - testGluten("SPARK-26893 Allow pushdown of partition pruning subquery filters to file source") { - withTable("a", "b") { - spark.range(4).selectExpr("id", "id % 2 AS p").write.partitionBy("p").saveAsTable("a") - spark.range(2).write.saveAsTable("b") - - // need to execute the query before we can examine fs.inputRDDs() - val df = sql("SELECT * FROM a WHERE p <= (SELECT MIN(id) FROM b)") - checkAnswer(df, Seq(Row(0, 0), Row(2, 0))) - assert(stripAQEPlan(df.queryExecution.executedPlan).collectFirst { - case t: WholeStageTransformer => t - } match { - case Some(WholeStageTransformer(fs: FileSourceScanExecTransformer, _)) => - fs.getPartitionArray - .exists(_.files.exists(_.getPath.toString.contains("p=0"))) - case _ => false - }) - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenTypedImperativeAggregateSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenTypedImperativeAggregateSuite.scala deleted file mode 100644 index cff309cfce2..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenTypedImperativeAggregateSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenTypedImperativeAggregateSuite - extends TypedImperativeAggregateSuite - with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenUnwrapCastInComparisonEndToEndSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenUnwrapCastInComparisonEndToEndSuite.scala deleted file mode 100644 index 2dcde94c13c..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenUnwrapCastInComparisonEndToEndSuite.scala +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenUnwrapCastInComparisonEndToEndSuite - extends UnwrapCastInComparisonEndToEndSuite - with GlutenSQLTestsTrait { - - import testImplicits._ - - testGluten("cases when literal is max") { - withTable(t) { - Seq[(Integer, java.lang.Short, java.lang.Float)]( - (1, 100.toShort, 3.14.toFloat), - (2, Short.MaxValue, Float.NaN), - (3, Short.MinValue, Float.PositiveInfinity), - (4, 0.toShort, Float.MaxValue), - (5, null, null)) - .toDF("c1", "c2", "c3") - .write - .saveAsTable(t) - val df = spark.table(t) - - val lit = Short.MaxValue.toInt - checkAnswer(df.where(s"c2 > $lit").select("c1"), Seq.empty) - checkAnswer(df.where(s"c2 >= $lit").select("c1"), Row(2)) - checkAnswer(df.where(s"c2 == $lit").select("c1"), Row(2)) - checkAnswer(df.where(s"c2 <=> $lit").select("c1"), Row(2)) - checkAnswer(df.where(s"c2 != $lit").select("c1"), Row(1) :: Row(3) :: Row(4) :: Nil) - checkAnswer(df.where(s"c2 <= $lit").select("c1"), Row(1) :: Row(2) :: Row(3) :: Row(4) :: Nil) - checkAnswer(df.where(s"c2 < $lit").select("c1"), Row(1) :: Row(3) :: Row(4) :: Nil) - - // NaN is not supported in velox, so unexpected result will be obtained. -// checkAnswer(df.where(s"c3 > double('nan')").select("c1"), Seq.empty) -// checkAnswer(df.where(s"c3 >= double('nan')").select("c1"), Row(2)) -// checkAnswer(df.where(s"c3 == double('nan')").select("c1"), Row(2)) -// checkAnswer(df.where(s"c3 <=> double('nan')").select("c1"), Row(2)) -// checkAnswer(df.where(s"c3 != double('nan')").select("c1"), Row(1) :: Row(3) :: Row(4) :: Nil) -// checkAnswer(df.where(s"c3 <= double('nan')").select("c1"), -// Row(1) :: Row(2) :: Row(3) :: Row(4) :: Nil) -// checkAnswer(df.where(s"c3 < double('nan')").select("c1"), Row(1) :: Row(3) :: Row(4) :: Nil) - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenXPathFunctionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenXPathFunctionsSuite.scala deleted file mode 100644 index 918a96c49e3..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenXPathFunctionsSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql - -class GlutenXPathFunctionsSuite extends XPathFunctionsSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenAnsiCastSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenAnsiCastSuite.scala deleted file mode 100644 index 687ff84aeaa..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenAnsiCastSuite.scala +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.GlutenTestsTrait -import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{ALL_TIMEZONES, UTC} -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.{DataType, StringType, TimestampType} -import org.apache.spark.util.DebuggableThreadUtils - -import java.sql.Timestamp -import java.time.LocalDateTime -import java.util.{Calendar, TimeZone} - -class GlutenCastSuiteWithAnsiModeOn extends AnsiCastSuiteBase with GlutenTestsTrait { - - override def beforeAll(): Unit = { - super.beforeAll() - SQLConf.get.setConf(SQLConf.ANSI_ENABLED, true) - } - - override def afterAll(): Unit = { - super.afterAll() - SQLConf.get.unsetConf(SQLConf.ANSI_ENABLED) - } - - override def cast(v: Any, targetType: DataType, timeZoneId: Option[String] = None): CastBase = { - v match { - case lit: Expression => Cast(lit, targetType, timeZoneId) - case _ => Cast(Literal(v), targetType, timeZoneId) - } - } - - override def setConfigurationHint: String = - s"set ${SQLConf.ANSI_ENABLED.key} as false" -} - -class GlutenAnsiCastSuiteWithAnsiModeOn extends AnsiCastSuiteBase with GlutenTestsTrait { - - override def beforeAll(): Unit = { - super.beforeAll() - SQLConf.get.setConf(SQLConf.ANSI_ENABLED, true) - } - - override def afterAll(): Unit = { - super.afterAll() - SQLConf.get.unsetConf(SQLConf.ANSI_ENABLED) - } - - override def cast(v: Any, targetType: DataType, timeZoneId: Option[String] = None): CastBase = { - v match { - case lit: Expression => AnsiCast(lit, targetType, timeZoneId) - case _ => AnsiCast(Literal(v), targetType, timeZoneId) - } - } - - override def setConfigurationHint: String = - s"set ${SQLConf.STORE_ASSIGNMENT_POLICY.key} as" + - s" ${SQLConf.StoreAssignmentPolicy.LEGACY.toString}" -} - -class GlutenAnsiCastSuiteWithAnsiModeOff extends AnsiCastSuiteBase with GlutenTestsTrait { - - override def beforeAll(): Unit = { - super.beforeAll() - SQLConf.get.setConf(SQLConf.ANSI_ENABLED, false) - } - - override def afterAll(): Unit = { - super.afterAll() - SQLConf.get.unsetConf(SQLConf.ANSI_ENABLED) - } - - override def cast(v: Any, targetType: DataType, timeZoneId: Option[String] = None): CastBase = { - v match { - case lit: Expression => AnsiCast(lit, targetType, timeZoneId) - case _ => AnsiCast(Literal(v), targetType, timeZoneId) - } - } - - override def setConfigurationHint: String = - s"set ${SQLConf.STORE_ASSIGNMENT_POLICY.key} as" + - s" ${SQLConf.StoreAssignmentPolicy.LEGACY.toString}" -} - -class GlutenTryCastSuite extends TryCastSuite with GlutenTestsTrait { - - private val specialTs = Seq( - "0001-01-01T00:00:00", // the fist timestamp of Common Era - "1582-10-15T23:59:59", // the cutover date from Julian to Gregorian calendar - "1970-01-01T00:00:00", // the epoch timestamp - "9999-12-31T23:59:59" // the last supported timestamp according to SQL standard - ) - - testGluten("SPARK-35698: cast timestamp without time zone to string") { - specialTs.foreach { - s => checkEvaluation(cast(LocalDateTime.parse(s), StringType), s.replace("T", " ")) - } - } - - testGluten("cast string to timestamp") { - DebuggableThreadUtils.parmap( - ALL_TIMEZONES - .filterNot(_.getId.contains("SystemV")) - .filterNot(_.getId.contains("Europe/Kyiv")) - .filterNot(_.getId.contains("America/Ciudad_Juarez")) - .filterNot(_.getId.contains("Antarctica/Vostok")) - .filterNot(_.getId.contains("Pacific/Kanton")) - .filterNot(_.getId.contains("Asia/Tehran")) - .filterNot(_.getId.contains("Iran")), - prefix = "CastSuiteBase-cast-string-to-timestamp", - maxThreads = 1 - ) { - zid => - withSQLConf( - SQLConf.SESSION_LOCAL_TIMEZONE.key -> zid.getId - ) { - def checkCastStringToTimestamp(str: String, expected: Timestamp): Unit = { - checkEvaluation(cast(Literal(str), TimestampType, Option(zid.getId)), expected) - } - - val tz = TimeZone.getTimeZone(zid) - var c = Calendar.getInstance(tz) - c.set(2015, 0, 1, 0, 0, 0) - c.set(Calendar.MILLISECOND, 0) - checkCastStringToTimestamp("2015", new Timestamp(c.getTimeInMillis)) - c = Calendar.getInstance(tz) - c.set(2015, 2, 1, 0, 0, 0) - c.set(Calendar.MILLISECOND, 0) - checkCastStringToTimestamp("2015-03", new Timestamp(c.getTimeInMillis)) - c = Calendar.getInstance(tz) - c.set(2015, 2, 18, 0, 0, 0) - c.set(Calendar.MILLISECOND, 0) - checkCastStringToTimestamp("2015-03-18", new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp("2015-03-18 ", new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(tz) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 0) - checkCastStringToTimestamp("2015-03-18 12:03:17", new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp("2015-03-18T12:03:17", new Timestamp(c.getTimeInMillis)) - - // If the string value includes timezone string, it represents the timestamp string - // in the timezone regardless of the timeZoneId parameter. - c = Calendar.getInstance(TimeZone.getTimeZone(UTC)) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 0) - checkCastStringToTimestamp("2015-03-18T12:03:17Z", new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp("2015-03-18 12:03:17Z", new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(TimeZone.getTimeZone("GMT-01:00")) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 0) - // Unsupported timezone format for Velox backend. - // checkCastStringToTimestamp("2015-03-18T12:03:17-1:0", new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp("2015-03-18T12:03:17-01:00", new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:30")) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 0) - checkCastStringToTimestamp("2015-03-18T12:03:17+07:30", new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:03")) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 0) - // Unsupported timezone format for Velox backend. - // checkCastStringToTimestamp("2015-03-18T12:03:17+7:3", - // new Timestamp(c.getTimeInMillis)) - - // tests for the string including milliseconds. - c = Calendar.getInstance(tz) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 123) - checkCastStringToTimestamp("2015-03-18 12:03:17.123", new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp("2015-03-18T12:03:17.123", new Timestamp(c.getTimeInMillis)) - - // If the string value includes timezone string, it represents the timestamp string - // in the timezone regardless of the timeZoneId parameter. - c = Calendar.getInstance(TimeZone.getTimeZone(UTC)) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 456) - checkCastStringToTimestamp("2015-03-18T12:03:17.456Z", new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp("2015-03-18 12:03:17.456Z", new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(TimeZone.getTimeZone("GMT-01:00")) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 123) - // Unsupported timezone format for Velox backend. - // checkCastStringToTimestamp("2015-03-18T12:03:17.123-1:0", - // new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp( - "2015-03-18T12:03:17.123-01:00", - new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:30")) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 123) - checkCastStringToTimestamp( - "2015-03-18T12:03:17.123+07:30", - new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:03")) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 123) - // Unsupported timezone format for Velox backend. - // checkCastStringToTimestamp("2015-03-18T12:03:17.123+7:3", - // new Timestamp(c.getTimeInMillis)) - } - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenArithmeticExpressionSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenArithmeticExpressionSuite.scala deleted file mode 100644 index 14079037518..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenArithmeticExpressionSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.GlutenTestsTrait - -class GlutenArithmeticExpressionSuite extends ArithmeticExpressionSuite with GlutenTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenBitwiseExpressionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenBitwiseExpressionsSuite.scala deleted file mode 100644 index fd9827ddf50..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenBitwiseExpressionsSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.GlutenTestsTrait - -class GlutenBitwiseExpressionsSuite extends BitwiseExpressionsSuite with GlutenTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala deleted file mode 100644 index edfc1828869..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala +++ /dev/null @@ -1,306 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.GlutenTestsTrait -import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, ALL_TIMEZONES, UTC, UTC_OPT} -import org.apache.spark.sql.catalyst.util.DateTimeUtils.{fromJavaTimestamp, millisToMicros, TimeZoneUTC} -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types._ -import org.apache.spark.util.DebuggableThreadUtils - -import java.sql.{Date, Timestamp} -import java.util.{Calendar, TimeZone} - -class GlutenCastSuite extends CastSuite with GlutenTestsTrait { - override def cast(v: Any, targetType: DataType, timeZoneId: Option[String] = None): CastBase = { - v match { - case lit: Expression => - logDebug(s"Cast from: ${lit.dataType.typeName}, to: ${targetType.typeName}") - Cast(lit, targetType, timeZoneId) - case _ => - val lit = Literal(v) - logDebug(s"Cast from: ${lit.dataType.typeName}, to: ${targetType.typeName}") - Cast(lit, targetType, timeZoneId) - } - } - - // Register UDT For test("SPARK-32828") - UDTRegistration.register(classOf[IExampleBaseType].getName, classOf[ExampleBaseTypeUDT].getName) - UDTRegistration.register(classOf[IExampleSubType].getName, classOf[ExampleSubTypeUDT].getName) - - testGluten("missing cases - from boolean") { - (DataTypeTestUtils.numericTypeWithoutDecimal + BooleanType).foreach { - t => - t match { - case BooleanType => - checkEvaluation(cast(cast(true, BooleanType), t), true) - checkEvaluation(cast(cast(false, BooleanType), t), false) - case _ => - checkEvaluation(cast(cast(true, BooleanType), t), 1) - checkEvaluation(cast(cast(false, BooleanType), t), 0) - } - } - } - - testGluten("missing cases - from byte") { - DataTypeTestUtils.numericTypeWithoutDecimal.foreach { - t => - checkEvaluation(cast(cast(0, ByteType), t), 0) - checkEvaluation(cast(cast(-1, ByteType), t), -1) - checkEvaluation(cast(cast(1, ByteType), t), 1) - } - } - - testGluten("missing cases - from short") { - DataTypeTestUtils.numericTypeWithoutDecimal.foreach { - t => - checkEvaluation(cast(cast(0, ShortType), t), 0) - checkEvaluation(cast(cast(-1, ShortType), t), -1) - checkEvaluation(cast(cast(1, ShortType), t), 1) - } - } - - testGluten("missing cases - date self check") { - val d = Date.valueOf("1970-01-01") - checkEvaluation(cast(d, DateType), d) - } - - testGluten("data type casting") { - val sd = "1970-01-01" - val d = Date.valueOf(sd) - val zts = sd + " 00:00:00" - val sts = sd + " 00:00:02" - val nts = sts + ".1" - val ts = withDefaultTimeZone(UTC)(Timestamp.valueOf(nts)) - - // SystemV timezones are a legacy way of specifying timezones in Unix-like OS. - // It is not supported by Velox. - for (tz <- ALL_TIMEZONES.filterNot(_.getId.contains("SystemV"))) { - withSQLConf( - SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz.getId - ) { - val timeZoneId = Option(tz.getId) - var c = Calendar.getInstance(TimeZoneUTC) - c.set(2015, 2, 8, 2, 30, 0) - checkEvaluation( - cast( - cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId), - TimestampType, - timeZoneId), - millisToMicros(c.getTimeInMillis)) - c = Calendar.getInstance(TimeZoneUTC) - c.set(2015, 10, 1, 2, 30, 0) - checkEvaluation( - cast( - cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId), - TimestampType, - timeZoneId), - millisToMicros(c.getTimeInMillis)) - } - } - - checkEvaluation(cast("abdef", StringType), "abdef") - checkEvaluation(cast("12.65", DecimalType.SYSTEM_DEFAULT), Decimal(12.65)) - - checkEvaluation(cast(cast(sd, DateType), StringType), sd) - checkEvaluation(cast(cast(d, StringType), DateType), 0) - - withSQLConf( - SQLConf.SESSION_LOCAL_TIMEZONE.key -> UTC_OPT.get - ) { - checkEvaluation(cast(cast(nts, TimestampType, UTC_OPT), StringType, UTC_OPT), nts) - checkEvaluation( - cast(cast(ts, StringType, UTC_OPT), TimestampType, UTC_OPT), - fromJavaTimestamp(ts)) - - // all convert to string type to check - checkEvaluation( - cast(cast(cast(nts, TimestampType, UTC_OPT), DateType, UTC_OPT), StringType), - sd) - checkEvaluation( - cast(cast(cast(ts, DateType, UTC_OPT), TimestampType, UTC_OPT), StringType, UTC_OPT), - zts) - } - - checkEvaluation(cast(cast("abdef", BinaryType), StringType), "abdef") - - checkEvaluation( - cast( - cast(cast(cast(cast(cast("5", ByteType), ShortType), IntegerType), FloatType), DoubleType), - LongType), - 5.toLong) - - checkEvaluation(cast("23", DoubleType), 23d) - checkEvaluation(cast("23", IntegerType), 23) - checkEvaluation(cast("23", FloatType), 23f) - checkEvaluation(cast("23", DecimalType.USER_DEFAULT), Decimal(23)) - checkEvaluation(cast("23", ByteType), 23.toByte) - checkEvaluation(cast("23", ShortType), 23.toShort) - checkEvaluation(cast(123, IntegerType), 123) - - checkEvaluation(cast(Literal.create(null, IntegerType), ShortType), null) - } - - test("cast from boolean to timestamp") { - val tsTrue = new Timestamp(0) - tsTrue.setNanos(1000) - - val tsFalse = new Timestamp(0) - - checkEvaluation(cast(true, TimestampType), tsTrue) - - checkEvaluation(cast(false, TimestampType), tsFalse) - } - - test("cast timestamp to Int64 with floor division") { - val originalDefaultTz = TimeZone.getDefault - try { - TimeZone.setDefault(TimeZone.getTimeZone("UTC")) - val testCases = Seq( - ("1970-01-01 00:00:00.000", 0L), - ("1970-01-01 00:00:00.999", 0L), - ("1970-01-01 00:00:01.000", 1L), - ("1970-01-01 00:00:59.999", 59L), - ("1970-01-01 00:01:00.000", 60L), - ("2000-01-01 00:00:00.000", 946684800L), - ("2024-02-16 12:34:56.789", 1708086896L), - ("9999-12-31 23:59:59.999", 253402300799L), - ("1969-12-31 23:59:59.999", -1L), - ("1969-12-31 23:59:58.500", -2L), - ("1900-01-01 12:00:00.000", -2208945600L) - ) - - for ((inputStr, expectedOutput) <- testCases) { - checkEvaluation(cast(Timestamp.valueOf(inputStr), LongType), expectedOutput) - } - } finally { - TimeZone.setDefault(originalDefaultTz) - } - } - - testGluten("cast string to timestamp") { - DebuggableThreadUtils.parmap( - ALL_TIMEZONES - .filterNot(_.getId.contains("SystemV")) - .filterNot(_.getId.contains("Europe/Kyiv")) - .filterNot(_.getId.contains("America/Ciudad_Juarez")) - .filterNot(_.getId.contains("Antarctica/Vostok")) - .filterNot(_.getId.contains("Pacific/Kanton")) - .filterNot(_.getId.contains("Asia/Tehran")) - .filterNot(_.getId.contains("Iran")), - prefix = "CastSuiteBase-cast-string-to-timestamp", - maxThreads = 1 - ) { - zid => - withSQLConf( - SQLConf.SESSION_LOCAL_TIMEZONE.key -> zid.getId - ) { - def checkCastStringToTimestamp(str: String, expected: Timestamp): Unit = { - checkEvaluation(cast(Literal(str), TimestampType, Option(zid.getId)), expected) - } - - val tz = TimeZone.getTimeZone(zid) - var c = Calendar.getInstance(tz) - c.set(2015, 0, 1, 0, 0, 0) - c.set(Calendar.MILLISECOND, 0) - checkCastStringToTimestamp("2015", new Timestamp(c.getTimeInMillis)) - c = Calendar.getInstance(tz) - c.set(2015, 2, 1, 0, 0, 0) - c.set(Calendar.MILLISECOND, 0) - checkCastStringToTimestamp("2015-03", new Timestamp(c.getTimeInMillis)) - c = Calendar.getInstance(tz) - c.set(2015, 2, 18, 0, 0, 0) - c.set(Calendar.MILLISECOND, 0) - checkCastStringToTimestamp("2015-03-18", new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp("2015-03-18 ", new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(tz) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 0) - checkCastStringToTimestamp("2015-03-18 12:03:17", new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp("2015-03-18T12:03:17", new Timestamp(c.getTimeInMillis)) - - // If the string value includes timezone string, it represents the timestamp string - // in the timezone regardless of the timeZoneId parameter. - c = Calendar.getInstance(TimeZone.getTimeZone(UTC)) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 0) - checkCastStringToTimestamp("2015-03-18T12:03:17Z", new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp("2015-03-18 12:03:17Z", new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(TimeZone.getTimeZone("GMT-01:00")) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 0) - // Unsupported timezone format for Velox backend. - // checkCastStringToTimestamp("2015-03-18T12:03:17-1:0", new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp("2015-03-18T12:03:17-01:00", new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:30")) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 0) - checkCastStringToTimestamp("2015-03-18T12:03:17+07:30", new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:03")) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 0) - // Unsupported timezone format for Velox backend. - // checkCastStringToTimestamp("2015-03-18T12:03:17+7:3", - // new Timestamp(c.getTimeInMillis)) - - // tests for the string including milliseconds. - c = Calendar.getInstance(tz) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 123) - checkCastStringToTimestamp("2015-03-18 12:03:17.123", new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp("2015-03-18T12:03:17.123", new Timestamp(c.getTimeInMillis)) - - // If the string value includes timezone string, it represents the timestamp string - // in the timezone regardless of the timeZoneId parameter. - c = Calendar.getInstance(TimeZone.getTimeZone(UTC)) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 456) - checkCastStringToTimestamp("2015-03-18T12:03:17.456Z", new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp("2015-03-18 12:03:17.456Z", new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(TimeZone.getTimeZone("GMT-01:00")) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 123) - // Unsupported timezone format for Velox backend. - // checkCastStringToTimestamp("2015-03-18T12:03:17.123-1:0", - // new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp( - "2015-03-18T12:03:17.123-01:00", - new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:30")) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 123) - checkCastStringToTimestamp( - "2015-03-18T12:03:17.123+07:30", - new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:03")) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 123) - // Unsupported timezone format for Velox backend. - // checkCastStringToTimestamp("2015-03-18T12:03:17.123+7:3", - // new Timestamp(c.getTimeInMillis)) - } - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCollectionExpressionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCollectionExpressionsSuite.scala deleted file mode 100644 index fcbbf2a5b35..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCollectionExpressionsSuite.scala +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.GlutenTestsTrait -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.analysis.TypeCheckResult -import org.apache.spark.sql.types._ - -import scala.util.Random - -class GlutenCollectionExpressionsSuite extends CollectionExpressionsSuite with GlutenTestsTrait { - testGluten("Shuffle") { - // Primitive-type elements - val ai0 = Literal.create(Seq(1, 2, 3, 4, 5), ArrayType(IntegerType, containsNull = false)) - val ai1 = Literal.create(Seq(1, 2, 3), ArrayType(IntegerType, containsNull = false)) - val ai2 = Literal.create(Seq(null, 1, null, 3), ArrayType(IntegerType, containsNull = true)) - val ai3 = Literal.create(Seq(2, null, 4, null), ArrayType(IntegerType, containsNull = true)) - val ai4 = Literal.create(Seq(null, null, null), ArrayType(IntegerType, containsNull = true)) - val ai5 = Literal.create(Seq(1), ArrayType(IntegerType, containsNull = false)) - val ai6 = Literal.create(Seq.empty, ArrayType(IntegerType, containsNull = false)) - val ai7 = Literal.create(null, ArrayType(IntegerType, containsNull = true)) - - checkEvaluation(Shuffle(ai0, Some(0)), Array(2, 1, 5, 4, 3)) - checkEvaluation(Shuffle(ai1, Some(0)), Array(2, 1, 3)) - checkEvaluation(Shuffle(ai2, Some(0)), Array(1, null, null, 3)) - checkEvaluation(Shuffle(ai3, Some(0)), Array(null, 2, 4, null)) - checkEvaluation(Shuffle(ai4, Some(0)), Array(null, null, null)) - checkEvaluation(Shuffle(ai5, Some(0)), Array(1)) - checkEvaluation(Shuffle(ai6, Some(0)), Array.empty) - checkEvaluation(Shuffle(ai7, Some(0)), null) - - // Non-primitive-type elements - val as0 = Literal.create(Seq("a", "b", "c", "d"), ArrayType(StringType, containsNull = false)) - val as1 = Literal.create(Seq("a", "b", "c"), ArrayType(StringType, containsNull = false)) - val as2 = Literal.create(Seq(null, "a", null, "c"), ArrayType(StringType, containsNull = true)) - val as3 = Literal.create(Seq("b", null, "d", null), ArrayType(StringType, containsNull = true)) - val as4 = Literal.create(Seq(null, null, null), ArrayType(StringType, containsNull = true)) - val as5 = Literal.create(Seq("a"), ArrayType(StringType, containsNull = false)) - val as6 = Literal.create(Seq.empty, ArrayType(StringType, containsNull = false)) - val as7 = Literal.create(null, ArrayType(StringType, containsNull = true)) - val aa = - Literal.create(Seq(Seq("a", "b"), Seq("c", "d"), Seq("e")), ArrayType(ArrayType(StringType))) - - checkEvaluation(Shuffle(as0, Some(0)), Array("b", "a", "c", "d")) - checkEvaluation(Shuffle(as1, Some(0)), Array("b", "a", "c")) - checkEvaluation(Shuffle(as2, Some(0)), Array("a", null, null, "c")) - checkEvaluation(Shuffle(as3, Some(0)), Array(null, "b", "d", null)) - checkEvaluation(Shuffle(as4, Some(0)), Array(null, null, null)) - checkEvaluation(Shuffle(as5, Some(0)), Array("a")) - checkEvaluation(Shuffle(as6, Some(0)), Array.empty) - checkEvaluation(Shuffle(as7, Some(0)), null) - checkEvaluation(Shuffle(aa, Some(0)), Array(Array("c", "d"), Array("a", "b"), Array("e"))) - - val r = new Random(1234) - val seed1 = Some(r.nextLong()) - assert( - evaluateWithoutCodegen(Shuffle(ai0, seed1)) === - evaluateWithoutCodegen(Shuffle(ai0, seed1))) - assert( - evaluateWithMutableProjection(Shuffle(ai0, seed1)) === - evaluateWithMutableProjection(Shuffle(ai0, seed1))) - assert( - evaluateWithUnsafeProjection(Shuffle(ai0, seed1)) === - evaluateWithUnsafeProjection(Shuffle(ai0, seed1))) - - val seed2 = Some(r.nextLong()) - assert( - evaluateWithoutCodegen(Shuffle(ai0, seed1)) !== - evaluateWithoutCodegen(Shuffle(ai0, seed2))) - assert( - evaluateWithMutableProjection(Shuffle(ai0, seed1)) !== - evaluateWithMutableProjection(Shuffle(ai0, seed2))) - assert( - evaluateWithUnsafeProjection(Shuffle(ai0, seed1)) !== - evaluateWithUnsafeProjection(Shuffle(ai0, seed2))) - } - - testGluten("MapFromEntries") { - def arrayType(keyType: DataType, valueType: DataType): DataType = { - ArrayType(StructType(Seq(StructField("a", keyType), StructField("b", valueType))), true) - } - - def row(values: Any*): InternalRow = create_row(values: _*) - - // Primitive-type keys and values - val aiType = arrayType(IntegerType, IntegerType) - val ai0 = Literal.create(Seq(row(1, 10), row(2, 20), row(3, 20)), aiType) - val ai1 = Literal.create(Seq(row(1, null), row(2, 20), row(3, null)), aiType) - val ai2 = Literal.create(Seq.empty, aiType) - val ai3 = Literal.create(null, aiType) - // Ignore duplicated key as 'last_win' not supported by Velox for now - // val ai4 = Literal.create(Seq(row(1, 10), row(1, 20)), aiType) - // The map key is null - val ai5 = Literal.create(Seq(row(1, 10), row(null, 20)), aiType) - val ai6 = Literal.create(Seq(null, row(2, 20), null), aiType) - - checkEvaluation(MapFromEntries(ai0), create_map(1 -> 10, 2 -> 20, 3 -> 20)) - checkEvaluation(MapFromEntries(ai1), create_map(1 -> null, 2 -> 20, 3 -> null)) - checkEvaluation(MapFromEntries(ai2), Map.empty) - checkEvaluation(MapFromEntries(ai3), null) - - // Map key can't be null - checkExceptionInExpression[RuntimeException](MapFromEntries(ai5), "Cannot use null as map key") - checkEvaluation(MapFromEntries(ai6), null) - - // Non-primitive-type keys and values - val asType = arrayType(StringType, StringType) - val as0 = Literal.create(Seq(row("a", "aa"), row("b", "bb"), row("c", "bb")), asType) - val as1 = Literal.create(Seq(row("a", null), row("b", "bb"), row("c", null)), asType) - val as2 = Literal.create(Seq.empty, asType) - val as3 = Literal.create(null, asType) - val as5 = Literal.create(Seq(row("a", "aa"), row(null, "bb")), asType) - val as6 = Literal.create(Seq(null, row("b", "bb"), null), asType) - - checkEvaluation(MapFromEntries(as0), create_map("a" -> "aa", "b" -> "bb", "c" -> "bb")) - checkEvaluation(MapFromEntries(as1), create_map("a" -> null, "b" -> "bb", "c" -> null)) - checkEvaluation(MapFromEntries(as2), Map.empty) - checkEvaluation(MapFromEntries(as3), null) - - // Map key can't be null - checkExceptionInExpression[RuntimeException](MapFromEntries(as5), "Cannot use null as map key") - checkEvaluation(MapFromEntries(as6), null) - - // map key can't be map - val structOfMap = row(create_map(1 -> 1), 1) - val map = MapFromEntries( - Literal.create( - Seq(structOfMap), - arrayType(keyType = MapType(IntegerType, IntegerType), valueType = IntegerType))) - map.checkInputDataTypes() match { - case TypeCheckResult.TypeCheckSuccess => fail("should not allow map as map key") - case TypeCheckResult.TypeCheckFailure(msg) => - assert(msg.contains("The key of map cannot be/contain map")) - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenComplexTypeSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenComplexTypeSuite.scala deleted file mode 100644 index f5f278361e1..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenComplexTypeSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.GlutenTestsTrait - -class GlutenComplexTypeSuite extends ComplexTypeSuite with GlutenTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenConditionalExpressionSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenConditionalExpressionSuite.scala deleted file mode 100644 index 923f5f87bcc..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenConditionalExpressionSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.GlutenTestsTrait - -class GlutenConditionalExpressionSuite extends ConditionalExpressionSuite with GlutenTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala deleted file mode 100644 index 8c494699c12..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala +++ /dev/null @@ -1,578 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.GlutenTestsTrait -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection -import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._ -import org.apache.spark.sql.catalyst.util.DateTimeUtils -import org.apache.spark.sql.catalyst.util.DateTimeUtils.{getZoneId, TimeZoneUTC} -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types._ -import org.apache.spark.unsafe.types.UTF8String - -import java.sql.{Date, Timestamp} -import java.text.SimpleDateFormat -import java.time.{LocalDateTime, ZoneId} -import java.util.{Calendar, Locale, TimeZone} -import java.util.concurrent.TimeUnit._ - -class GlutenDateExpressionsSuite extends DateExpressionsSuite with GlutenTestsTrait { - override def testIntegralInput(testFunc: Number => Unit): Unit = { - def checkResult(input: Long): Unit = { - if (input.toByte == input) { - testFunc(input.toByte) - } else if (input.toShort == input) { - testFunc(input.toShort) - } else if (input.toInt == input) { - testFunc(input.toInt) - } else { - testFunc(input) - } - } - - checkResult(0) - checkResult(Byte.MaxValue) - checkResult(Byte.MinValue) - checkResult(Short.MaxValue) - checkResult(Short.MinValue) - // Spark collect causes integer overflow. - // checkResult(Int.MaxValue) - // checkResult(Int.MinValue) - // checkResult(Int.MaxValue.toLong + 100) - // checkResult(Int.MinValue.toLong - 100) - } - - testGluten("TIMESTAMP_MICROS") { - def testIntegralFunc(value: Number): Unit = { - checkEvaluation(MicrosToTimestamp(Literal(value)), value.longValue()) - } - - // test null input - checkEvaluation(MicrosToTimestamp(Literal(null, IntegerType)), null) - - // test integral input - testIntegralInput(testIntegralFunc) - // test max/min input - // Spark collect causes long overflow. - // testIntegralFunc(Long.MaxValue) - // testIntegralFunc(Long.MinValue) - } - - val outstandingTimezonesIds: Seq[String] = Seq( - // Velox doesn't support timezones like UTC. - // "UTC", - PST.getId, - CET.getId, - "Africa/Dakar", - LA.getId, - "Asia/Urumqi", - "Asia/Hong_Kong", - "Europe/Brussels") - val outstandingZoneIds: Seq[ZoneId] = outstandingTimezonesIds.map(getZoneId) - - testGluten("unix_timestamp") { - Seq("legacy", "corrected").foreach { - legacyParserPolicy => - withDefaultTimeZone(UTC) { - for (zid <- outstandingZoneIds) { - withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, - SQLConf.SESSION_LOCAL_TIMEZONE.key -> zid.getId - ) { - val sdf1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US) - val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS" - val sdf2 = new SimpleDateFormat(fmt2, Locale.US) - val fmt3 = "yy-MM-dd" - val sdf3 = new SimpleDateFormat(fmt3, Locale.US) - sdf3.setTimeZone(TimeZoneUTC) - - val timeZoneId = Option(zid.getId) - val tz = TimeZone.getTimeZone(zid) - sdf1.setTimeZone(tz) - sdf2.setTimeZone(tz) - - val date1 = Date.valueOf("2015-07-24") - checkEvaluation( - UnixTimestamp( - Literal(sdf1.format(new Timestamp(0))), - Literal("yyyy-MM-dd HH:mm:ss"), - timeZoneId), - 0L) - checkEvaluation( - UnixTimestamp( - Literal(sdf1.format(new Timestamp(1000000))), - Literal("yyyy-MM-dd HH:mm:ss"), - timeZoneId), - 1000L) - checkEvaluation( - UnixTimestamp( - Literal(new Timestamp(1000000)), - Literal("yyyy-MM-dd HH:mm:ss"), - timeZoneId), - 1000L) - checkEvaluation( - UnixTimestamp( - Literal( - DateTimeUtils.microsToLocalDateTime(DateTimeUtils.millisToMicros(1000000))), - Literal("yyyy-MM-dd HH:mm:ss"), - timeZoneId), - 1000L) - checkEvaluation( - UnixTimestamp(Literal(date1), Literal("yyyy-MM-dd HH:mm:ss"), timeZoneId), - MICROSECONDS.toSeconds( - DateTimeUtils.daysToMicros(DateTimeUtils.fromJavaDate(date1), tz.toZoneId)) - ) - checkEvaluation( - UnixTimestamp( - Literal(sdf2.format(new Timestamp(-1000000))), - Literal(fmt2), - timeZoneId), - -1000L) - checkEvaluation( - UnixTimestamp( - Literal(sdf3.format(Date.valueOf("2015-07-24"))), - Literal(fmt3), - timeZoneId), - MICROSECONDS.toSeconds( - DateTimeUtils.daysToMicros( - DateTimeUtils.fromJavaDate(Date.valueOf("2015-07-24")), - tz.toZoneId)) - ) - val t1 = UnixTimestamp(CurrentTimestamp(), Literal("yyyy-MM-dd HH:mm:ss")) - .eval() - .asInstanceOf[Long] - val t2 = UnixTimestamp(CurrentTimestamp(), Literal("yyyy-MM-dd HH:mm:ss")) - .eval() - .asInstanceOf[Long] - assert(t2 - t1 <= 1) - checkEvaluation( - UnixTimestamp( - Literal.create(null, DateType), - Literal.create(null, StringType), - timeZoneId), - null) - checkEvaluation( - UnixTimestamp( - Literal.create(null, DateType), - Literal("yyyy-MM-dd HH:mm:ss"), - timeZoneId), - null) - checkEvaluation( - UnixTimestamp(Literal(date1), Literal.create(null, StringType), timeZoneId), - MICROSECONDS.toSeconds( - DateTimeUtils.daysToMicros(DateTimeUtils.fromJavaDate(date1), tz.toZoneId)) - ) - } - } - } - } - // Test escaping of format - GenerateUnsafeProjection.generate( - UnixTimestamp(Literal("2015-07-24"), Literal("\""), UTC_OPT) :: Nil) - } - - testGluten("to_unix_timestamp") { - withDefaultTimeZone(UTC) { - for (zid <- outstandingZoneIds) { - Seq("legacy", "corrected").foreach { - legacyParserPolicy => - withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, - SQLConf.SESSION_LOCAL_TIMEZONE.key -> zid.getId - ) { - val fmt1 = "yyyy-MM-dd HH:mm:ss" - val sdf1 = new SimpleDateFormat(fmt1, Locale.US) - val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS" - val sdf2 = new SimpleDateFormat(fmt2, Locale.US) - val fmt3 = "yy-MM-dd" - val sdf3 = new SimpleDateFormat(fmt3, Locale.US) - sdf3.setTimeZone(TimeZoneUTC) - - val timeZoneId = Option(zid.getId) - val tz = TimeZone.getTimeZone(zid) - sdf1.setTimeZone(tz) - sdf2.setTimeZone(tz) - - val date1 = Date.valueOf("2015-07-24") - checkEvaluation( - ToUnixTimestamp(Literal(sdf1.format(new Timestamp(0))), Literal(fmt1), timeZoneId), - 0L) - checkEvaluation( - ToUnixTimestamp( - Literal(sdf1.format(new Timestamp(1000000))), - Literal(fmt1), - timeZoneId), - 1000L) - checkEvaluation( - ToUnixTimestamp(Literal(new Timestamp(1000000)), Literal(fmt1)), - 1000L) - checkEvaluation( - ToUnixTimestamp( - Literal( - DateTimeUtils.microsToLocalDateTime(DateTimeUtils.millisToMicros(1000000))), - Literal(fmt1)), - 1000L) - checkEvaluation( - ToUnixTimestamp(Literal(date1), Literal(fmt1), timeZoneId), - MICROSECONDS.toSeconds( - DateTimeUtils.daysToMicros(DateTimeUtils.fromJavaDate(date1), zid))) - checkEvaluation( - ToUnixTimestamp( - Literal(sdf2.format(new Timestamp(-1000000))), - Literal(fmt2), - timeZoneId), - -1000L) - checkEvaluation( - ToUnixTimestamp( - Literal(sdf3.format(Date.valueOf("2015-07-24"))), - Literal(fmt3), - timeZoneId), - MICROSECONDS.toSeconds(DateTimeUtils - .daysToMicros(DateTimeUtils.fromJavaDate(Date.valueOf("2015-07-24")), zid)) - ) - val t1 = ToUnixTimestamp(CurrentTimestamp(), Literal(fmt1)).eval().asInstanceOf[Long] - val t2 = ToUnixTimestamp(CurrentTimestamp(), Literal(fmt1)).eval().asInstanceOf[Long] - assert(t2 - t1 <= 1) - checkEvaluation( - ToUnixTimestamp( - Literal.create(null, DateType), - Literal.create(null, StringType), - timeZoneId), - null) - checkEvaluation( - ToUnixTimestamp(Literal.create(null, DateType), Literal(fmt1), timeZoneId), - null) - checkEvaluation( - ToUnixTimestamp(Literal(date1), Literal.create(null, StringType), timeZoneId), - MICROSECONDS.toSeconds( - DateTimeUtils.daysToMicros(DateTimeUtils.fromJavaDate(date1), zid)) - ) - - // SPARK-28072 The codegen path for non-literal input should also work - checkEvaluation( - expression = ToUnixTimestamp( - BoundReference(ordinal = 0, dataType = StringType, nullable = true), - BoundReference(ordinal = 1, dataType = StringType, nullable = true), - timeZoneId), - expected = 0L, - inputRow = InternalRow( - UTF8String.fromString(sdf1.format(new Timestamp(0))), - UTF8String.fromString(fmt1)) - ) - } - } - } - } - // Test escaping of format - GenerateUnsafeProjection.generate( - ToUnixTimestamp(Literal("2015-07-24"), Literal("\""), UTC_OPT) :: Nil) - } - - // Modified based on vanilla spark to explicitly set timezone in config. - testGluten("DateFormat") { - val PST_OPT = Option("America/Los_Angeles") - val JST_OPT = Option("Asia/Tokyo") - - Seq("legacy", "corrected").foreach { - legacyParserPolicy => - withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, - SQLConf.SESSION_LOCAL_TIMEZONE.key -> UTC_OPT.get) { - checkEvaluation( - DateFormatClass(Literal.create(null, TimestampType), Literal("y"), UTC_OPT), - null) - checkEvaluation( - DateFormatClass( - Cast(Literal(d), TimestampType, UTC_OPT), - Literal.create(null, StringType), - UTC_OPT), - null) - - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT), Literal("y"), UTC_OPT), - "2015") - checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), UTC_OPT), "2013") - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT), Literal("H"), UTC_OPT), - "0") - checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), UTC_OPT), "13") - } - - withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, - SQLConf.SESSION_LOCAL_TIMEZONE.key -> PST_OPT.get) { - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT), Literal("y"), PST_OPT), - "2015") - checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), PST_OPT), "2013") - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT), Literal("H"), PST_OPT), - "0") - checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), PST_OPT), "5") - } - - withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, - SQLConf.SESSION_LOCAL_TIMEZONE.key -> JST_OPT.get) { - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT), Literal("y"), JST_OPT), - "2015") - checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), JST_OPT), "2013") - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT), Literal("H"), JST_OPT), - "0") - checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), JST_OPT), "22") - } - } - } - - testGluten("from_unixtime") { - val outstandingTimezonesIds: Seq[String] = Seq( - // Velox doesn't support timezones like "UTC". - // "UTC", - // Not supported in velox. - // PST.getId, - // CET.getId, - "Africa/Dakar", - LA.getId, - "Asia/Urumqi", - "Asia/Hong_Kong", - "Europe/Brussels" - ) - val outstandingZoneIds: Seq[ZoneId] = outstandingTimezonesIds.map(getZoneId) - Seq("legacy", "corrected").foreach { - legacyParserPolicy => - for (zid <- outstandingZoneIds) { - withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, - SQLConf.SESSION_LOCAL_TIMEZONE.key -> zid.getId) { - val fmt1 = "yyyy-MM-dd HH:mm:ss" - val sdf1 = new SimpleDateFormat(fmt1, Locale.US) - val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS" - val sdf2 = new SimpleDateFormat(fmt2, Locale.US) - val timeZoneId = Option(zid.getId) - val tz = TimeZone.getTimeZone(zid) - sdf1.setTimeZone(tz) - sdf2.setTimeZone(tz) - - checkEvaluation( - FromUnixTime(Literal(0L), Literal(fmt1), timeZoneId), - sdf1.format(new Timestamp(0))) - checkEvaluation( - FromUnixTime(Literal(1000L), Literal(fmt1), timeZoneId), - sdf1.format(new Timestamp(1000000))) - checkEvaluation( - FromUnixTime(Literal(-1000L), Literal(fmt2), timeZoneId), - sdf2.format(new Timestamp(-1000000))) - checkEvaluation( - FromUnixTime(Literal(Long.MaxValue), Literal(fmt2), timeZoneId), - sdf2.format(new Timestamp(-1000))) - checkEvaluation( - FromUnixTime( - Literal.create(null, LongType), - Literal.create(null, StringType), - timeZoneId), - null) - checkEvaluation( - FromUnixTime(Literal.create(null, LongType), Literal(fmt1), timeZoneId), - null) - checkEvaluation( - FromUnixTime(Literal(1000L), Literal.create(null, StringType), timeZoneId), - null) - - // SPARK-28072 The codegen path for non-literal input should also work - checkEvaluation( - expression = FromUnixTime( - BoundReference(ordinal = 0, dataType = LongType, nullable = true), - BoundReference(ordinal = 1, dataType = StringType, nullable = true), - timeZoneId), - expected = UTF8String.fromString(sdf1.format(new Timestamp(0))), - inputRow = InternalRow(0L, UTF8String.fromString(fmt1)) - ) - } - } - } - // Test escaping of format - GenerateUnsafeProjection.generate(FromUnixTime(Literal(0L), Literal("\""), UTC_OPT) :: Nil) - } - - testGluten("Hour") { - val outstandingTimezonesIds: Seq[String] = Seq( - // Velox doesn't support timezones like "UTC". - // "UTC", - // Due to known issue: "-08:00/+01:00 not found in timezone database", - // skip check PST, CET timezone here. - // https://github.com/facebookincubator/velox/issues/7804 - // PST.getId, CET.getId, - "Africa/Dakar", - LA.getId, - "Asia/Urumqi", - "Asia/Hong_Kong", - "Europe/Brussels" - ) - withDefaultTimeZone(UTC) { - Seq("legacy", "corrected").foreach { - legacyParserPolicy => - withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy - ) { - assert(Hour(Literal.create(null, DateType), UTC_OPT).resolved === false) - assert(Hour(Literal(ts), UTC_OPT).resolved) - Seq(TimestampType, TimestampNTZType).foreach { - dt => - checkEvaluation(Hour(Cast(Literal(d), dt, UTC_OPT), UTC_OPT), 0) - checkEvaluation(Hour(Cast(Literal(date), dt, UTC_OPT), UTC_OPT), 13) - } - checkEvaluation(Hour(Literal(ts), UTC_OPT), 13) - } - - val c = Calendar.getInstance() - outstandingTimezonesIds.foreach { - zid => - withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, - SQLConf.SESSION_LOCAL_TIMEZONE.key -> zid - ) { - val timeZoneId = Option(zid) - c.setTimeZone(TimeZone.getTimeZone(zid)) - (0 to 24 by 5).foreach { - h => - // validate timestamp with local time zone - c.set(2015, 18, 3, h, 29, 59) - checkEvaluation( - Hour(Literal(new Timestamp(c.getTimeInMillis)), timeZoneId), - c.get(Calendar.HOUR_OF_DAY)) - - // validate timestamp without time zone - val localDateTime = LocalDateTime.of(2015, 1, 3, h, 29, 59) - checkEvaluation(Hour(Literal(localDateTime), timeZoneId), h) - } - Seq(TimestampType, TimestampNTZType).foreach { - dt => - checkConsistencyBetweenInterpretedAndCodegen( - (child: Expression) => Hour(child, timeZoneId), - dt) - } - } - } - } - } - } - - test("test timestamp add") { - // Check case-insensitivity - checkEvaluation( - TimestampAdd("SECOND", Literal(1), Literal(Timestamp.valueOf("2022-02-15 12:57:00"))), - Timestamp.valueOf("2022-02-15 12:57:01")) - checkEvaluation( - TimestampAdd("MINUTE", Literal(1), Literal(Timestamp.valueOf("2022-02-15 12:57:00"))), - Timestamp.valueOf("2022-02-15 12:58:00")) - checkEvaluation( - TimestampAdd("HOUR", Literal(1), Literal(Timestamp.valueOf("2022-02-15 12:57:00"))), - Timestamp.valueOf("2022-02-15 13:57:00")) - checkEvaluation( - TimestampAdd("DAY", Literal(1), Literal(Timestamp.valueOf("2022-02-15 12:57:00"))), - Timestamp.valueOf("2022-02-16 12:57:00")) - checkEvaluation( - TimestampAdd("MONTH", Literal(1), Literal(Timestamp.valueOf("2022-02-15 12:57:00"))), - Timestamp.valueOf("2022-03-15 12:57:00")) - checkEvaluation( - TimestampAdd("YEAR", Literal(1), Literal(Timestamp.valueOf("2022-02-15 12:57:00"))), - Timestamp.valueOf("2023-02-15 12:57:00")) - } - - testGluten("months_between") { - val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US) - for (zid <- outstandingZoneIds) { - withSQLConf( - SQLConf.SESSION_LOCAL_TIMEZONE.key -> zid.getId - ) { - val timeZoneId = Option(zid.getId) - sdf.setTimeZone(TimeZone.getTimeZone(zid)) - - checkEvaluation( - MonthsBetween( - Literal(new Timestamp(sdf.parse("1997-02-28 10:30:00").getTime)), - Literal(new Timestamp(sdf.parse("1996-10-30 00:00:00").getTime)), - Literal.TrueLiteral, - timeZoneId = timeZoneId - ), - 3.94959677 - ) - checkEvaluation( - MonthsBetween( - Literal(new Timestamp(sdf.parse("1997-02-28 10:30:00").getTime)), - Literal(new Timestamp(sdf.parse("1996-10-30 00:00:00").getTime)), - Literal.FalseLiteral, - timeZoneId = timeZoneId - ), - 3.9495967741935485 - ) - - Seq(Literal.FalseLiteral, Literal.TrueLiteral).foreach { - roundOff => - checkEvaluation( - MonthsBetween( - Literal(new Timestamp(sdf.parse("2015-01-30 11:52:00").getTime)), - Literal(new Timestamp(sdf.parse("2015-01-30 11:50:00").getTime)), - roundOff, - timeZoneId = timeZoneId - ), - 0.0 - ) - checkEvaluation( - MonthsBetween( - Literal(new Timestamp(sdf.parse("2015-01-31 00:00:00").getTime)), - Literal(new Timestamp(sdf.parse("2015-03-31 22:00:00").getTime)), - roundOff, - timeZoneId = timeZoneId - ), - -2.0 - ) - checkEvaluation( - MonthsBetween( - Literal(new Timestamp(sdf.parse("2015-03-31 22:00:00").getTime)), - Literal(new Timestamp(sdf.parse("2015-02-28 00:00:00").getTime)), - roundOff, - timeZoneId = timeZoneId - ), - 1.0 - ) - } - val t = Literal(Timestamp.valueOf("2015-03-31 22:00:00")) - val tnull = Literal.create(null, TimestampType) - checkEvaluation(MonthsBetween(t, tnull, Literal.TrueLiteral, timeZoneId = timeZoneId), null) - checkEvaluation(MonthsBetween(tnull, t, Literal.TrueLiteral, timeZoneId = timeZoneId), null) - checkEvaluation( - MonthsBetween(tnull, tnull, Literal.TrueLiteral, timeZoneId = timeZoneId), - null) - checkEvaluation( - MonthsBetween(t, t, Literal.create(null, BooleanType), timeZoneId = timeZoneId), - null) - checkConsistencyBetweenInterpretedAndCodegen( - (time1: Expression, time2: Expression, roundOff: Expression) => - MonthsBetween(time1, time2, roundOff, timeZoneId = timeZoneId), - TimestampType, - TimestampType, - BooleanType - ) - } - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDecimalExpressionSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDecimalExpressionSuite.scala deleted file mode 100644 index 8f9054928e4..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDecimalExpressionSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.GlutenTestsTrait - -class GlutenDecimalExpressionSuite extends DecimalExpressionSuite with GlutenTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDecimalPrecisionSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDecimalPrecisionSuite.scala deleted file mode 100644 index 97e752d7d04..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDecimalPrecisionSuite.scala +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.gluten.expression._ - -import org.apache.spark.sql.GlutenTestsTrait -import org.apache.spark.sql.catalyst.analysis.{Analyzer, EmptyFunctionRegistry, UnresolvedAttribute} -import org.apache.spark.sql.catalyst.catalog.{InMemoryCatalog, SessionCatalog} -import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, Project} -import org.apache.spark.sql.types._ - -class GlutenDecimalPrecisionSuite extends GlutenTestsTrait { - private val catalog = new SessionCatalog(new InMemoryCatalog, EmptyFunctionRegistry) - private val analyzer = new Analyzer(catalog) - - private val relation = LocalRelation( - AttributeReference("i", IntegerType)(), - AttributeReference("d1", DecimalType(2, 1))(), - AttributeReference("d2", DecimalType(5, 2))(), - AttributeReference("u", DecimalType.SYSTEM_DEFAULT)(), - AttributeReference("f", FloatType)(), - AttributeReference("b", DoubleType)() - ) - - private val i: Expression = UnresolvedAttribute("i") - private val d1: Expression = UnresolvedAttribute("d1") - private val d2: Expression = UnresolvedAttribute("d2") - private val u: Expression = UnresolvedAttribute("u") - private val f: Expression = UnresolvedAttribute("f") - private val b: Expression = UnresolvedAttribute("b") - - private def checkType(expression: Expression, expectedType: DataType): Unit = { - val plan = analyzer.execute(Project(Seq(Alias(expression, "c")()), relation)) - assert(plan.isInstanceOf[Project]) - val expr = plan.asInstanceOf[Project].projectList.head - assert(expr.dataType == expectedType) - val transformedExpr = - ExpressionConverter.replaceWithExpressionTransformer(expr, plan.inputSet.toSeq) - assert(transformedExpr.dataType == expectedType) - } - - private def stripAlias(expr: Expression): Expression = { - expr match { - case a: Alias => stripAlias(a.child) - case _ => expr - } - } - - private def checkComparison(expression: Expression, expectedType: DataType): Unit = { - val plan = analyzer.execute(Project(Alias(expression, "c")() :: Nil, relation)) - assert(plan.isInstanceOf[Project]) - val expr = stripAlias(plan.asInstanceOf[Project].projectList.head) - val transformedExpr = - ExpressionConverter.replaceWithExpressionTransformer(expr, plan.inputSet.toSeq) - assert(transformedExpr.isInstanceOf[GenericExpressionTransformer]) - val binaryComparison = transformedExpr.asInstanceOf[GenericExpressionTransformer] - assert(binaryComparison.original.isInstanceOf[BinaryComparison]) - assert(binaryComparison.children.size == 2) - assert(binaryComparison.children.forall(_.dataType == expectedType)) - } - - test("basic operations") { - checkType(Add(d1, d2), DecimalType(6, 2)) - checkType(Subtract(d1, d2), DecimalType(6, 2)) - checkType(Multiply(d1, d2), DecimalType(8, 3)) - checkType(Divide(d1, d2), DecimalType(10, 7)) - checkType(Divide(d2, d1), DecimalType(10, 6)) - - checkType(Add(Add(d1, d2), d1), DecimalType(7, 2)) - checkType(Add(Add(d1, d1), d1), DecimalType(4, 1)) - checkType(Add(d1, Add(d1, d1)), DecimalType(4, 1)) - checkType(Add(Add(Add(d1, d2), d1), d2), DecimalType(8, 2)) - checkType(Add(Add(d1, d2), Add(d1, d2)), DecimalType(7, 2)) - checkType(Subtract(Subtract(d2, d1), d1), DecimalType(7, 2)) - checkType(Multiply(Multiply(d1, d1), d2), DecimalType(11, 4)) - checkType(Divide(d2, Add(d1, d1)), DecimalType(10, 6)) - } - - test("Comparison operations") { - checkComparison(EqualTo(i, d1), DecimalType(11, 1)) - checkComparison(EqualNullSafe(d2, d1), DecimalType(5, 2)) - checkComparison(LessThan(i, d1), DecimalType(11, 1)) - checkComparison(LessThanOrEqual(d1, d2), DecimalType(5, 2)) - checkComparison(GreaterThan(d2, u), DecimalType.SYSTEM_DEFAULT) - checkComparison(GreaterThanOrEqual(d1, f), DoubleType) - checkComparison(GreaterThan(d2, d2), DecimalType(5, 2)) - } - - test("bringing in primitive types") { - checkType(Add(d1, i), DecimalType(12, 1)) - checkType(Add(d1, f), DoubleType) - checkType(Add(i, d1), DecimalType(12, 1)) - checkType(Add(f, d1), DoubleType) - checkType(Add(d1, Cast(i, LongType)), DecimalType(22, 1)) - checkType(Add(d1, Cast(i, ShortType)), DecimalType(7, 1)) - checkType(Add(d1, Cast(i, ByteType)), DecimalType(5, 1)) - checkType(Add(d1, Cast(i, DoubleType)), DoubleType) - } - - test("maximum decimals") { - for (expr <- Seq(d1, d2, i, u)) { - checkType(Add(expr, u), DecimalType(38, 17)) - checkType(Subtract(expr, u), DecimalType(38, 17)) - } - - checkType(Multiply(d1, u), DecimalType(38, 16)) - checkType(Multiply(d2, u), DecimalType(38, 14)) - checkType(Multiply(i, u), DecimalType(38, 7)) - checkType(Multiply(u, u), DecimalType(38, 6)) - - checkType(Divide(u, d1), DecimalType(38, 17)) - checkType(Divide(u, d2), DecimalType(38, 16)) - checkType(Divide(u, i), DecimalType(38, 18)) - checkType(Divide(u, u), DecimalType(38, 6)) - - for (expr <- Seq(f, b)) { - checkType(Add(expr, u), DoubleType) - checkType(Subtract(expr, u), DoubleType) - checkType(Multiply(expr, u), DoubleType) - checkType(Divide(expr, u), DoubleType) - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenGeneratorExpressionSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenGeneratorExpressionSuite.scala deleted file mode 100644 index d1867936c14..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenGeneratorExpressionSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.GlutenTestsTrait - -class GlutenGeneratorExpressionSuite extends GeneratorExpressionSuite with GlutenTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenHashExpressionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenHashExpressionsSuite.scala deleted file mode 100644 index 4f9d1ffff27..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenHashExpressionsSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.GlutenTestsTrait - -class GlutenHashExpressionsSuite extends HashExpressionsSuite with GlutenTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenHigherOrderFunctionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenHigherOrderFunctionsSuite.scala deleted file mode 100644 index 6687e707924..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenHigherOrderFunctionsSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.GlutenTestsTrait - -class GlutenHigherOrderFunctionsSuite extends HigherOrderFunctionsSuite with GlutenTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenIntervalExpressionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenIntervalExpressionsSuite.scala deleted file mode 100644 index 2b8aec03d7b..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenIntervalExpressionsSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.GlutenTestsTrait - -class GlutenIntervalExpressionsSuite extends IntervalExpressionsSuite with GlutenTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenJsonExpressionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenJsonExpressionsSuite.scala deleted file mode 100644 index f9d314e508e..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenJsonExpressionsSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.GlutenTestsTrait - -class GlutenJsonExpressionsSuite extends JsonExpressionsSuite with GlutenTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenLiteralExpressionSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenLiteralExpressionSuite.scala deleted file mode 100644 index f81ef0b6ff3..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenLiteralExpressionSuite.scala +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.GlutenTestsTrait -import org.apache.spark.sql.Row -import org.apache.spark.sql.catalyst.util.DateTimeUtils -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types._ -import org.apache.spark.unsafe.types.CalendarInterval - -import java.nio.charset.StandardCharsets -import java.time.{Instant, LocalDate} - -class GlutenLiteralExpressionSuite extends LiteralExpressionSuite with GlutenTestsTrait { - testGluten("default") { - checkEvaluation(Literal.default(BooleanType), false) - checkEvaluation(Literal.default(ByteType), 0.toByte) - checkEvaluation(Literal.default(ShortType), 0.toShort) - checkEvaluation(Literal.default(IntegerType), 0) - checkEvaluation(Literal.default(LongType), 0L) - checkEvaluation(Literal.default(FloatType), 0.0f) - checkEvaluation(Literal.default(DoubleType), 0.0) - checkEvaluation(Literal.default(StringType), "") - checkEvaluation(Literal.default(BinaryType), "".getBytes(StandardCharsets.UTF_8)) - checkEvaluation(Literal.default(DecimalType.USER_DEFAULT), Decimal(0)) - checkEvaluation(Literal.default(DecimalType.SYSTEM_DEFAULT), Decimal(0)) - withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "false") { - checkEvaluation(Literal.default(DateType), DateTimeUtils.toJavaDate(0)) - checkEvaluation(Literal.default(TimestampType), DateTimeUtils.toJavaTimestamp(0L)) - } - withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") { - checkEvaluation(Literal.default(DateType), LocalDate.ofEpochDay(0)) - checkEvaluation(Literal.default(TimestampType), Instant.ofEpochSecond(0)) - } - checkEvaluation(Literal.default(CalendarIntervalType), new CalendarInterval(0, 0, 0L)) - checkEvaluation(Literal.default(YearMonthIntervalType()), 0) - checkEvaluation(Literal.default(DayTimeIntervalType()), 0L) - checkEvaluation(Literal.default(ArrayType(StringType)), Array()) - checkEvaluation(Literal.default(MapType(IntegerType, StringType)), Map()) - checkEvaluation(Literal.default(StructType(StructField("a", StringType) :: Nil)), Row("")) - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala deleted file mode 100644 index a256b80ef25..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala +++ /dev/null @@ -1,340 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.gluten.utils.BackendTestUtils - -import org.apache.spark.sql.GlutenQueryTestUtil.isNaNOrInf -import org.apache.spark.sql.GlutenTestsTrait -import org.apache.spark.sql.catalyst.dsl.expressions._ -import org.apache.spark.sql.types._ - -import org.apache.commons.math3.util.Precision - -import java.nio.charset.StandardCharsets - -class GlutenMathExpressionsSuite extends MathExpressionsSuite with GlutenTestsTrait { - override protected def checkResult( - result: Any, - expected: Any, - exprDataType: DataType, - exprNullable: Boolean): Boolean = { - if (BackendTestUtils.isVeloxBackendLoaded()) { - super.checkResult(result, expected, exprDataType, exprNullable) - } else { - // The result is null for a non-nullable expression - assert(result != null || exprNullable, "exprNullable should be true if result is null") - (result, expected) match { - case (result: Double, expected: Double) => - if ( - (isNaNOrInf(result) || isNaNOrInf(expected)) - || (result == -0.0) || (expected == -0.0) - ) { - java.lang.Double.doubleToRawLongBits(result) == - java.lang.Double.doubleToRawLongBits(expected) - } else { - Precision.equalsWithRelativeTolerance(result, expected, 0.00001d) || - Precision.equals(result, expected, 0.00001d) - } - case _ => - super.checkResult(result, expected, exprDataType, exprNullable) - } - } - } - - testGluten("round/bround/floor/ceil") { - val scales = -6 to 6 - val doublePi: Double = math.Pi - val shortPi: Short = 31415 - val intPi: Int = 314159265 - val longPi: Long = 31415926535897932L - val bdPi: BigDecimal = BigDecimal(31415927L, 7) - val floatPi: Float = 3.1415f - - val doubleResults: Seq[Double] = - Seq(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 3.0, 3.1, 3.14, 3.142, 3.1416, 3.14159, 3.141593) - - val floatResults: Seq[Float] = - Seq(0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 3.0f, 3.1f, 3.14f, 3.142f, 3.1415f, 3.1415f, 3.1415f) - - val bRoundFloatResults: Seq[Float] = - Seq(0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 3.0f, 3.1f, 3.14f, 3.141f, 3.1415f, 3.1415f, 3.1415f) - - val shortResults: Seq[Short] = Seq[Short](0, 0, 30000, 31000, 31400, 31420) ++ - Seq.fill[Short](7)(31415) - - val intResults: Seq[Int] = - Seq(314000000, 314200000, 314160000, 314159000, 314159300, 314159270) ++ Seq.fill(7)( - 314159265) - - val longResults: Seq[Long] = Seq(31415926536000000L, 31415926535900000L, 31415926535900000L, - 31415926535898000L, 31415926535897900L, 31415926535897930L) ++ - Seq.fill(7)(31415926535897932L) - - val intResultsB: Seq[Int] = - Seq(314000000, 314200000, 314160000, 314159000, 314159300, 314159260) ++ Seq.fill(7)( - 314159265) - - def doubleResultsFloor(i: Int): Decimal = { - val results = Seq(0, 0, 0, 0, 0, 0, 3, 3.1, 3.14, 3.141, 3.1415, 3.14159, 3.141592) - Decimal(results(i)) - } - - def doubleResultsCeil(i: Int): Any = { - val results = - Seq(1000000, 100000, 10000, 1000, 100, 10, 4, 3.2, 3.15, 3.142, 3.1416, 3.1416, 3.141593) - Decimal(results(i)) - } - - def floatResultsFloor(i: Int): Any = { - val results = Seq(0, 0, 0, 0, 0, 0, 3, 3.1, 3.14, 3.141, 3.1415, 3.1415, 3.1415) - Decimal(results(i)) - } - - def floatResultsCeil(i: Int): Any = { - val results = - Seq(1000000, 100000, 10000, 1000, 100, 10, 4, 3.2, 3.15, 3.142, 3.1415, 3.1415, 3.1415) - Decimal(results(i)) - } - - def shortResultsFloor(i: Int): Decimal = { - val results = Seq(0, 0, 30000, 31000, 31400, 31410) ++ Seq.fill(7)(31415) - Decimal(results(i)) - } - - def shortResultsCeil(i: Int): Decimal = { - val results = Seq(1000000, 100000, 40000, 32000, 31500, 31420) ++ Seq.fill(7)(31415) - Decimal(results(i)) - } - - def longResultsFloor(i: Int): Decimal = { - val results = Seq(31415926535000000L, 31415926535800000L, 31415926535890000L, - 31415926535897000L, 31415926535897900L, 31415926535897930L, 31415926535897932L) ++ - Seq.fill(6)(31415926535897932L) - Decimal(results(i)) - } - - def longResultsCeil(i: Int): Decimal = { - val results = Seq(31415926536000000L, 31415926535900000L, 31415926535900000L, - 31415926535898000L, 31415926535898000L, 31415926535897940L) ++ - Seq.fill(7)(31415926535897932L) - Decimal(results(i)) - } - - def intResultsFloor(i: Int): Decimal = { - val results = - Seq(314000000, 314100000, 314150000, 314159000, 314159200, 314159260) ++ Seq.fill(7)( - 314159265) - Decimal(results(i)) - } - - def intResultsCeil(i: Int): Decimal = { - val results = - Seq(315000000, 314200000, 314160000, 314160000, 314159300, 314159270) ++ Seq.fill(7)( - 314159265) - Decimal(results(i)) - } - - scales.zipWithIndex.foreach { - case (scale, i) => - checkEvaluation(Round(doublePi, scale), doubleResults(i), EmptyRow) - checkEvaluation(Round(shortPi, scale), shortResults(i), EmptyRow) - checkEvaluation(Round(intPi, scale), intResults(i), EmptyRow) - checkEvaluation(Round(longPi, scale), longResults(i), EmptyRow) - checkEvaluation(Round(floatPi, scale), floatResults(i), EmptyRow) - checkEvaluation(BRound(doublePi, scale), doubleResults(i), EmptyRow) - checkEvaluation(BRound(shortPi, scale), shortResults(i), EmptyRow) - checkEvaluation(BRound(intPi, scale), intResultsB(i), EmptyRow) - checkEvaluation(BRound(longPi, scale), longResults(i), EmptyRow) - checkEvaluation( - BRound(floatPi, scale), - // the velox backend will fallback when executing bround, - // so uses the same excepted results with the vanilla spark - if (BackendTestUtils.isCHBackendLoaded()) floatResults(i) else bRoundFloatResults(i), - EmptyRow - ) - checkEvaluation( - checkDataTypeAndCast(RoundFloor(Literal(doublePi), Literal(scale))), - doubleResultsFloor(i), - EmptyRow) - checkEvaluation( - checkDataTypeAndCast(RoundFloor(Literal(shortPi), Literal(scale))), - shortResultsFloor(i), - EmptyRow) - checkEvaluation( - checkDataTypeAndCast(RoundFloor(Literal(intPi), Literal(scale))), - intResultsFloor(i), - EmptyRow) - checkEvaluation( - checkDataTypeAndCast(RoundFloor(Literal(longPi), Literal(scale))), - longResultsFloor(i), - EmptyRow) - checkEvaluation( - checkDataTypeAndCast(RoundFloor(Literal(floatPi), Literal(scale))), - floatResultsFloor(i), - EmptyRow) - checkEvaluation( - checkDataTypeAndCast(RoundCeil(Literal(doublePi), Literal(scale))), - doubleResultsCeil(i), - EmptyRow) - checkEvaluation( - checkDataTypeAndCast(RoundCeil(Literal(shortPi), Literal(scale))), - shortResultsCeil(i), - EmptyRow) - checkEvaluation( - checkDataTypeAndCast(RoundCeil(Literal(intPi), Literal(scale))), - intResultsCeil(i), - EmptyRow) - checkEvaluation( - checkDataTypeAndCast(RoundCeil(Literal(longPi), Literal(scale))), - longResultsCeil(i), - EmptyRow) - checkEvaluation( - checkDataTypeAndCast(RoundCeil(Literal(floatPi), Literal(scale))), - floatResultsCeil(i), - EmptyRow) - } - - val bdResults: Seq[BigDecimal] = Seq( - BigDecimal(3), - BigDecimal("3.1"), - BigDecimal("3.14"), - BigDecimal("3.142"), - BigDecimal("3.1416"), - BigDecimal("3.14159"), - BigDecimal("3.141593"), - BigDecimal("3.1415927") - ) - - val bdResultsFloor: Seq[BigDecimal] = - Seq( - BigDecimal(3), - BigDecimal("3.1"), - BigDecimal("3.14"), - BigDecimal("3.141"), - BigDecimal("3.1415"), - BigDecimal("3.14159"), - BigDecimal("3.141592"), - BigDecimal("3.1415927") - ) - - val bdResultsCeil: Seq[BigDecimal] = Seq( - BigDecimal(4), - BigDecimal("3.2"), - BigDecimal("3.15"), - BigDecimal("3.142"), - BigDecimal("3.1416"), - BigDecimal("3.14160"), - BigDecimal("3.141593"), - BigDecimal("3.1415927") - ) - - (0 to 7).foreach { - i => - checkEvaluation(Round(bdPi, i), bdResults(i), EmptyRow) - checkEvaluation(BRound(bdPi, i), bdResults(i), EmptyRow) - checkEvaluation(RoundFloor(bdPi, i), bdResultsFloor(i), EmptyRow) - checkEvaluation(RoundCeil(bdPi, i), bdResultsCeil(i), EmptyRow) - } - (8 to 10).foreach { - scale => - checkEvaluation(Round(bdPi, scale), bdPi, EmptyRow) - checkEvaluation(BRound(bdPi, scale), bdPi, EmptyRow) - checkEvaluation(RoundFloor(bdPi, scale), bdPi, EmptyRow) - checkEvaluation(RoundCeil(bdPi, scale), bdPi, EmptyRow) - } - - DataTypeTestUtils.numericTypes.foreach { - dataType => - checkEvaluation(Round(Literal.create(null, dataType), Literal(2)), null) - checkEvaluation( - Round(Literal.create(null, dataType), Literal.create(null, IntegerType)), - null) - checkEvaluation(BRound(Literal.create(null, dataType), Literal(2)), null) - checkEvaluation( - BRound(Literal.create(null, dataType), Literal.create(null, IntegerType)), - null) - checkEvaluation( - checkDataTypeAndCast(RoundFloor(Literal.create(null, dataType), Literal(2))), - null) - checkEvaluation( - checkDataTypeAndCast(RoundCeil(Literal.create(null, dataType), Literal(2))), - null) - } - - checkEvaluation(Round(2.5, 0), 3.0) - checkEvaluation(Round(3.5, 0), 4.0) - checkEvaluation(Round(-2.5, 0), -3.0) - checkEvaluation(Round(-3.5, 0), -4.0) - checkEvaluation(Round(-0.35, 1), -0.4) - checkEvaluation(Round(-35, -1), -40) - checkEvaluation(Round(1.12345678901234567, 8), 1.12345679) - checkEvaluation(Round(-0.98765432109876543, 5), -0.98765) - checkEvaluation(Round(12345.67890123456789, 6), 12345.678901) - // Enable the test after fixing https://github.com/apache/gluten/issues/6827 - // checkEvaluation(Round(0.5549999999999999, 2), 0.55) - checkEvaluation(Round(-35, -1), -40) - checkEvaluation(Round(44, -1), 40) - checkEvaluation(Round(78, 1), 78) - checkEvaluation(Round(BigDecimal("45.00"), -1), BigDecimal(50)) - checkEvaluation(BRound(2.5, 0), 2.0) - checkEvaluation(BRound(3.5, 0), 4.0) - checkEvaluation(BRound(-2.5, 0), -2.0) - checkEvaluation(BRound(-3.5, 0), -4.0) - checkEvaluation(BRound(-0.35, 1), -0.4) - checkEvaluation(BRound(-35, -1), -40) - checkEvaluation(BRound(BigDecimal("45.00"), -1), BigDecimal(40)) - checkEvaluation(checkDataTypeAndCast(RoundFloor(Literal(2.5), Literal(0))), Decimal(2)) - checkEvaluation(checkDataTypeAndCast(RoundFloor(Literal(3.5), Literal(0))), Decimal(3)) - checkEvaluation(checkDataTypeAndCast(RoundFloor(Literal(-2.5), Literal(0))), Decimal(-3L)) - checkEvaluation(checkDataTypeAndCast(RoundFloor(Literal(-3.5), Literal(0))), Decimal(-4L)) - checkEvaluation(checkDataTypeAndCast(RoundFloor(Literal(-0.35), Literal(1))), Decimal(-0.4)) - checkEvaluation(checkDataTypeAndCast(RoundFloor(Literal(-35), Literal(-1))), Decimal(-40)) - checkEvaluation(checkDataTypeAndCast(RoundFloor(Literal(-0.1), Literal(0))), Decimal(-1)) - checkEvaluation(checkDataTypeAndCast(RoundFloor(Literal(5), Literal(0))), Decimal(5)) - checkEvaluation(checkDataTypeAndCast(RoundFloor(Literal(3.1411), Literal(-3))), Decimal(0)) - checkEvaluation(checkDataTypeAndCast(RoundFloor(Literal(135.135), Literal(-2))), Decimal(100)) - checkEvaluation(checkDataTypeAndCast(RoundCeil(Literal(2.5), Literal(0))), Decimal(3)) - checkEvaluation(checkDataTypeAndCast(RoundCeil(Literal(3.5), Literal(0))), Decimal(4L)) - checkEvaluation(checkDataTypeAndCast(RoundCeil(Literal(-2.5), Literal(0))), Decimal(-2L)) - checkEvaluation(checkDataTypeAndCast(RoundCeil(Literal(-3.5), Literal(0))), Decimal(-3L)) - checkEvaluation(checkDataTypeAndCast(RoundCeil(Literal(-0.35), Literal(1))), Decimal(-0.3)) - checkEvaluation(checkDataTypeAndCast(RoundCeil(Literal(-35), Literal(-1))), Decimal(-30)) - checkEvaluation(checkDataTypeAndCast(RoundCeil(Literal(-0.1), Literal(0))), Decimal(0)) - checkEvaluation(checkDataTypeAndCast(RoundCeil(Literal(5), Literal(0))), Decimal(5)) - checkEvaluation(checkDataTypeAndCast(RoundCeil(Literal(3.1411), Literal(-3))), Decimal(1000)) - checkEvaluation(checkDataTypeAndCast(RoundCeil(Literal(135.135), Literal(-2))), Decimal(200)) - } - - testGluten("unhex") { - checkEvaluation(Unhex(Literal.create(null, StringType)), null) - checkEvaluation(Unhex(Literal("737472696E67")), "string".getBytes(StandardCharsets.UTF_8)) - checkEvaluation(Unhex(Literal("")), new Array[Byte](0)) - checkEvaluation(Unhex(Literal("F")), Array[Byte](15)) - checkEvaluation(Unhex(Literal("ff")), Array[Byte](-1)) - -// checkEvaluation(Unhex(Literal("GG")), null) - checkEvaluation(Unhex(Literal("123")), Array[Byte](1, 35)) - checkEvaluation(Unhex(Literal("12345")), Array[Byte](1, 35, 69)) - // scalastyle:off - // Turn off scala style for non-ascii chars - checkEvaluation(Unhex(Literal("E4B889E9878DE79A84")), "三重的".getBytes(StandardCharsets.UTF_8)) -// checkEvaluation(Unhex(Literal("三重的")), null) - // scalastyle:on - checkConsistencyBetweenInterpretedAndCodegen(Unhex, StringType) - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMiscExpressionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMiscExpressionsSuite.scala deleted file mode 100644 index c734a9cfbbd..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMiscExpressionsSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.GlutenTestsTrait - -class GlutenMiscExpressionsSuite extends MiscExpressionsSuite with GlutenTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenNondeterministicSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenNondeterministicSuite.scala deleted file mode 100644 index 34830b368ca..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenNondeterministicSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.GlutenTestsTrait - -class GlutenNondeterministicSuite extends NondeterministicSuite with GlutenTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenNullExpressionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenNullExpressionsSuite.scala deleted file mode 100644 index 900fd764d0d..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenNullExpressionsSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.GlutenTestsTrait - -class GlutenNullExpressionsSuite extends NullExpressionsSuite with GlutenTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenPredicateSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenPredicateSuite.scala deleted file mode 100644 index 90e93f3593e..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenPredicateSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.GlutenTestsTrait - -class GlutenPredicateSuite extends PredicateSuite with GlutenTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenRandomSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenRandomSuite.scala deleted file mode 100644 index 95d2e71ffe5..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenRandomSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.GlutenTestsTrait - -class GlutenRandomSuite extends RandomSuite with GlutenTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenRegexpExpressionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenRegexpExpressionsSuite.scala deleted file mode 100644 index 33cb9a78358..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenRegexpExpressionsSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.GlutenTestsTrait - -class GlutenRegexpExpressionsSuite extends RegexpExpressionsSuite with GlutenTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenSortOrderExpressionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenSortOrderExpressionsSuite.scala deleted file mode 100644 index 37c630f495f..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenSortOrderExpressionsSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.GlutenTestsTrait - -class GlutenSortOrderExpressionsSuite extends SortOrderExpressionsSuite with GlutenTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenStringExpressionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenStringExpressionsSuite.scala deleted file mode 100644 index cdb67efeccf..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenStringExpressionsSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.GlutenTestsTrait - -class GlutenStringExpressionsSuite extends StringExpressionsSuite with GlutenTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/GlutenPercentileSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/GlutenPercentileSuite.scala deleted file mode 100644 index 5f89c2810e6..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/GlutenPercentileSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions.aggregate - -import org.apache.spark.sql.GlutenTestsTrait - -class GlutenPercentileSuite extends PercentileSuite with GlutenTestsTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenDataSourceV2DataFrameSessionCatalogSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenDataSourceV2DataFrameSessionCatalogSuite.scala deleted file mode 100644 index 4099ea13822..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenDataSourceV2DataFrameSessionCatalogSuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.connector - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenDataSourceV2DataFrameSessionCatalogSuite - extends DataSourceV2DataFrameSessionCatalogSuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenDataSourceV2DataFrameSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenDataSourceV2DataFrameSuite.scala deleted file mode 100644 index 327c930bfb3..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenDataSourceV2DataFrameSuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.connector - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenDataSourceV2DataFrameSuite - extends DataSourceV2DataFrameSuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenDataSourceV2FunctionSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenDataSourceV2FunctionSuite.scala deleted file mode 100644 index 10f4d90f54f..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenDataSourceV2FunctionSuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.connector - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenDataSourceV2FunctionSuite - extends DataSourceV2FunctionSuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenDataSourceV2SQLSessionCatalogSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenDataSourceV2SQLSessionCatalogSuite.scala deleted file mode 100644 index 7e1a1cdaca9..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenDataSourceV2SQLSessionCatalogSuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.connector - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenDataSourceV2SQLSessionCatalogSuite - extends DataSourceV2SQLSessionCatalogSuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenDataSourceV2SQLSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenDataSourceV2SQLSuite.scala deleted file mode 100644 index c3666f80fae..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenDataSourceV2SQLSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.connector - -import org.apache.spark.sql._ - -class GlutenDataSourceV2SQLSuite extends DataSourceV2SQLSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenDataSourceV2Suite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenDataSourceV2Suite.scala deleted file mode 100644 index eeb77133e4d..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenDataSourceV2Suite.scala +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.connector - -import org.apache.spark.sql.{GlutenSQLTestsBaseTrait, Row} -import org.apache.spark.sql.execution.ColumnarShuffleExchangeExec -import org.apache.spark.sql.internal.SQLConf - -import test.org.apache.spark.sql.connector.JavaPartitionAwareDataSource - -class GlutenDataSourceV2Suite extends DataSourceV2Suite with GlutenSQLTestsBaseTrait { - import testImplicits._ - - testGluten("partitioning reporting") { - import org.apache.spark.sql.functions.{count, sum} - withSQLConf(SQLConf.V2_BUCKETING_ENABLED.key -> "true") { - Seq(classOf[PartitionAwareDataSource], classOf[JavaPartitionAwareDataSource]).foreach { - cls => - withClue(cls.getName) { - val df = spark.read.format(cls.getName).load() - checkAnswer(df, Seq(Row(1, 4), Row(1, 4), Row(3, 6), Row(2, 6), Row(4, 2), Row(4, 2))) - - val groupByColA = df.groupBy('i).agg(sum('j)) - checkAnswer(groupByColA, Seq(Row(1, 8), Row(2, 6), Row(3, 6), Row(4, 4))) - assert(collectFirst(groupByColA.queryExecution.executedPlan) { - case e: ColumnarShuffleExchangeExec => e - }.isEmpty) - - val groupByColAB = df.groupBy('i, 'j).agg(count("*")) - checkAnswer(groupByColAB, Seq(Row(1, 4, 2), Row(2, 6, 1), Row(3, 6, 1), Row(4, 2, 2))) - assert(collectFirst(groupByColAB.queryExecution.executedPlan) { - case e: ColumnarShuffleExchangeExec => e - }.isEmpty) - - val groupByColB = df.groupBy('j).agg(sum('i)) - checkAnswer(groupByColB, Seq(Row(2, 8), Row(4, 2), Row(6, 5))) - assert(collectFirst(groupByColB.queryExecution.executedPlan) { - case e: ColumnarShuffleExchangeExec => e - }.isDefined) - - val groupByAPlusB = df.groupBy('i + 'j).agg(count("*")) - checkAnswer(groupByAPlusB, Seq(Row(5, 2), Row(6, 2), Row(8, 1), Row(9, 1))) - assert(collectFirst(groupByAPlusB.queryExecution.executedPlan) { - case e: ColumnarShuffleExchangeExec => e - }.isDefined) - } - } - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenDeleteFromTableSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenDeleteFromTableSuite.scala deleted file mode 100644 index ea2fc4e943e..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenDeleteFromTableSuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.connector - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenDeleteFromTableSuite - extends GroupBasedDeleteFromTableSuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenFileDataSourceV2FallBackSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenFileDataSourceV2FallBackSuite.scala deleted file mode 100644 index 02198099aa4..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenFileDataSourceV2FallBackSuite.scala +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.connector - -import org.apache.gluten.execution.FileSourceScanExecTransformer - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.execution.QueryExecution -import org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand -import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat -import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetDataSourceV2 -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.util.QueryExecutionListener - -import scala.collection.Seq -import scala.collection.mutable.ArrayBuffer - -class GlutenFileDataSourceV2FallBackSuite - extends FileDataSourceV2FallBackSuite - with GlutenSQLTestsBaseTrait { - - testGluten("Fallback Parquet V2 to V1") { - Seq("parquet", classOf[ParquetDataSourceV2].getCanonicalName).foreach { - format => - withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> format) { - val commands = ArrayBuffer.empty[(String, LogicalPlan)] - val exceptions = ArrayBuffer.empty[(String, Exception)] - val listener = new QueryExecutionListener { - override def onFailure( - funcName: String, - qe: QueryExecution, - exception: Exception): Unit = { - exceptions += funcName -> exception - } - - override def onSuccess(funcName: String, qe: QueryExecution, duration: Long): Unit = { - commands += funcName -> qe.logical - } - } - spark.listenerManager.register(listener) - - try { - withTempPath { - path => - val inputData = spark.range(10) - inputData.write.format(format).save(path.getCanonicalPath) - sparkContext.listenerBus.waitUntilEmpty() - assert(commands.length == 1) - assert(commands.head._1 == "command") - assert(commands.head._2.isInstanceOf[InsertIntoHadoopFsRelationCommand]) - assert( - commands.head._2 - .asInstanceOf[InsertIntoHadoopFsRelationCommand] - .fileFormat - .isInstanceOf[ParquetFileFormat]) - val df = spark.read.format(format).load(path.getCanonicalPath) - checkAnswer(df, inputData.toDF()) - assert( - df.queryExecution.executedPlan.exists( - _.isInstanceOf[FileSourceScanExecTransformer])) - } - } finally { - spark.listenerManager.unregister(listener) - } - } - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenKeyGroupedPartitioningSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenKeyGroupedPartitioningSuite.scala deleted file mode 100644 index 9cea1d9dc66..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenKeyGroupedPartitioningSuite.scala +++ /dev/null @@ -1,292 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.connector - -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.execution.SortMergeJoinExecTransformerBase - -import org.apache.spark.SparkConf -import org.apache.spark.sql.GlutenSQLTestsBaseTrait -import org.apache.spark.sql.Row -import org.apache.spark.sql.connector.catalog.Identifier -import org.apache.spark.sql.connector.catalog.InMemoryTableCatalog -import org.apache.spark.sql.connector.distributions.Distributions -import org.apache.spark.sql.connector.expressions._ -import org.apache.spark.sql.connector.expressions.Expressions._ -import org.apache.spark.sql.execution.{ColumnarShuffleExchangeExec, SparkPlan} -import org.apache.spark.sql.execution.joins.SortMergeJoinExec -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types._ - -import java.util.Collections - -class GlutenKeyGroupedPartitioningSuite - extends KeyGroupedPartitioningSuite - with GlutenSQLTestsBaseTrait { - override def sparkConf: SparkConf = { - // Native SQL configs - super.sparkConf - .set(GlutenConfig.COLUMNAR_FORCE_SHUFFLED_HASH_JOIN_ENABLED.key, "false") - .set("spark.sql.adaptive.enabled", "false") - .set("spark.sql.shuffle.partitions", "5") - } - - private val emptyProps: java.util.Map[String, String] = { - Collections.emptyMap[String, String] - } - private def createTable( - table: String, - schema: StructType, - partitions: Array[Transform], - catalog: InMemoryTableCatalog = catalog): Unit = { - catalog.createTable( - Identifier.of(Array("ns"), table), - schema, - partitions, - emptyProps, - Distributions.unspecified(), - Array.empty, - None) - } - - private val customers: String = "customers" - private val customers_schema = new StructType() - .add("customer_name", StringType) - .add("customer_age", IntegerType) - .add("customer_id", LongType) - - private val orders: String = "orders" - private val orders_schema = new StructType() - .add("order_amount", DoubleType) - .add("customer_id", LongType) - - private def testWithCustomersAndOrders( - customers_partitions: Array[Transform], - orders_partitions: Array[Transform], - expectedNumOfShuffleExecs: Int): Unit = { - createTable(customers, customers_schema, customers_partitions) - sql( - s"INSERT INTO testcat.ns.$customers VALUES " + - s"('aaa', 10, 1), ('bbb', 20, 2), ('ccc', 30, 3)") - - createTable(orders, orders_schema, orders_partitions) - sql( - s"INSERT INTO testcat.ns.$orders VALUES " + - s"(100.0, 1), (200.0, 1), (150.0, 2), (250.0, 2), (350.0, 2), (400.50, 3)") - - val df = sql( - "SELECT customer_name, customer_age, order_amount " + - s"FROM testcat.ns.$customers c JOIN testcat.ns.$orders o " + - "ON c.customer_id = o.customer_id ORDER BY c.customer_id, order_amount") - - val shuffles = collectColumnarShuffleExchangeExec(df.queryExecution.executedPlan) - assert(shuffles.length == expectedNumOfShuffleExecs) - - checkAnswer( - df, - Seq( - Row("aaa", 10, 100.0), - Row("aaa", 10, 200.0), - Row("bbb", 20, 150.0), - Row("bbb", 20, 250.0), - Row("bbb", 20, 350.0), - Row("ccc", 30, 400.50))) - } - - private def collectColumnarShuffleExchangeExec( - plan: SparkPlan): Seq[ColumnarShuffleExchangeExec] = { - // here we skip collecting shuffle operators that are not associated with SMJ - collect(plan) { - case s: SortMergeJoinExecTransformerBase => s - case s: SortMergeJoinExec => s - }.flatMap(smj => collect(smj) { case s: ColumnarShuffleExchangeExec => s }) - } - - testGluten("partitioned join: only one side reports partitioning") { - val customers_partitions = Array(bucket(4, "customer_id")) - val orders_partitions = Array(bucket(2, "customer_id")) - - testWithCustomersAndOrders(customers_partitions, orders_partitions, 2) - } - testGluten("partitioned join: exact distribution (same number of buckets) from both sides") { - val customers_partitions = Array(bucket(4, "customer_id")) - val orders_partitions = Array(bucket(4, "customer_id")) - - testWithCustomersAndOrders(customers_partitions, orders_partitions, 0) - } - - private val items: String = "items" - private val items_schema: StructType = new StructType() - .add("id", LongType) - .add("name", StringType) - .add("price", FloatType) - .add("arrive_time", TimestampType) - - private val purchases: String = "purchases" - private val purchases_schema: StructType = new StructType() - .add("item_id", LongType) - .add("price", FloatType) - .add("time", TimestampType) - - testGluten("partitioned join: join with two partition keys and matching & sorted partitions") { - val items_partitions = Array(bucket(8, "id"), days("arrive_time")) - createTable(items, items_schema, items_partitions) - sql( - s"INSERT INTO testcat.ns.$items VALUES " + - s"(1, 'aa', 40.0, cast('2020-01-01' as timestamp)), " + - s"(1, 'aa', 41.0, cast('2020-01-15' as timestamp)), " + - s"(2, 'bb', 10.0, cast('2020-01-01' as timestamp)), " + - s"(2, 'bb', 10.5, cast('2020-01-01' as timestamp)), " + - s"(3, 'cc', 15.5, cast('2020-02-01' as timestamp))") - - val purchases_partitions = Array(bucket(8, "item_id"), days("time")) - createTable(purchases, purchases_schema, purchases_partitions) - sql( - s"INSERT INTO testcat.ns.$purchases VALUES " + - s"(1, 42.0, cast('2020-01-01' as timestamp)), " + - s"(1, 44.0, cast('2020-01-15' as timestamp)), " + - s"(1, 45.0, cast('2020-01-15' as timestamp)), " + - s"(2, 11.0, cast('2020-01-01' as timestamp)), " + - s"(3, 19.5, cast('2020-02-01' as timestamp))") - - val df = sql( - "SELECT id, name, i.price as purchase_price, p.price as sale_price " + - s"FROM testcat.ns.$items i JOIN testcat.ns.$purchases p " + - "ON i.id = p.item_id AND i.arrive_time = p.time ORDER BY id, purchase_price, sale_price") - - val shuffles = collectColumnarShuffleExchangeExec(df.queryExecution.executedPlan) - assert(shuffles.isEmpty, "should not add shuffle for both sides of the join") - checkAnswer( - df, - Seq( - Row(1, "aa", 40.0, 42.0), - Row(1, "aa", 41.0, 44.0), - Row(1, "aa", 41.0, 45.0), - Row(2, "bb", 10.0, 11.0), - Row(2, "bb", 10.5, 11.0), - Row(3, "cc", 15.5, 19.5))) - } - - testGluten("partitioned join: join with two partition keys and unsorted partitions") { - val items_partitions = Array(bucket(8, "id"), days("arrive_time")) - createTable(items, items_schema, items_partitions) - sql( - s"INSERT INTO testcat.ns.$items VALUES " + - s"(3, 'cc', 15.5, cast('2020-02-01' as timestamp)), " + - s"(1, 'aa', 40.0, cast('2020-01-01' as timestamp)), " + - s"(1, 'aa', 41.0, cast('2020-01-15' as timestamp)), " + - s"(2, 'bb', 10.0, cast('2020-01-01' as timestamp)), " + - s"(2, 'bb', 10.5, cast('2020-01-01' as timestamp))") - - val purchases_partitions = Array(bucket(8, "item_id"), days("time")) - createTable(purchases, purchases_schema, purchases_partitions) - sql( - s"INSERT INTO testcat.ns.$purchases VALUES " + - s"(2, 11.0, cast('2020-01-01' as timestamp)), " + - s"(1, 42.0, cast('2020-01-01' as timestamp)), " + - s"(1, 44.0, cast('2020-01-15' as timestamp)), " + - s"(1, 45.0, cast('2020-01-15' as timestamp)), " + - s"(3, 19.5, cast('2020-02-01' as timestamp))") - - val df = sql( - "SELECT id, name, i.price as purchase_price, p.price as sale_price " + - s"FROM testcat.ns.$items i JOIN testcat.ns.$purchases p " + - "ON i.id = p.item_id AND i.arrive_time = p.time ORDER BY id, purchase_price, sale_price") - - val shuffles = collectColumnarShuffleExchangeExec(df.queryExecution.executedPlan) - assert(shuffles.isEmpty, "should not add shuffle for both sides of the join") - checkAnswer( - df, - Seq( - Row(1, "aa", 40.0, 42.0), - Row(1, "aa", 41.0, 44.0), - Row(1, "aa", 41.0, 45.0), - Row(2, "bb", 10.0, 11.0), - Row(2, "bb", 10.5, 11.0), - Row(3, "cc", 15.5, 19.5))) - } - - testGluten("partitioned join: join with two partition keys and different # of partition keys") { - val items_partitions = Array(bucket(8, "id"), days("arrive_time")) - createTable(items, items_schema, items_partitions) - - sql( - s"INSERT INTO testcat.ns.$items VALUES " + - s"(1, 'aa', 40.0, cast('2020-01-01' as timestamp)), " + - s"(2, 'bb', 10.0, cast('2020-01-01' as timestamp)), " + - s"(3, 'cc', 15.5, cast('2020-02-01' as timestamp))") - - val purchases_partitions = Array(bucket(8, "item_id"), days("time")) - createTable(purchases, purchases_schema, purchases_partitions) - sql( - s"INSERT INTO testcat.ns.$purchases VALUES " + - s"(1, 42.0, cast('2020-01-01' as timestamp)), " + - s"(2, 11.0, cast('2020-01-01' as timestamp))") - - val df = sql( - "SELECT id, name, i.price as purchase_price, p.price as sale_price " + - s"FROM testcat.ns.$items i JOIN testcat.ns.$purchases p " + - "ON i.id = p.item_id AND i.arrive_time = p.time ORDER BY id, purchase_price, sale_price") - - val shuffles = collectColumnarShuffleExchangeExec(df.queryExecution.executedPlan) - assert(shuffles.nonEmpty, "should add shuffle when partition keys mismatch") - } - - testGluten("data source partitioning + dynamic partition filtering") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", - SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true", - SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false", - SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "10" - ) { - val items_partitions = Array(identity("id")) - createTable(items, items_schema, items_partitions) - sql( - s"INSERT INTO testcat.ns.$items VALUES " + - s"(1, 'aa', 40.0, cast('2020-01-01' as timestamp)), " + - s"(1, 'aa', 41.0, cast('2020-01-15' as timestamp)), " + - s"(2, 'bb', 10.0, cast('2020-01-01' as timestamp)), " + - s"(2, 'bb', 10.5, cast('2020-01-01' as timestamp)), " + - s"(3, 'cc', 15.5, cast('2020-02-01' as timestamp))") - - val purchases_partitions = Array(identity("item_id")) - createTable(purchases, purchases_schema, purchases_partitions) - sql( - s"INSERT INTO testcat.ns.$purchases VALUES " + - s"(1, 42.0, cast('2020-01-01' as timestamp)), " + - s"(1, 44.0, cast('2020-01-15' as timestamp)), " + - s"(1, 45.0, cast('2020-01-15' as timestamp)), " + - s"(2, 11.0, cast('2020-01-01' as timestamp)), " + - s"(3, 19.5, cast('2020-02-01' as timestamp))") - - // number of unique partitions changed after dynamic filtering - should throw exception - var df = sql( - s"SELECT sum(p.price) from testcat.ns.$items i, testcat.ns.$purchases p WHERE " + - s"i.id = p.item_id AND i.price > 40.0") - val e = intercept[Exception](df.collect()) - assert(e.getMessage.contains("number of unique partition values")) - - // dynamic filtering doesn't change partitioning so storage-partitioned join should kick in - df = sql( - s"SELECT sum(p.price) from testcat.ns.$items i, testcat.ns.$purchases p WHERE " + - s"i.id = p.item_id AND i.price >= 10.0") - val shuffles = collectColumnarShuffleExchangeExec(df.queryExecution.executedPlan) - assert(shuffles.isEmpty, "should not add shuffle for both sides of the join") - checkAnswer(df, Seq(Row(303.5))) - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenLocalScanSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenLocalScanSuite.scala deleted file mode 100644 index 735b5d1a0e1..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenLocalScanSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.connector - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenLocalScanSuite extends LocalScanSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenMetadataColumnSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenMetadataColumnSuite.scala deleted file mode 100644 index 59a14fb11c0..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenMetadataColumnSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.connector - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenMetadataColumnSuite extends MetadataColumnSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenSupportsCatalogOptionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenSupportsCatalogOptionsSuite.scala deleted file mode 100644 index 92f2a04cebe..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenSupportsCatalogOptionsSuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.connector - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenSupportsCatalogOptionsSuite - extends SupportsCatalogOptionsSuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenTableCapabilityCheckSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenTableCapabilityCheckSuite.scala deleted file mode 100644 index 93502b7adb0..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenTableCapabilityCheckSuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.connector - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenTableCapabilityCheckSuite - extends TableCapabilityCheckSuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenWriteDistributionAndOrderingSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenWriteDistributionAndOrderingSuite.scala deleted file mode 100644 index f96ec9a6d1d..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/connector/GlutenWriteDistributionAndOrderingSuite.scala +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.connector - -import org.apache.spark.SparkConf -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenWriteDistributionAndOrderingSuite - extends WriteDistributionAndOrderingSuite - with GlutenSQLTestsBaseTrait { - override def sparkConf: SparkConf = { - // Native SQL configs - super.sparkConf - .set("spark.sql.shuffle.partitions", "5") - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/errors/GlutenQueryCompilationErrorsDSv2Suite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/errors/GlutenQueryCompilationErrorsDSv2Suite.scala deleted file mode 100644 index 6c14c16664a..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/errors/GlutenQueryCompilationErrorsDSv2Suite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.errors - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenQueryCompilationErrorsDSv2Suite - extends QueryCompilationErrorsDSv2Suite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/errors/GlutenQueryCompilationErrorsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/errors/GlutenQueryCompilationErrorsSuite.scala deleted file mode 100644 index 7ccb3b059ac..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/errors/GlutenQueryCompilationErrorsSuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.errors - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenQueryCompilationErrorsSuite - extends QueryCompilationErrorsSuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/errors/GlutenQueryExecutionErrorsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/errors/GlutenQueryExecutionErrorsSuite.scala deleted file mode 100644 index 8896541c29d..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/errors/GlutenQueryExecutionErrorsSuite.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.errors - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenQueryExecutionErrorsSuite - extends QueryExecutionErrorsSuite - with GlutenSQLTestsBaseTrait { - override protected def getResourceParquetFilePath(name: String): String = { - getWorkspaceFilePath("sql", "core", "src", "test", "resources").toString + "/" + name - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/errors/GlutenQueryParsingErrorsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/errors/GlutenQueryParsingErrorsSuite.scala deleted file mode 100644 index 307a740396e..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/errors/GlutenQueryParsingErrorsSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.errors - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenQueryParsingErrorsSuite extends QueryParsingErrorsSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala deleted file mode 100644 index 91766cf1eb4..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala +++ /dev/null @@ -1,259 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution - -import org.apache.gluten.backendsapi.BackendsApiManager -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.execution.{BasicScanExecTransformer, GlutenPlan} -import org.apache.gluten.extension.GlutenSessionExtensions -import org.apache.gluten.extension.caller.CallerInfo -import org.apache.gluten.extension.columnar.{FallbackTags, RemoveFallbackTagRule} -import org.apache.gluten.extension.columnar.ColumnarRuleApplier.ColumnarRuleCall -import org.apache.gluten.extension.columnar.MiscColumnarRules.RemoveTopmostColumnarToRow -import org.apache.gluten.extension.columnar.heuristic.{ExpandFallbackPolicy, HeuristicApplier} -import org.apache.gluten.extension.columnar.transition.{Convention, InsertBackendTransitions} - -import org.apache.spark.rdd.RDD -import org.apache.spark.sql.{GlutenSQLTestsTrait, SparkSession} -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.catalyst.rules.Rule - -class FallbackStrategiesSuite extends GlutenSQLTestsTrait { - import FallbackStrategiesSuite._ - testGluten("Fall back the whole query if one unsupported") { - withSQLConf((GlutenConfig.COLUMNAR_QUERY_FALLBACK_THRESHOLD.key, "1")) { - val originalPlan = UnaryOp2(UnaryOp1(UnaryOp2(UnaryOp1(LeafOp())))) - val rule = newRuleApplier( - spark, - List( - _ => - _ => { - UnaryOp2(UnaryOp1Transformer(UnaryOp2(UnaryOp1Transformer(LeafOp())))) - }, - c => InsertBackendTransitions(c.outputsColumnar))) - val outputPlan = rule.apply(originalPlan, false) - // Expect to fall back the entire plan. - assert(outputPlan == originalPlan) - } - } - - testGluten("Fall back the whole plan if meeting the configured threshold") { - withSQLConf((GlutenConfig.COLUMNAR_WHOLESTAGE_FALLBACK_THRESHOLD.key, "1")) { - CallerInfo.withLocalValue(isAqe = true, isCache = false) { - val originalPlan = UnaryOp2(UnaryOp1(UnaryOp2(UnaryOp1(LeafOp())))) - val rule = newRuleApplier( - spark, - List( - _ => - _ => { - UnaryOp2(UnaryOp1Transformer(UnaryOp2(UnaryOp1Transformer(LeafOp())))) - }, - c => InsertBackendTransitions(c.outputsColumnar))) - val outputPlan = rule.apply(originalPlan, false) - // Expect to fall back the entire plan. - assert(outputPlan == originalPlan) - } - } - } - - testGluten("Don't fall back the whole plan if NOT meeting the configured threshold") { - withSQLConf((GlutenConfig.COLUMNAR_WHOLESTAGE_FALLBACK_THRESHOLD.key, "4")) { - CallerInfo.withLocalValue(isAqe = true, isCache = false) { - val originalPlan = UnaryOp2(UnaryOp1(UnaryOp2(UnaryOp1(LeafOp())))) - val rule = newRuleApplier( - spark, - List( - _ => - _ => { - UnaryOp2(UnaryOp1Transformer(UnaryOp2(UnaryOp1Transformer(LeafOp())))) - }, - c => InsertBackendTransitions(c.outputsColumnar))) - val outputPlan = rule.apply(originalPlan, false) - // Expect to get the plan with columnar rule applied. - assert(outputPlan != originalPlan) - } - } - } - - testGluten( - "Fall back the whole plan if meeting the configured threshold (leaf node is" + - " transformable)") { - withSQLConf((GlutenConfig.COLUMNAR_WHOLESTAGE_FALLBACK_THRESHOLD.key, "2")) { - CallerInfo.withLocalValue(isAqe = true, isCache = false) { - val originalPlan = UnaryOp2(UnaryOp1(UnaryOp2(UnaryOp1(LeafOp())))) - val rule = newRuleApplier( - spark, - List( - _ => - _ => { - UnaryOp2(UnaryOp1Transformer(UnaryOp2(UnaryOp1Transformer(LeafOpTransformer())))) - }, - c => InsertBackendTransitions(c.outputsColumnar)) - ) - val outputPlan = rule.apply(originalPlan, false) - // Expect to fall back the entire plan. - assert(outputPlan == originalPlan) - } - } - } - - testGluten( - "Don't Fall back the whole plan if NOT meeting the configured threshold (" + - "leaf node is transformable)") { - withSQLConf((GlutenConfig.COLUMNAR_WHOLESTAGE_FALLBACK_THRESHOLD.key, "3")) { - CallerInfo.withLocalValue(isAqe = true, isCache = false) { - val originalPlan = UnaryOp2(UnaryOp1(UnaryOp2(UnaryOp1(LeafOp())))) - val rule = newRuleApplier( - spark, - List( - _ => - _ => { - UnaryOp2(UnaryOp1Transformer(UnaryOp2(UnaryOp1Transformer(LeafOpTransformer())))) - }, - c => InsertBackendTransitions(c.outputsColumnar)) - ) - val outputPlan = rule.apply(originalPlan, false) - // Expect to get the plan with columnar rule applied. - assert(outputPlan != originalPlan) - } - } - } - - testGluten("Tag not transformable more than once") { - val originalPlan = UnaryOp1(LeafOp(supportsColumnar = true)) - FallbackTags.add(originalPlan, "fake reason") - val rule = FallbackEmptySchemaRelation() - val newPlan = rule.apply(originalPlan) - val reason = FallbackTags.get(newPlan).reason() - assert( - reason.contains("fake reason") && - reason.contains("at least one of its children has empty output")) - } - - testGluten("test enabling/disabling Gluten at thread level") { - spark.sql("create table fallback_by_thread_config (a int) using parquet") - spark.sql("insert overwrite fallback_by_thread_config select id as a from range(3)") - val sql = - """ - |select * - |from fallback_by_thread_config as t0 - |""".stripMargin - - val noFallbackPlan = spark.sql(sql).queryExecution.executedPlan - val noFallbackScanExec = noFallbackPlan.collect { case _: BasicScanExecTransformer => true } - assert(noFallbackScanExec.size == 1) - - val thread = new Thread( - () => { - spark.sparkContext - .setLocalProperty(GlutenSessionExtensions.GLUTEN_ENABLE_FOR_THREAD_KEY, "false") - val fallbackPlan = spark.sql(sql).queryExecution.executedPlan - val fallbackScanExec = fallbackPlan.collect { - case e: FileSourceScanExec if !e.isInstanceOf[BasicScanExecTransformer] => true - } - assert(fallbackScanExec.size == 1) - - spark.sparkContext - .setLocalProperty(GlutenSessionExtensions.GLUTEN_ENABLE_FOR_THREAD_KEY, null) - val noFallbackPlan = spark.sql(sql).queryExecution.executedPlan - val noFallbackScanExec = noFallbackPlan.collect { case _: BasicScanExecTransformer => true } - assert(noFallbackScanExec.size == 1) - }) - thread.start() - thread.join(10000) - } -} - -private object FallbackStrategiesSuite { - def newRuleApplier( - spark: SparkSession, - transformBuilders: Seq[ColumnarRuleCall => Rule[SparkPlan]]): HeuristicApplier = { - new HeuristicApplier( - spark, - Nil, - transformBuilders, - List(c => p => ExpandFallbackPolicy(c.caller.isAqe(), p)), - List( - c => RemoveTopmostColumnarToRow(c.session, c.caller.isAqe()), - _ => ColumnarCollapseTransformStages(GlutenConfig.get) - ), - List(_ => RemoveFallbackTagRule()), - Nil - ) - } - - // TODO: Generalize the code among shim versions. - case class FallbackEmptySchemaRelation() extends Rule[SparkPlan] { - override def apply(plan: SparkPlan): SparkPlan = plan.transformDown { - case p => - if (p.children.exists(_.output.isEmpty)) { - // Some backends are not eligible to offload plan with zero-column input. - // If any child have empty output, mark the plan and that child as UNSUPPORTED. - FallbackTags.add(p, "at least one of its children has empty output") - p.children.foreach { - child => - if (child.output.isEmpty) { - FallbackTags.add(child, "at least one of its children has empty output") - } - } - } - p - } - } - - case class LeafOp(override val supportsColumnar: Boolean = false) extends LeafExecNode { - override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() - override def output: Seq[Attribute] = Seq.empty - } - - case class UnaryOp1(child: SparkPlan, override val supportsColumnar: Boolean = false) - extends UnaryExecNode { - override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() - override def output: Seq[Attribute] = child.output - override protected def withNewChildInternal(newChild: SparkPlan): UnaryOp1 = - copy(child = newChild) - } - - case class UnaryOp2(child: SparkPlan, override val supportsColumnar: Boolean = false) - extends UnaryExecNode { - override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() - override def output: Seq[Attribute] = child.output - override protected def withNewChildInternal(newChild: SparkPlan): UnaryOp2 = - copy(child = newChild) - } - - // For replacing LeafOp. - case class LeafOpTransformer() extends LeafExecNode with GlutenPlan { - override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType - override def rowType(): Convention.RowType = Convention.RowType.None - override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() - override def output: Seq[Attribute] = Seq.empty - } - - // For replacing UnaryOp1. - case class UnaryOp1Transformer(override val child: SparkPlan) - extends UnaryExecNode - with GlutenPlan { - override def batchType(): Convention.BatchType = BackendsApiManager.getSettings.primaryBatchType - override def rowType(): Convention.RowType = Convention.RowType.None - override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() - override def output: Seq[Attribute] = child.output - override protected def withNewChildInternal(newChild: SparkPlan): UnaryOp1Transformer = - copy(child = newChild) - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenBroadcastExchangeSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenBroadcastExchangeSuite.scala deleted file mode 100644 index 48186335422..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenBroadcastExchangeSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenBroadcastExchangeSuite extends BroadcastExchangeSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenCoalesceShufflePartitionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenCoalesceShufflePartitionsSuite.scala deleted file mode 100644 index cea110bbad1..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenCoalesceShufflePartitionsSuite.scala +++ /dev/null @@ -1,294 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution - -import org.apache.spark.SparkConf -import org.apache.spark.internal.config.IO_ENCRYPTION_ENABLED -import org.apache.spark.internal.config.UI.UI_ENABLED -import org.apache.spark.sql.{GlutenTestsCommonTrait, QueryTest, SparkSession} -import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec -import org.apache.spark.sql.functions.col -import org.apache.spark.sql.internal.SQLConf - -class GlutenCoalesceShufflePartitionsSuite - extends CoalesceShufflePartitionsSuite - with GlutenTestsCommonTrait { - - override protected def afterAll(): Unit = {} - - override def withSparkSession( - f: SparkSession => Unit, - targetPostShuffleInputSize: Int, - minNumPostShufflePartitions: Option[Int], - enableIOEncryption: Boolean = false): Unit = { - val sparkConf = - new SparkConf(false) - .setMaster("local[*]") - .setAppName("test") - .set(UI_ENABLED, false) - .set(IO_ENCRYPTION_ENABLED, enableIOEncryption) - .set(SQLConf.SHUFFLE_PARTITIONS.key, "5") - .set(SQLConf.COALESCE_PARTITIONS_INITIAL_PARTITION_NUM.key, "5") - .set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "true") - .set(SQLConf.FETCH_SHUFFLE_BLOCKS_IN_BATCH.key, "true") - .set(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key, "-1") - .set(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key, targetPostShuffleInputSize.toString) - .set(SQLConf.COALESCE_PARTITIONS_ENABLED.key, "true") - // Gluten config - .set("spark.plugins", "org.apache.gluten.GlutenPlugin") - .set("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager") - .set("spark.memory.offHeap.enabled", "true") - .set("spark.memory.offHeap.size", "5g") - minNumPostShufflePartitions match { - case Some(numPartitions) => - sparkConf.set(SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM.key, numPartitions.toString) - case None => - sparkConf.set(SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM.key, "1") - } - - val spark = SparkSession - .builder() - .config(sparkConf) - .getOrCreate() - try f(spark) - finally { - spark.stop() - } - } - - Seq(Some(5), None).foreach { - minNumPostShufflePartitions => - val testNameNote = minNumPostShufflePartitions match { - case Some(numPartitions) => "(minNumPostShufflePartitions: " + numPartitions + ")" - case None => "" - } - - // Ported from vanilla spark with targetPostShuffleInputSize changed. - testGluten(s"determining the number of reducers: aggregate operator$testNameNote") { - val test: SparkSession => Unit = { - spark: SparkSession => - val df = - spark - .range(0, 1000, 1, numInputPartitions) - .selectExpr("id % 20 as key", "id as value") - val agg = df.groupBy("key").count() - - // Check the answer first. - QueryTest.checkAnswer(agg, spark.range(0, 20).selectExpr("id", "50 as cnt").collect()) - - // Then, let's look at the number of post-shuffle partitions estimated - // by the ExchangeCoordinator. - val finalPlan = agg.queryExecution.executedPlan - .asInstanceOf[AdaptiveSparkPlanExec] - .executedPlan - val shuffleReads = finalPlan.collect { case r @ CoalescedShuffleRead() => r } - - minNumPostShufflePartitions match { - case Some(numPartitions) => - assert(shuffleReads.isEmpty) - case None => - assert(shuffleReads.length === 1) - shuffleReads.foreach(read => assert(read.outputPartitioning.numPartitions === 3)) - } - } - // Change the original value 2000 to 2500 for gluten. The test depends on the calculation - // for bytesByPartitionId in MapOutputStatistics. Gluten has a different statistic result. - // See ShufflePartitionsUtil.coalescePartitions & GlutenColumnarShuffleWriter's mapStatus. - withSparkSession(test, 2500, minNumPostShufflePartitions) - } - - testGluten(s"determining the number of reducers: join operator$testNameNote") { - val test: SparkSession => Unit = { - spark: SparkSession => - val df1 = - spark - .range(0, 1000, 1, numInputPartitions) - .selectExpr("id % 500 as key1", "id as value1") - val df2 = - spark - .range(0, 1000, 1, numInputPartitions) - .selectExpr("id % 500 as key2", "id as value2") - - val join = df1.join(df2, col("key1") === col("key2")).select(col("key1"), col("value2")) - - // Check the answer first. - val expectedAnswer = - spark - .range(0, 1000) - .selectExpr("id % 500 as key", "id as value") - .union(spark.range(0, 1000).selectExpr("id % 500 as key", "id as value")) - QueryTest.checkAnswer(join, expectedAnswer.collect()) - - // Then, let's look at the number of post-shuffle partitions estimated - // by the ExchangeCoordinator. - val finalPlan = join.queryExecution.executedPlan - .asInstanceOf[AdaptiveSparkPlanExec] - .executedPlan - val shuffleReads = finalPlan.collect { case r @ CoalescedShuffleRead() => r } - - minNumPostShufflePartitions match { - case Some(numPartitions) => - assert(shuffleReads.isEmpty) - - case None => - assert(shuffleReads.length === 2) - shuffleReads.foreach(read => assert(read.outputPartitioning.numPartitions === 2)) - } - } - // Change the original value 16384 to 20000 for gluten. The test depends on the calculation - // for bytesByPartitionId in MapOutputStatistics. Gluten has a different statistic result. - // See ShufflePartitionsUtil.coalescePartitions & GlutenColumnarShuffleWriter's mapStatus. - withSparkSession(test, 20000, minNumPostShufflePartitions) - } - - testGluten(s"determining the number of reducers: complex query 1$testNameNote") { - val test: (SparkSession) => Unit = { - spark: SparkSession => - val df1 = - spark - .range(0, 1000, 1, numInputPartitions) - .selectExpr("id % 500 as key1", "id as value1") - .groupBy("key1") - .count() - .toDF("key1", "cnt1") - val df2 = - spark - .range(0, 1000, 1, numInputPartitions) - .selectExpr("id % 500 as key2", "id as value2") - .groupBy("key2") - .count() - .toDF("key2", "cnt2") - - val join = df1.join(df2, col("key1") === col("key2")).select(col("key1"), col("cnt2")) - - // Check the answer first. - val expectedAnswer = - spark - .range(0, 500) - .selectExpr("id", "2 as cnt") - QueryTest.checkAnswer(join, expectedAnswer.collect()) - - // Then, let's look at the number of post-shuffle partitions estimated - // by the ExchangeCoordinator. - val finalPlan = join.queryExecution.executedPlan - .asInstanceOf[AdaptiveSparkPlanExec] - .executedPlan - val shuffleReads = finalPlan.collect { case r @ CoalescedShuffleRead() => r } - - minNumPostShufflePartitions match { - case Some(numPartitions) => - assert(shuffleReads.isEmpty) - - case None => - assert(shuffleReads.length === 2) - shuffleReads.foreach(read => assert(read.outputPartitioning.numPartitions === 2)) - } - } - - // Change the original value 16384 to 20000 for gluten. The test depends on the calculation - // for bytesByPartitionId in MapOutputStatistics. Gluten has a different statistic result. - // See ShufflePartitionsUtil.coalescePartitions & GlutenColumnarShuffleWriter's mapStatus. - withSparkSession(test, 20000, minNumPostShufflePartitions) - } - - testGluten(s"determining the number of reducers: complex query 2$testNameNote") { - val test: (SparkSession) => Unit = { - spark: SparkSession => - val df1 = - spark - .range(0, 1000, 1, numInputPartitions) - .selectExpr("id % 500 as key1", "id as value1") - .groupBy("key1") - .count() - .toDF("key1", "cnt1") - val df2 = - spark - .range(0, 1000, 1, numInputPartitions) - .selectExpr("id % 500 as key2", "id as value2") - - val join = - df1 - .join(df2, col("key1") === col("key2")) - .select(col("key1"), col("cnt1"), col("value2")) - - // Check the answer first. - val expectedAnswer = - spark - .range(0, 1000) - .selectExpr("id % 500 as key", "2 as cnt", "id as value") - QueryTest.checkAnswer(join, expectedAnswer.collect()) - - // Then, let's look at the number of post-shuffle partitions estimated - // by the ExchangeCoordinator. - val finalPlan = join.queryExecution.executedPlan - .asInstanceOf[AdaptiveSparkPlanExec] - .executedPlan - val shuffleReads = finalPlan.collect { case r @ CoalescedShuffleRead() => r } - - minNumPostShufflePartitions match { - case Some(numPartitions) => - assert(shuffleReads.isEmpty) - - case None => - assert(shuffleReads.length === 2) - shuffleReads.foreach(read => assert(read.outputPartitioning.numPartitions === 3)) - } - } - - // Change the original value 12000 to 16000 for gluten. The test depends on the calculation - // for bytesByPartitionId in MapOutputStatistics. Gluten has a different statistic result. - // See ShufflePartitionsUtil.coalescePartitions & GlutenColumnarShuffleWriter's mapStatus. - withSparkSession(test, 16000, minNumPostShufflePartitions) - } - - testGluten( - "determining the number of reducers:" + - s" plan already partitioned$testNameNote") { - val test: SparkSession => Unit = { - spark: SparkSession => - try { - spark.range(1000).write.bucketBy(30, "id").saveAsTable("t") - // `df1` is hash partitioned by `id`. - val df1 = spark.read.table("t") - val df2 = - spark - .range(0, 1000, 1, numInputPartitions) - .selectExpr("id % 500 as key2", "id as value2") - - val join = df1.join(df2, col("id") === col("key2")).select(col("id"), col("value2")) - - // Check the answer first. - val expectedAnswer = spark - .range(0, 500) - .selectExpr("id % 500", "id as value") - .union(spark.range(500, 1000).selectExpr("id % 500", "id as value")) - QueryTest.checkAnswer(join, expectedAnswer.collect()) - - // Then, let's make sure we do not reduce number of post shuffle partitions. - val finalPlan = join.queryExecution.executedPlan - .asInstanceOf[AdaptiveSparkPlanExec] - .executedPlan - val shuffleReads = finalPlan.collect { case r @ CoalescedShuffleRead() => r } - assert(shuffleReads.length === 0) - } finally { - spark.sql("drop table t") - } - } - withSparkSession(test, 12000, minNumPostShufflePartitions) - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenExchangeSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenExchangeSuite.scala deleted file mode 100644 index bc15153cca0..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenExchangeSuite.scala +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait -import org.apache.spark.sql.execution.exchange.{Exchange, ReusedExchangeExec} -import org.apache.spark.sql.internal.SQLConf - -class GlutenExchangeSuite extends ExchangeSuite with GlutenSQLTestsBaseTrait { - - testGluten("Exchange reuse across the whole plan with shuffle partition 2") { - // The shuffle exchange will be inserted between Aggregate - // when shuffle partition is > 1. - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", - SQLConf.SHUFFLE_PARTITIONS.key -> "2") { - val df = sql(""" - |SELECT - | (SELECT max(a.key) FROM testData AS a JOIN testData AS b ON b.key = a.key), - | a.key - |FROM testData AS a - |JOIN testData AS b ON b.key = a.key - """.stripMargin) - - val plan = df.queryExecution.executedPlan - - val exchangeIds = plan.collectWithSubqueries { case e: Exchange => e.id } - val reusedExchangeIds = plan.collectWithSubqueries { - case re: ReusedExchangeExec => re.child.id - } - - assert(exchangeIds.size == 2, "Whole plan exchange reusing not working correctly") - assert(reusedExchangeIds.size == 3, "Whole plan exchange reusing not working correctly") - assert( - reusedExchangeIds.forall(exchangeIds.contains(_)), - "ReusedExchangeExec should reuse an existing exchange") - - val df2 = sql(""" - |SELECT - | (SELECT min(a.key) FROM testData AS a JOIN testData AS b ON b.key = a.key), - | (SELECT max(a.key) FROM testData AS a JOIN testData2 AS b ON b.a = a.key) - """.stripMargin) - - val plan2 = df2.queryExecution.executedPlan - - val exchangeIds2 = plan2.collectWithSubqueries { case e: Exchange => e.id } - val reusedExchangeIds2 = plan2.collectWithSubqueries { - case re: ReusedExchangeExec => re.child.id - } - - assert(exchangeIds2.size == 4, "Whole plan exchange reusing not working correctly") - assert(reusedExchangeIds2.size == 2, "Whole plan exchange reusing not working correctly") - assert( - reusedExchangeIds2.forall(exchangeIds2.contains(_)), - "ReusedExchangeExec should reuse an existing exchange") - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenQueryExecutionSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenQueryExecutionSuite.scala deleted file mode 100644 index 005386634b9..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenQueryExecutionSuite.scala +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.util.Utils - -import org.apache.logging.log4j.{Level, LogManager} -import org.apache.logging.log4j.core.LoggerContext - -import scala.io.Source - -class GlutenQueryExecutionSuite extends QueryExecutionSuite with GlutenSQLTestsBaseTrait { - - override def checkDumpedPlans(path: String, expected: Int): Unit = - Utils.tryWithResource(Source.fromFile(path)) { - source => - assert( - source.getLines.toList - .takeWhile(_ != "== Whole Stage Codegen ==") - .map(_.replaceAll("#\\d+", "#x")) == List( - "== Parsed Logical Plan ==", - s"Range (0, $expected, step=1, splits=Some(2))", - "", - "== Analyzed Logical Plan ==", - "id: bigint", - s"Range (0, $expected, step=1, splits=Some(2))", - "", - "== Optimized Logical Plan ==", - s"Range (0, $expected, step=1, splits=Some(2))", - "", - "== Physical Plan ==", - "*(1) ColumnarToRow", - s"+- ColumnarRange 0, $expected, 1, 2, $expected, [id#xL]", - "" - )) - } - - testGluten("dumping query execution info to a file - explainMode=formatted") { - withTempDir { - dir => - val path = dir.getCanonicalPath + "/plans.txt" - val df = spark.range(0, 10) - df.queryExecution.debug.toFile(path, explainMode = Option("formatted")) - val lines = Utils.tryWithResource(Source.fromFile(path))(_.getLines().toList) - assert( - lines - .takeWhile(_ != "== Whole Stage Codegen ==") - .map(_.replaceAll("#\\d+", "#x")) == List( - "== Physical Plan ==", - "* ColumnarToRow (2)", - "+- ColumnarRange (1)", - "", - "", - "(1) ColumnarRange", - "Output [1]: [id#xL]", - "Arguments: Range (0, 10, step=1, splits=Some(2))", - "", - "(2) ColumnarToRow [codegen id : 1]", - "Input [1]: [id#xL]", - "", - "" - )) - } - } - - testGluten("Logging plan changes for execution") { - val ctx = LogManager.getContext(false).asInstanceOf[LoggerContext] - val config = ctx.getConfiguration - val loggerConfig = config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME) - loggerConfig.setLevel(Level.INFO) - ctx.updateLoggers() - - val testAppender = new LogAppender("plan changes") - withLogAppender(testAppender) { - withSQLConf( - SQLConf.PLAN_CHANGE_LOG_LEVEL.key -> "INFO" - ) { - spark.range(1).groupBy("id").count().queryExecution.executedPlan - } - } - Seq("=== Applying Rule org.apache.spark.sql.execution", "=== Result of Batch Preparations ===") - .foreach { - expectedMsg => - assert( - testAppender.loggingEvents.exists( - _.getMessage.getFormattedMessage.contains(expectedMsg) - ) - ) - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenReplaceHashWithSortAggSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenReplaceHashWithSortAggSuite.scala deleted file mode 100644 index 33bf1a1ec97..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenReplaceHashWithSortAggSuite.scala +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution - -import org.apache.gluten.execution.HashAggregateExecBaseTransformer - -import org.apache.spark.sql.{DataFrame, GlutenSQLTestsBaseTrait} -import org.apache.spark.sql.execution.aggregate.{ObjectHashAggregateExec, SortAggregateExec} -import org.apache.spark.sql.internal.SQLConf - -class GlutenReplaceHashWithSortAggSuite - extends ReplaceHashWithSortAggSuite - with GlutenSQLTestsBaseTrait { - - private def checkNumAggs(df: DataFrame, hashAggCount: Int, sortAggCount: Int): Unit = { - val plan = df.queryExecution.executedPlan - assert(collectWithSubqueries(plan) { - case s @ (_: HashAggregateExecBaseTransformer | _: ObjectHashAggregateExec) => s - }.length == hashAggCount) - assert(collectWithSubqueries(plan) { case s: SortAggregateExec => s }.length == sortAggCount) - } - - private def checkAggs( - query: String, - enabledHashAggCount: Int, - enabledSortAggCount: Int, - disabledHashAggCount: Int, - disabledSortAggCount: Int): Unit = { - withSQLConf(SQLConf.REPLACE_HASH_WITH_SORT_AGG_ENABLED.key -> "true") { - val df = sql(query) - checkNumAggs(df, enabledHashAggCount, enabledSortAggCount) - val result = df.collect() - withSQLConf(SQLConf.REPLACE_HASH_WITH_SORT_AGG_ENABLED.key -> "false") { - val df = sql(query) - checkNumAggs(df, disabledHashAggCount, disabledSortAggCount) - checkAnswer(df, result) - } - } - } - - // === Following cases override super class's cases === - - testGluten("replace partial hash aggregate with sort aggregate") { - withTempView("t") { - spark.range(100).selectExpr("id as key").repartition(10).createOrReplaceTempView("t") - - Seq("FIRST", "COLLECT_LIST").foreach { - aggExpr => - // Because repartition modification causing the result sort order not same and the - // result not same, so we add order by key before comparing the result. - val query = - s""" - |SELECT key, $aggExpr(key) - |FROM - |( - | SELECT key - | FROM t - | WHERE key > 10 - | SORT BY key - |) - |GROUP BY key - |ORDER BY key - """.stripMargin - checkAggs(query, 2, 0, 2, 0) - } - } - } - - testGluten("replace partial and final hash aggregate together with sort aggregate") { - withTempView("t1", "t2") { - spark.range(100).selectExpr("id as key").createOrReplaceTempView("t1") - spark.range(50).selectExpr("id as key").createOrReplaceTempView("t2") - Seq(("COUNT", 1, 0, 1, 0), ("COLLECT_LIST", 1, 0, 1, 0)).foreach { - aggExprInfo => - val query = - s""" - |SELECT key, ${aggExprInfo._1}(key) - |FROM - |( - | SELECT /*+ SHUFFLE_MERGE(t1) */ t1.key AS key - | FROM t1 - | JOIN t2 - | ON t1.key = t2.key - |) - |GROUP BY key - """.stripMargin - checkAggs(query, aggExprInfo._2, aggExprInfo._3, aggExprInfo._4, aggExprInfo._5) - } - } - } - - testGluten("do not replace hash aggregate if child does not have sort order") { - withTempView("t1", "t2") { - spark.range(100).selectExpr("id as key").createOrReplaceTempView("t1") - spark.range(50).selectExpr("id as key").createOrReplaceTempView("t2") - Seq("COUNT", "COLLECT_LIST").foreach { - aggExpr => - val query = - s""" - |SELECT key, $aggExpr(key) - |FROM - |( - | SELECT /*+ BROADCAST(t1) */ t1.key AS key - | FROM t1 - | JOIN t2 - | ON t1.key = t2.key - |) - |GROUP BY key - """.stripMargin - checkAggs(query, 2, 0, 2, 0) - } - } - } - - testGluten("do not replace hash aggregate if there is no group-by column") { - withTempView("t1") { - spark.range(100).selectExpr("id as key").createOrReplaceTempView("t1") - Seq("COUNT", "COLLECT_LIST").foreach { - aggExpr => - val query = - s""" - |SELECT $aggExpr(key) - |FROM t1 - """.stripMargin - checkAggs(query, 2, 0, 2, 0) - } - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenReuseExchangeAndSubquerySuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenReuseExchangeAndSubquerySuite.scala deleted file mode 100644 index d7232f6a06c..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenReuseExchangeAndSubquerySuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenReuseExchangeAndSubquerySuite - extends ReuseExchangeAndSubquerySuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenSQLAggregateFunctionSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenSQLAggregateFunctionSuite.scala deleted file mode 100644 index 0dd56ae1574..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenSQLAggregateFunctionSuite.scala +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution - -import org.apache.gluten.execution.HashAggregateExecBaseTransformer - -import org.apache.spark.sql.{GlutenSQLTestsTrait, Row} -import org.apache.spark.sql.internal.SQLConf - -class GlutenSQLAggregateFunctionSuite extends GlutenSQLTestsTrait { - - testGluten("GLUTEN-4853: The result order is reversed for count and count distinct") { - val query = - """ - |select count(distinct if(sex = 'x', id, null)) as uv, count(if(sex = 'x', id, null)) as pv - |from values (1, 'x'), (1, 'x'), (2, 'y'), (3, 'x'), (3, 'x'), (4, 'y'), (5, 'x') - |AS tab(id, sex) - |""".stripMargin - val df = sql(query) - checkAnswer(df, Seq(Row(3, 5))) - assert(getExecutedPlan(df).count(_.isInstanceOf[HashAggregateExecBaseTransformer]) == 4) - } - - testGluten("Return NaN or null when dividing by zero") { - val query = - """ - |select skewness(value), kurtosis(value) - |from values (1), (1) - |AS tab(value) - |""".stripMargin - val df = sql(query) - - withSQLConf( - SQLConf.LEGACY_STATISTICAL_AGGREGATE.key -> "true" - ) { - checkAnswer(df, Seq(Row(Double.NaN, Double.NaN))) - assert(getExecutedPlan(df).count(_.isInstanceOf[HashAggregateExecBaseTransformer]) == 2) - } - - withSQLConf( - SQLConf.LEGACY_STATISTICAL_AGGREGATE.key -> - SQLConf.LEGACY_STATISTICAL_AGGREGATE.defaultValueString - ) { - checkAnswer(df, Seq(Row(null, null))) - assert(getExecutedPlan(df).count(_.isInstanceOf[HashAggregateExecBaseTransformer]) == 2) - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenSQLCollectLimitExecSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenSQLCollectLimitExecSuite.scala deleted file mode 100644 index 069dea32f48..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenSQLCollectLimitExecSuite.scala +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution - -import org.apache.gluten.execution.ColumnarCollectLimitBaseExec - -import org.apache.spark.SparkConf -import org.apache.spark.sql.{DataFrame, GlutenSQLTestsTrait, Row} - -class GlutenSQLCollectLimitExecSuite extends GlutenSQLTestsTrait { - - override def sparkConf: SparkConf = { - super.sparkConf - .set("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager") - } - - private def assertGlutenOperatorMatch[T: reflect.ClassTag]( - df: DataFrame, - checkMatch: Boolean): Unit = { - val executedPlan = getExecutedPlan(df) - - val operatorFound = executedPlan.exists { - plan => - try { - implicitly[reflect.ClassTag[T]].runtimeClass.isInstance(plan) - } catch { - case _: Throwable => false - } - } - - val assertionCondition = operatorFound == checkMatch - val assertionMessage = - if (checkMatch) { - s"Operator ${implicitly[reflect.ClassTag[T]].runtimeClass.getSimpleName} not found " + - s"in executed plan:\n $executedPlan" - } else { - s"Operator ${implicitly[reflect.ClassTag[T]].runtimeClass.getSimpleName} was found " + - s"in executed plan:\n $executedPlan" - } - - assert(assertionCondition, assertionMessage) - } - - test("ColumnarCollectLimitExec - basic limit test") { - val df = spark.range(0, 1000, 1).toDF("id").limit(5) - val expectedData = Seq(Row(0L), Row(1L), Row(2L), Row(3L), Row(4L)) - - checkAnswer(df, expectedData) - - assertGlutenOperatorMatch[ColumnarCollectLimitBaseExec](df, checkMatch = true) - } - - test("ColumnarCollectLimitExec - with filter") { - val df = spark - .range(0, 20, 1) - .toDF("id") - .filter("id % 2 == 0") - .limit(5) - val expectedData = Seq(Row(0L), Row(2L), Row(4L), Row(6L), Row(8L)) - - checkAnswer(df, expectedData) - - assertGlutenOperatorMatch[ColumnarCollectLimitBaseExec](df, checkMatch = true) - } - - test("ColumnarCollectLimitExec - range with repartition") { - - val df = spark - .range(0, 10, 1) - .toDF("id") - .repartition(3) - .orderBy("id") - .limit(3) - val expectedData = Seq(Row(0L), Row(1L), Row(2L)) - - checkAnswer(df, expectedData) - } - - test("ColumnarCollectLimitExec - with distinct values") { - val df = spark - .range(0, 10, 1) - .toDF("id") - .select("id") - .distinct() - .limit(5) - val expectedData = Seq(Row(0L), Row(1L), Row(2L), Row(3L), Row(4L)) - - checkAnswer(df, expectedData) - - assertGlutenOperatorMatch[ColumnarCollectLimitBaseExec](df, checkMatch = true) - } - - test("ColumnarCollectLimitExec - chained limit") { - val df = spark - .range(0, 10, 1) - .toDF("id") - .limit(8) - .limit(3) - val expectedData = Seq(Row(0L), Row(1L), Row(2L)) - - checkAnswer(df, expectedData) - - assertGlutenOperatorMatch[ColumnarCollectLimitBaseExec](df, checkMatch = true) - } - - test("ColumnarCollectLimitExec - limit after union") { - val df1 = spark.range(0, 5).toDF("id") - val df2 = spark.range(5, 10).toDF("id") - val unionDf = df1.union(df2).limit(3) - - val expectedData = Seq(Row(0L), Row(1L), Row(2L)) - - checkAnswer(unionDf, expectedData) - - assertGlutenOperatorMatch[ColumnarCollectLimitBaseExec](unionDf, checkMatch = true) - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenSQLWindowFunctionSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenSQLWindowFunctionSuite.scala deleted file mode 100644 index 6665174207b..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenSQLWindowFunctionSuite.scala +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution - -import org.apache.gluten.execution.WindowExecTransformer - -import org.apache.spark.sql.GlutenSQLTestsTrait -import org.apache.spark.sql.Row -import org.apache.spark.sql.types._ - -class GlutenSQLWindowFunctionSuite extends SQLWindowFunctionSuite with GlutenSQLTestsTrait { - - private def decimal(v: BigDecimal): Decimal = Decimal(v, 7, 2) - - val customerSchema = StructType( - List( - StructField("c_custkey", IntegerType), - StructField("c_nationkey", IntegerType), - StructField("c_acctbal", DecimalType(7, 2)) - ) - ) - - val customerData = Seq( - Row(4553, 11, decimal(6388.41)), - Row(4953, 10, decimal(6037.28)), - Row(35403, 5, decimal(6034.70)), - Row(35803, 12, decimal(5284.87)), - Row(60865, 5, decimal(-227.82)), - Row(61065, 13, decimal(7284.77)), - Row(127412, 13, decimal(4621.41)), - Row(148303, 10, decimal(4302.30)), - Row(9954, 5, decimal(7587.25)), - Row(95337, 12, decimal(915.61)) - ) - - testGluten("Literal in window partition by and sort") { - withTable("customer") { - val rdd = spark.sparkContext.parallelize(customerData) - val customerDF = spark.createDataFrame(rdd, customerSchema) - customerDF.createOrReplaceTempView("customer") - val query = - """ - |SELECT - | c_custkey, - | c_acctbal, - | row_number() OVER ( - | PARTITION BY c_nationkey, - | "a" - | ORDER BY - | c_custkey, - | "a" - | ) AS row_num - |FROM - | customer - |ORDER BY 1, 2; - |""".stripMargin - val df = sql(query) - checkAnswer( - df, - Seq( - Row(4553, BigDecimal(638841L, 2), 1), - Row(4953, BigDecimal(603728L, 2), 1), - Row(9954, BigDecimal(758725L, 2), 1), - Row(35403, BigDecimal(603470L, 2), 2), - Row(35803, BigDecimal(528487L, 2), 1), - Row(60865, BigDecimal(-22782L, 2), 3), - Row(61065, BigDecimal(728477L, 2), 1), - Row(95337, BigDecimal(91561L, 2), 2), - Row(127412, BigDecimal(462141L, 2), 2), - Row(148303, BigDecimal(430230L, 2), 2) - ) - ) - assert( - getExecutedPlan(df).exists { - case _: WindowExecTransformer => true - case _ => false - } - ) - } - } - - testGluten("Expression in WindowExpression") { - withTable("customer") { - val rdd = spark.sparkContext.parallelize(customerData) - val customerDF = spark.createDataFrame(rdd, customerSchema) - customerDF.createOrReplaceTempView("customer") - val query = - """ - |SELECT - | c_custkey, - | avg(c_acctbal) OVER ( - | PARTITION BY c_nationkey - | ORDER BY c_custkey - | ) - |FROM - | customer - |ORDER BY 1, 2; - |""".stripMargin - val df = sql(query) - checkAnswer( - df, - Seq( - Row(4553, BigDecimal(6388410000L, 6)), - Row(4953, BigDecimal(6037280000L, 6)), - Row(9954, BigDecimal(7587250000L, 6)), - Row(35403, BigDecimal(6810975000L, 6)), - Row(35803, BigDecimal(5284870000L, 6)), - Row(60865, BigDecimal(4464710000L, 6)), - Row(61065, BigDecimal(7284770000L, 6)), - Row(95337, BigDecimal(3100240000L, 6)), - Row(127412, BigDecimal(5953090000L, 6)), - Row(148303, BigDecimal(5169790000L, 6)) - ) - ) - assert( - getExecutedPlan(df).exists { - case _: WindowExecTransformer => true - case _ => false - } - ) - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenSameResultSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenSameResultSuite.scala deleted file mode 100644 index de9a897ffb0..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenSameResultSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenSameResultSuite extends SameResultSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenSortSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenSortSuite.scala deleted file mode 100644 index 2bfb3616bd5..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenSortSuite.scala +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution - -import org.apache.gluten.execution.SortExecTransformer - -import org.apache.spark.sql.{catalyst, GlutenQueryTestUtil, GlutenSQLTestsBaseTrait, Row} -import org.apache.spark.sql.catalyst.analysis.{Resolver, UnresolvedAttribute} -import org.apache.spark.sql.catalyst.expressions.{Length, SortOrder} -import org.apache.spark.sql.catalyst.plans.QueryPlan -import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper -import org.apache.spark.sql.functions.length - -class GlutenSortSuite extends SortSuite with GlutenSQLTestsBaseTrait with AdaptiveSparkPlanHelper { - import testImplicits._ - - protected val resolver: Resolver = conf.resolver - - protected def attr(name: String): UnresolvedAttribute = { - UnresolvedAttribute(name) - } - - protected def resolveAttrs[T <: QueryPlan[T]]( - expr: catalyst.expressions.Expression, - plan: QueryPlan[T]): catalyst.expressions.Expression = { - - expr.transform { - case UnresolvedAttribute(Seq(attrName)) => - plan.output.find(attr => resolver(attr.name, attrName)).get - case UnresolvedAttribute(nameParts) => - val attrName = nameParts.mkString(".") - fail(s"cannot resolve a nested attr: $attrName") - } - } - - testGluten("post-project outputOrdering check") { - val input = Seq( - ("Hello", 4, 2.0), - ("Hello Bob", 10, 1.0), - ("Hello Bob", 1, 3.0) - ) - - val df = input.toDF("a", "b", "c").orderBy(length($"a").desc, $"b".desc) - GlutenQueryTestUtil.checkAnswer( - df, - Seq( - Row("Hello Bob", 10, 1.0), - Row("Hello Bob", 1, 3.0), - Row("Hello", 4, 2.0) - ) - ) - - val ordering = Seq( - catalyst.expressions.SortOrder( - Length(attr("a")), - catalyst.expressions.Descending, - catalyst.expressions.NullsLast, - Seq.empty - ), - catalyst.expressions.SortOrder( - attr("b"), - catalyst.expressions.Descending, - catalyst.expressions.NullsLast, - Seq.empty - ) - ) - - assert( - getExecutedPlan(df).exists { - case _: SortExecTransformer => true - case _ => false - } - ) - val plan = stripAQEPlan(df.queryExecution.executedPlan) - val actualOrdering = plan.outputOrdering - val expectedOrdering = ordering.map(resolveAttrs(_, plan).asInstanceOf[SortOrder]) - assert(actualOrdering.length == expectedOrdering.length) - actualOrdering.zip(expectedOrdering).foreach { - case (actual, expected) => - assert(actual.satisfies(expected), "ordering must satisfy") - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenTakeOrderedAndProjectSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenTakeOrderedAndProjectSuite.scala deleted file mode 100644 index bc231e52adc..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenTakeOrderedAndProjectSuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenTakeOrderedAndProjectSuite - extends TakeOrderedAndProjectSuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/adaptive/clickhouse/ClickHouseAdaptiveQueryExecSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/adaptive/clickhouse/ClickHouseAdaptiveQueryExecSuite.scala deleted file mode 100644 index 3ef4d574e01..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/adaptive/clickhouse/ClickHouseAdaptiveQueryExecSuite.scala +++ /dev/null @@ -1,1666 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.adaptive.clickhouse - -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.execution.{BroadcastHashJoinExecTransformerBase, ColumnarToCarrierRowExecBase, ShuffledHashJoinExecTransformerBase, SortExecTransformer, SortMergeJoinExecTransformerBase} - -import org.apache.spark.SparkConf -import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent} -import org.apache.spark.sql.{Dataset, GlutenSQLTestsTrait, Row} -import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight} -import org.apache.spark.sql.execution._ -import org.apache.spark.sql.execution.adaptive._ -import org.apache.spark.sql.execution.command.DataWritingCommandExec -import org.apache.spark.sql.execution.datasources.noop.NoopDataSource -import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec -import org.apache.spark.sql.execution.exchange._ -import org.apache.spark.sql.execution.joins.{BaseJoinExec, BroadcastHashJoinExec, ShuffledHashJoinExec, SortMergeJoinExec} -import org.apache.spark.sql.execution.metric.SQLShuffleReadMetricsReporter -import org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate -import org.apache.spark.sql.functions.when -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.internal.SQLConf.PartitionOverwriteMode -import org.apache.spark.sql.test.SQLTestData.TestData -import org.apache.spark.sql.types.{IntegerType, StructType} -import org.apache.spark.sql.util.QueryExecutionListener - -import org.apache.logging.log4j.Level - -class ClickHouseAdaptiveQueryExecSuite extends AdaptiveQueryExecSuite with GlutenSQLTestsTrait { - import testImplicits._ - - override def sparkConf: SparkConf = { - super.sparkConf - .set(GlutenConfig.COLUMNAR_FORCE_SHUFFLED_HASH_JOIN_ENABLED.key, "false") - .set(SQLConf.SHUFFLE_PARTITIONS.key, "5") - } - - private def runAdaptiveAndVerifyResult(query: String): (SparkPlan, SparkPlan) = { - var finalPlanCnt = 0 - val listener = new SparkListener { - override def onOtherEvent(event: SparkListenerEvent): Unit = { - event match { - case SparkListenerSQLAdaptiveExecutionUpdate(_, _, sparkPlanInfo) => - if (sparkPlanInfo.simpleString.startsWith("AdaptiveSparkPlan isFinalPlan=true")) { - finalPlanCnt += 1 - } - case _ => // ignore other events - } - } - } - spark.sparkContext.addSparkListener(listener) - - val dfAdaptive = sql(query) - val planBefore = dfAdaptive.queryExecution.executedPlan - assert(planBefore.toString.startsWith("AdaptiveSparkPlan isFinalPlan=false")) - val result = dfAdaptive.collect() - withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") { - val df = sql(query) - checkAnswer(df, result) - } - val planAfter = dfAdaptive.queryExecution.executedPlan - assert(planAfter.toString.startsWith("AdaptiveSparkPlan isFinalPlan=true")) - val adaptivePlan = planAfter.asInstanceOf[AdaptiveSparkPlanExec].executedPlan - - spark.sparkContext.listenerBus.waitUntilEmpty() - // AQE will post `SparkListenerSQLAdaptiveExecutionUpdate` twice in case of subqueries that - // exist out of query stages. - val expectedFinalPlanCnt = adaptivePlan.find(_.subqueries.nonEmpty).map(_ => 2).getOrElse(1) - assert(finalPlanCnt == expectedFinalPlanCnt) - spark.sparkContext.removeSparkListener(listener) - - val exchanges = adaptivePlan.collect { case e: Exchange => e } - assert(exchanges.isEmpty, "The final plan should not contain any Exchange node.") - (dfAdaptive.queryExecution.sparkPlan, adaptivePlan) - } - - private def broadcastHashJoinSize(plan: SparkPlan): Int = { - findTopLevelBroadcastHashJoinTransform(plan).size + findTopLevelBroadcastHashJoin(plan).size - } - - private def findTopLevelBroadcastHashJoinTransform( - plan: SparkPlan): Seq[BroadcastHashJoinExecTransformerBase] = { - collect(plan) { case j: BroadcastHashJoinExecTransformerBase => j } - } - - private def findTopLevelBroadcastHashJoin(plan: SparkPlan): Seq[BroadcastHashJoinExec] = { - collect(plan) { case j: BroadcastHashJoinExec => j } - } - - private def findTopLevelSortMergeJoin(plan: SparkPlan): Seq[SortMergeJoinExec] = { - collect(plan) { case j: SortMergeJoinExec => j } - } - - private def findTopLevelSortMergeJoinTransform( - plan: SparkPlan): Seq[SortMergeJoinExecTransformerBase] = { - collect(plan) { case j: SortMergeJoinExecTransformerBase => j } - } - - private def sortMergeJoinSize(plan: SparkPlan): Int = { - findTopLevelSortMergeJoinTransform(plan).size + findTopLevelSortMergeJoin(plan).size - } - - private def findTopLevelShuffledHashJoin(plan: SparkPlan): Seq[ShuffledHashJoinExec] = { - collect(plan) { case j: ShuffledHashJoinExec => j } - } - - private def findTopLevelShuffledHashJoinTransform( - plan: SparkPlan): Seq[ShuffledHashJoinExecTransformerBase] = { - collect(plan) { case j: ShuffledHashJoinExecTransformerBase => j } - } - - private def findTopLevelBaseJoin(plan: SparkPlan): Seq[BaseJoinExec] = { - collect(plan) { case j: BaseJoinExec => j } - } - - private def findTopLevelSort(plan: SparkPlan): Seq[SortExec] = { - collect(plan) { case s: SortExec => s } - } - - private def findTopLevelSortTransform(plan: SparkPlan): Seq[SortExecTransformer] = { - collect(plan) { case s: SortExecTransformer => s } - } - - private def findReusedExchange(plan: SparkPlan): Seq[ReusedExchangeExec] = { - collectWithSubqueries(plan) { - case ShuffleQueryStageExec(_, e: ReusedExchangeExec, _) => e - case BroadcastQueryStageExec(_, e: ReusedExchangeExec, _) => e - } - } - - private def findReusedSubquery(plan: SparkPlan): Seq[ReusedSubqueryExec] = { - collectWithSubqueries(plan) { case e: ReusedSubqueryExec => e } - } - - private def checkNumLocalShuffleReads( - plan: SparkPlan, - numShufflesWithoutLocalRead: Int = 0): Unit = { - val numShuffles = collect(plan) { case s: ShuffleQueryStageExec => s }.length - - val numLocalReads = collect(plan) { - case r: AQEShuffleReadExec if r.isLocalRead => r - } - // because columnar local reads cannot execute - numLocalReads.foreach { - r => - val rdd = r.executeColumnar() - val parts = rdd.partitions - assert(parts.forall(rdd.preferredLocations(_).nonEmpty)) - } - assert(numShuffles === (numLocalReads.length + numShufflesWithoutLocalRead)) - } - - testGluten("Change merge join to broadcast join") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300" - ) { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - "SELECT * FROM testData join testData2 ON key = a where value = '1'") - val smj = findTopLevelSortMergeJoin(plan) - assert(smj.size == 1) - val bhj = findTopLevelBroadcastHashJoinTransform(adaptivePlan) - assert(bhj.size == 1) - checkNumLocalShuffleReads(adaptivePlan) - } - } - - testGluten("Change broadcast join to merge join") { - withTable("t1", "t2") { - withSQLConf( - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "10000", - SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - SQLConf.SHUFFLE_PARTITIONS.key -> "1") { - sql("CREATE TABLE t1 USING PARQUET AS SELECT 1 c1") - sql("CREATE TABLE t2 USING PARQUET AS SELECT 1 c1") - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult(""" - |SELECT * FROM ( - | SELECT distinct c1 from t1 - | ) tmp1 JOIN ( - | SELECT distinct c1 from t2 - | ) tmp2 ON tmp1.c1 = tmp2.c1 - |""".stripMargin) - assert(broadcastHashJoinSize(plan) == 1) - assert(broadcastHashJoinSize(adaptivePlan) == 0) - assert(findTopLevelSortMergeJoinTransform(adaptivePlan).size == 1) - } - } - } - - testGluten("Reuse the parallelism of coalesced shuffle in local shuffle read") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300", - SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "10") { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - "SELECT * FROM testData join testData2 ON key = a where value = '1'") - - assert(sortMergeJoinSize(plan) == 1) - assert(broadcastHashJoinSize(adaptivePlan) == 1) - val localReads = collect(adaptivePlan) { - case read: AQEShuffleReadExec if read.isLocalRead => read - } - assert(localReads.length == 2) - } - } - - testGluten("Reuse the default parallelism in local shuffle read") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300", - SQLConf.COALESCE_PARTITIONS_ENABLED.key -> "false") { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - "SELECT * FROM testData join testData2 ON key = a where value = '1'") - val smj = findTopLevelSortMergeJoin(plan) - assert(smj.size == 1) - val bhj = findTopLevelBroadcastHashJoinTransform(adaptivePlan) - assert(bhj.size == 1) - val localReads = collect(adaptivePlan) { - case read: AQEShuffleReadExec if read.isLocalRead => read - } - assert(localReads.length == 2) - val localShuffleRDD0 = localReads(0) - .executeColumnar() - .asInstanceOf[ShuffledColumnarBatchRDD] - val localShuffleRDD1 = localReads(1) - .executeColumnar() - .asInstanceOf[ShuffledColumnarBatchRDD] - // the final parallelism is math.max(1, numReduces / numMappers): math.max(1, 5/2) = 2 - // and the partitions length is 2 * numMappers = 4 - assert(localShuffleRDD0.getPartitions.length == 4) - // the final parallelism is math.max(1, numReduces / numMappers): math.max(1, 5/2) = 2 - // and the partitions length is 2 * numMappers = 4 - assert(localShuffleRDD1.getPartitions.length == 4) - } - } - - testGluten("Empty stage coalesced to 1-partition RDD") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.COALESCE_PARTITIONS_ENABLED.key -> "true", - SQLConf.ADAPTIVE_OPTIMIZER_EXCLUDED_RULES.key -> AQEPropagateEmptyRelation.ruleName - ) { - val df1 = spark.range(10).withColumn("a", 'id) - val df2 = spark.range(10).withColumn("b", 'id) - withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") { - val testDf = df1 - .where('a > 10) - .join(df2.where('b > 10), Seq("id"), "left_outer") - .groupBy('a) - .count() - checkAnswer(testDf, Seq()) - val plan = testDf.queryExecution.executedPlan - assert(find(plan)(_.isInstanceOf[SortMergeJoinExecTransformerBase]).isDefined) - } - - withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "1") { - val testDf = df1 - .where('a > 10) - .join(df2.where('b > 10), Seq("id"), "left_outer") - .groupBy('a) - .count() - checkAnswer(testDf, Seq()) - val plan = testDf.queryExecution.executedPlan - assert(find(plan)(_.isInstanceOf[BroadcastHashJoinExecTransformerBase]).isDefined) - val coalescedReads = collect(plan) { case r: AQEShuffleReadExec => r } - assert(coalescedReads.length == 3, s"$plan") - coalescedReads.foreach(r => assert(r.isLocalRead || r.partitionSpecs.length == 1)) - } - } - } - - testGluten("Scalar subquery") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300") { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - "SELECT * FROM testData join testData2 ON key = a " + - "where value = (SELECT max(a) from testData3)") - assert(sortMergeJoinSize(plan) == 1) - assert(broadcastHashJoinSize(adaptivePlan) == 1) - checkNumLocalShuffleReads(adaptivePlan) - } - } - - testGluten("Scalar subquery in later stages") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300") { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - "SELECT * FROM testData join testData2 ON key = a " + - "where (value + a) = (SELECT max(a) from testData3)") - assert(sortMergeJoinSize(plan) == 1) - assert(broadcastHashJoinSize(adaptivePlan) == 1) - - checkNumLocalShuffleReads(adaptivePlan) - } - } - - testGluten("multiple joins") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300") { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - """ - |WITH t4 AS ( - | SELECT * FROM lowercaseData t2 JOIN testData3 t3 ON t2.n = t3.a where t2.n = '1' - |) - |SELECT * FROM testData - |JOIN testData2 t2 ON key = t2.a - |JOIN t4 ON t2.b = t4.a - |WHERE value = 1 - """.stripMargin) - assert(sortMergeJoinSize(plan) == 3) - assert(broadcastHashJoinSize(adaptivePlan) == 3) - - // A possible resulting query plan: - // BroadcastHashJoin - // +- BroadcastExchange - // +- LocalShuffleReader* - // +- ShuffleExchange - // +- BroadcastHashJoin - // +- BroadcastExchange - // +- LocalShuffleReader* - // +- ShuffleExchange - // +- LocalShuffleReader* - // +- ShuffleExchange - // +- BroadcastHashJoin - // +- LocalShuffleReader* - // +- ShuffleExchange - // +- BroadcastExchange - // +-LocalShuffleReader* - // +- ShuffleExchange - - // After applied the 'OptimizeShuffleWithLocalRead' rule, we can convert all the four - // shuffle read to local shuffle read in the bottom two 'BroadcastHashJoin'. - // For the top level 'BroadcastHashJoin', the probe side is not shuffle query stage - // and the build side shuffle query stage is also converted to local shuffle read. - checkNumLocalShuffleReads(adaptivePlan, 0) - } - } - - testGluten("multiple joins with aggregate") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300") { - val (plan, adaptivePlan) = - runAdaptiveAndVerifyResult(""" - |WITH t4 AS ( - | SELECT * FROM lowercaseData t2 JOIN ( - | select a, sum(b) from testData3 group by a - | ) t3 ON t2.n = t3.a where t2.n = '1' - |) - |SELECT * FROM testData - |JOIN testData2 t2 ON key = t2.a - |JOIN t4 ON t2.b = t4.a - |WHERE value = 1 - """.stripMargin) - assert(sortMergeJoinSize(plan) == 3) - assert(broadcastHashJoinSize(adaptivePlan) == 3) - - // A possible resulting query plan: - // BroadcastHashJoin - // +- BroadcastExchange - // +- LocalShuffleReader* - // +- ShuffleExchange - // +- BroadcastHashJoin - // +- BroadcastExchange - // +- LocalShuffleReader* - // +- ShuffleExchange - // +- LocalShuffleReader* - // +- ShuffleExchange - // +- BroadcastHashJoin - // +- LocalShuffleReader* - // +- ShuffleExchange - // +- BroadcastExchange - // +-HashAggregate - // +- CoalescedShuffleReader - // +- ShuffleExchange - - // The shuffle added by Aggregate can't apply local read. - checkNumLocalShuffleReads(adaptivePlan, 1) - } - } - - testGluten("multiple joins with aggregate 2") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "500") { - val (plan, adaptivePlan) = - runAdaptiveAndVerifyResult(""" - |WITH t4 AS ( - | SELECT * FROM lowercaseData t2 JOIN ( - | select a, max(b) b from testData2 group by a - | ) t3 ON t2.n = t3.b - |) - |SELECT * FROM testData - |JOIN testData2 t2 ON key = t2.a - |JOIN t4 ON value = t4.a - |WHERE value = 1 - """.stripMargin) - assert(sortMergeJoinSize(plan) == 3) - assert(broadcastHashJoinSize(adaptivePlan) == 3) - - // A possible resulting query plan: - // BroadcastHashJoin - // +- BroadcastExchange - // +- LocalShuffleReader* - // +- ShuffleExchange - // +- BroadcastHashJoin - // +- BroadcastExchange - // +- LocalShuffleReader* - // +- ShuffleExchange - // +- LocalShuffleReader* - // +- ShuffleExchange - // +- BroadcastHashJoin - // +- Filter - // +- HashAggregate - // +- CoalescedShuffleReader - // +- ShuffleExchange - // +- BroadcastExchange - // +-LocalShuffleReader* - // +- ShuffleExchange - - // The shuffle added by Aggregate can't apply local read. - checkNumLocalShuffleReads(adaptivePlan, 1) - } - } - - testGluten("Exchange reuse") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - // magic threshold, ch backend has two bhj when threshold is 100 - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "70", - SQLConf.SHUFFLE_PARTITIONS.key -> "5" - ) { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - "SELECT value FROM testData join testData2 ON key = a " + - "join (SELECT value v from testData join testData3 ON key = a) on value = v") - assert(sortMergeJoinSize(plan) == 3) - // TODO: vanilla spark has 2 bhj, and 1 smj, but gluten has 3 bhj, - // make sure this will not cause performance regression and why it is bhj - assert(broadcastHashJoinSize(adaptivePlan) == 1) - // Vanilla spark still a SMJ, and its two shuffles can't apply local read. - checkNumLocalShuffleReads(adaptivePlan, 4) - // Even with local shuffle read, the query stage reuse can also work. - val ex = findReusedExchange(adaptivePlan) - assert(ex.size == 1) - } - } - - testGluten("Exchange reuse with subqueries") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300") { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - "SELECT a FROM testData join testData2 ON key = a " + - "where value = (SELECT max(a) from testData join testData2 ON key = a)") - assert(sortMergeJoinSize(plan) == 1) - assert(broadcastHashJoinSize(adaptivePlan) == 1) - checkNumLocalShuffleReads(adaptivePlan) - // // Even with local shuffle read, the query stage reuse can also work. - // gluten change the smj to bhj, stage is changed, so we cannot find the stage with old - // ReuseExchange from stageCache, then the reuse is removed - // https://github.com/apache/spark/pull/24706/ - // files#diff-ec42cd27662f3f528832c298a60fffa1d341feb04aa1d8c80044b70cbe0ebbfcR224 - // maybe vanilla spark should checkReuse rile again - // val ex = findReusedExchange(adaptivePlan) - // assert(ex.size == 1) - } - } - - testGluten("Exchange reuse across subqueries") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300", - SQLConf.SUBQUERY_REUSE_ENABLED.key -> "false") { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - "SELECT a FROM testData join testData2 ON key = a " + - "where value >= (SELECT max(a) from testData join testData2 ON key = a) " + - "and a <= (SELECT max(a) from testData join testData2 ON key = a)") - assert(sortMergeJoinSize(plan) == 1) - assert(broadcastHashJoinSize(adaptivePlan) == 1) - checkNumLocalShuffleReads(adaptivePlan) - // Even with local shuffle read, the query stage reuse can also work. - val ex = findReusedExchange(adaptivePlan) - assert(ex.nonEmpty) - val sub = findReusedSubquery(adaptivePlan) - assert(sub.isEmpty) - } - } - - testGluten("Subquery reuse") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300") { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - "SELECT a FROM testData join testData2 ON key = a " + - "where value >= (SELECT max(a) from testData join testData2 ON key = a) " + - "and a <= (SELECT max(a) from testData join testData2 ON key = a)") - assert(sortMergeJoinSize(plan) == 1) - assert(broadcastHashJoinSize(adaptivePlan) == 1) - checkNumLocalShuffleReads(adaptivePlan) - // Even with local shuffle read, the query stage reuse can also work. - val ex = findReusedExchange(adaptivePlan) - assert(ex.isEmpty) - val sub = findReusedSubquery(adaptivePlan) - assert(sub.nonEmpty) - } - } - - testGluten("Broadcast exchange reuse across subqueries") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "20000000", - SQLConf.SUBQUERY_REUSE_ENABLED.key -> "false") { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - "SELECT a FROM testData join testData2 ON key = a " + - "where value >= (" + - "SELECT /*+ broadcast(testData2) */ max(key) from testData join testData2 ON key = a) " + - "and a <= (" + - "SELECT /*+ broadcast(testData2) */ max(value) from testData join testData2 ON key = a)") - assert(sortMergeJoinSize(plan) == 1) - assert(broadcastHashJoinSize(adaptivePlan) == 1) - checkNumLocalShuffleReads(adaptivePlan) - // Even with local shuffle read, the query stage reuse can also work. - val ex = findReusedExchange(adaptivePlan) - assert(ex.nonEmpty) - assert(ex.head.child.isInstanceOf[ColumnarBroadcastExchangeExec]) - val sub = findReusedSubquery(adaptivePlan) - assert(sub.isEmpty) - } - } - - // Cost is equal, not test cost is greater, need new test, but other test may contain cost change, - // so it maybe not essential - testGluten("Avoid plan change if cost is greater") {} - - testGluten("Change merge join to broadcast join without local shuffle read") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.LOCAL_SHUFFLE_READER_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300") { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - """ - |SELECT * FROM testData t1 join testData2 t2 - |ON t1.key = t2.a join testData3 t3 on t2.a = t3.a - |where t1.value = 1 - """.stripMargin - ) - assert(sortMergeJoinSize(plan) == 2) - val bhj = findTopLevelBroadcastHashJoinTransform(adaptivePlan) - assert(bhj.size == 2) - // There is still a SMJ, and its two shuffles can't apply local read. - checkNumLocalShuffleReads(adaptivePlan, 0) - } - } - - testGluten( - "Avoid changing merge join to broadcast join if too many empty partitions " + - "on build plan") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.NON_EMPTY_PARTITION_RATIO_FOR_BROADCAST_JOIN.key -> "0.5", - // this config will make some empty partitions - SQLConf.SHUFFLE_PARTITIONS.key -> "5" - ) { - // `testData` is small enough to be broadcast but has empty partition ratio over the config. - // because testData2 in gluten sizeInBytes(from ColumnarShuffleExchangeExec plan stats) - // is 78B sometimes, so change the threshold from 80 to 60 - withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "60") { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - "SELECT * FROM testData join testData2 ON key = a where value = '1'") - assert(sortMergeJoinSize(plan) == 1) - val bhj = findTopLevelBroadcastHashJoinTransform(adaptivePlan) - assert(bhj.isEmpty) - } - // It is still possible to broadcast `testData2`. - withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "2000") { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - "SELECT * FROM testData join testData2 ON key = a where value = '1'") - assert(sortMergeJoinSize(plan) == 1) - val bhj = findTopLevelBroadcastHashJoinTransform(adaptivePlan) - assert(bhj.size == 1) - assert(bhj.head.joinBuildSide == BuildRight) - } - } - } - - testGluten("SPARK-30524: Do not optimize skew join if introduce additional shuffle") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "100", - SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "100" - ) { - withTempView("skewData1", "skewData2") { - spark - .range(0, 1000, 1, 10) - .selectExpr("id % 3 as key1", "id as value1") - .createOrReplaceTempView("skewData1") - spark - .range(0, 1000, 1, 10) - .selectExpr("id % 1 as key2", "id as value2") - .createOrReplaceTempView("skewData2") - - def checkSkewJoin(query: String, optimizeSkewJoin: Boolean): Unit = { - val (_, innerAdaptivePlan) = runAdaptiveAndVerifyResult(query) - val innerSmj = findTopLevelSortMergeJoinTransform(innerAdaptivePlan) - assert(innerSmj.size == 1 && innerSmj.head.isSkewJoin == optimizeSkewJoin) - } - - // OptimizeSkewedJoin check the map status, because the - checkSkewJoin("SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2", true) - // Additional shuffle introduced, so disable the "OptimizeSkewedJoin" optimization - checkSkewJoin( - "SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2 GROUP BY key1", - false) - } - } - } - - testGluten("SPARK-29544: adaptive skew join with different join types") { - Seq("SHUFFLE_MERGE", "SHUFFLE_HASH").foreach { - joinHint => - def getJoinNode(plan: SparkPlan): Seq[BinaryExecNode] = if (joinHint == "SHUFFLE_MERGE") { - findTopLevelSortMergeJoinTransform(plan) - } else { - findTopLevelShuffledHashJoinTransform(plan) - } - - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM.key -> "1", - SQLConf.SHUFFLE_PARTITIONS.key -> "100", - SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "800", - SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "800" - ) { - withTempView("skewData1", "skewData2") { - spark - .range(0, 1000, 1, 10) - .select( - when('id < 250, 249) - .when('id >= 750, 1000) - .otherwise('id) - .as("key1"), - 'id.as("value1")) - .createOrReplaceTempView("skewData1") - spark - .range(0, 1000, 1, 10) - .select( - when('id < 250, 249) - .otherwise('id) - .as("key2"), - 'id.as("value2")) - .createOrReplaceTempView("skewData2") - - def checkSkewJoin( - joins: Seq[BinaryExecNode], - leftSkewNum: Int, - rightSkewNum: Int): Unit = { - assert(joins.size == 1) - joins.head match { - case s: SortMergeJoinExecTransformerBase => assert(s.isSkewJoin) - case g: ShuffledHashJoinExecTransformerBase => assert(g.isSkewJoin) - case _ => assert(false) - } - assert( - joins.head.left - .collect { case r: AQEShuffleReadExec => r } - .head - .partitionSpecs - .collect { case p: PartialReducerPartitionSpec => p.reducerIndex } - .distinct - .length == leftSkewNum) - assert( - joins.head.right - .collect { case r: AQEShuffleReadExec => r } - .head - .partitionSpecs - .collect { case p: PartialReducerPartitionSpec => p.reducerIndex } - .distinct - .length == rightSkewNum) - } - - // skewed inner join optimization - val (_, innerAdaptivePlan) = runAdaptiveAndVerifyResult( - s"SELECT /*+ $joinHint(skewData1) */ * FROM skewData1 " + - "JOIN skewData2 ON key1 = key2") - val inner = getJoinNode(innerAdaptivePlan) - // checkSkewJoin(inner, 2, 1) - - // skewed left outer join optimization - val (_, leftAdaptivePlan) = runAdaptiveAndVerifyResult( - s"SELECT /*+ $joinHint(skewData2) */ * FROM skewData1 " + - "LEFT OUTER JOIN skewData2 ON key1 = key2") - val leftJoin = getJoinNode(leftAdaptivePlan) - // checkSkewJoin(leftJoin, 2, 0) - - // skewed right outer join optimization - val (_, rightAdaptivePlan) = runAdaptiveAndVerifyResult( - s"SELECT /*+ $joinHint(skewData1) */ * FROM skewData1 " + - "RIGHT OUTER JOIN skewData2 ON key1 = key2") - val rightJoin = getJoinNode(rightAdaptivePlan) - // checkSkewJoin(rightJoin, 0, 1) - } - } - } - } - - testGluten("SPARK-34682: AQEShuffleReadExec operating on canonicalized plan") { - withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true") { - val (_, adaptivePlan) = runAdaptiveAndVerifyResult("SELECT key FROM testData GROUP BY key") - val reads = collect(adaptivePlan) { case r: AQEShuffleReadExec => r } - assert(reads.length == 1) - val read = reads.head - val c = read.canonicalized.asInstanceOf[AQEShuffleReadExec] - // we can't just call execute() because that has separate checks for canonicalized plans - val ex = intercept[IllegalStateException] { - val doExecute = PrivateMethod[Unit](Symbol("doExecuteColumnar")) - c.invokePrivate(doExecute()) - } - assert(ex.getMessage === "operating on canonicalized plan") - } - } - - testGluten("metrics of the shuffle read") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.SHUFFLE_PARTITIONS.key -> "5") { - val (_, adaptivePlan) = runAdaptiveAndVerifyResult("SELECT key FROM testData GROUP BY key") - val reads = collect(adaptivePlan) { case r: AQEShuffleReadExec => r } - assert(reads.length == 1) - val read = reads.head - assert(!read.isLocalRead) - assert(!read.hasSkewedPartition) - assert(read.hasCoalescedPartition) - assert( - read.metrics.keys.toSeq.sorted == Seq( - "numCoalescedPartitions", - "numPartitions", - "partitionDataSize")) - assert(read.metrics("numCoalescedPartitions").value == 1) - assert(read.metrics("numPartitions").value == read.partitionSpecs.length) - assert(read.metrics("partitionDataSize").value > 0) - - withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300") { - val (_, adaptivePlan) = runAdaptiveAndVerifyResult( - "SELECT * FROM testData join testData2 ON key = a where value = '1'") - val join = collect(adaptivePlan) { case j: BroadcastHashJoinExecTransformerBase => j }.head - assert(join.joinBuildSide == BuildLeft) - - val reads = collect(join.right) { case r: AQEShuffleReadExec => r } - assert(reads.length == 1) - val read = reads.head - assert(read.isLocalRead) - assert(read.metrics.keys.toSeq == Seq("numPartitions")) - assert(read.metrics("numPartitions").value == read.partitionSpecs.length) - } - - withSQLConf( - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - SQLConf.SHUFFLE_PARTITIONS.key -> "100", - SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "800", - SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "1000" - ) { - withTempView("skewData1", "skewData2") { - spark - .range(0, 1000, 1, 10) - .select( - when('id < 250, 249) - .when('id >= 750, 1000) - .otherwise('id) - .as("key1"), - 'id.as("value1")) - .createOrReplaceTempView("skewData1") - spark - .range(0, 1000, 1, 10) - .select( - when('id < 250, 249) - .otherwise('id) - .as("key2"), - 'id.as("value2")) - .createOrReplaceTempView("skewData2") - val (_, adaptivePlan) = - runAdaptiveAndVerifyResult("SELECT * FROM skewData1 join skewData2 ON key1 = key2") - } - } - } - } - - testGluten("SPARK-32717: AQEOptimizer should respect excludedRules configuration") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> Long.MaxValue.toString, - // This test is a copy of test(SPARK-32573), in order to test the configuration - // `spark.sql.adaptive.optimizer.excludedRules` works as expect. - "spark.gluten.sql.columnar.backend.ch.aqe.propagate.empty.relation" -> "false" - ) { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - "SELECT * FROM testData2 t1 WHERE t1.b NOT IN (SELECT b FROM testData3)") - val bhj = findTopLevelBroadcastHashJoin(plan) - assert(bhj.size == 1) - val join = findTopLevelBaseJoin(adaptivePlan) - // this is different compares to test(SPARK-32573) due to the rule - // `EliminateUnnecessaryJoin` has been excluded. - assert(join.nonEmpty) - checkNumLocalShuffleReads(adaptivePlan) - } - } - - testGluten("SPARK-32753: Only copy tags to node with no tags") { - withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true") { - withTempView("v1") { - spark.range(10).union(spark.range(10)).createOrReplaceTempView("v1") - - val (_, adaptivePlan) = - runAdaptiveAndVerifyResult("SELECT id FROM v1 GROUP BY id DISTRIBUTE BY id") - assert(collect(adaptivePlan) { case s: ColumnarShuffleExchangeExec => s }.length == 1) - } - } - } - - testGluten("Logging plan changes for AQE") { - val testAppender = new LogAppender("plan changes") - withLogAppender(testAppender) { - withSQLConf( - // this test default level is WARN, so we should check warn level - SQLConf.PLAN_CHANGE_LOG_LEVEL.key -> "WARN", - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80" - ) { - sql( - "SELECT * FROM testData JOIN testData2 ON key = a " + - "WHERE value = (SELECT max(a) FROM testData3)").collect() - } - Seq( - "=== Result of Batch AQE Preparations ===", - "=== Result of Batch AQE Post Stage Creation ===", - "=== Result of Batch AQE Replanning ===", - "=== Result of Batch AQE Query Stage Optimization ===" - ).foreach { - expectedMsg => - assert( - testAppender.loggingEvents.exists( - _.getMessage.getFormattedMessage.contains(expectedMsg))) - } - } - } - - testGluten("SPARK-33551: Do not use AQE shuffle read for repartition") { - def hasRepartitionShuffle(plan: SparkPlan): Boolean = { - find(plan) { - case s: ShuffleExchangeLike => - s.shuffleOrigin == REPARTITION_BY_COL || s.shuffleOrigin == REPARTITION_BY_NUM - case _ => false - }.isDefined - } - - def checkBHJ( - df: Dataset[Row], - optimizeOutRepartition: Boolean, - probeSideLocalRead: Boolean, - probeSideCoalescedRead: Boolean): Unit = { - df.collect() - val plan = df.queryExecution.executedPlan - // There should be only one shuffle that can't do local read, which is either the top shuffle - // from repartition, or BHJ probe side shuffle. - checkNumLocalShuffleReads(plan, 1) - assert(hasRepartitionShuffle(plan) == !optimizeOutRepartition) - val bhj = findTopLevelBroadcastHashJoinTransform(plan) - assert(bhj.length == 1) - - // Build side should do local read. - val buildSide = find(bhj.head.left)(_.isInstanceOf[AQEShuffleReadExec]) - assert(buildSide.isDefined) - assert(buildSide.get.asInstanceOf[AQEShuffleReadExec].isLocalRead) - - val probeSide = find(bhj.head.right)(_.isInstanceOf[AQEShuffleReadExec]) - if (probeSideLocalRead || probeSideCoalescedRead) { - assert(probeSide.isDefined) - if (probeSideLocalRead) { - assert(probeSide.get.asInstanceOf[AQEShuffleReadExec].isLocalRead) - } else { - assert(probeSide.get.asInstanceOf[AQEShuffleReadExec].hasCoalescedPartition) - } - } else { - assert(probeSide.isEmpty) - } - } - - def checkSMJ( - df: Dataset[Row], - optimizeOutRepartition: Boolean, - optimizeSkewJoin: Boolean, - coalescedRead: Boolean): Unit = { - df.collect() - val plan = df.queryExecution.executedPlan - assert(hasRepartitionShuffle(plan) == !optimizeOutRepartition) - val smj = findTopLevelSortMergeJoin(plan) - assert(smj.length == 1) - assert(smj.head.isSkewJoin == optimizeSkewJoin) - val aqeReads = collect(smj.head) { case c: AQEShuffleReadExec => c } - if (coalescedRead || optimizeSkewJoin) { - assert(aqeReads.length == 2) - if (coalescedRead) assert(aqeReads.forall(_.hasCoalescedPartition)) - } else { - assert(aqeReads.isEmpty) - } - } - - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.SHUFFLE_PARTITIONS.key -> "5") { - val df = sql(""" - |SELECT * FROM ( - | SELECT * FROM testData WHERE key = 1 - |) - |RIGHT OUTER JOIN testData2 - |ON value = b - """.stripMargin) - - withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300") { - // Repartition with no partition num specified. - checkBHJ( - df.repartition('b), - // The top shuffle from repartition is optimized out. - optimizeOutRepartition = true, - probeSideLocalRead = false, - probeSideCoalescedRead = true - ) - - // Repartition with default partition num (5 in test env) specified. - checkBHJ( - df.repartition(5, 'b), - // The top shuffle from repartition is optimized out - // The final plan must have 5 partitions, no optimization can be made to the probe side. - optimizeOutRepartition = true, - probeSideLocalRead = false, - probeSideCoalescedRead = false - ) - - // Repartition with non-default partition num specified. - checkBHJ( - df.repartition(4, 'b), - // The top shuffle from repartition is not optimized out - optimizeOutRepartition = false, - probeSideLocalRead = true, - probeSideCoalescedRead = true - ) - - // Repartition by col and project away the partition cols - checkBHJ( - df.repartition('b).select('key), - // The top shuffle from repartition is not optimized out - optimizeOutRepartition = false, - probeSideLocalRead = true, - probeSideCoalescedRead = true - ) - } - - // Force skew join - withSQLConf( - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - SQLConf.SKEW_JOIN_ENABLED.key -> "true", - SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "1", - SQLConf.SKEW_JOIN_SKEWED_PARTITION_FACTOR.key -> "0", - SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "10" - ) { - // Repartition with no partition num specified. - checkSMJ( - df.repartition('b), - // The top shuffle from repartition is optimized out. - optimizeOutRepartition = true, - optimizeSkewJoin = false, - coalescedRead = true) - - // Repartition with default partition num (5 in test env) specified. - checkSMJ( - df.repartition(5, 'b), - // The top shuffle from repartition is optimized out. - // The final plan must have 5 partitions, can't do coalesced read. - optimizeOutRepartition = true, - optimizeSkewJoin = false, - coalescedRead = false - ) - - // Repartition with non-default partition num specified. - checkSMJ( - df.repartition(4, 'b), - // The top shuffle from repartition is not optimized out. - optimizeOutRepartition = false, - optimizeSkewJoin = true, - coalescedRead = false - ) - - // Repartition by col and project away the partition cols - checkSMJ( - df.repartition('b).select('key), - // The top shuffle from repartition is not optimized out. - optimizeOutRepartition = false, - optimizeSkewJoin = true, - coalescedRead = false - ) - } - } - } - - testGluten("SPARK-34091: Batch shuffle fetch in AQE partition coalescing") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.SHUFFLE_PARTITIONS.key -> "10", - SQLConf.FETCH_SHUFFLE_BLOCKS_IN_BATCH.key -> "true") { - withTable("t1") { - spark.range(100).selectExpr("id + 1 as a").write.format("parquet").saveAsTable("t1") - val query = "SELECT SUM(a) FROM t1 GROUP BY a" - val (_, adaptivePlan) = runAdaptiveAndVerifyResult(query) - val metricName = SQLShuffleReadMetricsReporter.LOCAL_BLOCKS_FETCHED - val blocksFetchedMetric = collectFirst(adaptivePlan) { - case p if p.metrics.contains(metricName) => p.metrics(metricName) - } - assert(blocksFetchedMetric.isDefined) - val blocksFetched = blocksFetchedMetric.get.value - withSQLConf(SQLConf.FETCH_SHUFFLE_BLOCKS_IN_BATCH.key -> "false") { - val (_, adaptivePlan2) = runAdaptiveAndVerifyResult(query) - val blocksFetchedMetric2 = collectFirst(adaptivePlan2) { - case p if p.metrics.contains(metricName) => p.metrics(metricName) - } - assert(blocksFetchedMetric2.isDefined) - val blocksFetched2 = blocksFetchedMetric2.get.value - assert(blocksFetched == blocksFetched2) - } - } - } - } - - testGluten("SPARK-34899: Use origin plan if we can not coalesce shuffle partition") { - def checkNoCoalescePartitions(ds: Dataset[Row], origin: ShuffleOrigin): Unit = { - assert(collect(ds.queryExecution.executedPlan) { - case s: ShuffleExchangeExec if s.shuffleOrigin == origin && s.numPartitions == 2 => s - }.size == 1) - ds.collect() - val plan = ds.queryExecution.executedPlan - assert(collect(plan) { - case s: ColumnarShuffleExchangeExec if s.shuffleOrigin == origin && s.numPartitions == 2 => - s - }.size == 1) - checkAnswer(ds, testData) - } - - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.COALESCE_PARTITIONS_ENABLED.key -> "true", - // Pick a small value so that no coalesce can happen. - SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "100", - SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM.key -> "1", - SQLConf.SHUFFLE_PARTITIONS.key -> "2" - ) { - val df = - spark.sparkContext.parallelize((1 to 100).map(i => TestData(i, i.toString)), 10).toDF() - - // partition size [1420, 1420] - checkNoCoalescePartitions(df.repartition($"key"), REPARTITION_BY_COL) - // partition size [1140, 1119] - checkNoCoalescePartitions(df.sort($"key"), ENSURE_REQUIREMENTS) - } - } - - testGluten("SPARK-35239: Coalesce shuffle partition should handle empty input RDD") { - withTable("t") { - withSQLConf( - SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM.key -> "1", - SQLConf.SHUFFLE_PARTITIONS.key -> "2", - SQLConf.ADAPTIVE_OPTIMIZER_EXCLUDED_RULES.key -> AQEPropagateEmptyRelation.ruleName - ) { - spark.sql("CREATE TABLE t (c1 int) USING PARQUET") - val (_, adaptive) = runAdaptiveAndVerifyResult("SELECT c1, count(*) FROM t GROUP BY c1") - assert( - collect(adaptive) { - case c @ AQEShuffleReadExec(_, partitionSpecs) if partitionSpecs.length == 1 => - assert(c.hasCoalescedPartition) - c - }.length == 1 - ) - } - } - } - - testGluten("SPARK-35264: Support AQE side broadcastJoin threshold") { - withTempView("t1", "t2") { - def checkJoinStrategy(shouldBroadcast: Boolean): Unit = { - withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") { - val (origin, adaptive) = - runAdaptiveAndVerifyResult("SELECT t1.c1, t2.c1 FROM t1 JOIN t2 ON t1.c1 = t2.c1") - assert(findTopLevelSortMergeJoin(origin).size == 1) - if (shouldBroadcast) { - assert(findTopLevelBroadcastHashJoinTransform(adaptive).size == 1) - } else { - assert(findTopLevelSortMergeJoinTransform(adaptive).size == 1) - } - } - } - - // t1: 1600 bytes - // t2: 160 bytes - spark.sparkContext - .parallelize((1 to 100).map(i => TestData(i, i.toString)), 10) - .toDF("c1", "c2") - .createOrReplaceTempView("t1") - spark.sparkContext - .parallelize((1 to 10).map(i => TestData(i, i.toString)), 5) - .toDF("c1", "c2") - .createOrReplaceTempView("t2") - - checkJoinStrategy(false) - withSQLConf(SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") { - checkJoinStrategy(false) - } - - withSQLConf(SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "400") { - checkJoinStrategy(true) - } - } - } - - // table partition size is different with spark - testGluten("SPARK-35264: Support AQE side shuffled hash join formula") { - withTempView("t1", "t2") { - def checkJoinStrategy(shouldShuffleHashJoin: Boolean): Unit = { - Seq("100", "100000").foreach { - size => - withSQLConf(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> size) { - val (origin1, adaptive1) = - runAdaptiveAndVerifyResult("SELECT t1.c1, t2.c1 FROM t1 JOIN t2 ON t1.c1 = t2.c1") - assert(findTopLevelSortMergeJoin(origin1).size === 1) - if (shouldShuffleHashJoin && size.toInt < 100000) { - val shj = findTopLevelShuffledHashJoinTransform(adaptive1) - assert(shj.size === 1) - assert(shj.head.joinBuildSide == BuildRight) - } else { - assert(findTopLevelSortMergeJoinTransform(adaptive1).size === 1) - } - } - } - // respect user specified join hint - val (origin2, adaptive2) = runAdaptiveAndVerifyResult( - "SELECT /*+ MERGE(t1) */ t1.c1, t2.c1 FROM t1 JOIN t2 ON t1.c1 = t2.c1") - assert(findTopLevelSortMergeJoin(origin2).size === 1) - assert(findTopLevelSortMergeJoinTransform(adaptive2).size === 1) - } - - spark.sparkContext - .parallelize((1 to 100).map(i => TestData(i, i.toString)), 10) - .toDF("c1", "c2") - .createOrReplaceTempView("t1") - spark.sparkContext - .parallelize((1 to 10).map(i => TestData(i, i.toString)), 5) - .toDF("c1", "c2") - .createOrReplaceTempView("t2") - - // t1 partition size: [395, 316, 313] - // t2 partition size: [140, 50, 0] - withSQLConf( - SQLConf.SHUFFLE_PARTITIONS.key -> "3", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - SQLConf.PREFER_SORTMERGEJOIN.key -> "true") { - // check default value - checkJoinStrategy(false) - // t1 no hint. - // t2 partition size are all smaller than 200, t2 has SHJ hint. The result is true. - withSQLConf(SQLConf.ADAPTIVE_MAX_SHUFFLE_HASH_JOIN_LOCAL_MAP_THRESHOLD.key -> "200") { - checkJoinStrategy(true) - } - // t1 no hint. - // Not all partition size of t2 are smaller than 100, t2 no hint. The result is false. - withSQLConf(SQLConf.ADAPTIVE_MAX_SHUFFLE_HASH_JOIN_LOCAL_MAP_THRESHOLD.key -> "100") { - checkJoinStrategy(false) - } - // t1, t2 partition size are all smaller than 1000, t1 and t2 can use SHJ. - // The result is true. - withSQLConf(SQLConf.ADAPTIVE_MAX_SHUFFLE_HASH_JOIN_LOCAL_MAP_THRESHOLD.key -> "1000") { - checkJoinStrategy(true) - } - } - } - } - - testGluten("SPARK-32932: Do not use local shuffle read at final stage on write command") { - withSQLConf( - SQLConf.PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.DYNAMIC.toString, - SQLConf.SHUFFLE_PARTITIONS.key -> "5", - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true" - ) { - val data = - for ( - i <- 1L to 10L; - j <- 1L to 3L - ) yield (i, j) - - val df = data.toDF("i", "j").repartition($"j") - var noLocalread: Boolean = false - val listener = new QueryExecutionListener { - override def onSuccess(funcName: String, qe: QueryExecution, durationNs: Long): Unit = { - qe.executedPlan match { - case plan @ (_: DataWritingCommandExec | _: V2TableWriteExec) => - noLocalread = collect(plan) { - case exec: AQEShuffleReadExec if exec.isLocalRead => exec - }.isEmpty - case _ => // ignore other events - } - } - override def onFailure( - funcName: String, - qe: QueryExecution, - exception: Exception): Unit = {} - } - spark.listenerManager.register(listener) - - withTable("t") { - df.write.partitionBy("j").saveAsTable("t") - sparkContext.listenerBus.waitUntilEmpty() - assert(noLocalread) - noLocalread = false - } - - // Test DataSource v2 - val format = classOf[NoopDataSource].getName - df.write.format(format).mode("overwrite").save() - sparkContext.listenerBus.waitUntilEmpty() - assert(noLocalread) - noLocalread = false - - spark.listenerManager.unregister(listener) - } - } - - testGluten( - "SPARK-30953: InsertAdaptiveSparkPlan should apply AQE on child plan of v2 write commands") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.ADAPTIVE_EXECUTION_FORCE_APPLY.key -> "true") { - var plan: SparkPlan = null - val listener = new QueryExecutionListener { - override def onSuccess(funcName: String, qe: QueryExecution, durationNs: Long): Unit = { - plan = qe.executedPlan - } - override def onFailure( - funcName: String, - qe: QueryExecution, - exception: Exception): Unit = {} - } - spark.listenerManager.register(listener) - withTable("t1") { - val format = classOf[NoopDataSource].getName - Seq((0, 1)).toDF("x", "y").write.format(format).mode("overwrite").save() - - sparkContext.listenerBus.waitUntilEmpty() - assert(plan.isInstanceOf[V2TableWriteExec]) - val childPlan = plan.asInstanceOf[V2TableWriteExec].child - assert(childPlan.isInstanceOf[ColumnarToCarrierRowExecBase]) - assert( - childPlan - .asInstanceOf[ColumnarToCarrierRowExecBase] - .child - .isInstanceOf[AdaptiveSparkPlanExec]) - - spark.listenerManager.unregister(listener) - } - } - } - - testGluten("SPARK-35650: Coalesce number of partitions by AEQ") { - withSQLConf(SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM.key -> "1") { - Seq("REPARTITION", "REBALANCE(key)") - .foreach { - repartition => - val query = s"SELECT /*+ $repartition */ * FROM testData" - val (_, adaptivePlan) = runAdaptiveAndVerifyResult(query) - collect(adaptivePlan) { case r: AQEShuffleReadExec => r } match { - case Seq(aqeShuffleRead) => - assert(aqeShuffleRead.partitionSpecs.size === 1) - assert(!aqeShuffleRead.isLocalRead) - case _ => - fail("There should be a AQEShuffleReadExec") - } - } - } - } - - testGluten("SPARK-35650: Use local shuffle read if can not coalesce number of partitions") { - withSQLConf(SQLConf.COALESCE_PARTITIONS_ENABLED.key -> "false") { - val query = "SELECT /*+ REPARTITION */ * FROM testData" - val (_, adaptivePlan) = runAdaptiveAndVerifyResult(query) - collect(adaptivePlan) { case r: AQEShuffleReadExec => r } match { - case Seq(aqeShuffleRead) => - assert(aqeShuffleRead.partitionSpecs.size === 4) - assert(aqeShuffleRead.isLocalRead) - case _ => - fail("There should be a AQEShuffleReadExec") - } - } - } - - testGluten("SPARK-35725: Support optimize skewed partitions in RebalancePartitions") { - withTempView("v") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.COALESCE_PARTITIONS_ENABLED.key -> "true", - SQLConf.ADAPTIVE_OPTIMIZE_SKEWS_IN_REBALANCE_PARTITIONS_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - SQLConf.SHUFFLE_PARTITIONS.key -> "5", - SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM.key -> "1" - ) { - - spark.sparkContext - .parallelize((1 to 10).map(i => TestData(if (i > 4) 5 else i, i.toString)), 3) - .toDF("c1", "c2") - .createOrReplaceTempView("v") - - def checkPartitionNumber( - query: String, - skewedPartitionNumber: Int, - totalNumber: Int): Unit = { - val (_, adaptive) = runAdaptiveAndVerifyResult(query) - val read = collect(adaptive) { case read: AQEShuffleReadExec => read } - assert(read.size == 1) - assert( - read.head.partitionSpecs.count(_.isInstanceOf[PartialReducerPartitionSpec]) == - skewedPartitionNumber) - assert(read.head.partitionSpecs.size == totalNumber) - } - - // Changed ADVISORY_PARTITION_SIZE_IN_BYTES from 150 to 120 because Gluten has smaller - // partition size. - withSQLConf(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "120") { - // partition size [0,208,54,54,54] - checkPartitionNumber("SELECT /*+ REBALANCE(c1) */ * FROM v", 2, 4) - // partition size [108, 54, 60, 108, 54, 108, 54] - checkPartitionNumber("SELECT /*+ REBALANCE */ * FROM v", 6, 7) - } - - // no skewed partition should be optimized - withSQLConf(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "10000") { - checkPartitionNumber("SELECT /*+ REBALANCE(c1) */ * FROM v", 0, 1) - } - } - } - } - - testGluten("SPARK-35888: join with a 0-partition table") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM.key -> "1", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - SQLConf.ADAPTIVE_OPTIMIZER_EXCLUDED_RULES.key -> AQEPropagateEmptyRelation.ruleName - ) { - withTempView("t2") { - // create a temp view with 0 partition - spark - .createDataFrame(sparkContext.emptyRDD[Row], new StructType().add("b", IntegerType)) - .createOrReplaceTempView("t2") - val (_, adaptive) = - runAdaptiveAndVerifyResult("SELECT * FROM testData2 t1 left semi join t2 ON t1.a=t2.b") - val aqeReads = collect(adaptive) { case c: AQEShuffleReadExec => c } - assert(aqeReads.length == 2) - aqeReads.foreach { - c => - val stats = c.child.asInstanceOf[QueryStageExec].getRuntimeStatistics - assert(stats.sizeInBytes >= 0) - assert(stats.rowCount.get >= 0) - } - } - } - } - - testGluten("SPARK-35968: AQE coalescing should not produce too small partitions by default") { - withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true") { - val (_, adaptive) = - runAdaptiveAndVerifyResult("SELECT sum(id) FROM RANGE(10) GROUP BY id % 3") - val coalesceRead = collect(adaptive) { - case r: AQEShuffleReadExec if r.hasCoalescedPartition => r - } - assert(coalesceRead.length == 1) - // RANGE(10) is a very small dataset and AQE coalescing should produce one partition. - assert(coalesceRead.head.partitionSpecs.length == 1) - } - } - - testGluten("SPARK-35794: Allow custom plugin for cost evaluator") { - CostEvaluator.instantiate( - classOf[SimpleShuffleSortCostEvaluator].getCanonicalName, - spark.sparkContext.getConf) - intercept[IllegalArgumentException] { - CostEvaluator.instantiate( - classOf[InvalidCostEvaluator].getCanonicalName, - spark.sparkContext.getConf) - } - - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300") { - val query = "SELECT * FROM testData join testData2 ON key = a where value = '1'" - - withSQLConf( - SQLConf.ADAPTIVE_CUSTOM_COST_EVALUATOR_CLASS.key -> - "org.apache.spark.sql.execution.adaptive.SimpleShuffleSortCostEvaluator") { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult(query) - val smj = findTopLevelSortMergeJoin(plan) - assert(smj.size == 1) - val bhj = findTopLevelBroadcastHashJoinTransform(adaptivePlan) - assert(bhj.size == 1) - checkNumLocalShuffleReads(adaptivePlan) - } - - withSQLConf( - SQLConf.ADAPTIVE_CUSTOM_COST_EVALUATOR_CLASS.key -> - "org.apache.spark.sql.execution.adaptive.InvalidCostEvaluator") { - intercept[IllegalArgumentException] { - runAdaptiveAndVerifyResult(query) - } - } - } - } - - testGluten("SPARK-36020: Check logical link in remove redundant projects") { - withTempView("t") { - spark - .range(10) - .selectExpr( - "id % 10 as key", - "cast(id * 2 as int) as a", - "cast(id * 3 as int) as b", - "array(id, id + 1, id + 3) as c") - .createOrReplaceTempView("t") - withSQLConf( - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "800") { - val query = - """ - |WITH tt AS ( - | SELECT key, a, b, explode(c) AS c FROM t - |) - |SELECT t1.key, t1.c, t2.key, t2.c - |FROM (SELECT a, b, c, key FROM tt WHERE a > 1) t1 - |JOIN (SELECT a, b, c, key FROM tt) t2 - | ON t1.key = t2.key - |""".stripMargin - val (origin, adaptive) = runAdaptiveAndVerifyResult(query) - assert(findTopLevelSortMergeJoin(origin).size == 1) - assert(findTopLevelBroadcastHashJoinTransform(adaptive).size == 1) - } - } - } - - testGluten( - "SPARK-36032: Use inputPlan instead of currentPhysicalPlan to initialize logical link") { - withTempView("v") { - spark.sparkContext - .parallelize((1 to 10).map(i => TestData(i, i.toString)), 2) - .toDF("c1", "c2") - .createOrReplaceTempView("v") - - Seq("-1", "10000").foreach { - aqeBhj => - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> aqeBhj, - SQLConf.SHUFFLE_PARTITIONS.key -> "1" - ) { - val (origin, adaptive) = runAdaptiveAndVerifyResult(""" - |SELECT * FROM v t1 JOIN ( - | SELECT c1 + 1 as c3 FROM v - |)t2 ON t1.c1 = t2.c3 - |SORT BY c1 - """.stripMargin) - if (aqeBhj.toInt < 0) { - // 1 sort since spark plan has no shuffle for SMJ - assert(findTopLevelSort(origin).size == 1) - // 2 sorts in SMJ - assert(findTopLevelSortTransform(adaptive).size == 2) - } else { - assert(findTopLevelSort(origin).size == 1) - // 1 sort at top node and BHJ has no sort - assert(findTopLevelSortTransform(adaptive).size == 1) - } - } - } - } - } - - testGluten("SPARK-37742: AQE reads invalid InMemoryRelation stats and mistakenly plans BHJ") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "1048584", - SQLConf.ADAPTIVE_OPTIMIZER_EXCLUDED_RULES.key -> AQEPropagateEmptyRelation.ruleName - ) { - // Spark estimates a string column as 20 bytes so with 60k rows, these relations should be - // estimated at ~120m bytes which is greater than the broadcast join threshold. - val joinKeyOne = "00112233445566778899" - val joinKeyTwo = "11223344556677889900" - Seq - .fill(60000)(joinKeyOne) - .toDF("key") - .createOrReplaceTempView("temp") - Seq - .fill(60000)(joinKeyTwo) - .toDF("key") - .createOrReplaceTempView("temp2") - - Seq(joinKeyOne).toDF("key").createOrReplaceTempView("smallTemp") - spark.sql("SELECT key as newKey FROM temp").persist() - - // This query is trying to set up a situation where there are three joins. - // The first join will join the cached relation with a smaller relation. - // The first join is expected to be a broadcast join since the smaller relation will - // fit under the broadcast join threshold. - // The second join will join the first join with another relation and is expected - // to remain as a sort-merge join. - // The third join will join the cached relation with another relation and is expected - // to remain as a sort-merge join. - val query = - s""" - |SELECT t3.newKey - |FROM - | (SELECT t1.newKey - | FROM (SELECT key as newKey FROM temp) as t1 - | JOIN - | (SELECT key FROM smallTemp) as t2 - | ON t1.newKey = t2.key - | ) as t3 - | JOIN - | (SELECT key FROM temp2) as t4 - | ON t3.newKey = t4.key - |UNION - |SELECT t1.newKey - |FROM - | (SELECT key as newKey FROM temp) as t1 - | JOIN - | (SELECT key FROM temp2) as t2 - | ON t1.newKey = t2.key - |""".stripMargin - val df = spark.sql(query) - df.collect() - val adaptivePlan = df.queryExecution.executedPlan - val bhj = findTopLevelBroadcastHashJoinTransform(adaptivePlan) - assert(bhj.length == 1) - } - } - - testGluten("test log level") { - def verifyLog(expectedLevel: Level): Unit = { - val logAppender = new LogAppender("adaptive execution") - logAppender.setThreshold(expectedLevel) - withLogAppender( - logAppender, - loggerNames = Seq(AdaptiveSparkPlanExec.getClass.getName.dropRight(1)), - level = Some(Level.TRACE)) { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300") { - sql("SELECT * FROM testData join testData2 ON key = a where value = '1'").collect() - } - } - Seq("Plan changed", "Final plan").foreach { - msg => - assert(logAppender.loggingEvents.exists { - event => - event.getMessage.getFormattedMessage.contains(msg) && event.getLevel == expectedLevel - }) - } - } - - // Verify default log level - verifyLog(Level.DEBUG) - - // Verify custom log level - val levels = Seq( - "TRACE" -> Level.TRACE, - "trace" -> Level.TRACE, - "DEBUG" -> Level.DEBUG, - "debug" -> Level.DEBUG, - "INFO" -> Level.INFO, - "info" -> Level.INFO, - "WARN" -> Level.WARN, - "warn" -> Level.WARN, - "ERROR" -> Level.ERROR, - "error" -> Level.ERROR, - "deBUG" -> Level.DEBUG - ) - - levels.foreach { - level => - withSQLConf(SQLConf.ADAPTIVE_EXECUTION_LOG_LEVEL.key -> level._1) { - verifyLog(level._2) - } - } - } - - testGluten("SPARK-37652: optimize skewed join through union") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "100", - SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "100" - ) { - withTempView("skewData1", "skewData2") { - spark - .range(0, 1000, 1, 10) - .selectExpr("id % 3 as key1", "id as value1") - .createOrReplaceTempView("skewData1") - spark - .range(0, 1000, 1, 10) - .selectExpr("id % 1 as key2", "id as value2") - .createOrReplaceTempView("skewData2") - - def checkSkewJoin(query: String, joinNums: Int, optimizeSkewJoinNums: Int): Unit = { - val (_, innerAdaptivePlan) = runAdaptiveAndVerifyResult(query) - val joins = findTopLevelSortMergeJoinTransform(innerAdaptivePlan) - val optimizeSkewJoins = joins.filter(_.isSkewJoin) - assert(joins.size == joinNums && optimizeSkewJoins.size == optimizeSkewJoinNums) - } - - // skewJoin union skewJoin - checkSkewJoin( - "SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2 " + - "UNION ALL SELECT key2 FROM skewData1 JOIN skewData2 ON key1 = key2", - 2, - 2) - - // skewJoin union aggregate - checkSkewJoin( - "SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2 " + - "UNION ALL SELECT key2 FROM skewData2 GROUP BY key2", - 1, - 1) - - // skewJoin1 union (skewJoin2 join aggregate) - // skewJoin2 will lead to extra shuffles, but skew1 cannot be optimized - checkSkewJoin( - "SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2 UNION ALL " + - "SELECT key1 from (SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2) tmp1 " + - "JOIN (SELECT key2 FROM skewData2 GROUP BY key2) tmp2 ON key1 = key2", - 3, - 0 - ) - } - } - } - -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/adaptive/velox/VeloxAdaptiveQueryExecSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/adaptive/velox/VeloxAdaptiveQueryExecSuite.scala deleted file mode 100644 index 64f7a14153c..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/adaptive/velox/VeloxAdaptiveQueryExecSuite.scala +++ /dev/null @@ -1,1602 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.adaptive.velox - -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.execution.{BroadcastHashJoinExecTransformerBase, ColumnarToCarrierRowExecBase, ShuffledHashJoinExecTransformerBase, SortExecTransformer, SortMergeJoinExecTransformer} - -import org.apache.spark.SparkConf -import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent} -import org.apache.spark.sql.{Dataset, GlutenSQLTestsTrait, Row} -import org.apache.spark.sql.GlutenTestConstants.GLUTEN_TEST -import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight} -import org.apache.spark.sql.execution._ -import org.apache.spark.sql.execution.adaptive._ -import org.apache.spark.sql.execution.command.DataWritingCommandExec -import org.apache.spark.sql.execution.datasources.noop.NoopDataSource -import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec -import org.apache.spark.sql.execution.exchange._ -import org.apache.spark.sql.execution.joins.{BaseJoinExec, BroadcastHashJoinExec, ShuffledHashJoinExec, SortMergeJoinExec} -import org.apache.spark.sql.execution.metric.SQLShuffleReadMetricsReporter -import org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate -import org.apache.spark.sql.functions.when -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.internal.SQLConf.PartitionOverwriteMode -import org.apache.spark.sql.test.SQLTestData.TestData -import org.apache.spark.sql.types.{IntegerType, StructType} -import org.apache.spark.sql.util.QueryExecutionListener - -import org.apache.logging.log4j.Level - -class VeloxAdaptiveQueryExecSuite extends AdaptiveQueryExecSuite with GlutenSQLTestsTrait { - import testImplicits._ - - override def sparkConf: SparkConf = { - super.sparkConf - .set(GlutenConfig.COLUMNAR_FORCE_SHUFFLED_HASH_JOIN_ENABLED.key, "false") - .set(SQLConf.SHUFFLE_PARTITIONS.key, "5") - } - - private def runAdaptiveAndVerifyResult(query: String): (SparkPlan, SparkPlan) = { - var finalPlanCnt = 0 - val listener = new SparkListener { - override def onOtherEvent(event: SparkListenerEvent): Unit = { - event match { - case SparkListenerSQLAdaptiveExecutionUpdate(_, _, sparkPlanInfo) => - if (sparkPlanInfo.simpleString.startsWith("AdaptiveSparkPlan isFinalPlan=true")) { - finalPlanCnt += 1 - } - case _ => // ignore other events - } - } - } - spark.sparkContext.addSparkListener(listener) - - val dfAdaptive = sql(query) - val planBefore = dfAdaptive.queryExecution.executedPlan - assert(planBefore.toString.startsWith("AdaptiveSparkPlan isFinalPlan=false")) - val result = dfAdaptive.collect() - withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") { - val df = sql(query) - checkAnswer(df, result) - } - val planAfter = dfAdaptive.queryExecution.executedPlan - assert(planAfter.toString.startsWith("AdaptiveSparkPlan isFinalPlan=true")) - val adaptivePlan = planAfter.asInstanceOf[AdaptiveSparkPlanExec].executedPlan - - spark.sparkContext.listenerBus.waitUntilEmpty() - // AQE will post `SparkListenerSQLAdaptiveExecutionUpdate` twice in case of subqueries that - // exist out of query stages. - val expectedFinalPlanCnt = adaptivePlan.find(_.subqueries.nonEmpty).map(_ => 2).getOrElse(1) - assert(finalPlanCnt == expectedFinalPlanCnt) - spark.sparkContext.removeSparkListener(listener) - - val exchanges = adaptivePlan.collect { case e: Exchange => e } - assert(exchanges.isEmpty, "The final plan should not contain any Exchange node.") - (dfAdaptive.queryExecution.sparkPlan, adaptivePlan) - } - - private def broadcastHashJoinSize(plan: SparkPlan): Int = { - findTopLevelBroadcastHashJoinTransform(plan).size + findTopLevelBroadcastHashJoin(plan).size - } - - private def findTopLevelBroadcastHashJoinTransform( - plan: SparkPlan): Seq[BroadcastHashJoinExecTransformerBase] = { - collect(plan) { case j: BroadcastHashJoinExecTransformerBase => j } - } - - private def findTopLevelBroadcastHashJoin(plan: SparkPlan): Seq[BroadcastHashJoinExec] = { - collect(plan) { case j: BroadcastHashJoinExec => j } - } - - private def findTopLevelSortMergeJoin(plan: SparkPlan): Seq[SortMergeJoinExec] = { - collect(plan) { case j: SortMergeJoinExec => j } - } - - private def findTopLevelSortMergeJoinTransform( - plan: SparkPlan): Seq[SortMergeJoinExecTransformer] = { - collect(plan) { case j: SortMergeJoinExecTransformer => j } - } - - private def sortMergeJoinSize(plan: SparkPlan): Int = { - findTopLevelSortMergeJoinTransform(plan).size + findTopLevelSortMergeJoin(plan).size - } - - private def findTopLevelShuffledHashJoin(plan: SparkPlan): Seq[ShuffledHashJoinExec] = { - collect(plan) { case j: ShuffledHashJoinExec => j } - } - - private def findTopLevelShuffledHashJoinTransform( - plan: SparkPlan): Seq[ShuffledHashJoinExecTransformerBase] = { - collect(plan) { case j: ShuffledHashJoinExecTransformerBase => j } - } - - private def findTopLevelBaseJoin(plan: SparkPlan): Seq[BaseJoinExec] = { - collect(plan) { case j: BaseJoinExec => j } - } - - private def findTopLevelSort(plan: SparkPlan): Seq[SortExec] = { - collect(plan) { case s: SortExec => s } - } - - private def findTopLevelSortTransform(plan: SparkPlan): Seq[SortExecTransformer] = { - collect(plan) { case s: SortExecTransformer => s } - } - - private def findReusedExchange(plan: SparkPlan): Seq[ReusedExchangeExec] = { - collectWithSubqueries(plan) { - case ShuffleQueryStageExec(_, e: ReusedExchangeExec, _) => e - case BroadcastQueryStageExec(_, e: ReusedExchangeExec, _) => e - } - } - - private def findReusedSubquery(plan: SparkPlan): Seq[ReusedSubqueryExec] = { - collectWithSubqueries(plan) { case e: ReusedSubqueryExec => e } - } - - private def checkNumLocalShuffleReads( - plan: SparkPlan, - numShufflesWithoutLocalRead: Int = 0): Unit = { - val numShuffles = collect(plan) { case s: ShuffleQueryStageExec => s }.length - - val numLocalReads = collect(plan) { - case r: AQEShuffleReadExec if r.isLocalRead => r - } - // because columnar local reads cannot execute - numLocalReads.foreach { - r => - val rdd = r.executeColumnar() - val parts = rdd.partitions - assert(parts.forall(rdd.preferredLocations(_).nonEmpty)) - } - assert(numShuffles === (numLocalReads.length + numShufflesWithoutLocalRead)) - } - - testGluten("Change merge join to broadcast join") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300" - ) { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - "SELECT * FROM testData join testData2 ON key = a where value = '1'") - val smj = findTopLevelSortMergeJoin(plan) - assert(smj.size == 1) - val bhj = findTopLevelBroadcastHashJoinTransform(adaptivePlan) - assert(bhj.size == 1) - checkNumLocalShuffleReads(adaptivePlan) - } - } - - testGluten("Change broadcast join to merge join") { - withTable("t1", "t2") { - withSQLConf( - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "10000", - SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - SQLConf.SHUFFLE_PARTITIONS.key -> "1") { - sql("CREATE TABLE t1 USING PARQUET AS SELECT 1 c1") - sql("CREATE TABLE t2 USING PARQUET AS SELECT 1 c1") - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult(""" - |SELECT * FROM ( - | SELECT distinct c1 from t1 - | ) tmp1 JOIN ( - | SELECT distinct c1 from t2 - | ) tmp2 ON tmp1.c1 = tmp2.c1 - |""".stripMargin) - assert(broadcastHashJoinSize(plan) == 1) - assert(broadcastHashJoinSize(adaptivePlan) == 0) - assert(findTopLevelSortMergeJoinTransform(adaptivePlan).size == 1) - } - } - } - - testGluten("Reuse the parallelism of coalesced shuffle in local shuffle read") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300", - SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "10") { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - "SELECT * FROM testData join testData2 ON key = a where value = '1'") - - assert(sortMergeJoinSize(plan) == 1) - assert(broadcastHashJoinSize(adaptivePlan) == 1) - val localReads = collect(adaptivePlan) { - case read: AQEShuffleReadExec if read.isLocalRead => read - } - assert(localReads.length == 2) - } - } - - testGluten("Reuse the default parallelism in local shuffle read") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300", - SQLConf.COALESCE_PARTITIONS_ENABLED.key -> "false") { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - "SELECT * FROM testData join testData2 ON key = a where value = '1'") - val smj = findTopLevelSortMergeJoin(plan) - assert(smj.size == 1) - val bhj = findTopLevelBroadcastHashJoinTransform(adaptivePlan) - assert(bhj.size == 1) - val localReads = collect(adaptivePlan) { - case read: AQEShuffleReadExec if read.isLocalRead => read - } - assert(localReads.length == 2) - val localShuffleRDD0 = localReads(0) - .executeColumnar() - .asInstanceOf[ShuffledColumnarBatchRDD] - val localShuffleRDD1 = localReads(1) - .executeColumnar() - .asInstanceOf[ShuffledColumnarBatchRDD] - // the final parallelism is math.max(1, numReduces / numMappers): math.max(1, 5/2) = 2 - // and the partitions length is 2 * numMappers = 4 - assert(localShuffleRDD0.getPartitions.length == 4) - // the final parallelism is math.max(1, numReduces / numMappers): math.max(1, 5/2) = 2 - // and the partitions length is 2 * numMappers = 4 - assert(localShuffleRDD1.getPartitions.length == 4) - } - } - - testGluten("Empty stage coalesced to 1-partition RDD") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.COALESCE_PARTITIONS_ENABLED.key -> "true", - SQLConf.ADAPTIVE_OPTIMIZER_EXCLUDED_RULES.key -> AQEPropagateEmptyRelation.ruleName - ) { - val df1 = spark.range(10).withColumn("a", 'id) - val df2 = spark.range(10).withColumn("b", 'id) - withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") { - val testDf = df1 - .where('a > 10) - .join(df2.where('b > 10), Seq("id"), "left_outer") - .groupBy('a) - .count() - checkAnswer(testDf, Seq()) - val plan = testDf.queryExecution.executedPlan - assert(find(plan)(_.isInstanceOf[SortMergeJoinExecTransformer]).isDefined) - } - - withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "1") { - val testDf = df1 - .where('a > 10) - .join(df2.where('b > 10), Seq("id"), "left_outer") - .groupBy('a) - .count() - checkAnswer(testDf, Seq()) - val plan = testDf.queryExecution.executedPlan - assert(find(plan)(_.isInstanceOf[BroadcastHashJoinExecTransformerBase]).isDefined) - val coalescedReads = collect(plan) { case r: AQEShuffleReadExec => r } - assert(coalescedReads.length == 3, s"$plan") - coalescedReads.foreach(r => assert(r.isLocalRead || r.partitionSpecs.length == 1)) - } - } - } - - testGluten("Scalar subquery") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300") { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - "SELECT * FROM testData join testData2 ON key = a " + - "where value = (SELECT max(a) from testData3)") - assert(sortMergeJoinSize(plan) == 1) - assert(broadcastHashJoinSize(adaptivePlan) == 1) - checkNumLocalShuffleReads(adaptivePlan) - } - } - - testGluten("Scalar subquery in later stages") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300") { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - "SELECT * FROM testData join testData2 ON key = a " + - "where (value + a) = (SELECT max(a) from testData3)") - assert(sortMergeJoinSize(plan) == 1) - assert(broadcastHashJoinSize(adaptivePlan) == 1) - - checkNumLocalShuffleReads(adaptivePlan) - } - } - - testGluten("multiple joins") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300") { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - """ - |WITH t4 AS ( - | SELECT * FROM lowercaseData t2 JOIN testData3 t3 ON t2.n = t3.a where t2.n = '1' - |) - |SELECT * FROM testData - |JOIN testData2 t2 ON key = t2.a - |JOIN t4 ON t2.b = t4.a - |WHERE value = 1 - """.stripMargin) - assert(sortMergeJoinSize(plan) == 3) - assert(broadcastHashJoinSize(adaptivePlan) == 3) - - // A possible resulting query plan: - // BroadcastHashJoin - // +- BroadcastExchange - // +- LocalShuffleReader* - // +- ShuffleExchange - // +- BroadcastHashJoin - // +- BroadcastExchange - // +- LocalShuffleReader* - // +- ShuffleExchange - // +- LocalShuffleReader* - // +- ShuffleExchange - // +- BroadcastHashJoin - // +- LocalShuffleReader* - // +- ShuffleExchange - // +- BroadcastExchange - // +-LocalShuffleReader* - // +- ShuffleExchange - - // After applied the 'OptimizeShuffleWithLocalRead' rule, we can convert all the four - // shuffle read to local shuffle read in the bottom two 'BroadcastHashJoin'. - // For the top level 'BroadcastHashJoin', the probe side is not shuffle query stage - // and the build side shuffle query stage is also converted to local shuffle read. - checkNumLocalShuffleReads(adaptivePlan, 0) - } - } - - testGluten("multiple joins with aggregate") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300") { - val (plan, adaptivePlan) = - runAdaptiveAndVerifyResult(""" - |WITH t4 AS ( - | SELECT * FROM lowercaseData t2 JOIN ( - | select a, sum(b) from testData3 group by a - | ) t3 ON t2.n = t3.a where t2.n = '1' - |) - |SELECT * FROM testData - |JOIN testData2 t2 ON key = t2.a - |JOIN t4 ON t2.b = t4.a - |WHERE value = 1 - """.stripMargin) - assert(sortMergeJoinSize(plan) == 3) - assert(broadcastHashJoinSize(adaptivePlan) == 3) - - // A possible resulting query plan: - // BroadcastHashJoin - // +- BroadcastExchange - // +- LocalShuffleReader* - // +- ShuffleExchange - // +- BroadcastHashJoin - // +- BroadcastExchange - // +- LocalShuffleReader* - // +- ShuffleExchange - // +- LocalShuffleReader* - // +- ShuffleExchange - // +- BroadcastHashJoin - // +- LocalShuffleReader* - // +- ShuffleExchange - // +- BroadcastExchange - // +-HashAggregate - // +- CoalescedShuffleReader - // +- ShuffleExchange - - // The shuffle added by Aggregate can't apply local read. - checkNumLocalShuffleReads(adaptivePlan, 1) - } - } - - testGluten("multiple joins with aggregate 2") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "500") { - val (plan, adaptivePlan) = - runAdaptiveAndVerifyResult(""" - |WITH t4 AS ( - | SELECT * FROM lowercaseData t2 JOIN ( - | select a, max(b) b from testData2 group by a - | ) t3 ON t2.n = t3.b - |) - |SELECT * FROM testData - |JOIN testData2 t2 ON key = t2.a - |JOIN t4 ON value = t4.a - |WHERE value = 1 - """.stripMargin) - assert(sortMergeJoinSize(plan) == 3) - assert(broadcastHashJoinSize(adaptivePlan) == 3) - - // A possible resulting query plan: - // BroadcastHashJoin - // +- BroadcastExchange - // +- LocalShuffleReader* - // +- ShuffleExchange - // +- BroadcastHashJoin - // +- BroadcastExchange - // +- LocalShuffleReader* - // +- ShuffleExchange - // +- LocalShuffleReader* - // +- ShuffleExchange - // +- BroadcastHashJoin - // +- Filter - // +- HashAggregate - // +- CoalescedShuffleReader - // +- ShuffleExchange - // +- BroadcastExchange - // +-LocalShuffleReader* - // +- ShuffleExchange - - // The shuffle added by Aggregate can't apply local read. - checkNumLocalShuffleReads(adaptivePlan, 1) - } - } - - testGluten("Exchange reuse") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "20" - ) { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - "SELECT value FROM testData join testData2 ON key = a " + - "join (SELECT value v from testData join testData3 ON key = a) on value = v") - assert(sortMergeJoinSize(plan) == 3) - assert(broadcastHashJoinSize(adaptivePlan) == 2) - // There is still a SMJ, and its two shuffles can't apply local read. - checkNumLocalShuffleReads(adaptivePlan, 2) - // Even with local shuffle read, the query stage reuse can also work. - val ex = findReusedExchange(adaptivePlan) - assert(ex.size == 1) - } - } - - testGluten("Exchange reuse with subqueries") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300") { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - "SELECT a FROM testData join testData2 ON key = a " + - "where value = (SELECT max(a) from testData join testData2 ON key = a)") - assert(sortMergeJoinSize(plan) == 1) - assert(broadcastHashJoinSize(adaptivePlan) == 1) - checkNumLocalShuffleReads(adaptivePlan) - // // Even with local shuffle read, the query stage reuse can also work. - // gluten change the smj to bhj, stage is changed, so we cannot find the stage with old - // ReuseExchange from stageCache, then the reuse is removed - // https://github.com/apache/spark/pull/24706/ - // files#diff-ec42cd27662f3f528832c298a60fffa1d341feb04aa1d8c80044b70cbe0ebbfcR224 - // maybe vanilla spark should checkReuse rile again - // val ex = findReusedExchange(adaptivePlan) - // assert(ex.size == 1) - } - } - - testGluten("Exchange reuse across subqueries") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300", - SQLConf.SUBQUERY_REUSE_ENABLED.key -> "false") { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - "SELECT a FROM testData join testData2 ON key = a " + - "where value >= (SELECT max(a) from testData join testData2 ON key = a) " + - "and a <= (SELECT max(a) from testData join testData2 ON key = a)") - assert(sortMergeJoinSize(plan) == 1) - assert(broadcastHashJoinSize(adaptivePlan) == 1) - checkNumLocalShuffleReads(adaptivePlan) - // Even with local shuffle read, the query stage reuse can also work. - val ex = findReusedExchange(adaptivePlan) - assert(ex.nonEmpty) - val sub = findReusedSubquery(adaptivePlan) - assert(sub.isEmpty) - } - } - - testGluten("Subquery reuse") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300") { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - "SELECT a FROM testData join testData2 ON key = a " + - "where value >= (SELECT max(a) from testData join testData2 ON key = a) " + - "and a <= (SELECT max(a) from testData join testData2 ON key = a)") - assert(sortMergeJoinSize(plan) == 1) - assert(broadcastHashJoinSize(adaptivePlan) == 1) - checkNumLocalShuffleReads(adaptivePlan) - // Even with local shuffle read, the query stage reuse can also work. - val ex = findReusedExchange(adaptivePlan) - assert(ex.isEmpty) - val sub = findReusedSubquery(adaptivePlan) - assert(sub.nonEmpty) - } - } - - testGluten("Broadcast exchange reuse across subqueries") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "20000000", - SQLConf.SUBQUERY_REUSE_ENABLED.key -> "false") { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - "SELECT a FROM testData join testData2 ON key = a " + - "where value >= (" + - "SELECT /*+ broadcast(testData2) */ max(key) from testData join testData2 ON key = a) " + - "and a <= (" + - "SELECT /*+ broadcast(testData2) */ max(value) from testData join testData2 ON key = a)") - assert(sortMergeJoinSize(plan) == 1) - assert(broadcastHashJoinSize(adaptivePlan) == 1) - checkNumLocalShuffleReads(adaptivePlan) - // Even with local shuffle read, the query stage reuse can also work. - val ex = findReusedExchange(adaptivePlan) - assert(ex.nonEmpty) - assert(ex.head.child.isInstanceOf[ColumnarBroadcastExchangeExec]) - val sub = findReusedSubquery(adaptivePlan) - assert(sub.isEmpty) - } - } - - // Cost is equal, not test cost is greater, need new test, but other test may contain cost change, - // so it maybe not essential - testGluten("Avoid plan change if cost is greater") {} - - testGluten("Change merge join to broadcast join without local shuffle read") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.LOCAL_SHUFFLE_READER_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300") { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - """ - |SELECT * FROM testData t1 join testData2 t2 - |ON t1.key = t2.a join testData3 t3 on t2.a = t3.a - |where t1.value = 1 - """.stripMargin - ) - assert(sortMergeJoinSize(plan) == 2) - val bhj = findTopLevelBroadcastHashJoinTransform(adaptivePlan) - assert(bhj.size == 2) - // There is still a SMJ, and its two shuffles can't apply local read. - checkNumLocalShuffleReads(adaptivePlan, 0) - } - } - - testGluten( - "Avoid changing merge join to broadcast join if too many empty partitions " + - "on build plan") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.NON_EMPTY_PARTITION_RATIO_FOR_BROADCAST_JOIN.key -> "0.5", - // this config will make some empty partitions - SQLConf.SHUFFLE_PARTITIONS.key -> "5" - ) { - // `testData` is small enough to be broadcast but has empty partition ratio over the config. - // because testData2 in gluten sizeInBytes(from ColumnarShuffleExchangeExec plan stats) - // is 24B sometimes, so change the threshold from 80 to 20 - withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "20") { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - "SELECT * FROM testData join testData2 ON key = a where value = '1'") - assert(sortMergeJoinSize(plan) == 1) - val bhj = findTopLevelBroadcastHashJoinTransform(adaptivePlan) - assert(bhj.isEmpty) - } - // It is still possible to broadcast `testData2`. - withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "2000") { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( - "SELECT * FROM testData join testData2 ON key = a where value = '1'") - assert(sortMergeJoinSize(plan) == 1) - val bhj = findTopLevelBroadcastHashJoinTransform(adaptivePlan) - assert(bhj.size == 1) - assert(bhj.head.joinBuildSide == BuildRight) - } - } - } - - testGluten("SPARK-30524: Do not optimize skew join if introduce additional shuffle") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "100", - SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "100" - ) { - withTempView("skewData1", "skewData2") { - spark - .range(0, 1000, 1, 10) - .selectExpr("id % 3 as key1", "id as value1") - .createOrReplaceTempView("skewData1") - spark - .range(0, 1000, 1, 10) - .selectExpr("id % 1 as key2", "id as value2") - .createOrReplaceTempView("skewData2") - - def checkSkewJoin(query: String, optimizeSkewJoin: Boolean): Unit = { - val (_, innerAdaptivePlan) = runAdaptiveAndVerifyResult(query) - val innerSmj = findTopLevelSortMergeJoinTransform(innerAdaptivePlan) - assert(innerSmj.size == 1 && innerSmj.head.isSkewJoin == optimizeSkewJoin) - } - - // OptimizeSkewedJoin check the map status, because the - checkSkewJoin("SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2", true) - // Additional shuffle introduced, so disable the "OptimizeSkewedJoin" optimization - checkSkewJoin( - "SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2 GROUP BY key1", - false) - } - } - } - - testGluten("SPARK-29544: adaptive skew join with different join types") { - Seq("SHUFFLE_MERGE", "SHUFFLE_HASH").foreach { - joinHint => - def getJoinNode(plan: SparkPlan): Seq[BinaryExecNode] = if (joinHint == "SHUFFLE_MERGE") { - findTopLevelSortMergeJoinTransform(plan) - } else { - findTopLevelShuffledHashJoinTransform(plan) - } - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM.key -> "1", - SQLConf.SHUFFLE_PARTITIONS.key -> "100", - SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "800", - SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "800" - ) { - withTempView("skewData1", "skewData2") { - spark - .range(0, 1000, 1, 10) - .select( - when('id < 250, 249) - .when('id >= 750, 1000) - .otherwise('id) - .as("key1"), - 'id.as("value1")) - .createOrReplaceTempView("skewData1") - spark - .range(0, 1000, 1, 10) - .select( - when('id < 250, 249) - .otherwise('id) - .as("key2"), - 'id.as("value2")) - .createOrReplaceTempView("skewData2") - - def checkSkewJoin( - joins: Seq[BinaryExecNode], - leftSkewNum: Int, - rightSkewNum: Int): Unit = { - assert(joins.size == 1) - joins.head match { - case s: SortMergeJoinExecTransformer => assert(s.isSkewJoin) - case g: ShuffledHashJoinExecTransformerBase => assert(g.isSkewJoin) - case _ => assert(false) - } - assert( - joins.head.left - .collect { case r: AQEShuffleReadExec => r } - .head - .partitionSpecs - .collect { case p: PartialReducerPartitionSpec => p.reducerIndex } - .distinct - .length == leftSkewNum) - assert( - joins.head.right - .collect { case r: AQEShuffleReadExec => r } - .head - .partitionSpecs - .collect { case p: PartialReducerPartitionSpec => p.reducerIndex } - .distinct - .length == rightSkewNum) - } - - // skewed inner join optimization - val (_, innerAdaptivePlan) = runAdaptiveAndVerifyResult( - s"SELECT /*+ $joinHint(skewData1) */ * FROM skewData1 " + - "JOIN skewData2 ON key1 = key2") - val inner = getJoinNode(innerAdaptivePlan) - // checkSkewJoin(inner, 2, 1) - - // skewed left outer join optimization - val (_, leftAdaptivePlan) = runAdaptiveAndVerifyResult( - s"SELECT /*+ $joinHint(skewData2) */ * FROM skewData1 " + - "LEFT OUTER JOIN skewData2 ON key1 = key2") - val leftJoin = getJoinNode(leftAdaptivePlan) - // checkSkewJoin(leftJoin, 2, 0) - - // skewed right outer join optimization - val (_, rightAdaptivePlan) = runAdaptiveAndVerifyResult( - s"SELECT /*+ $joinHint(skewData1) */ * FROM skewData1 " + - "RIGHT OUTER JOIN skewData2 ON key1 = key2") - val rightJoin = getJoinNode(rightAdaptivePlan) - // checkSkewJoin(rightJoin, 0, 1) - } - } - } - } - - testGluten("SPARK-34682: AQEShuffleReadExec operating on canonicalized plan") { - withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true") { - val (_, adaptivePlan) = runAdaptiveAndVerifyResult("SELECT key FROM testData GROUP BY key") - val reads = collect(adaptivePlan) { case r: AQEShuffleReadExec => r } - assert(reads.length == 1) - val read = reads.head - val c = read.canonicalized.asInstanceOf[AQEShuffleReadExec] - // we can't just call execute() because that has separate checks for canonicalized plans - val ex = intercept[IllegalStateException] { - val doExecute = PrivateMethod[Unit](Symbol("doExecuteColumnar")) - c.invokePrivate(doExecute()) - } - assert(ex.getMessage === "operating on canonicalized plan") - } - } - - testGluten("metrics of the shuffle read") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.SHUFFLE_PARTITIONS.key -> "5") { - val (_, adaptivePlan) = runAdaptiveAndVerifyResult("SELECT key FROM testData GROUP BY key") - val reads = collect(adaptivePlan) { case r: AQEShuffleReadExec => r } - assert(reads.length == 1) - val read = reads.head - assert(!read.isLocalRead) - assert(!read.hasSkewedPartition) - assert(read.hasCoalescedPartition) - assert( - read.metrics.keys.toSeq.sorted == Seq( - "numCoalescedPartitions", - "numPartitions", - "partitionDataSize")) - assert(read.metrics("numCoalescedPartitions").value == 1) - assert(read.metrics("numPartitions").value == read.partitionSpecs.length) - assert(read.metrics("partitionDataSize").value > 0) - - // Gluten has smaller shuffle data size, and the right side is materialized before the left - // side. Need to lower the threshold to avoid the planner broadcasting the right side first. - withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "40") { - val (_, adaptivePlan) = runAdaptiveAndVerifyResult( - "SELECT * FROM testData join testData2 ON key = a where value = '1'") - val join = collect(adaptivePlan) { case j: BroadcastHashJoinExecTransformerBase => j }.head - assert(join.joinBuildSide == BuildLeft) - - val reads = collect(join.right) { case r: AQEShuffleReadExec => r } - assert(reads.length == 1) - val read = reads.head - assert(read.isLocalRead) - assert(read.metrics.keys.toSeq == Seq("numPartitions")) - assert(read.metrics("numPartitions").value == read.partitionSpecs.length) - } - - withSQLConf( - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - SQLConf.SHUFFLE_PARTITIONS.key -> "100", - SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "800", - SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "1000" - ) { - withTempView("skewData1", "skewData2") { - spark - .range(0, 1000, 1, 10) - .select( - when('id < 250, 249) - .when('id >= 750, 1000) - .otherwise('id) - .as("key1"), - 'id.as("value1")) - .createOrReplaceTempView("skewData1") - spark - .range(0, 1000, 1, 10) - .select( - when('id < 250, 249) - .otherwise('id) - .as("key2"), - 'id.as("value2")) - .createOrReplaceTempView("skewData2") - val (_, adaptivePlan) = - runAdaptiveAndVerifyResult("SELECT * FROM skewData1 join skewData2 ON key1 = key2") - } - } - } - } - - // because gluten use columnar format, which cannot execute to get rowIterator, then get the key - // null status - ignore( - GLUTEN_TEST + "SPARK-32573: Eliminate NAAJ when BuildSide is HashedRelationWithAllNullKeys") {} - - // EmptyRelation case - ignore( - GLUTEN_TEST + "SPARK-35455: Unify empty relation optimization " + - "between normal and AQE optimizer - single join") {} - - testGluten("SPARK-32753: Only copy tags to node with no tags") { - withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true") { - withTempView("v1") { - spark.range(10).union(spark.range(10)).createOrReplaceTempView("v1") - - val (_, adaptivePlan) = - runAdaptiveAndVerifyResult("SELECT id FROM v1 GROUP BY id DISTRIBUTE BY id") - assert(collect(adaptivePlan) { case s: ColumnarShuffleExchangeExec => s }.length == 1) - } - } - } - - testGluten("Logging plan changes for AQE") { - val testAppender = new LogAppender("plan changes") - withLogAppender(testAppender) { - withSQLConf( - // this test default level is WARN, so we should check warn level - SQLConf.PLAN_CHANGE_LOG_LEVEL.key -> "WARN", - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80" - ) { - sql( - "SELECT * FROM testData JOIN testData2 ON key = a " + - "WHERE value = (SELECT max(a) FROM testData3)").collect() - } - Seq( - "=== Result of Batch AQE Preparations ===", - "=== Result of Batch AQE Post Stage Creation ===", - "=== Result of Batch AQE Replanning ===", - "=== Result of Batch AQE Query Stage Optimization ===" - ).foreach { - expectedMsg => - assert( - testAppender.loggingEvents.exists( - _.getMessage.getFormattedMessage.contains(expectedMsg))) - } - } - } - - testGluten("SPARK-33551: Do not use AQE shuffle read for repartition") { - def hasRepartitionShuffle(plan: SparkPlan): Boolean = { - find(plan) { - case s: ShuffleExchangeLike => - s.shuffleOrigin == REPARTITION_BY_COL || s.shuffleOrigin == REPARTITION_BY_NUM - case _ => false - }.isDefined - } - - def checkBHJ( - df: Dataset[Row], - optimizeOutRepartition: Boolean, - probeSideLocalRead: Boolean, - probeSideCoalescedRead: Boolean): Unit = { - df.collect() - val plan = df.queryExecution.executedPlan - // There should be only one shuffle that can't do local read, which is either the top shuffle - // from repartition, or BHJ probe side shuffle. - checkNumLocalShuffleReads(plan, 1) - assert(hasRepartitionShuffle(plan) == !optimizeOutRepartition) - val bhj = findTopLevelBroadcastHashJoinTransform(plan) - assert(bhj.length == 1) - - // Build side should do local read. - val buildSide = find(bhj.head.left)(_.isInstanceOf[AQEShuffleReadExec]) - assert(buildSide.isDefined) - assert(buildSide.get.asInstanceOf[AQEShuffleReadExec].isLocalRead) - - val probeSide = find(bhj.head.right)(_.isInstanceOf[AQEShuffleReadExec]) - if (probeSideLocalRead || probeSideCoalescedRead) { - assert(probeSide.isDefined) - if (probeSideLocalRead) { - assert(probeSide.get.asInstanceOf[AQEShuffleReadExec].isLocalRead) - } else { - assert(probeSide.get.asInstanceOf[AQEShuffleReadExec].hasCoalescedPartition) - } - } else { - assert(probeSide.isEmpty) - } - } - - def checkSMJ( - df: Dataset[Row], - optimizeOutRepartition: Boolean, - optimizeSkewJoin: Boolean, - coalescedRead: Boolean): Unit = { - df.collect() - val plan = df.queryExecution.executedPlan - assert(hasRepartitionShuffle(plan) == !optimizeOutRepartition) - val smj = findTopLevelSortMergeJoinTransform(plan) - assert(smj.length == 1) - assert(smj.head.isSkewJoin == optimizeSkewJoin) - val aqeReads = collect(smj.head) { case c: AQEShuffleReadExec => c } - if (coalescedRead || optimizeSkewJoin) { - assert(aqeReads.length == 2) - if (coalescedRead) assert(aqeReads.forall(_.hasCoalescedPartition)) - } else { - assert(aqeReads.isEmpty) - } - } - - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.SHUFFLE_PARTITIONS.key -> "5") { - val df = sql(""" - |SELECT * FROM ( - | SELECT * FROM testData WHERE key = 1 - |) - |RIGHT OUTER JOIN testData2 - |ON value = b - """.stripMargin) - - withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300") { - // Repartition with no partition num specified. - checkBHJ( - df.repartition('b), - // The top shuffle from repartition is optimized out. - optimizeOutRepartition = true, - probeSideLocalRead = false, - probeSideCoalescedRead = true - ) - - // Repartition with default partition num (5 in test env) specified. - checkBHJ( - df.repartition(5, 'b), - // The top shuffle from repartition is optimized out - // The final plan must have 5 partitions, no optimization can be made to the probe side. - optimizeOutRepartition = true, - probeSideLocalRead = false, - probeSideCoalescedRead = false - ) - - // Repartition with non-default partition num specified. - checkBHJ( - df.repartition(4, 'b), - // The top shuffle from repartition is not optimized out - optimizeOutRepartition = false, - probeSideLocalRead = true, - probeSideCoalescedRead = true - ) - - // Repartition by col and project away the partition cols - checkBHJ( - df.repartition('b).select('key), - // The top shuffle from repartition is not optimized out - optimizeOutRepartition = false, - probeSideLocalRead = true, - probeSideCoalescedRead = true - ) - } - - // Force skew join - withSQLConf( - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - SQLConf.SKEW_JOIN_ENABLED.key -> "true", - SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "1", - SQLConf.SKEW_JOIN_SKEWED_PARTITION_FACTOR.key -> "0", - SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "10" - ) { - // Repartition with no partition num specified. - checkSMJ( - df.repartition('b), - // The top shuffle from repartition is optimized out. - optimizeOutRepartition = true, - optimizeSkewJoin = false, - coalescedRead = true) - - // Repartition with default partition num (5 in test env) specified. - checkSMJ( - df.repartition(5, 'b), - // The top shuffle from repartition is optimized out. - // The final plan must have 5 partitions, can't do coalesced read. - optimizeOutRepartition = true, - optimizeSkewJoin = false, - coalescedRead = false - ) - - // Repartition with non-default partition num specified. - checkSMJ( - df.repartition(4, 'b), - // The top shuffle from repartition is not optimized out. - optimizeOutRepartition = false, - optimizeSkewJoin = true, - coalescedRead = false - ) - - // Repartition by col and project away the partition cols - checkSMJ( - df.repartition('b).select('key), - // The top shuffle from repartition is not optimized out. - optimizeOutRepartition = false, - optimizeSkewJoin = true, - coalescedRead = false - ) - } - } - } - - testGluten("SPARK-34091: Batch shuffle fetch in AQE partition coalescing") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.SHUFFLE_PARTITIONS.key -> "10", - SQLConf.FETCH_SHUFFLE_BLOCKS_IN_BATCH.key -> "true") { - withTable("t1") { - spark.range(100).selectExpr("id + 1 as a").write.format("parquet").saveAsTable("t1") - val query = "SELECT SUM(a) FROM t1 GROUP BY a" - val (_, adaptivePlan) = runAdaptiveAndVerifyResult(query) - val metricName = SQLShuffleReadMetricsReporter.LOCAL_BLOCKS_FETCHED - val blocksFetchedMetric = collectFirst(adaptivePlan) { - case p if p.metrics.contains(metricName) => p.metrics(metricName) - } - assert(blocksFetchedMetric.isDefined) - val blocksFetched = blocksFetchedMetric.get.value - withSQLConf(SQLConf.FETCH_SHUFFLE_BLOCKS_IN_BATCH.key -> "false") { - val (_, adaptivePlan2) = runAdaptiveAndVerifyResult(query) - val blocksFetchedMetric2 = collectFirst(adaptivePlan2) { - case p if p.metrics.contains(metricName) => p.metrics(metricName) - } - assert(blocksFetchedMetric2.isDefined) - val blocksFetched2 = blocksFetchedMetric2.get.value - assert(blocksFetched == blocksFetched2) - } - } - } - } - - testGluten("SPARK-34899: Use origin plan if we can not coalesce shuffle partition") { - def checkNoCoalescePartitions(ds: Dataset[Row], origin: ShuffleOrigin): Unit = { - assert(collect(ds.queryExecution.executedPlan) { - case s: ShuffleExchangeExec if s.shuffleOrigin == origin && s.numPartitions == 2 => s - }.size == 1) - ds.collect() - val plan = ds.queryExecution.executedPlan - assert(collect(plan) { - case s: ColumnarShuffleExchangeExec if s.shuffleOrigin == origin && s.numPartitions == 2 => - s - }.size == 1) - checkAnswer(ds, testData) - } - - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.COALESCE_PARTITIONS_ENABLED.key -> "true", - // Pick a small value so that no coalesce can happen. - SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "100", - SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM.key -> "1", - SQLConf.SHUFFLE_PARTITIONS.key -> "2" - ) { - val df = - spark.sparkContext.parallelize((1 to 100).map(i => TestData(i, i.toString)), 10).toDF() - - // partition size [1420, 1420] - checkNoCoalescePartitions(df.repartition($"key"), REPARTITION_BY_COL) - // partition size [1140, 1119] - checkNoCoalescePartitions(df.sort($"key"), ENSURE_REQUIREMENTS) - } - } - - testGluten("SPARK-35239: Coalesce shuffle partition should handle empty input RDD") { - withTable("t") { - withSQLConf( - SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM.key -> "1", - SQLConf.SHUFFLE_PARTITIONS.key -> "2", - SQLConf.ADAPTIVE_OPTIMIZER_EXCLUDED_RULES.key -> AQEPropagateEmptyRelation.ruleName - ) { - spark.sql("CREATE TABLE t (c1 int) USING PARQUET") - val (_, adaptive) = runAdaptiveAndVerifyResult("SELECT c1, count(*) FROM t GROUP BY c1") - assert( - collect(adaptive) { - case c @ AQEShuffleReadExec(_, partitionSpecs) if partitionSpecs.length == 1 => - assert(c.hasCoalescedPartition) - c - }.length == 1 - ) - } - } - } - - testGluten("SPARK-35264: Support AQE side broadcastJoin threshold") { - withTempView("t1", "t2") { - def checkJoinStrategy(shouldBroadcast: Boolean): Unit = { - withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") { - val (origin, adaptive) = - runAdaptiveAndVerifyResult("SELECT t1.c1, t2.c1 FROM t1 JOIN t2 ON t1.c1 = t2.c1") - assert(findTopLevelSortMergeJoin(origin).size == 1) - if (shouldBroadcast) { - assert(findTopLevelBroadcastHashJoinTransform(adaptive).size == 1) - } else { - assert(findTopLevelSortMergeJoinTransform(adaptive).size == 1) - } - } - } - - // t1: 1600 bytes - // t2: 160 bytes - spark.sparkContext - .parallelize((1 to 100).map(i => TestData(i, i.toString)), 10) - .toDF("c1", "c2") - .createOrReplaceTempView("t1") - spark.sparkContext - .parallelize((1 to 10).map(i => TestData(i, i.toString)), 5) - .toDF("c1", "c2") - .createOrReplaceTempView("t2") - - checkJoinStrategy(false) - withSQLConf(SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") { - checkJoinStrategy(false) - } - - withSQLConf(SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "400") { - checkJoinStrategy(true) - } - } - } - - // table partition size is different with spark - testGluten("SPARK-35264: Support AQE side shuffled hash join formula") { - withTempView("t1", "t2") { - def checkJoinStrategy(shouldShuffleHashJoin: Boolean): Unit = { - Seq("100", "100000").foreach { - size => - withSQLConf(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> size) { - val (origin1, adaptive1) = - runAdaptiveAndVerifyResult("SELECT t1.c1, t2.c1 FROM t1 JOIN t2 ON t1.c1 = t2.c1") - assert(findTopLevelSortMergeJoin(origin1).size === 1) - if (shouldShuffleHashJoin && size.toInt < 100000) { - val shj = findTopLevelShuffledHashJoinTransform(adaptive1) - assert(shj.size === 1) - assert(shj.head.joinBuildSide == BuildRight) - } else { - assert(findTopLevelSortMergeJoinTransform(adaptive1).size === 1) - } - } - } - // respect user specified join hint - val (origin2, adaptive2) = runAdaptiveAndVerifyResult( - "SELECT /*+ MERGE(t1) */ t1.c1, t2.c1 FROM t1 JOIN t2 ON t1.c1 = t2.c1") - assert(findTopLevelSortMergeJoin(origin2).size === 1) - assert(findTopLevelSortMergeJoinTransform(adaptive2).size === 1) - } - - spark.sparkContext - .parallelize((1 to 100).map(i => TestData(i, i.toString)), 10) - .toDF("c1", "c2") - .createOrReplaceTempView("t1") - spark.sparkContext - .parallelize((1 to 10).map(i => TestData(i, i.toString)), 5) - .toDF("c1", "c2") - .createOrReplaceTempView("t2") - - // t1 partition size: [395, 316, 313] - // t2 partition size: [140, 50, 0] - withSQLConf( - SQLConf.SHUFFLE_PARTITIONS.key -> "3", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - SQLConf.PREFER_SORTMERGEJOIN.key -> "true") { - // check default value - checkJoinStrategy(false) - // t1 no hint. - // t2 partition size are all smaller than 200, t2 has SHJ hint. The result is true. - withSQLConf(SQLConf.ADAPTIVE_MAX_SHUFFLE_HASH_JOIN_LOCAL_MAP_THRESHOLD.key -> "200") { - checkJoinStrategy(true) - } - // t1 no hint. - // Not all partition size of t2 are smaller than 100, t2 no hint. The result is false. - withSQLConf(SQLConf.ADAPTIVE_MAX_SHUFFLE_HASH_JOIN_LOCAL_MAP_THRESHOLD.key -> "100") { - checkJoinStrategy(false) - } - // t1, t2 partition size are all smaller than 1000, t1 and t2 can use SHJ. - // The result is true. - withSQLConf(SQLConf.ADAPTIVE_MAX_SHUFFLE_HASH_JOIN_LOCAL_MAP_THRESHOLD.key -> "1000") { - checkJoinStrategy(true) - } - } - } - } - - testGluten("SPARK-32932: Do not use local shuffle read at final stage on write command") { - withSQLConf( - SQLConf.PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.DYNAMIC.toString, - SQLConf.SHUFFLE_PARTITIONS.key -> "5", - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true" - ) { - val data = - for ( - i <- 1L to 10L; - j <- 1L to 3L - ) yield (i, j) - - val df = data.toDF("i", "j").repartition($"j") - var noLocalread: Boolean = false - val listener = new QueryExecutionListener { - override def onSuccess(funcName: String, qe: QueryExecution, durationNs: Long): Unit = { - qe.executedPlan match { - case plan @ (_: DataWritingCommandExec | _: V2TableWriteExec) => - noLocalread = collect(plan) { - case exec: AQEShuffleReadExec if exec.isLocalRead => exec - }.isEmpty - case _ => // ignore other events - } - } - override def onFailure( - funcName: String, - qe: QueryExecution, - exception: Exception): Unit = {} - } - spark.listenerManager.register(listener) - - withTable("t") { - df.write.partitionBy("j").saveAsTable("t") - sparkContext.listenerBus.waitUntilEmpty() - assert(noLocalread) - noLocalread = false - } - - // Test DataSource v2 - val format = classOf[NoopDataSource].getName - df.write.format(format).mode("overwrite").save() - sparkContext.listenerBus.waitUntilEmpty() - assert(noLocalread) - noLocalread = false - - spark.listenerManager.unregister(listener) - } - } - - testGluten( - "SPARK-30953: InsertAdaptiveSparkPlan should apply AQE on child plan of v2 write commands") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.ADAPTIVE_EXECUTION_FORCE_APPLY.key -> "true") { - var plan: SparkPlan = null - val listener = new QueryExecutionListener { - override def onSuccess(funcName: String, qe: QueryExecution, durationNs: Long): Unit = { - plan = qe.executedPlan - } - override def onFailure( - funcName: String, - qe: QueryExecution, - exception: Exception): Unit = {} - } - spark.listenerManager.register(listener) - withTable("t1") { - val format = classOf[NoopDataSource].getName - Seq((0, 1)).toDF("x", "y").write.format(format).mode("overwrite").save() - - sparkContext.listenerBus.waitUntilEmpty() - assert(plan.isInstanceOf[V2TableWriteExec]) - val childPlan = plan.asInstanceOf[V2TableWriteExec].child - assert(childPlan.isInstanceOf[ColumnarToCarrierRowExecBase]) - assert( - childPlan - .asInstanceOf[ColumnarToCarrierRowExecBase] - .child - .isInstanceOf[AdaptiveSparkPlanExec]) - - spark.listenerManager.unregister(listener) - } - } - } - - testGluten("SPARK-35650: Coalesce number of partitions by AEQ") { - withSQLConf(SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM.key -> "1") { - Seq("REPARTITION", "REBALANCE(key)") - .foreach { - repartition => - val query = s"SELECT /*+ $repartition */ * FROM testData" - val (_, adaptivePlan) = runAdaptiveAndVerifyResult(query) - collect(adaptivePlan) { case r: AQEShuffleReadExec => r } match { - case Seq(aqeShuffleRead) => - assert(aqeShuffleRead.partitionSpecs.size === 1) - assert(!aqeShuffleRead.isLocalRead) - case _ => - fail("There should be a AQEShuffleReadExec") - } - } - } - } - - testGluten("SPARK-35650: Use local shuffle read if can not coalesce number of partitions") { - withSQLConf(SQLConf.COALESCE_PARTITIONS_ENABLED.key -> "false") { - val query = "SELECT /*+ REPARTITION */ * FROM testData" - val (_, adaptivePlan) = runAdaptiveAndVerifyResult(query) - collect(adaptivePlan) { case r: AQEShuffleReadExec => r } match { - case Seq(aqeShuffleRead) => - assert(aqeShuffleRead.partitionSpecs.size === 4) - assert(aqeShuffleRead.isLocalRead) - case _ => - fail("There should be a AQEShuffleReadExec") - } - } - } - - testGluten("SPARK-35725: Support optimize skewed partitions in RebalancePartitions") { - withTempView("v") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.COALESCE_PARTITIONS_ENABLED.key -> "true", - SQLConf.ADAPTIVE_OPTIMIZE_SKEWS_IN_REBALANCE_PARTITIONS_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - SQLConf.SHUFFLE_PARTITIONS.key -> "5", - SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM.key -> "1" - ) { - - spark.sparkContext - .parallelize((1 to 10).map(i => TestData(if (i > 4) 5 else i, i.toString)), 3) - .toDF("c1", "c2") - .createOrReplaceTempView("v") - - def checkPartitionNumber( - query: String, - skewedPartitionNumber: Int, - totalNumber: Int): Unit = { - val (_, adaptive) = runAdaptiveAndVerifyResult(query) - val read = collect(adaptive) { case read: AQEShuffleReadExec => read } - assert(read.size == 1) - assert( - read.head.partitionSpecs.count(_.isInstanceOf[PartialReducerPartitionSpec]) == - skewedPartitionNumber) - assert(read.head.partitionSpecs.size == totalNumber) - } - - // Changed ADVISORY_PARTITION_SIZE_IN_BYTES from 150 to 120 because Gluten has smaller - // partition size. - withSQLConf(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "120") { - // partition size [0,208,54,54,54] - checkPartitionNumber("SELECT /*+ REBALANCE(c1) */ * FROM v", 2, 4) - // partition size [108, 54, 60, 108, 54, 108, 54] - checkPartitionNumber("SELECT /*+ REBALANCE */ * FROM v", 6, 7) - } - - // no skewed partition should be optimized - withSQLConf(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "10000") { - checkPartitionNumber("SELECT /*+ REBALANCE(c1) */ * FROM v", 0, 1) - } - } - } - } - - testGluten("SPARK-35888: join with a 0-partition table") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM.key -> "1", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - SQLConf.ADAPTIVE_OPTIMIZER_EXCLUDED_RULES.key -> AQEPropagateEmptyRelation.ruleName - ) { - withTempView("t2") { - // create a temp view with 0 partition - spark - .createDataFrame(sparkContext.emptyRDD[Row], new StructType().add("b", IntegerType)) - .createOrReplaceTempView("t2") - val (_, adaptive) = - runAdaptiveAndVerifyResult("SELECT * FROM testData2 t1 left semi join t2 ON t1.a=t2.b") - val aqeReads = collect(adaptive) { case c: AQEShuffleReadExec => c } - assert(aqeReads.length == 2) - aqeReads.foreach { - c => - val stats = c.child.asInstanceOf[QueryStageExec].getRuntimeStatistics - assert(stats.sizeInBytes >= 0) - assert(stats.rowCount.get >= 0) - } - } - } - } - - testGluten("SPARK-35968: AQE coalescing should not produce too small partitions by default") { - withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true") { - val (_, adaptive) = - runAdaptiveAndVerifyResult("SELECT sum(id) FROM RANGE(10) GROUP BY id % 3") - val coalesceRead = collect(adaptive) { - case r: AQEShuffleReadExec if r.hasCoalescedPartition => r - } - assert(coalesceRead.length == 1) - // RANGE(10) is a very small dataset and AQE coalescing should produce one partition. - assert(coalesceRead.head.partitionSpecs.length == 1) - } - } - - testGluten("SPARK-35794: Allow custom plugin for cost evaluator") { - CostEvaluator.instantiate( - classOf[SimpleShuffleSortCostEvaluator].getCanonicalName, - spark.sparkContext.getConf) - intercept[IllegalArgumentException] { - CostEvaluator.instantiate( - classOf[InvalidCostEvaluator].getCanonicalName, - spark.sparkContext.getConf) - } - - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300") { - val query = "SELECT * FROM testData join testData2 ON key = a where value = '1'" - - withSQLConf( - SQLConf.ADAPTIVE_CUSTOM_COST_EVALUATOR_CLASS.key -> - "org.apache.spark.sql.execution.adaptive.SimpleShuffleSortCostEvaluator") { - val (plan, adaptivePlan) = runAdaptiveAndVerifyResult(query) - val smj = findTopLevelSortMergeJoin(plan) - assert(smj.size == 1) - val bhj = findTopLevelBroadcastHashJoinTransform(adaptivePlan) - assert(bhj.size == 1) - checkNumLocalShuffleReads(adaptivePlan) - } - - withSQLConf( - SQLConf.ADAPTIVE_CUSTOM_COST_EVALUATOR_CLASS.key -> - "org.apache.spark.sql.execution.adaptive.InvalidCostEvaluator") { - intercept[IllegalArgumentException] { - runAdaptiveAndVerifyResult(query) - } - } - } - } - - testGluten("SPARK-36020: Check logical link in remove redundant projects") { - withTempView("t") { - spark - .range(10) - .selectExpr( - "id % 10 as key", - "cast(id * 2 as int) as a", - "cast(id * 3 as int) as b", - "array(id, id + 1, id + 3) as c") - .createOrReplaceTempView("t") - withSQLConf( - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "800") { - val query = - """ - |WITH tt AS ( - | SELECT key, a, b, explode(c) AS c FROM t - |) - |SELECT t1.key, t1.c, t2.key, t2.c - |FROM (SELECT a, b, c, key FROM tt WHERE a > 1) t1 - |JOIN (SELECT a, b, c, key FROM tt) t2 - | ON t1.key = t2.key - |""".stripMargin - val (origin, adaptive) = runAdaptiveAndVerifyResult(query) - assert(findTopLevelSortMergeJoin(origin).size == 1) - assert(findTopLevelBroadcastHashJoinTransform(adaptive).size == 1) - } - } - } - - testGluten( - "SPARK-36032: Use inputPlan instead of currentPhysicalPlan to initialize logical link") { - withTempView("v") { - spark.sparkContext - .parallelize((1 to 10).map(i => TestData(i, i.toString)), 2) - .toDF("c1", "c2") - .createOrReplaceTempView("v") - - Seq("-1", "10000").foreach { - aqeBhj => - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> aqeBhj, - SQLConf.SHUFFLE_PARTITIONS.key -> "1" - ) { - val (origin, adaptive) = runAdaptiveAndVerifyResult(""" - |SELECT * FROM v t1 JOIN ( - | SELECT c1 + 1 as c3 FROM v - |)t2 ON t1.c1 = t2.c3 - |SORT BY c1 - """.stripMargin) - if (aqeBhj.toInt < 0) { - // 1 sort since spark plan has no shuffle for SMJ - assert(findTopLevelSort(origin).size == 1) - // 2 sorts in SMJ - assert(findTopLevelSortTransform(adaptive).size == 2) - } else { - assert(findTopLevelSort(origin).size == 1) - // 1 sort at top node and BHJ has no sort - assert(findTopLevelSortTransform(adaptive).size == 1) - } - } - } - } - } - - testGluten("SPARK-37742: AQE reads invalid InMemoryRelation stats and mistakenly plans BHJ") { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "1048584", - SQLConf.ADAPTIVE_OPTIMIZER_EXCLUDED_RULES.key -> AQEPropagateEmptyRelation.ruleName - ) { - // Spark estimates a string column as 20 bytes so with 60k rows, these relations should be - // estimated at ~120m bytes which is greater than the broadcast join threshold. - val joinKeyOne = "00112233445566778899" - val joinKeyTwo = "11223344556677889900" - Seq - .fill(60000)(joinKeyOne) - .toDF("key") - .createOrReplaceTempView("temp") - Seq - .fill(60000)(joinKeyTwo) - .toDF("key") - .createOrReplaceTempView("temp2") - - Seq(joinKeyOne).toDF("key").createOrReplaceTempView("smallTemp") - spark.sql("SELECT key as newKey FROM temp").persist() - - // This query is trying to set up a situation where there are three joins. - // The first join will join the cached relation with a smaller relation. - // The first join is expected to be a broadcast join since the smaller relation will - // fit under the broadcast join threshold. - // The second join will join the first join with another relation and is expected - // to remain as a sort-merge join. - // The third join will join the cached relation with another relation and is expected - // to remain as a sort-merge join. - val query = - s""" - |SELECT t3.newKey - |FROM - | (SELECT t1.newKey - | FROM (SELECT key as newKey FROM temp) as t1 - | JOIN - | (SELECT key FROM smallTemp) as t2 - | ON t1.newKey = t2.key - | ) as t3 - | JOIN - | (SELECT key FROM temp2) as t4 - | ON t3.newKey = t4.key - |UNION - |SELECT t1.newKey - |FROM - | (SELECT key as newKey FROM temp) as t1 - | JOIN - | (SELECT key FROM temp2) as t2 - | ON t1.newKey = t2.key - |""".stripMargin - val df = spark.sql(query) - df.collect() - val adaptivePlan = df.queryExecution.executedPlan - val bhj = findTopLevelBroadcastHashJoinTransform(adaptivePlan) - assert(bhj.length == 1) - } - } - - testGluten("test log level") { - def verifyLog(expectedLevel: Level): Unit = { - val logAppender = new LogAppender("adaptive execution") - logAppender.setThreshold(expectedLevel) - withLogAppender( - logAppender, - loggerNames = Seq(AdaptiveSparkPlanExec.getClass.getName.dropRight(1)), - level = Some(Level.TRACE)) { - withSQLConf( - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "300") { - sql("SELECT * FROM testData join testData2 ON key = a where value = '1'").collect() - } - } - Seq("Plan changed", "Final plan").foreach { - msg => - assert(logAppender.loggingEvents.exists { - event => - event.getMessage.getFormattedMessage.contains(msg) && event.getLevel == expectedLevel - }) - } - } - - // Verify default log level - verifyLog(Level.DEBUG) - - // Verify custom log level - val levels = Seq( - "TRACE" -> Level.TRACE, - "trace" -> Level.TRACE, - "DEBUG" -> Level.DEBUG, - "debug" -> Level.DEBUG, - "INFO" -> Level.INFO, - "info" -> Level.INFO, - "WARN" -> Level.WARN, - "warn" -> Level.WARN, - "ERROR" -> Level.ERROR, - "error" -> Level.ERROR, - "deBUG" -> Level.DEBUG - ) - - levels.foreach { - level => - withSQLConf(SQLConf.ADAPTIVE_EXECUTION_LOG_LEVEL.key -> level._1) { - verifyLog(level._2) - } - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala deleted file mode 100644 index a305887c9b1..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.benchmarks - -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.execution.{FileSourceScanExecTransformer, WholeStageTransformer} -import org.apache.gluten.extension.columnar.transition.Transitions -import org.apache.gluten.utils.BackendTestUtils - -import org.apache.spark.SparkConf -import org.apache.spark.benchmark.Benchmark -import org.apache.spark.rdd.RDD -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.UnsafeProjection -import org.apache.spark.sql.execution.FileSourceScanExec -import org.apache.spark.sql.execution.benchmark.SqlBasedBenchmark -import org.apache.spark.sql.execution.datasources.{FilePartition, FileScanRDD, PartitionedFile} -import org.apache.spark.sql.vectorized.ColumnarBatch - -import scala.collection.JavaConverters._ - -/** - * Benchmark to measure native parquet read performance. To run this benchmark: - * {{{ - * 1. Run in IDEA: run this class directly; - * 2. Run without IDEA: bin/spark-submit --class - * --jars ,, - * --conf xxxx=xxx - * gluten-ut-XXX-tests.jar - * parameters - * - * Parameters: - * 1. parquet files dir; - * 2. the fields to read; - * 3. the execution count; - * 4. whether to run vanilla spark benchmarks; - * }}} - */ -object ParquetReadBenchmark extends SqlBasedBenchmark { - - protected lazy val thrdNum = "1" - protected lazy val memorySize = "4G" - protected lazy val offheapSize = "4G" - - def beforeAll(): Unit = {} - - override def getSparkSession: SparkSession = { - beforeAll() - val conf = new SparkConf() - .setAppName("ParquetReadBenchmark") - .setIfMissing("spark.master", s"local[$thrdNum]") - .set("spark.plugins", "org.apache.gluten.GlutenPlugin") - .set("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager") - .set("spark.memory.offHeap.enabled", "true") - .setIfMissing("spark.memory.offHeap.size", offheapSize) - .setIfMissing("spark.sql.columnVector.offheap.enabled", "true") - .set("spark.sql.adaptive.enabled", "false") - .setIfMissing("spark.driver.memory", memorySize) - .setIfMissing("spark.executor.memory", memorySize) - .setIfMissing("spark.sql.files.maxPartitionBytes", "1G") - .setIfMissing("spark.sql.files.openCostInBytes", "1073741824") - - if (BackendTestUtils.isCHBackendLoaded()) { - conf - .set("spark.io.compression.codec", "LZ4") - .set(GlutenConfig.NATIVE_VALIDATION_ENABLED.key, "false") - .set("spark.gluten.sql.columnar.backend.ch.worker.id", "1") - .set("spark.gluten.sql.columnar.separate.scan.rdd.for.ch", "false") - .set( - "spark.sql.catalog.spark_catalog", - "org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseSparkCatalog") - .set("spark.databricks.delta.maxSnapshotLineageLength", "20") - .set("spark.databricks.delta.snapshotPartitions", "1") - .set("spark.databricks.delta.properties.defaults.checkpointInterval", "5") - .set("spark.databricks.delta.stalenessLimit", "3600000") - } - - SparkSession.builder.config(conf).getOrCreate() - } - - override def runBenchmarkSuite(mainArgs: Array[String]): Unit = { - val (parquetDir, scanSchema, executedCnt, executedVanilla) = - if (mainArgs.isEmpty) { - ("/data/tpch-data-sf10/lineitem", "l_orderkey,l_receiptdate", 5, true) - } else { - (mainArgs(0), mainArgs(1), mainArgs(2).toInt, mainArgs(3).toBoolean) - } - - val parquetReadDf = spark.sql(s""" - |select $scanSchema from parquet.`$parquetDir` - | - |""".stripMargin) - // Get the `FileSourceScanExecTransformer` - val fileScan = parquetReadDf.queryExecution.executedPlan.collect { - case scan: FileSourceScanExecTransformer => scan - }.head - - val filePartitions = fileScan.getPartitions - .map(_.asInstanceOf[FilePartition]) - - val wholeStageTransform = parquetReadDf.queryExecution.executedPlan.collect { - case wholeStage: WholeStageTransformer => wholeStage - }.head - - // remove ProjectExecTransformer - val newWholeStage = wholeStageTransform.withNewChildren(Seq(fileScan)) - - // generate ColumnarToRow - val columnarToRowPlan = Transitions.toRowPlan(newWholeStage) - - val newWholeStageRDD = newWholeStage.executeColumnar() - val newColumnarToRowRDD = columnarToRowPlan.execute() - - // Get the total row count - val totalRowCnt = newWholeStageRDD - .mapPartitionsInternal( - batches => { - batches.map(batch => batch.numRows().toLong) - }) - .collect() - .sum - - val parquetReadBenchmark = - new Benchmark( - s"Parquet Read files, fields: $scanSchema, total $totalRowCnt records", - totalRowCnt, - output = output) - - parquetReadBenchmark.addCase(s"Native Parquet Read", executedCnt) { - _ => - val resultRDD: RDD[Long] = newWholeStageRDD.mapPartitionsInternal { - batches => - batches.foreach(batch => batch.numRows().toLong) - Iterator.empty - } - resultRDD.collect() - } - - parquetReadBenchmark.addCase(s"Native Parquet Read to Rows", executedCnt) { - _ => - val resultRDD: RDD[Int] = newColumnarToRowRDD.mapPartitionsInternal { - rows => - rows.foreach(_.numFields) - Iterator.empty - } - resultRDD.collect() - } - - if (executedVanilla) { - spark.conf.set(GlutenConfig.GLUTEN_ENABLED.key, "false") - - val vanillaParquet = spark.sql(s""" - |select $scanSchema from parquet.`$parquetDir` - | - |""".stripMargin) - - val vanillaScanPlan = vanillaParquet.queryExecution.executedPlan.collect { - case scan: FileSourceScanExec => scan - } - - val fileScan = vanillaScanPlan.head - val fileScanOutput = fileScan.output - val relation = fileScan.relation - val readFile: (PartitionedFile) => Iterator[InternalRow] = - relation.fileFormat.buildReaderWithPartitionValues( - sparkSession = relation.sparkSession, - dataSchema = relation.dataSchema, - partitionSchema = relation.partitionSchema, - requiredSchema = fileScan.requiredSchema, - filters = Seq.empty, - options = relation.options, - hadoopConf = relation.sparkSession.sessionState.newHadoopConfWithOptions(relation.options) - ) - - val newFileScanRDD = new FileScanRDD(spark, readFile, filePartitions, fileScan.requiredSchema) - .asInstanceOf[RDD[ColumnarBatch]] - - val rowCnt = newFileScanRDD - .mapPartitionsInternal(batches => batches.map(batch => batch.numRows().toLong)) - .collect() - .sum - assert(totalRowCnt == rowCnt, "The row count of the benchmark is not equal.") - - parquetReadBenchmark.addCase(s"Vanilla Spark Parquet Read", executedCnt) { - _ => - val resultRDD: RDD[Long] = newFileScanRDD.mapPartitionsInternal { - batches => - batches.foreach(_.numRows().toLong) - Iterator.empty - } - resultRDD.collect() - } - - parquetReadBenchmark.addCase(s"Vanilla Spark Parquet Read to Rows", executedCnt) { - _ => - val resultRDD: RDD[Long] = newFileScanRDD.mapPartitionsInternal { - batches => - val toUnsafe = UnsafeProjection.create(fileScanOutput, fileScanOutput) - batches.foreach(_.rowIterator().asScala.map(toUnsafe).foreach(_.numFields)) - Iterator.empty - } - resultRDD.collect() - } - } - - parquetReadBenchmark.run() - } - - override def afterAll(): Unit = { - super.afterAll() - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenBucketingUtilsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenBucketingUtilsSuite.scala deleted file mode 100644 index 37a786e34c5..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenBucketingUtilsSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenBucketingUtilsSuite extends BucketingUtilsSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenDataSourceStrategySuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenDataSourceStrategySuite.scala deleted file mode 100644 index eeb63436c1e..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenDataSourceStrategySuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenDataSourceStrategySuite extends DataSourceStrategySuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenDataSourceSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenDataSourceSuite.scala deleted file mode 100644 index 6435d17de2a..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenDataSourceSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenDataSourceSuite extends DataSourceSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileFormatWriterSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileFormatWriterSuite.scala deleted file mode 100644 index c0ba24f2be1..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileFormatWriterSuite.scala +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait -import org.apache.spark.sql.catalyst.plans.CodegenInterpretedPlanTest - -class GlutenFileFormatWriterSuite - extends FileFormatWriterSuite - with GlutenSQLTestsBaseTrait - with CodegenInterpretedPlanTest {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileIndexSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileIndexSuite.scala deleted file mode 100644 index c1c57eaa914..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileIndexSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenFileIndexSuite extends FileIndexSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala deleted file mode 100644 index ed347d024c1..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources - -import org.apache.gluten.backendsapi.BackendsApiManager -import org.apache.gluten.execution.{FileSourceScanExecTransformer, FilterExecTransformer} - -import org.apache.spark.sql.{Column, DataFrame, Row} -import org.apache.spark.sql.GlutenSQLTestsBaseTrait -import org.apache.spark.sql.execution.FileSourceScanExec -import org.apache.spark.sql.types.{IntegerType, LongType, StringType, StructField, StructType} - -import java.io.File -import java.sql.Timestamp - -import scala.reflect.ClassTag - -class GlutenFileMetadataStructSuite extends FileMetadataStructSuite with GlutenSQLTestsBaseTrait { - - val schemaWithFilePathField: StructType = new StructType() - .add(StructField("file_path", StringType)) - .add(StructField("age", IntegerType)) - .add( - StructField( - "info", - new StructType() - .add(StructField("id", LongType)) - .add(StructField("university", StringType)))) - - private val METADATA_FILE_PATH = "_metadata.file_path" - private val METADATA_FILE_NAME = "_metadata.file_name" - private val METADATA_FILE_SIZE = "_metadata.file_size" - private val METADATA_FILE_MODIFICATION_TIME = "_metadata.file_modification_time" - - private def getMetadataForFile(f: File): Map[String, Any] = { - Map( - METADATA_FILE_PATH -> f.toURI.toString, - METADATA_FILE_NAME -> f.getName, - METADATA_FILE_SIZE -> f.length(), - METADATA_FILE_MODIFICATION_TIME -> new Timestamp(f.lastModified()) - ) - } - - private def metadataColumnsNativeTest(testName: String, fileSchema: StructType)( - f: (DataFrame, Map[String, Any], Map[String, Any]) => Unit): Unit = { - Seq("parquet").foreach { - testFileFormat => - testGluten(s"metadata struct ($testFileFormat): " + testName) { - withTempDir { - dir => - import scala.collection.JavaConverters._ - - // 1. create df0 and df1 and save under /data/f0 and /data/f1 - val df0 = spark.createDataFrame(data0.asJava, fileSchema) - val f0 = new File(dir, "data/f0").getCanonicalPath - df0.coalesce(1).write.format(testFileFormat).save(f0) - - val df1 = spark.createDataFrame(data1.asJava, fileSchema) - val f1 = new File(dir, "data/f1 gluten").getCanonicalPath - df1.coalesce(1).write.format(testFileFormat).save(f1) - - // 2. read both f0 and f1 - val df = spark.read - .format(testFileFormat) - .schema(fileSchema) - .load(new File(dir, "data").getCanonicalPath + "/*") - val realF0 = new File(dir, "data/f0") - .listFiles() - .filter(_.getName.endsWith(s".$testFileFormat")) - .head - val realF1 = new File(dir, "data/f1 gluten") - .listFiles() - .filter(_.getName.endsWith(s".$testFileFormat")) - .head - f(df, getMetadataForFile(realF0), getMetadataForFile(realF1)) - } - } - } - } - - def checkOperatorMatch[T](df: DataFrame)(implicit tag: ClassTag[T]): Unit = { - val executedPlan = getExecutedPlan(df) - assert(executedPlan.exists(plan => plan.getClass == tag.runtimeClass)) - } - - metadataColumnsNativeTest( - "plan check with metadata and user data select", - schemaWithFilePathField) { - (df, f0, f1) => - var dfWithMetadata = df.select( - METADATA_FILE_NAME, - METADATA_FILE_PATH, - METADATA_FILE_SIZE, - METADATA_FILE_MODIFICATION_TIME, - "age") - dfWithMetadata.collect - if (BackendsApiManager.getSettings.supportNativeMetadataColumns()) { - checkOperatorMatch[FileSourceScanExecTransformer](dfWithMetadata) - } else { - checkOperatorMatch[FileSourceScanExec](dfWithMetadata) - } - - // would fallback - dfWithMetadata = df.select(METADATA_FILE_PATH, "file_path") - checkAnswer( - dfWithMetadata, - Seq( - Row(f0(METADATA_FILE_PATH), "jack"), - Row(f1(METADATA_FILE_PATH), "lily") - ) - ) - checkOperatorMatch[FileSourceScanExec](dfWithMetadata) - } - - metadataColumnsNativeTest("plan check with metadata filter", schemaWithFilePathField) { - (df, f0, f1) => - var filterDF = df - .select("file_path", "age", METADATA_FILE_NAME) - .where(Column(METADATA_FILE_NAME) === f0((METADATA_FILE_NAME))) - val ret = filterDF.collect - assert(ret.size == 1) - if (BackendsApiManager.getSettings.supportNativeMetadataColumns()) { - checkOperatorMatch[FileSourceScanExecTransformer](filterDF) - } else { - checkOperatorMatch[FileSourceScanExec](filterDF) - } - checkOperatorMatch[FilterExecTransformer](filterDF) - - // case to check if file_path is URI string - filterDF = - df.select(METADATA_FILE_PATH).where(Column(METADATA_FILE_NAME) === f1((METADATA_FILE_NAME))) - checkAnswer( - filterDF, - Seq( - Row(f1(METADATA_FILE_PATH)) - ) - ) - if (BackendsApiManager.getSettings.supportNativeMetadataColumns()) { - checkOperatorMatch[FileSourceScanExecTransformer](filterDF) - } else { - checkOperatorMatch[FileSourceScanExec](filterDF) - } - checkOperatorMatch[FilterExecTransformer](filterDF) - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileSourceAggregatePushDownSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileSourceAggregatePushDownSuite.scala deleted file mode 100644 index 54138564f95..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileSourceAggregatePushDownSuite.scala +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenParquetV1AggregatePushDownSuite - extends ParquetV1AggregatePushDownSuite - with GlutenSQLTestsBaseTrait {} - -class GlutenParquetV2AggregatePushDownSuite - extends ParquetV2AggregatePushDownSuite - with GlutenSQLTestsBaseTrait {} - -class GlutenOrcV1AggregatePushDownSuite - extends OrcV1AggregatePushDownSuite - with GlutenSQLTestsBaseTrait {} - -class GlutenOrcV2AggregatePushDownSuite - extends OrcV2AggregatePushDownSuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileSourceCodecSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileSourceCodecSuite.scala deleted file mode 100644 index 631be9c96fa..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileSourceCodecSuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenParquetCodecSuite extends ParquetCodecSuite with GlutenSQLTestsBaseTrait {} - -class GlutenOrcCodecSuite extends OrcCodecSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileSourceStrategySuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileSourceStrategySuite.scala deleted file mode 100644 index 171a27e31c4..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileSourceStrategySuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources - -import org.apache.spark.sql._ - -class GlutenFileSourceStrategySuite extends FileSourceStrategySuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenHadoopFileLinesReaderSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenHadoopFileLinesReaderSuite.scala deleted file mode 100644 index b283d44b03a..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenHadoopFileLinesReaderSuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenHadoopFileLinesReaderSuite - extends HadoopFileLinesReaderSuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenPathFilterStrategySuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenPathFilterStrategySuite.scala deleted file mode 100644 index f3554eb1cb0..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenPathFilterStrategySuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenPathFilterStrategySuite extends PathFilterStrategySuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenPathFilterSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenPathFilterSuite.scala deleted file mode 100644 index 4f4f9c76ee4..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenPathFilterSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenPathFilterSuite extends PathFilterSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenPruneFileSourcePartitionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenPruneFileSourcePartitionsSuite.scala deleted file mode 100644 index a108c4fe1ec..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenPruneFileSourcePartitionsSuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenPruneFileSourcePartitionsSuite - extends PruneFileSourcePartitionsSuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenReadSchemaSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenReadSchemaSuite.scala deleted file mode 100644 index 1ca70a2cb9a..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenReadSchemaSuite.scala +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait -import org.apache.spark.sql.internal.SQLConf - -import java.io.File - -class GlutenCSVReadSchemaSuite extends CSVReadSchemaSuite with GlutenSQLTestsBaseTrait {} - -class GlutenHeaderCSVReadSchemaSuite - extends HeaderCSVReadSchemaSuite - with GlutenSQLTestsBaseTrait {} - -class GlutenJsonReadSchemaSuite extends JsonReadSchemaSuite with GlutenSQLTestsBaseTrait {} - -class GlutenOrcReadSchemaSuite extends OrcReadSchemaSuite with GlutenSQLTestsBaseTrait {} - -class GlutenVectorizedOrcReadSchemaSuite - extends VectorizedOrcReadSchemaSuite - with GlutenSQLTestsBaseTrait { - - import testImplicits._ - - private lazy val values = 1 to 10 - private lazy val floatDF = values.map(_.toFloat).toDF("col1") - private lazy val doubleDF = values.map(_.toDouble).toDF("col1") - private lazy val unionDF = floatDF.union(doubleDF) - - testGluten("change column position") { - withTempPath { - dir => - withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> "false") { - val path = dir.getCanonicalPath - - val df1 = Seq(("1", "a"), ("2", "b"), ("3", "c")).toDF("col1", "col2") - val df2 = Seq(("d", "4"), ("e", "5"), ("f", "6")).toDF("col2", "col1") - val unionDF = df1.unionByName(df2) - - val dir1 = s"$path${File.separator}part=one" - val dir2 = s"$path${File.separator}part=two" - - df1.write.format(format).options(options).save(dir1) - df2.write.format(format).options(options).save(dir2) - - val df = spark.read - .schema(unionDF.schema) - .format(format) - .options(options) - .load(path) - .select("col1", "col2") - - checkAnswer(df, unionDF) - } - } - } - - testGluten("read byte, int, short, long together") { - withTempPath { - dir => - withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> "false") { - val path = dir.getCanonicalPath - - val byteDF = (Byte.MaxValue - 2 to Byte.MaxValue).map(_.toByte).toDF("col1") - val shortDF = (Short.MaxValue - 2 to Short.MaxValue).map(_.toShort).toDF("col1") - val intDF = (Int.MaxValue - 2 to Int.MaxValue).toDF("col1") - val longDF = (Long.MaxValue - 2 to Long.MaxValue).toDF("col1") - val unionDF = byteDF.union(shortDF).union(intDF).union(longDF) - - val byteDir = s"$path${File.separator}part=byte" - val shortDir = s"$path${File.separator}part=short" - val intDir = s"$path${File.separator}part=int" - val longDir = s"$path${File.separator}part=long" - - byteDF.write.format(format).options(options).save(byteDir) - shortDF.write.format(format).options(options).save(shortDir) - intDF.write.format(format).options(options).save(intDir) - longDF.write.format(format).options(options).save(longDir) - - val df = spark.read - .schema(unionDF.schema) - .format(format) - .options(options) - .load(path) - .select("col1") - - checkAnswer(df, unionDF) - } - } - } - - testGluten("read float and double together") { - withTempPath { - dir => - withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> "false") { - val path = dir.getCanonicalPath - - val floatDir = s"$path${File.separator}part=float" - val doubleDir = s"$path${File.separator}part=double" - - floatDF.write.format(format).options(options).save(floatDir) - doubleDF.write.format(format).options(options).save(doubleDir) - - val df = spark.read - .schema(unionDF.schema) - .format(format) - .options(options) - .load(path) - .select("col1") - - checkAnswer(df, unionDF) - } - } - } -} - -class GlutenMergedOrcReadSchemaSuite - extends MergedOrcReadSchemaSuite - with GlutenSQLTestsBaseTrait {} - -class GlutenParquetReadSchemaSuite extends ParquetReadSchemaSuite with GlutenSQLTestsBaseTrait {} - -class GlutenVectorizedParquetReadSchemaSuite - extends VectorizedParquetReadSchemaSuite - with GlutenSQLTestsBaseTrait {} - -class GlutenMergedParquetReadSchemaSuite - extends MergedParquetReadSchemaSuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/GlutenBinaryFileFormatSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/GlutenBinaryFileFormatSuite.scala deleted file mode 100644 index ee6ec1bea1a..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/GlutenBinaryFileFormatSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.binaryfile - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenBinaryFileFormatSuite extends BinaryFileFormatSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/csv/GlutenCSVSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/csv/GlutenCSVSuite.scala deleted file mode 100644 index 4181a32521c..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/csv/GlutenCSVSuite.scala +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.csv - -import org.apache.spark.SparkConf -import org.apache.spark.sql.GlutenSQLTestsBaseTrait -import org.apache.spark.sql.internal.SQLConf - -class GlutenCSVSuite extends CSVSuite with GlutenSQLTestsBaseTrait { - - /** Returns full path to the given file in the resource folder */ - override protected def testFile(fileName: String): String = { - getWorkspaceFilePath("sql", "core", "src", "test", "resources").toString + "/" + fileName - } -} - -class GlutenCSVv1Suite extends GlutenCSVSuite { - override def sparkConf: SparkConf = - super.sparkConf - .set(SQLConf.USE_V1_SOURCE_LIST, "csv") -} - -class GlutenCSVv2Suite extends GlutenCSVSuite { - override def sparkConf: SparkConf = - super.sparkConf - .set(SQLConf.USE_V1_SOURCE_LIST, "") -} - -class GlutenCSVLegacyTimeParserSuite extends GlutenCSVSuite { - override def sparkConf: SparkConf = - super.sparkConf - .set(SQLConf.LEGACY_TIME_PARSER_POLICY, "legacy") -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/exchange/GlutenValidateRequirementsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/exchange/GlutenValidateRequirementsSuite.scala deleted file mode 100644 index 132e80696cf..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/exchange/GlutenValidateRequirementsSuite.scala +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.exchange - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait -import org.apache.spark.sql.execution.exchange.ValidateRequirementsSuite - -class GlutenValidateRequirementsSuite - extends ValidateRequirementsSuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/json/GlutenJsonSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/json/GlutenJsonSuite.scala deleted file mode 100644 index 4b7e3cc54e8..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/json/GlutenJsonSuite.scala +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.json - -import org.apache.spark.SparkConf -import org.apache.spark.sql.{sources, GlutenSQLTestsBaseTrait} -import org.apache.spark.sql.execution.datasources.{InMemoryFileIndex, NoopCache} -import org.apache.spark.sql.execution.datasources.v2.json.JsonScanBuilder -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.{IntegerType, StructType} -import org.apache.spark.sql.util.CaseInsensitiveStringMap - -class GlutenJsonSuite extends JsonSuite with GlutenSQLTestsBaseTrait { - - /** Returns full path to the given file in the resource folder */ - override protected def testFile(fileName: String): String = { - getWorkspaceFilePath("sql", "core", "src", "test", "resources").toString + "/" + fileName - } -} - -class GlutenJsonV1Suite extends GlutenJsonSuite with GlutenSQLTestsBaseTrait { - override def sparkConf: SparkConf = - super.sparkConf - .set(SQLConf.USE_V1_SOURCE_LIST, "json") -} - -class GlutenJsonV2Suite extends GlutenJsonSuite with GlutenSQLTestsBaseTrait { - override def sparkConf: SparkConf = - super.sparkConf - .set(SQLConf.USE_V1_SOURCE_LIST, "") - - test("get pushed filters") { - val attr = "col" - def getBuilder(path: String): JsonScanBuilder = { - val fileIndex = new InMemoryFileIndex( - spark, - Seq(new org.apache.hadoop.fs.Path(path, "file.json")), - Map.empty, - None, - NoopCache) - val schema = new StructType().add(attr, IntegerType) - val options = CaseInsensitiveStringMap.empty() - new JsonScanBuilder(spark, fileIndex, schema, schema, options) - } - val filters: Array[sources.Filter] = Array(sources.IsNotNull(attr)) - withSQLConf(SQLConf.JSON_FILTER_PUSHDOWN_ENABLED.key -> "true") { - withTempPath { - file => - val scanBuilder = getBuilder(file.getCanonicalPath) - assert(scanBuilder.pushDataFilters(filters) === filters) - } - } - - withSQLConf(SQLConf.JSON_FILTER_PUSHDOWN_ENABLED.key -> "false") { - withTempPath { - file => - val scanBuilder = getBuilder(file.getCanonicalPath) - assert(scanBuilder.pushDataFilters(filters) === Array.empty[sources.Filter]) - } - } - } -} - -class GlutenJsonLegacyTimeParserSuite extends GlutenJsonSuite with GlutenSQLTestsBaseTrait { - override def sparkConf: SparkConf = - super.sparkConf - .set(SQLConf.LEGACY_TIME_PARSER_POLICY, "legacy") -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/orc/GlutenOrcColumnarBatchReaderSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/orc/GlutenOrcColumnarBatchReaderSuite.scala deleted file mode 100644 index e2e3818aad9..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/orc/GlutenOrcColumnarBatchReaderSuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.orc - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenOrcColumnarBatchReaderSuite - extends OrcColumnarBatchReaderSuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/orc/GlutenOrcFilterSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/orc/GlutenOrcFilterSuite.scala deleted file mode 100644 index f5a8db3395d..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/orc/GlutenOrcFilterSuite.scala +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.orc - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -/** A test suite that tests Apache ORC filter API based filter pushdown optimization. */ -class GlutenOrcFilterSuite extends OrcFilterSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/orc/GlutenOrcPartitionDiscoverySuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/orc/GlutenOrcPartitionDiscoverySuite.scala deleted file mode 100644 index a9848b7f444..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/orc/GlutenOrcPartitionDiscoverySuite.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.orc - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenOrcPartitionDiscoverySuite - extends OrcPartitionDiscoveryTest - with GlutenSQLTestsBaseTrait {} - -class GlutenOrcV1PartitionDiscoverySuite - extends OrcV1PartitionDiscoverySuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/orc/GlutenOrcQuerySuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/orc/GlutenOrcQuerySuite.scala deleted file mode 100644 index f186695b88b..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/orc/GlutenOrcQuerySuite.scala +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.orc - -import org.apache.spark.SparkConf -import org.apache.spark.sql.{GlutenSQLTestsBaseTrait, Row} -import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.internal.SQLConf - -class GlutenOrcQuerySuite extends OrcQuerySuite with GlutenSQLTestsBaseTrait { - testGluten("Simple selection form ORC table") { - val data = (1 to 10).map { - i => Person(s"name_$i", i, (0 to 1).map(m => Contact(s"contact_$m", s"phone_$m"))) - } - - withOrcTable(data, "t") { - withSQLConf("spark.sql.orc.enableVectorizedReader" -> "false") { - // ppd: - // leaf-0 = (LESS_THAN_EQUALS age 5) - // expr = leaf-0 - assert(sql("SELECT name FROM t WHERE age <= 5").count() === 5) - - // ppd: - // leaf-0 = (LESS_THAN_EQUALS age 5) - // expr = (not leaf-0) - assertResult(10) { - sql("SELECT name, contacts FROM t where age > 5").rdd - .flatMap(_.getAs[scala.collection.Seq[_]]("contacts")) - .count() - } - - // ppd: - // leaf-0 = (LESS_THAN_EQUALS age 5) - // leaf-1 = (LESS_THAN age 8) - // expr = (and (not leaf-0) leaf-1) - { - val df = sql("SELECT name, contacts FROM t WHERE age > 5 AND age < 8") - assert(df.count() === 2) - assertResult(4) { - df.rdd.flatMap(_.getAs[scala.collection.Seq[_]]("contacts")).count() - } - } - - // ppd: - // leaf-0 = (LESS_THAN age 2) - // leaf-1 = (LESS_THAN_EQUALS age 8) - // expr = (or leaf-0 (not leaf-1)) - { - val df = sql("SELECT name, contacts FROM t WHERE age < 2 OR age > 8") - assert(df.count() === 3) - assertResult(6) { - df.rdd.flatMap(_.getAs[scala.collection.Seq[_]]("contacts")).count() - } - } - } - } - } - - testGluten("simple select queries") { - withOrcTable((0 until 10).map(i => (i, i.toString)), "t") { - withSQLConf("spark.sql.orc.enableVectorizedReader" -> "false") { - checkAnswer(sql("SELECT `_1` FROM t where t.`_1` > 5"), (6 until 10).map(Row.apply(_))) - - checkAnswer( - sql("SELECT `_1` FROM t as tmp where tmp.`_1` < 5"), - (0 until 5).map(Row.apply(_))) - } - } - } - - testGluten("overwriting") { - val data = (0 until 10).map(i => (i, i.toString)) - spark.createDataFrame(data).toDF("c1", "c2").createOrReplaceTempView("tmp") - withOrcTable(data, "t") { - withSQLConf("spark.sql.orc.enableVectorizedReader" -> "false") { - sql("INSERT OVERWRITE TABLE t SELECT * FROM tmp") - checkAnswer(spark.table("t"), data.map(Row.fromTuple)) - } - } - spark.sessionState.catalog.dropTable( - TableIdentifier("tmp"), - ignoreIfNotExists = true, - purge = false) - } - - testGluten("self-join") { - // 4 rows, cells of column 1 of row 2 and row 4 are null - val data = (1 to 4).map { - i => - val maybeInt = if (i % 2 == 0) None else Some(i) - (maybeInt, i.toString) - } - - withOrcTable(data, "t") { - withSQLConf("spark.sql.orc.enableVectorizedReader" -> "false") { - val selfJoin = sql("SELECT * FROM t x JOIN t y WHERE x.`_1` = y.`_1`") - val queryOutput = selfJoin.queryExecution.analyzed.output - - assertResult(4, "Field count mismatches")(queryOutput.size) - assertResult(2, s"Duplicated expression ID in query plan:\n $selfJoin") { - queryOutput.filter(_.name == "_1").map(_.exprId).size - } - - checkAnswer(selfJoin, List(Row(1, "1", 1, "1"), Row(3, "3", 3, "3"))) - } - } - } - - testGluten("columns only referenced by pushed down filters should remain") { - withOrcTable((1 to 10).map(Tuple1.apply), "t") { - withSQLConf("spark.sql.orc.enableVectorizedReader" -> "false") { - checkAnswer(sql("SELECT `_1` FROM t WHERE `_1` < 10"), (1 to 9).map(Row.apply(_))) - } - } - } - - testGluten("SPARK-5309 strings stored using dictionary compression in orc") { - withOrcTable((0 until 1000).map(i => ("same", "run_" + i / 100, 1)), "t") { - withSQLConf("spark.sql.orc.enableVectorizedReader" -> "false") { - checkAnswer( - sql("SELECT `_1`, `_2`, SUM(`_3`) FROM t GROUP BY `_1`, `_2`"), - (0 until 10).map(i => Row("same", "run_" + i, 100))) - - checkAnswer( - sql("SELECT `_1`, `_2`, SUM(`_3`) FROM t WHERE `_2` = 'run_5' GROUP BY `_1`, `_2`"), - List(Row("same", "run_5", 100))) - } - } - } -} - -class GlutenOrcV1QuerySuite extends GlutenOrcQuerySuite { - override def sparkConf: SparkConf = - super.sparkConf - .set(SQLConf.USE_V1_SOURCE_LIST, "orc") -} - -class GlutenOrcV2QuerySuite extends GlutenOrcQuerySuite { - override def sparkConf: SparkConf = - super.sparkConf - .set(SQLConf.USE_V1_SOURCE_LIST, "") -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/orc/GlutenOrcSourceSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/orc/GlutenOrcSourceSuite.scala deleted file mode 100644 index ffb96d0f316..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/orc/GlutenOrcSourceSuite.scala +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.orc - -import org.apache.spark.sql.{GlutenSQLTestsBaseTrait, Row} -import org.apache.spark.sql.execution.FileSourceScanExec -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types._ - -import java.sql.Date -import java.time.{Duration, Period} - -class GlutenOrcSourceSuite extends OrcSourceSuite with GlutenSQLTestsBaseTrait { - import testImplicits._ - - override def withAllNativeOrcReaders(code: => Unit): Unit = { - // test the row-based reader - withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> "false")(code) - } - - testGluten("SPARK-31238: compatibility with Spark 2.4 in reading dates") { - Seq(false).foreach { - vectorized => - withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> vectorized.toString) { - checkAnswer( - readResourceOrcFile("test-data/before_1582_date_v2_4.snappy.orc"), - Row(java.sql.Date.valueOf("1200-01-01"))) - } - } - } - - testGluten("SPARK-31238, SPARK-31423: rebasing dates in write") { - withTempPath { - dir => - val path = dir.getAbsolutePath - Seq("1001-01-01", "1582-10-10") - .toDF("dateS") - .select($"dateS".cast("date").as("date")) - .write - .orc(path) - - Seq(false).foreach { - vectorized => - withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> vectorized.toString) { - checkAnswer( - spark.read.orc(path), - Seq(Row(Date.valueOf("1001-01-01")), Row(Date.valueOf("1582-10-15")))) - } - } - } - } - - testGluten("SPARK-31284: compatibility with Spark 2.4 in reading timestamps") { - Seq(false).foreach { - vectorized => - withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> vectorized.toString) { - checkAnswer( - readResourceOrcFile("test-data/before_1582_ts_v2_4.snappy.orc"), - Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456"))) - } - } - } - - testGluten("SPARK-31284, SPARK-31423: rebasing timestamps in write") { - withTempPath { - dir => - val path = dir.getAbsolutePath - Seq("1001-01-01 01:02:03.123456", "1582-10-10 11:12:13.654321") - .toDF("tsS") - .select($"tsS".cast("timestamp").as("ts")) - .write - .orc(path) - - Seq(false).foreach { - vectorized => - withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> vectorized.toString) { - checkAnswer( - spark.read.orc(path), - Seq( - Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456")), - Row(java.sql.Timestamp.valueOf("1582-10-15 11:12:13.654321"))) - ) - } - } - } - } - - testGluten("SPARK-34862: Support ORC vectorized reader for nested column") { - withTempPath { - dir => - val path = dir.getCanonicalPath - val df = spark - .range(10) - .map { - x => - val stringColumn = s"$x" * 10 - val structColumn = (x, s"$x" * 100) - val arrayColumn = (0 until 5).map(i => (x + i, s"$x" * 5)) - val mapColumn = Map( - s"$x" -> (x * 0.1, (x, s"$x" * 100)), - (s"$x" * 2) -> (x * 0.2, (x, s"$x" * 200)), - (s"$x" * 3) -> (x * 0.3, (x, s"$x" * 300))) - (x, stringColumn, structColumn, arrayColumn, mapColumn) - } - .toDF("int_col", "string_col", "struct_col", "array_col", "map_col") - df.write.format("orc").save(path) - - // Rewrite because Gluten does not support Spark's vectorized reading. - withSQLConf(SQLConf.ORC_VECTORIZED_READER_NESTED_COLUMN_ENABLED.key -> "false") { - val readDf = spark.read.orc(path) - val vectorizationEnabled = readDf.queryExecution.executedPlan.find { - case scan: FileSourceScanExec => scan.supportsColumnar - case _ => false - }.isDefined - assert(!vectorizationEnabled) - checkAnswer(readDf, df) - } - } - } - withAllNativeOrcReaders { - Seq(false).foreach { - vecReaderNestedColEnabled => - val vecReaderEnabled = SQLConf.get.orcVectorizedReaderEnabled - testGluten( - "SPARK-36931: Support reading and writing ANSI intervals (" + - s"${SQLConf.ORC_VECTORIZED_READER_ENABLED.key}=$vecReaderEnabled, " + - s"${SQLConf.ORC_VECTORIZED_READER_NESTED_COLUMN_ENABLED.key}" + - s"=$vecReaderNestedColEnabled)") { - - withSQLConf( - SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> - vecReaderEnabled.toString, - SQLConf.ORC_VECTORIZED_READER_NESTED_COLUMN_ENABLED.key -> - vecReaderNestedColEnabled.toString - ) { - Seq( - YearMonthIntervalType() -> ((i: Int) => Period.of(i, i, 0)), - DayTimeIntervalType() -> ((i: Int) => Duration.ofDays(i).plusSeconds(i)) - ).foreach { - case (it, f) => - val data = (1 to 10).map(i => Row(i, f(i))) - val schema = StructType( - Array(StructField("d", IntegerType, false), StructField("i", it, false))) - withTempPath { - file => - val df = spark.createDataFrame(sparkContext.parallelize(data), schema) - df.write.orc(file.getCanonicalPath) - val df2 = spark.read.orc(file.getCanonicalPath) - checkAnswer(df2, df.collect().toSeq) - } - } - - // Tests for ANSI intervals in complex types. - withTempPath { - file => - val df = spark.sql("""SELECT - | named_struct('interval', interval '1-2' year to month) a, - | array(interval '1 2:3' day to minute) b, - | map('key', interval '10' year) c, - | map(interval '20' second, 'value') d""".stripMargin) - df.write.orc(file.getCanonicalPath) - val df2 = spark.read.orc(file.getCanonicalPath) - checkAnswer(df2, df.collect().toSeq) - } - } - } - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/orc/GlutenOrcV1FilterSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/orc/GlutenOrcV1FilterSuite.scala deleted file mode 100644 index 3c2fb0b318f..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/orc/GlutenOrcV1FilterSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.orc - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenOrcV1FilterSuite extends OrcV1FilterSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/orc/GlutenOrcV1SchemaPruningSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/orc/GlutenOrcV1SchemaPruningSuite.scala deleted file mode 100644 index c142d33bdc2..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/orc/GlutenOrcV1SchemaPruningSuite.scala +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.orc - -import org.apache.gluten.execution.FileSourceScanExecTransformer - -import org.apache.spark.sql.{DataFrame, GlutenSQLTestsBaseTrait} -import org.apache.spark.sql.catalyst.parser.CatalystSqlParser -import org.apache.spark.sql.execution.FileSourceScanExec -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.tags.ExtendedSQLTest - -@ExtendedSQLTest -class GlutenOrcV1SchemaPruningSuite extends OrcV1SchemaPruningSuite with GlutenSQLTestsBaseTrait { - // disable column reader for nested type - override protected val vectorizedReaderNestedEnabledKey: String = - SQLConf.PARQUET_VECTORIZED_READER_NESTED_COLUMN_ENABLED.key + "_DISABLED" - - override def checkScanSchemata(df: DataFrame, expectedSchemaCatalogStrings: String*): Unit = { - val fileSourceScanSchemata = - collect(df.queryExecution.executedPlan) { - case scan: FileSourceScanExec => scan.requiredSchema - case scan: FileSourceScanExecTransformer => scan.requiredSchema - } - assert( - fileSourceScanSchemata.size === expectedSchemaCatalogStrings.size, - s"Found ${fileSourceScanSchemata.size} file sources in dataframe, " + - s"but expected $expectedSchemaCatalogStrings" - ) - fileSourceScanSchemata.zip(expectedSchemaCatalogStrings).foreach { - case (scanSchema, expectedScanSchemaCatalogString) => - val expectedScanSchema = CatalystSqlParser.parseDataType(expectedScanSchemaCatalogString) - implicit val equality = schemaEquality - assert(scanSchema === expectedScanSchema) - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/orc/GlutenOrcV2SchemaPruningSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/orc/GlutenOrcV2SchemaPruningSuite.scala deleted file mode 100644 index 76a9a6ef956..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/orc/GlutenOrcV2SchemaPruningSuite.scala +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.orc - -import org.apache.gluten.execution.BatchScanExecTransformer - -import org.apache.spark.sql.{DataFrame, GlutenSQLTestsBaseTrait} -import org.apache.spark.sql.catalyst.parser.CatalystSqlParser -import org.apache.spark.sql.execution.datasources.v2.BatchScanExec -import org.apache.spark.sql.execution.datasources.v2.orc.OrcScan -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.tags.ExtendedSQLTest - -@ExtendedSQLTest -class GlutenOrcV2SchemaPruningSuite extends OrcV2SchemaPruningSuite with GlutenSQLTestsBaseTrait { - // disable column reader for nested type - override protected val vectorizedReaderNestedEnabledKey: String = - SQLConf.PARQUET_VECTORIZED_READER_NESTED_COLUMN_ENABLED.key + "_DISABLED" - - override def checkScanSchemata(df: DataFrame, expectedSchemaCatalogStrings: String*): Unit = { - val fileSourceScanSchemata = - collect(df.queryExecution.executedPlan) { - case b: BatchScanExec if b.scan.isInstanceOf[OrcScan] => - b.scan.asInstanceOf[OrcScan].readDataSchema - case b: BatchScanExecTransformer if b.scan.isInstanceOf[OrcScan] => - b.scan.asInstanceOf[OrcScan].readDataSchema - } - assert( - fileSourceScanSchemata.size === expectedSchemaCatalogStrings.size, - s"Found ${fileSourceScanSchemata.size} file sources in dataframe, " + - s"but expected $expectedSchemaCatalogStrings" - ) - fileSourceScanSchemata.zip(expectedSchemaCatalogStrings).foreach { - case (scanSchema, expectedScanSchemaCatalogString) => - val expectedScanSchema = CatalystSqlParser.parseDataType(expectedScanSchemaCatalogString) - implicit val equality = schemaEquality - assert(scanSchema === expectedScanSchema) - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetColumnIndexSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetColumnIndexSuite.scala deleted file mode 100644 index 60e1ca04a2d..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetColumnIndexSuite.scala +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.parquet - -import org.apache.spark.sql.{DataFrame, GlutenSQLTestsBaseTrait} - -class GlutenParquetColumnIndexSuite extends ParquetColumnIndexSuite with GlutenSQLTestsBaseTrait { - private val actions: Seq[DataFrame => DataFrame] = Seq( - "_1 = 500", - "_1 = 500 or _1 = 1500", - "_1 = 500 or _1 = 501 or _1 = 1500", - "_1 = 500 or _1 = 501 or _1 = 1000 or _1 = 1500", - "_1 >= 500 and _1 < 1000", - "(_1 >= 500 and _1 < 1000) or (_1 >= 1500 and _1 < 1600)" - ).map(f => (df: DataFrame) => df.filter(f)) - - testGluten("test reading unaligned pages - test all types") { - val df = spark - .range(0, 2000) - .selectExpr( - "id as _1", - "cast(id as short) as _3", - "cast(id as int) as _4", - "cast(id as float) as _5", - "cast(id as double) as _6", - "cast(id as decimal(20,0)) as _7", - // We changed 1618161925000 to 1618161925 to avoid reaching the limitation of Velox: - // Timepoint is outside of supported year range. - "cast(cast(1618161925 + id * 60 * 60 * 24 as timestamp) as date) as _9" - ) - checkUnalignedPages(df)(actions: _*) - } - - testGluten("test reading unaligned pages - test all types (dict encode)") { - val df = spark - .range(0, 2000) - .selectExpr( - "id as _1", - "cast(id % 10 as byte) as _2", - "cast(id % 10 as short) as _3", - "cast(id % 10 as int) as _4", - "cast(id % 10 as float) as _5", - "cast(id % 10 as double) as _6", - "cast(id % 10 as decimal(20,0)) as _7", - "cast(id % 2 as boolean) as _8", - "cast(cast(1618161925 + (id % 10) * 60 * 60 * 24 as timestamp) as date) as _9", - "cast(1618161925 + (id % 10) as timestamp) as _10" - ) - checkUnalignedPages(df)(actions: _*) - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetCompressionCodecPrecedenceSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetCompressionCodecPrecedenceSuite.scala deleted file mode 100644 index 661d6aad8c3..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetCompressionCodecPrecedenceSuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.parquet - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenParquetCompressionCodecPrecedenceSuite - extends ParquetCompressionCodecPrecedenceSuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetDeltaByteArrayEncodingSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetDeltaByteArrayEncodingSuite.scala deleted file mode 100644 index 166f3255efd..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetDeltaByteArrayEncodingSuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.parquet - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenParquetDeltaByteArrayEncodingSuite - extends ParquetDeltaLengthByteArrayEncodingSuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetDeltaEncodingSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetDeltaEncodingSuite.scala deleted file mode 100644 index ccb69819a3a..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetDeltaEncodingSuite.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.parquet - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenParquetDeltaEncodingInteger - extends ParquetDeltaEncodingInteger - with GlutenSQLTestsBaseTrait {} - -class GlutenParquetDeltaEncodingLong - extends ParquetDeltaEncodingLong - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetDeltaLengthByteArrayEncodingSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetDeltaLengthByteArrayEncodingSuite.scala deleted file mode 100644 index 36928cee001..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetDeltaLengthByteArrayEncodingSuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.parquet - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenParquetDeltaLengthByteArrayEncodingSuite - extends ParquetDeltaLengthByteArrayEncodingSuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetEncodingSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetEncodingSuite.scala deleted file mode 100644 index 6c69c700bec..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetEncodingSuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.parquet - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -// TODO: this needs a lot more testing but it's currently not easy to test with the parquet -// writer abstractions. Revisit. -class GlutenParquetEncodingSuite extends ParquetEncodingSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetFieldIdIOSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetFieldIdIOSuite.scala deleted file mode 100644 index bd1c269843f..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetFieldIdIOSuite.scala +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.parquet - -import org.apache.spark.sql.{GlutenSQLTestsBaseTrait, Row} - -class GlutenParquetFieldIdIOSuite extends ParquetFieldIdIOSuite with GlutenSQLTestsBaseTrait { - testGluten("Parquet writer with ARRAY and MAP") { - spark.sql(""" - |CREATE TABLE T1 ( - | a INT, - | b ARRAY, - | c MAP - |) - |USING PARQUET - |""".stripMargin) - - spark.sql(""" - | INSERT OVERWRITE T1 VALUES - | (1, ARRAY(1, 2, 3), MAP("key1","value1")) - |""".stripMargin) - - checkAnswer( - spark.sql("SELECT * FROM T1"), - Row(1, Array("1", "2", "3"), Map("key1" -> "value1")) :: Nil - ) - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetFileFormatSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetFileFormatSuite.scala deleted file mode 100644 index b60850df240..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetFileFormatSuite.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.parquet - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenParquetFileFormatV1Suite - extends ParquetFileFormatV1Suite - with GlutenSQLTestsBaseTrait {} - -class GlutenParquetFileFormatV2Suite - extends ParquetFileFormatV2Suite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetFilterSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetFilterSuite.scala deleted file mode 100644 index 02b30a46a63..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetFilterSuite.scala +++ /dev/null @@ -1,580 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.parquet - -import org.apache.spark.SparkConf -import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.dsl.expressions._ -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.optimizer.InferFiltersFromConstraints -import org.apache.spark.sql.catalyst.planning.PhysicalOperation -import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.parseColumnPath -import org.apache.spark.sql.execution.datasources.{DataSourceStrategy, HadoopFsRelation, LogicalRelation, PushableColumnAndNestedColumn} -import org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanRelation -import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetScan -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy.{CORRECTED, LEGACY} -import org.apache.spark.sql.internal.SQLConf.ParquetOutputTimestampType.INT96 -import org.apache.spark.sql.types._ -import org.apache.spark.tags.ExtendedSQLTest -import org.apache.spark.util.Utils - -import org.apache.hadoop.fs.Path -import org.apache.parquet.filter2.predicate.{FilterApi, FilterPredicate} -import org.apache.parquet.filter2.predicate.FilterApi._ -import org.apache.parquet.filter2.predicate.Operators -import org.apache.parquet.filter2.predicate.Operators.{Column => _, Eq, Gt, GtEq, Lt, LtEq, NotEq} -import org.apache.parquet.hadoop.{ParquetFileReader, ParquetInputFormat, ParquetOutputFormat} -import org.apache.parquet.hadoop.util.HadoopInputFile - -import java.sql.{Date, Timestamp} -import java.time.LocalDate - -import scala.reflect.ClassTag -import scala.reflect.runtime.universe.TypeTag - -abstract class GlutenParquetFilterSuite extends ParquetFilterSuite with GlutenSQLTestsBaseTrait { - protected def checkFilterPredicate( - predicate: Predicate, - filterClass: Class[_ <: FilterPredicate], - expected: Seq[Row])(implicit df: DataFrame): Unit = { - checkFilterPredicate(df, predicate, filterClass, checkAnswer(_, _: Seq[Row]), expected) - } - - protected def checkFilterPredicate[T]( - predicate: Predicate, - filterClass: Class[_ <: FilterPredicate], - expected: T)(implicit df: DataFrame): Unit = { - checkFilterPredicate(predicate, filterClass, Seq(Row(expected)))(df) - } - - override protected def readResourceParquetFile(name: String): DataFrame = { - spark.read.parquet( - getWorkspaceFilePath("sql", "core", "src", "test", "resources").toString + "/" + name) - } - - testGluten("filter pushdown - timestamp") { - Seq(true, false).foreach { - java8Api => - Seq(CORRECTED, LEGACY).foreach { - rebaseMode => - val millisData = Seq( - "1000-06-14 08:28:53.123", - "1582-06-15 08:28:53.001", - "1900-06-16 08:28:53.0", - "2018-06-17 08:28:53.999") - // INT96 doesn't support pushdown - withSQLConf( - SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString, - SQLConf.PARQUET_INT96_REBASE_MODE_IN_WRITE.key -> rebaseMode.toString, - SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> INT96.toString - ) { - import testImplicits._ - withTempPath { - file => - millisData - .map(i => Tuple1(Timestamp.valueOf(i))) - .toDF - .write - .format(dataSourceName) - .save(file.getCanonicalPath) - readParquetFile(file.getCanonicalPath) { - df => - val schema = new SparkToParquetSchemaConverter(conf).convert(df.schema) - assertResult(None) { - createParquetFilters(schema).createFilter(sources.IsNull("_1")) - } - } - } - } - } - } - } - - testGluten("SPARK-12218: 'Not' is included in Parquet filter pushdown") { - import testImplicits._ - - withSQLConf(SQLConf.PARQUET_FILTER_PUSHDOWN_ENABLED.key -> "true") { - withTempPath { - dir => - val path = s"${dir.getCanonicalPath}/table1" - val df = (1 to 5).map(i => (i, (i % 2).toString)).toDF("a", "b") - df.show() - df.write.parquet(path) - - checkAnswer( - spark.read.parquet(path).where("not (a = 2) or not(b in ('1'))"), - (1 to 5).map(i => Row(i, (i % 2).toString))) - - checkAnswer( - spark.read.parquet(path).where("not (a = 2 and b in ('1'))"), - (1 to 5).map(i => Row(i, (i % 2).toString))) - } - } - } - - testGluten("SPARK-23852: Broken Parquet push-down for partially-written stats") { - withSQLConf(SQLConf.PARQUET_FILTER_PUSHDOWN_ENABLED.key -> "true") { - // parquet-1217.parquet contains a single column with values -1, 0, 1, 2 and null. - // The row-group statistics include null counts, but not min and max values, which - // triggers PARQUET-1217. - - val df = readResourceParquetFile("test-data/parquet-1217.parquet") - - // Will return 0 rows if PARQUET-1217 is not fixed. - assert(df.where("col > 0").count() === 2) - } - } - - testGluten("SPARK-17091: Convert IN predicate to Parquet filter push-down") { - val schema = StructType( - Seq( - StructField("a", IntegerType, nullable = false) - )) - - val parquetSchema = new SparkToParquetSchemaConverter(conf).convert(schema) - val parquetFilters = createParquetFilters(parquetSchema) - assertResult(Some(FilterApi.eq(intColumn("a"), null: Integer))) { - parquetFilters.createFilter(sources.In("a", Array(null))) - } - - assertResult(Some(FilterApi.eq(intColumn("a"), 10: Integer))) { - parquetFilters.createFilter(sources.In("a", Array(10))) - } - - // Remove duplicates - assertResult(Some(FilterApi.eq(intColumn("a"), 10: Integer))) { - parquetFilters.createFilter(sources.In("a", Array(10, 10))) - } - - assertResult( - Some( - or( - or(FilterApi.eq(intColumn("a"), 10: Integer), FilterApi.eq(intColumn("a"), 20: Integer)), - FilterApi.eq(intColumn("a"), 30: Integer)))) { - parquetFilters.createFilter(sources.In("a", Array(10, 20, 30))) - } - - Seq(0, 10).foreach { - threshold => - withSQLConf(SQLConf.PARQUET_FILTER_PUSHDOWN_INFILTERTHRESHOLD.key -> threshold.toString) { - assert( - createParquetFilters(parquetSchema) - .createFilter(sources.In("a", Array(10, 20, 30))) - .nonEmpty === threshold > 0) - } - } - - import testImplicits._ - withTempPath { - path => - val data = 0 to 1024 - data - .toDF("a") - .selectExpr("if (a = 1024, null, a) AS a") // convert 1024 to null - .coalesce(1) - .write - .option("parquet.block.size", 512) - .parquet(path.getAbsolutePath) - val df = spark.read.parquet(path.getAbsolutePath) - Seq(true, false).foreach { - pushEnabled => - withSQLConf(SQLConf.PARQUET_FILTER_PUSHDOWN_ENABLED.key -> pushEnabled.toString) { - Seq(1, 5, 10, 11, 100).foreach { - count => - val filter = s"a in(${Range(0, count).mkString(",")})" - assert(df.where(filter).count() === count) - val actual = stripSparkFilter(df.where(filter)).collect().length - assert(actual === count) - } - assert(df.where("a in(null)").count() === 0) - assert(df.where("a = null").count() === 0) - assert(df.where("a is null").count() === 1) - } - } - } - } - - testGluten("Support Parquet column index") { - // block 1: - // null count min max - // page-0 0 0 99 - // page-1 0 100 199 - // page-2 0 200 299 - // page-3 0 300 399 - // page-4 0 400 449 - // - // block 2: - // null count min max - // page-0 0 450 549 - // page-1 0 550 649 - // page-2 0 650 749 - // page-3 0 750 849 - // page-4 0 850 899 - withTempPath { - path => - spark - .range(900) - .repartition(1) - .write - .option(ParquetOutputFormat.PAGE_SIZE, "500") - .option(ParquetOutputFormat.BLOCK_SIZE, "2000") - .parquet(path.getCanonicalPath) - - val parquetFile = path.listFiles().filter(_.getName.startsWith("part")).last - val in = HadoopInputFile.fromPath( - new Path(parquetFile.getCanonicalPath), - spark.sessionState.newHadoopConf()) - - Utils.tryWithResource(ParquetFileReader.open(in)) { - reader => - val blocks = reader.getFooter.getBlocks - assert(blocks.size() > 1) - val columns = blocks.get(0).getColumns - assert(columns.size() === 1) - val columnIndex = reader.readColumnIndex(columns.get(0)) - assert(columnIndex.getMinValues.size() > 1) - - val rowGroupCnt = blocks.get(0).getRowCount - // Page count = Second page min value - first page min value - val pageCnt = columnIndex.getMinValues.get(1).asLongBuffer().get() - - columnIndex.getMinValues.get(0).asLongBuffer().get() - assert(pageCnt < rowGroupCnt) - Seq(true, false).foreach { - columnIndex => - withSQLConf(ParquetInputFormat.COLUMN_INDEX_FILTERING_ENABLED -> s"$columnIndex") { - val df = spark.read.parquet(parquetFile.getCanonicalPath).where("id = 1") - df.collect() - val plan = df.queryExecution.executedPlan - // Ignore metrics comparison. - /* - val metrics = plan.collectLeaves().head.metrics - val numOutputRows = metrics("numOutputRows").value - - if (columnIndex) { - assert(numOutputRows === pageCnt) - } else { - assert(numOutputRows === rowGroupCnt) - } - */ - } - } - } - } - } -} - -@ExtendedSQLTest -class GlutenParquetV1FilterSuite extends GlutenParquetFilterSuite with GlutenSQLTestsBaseTrait { - // TODO: enable Parquet V2 write path after file source V2 writers are workable. - override def sparkConf: SparkConf = - super.sparkConf - .set(SQLConf.USE_V1_SOURCE_LIST, "parquet") - override def checkFilterPredicate( - df: DataFrame, - predicate: Predicate, - filterClass: Class[_ <: FilterPredicate], - checker: (DataFrame, Seq[Row]) => Unit, - expected: Seq[Row]): Unit = { - val output = predicate.collect { case a: Attribute => a }.distinct - - Seq(("parquet", true), ("", false)).foreach { - case (pushdownDsList, nestedPredicatePushdown) => - withSQLConf( - SQLConf.PARQUET_FILTER_PUSHDOWN_ENABLED.key -> "true", - SQLConf.PARQUET_FILTER_PUSHDOWN_DATE_ENABLED.key -> "true", - SQLConf.PARQUET_FILTER_PUSHDOWN_TIMESTAMP_ENABLED.key -> "true", - SQLConf.PARQUET_FILTER_PUSHDOWN_DECIMAL_ENABLED.key -> "true", - SQLConf.PARQUET_FILTER_PUSHDOWN_STRING_STARTSWITH_ENABLED.key -> "true", - // Disable adding filters from constraints because it adds, for instance, - // is-not-null to pushed filters, which makes it hard to test if the pushed - // filter is expected or not (this had to be fixed with SPARK-13495). - SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> InferFiltersFromConstraints.ruleName, - SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false", - SQLConf.NESTED_PREDICATE_PUSHDOWN_FILE_SOURCE_LIST.key -> pushdownDsList - ) { - val query = df - .select(output.map(e => Column(e)): _*) - .where(Column(predicate)) - - val nestedOrAttributes = predicate.collectFirst { - case g: GetStructField => g - case a: Attribute => a - } - assert(nestedOrAttributes.isDefined, "No GetStructField nor Attribute is detected.") - - val parsed = - parseColumnPath(PushableColumnAndNestedColumn.unapply(nestedOrAttributes.get).get) - - val containsNestedColumnOrDot = parsed.length > 1 || parsed(0).contains(".") - - var maybeRelation: Option[HadoopFsRelation] = None - val maybeAnalyzedPredicate = query.queryExecution.optimizedPlan - .collect { - case PhysicalOperation( - _, - filters, - LogicalRelation(relation: HadoopFsRelation, _, _, _)) => - maybeRelation = Some(relation) - filters - } - .flatten - .reduceLeftOption(_ && _) - assert(maybeAnalyzedPredicate.isDefined, "No filter is analyzed from the given query") - - val (_, selectedFilters, _) = - DataSourceStrategy.selectFilters(maybeRelation.get, maybeAnalyzedPredicate.toSeq) - // If predicates contains nested column or dot, we push down the predicates only if - // "parquet" is in `NESTED_PREDICATE_PUSHDOWN_V1_SOURCE_LIST`. - if (nestedPredicatePushdown || !containsNestedColumnOrDot) { - assert(selectedFilters.nonEmpty, "No filter is pushed down") - val schema = new SparkToParquetSchemaConverter(conf).convert(df.schema) - val parquetFilters = createParquetFilters(schema) - // In this test suite, all the simple predicates are convertible here. - assert(parquetFilters.convertibleFilters(selectedFilters) === selectedFilters) - val pushedParquetFilters = selectedFilters.map { - pred => - val maybeFilter = parquetFilters.createFilter(pred) - assert(maybeFilter.isDefined, s"Couldn't generate filter predicate for $pred") - maybeFilter.get - } - // Doesn't bother checking type parameters here (e.g. `Eq[Integer]`) - assert( - pushedParquetFilters.exists(_.getClass === filterClass), - s"${pushedParquetFilters.map(_.getClass).toList} did not contain $filterClass.") - - checker(stripSparkFilter(query), expected) - } else { - assert(selectedFilters.isEmpty, "There is filter pushed down") - } - } - } - } -} - -@ExtendedSQLTest -class GlutenParquetV2FilterSuite extends GlutenParquetFilterSuite with GlutenSQLTestsBaseTrait { - // TODO: enable Parquet V2 write path after file source V2 writers are workable. - override def sparkConf: SparkConf = - super.sparkConf - .set(SQLConf.USE_V1_SOURCE_LIST, "") - - override def checkFilterPredicate( - df: DataFrame, - predicate: Predicate, - filterClass: Class[_ <: FilterPredicate], - checker: (DataFrame, Seq[Row]) => Unit, - expected: Seq[Row]): Unit = { - val output = predicate.collect { case a: Attribute => a }.distinct - - withSQLConf( - SQLConf.PARQUET_FILTER_PUSHDOWN_ENABLED.key -> "true", - SQLConf.PARQUET_FILTER_PUSHDOWN_DATE_ENABLED.key -> "true", - SQLConf.PARQUET_FILTER_PUSHDOWN_TIMESTAMP_ENABLED.key -> "true", - SQLConf.PARQUET_FILTER_PUSHDOWN_DECIMAL_ENABLED.key -> "true", - SQLConf.PARQUET_FILTER_PUSHDOWN_STRING_STARTSWITH_ENABLED.key -> "true", - // Disable adding filters from constraints because it adds, for instance, - // is-not-null to pushed filters, which makes it hard to test if the pushed - // filter is expected or not (this had to be fixed with SPARK-13495). - SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> InferFiltersFromConstraints.ruleName, - SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false" - ) { - val query = df - .select(output.map(e => Column(e)): _*) - .where(Column(predicate)) - - query.queryExecution.optimizedPlan.collectFirst { - case PhysicalOperation( - _, - filters, - DataSourceV2ScanRelation(_, scan: ParquetScan, _, None)) => - assert(filters.nonEmpty, "No filter is analyzed from the given query") - val sourceFilters = filters.flatMap(DataSourceStrategy.translateFilter(_, true)).toArray - val pushedFilters = scan.pushedFilters - assert(pushedFilters.nonEmpty, "No filter is pushed down") - val schema = new SparkToParquetSchemaConverter(conf).convert(df.schema) - val parquetFilters = createParquetFilters(schema) - // In this test suite, all the simple predicates are convertible here. - assert(parquetFilters.convertibleFilters(sourceFilters) === pushedFilters) - val pushedParquetFilters = pushedFilters.map { - pred => - val maybeFilter = parquetFilters.createFilter(pred) - assert(maybeFilter.isDefined, s"Couldn't generate filter predicate for $pred") - maybeFilter.get - } - // Doesn't bother checking type parameters here (e.g. `Eq[Integer]`) - assert( - pushedParquetFilters.exists(_.getClass === filterClass), - s"${pushedParquetFilters.map(_.getClass).toList} did not contain $filterClass.") - - checker(stripSparkFilter(query), expected) - - case _ => - throw new AnalysisException("Can not match ParquetTable in the query.") - } - } - } - - /** - * Takes a sequence of products `data` to generate multi-level nested dataframes as new test data. - * It tests both non-nested and nested dataframes which are written and read back with Parquet - * datasource. - * - * This is different from [[ParquetTest.withParquetDataFrame]] which does not test nested cases. - */ - private def withNestedParquetDataFrame[T <: Product: ClassTag: TypeTag](data: Seq[T])( - runTest: (DataFrame, String, Any => Any) => Unit): Unit = - withNestedParquetDataFrame(spark.createDataFrame(data))(runTest) - - private def withNestedParquetDataFrame(inputDF: DataFrame)( - runTest: (DataFrame, String, Any => Any) => Unit): Unit = { - withNestedDataFrame(inputDF).foreach { - case (newDF, colName, resultFun) => - withTempPath { - file => - newDF.write.format(dataSourceName).save(file.getCanonicalPath) - readParquetFile(file.getCanonicalPath)(df => runTest(df, colName, resultFun)) - } - } - } - - testGluten("filter pushdown - date") { - implicit class StringToDate(s: String) { - def date: Date = Date.valueOf(s) - } - - val data = Seq("1000-01-01", "2018-03-19", "2018-03-20", "2018-03-21") - import testImplicits._ - - // Velox backend does not support rebaseMode being LEGACY. - Seq(false, true).foreach { - java8Api => - Seq(CORRECTED).foreach { - rebaseMode => - withSQLConf( - SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString, - SQLConf.PARQUET_REBASE_MODE_IN_WRITE.key -> rebaseMode.toString) { - val dates = data.map(i => Tuple1(Date.valueOf(i))).toDF() - withNestedParquetDataFrame(dates) { - case (inputDF, colName, fun) => - implicit val df: DataFrame = inputDF - - def resultFun(dateStr: String): Any = { - val parsed = if (java8Api) LocalDate.parse(dateStr) else Date.valueOf(dateStr) - fun(parsed) - } - - val dateAttr: Expression = df(colName).expr - assert(df(colName).expr.dataType === DateType) - - checkFilterPredicate(dateAttr.isNull, classOf[Eq[_]], Seq.empty[Row]) - checkFilterPredicate( - dateAttr.isNotNull, - classOf[NotEq[_]], - data.map(i => Row.apply(resultFun(i)))) - - checkFilterPredicate( - dateAttr === "1000-01-01".date, - classOf[Eq[_]], - resultFun("1000-01-01")) - logWarning(s"java8Api: $java8Api, rebaseMode, $rebaseMode") - checkFilterPredicate( - dateAttr <=> "1000-01-01".date, - classOf[Eq[_]], - resultFun("1000-01-01")) - checkFilterPredicate( - dateAttr =!= "1000-01-01".date, - classOf[NotEq[_]], - Seq("2018-03-19", "2018-03-20", "2018-03-21").map(i => Row.apply(resultFun(i)))) - - checkFilterPredicate( - dateAttr < "2018-03-19".date, - classOf[Lt[_]], - resultFun("1000-01-01")) - checkFilterPredicate( - dateAttr > "2018-03-20".date, - classOf[Gt[_]], - resultFun("2018-03-21")) - checkFilterPredicate( - dateAttr <= "1000-01-01".date, - classOf[LtEq[_]], - resultFun("1000-01-01")) - checkFilterPredicate( - dateAttr >= "2018-03-21".date, - classOf[GtEq[_]], - resultFun("2018-03-21")) - - checkFilterPredicate( - Literal("1000-01-01".date) === dateAttr, - classOf[Eq[_]], - resultFun("1000-01-01")) - checkFilterPredicate( - Literal("1000-01-01".date) <=> dateAttr, - classOf[Eq[_]], - resultFun("1000-01-01")) - checkFilterPredicate( - Literal("2018-03-19".date) > dateAttr, - classOf[Lt[_]], - resultFun("1000-01-01")) - checkFilterPredicate( - Literal("2018-03-20".date) < dateAttr, - classOf[Gt[_]], - resultFun("2018-03-21")) - checkFilterPredicate( - Literal("1000-01-01".date) >= dateAttr, - classOf[LtEq[_]], - resultFun("1000-01-01")) - checkFilterPredicate( - Literal("2018-03-21".date) <= dateAttr, - classOf[GtEq[_]], - resultFun("2018-03-21")) - - checkFilterPredicate( - !(dateAttr < "2018-03-21".date), - classOf[GtEq[_]], - resultFun("2018-03-21")) - checkFilterPredicate( - dateAttr < "2018-03-19".date || dateAttr > "2018-03-20".date, - classOf[Operators.Or], - Seq(Row(resultFun("1000-01-01")), Row(resultFun("2018-03-21")))) - - Seq(3, 20).foreach { - threshold => - withSQLConf( - SQLConf.PARQUET_FILTER_PUSHDOWN_INFILTERTHRESHOLD.key -> s"$threshold") { - checkFilterPredicate( - In( - dateAttr, - Array( - "2018-03-19".date, - "2018-03-20".date, - "2018-03-21".date, - "2018-03-22".date).map(Literal.apply)), - if (threshold == 3) classOf[Operators.And] else classOf[Operators.Or], - Seq( - Row(resultFun("2018-03-19")), - Row(resultFun("2018-03-20")), - Row(resultFun("2018-03-21"))) - ) - } - } - } - } - } - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetIOSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetIOSuite.scala deleted file mode 100644 index ad1ae40f928..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetIOSuite.scala +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.parquet - -import org.apache.spark.sql._ - -/** A test suite that tests basic Parquet I/O. */ -class GlutenParquetIOSuite extends ParquetIOSuite with GlutenSQLTestsBaseTrait { - override protected def testFile(fileName: String): String = { - getWorkspaceFilePath("sql", "core", "src", "test", "resources").toString + "/" + fileName - } - - override protected def readResourceParquetFile(name: String): DataFrame = { - spark.read.parquet(testFile(name)) - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetInteroperabilitySuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetInteroperabilitySuite.scala deleted file mode 100644 index 051343dafb0..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetInteroperabilitySuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.parquet - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenParquetInteroperabilitySuite - extends ParquetInteroperabilitySuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetPartitionDiscoverySuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetPartitionDiscoverySuite.scala deleted file mode 100644 index 5af8fa48c53..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetPartitionDiscoverySuite.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.parquet - -import org.apache.spark.sql._ - -class GlutenParquetV1PartitionDiscoverySuite - extends ParquetV1PartitionDiscoverySuite - with GlutenSQLTestsBaseTrait {} - -class GlutenParquetV2PartitionDiscoverySuite - extends ParquetV2PartitionDiscoverySuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetProtobufCompatibilitySuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetProtobufCompatibilitySuite.scala deleted file mode 100644 index f175910792b..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetProtobufCompatibilitySuite.scala +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.parquet - -import org.apache.spark.sql.{DataFrame, GlutenSQLTestsBaseTrait} - -class GlutenParquetProtobufCompatibilitySuite - extends ParquetProtobufCompatibilitySuite - with GlutenSQLTestsBaseTrait { - override protected def readResourceParquetFile(name: String): DataFrame = { - spark.read.parquet( - getWorkspaceFilePath("sql", "core", "src", "test", "resources").toString + "/" + name) - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetQuerySuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetQuerySuite.scala deleted file mode 100644 index d5ba262ceb6..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetQuerySuite.scala +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.parquet - -import org.apache.spark.sql._ - -/** A test suite that tests various Parquet queries. */ -class GlutenParquetV1QuerySuite extends ParquetV1QuerySuite with GlutenSQLTestsBaseTrait { - import testImplicits._ - - testGluten( - "SPARK-26677: negated null-safe equality comparison should not filter matched row groups") { - withAllParquetReaders { - withTempPath { - path => - // Repeated values for dictionary encoding. - Seq(Some("A"), Some("A"), None).toDF.repartition(1).write.parquet(path.getAbsolutePath) - val df = spark.read.parquet(path.getAbsolutePath) - checkAnswer(stripSparkFilter(df.where("NOT (value <=> 'A')")), Seq(null: String).toDF) - } - } - } -} - -class GlutenParquetV2QuerySuite extends ParquetV2QuerySuite with GlutenSQLTestsBaseTrait { - import testImplicits._ - - testGluten( - "SPARK-26677: negated null-safe equality comparison should not filter matched row groups") { - withAllParquetReaders { - withTempPath { - path => - // Repeated values for dictionary encoding. - Seq(Some("A"), Some("A"), None).toDF.repartition(1).write.parquet(path.getAbsolutePath) - val df = spark.read.parquet(path.getAbsolutePath) - checkAnswer(stripSparkFilter(df.where("NOT (value <=> 'A')")), Seq(null: String).toDF) - } - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRebaseDatetimeSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRebaseDatetimeSuite.scala deleted file mode 100644 index a54af1f43da..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRebaseDatetimeSuite.scala +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.parquet - -import org.apache.spark.sql.{GlutenSQLTestsBaseTrait, Row} -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy -import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy.{CORRECTED, EXCEPTION, LEGACY} - -import java.sql.Date - -class GlutenParquetRebaseDatetimeV1Suite - extends ParquetRebaseDatetimeV1Suite - with GlutenSQLTestsBaseTrait { - - import testImplicits._ - - override protected def getResourceParquetFilePath(name: String): String = { - getWorkspaceFilePath("sql", "core", "src", "test", "resources").toString + "/" + name - } - - private def inReadConfToOptions( - conf: String, - mode: LegacyBehaviorPolicy.Value): Map[String, String] = conf match { - case SQLConf.PARQUET_INT96_REBASE_MODE_IN_READ.key => - Map(ParquetOptions.INT96_REBASE_MODE -> mode.toString) - case _ => Map(ParquetOptions.DATETIME_REBASE_MODE -> mode.toString) - } - - private def runInMode(conf: String, modes: Seq[LegacyBehaviorPolicy.Value])( - f: Map[String, String] => Unit): Unit = { - modes.foreach(mode => withSQLConf(conf -> mode.toString)(f(Map.empty))) - withSQLConf(conf -> EXCEPTION.toString) { - modes.foreach(mode => f(inReadConfToOptions(conf, mode))) - } - } - - // gluten does not consider file metadata which indicates needs rebase or not - // it only supports write the parquet file as CORRECTED - testGluten("SPARK-31159: rebasing dates in write") { - val N = 8 - Seq(false, true).foreach { - dictionaryEncoding => - withTempPath { - dir => - val path = dir.getAbsolutePath - withSQLConf(SQLConf.PARQUET_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString) { - Seq - .tabulate(N)(_ => "1001-01-01") - .toDF("dateS") - .select($"dateS".cast("date").as("date")) - .repartition(1) - .write - .option("parquet.enable.dictionary", dictionaryEncoding) - .parquet(path) - } - - withAllParquetReaders { - // The file metadata indicates if it needs rebase or not, so we can always get the - // correct result regardless of the "rebase mode" config. - runInMode( - SQLConf.PARQUET_REBASE_MODE_IN_READ.key, - Seq(LEGACY, CORRECTED, EXCEPTION)) { - options => - checkAnswer( - spark.read.options(options).parquet(path), - Seq.tabulate(N)(_ => Row(Date.valueOf("1001-01-01")))) - } - - // Force to not rebase to prove the written datetime values are rebased - // and we will get wrong result if we don't rebase while reading. - // gluten not support this mode -// withSQLConf("spark.test.forceNoRebase" -> "true") { -// checkAnswer( -// spark.read.parquet(path), -// Seq.tabulate(N)(_ => Row(Date.valueOf("1001-01-07")))) -// } - } - } - } - } -} - -class GlutenParquetRebaseDatetimeV2Suite - extends ParquetRebaseDatetimeV2Suite - with GlutenSQLTestsBaseTrait { - - override protected def getResourceParquetFilePath(name: String): String = { - getWorkspaceFilePath("sql", "core", "src", "test", "resources").toString + "/" + name - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetSchemaPruningSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetSchemaPruningSuite.scala deleted file mode 100644 index 495b4c3dfe1..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetSchemaPruningSuite.scala +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.parquet - -import org.apache.gluten.execution.{BatchScanExecTransformer, FileSourceScanExecTransformer} - -import org.apache.spark.SparkConf -import org.apache.spark.sql.{DataFrame, GlutenSQLTestsBaseTrait} -import org.apache.spark.sql.catalyst.parser.CatalystSqlParser -import org.apache.spark.sql.execution.FileSourceScanExec -import org.apache.spark.sql.execution.datasources.v2.BatchScanExec -import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetScan -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.tags.ExtendedSQLTest - -@ExtendedSQLTest -class GlutenParquetV1SchemaPruningSuite - extends ParquetV1SchemaPruningSuite - with GlutenSQLTestsBaseTrait { - // disable column reader for nested type - override protected val vectorizedReaderNestedEnabledKey: String = - SQLConf.PARQUET_VECTORIZED_READER_NESTED_COLUMN_ENABLED.key + "_DISABLED" - override def sparkConf: SparkConf = { - super.sparkConf.set("spark.memory.offHeap.size", "3g") - } - - override def checkScanSchemata(df: DataFrame, expectedSchemaCatalogStrings: String*): Unit = { - val fileSourceScanSchemata = - collect(df.queryExecution.executedPlan) { - case scan: FileSourceScanExec => scan.requiredSchema - case scan: FileSourceScanExecTransformer => scan.requiredSchema - } - assert( - fileSourceScanSchemata.size === expectedSchemaCatalogStrings.size, - s"Found ${fileSourceScanSchemata.size} file sources in dataframe, " + - s"but expected $expectedSchemaCatalogStrings" - ) - fileSourceScanSchemata.zip(expectedSchemaCatalogStrings).foreach { - case (scanSchema, expectedScanSchemaCatalogString) => - val expectedScanSchema = CatalystSqlParser.parseDataType(expectedScanSchemaCatalogString) - implicit val equality = schemaEquality - assert(scanSchema === expectedScanSchema) - } - } -} - -@ExtendedSQLTest -class GlutenParquetV2SchemaPruningSuite - extends ParquetV2SchemaPruningSuite - with GlutenSQLTestsBaseTrait { - override protected val vectorizedReaderNestedEnabledKey: String = - SQLConf.PARQUET_VECTORIZED_READER_NESTED_COLUMN_ENABLED.key + "_DISABLED" - override def sparkConf: SparkConf = { - super.sparkConf.set("spark.memory.offHeap.size", "3g") - } - - override def checkScanSchemata(df: DataFrame, expectedSchemaCatalogStrings: String*): Unit = { - val fileSourceScanSchemata = - collect(df.queryExecution.executedPlan) { - case scan: BatchScanExec => scan.scan.asInstanceOf[ParquetScan].readDataSchema - case scan: BatchScanExecTransformer => scan.scan.asInstanceOf[ParquetScan].readDataSchema - } - assert( - fileSourceScanSchemata.size === expectedSchemaCatalogStrings.size, - s"Found ${fileSourceScanSchemata.size} file sources in dataframe, " + - s"but expected $expectedSchemaCatalogStrings" - ) - fileSourceScanSchemata.zip(expectedSchemaCatalogStrings).foreach { - case (scanSchema, expectedScanSchemaCatalogString) => - val expectedScanSchema = CatalystSqlParser.parseDataType(expectedScanSchemaCatalogString) - implicit val equality = schemaEquality - assert(scanSchema === expectedScanSchema) - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetSchemaSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetSchemaSuite.scala deleted file mode 100644 index dbf520e9109..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetSchemaSuite.scala +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.parquet - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenParquetSchemaInferenceSuite - extends ParquetSchemaInferenceSuite - with GlutenSQLTestsBaseTrait {} - -class GlutenParquetSchemaSuite extends ParquetSchemaSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetThriftCompatibilitySuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetThriftCompatibilitySuite.scala deleted file mode 100644 index adb61baa49e..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetThriftCompatibilitySuite.scala +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.parquet - -import org.apache.spark.sql.{GlutenSQLTestsBaseTrait, Row} - -class GlutenParquetThriftCompatibilitySuite - extends ParquetThriftCompatibilitySuite - with GlutenSQLTestsBaseTrait { - - private val parquetFilePath = - getWorkspaceFilePath("sql", "core", "src", "test", "resources").toString + - "/test-data/parquet-thrift-compat.snappy.parquet" - - // TODO: https://github.com/apache/gluten/issues/11865 - ignoreGluten("Read Parquet file generated by parquet-thrift") { - logInfo(s"""Schema of the Parquet file written by parquet-thrift: - |${readParquetSchema(parquetFilePath.toString)} - """.stripMargin) - - checkAnswer( - spark.read.parquet(parquetFilePath.toString), - (0 until 10).map { - i => - val suits = Array("SPADES", "HEARTS", "DIAMONDS", "CLUBS") - - val nonNullablePrimitiveValues = Seq( - i % 2 == 0, - i.toByte, - (i + 1).toShort, - i + 2, - i.toLong * 10, - i.toDouble + 0.2d, - // Thrift `BINARY` values are actually unencoded `STRING` values, and thus are always - // treated as `BINARY (UTF8)` in parquet-thrift, since parquet-thrift always assume - // Thrift `STRING`s are encoded using UTF-8. - s"val_$i", - s"val_$i", - // Thrift ENUM values are converted to Parquet binaries containing UTF-8 strings - suits(i % 4) - ) - - val nullablePrimitiveValues = if (i % 3 == 0) { - Seq.fill(nonNullablePrimitiveValues.length)(null) - } else { - nonNullablePrimitiveValues - } - - val complexValues = Seq( - Seq.tabulate(3)(n => s"arr_${i + n}"), - // Thrift `SET`s are converted to Parquet `LIST`s - Seq(i), - Seq.tabulate(3)(n => (i + n: Integer) -> s"val_${i + n}").toMap, - Seq - .tabulate(3) { - n => - (i + n) -> Seq.tabulate(3) { - m => Row(Seq.tabulate(3)(j => i + j + m), s"val_${i + m}") - } - } - .toMap - ) - - Row(nonNullablePrimitiveValues ++ nullablePrimitiveValues ++ complexValues: _*) - } - ) - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetVectorizedSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetVectorizedSuite.scala deleted file mode 100644 index a0cf738e52a..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetVectorizedSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.parquet - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenParquetVectorizedSuite extends ParquetVectorizedSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/text/GlutenTextSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/text/GlutenTextSuite.scala deleted file mode 100644 index 24a18608137..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/text/GlutenTextSuite.scala +++ /dev/null @@ -1,282 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.text - -import org.apache.spark.{SparkConf, TestUtils} -import org.apache.spark.sql.{AnalysisException, DataFrame, GlutenSQLTestsBaseTrait, QueryTest, Row, SaveMode} -import org.apache.spark.sql.execution.datasources.CommonFileDataSourceSuite -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.test.SharedSparkSession -import org.apache.spark.sql.types.{StringType, StructType} -import org.apache.spark.util.Utils - -import org.apache.hadoop.io.SequenceFile.CompressionType -import org.apache.hadoop.io.compress.GzipCodec - -import java.io.File -import java.nio.charset.StandardCharsets -import java.nio.file.Files - -abstract class GlutenTextSuite - extends QueryTest - with SharedSparkSession - with CommonFileDataSourceSuite - with GlutenSQLTestsBaseTrait { - import testImplicits._ - - override protected def dataSourceFormat = "text" - - testGluten("reading text file") { - verifyFrame(spark.read.format("text").load(testFile)) - } - - testGluten("SQLContext.read.text() API") { - verifyFrame(spark.read.text(testFile)) - } - - testGluten("SPARK-12562 verify write.text() can handle column name beyond `value`") { - val df = spark.read.text(testFile).withColumnRenamed("value", "adwrasdf") - - val tempFile = Utils.createTempDir() - tempFile.delete() - df.write.text(tempFile.getCanonicalPath) - verifyFrame(spark.read.text(tempFile.getCanonicalPath)) - - Utils.deleteRecursively(tempFile) - } - - testGluten("error handling for invalid schema") { - val tempFile = Utils.createTempDir() - tempFile.delete() - - val df = spark.range(2) - intercept[AnalysisException] { - df.write.text(tempFile.getCanonicalPath) - } - - intercept[AnalysisException] { - spark.range(2).select(df("id"), df("id") + 1).write.text(tempFile.getCanonicalPath) - } - } - - testGluten("reading partitioned data using read.textFile()") { - val ds = spark.read.textFile(textPartitioned) - val data = ds.collect() - - assert(ds.schema == new StructType().add("value", StringType)) - assert(data.length == 2) - } - - testGluten("support for partitioned reading using read.text()") { - val df = spark.read.text(textPartitioned) - val data = df.filter("year = '2015'").select("value").collect() - - assert(data(0) == Row("2015-test")) - assert(data.length == 1) - } - - testGluten("SPARK-13503 Support to specify the option for compression codec for TEXT") { - val testDf = spark.read.text(testFile) - val extensionNameMap = Map("bzip2" -> ".bz2", "deflate" -> ".deflate", "gzip" -> ".gz") - extensionNameMap.foreach { - case (codecName, extension) => - val tempDir = Utils.createTempDir() - val tempDirPath = tempDir.getAbsolutePath - testDf.write.option("compression", codecName).mode(SaveMode.Overwrite).text(tempDirPath) - val compressedFiles = new File(tempDirPath).listFiles() - assert(compressedFiles.exists(_.getName.endsWith(s".txt$extension"))) - verifyFrame(spark.read.text(tempDirPath)) - } - - val errMsg = intercept[IllegalArgumentException] { - val tempDirPath = Utils.createTempDir().getAbsolutePath - testDf.write.option("compression", "illegal").mode(SaveMode.Overwrite).text(tempDirPath) - } - assert( - errMsg.getMessage.contains("Codec [illegal] is not available. " + - "Known codecs are")) - } - - testGluten("SPARK-13543 Write the output as uncompressed via option()") { - val extraOptions = Map[String, String]( - "mapreduce.output.fileoutputformat.compress" -> "true", - "mapreduce.output.fileoutputformat.compress.type" -> CompressionType.BLOCK.toString, - "mapreduce.map.output.compress" -> "true", - "mapreduce.output.fileoutputformat.compress.codec" -> classOf[GzipCodec].getName, - "mapreduce.map.output.compress.codec" -> classOf[GzipCodec].getName - ) - withTempDir { - dir => - val testDf = spark.read.text(testFile) - val tempDirPath = dir.getAbsolutePath - testDf.write - .option("compression", "none") - .options(extraOptions) - .mode(SaveMode.Overwrite) - .text(tempDirPath) - val compressedFiles = new File(tempDirPath).listFiles() - assert(compressedFiles.exists(!_.getName.endsWith(".txt.gz"))) - verifyFrame(spark.read.options(extraOptions).text(tempDirPath)) - } - } - - testGluten("case insensitive option") { - val extraOptions = Map[String, String]( - "mApReDuCe.output.fileoutputformat.compress" -> "true", - "mApReDuCe.output.fileoutputformat.compress.type" -> CompressionType.BLOCK.toString, - "mApReDuCe.map.output.compress" -> "true", - "mApReDuCe.output.fileoutputformat.compress.codec" -> classOf[GzipCodec].getName, - "mApReDuCe.map.output.compress.codec" -> classOf[GzipCodec].getName - ) - withTempDir { - dir => - val testDf = spark.read.text(testFile) - val tempDirPath = dir.getAbsolutePath - testDf.write - .option("CoMpReSsIoN", "none") - .options(extraOptions) - .mode(SaveMode.Overwrite) - .text(tempDirPath) - val compressedFiles = new File(tempDirPath).listFiles() - assert(compressedFiles.exists(!_.getName.endsWith(".txt.gz"))) - verifyFrame(spark.read.options(extraOptions).text(tempDirPath)) - } - } - - testGluten("SPARK-14343: select partitioning column") { - withTempPath { - dir => - val path = dir.getCanonicalPath - val ds1 = spark.range(1).selectExpr("CONCAT('val_', id)") - ds1.write.text(s"$path/part=a") - ds1.write.text(s"$path/part=b") - - checkAnswer( - spark.read.format("text").load(path).select($"part"), - Row("a") :: Row("b") :: Nil) - } - } - - testGluten("SPARK-15654: should not split gz files") { - withTempDir { - dir => - val path = dir.getCanonicalPath - val df1 = spark.range(0, 1000).selectExpr("CAST(id AS STRING) AS s") - df1.write.option("compression", "gzip").mode("overwrite").text(path) - - val expected = df1.collect() - Seq(10, 100, 1000).foreach { - bytes => - withSQLConf(SQLConf.FILES_MAX_PARTITION_BYTES.key -> bytes.toString) { - val df2 = spark.read.format("text").load(path) - checkAnswer(df2, expected) - } - } - } - } - - def testLineSeparator(lineSep: String): Unit = { - testGluten(s"SPARK-23577: Support line separator - lineSep: '$lineSep'") { - // Read - val values = Seq("a", "b", "\nc") - val data = values.mkString(lineSep) - val dataWithTrailingLineSep = s"$data$lineSep" - Seq(data, dataWithTrailingLineSep).foreach { - lines => - withTempPath { - path => - Files.write(path.toPath, lines.getBytes(StandardCharsets.UTF_8)) - val df = spark.read.option("lineSep", lineSep).text(path.getAbsolutePath) - checkAnswer(df, Seq("a", "b", "\nc").toDF()) - } - } - - // Write - withTempPath { - path => - values.toDF().coalesce(1).write.option("lineSep", lineSep).text(path.getAbsolutePath) - val partFile = - TestUtils.recursiveList(path).filter(f => f.getName.startsWith("part-")).head - val readBack = new String(Files.readAllBytes(partFile.toPath), StandardCharsets.UTF_8) - assert(readBack === s"a${lineSep}b$lineSep\nc$lineSep") - } - - // Roundtrip - withTempPath { - path => - val df = values.toDF() - df.write.option("lineSep", lineSep).text(path.getAbsolutePath) - val readBack = spark.read.option("lineSep", lineSep).text(path.getAbsolutePath) - checkAnswer(df, readBack) - } - } - } - - // scalastyle:off nonascii - Seq("|", "^", "::", "!!!@3", 0x1e.toChar.toString, "아").foreach { - lineSep => testLineSeparator(lineSep) - } - // scalastyle:on nonascii - - // Rewrite for file locating. - private def testFile: String = { - getWorkspaceFilePath( - "sql", - "core", - "src", - "test", - "resources").toString + "/test-data/text-suite.txt" - } - - // Added for file locating. - private def textPartitioned: String = { - getWorkspaceFilePath( - "sql", - "core", - "src", - "test", - "resources").toString + "/test-data/text-partitioned" - } - - /** Verifies data and schema. */ - private def verifyFrame(df: DataFrame): Unit = { - // schema - assert(df.schema == new StructType().add("value", StringType)) - - // verify content - val data = df.collect() - assert(data(0) == Row("This is a test file for the text data source")) - assert(data(1) == Row("1+1")) - // scalastyle:off nonascii - assert(data(2) == Row("数据砖头")) - // scalastyle:on nonascii - assert(data(3) == Row("\"doh\"")) - assert(data.length == 4) - } -} - -class GlutenTextV1Suite extends GlutenTextSuite with GlutenSQLTestsBaseTrait { - override def sparkConf: SparkConf = - super.sparkConf - .set(SQLConf.USE_V1_SOURCE_LIST, "text") -} - -class GlutenTextV2Suite extends GlutenTextSuite with GlutenSQLTestsBaseTrait { - override def sparkConf: SparkConf = - super.sparkConf - .set(SQLConf.USE_V1_SOURCE_LIST, "") -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/v2/GlutenDataSourceV2StrategySuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/v2/GlutenDataSourceV2StrategySuite.scala deleted file mode 100644 index f6d7db3849e..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/v2/GlutenDataSourceV2StrategySuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.v2 - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenDataSourceV2StrategySuite - extends DataSourceV2StrategySuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/v2/GlutenFileTableSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/v2/GlutenFileTableSuite.scala deleted file mode 100644 index bc6fcc3c0e9..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/v2/GlutenFileTableSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.v2 - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenFileTableSuite extends FileTableSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/v2/GlutenV2PredicateSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/v2/GlutenV2PredicateSuite.scala deleted file mode 100644 index e2d8186f687..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/v2/GlutenV2PredicateSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.v2 - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenV2PredicateSuite extends V2PredicateSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/exchange/GlutenEnsureRequirementsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/exchange/GlutenEnsureRequirementsSuite.scala deleted file mode 100644 index dc9e648fee8..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/exchange/GlutenEnsureRequirementsSuite.scala +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.exchange - -import org.apache.spark.SparkConf -import org.apache.spark.sql.GlutenSQLTestsBaseTrait -import org.apache.spark.sql.internal.SQLConf - -class GlutenEnsureRequirementsSuite extends EnsureRequirementsSuite with GlutenSQLTestsBaseTrait { - override def sparkConf: SparkConf = { - // Native SQL configs - super.sparkConf - .set("spark.sql.shuffle.partitions", "5") - } - - testGluten( - "SPARK-35675: EnsureRequirements remove shuffle should respect PartitioningCollection") { - import testImplicits._ - withSQLConf( - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - SQLConf.SHUFFLE_PARTITIONS.key -> "5", - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") { - val df1 = Seq((1, 2)).toDF("c1", "c2") - val df2 = Seq((1, 3)).toDF("c3", "c4") - val res = df1.join(df2, $"c1" === $"c3").repartition($"c1") - assert(res.queryExecution.executedPlan.collect { case s: ShuffleExchangeLike => s }.size == 2) - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala deleted file mode 100644 index c7aaf9ec5c0..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.joins - -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.utils.BackendTestUtils - -import org.apache.spark.sql.{GlutenTestsCommonTrait, SparkSession} -import org.apache.spark.sql.catalyst.optimizer.{ConstantFolding, ConvertToLocalRelation, NullPropagation} -import org.apache.spark.sql.internal.SQLConf - -/** - * This test needs setting for spark test home (its source code), e.g., appending the following - * setting for `mvn test`: -DargLine="-Dspark.test.home=/home/sparkuser/spark/". - * - * In addition, you also need build spark source code before running this test, e.g., with - * `./build/mvn -DskipTests clean package`. - */ -class GlutenBroadcastJoinSuite extends BroadcastJoinSuite with GlutenTestsCommonTrait { - - /** - * Create a new [[SparkSession]] running in local-cluster mode with unsafe and codegen enabled. - */ - override def beforeAll(): Unit = { - super.beforeAll() - val sparkBuilder = SparkSession - .builder() - .master("local-cluster[2,1,1024]") - .appName("Gluten-UT") - .master(s"local[2]") - .config(SQLConf.OPTIMIZER_EXCLUDED_RULES.key, ConvertToLocalRelation.ruleName) - .config("spark.driver.memory", "1G") - .config("spark.sql.adaptive.enabled", "true") - .config("spark.sql.shuffle.partitions", "1") - .config("spark.sql.files.maxPartitionBytes", "134217728") - .config("spark.memory.offHeap.enabled", "true") - .config("spark.memory.offHeap.size", "1024MB") - .config("spark.plugins", "org.apache.gluten.GlutenPlugin") - .config("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager") - .config("spark.sql.warehouse.dir", warehouse) - // Avoid static evaluation for literal input by spark catalyst. - .config( - "spark.sql.optimizer.excludedRules", - ConstantFolding.ruleName + "," + - NullPropagation.ruleName) - // Avoid the code size overflow error in Spark code generation. - .config("spark.sql.codegen.wholeStage", "false") - - spark = if (BackendTestUtils.isCHBackendLoaded()) { - sparkBuilder - .config("spark.io.compression.codec", "LZ4") - .config("spark.gluten.sql.columnar.backend.ch.worker.id", "1") - .config(GlutenConfig.NATIVE_VALIDATION_ENABLED.key, "false") - .config("spark.sql.files.openCostInBytes", "134217728") - .config("spark.unsafe.exceptionOnMemoryLeak", "true") - .getOrCreate() - } else { - sparkBuilder - .config("spark.unsafe.exceptionOnMemoryLeak", "true") - .getOrCreate() - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/joins/GlutenExistenceJoinSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/joins/GlutenExistenceJoinSuite.scala deleted file mode 100644 index 309af61a43a..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/joins/GlutenExistenceJoinSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.joins - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenExistenceJoinSuite extends ExistenceJoinSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/joins/GlutenInnerJoinSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/joins/GlutenInnerJoinSuite.scala deleted file mode 100644 index 745f550ae35..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/joins/GlutenInnerJoinSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.joins - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenInnerJoinSuite extends InnerJoinSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/joins/GlutenOuterJoinSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/joins/GlutenOuterJoinSuite.scala deleted file mode 100644 index c915c73695b..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/joins/GlutenOuterJoinSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.joins - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenOuterJoinSuite extends OuterJoinSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/python/GlutenBatchEvalPythonExecSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/python/GlutenBatchEvalPythonExecSuite.scala deleted file mode 100644 index d364262be58..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/python/GlutenBatchEvalPythonExecSuite.scala +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.python - -import org.apache.gluten.execution.{ColumnarToRowExecBase, FilterExecTransformer, RowToColumnarExecBase, WholeStageTransformer} - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait -import org.apache.spark.sql.catalyst.expressions.{And, AttributeReference, GreaterThan, In} -import org.apache.spark.sql.execution.{ColumnarInputAdapter, InputIteratorTransformer} - -class GlutenBatchEvalPythonExecSuite extends BatchEvalPythonExecSuite with GlutenSQLTestsBaseTrait { - - import testImplicits._ - - testGluten("Python UDF: push down deterministic FilterExecTransformer predicates") { - val df = Seq(("Hello", 4)) - .toDF("a", "b") - .where("dummyPythonUDF(b) and dummyPythonUDF(a) and a in (3, 4)") - val qualifiedPlanNodes = df.queryExecution.executedPlan.collect { - case f @ FilterExecTransformer( - And(_: AttributeReference, _: AttributeReference), - InputIteratorTransformer(ColumnarInputAdapter(r: RowToColumnarExecBase))) - if r.child.isInstanceOf[BatchEvalPythonExec] => - f - case b @ BatchEvalPythonExec(_, _, c: ColumnarToRowExecBase) => - c.child match { - case WholeStageTransformer(FilterExecTransformer(_: In, _), _) => b - } - } - assert(qualifiedPlanNodes.size == 2) - } - - testGluten("Nested Python UDF: push down deterministic FilterExecTransformer predicates") { - val df = Seq(("Hello", 4)) - .toDF("a", "b") - .where("dummyPythonUDF(a, dummyPythonUDF(a, b)) and a in (3, 4)") - val qualifiedPlanNodes = df.queryExecution.executedPlan.collect { - case f @ FilterExecTransformer( - _: AttributeReference, - InputIteratorTransformer(ColumnarInputAdapter(r: RowToColumnarExecBase))) - if r.child.isInstanceOf[BatchEvalPythonExec] => - f - case b @ BatchEvalPythonExec(_, _, c: ColumnarToRowExecBase) => - c.child match { - case WholeStageTransformer(FilterExecTransformer(_: In, _), _) => b - } - } - assert(qualifiedPlanNodes.size == 2) - } - - testGluten("Python UDF: no push down on non-deterministic") { - val df = Seq(("Hello", 4)) - .toDF("a", "b") - .where("b > 4 and dummyPythonUDF(a) and rand() > 0.3") - val qualifiedPlanNodes = df.queryExecution.executedPlan.collect { - case f @ FilterExecTransformer( - And(_: AttributeReference, _: GreaterThan), - InputIteratorTransformer(ColumnarInputAdapter(r: RowToColumnarExecBase))) - if r.child.isInstanceOf[BatchEvalPythonExec] => - f - case b @ BatchEvalPythonExec(_, _, c: ColumnarToRowExecBase) => - c.child match { - case WholeStageTransformer(_: FilterExecTransformer, _) => b - } - } - assert(qualifiedPlanNodes.size == 2) - } - - testGluten( - "Python UDF: push down on deterministic predicates after the first non-deterministic") { - val df = Seq(("Hello", 4)) - .toDF("a", "b") - .where("dummyPythonUDF(a) and rand() > 0.3 and b > 4") - - val qualifiedPlanNodes = df.queryExecution.executedPlan.collect { - case f @ FilterExecTransformer( - And(_: AttributeReference, _: GreaterThan), - InputIteratorTransformer(ColumnarInputAdapter(r: RowToColumnarExecBase))) - if r.child.isInstanceOf[BatchEvalPythonExec] => - f - case b @ BatchEvalPythonExec(_, _, c: ColumnarToRowExecBase) => - c.child match { - case WholeStageTransformer(_: FilterExecTransformer, _) => b - } - } - assert(qualifiedPlanNodes.size == 2) - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/python/GlutenExtractPythonUDFsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/python/GlutenExtractPythonUDFsSuite.scala deleted file mode 100644 index 1cd34bbf785..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/python/GlutenExtractPythonUDFsSuite.scala +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.python - -import org.apache.gluten.execution.{BatchScanExecTransformer, FileSourceScanExecTransformer} - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait -import org.apache.spark.sql.execution.SparkPlan -import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetScan -import org.apache.spark.sql.functions.col -import org.apache.spark.sql.internal.SQLConf - -class GlutenExtractPythonUDFsSuite extends ExtractPythonUDFsSuite with GlutenSQLTestsBaseTrait { - - import testImplicits._ - - def collectBatchExec(plan: SparkPlan): Seq[BatchEvalPythonExec] = plan.collect { - case b: BatchEvalPythonExec => b - } - - def collectColumnarArrowExec(plan: SparkPlan): Seq[EvalPythonExec] = plan.collect { - // To check for ColumnarArrowEvalPythonExec - case b: EvalPythonExec - if !b.isInstanceOf[ArrowEvalPythonExec] && !b.isInstanceOf[BatchEvalPythonExec] => - b - } - - testGluten("Chained Scalar Pandas UDFs should be combined to a single physical node") { - val df = Seq(("Hello", 4)).toDF("a", "b") - val df2 = df - .withColumn("c", scalarPandasUDF(col("a"))) - .withColumn("d", scalarPandasUDF(col("c"))) - val arrowEvalNodes = collectColumnarArrowExec(df2.queryExecution.executedPlan) - assert(arrowEvalNodes.size == 1) - } - - testGluten("Mixed Batched Python UDFs and Pandas UDF should be separate physical node") { - val df = Seq(("Hello", 4)).toDF("a", "b") - val df2 = df - .withColumn("c", batchedPythonUDF(col("a"))) - .withColumn("d", scalarPandasUDF(col("b"))) - - val pythonEvalNodes = collectBatchExec(df2.queryExecution.executedPlan) - val arrowEvalNodes = collectColumnarArrowExec(df2.queryExecution.executedPlan) - assert(pythonEvalNodes.size == 1) - assert(arrowEvalNodes.size == 1) - } - - testGluten( - "Independent Batched Python UDFs and Scalar Pandas UDFs should be combined separately") { - val df = Seq(("Hello", 4)).toDF("a", "b") - val df2 = df - .withColumn("c1", batchedPythonUDF(col("a"))) - .withColumn("c2", batchedPythonUDF(col("c1"))) - .withColumn("d1", scalarPandasUDF(col("a"))) - .withColumn("d2", scalarPandasUDF(col("d1"))) - - val pythonEvalNodes = collectBatchExec(df2.queryExecution.executedPlan) - val arrowEvalNodes = collectColumnarArrowExec(df2.queryExecution.executedPlan) - assert(pythonEvalNodes.size == 1) - assert(arrowEvalNodes.size == 1) - } - - testGluten("Dependent Batched Python UDFs and Scalar Pandas UDFs should not be combined") { - val df = Seq(("Hello", 4)).toDF("a", "b") - val df2 = df - .withColumn("c1", batchedPythonUDF(col("a"))) - .withColumn("d1", scalarPandasUDF(col("c1"))) - .withColumn("c2", batchedPythonUDF(col("d1"))) - .withColumn("d2", scalarPandasUDF(col("c2"))) - - val pythonEvalNodes = collectBatchExec(df2.queryExecution.executedPlan) - val arrowEvalNodes = collectColumnarArrowExec(df2.queryExecution.executedPlan) - assert(pythonEvalNodes.size == 2) - assert(arrowEvalNodes.size == 2) - } - - testGluten("Python UDF should not break column pruning/filter pushdown -- Parquet V2") { - withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") { - withTempPath { - f => - spark.range(10).select($"id".as("a"), $"id".as("b")).write.parquet(f.getCanonicalPath) - val df = spark.read.parquet(f.getCanonicalPath) - - withClue("column pruning") { - val query = df.filter(batchedPythonUDF($"a")).select($"a") - - val pythonEvalNodes = collectBatchExec(query.queryExecution.executedPlan) - assert(pythonEvalNodes.length == 1) - - val scanNodes = query.queryExecution.executedPlan.collect { - case scan: BatchScanExecTransformer => scan - } - assert(scanNodes.length == 1) - assert(scanNodes.head.output.map(_.name) == Seq("a")) - } - - withClue("filter pushdown") { - val query = df.filter($"a" > 1 && batchedPythonUDF($"a")) - val pythonEvalNodes = collectBatchExec(query.queryExecution.executedPlan) - assert(pythonEvalNodes.length == 1) - - val scanNodes = query.queryExecution.executedPlan.collect { - case scan: BatchScanExecTransformer => scan - } - assert(scanNodes.length == 1) - // $"a" is not null and $"a" > 1 - val filters = scanNodes.head.scan.asInstanceOf[ParquetScan].pushedFilters - assert(filters.length == 2) - assert(filters.flatMap(_.references).distinct === Array("a")) - } - } - } - } - - testGluten("Python UDF should not break column pruning/filter pushdown -- Parquet V1") { - withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "parquet") { - withTempPath { - f => - spark.range(10).select($"id".as("a"), $"id".as("b")).write.parquet(f.getCanonicalPath) - val df = spark.read.parquet(f.getCanonicalPath) - - withClue("column pruning") { - val query = df.filter(batchedPythonUDF($"a")).select($"a") - - val pythonEvalNodes = collectBatchExec(query.queryExecution.executedPlan) - assert(pythonEvalNodes.length == 1) - - val scanNodes = query.queryExecution.executedPlan.collect { - case scan: FileSourceScanExecTransformer => scan - } - assert(scanNodes.length == 1) - assert(scanNodes.head.output.map(_.name) == Seq("a")) - } - - withClue("filter pushdown") { - val query = df.filter($"a" > 1 && batchedPythonUDF($"a")) - val pythonEvalNodes = collectBatchExec(query.queryExecution.executedPlan) - assert(pythonEvalNodes.length == 1) - - val scanNodes = query.queryExecution.executedPlan.collect { - case scan: FileSourceScanExecTransformer => scan - } - assert(scanNodes.length == 1) - // $"a" is not null and $"a" > 1 - assert(scanNodes.head.dataFilters.length == 2) - assert( - scanNodes.head.dataFilters.flatMap(_.references.map(_.name)).distinct == Seq("a")) - } - } - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenCollapseProjectExecTransformerSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenCollapseProjectExecTransformerSuite.scala deleted file mode 100644 index 755cc26cb72..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenCollapseProjectExecTransformerSuite.scala +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.extension - -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.execution.ProjectExecTransformer -import org.apache.gluten.extension.columnar.CollapseProjectExecTransformer - -import org.apache.spark.sql.GlutenSQLTestsTrait -import org.apache.spark.sql.Row -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.execution.LocalTableScanExec -import org.apache.spark.sql.types._ - -class GlutenCollapseProjectExecTransformerSuite extends GlutenSQLTestsTrait { - - import testImplicits._ - - testGluten("Support ProjectExecTransformer collapse") { - val query = - """ - |SELECT - | o_orderpriority - |FROM - | orders - |WHERE - | o_shippriority >= 0 - | AND EXISTS ( - | SELECT - | * - | FROM - | lineitem - | WHERE - | l_orderkey = o_orderkey - | AND l_linenumber < 10 - | ) - |ORDER BY - | o_orderpriority - |LIMIT - | 100; - |""".stripMargin - - val ordersData = Seq[(Int, Int, String)]( - (30340, 1, "3-MEDIUM"), - (31140, 1, "1-URGENT"), - (31940, 1, "2-HIGH"), - (32740, 1, "3-MEDIUM"), - (33540, 1, "5-LOW"), - (34340, 1, "2-HIGH"), - (35140, 1, "3-MEDIUM"), - (35940, 1, "1-URGENT"), - (36740, 1, "3-MEDIUM"), - (37540, 1, "4-NOT SPECIFIED") - ) - val lineitemData = Seq[(Int, Int, String)]( - (30340, 1, "F"), - (31140, 4, "F"), - (31940, 7, "O"), - (32740, 6, "O"), - (33540, 2, "F"), - (34340, 3, "F"), - (35140, 1, "O"), - (35940, 2, "F"), - (36740, 3, "F"), - (37540, 5, "O") - ) - withTable("orders", "lineitem") { - ordersData - .toDF("o_orderkey", "o_shippriority", "o_orderpriority") - .write - .format("parquet") - .saveAsTable("orders") - lineitemData - .toDF("l_orderkey", "l_linenumber", "l_linestatus") - .write - .format("parquet") - .saveAsTable("lineitem") - Seq(true, false).foreach { - collapsed => - withSQLConf( - GlutenConfig.ENABLE_COLUMNAR_PROJECT_COLLAPSE.key -> collapsed.toString, - "spark.sql.autoBroadcastJoinThreshold" -> "-1") { - val df = sql(query) - checkAnswer( - df, - Seq( - Row("1-URGENT"), - Row("1-URGENT"), - Row("2-HIGH"), - Row("2-HIGH"), - Row("3-MEDIUM"), - Row("3-MEDIUM"), - Row("3-MEDIUM"), - Row("3-MEDIUM"), - Row("4-NOT SPECIFIED"), - Row("5-LOW") - ) - ) - assert( - getExecutedPlan(df).exists { - case _ @ProjectExecTransformer(_, _: ProjectExecTransformer) => true - case _ => false - } == !collapsed - ) - } - } - } - } - - testGluten("Collapse is blocked when CreateNamedStruct is nested inside wrapper expression") { - withSQLConf(GlutenConfig.ENABLE_COLUMNAR_PROJECT_COLLAPSE.key -> "true") { - val nameAttr = AttributeReference("name", StringType, nullable = true)() - val valueAttr = AttributeReference("value", IntegerType, nullable = false)() - val leaf = LocalTableScanExec(Seq(nameAttr, valueAttr), Seq.empty) - - // Inner project: Alias(If(IsNull(name), null, CreateNamedStruct(...)), "info") - val cns = CreateNamedStruct(Seq(Literal("n"), nameAttr, Literal("v"), valueAttr)) - val wrappedCns = If(IsNull(nameAttr), Literal.create(null, cns.dataType), cns) - val innerAlias = Alias(wrappedCns, "info")() - val innerProject = ProjectExecTransformer.createUnsafe(Seq(innerAlias), leaf) - - // Outer project: GetStructField(info, 0) AS n - val infoAttr = innerProject.output.find(_.name == "info").get - val outerExpr = Alias(GetStructField(infoAttr, 0, Some("n")), "n")() - val outerProject = ProjectExecTransformer.createUnsafe(Seq(outerExpr), innerProject) - - // Apply collapse rule - guard should block collapse - val result = CollapseProjectExecTransformer.apply(outerProject) - assert( - result match { - case ProjectExecTransformer(_, _: ProjectExecTransformer) => true - case _ => false - }, - "Expected stacked projects to remain uncollapsed when CreateNamedStruct " + - "is nested inside wrapper expression" - ) - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenSessionExtensionSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenSessionExtensionSuite.scala deleted file mode 100644 index 4924ee4c4f5..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenSessionExtensionSuite.scala +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.extension - -import org.apache.gluten.extension.injector.InjectorControl -import org.apache.gluten.utils.BackendTestUtils - -import org.apache.spark.SparkConf -import org.apache.spark.sql._ -import org.apache.spark.sql.internal.StaticSQLConf.SPARK_SESSION_EXTENSIONS - -class GlutenSessionExtensionSuite extends GlutenSQLTestsTrait { - - override def sparkConf: SparkConf = { - super.sparkConf - .set(SPARK_SESSION_EXTENSIONS.key, classOf[MyExtensions].getCanonicalName) - } - - testGluten("test gluten extensions") { - assert( - spark.sessionState.columnarRules - .exists(_.isInstanceOf[InjectorControl.DisablerAware])) - - assert(spark.sessionState.planner.strategies.contains(MySparkStrategy(spark))) - assert(spark.sessionState.analyzer.extendedResolutionRules.contains(MyRule(spark))) - assert(spark.sessionState.analyzer.postHocResolutionRules.contains(MyRule(spark))) - assert(spark.sessionState.analyzer.extendedCheckRules.contains(MyCheckRule(spark))) - assert(spark.sessionState.optimizer.batches.flatMap(_.rules).contains(MyRule(spark))) - if (BackendTestUtils.isCHBackendLoaded()) { - assert(spark.sessionState.sqlParser.isInstanceOf[InjectorControl.DisablerAware]) - } else { - assert(spark.sessionState.sqlParser.isInstanceOf[MyParser]) - } - assert( - spark.sessionState.functionRegistry - .lookupFunction(MyExtensions.myFunction._1) - .isDefined) - assert( - spark.sessionState.columnarRules.contains( - MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule()))) - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala deleted file mode 100644 index 2da437c45dc..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.gluten - -import org.apache.gluten.GlutenBuildInfo -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.events.GlutenPlanFallbackEvent -import org.apache.gluten.execution.FileSourceScanExecTransformer -import org.apache.gluten.utils.BackendTestUtils - -import org.apache.spark.SparkConf -import org.apache.spark.internal.config.UI.UI_ENABLED -import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent} -import org.apache.spark.sql.{GlutenSQLTestsTrait, Row} -import org.apache.spark.sql.execution.ProjectExec -import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper -import org.apache.spark.sql.execution.aggregate.HashAggregateExec -import org.apache.spark.sql.execution.ui.{GlutenSQLAppStatusStore, SparkListenerSQLExecutionStart} -import org.apache.spark.status.ElementTrackingStore - -import scala.collection.mutable.ArrayBuffer - -class GlutenFallbackSuite extends GlutenSQLTestsTrait with AdaptiveSparkPlanHelper { - override def sparkConf: SparkConf = { - super.sparkConf - .set(GlutenConfig.GLUTEN_UI_ENABLED.key, "true") - // The gluten ui event test suite expects the spark ui to be enable - .set(UI_ENABLED, true) - } - - testGluten("test fallback logging") { - val testAppender = new LogAppender("fallback reason") - withLogAppender(testAppender) { - withSQLConf( - GlutenConfig.COLUMNAR_FILESCAN_ENABLED.key -> "false", - GlutenConfig.VALIDATION_LOG_LEVEL.key -> "error") { - withTable("t") { - spark.range(10).write.format("parquet").saveAsTable("t") - sql("SELECT * FROM t").collect() - } - } - val msgRegex = """Validation failed for plan: Scan parquet default\.t\[QueryId=[0-9]+\],""" + - """ due to: \[FallbackByUserOptions\] Validation failed on node Scan parquet default\.t""" - assert(testAppender.loggingEvents.exists(_.getMessage.getFormattedMessage.matches(msgRegex))) - } - } - - testGluten("test fallback event") { - val kvStore = spark.sparkContext.statusStore.store.asInstanceOf[ElementTrackingStore] - val glutenStore = new GlutenSQLAppStatusStore(kvStore) - assert( - glutenStore - .buildInfo() - .info - .find(_._1 == "Gluten Version") - .exists(_._2 == GlutenBuildInfo.VERSION)) - - def runExecution(sqlString: String): Long = { - var id = 0L - val listener = new SparkListener { - override def onOtherEvent(event: SparkListenerEvent): Unit = { - event match { - case e: SparkListenerSQLExecutionStart => id = e.executionId - case _ => - } - } - } - spark.sparkContext.addSparkListener(listener) - try { - sql(sqlString).collect() - spark.sparkContext.listenerBus.waitUntilEmpty() - } finally { - spark.sparkContext.removeSparkListener(listener) - } - id - } - - withTable("t") { - spark.range(10).write.format("parquet").saveAsTable("t") - val id = runExecution("SELECT * FROM t") - val execution = glutenStore.execution(id) - assert(execution.isDefined) - assert(execution.get.numGlutenNodes == 1) - assert(execution.get.numFallbackNodes == 0) - assert(execution.get.fallbackNodeToReason.isEmpty) - - withSQLConf(GlutenConfig.COLUMNAR_FILESCAN_ENABLED.key -> "false") { - val id = runExecution("SELECT * FROM t") - val execution = glutenStore.execution(id) - assert(execution.isDefined) - assert(execution.get.numGlutenNodes == 0) - assert(execution.get.numFallbackNodes == 1) - val fallbackReason = execution.get.fallbackNodeToReason.head - assert(fallbackReason._1.contains("Scan parquet default.t")) - assert( - fallbackReason._2.contains( - "[FallbackByUserOptions] Validation failed on node Scan parquet default.t")) - } - } - - withTable("t1", "t2") { - spark.range(10).write.format("parquet").saveAsTable("t1") - spark.range(10).write.format("parquet").saveAsTable("t2") - - val id = runExecution("SELECT * FROM t1 FULL OUTER JOIN t2") - val execution = glutenStore.execution(id) - if (BackendTestUtils.isVeloxBackendLoaded()) { - assert(execution.get.numFallbackNodes == 1) - assert( - execution.get.fallbackNodeToReason.head._2 - .contains("FullOuter join is not supported with BroadcastNestedLoopJoin")) - } else { - assert(execution.get.numFallbackNodes == 0) - } - } - - // [GLUTEN-4119] Skip add ReusedExchange to fallback node - withTable("t1") { - spark.range(10).write.format("parquet").saveAsTable("t1") - val sql = - "WITH sub1 AS (SELECT * FROM t1), sub2 AS (SELECT * FROM t1) SELECT * FROM sub1 JOIN sub2;" - val id = runExecution(sql) - val execution = glutenStore.execution(id) - assert(!execution.get.fallbackNodeToReason.exists(_._1.contains("ReusedExchange"))) - } - } - - testGluten("Improve merge fallback reason") { - spark.sql("create table t using parquet as select 1 as c1, timestamp '2023-01-01' as c2") - withTable("t") { - val events = new ArrayBuffer[GlutenPlanFallbackEvent] - val listener = new SparkListener { - override def onOtherEvent(event: SparkListenerEvent): Unit = { - event match { - case e: GlutenPlanFallbackEvent => events.append(e) - case _ => - } - } - } - spark.sparkContext.addSparkListener(listener) - withSQLConf(GlutenConfig.COLUMNAR_WHOLESTAGE_FALLBACK_THRESHOLD.key -> "1") { - try { - val df = - spark.sql("select c1, count(*) from t where c2 > timestamp '2022-01-01' group by c1") - checkAnswer(df, Row(1, 1)) - spark.sparkContext.listenerBus.waitUntilEmpty() - - // avoid failing when we support transform timestamp filter in future - val isFallback = find(df.queryExecution.executedPlan) { - _.isInstanceOf[FileSourceScanExecTransformer] - }.isEmpty - if (isFallback) { - events.exists( - _.fallbackNodeToReason.values.exists( - _.contains("Subfield filters creation not supported for input type 'TIMESTAMP'"))) - events.exists( - _.fallbackNodeToReason.values.exists( - _.contains("Timestamp is not fully supported in Filter"))) - } - } finally { - spark.sparkContext.removeSparkListener(listener) - } - } - } - } - - test("Add logical link to rewritten spark plan") { - val events = new ArrayBuffer[GlutenPlanFallbackEvent] - val listener = new SparkListener { - override def onOtherEvent(event: SparkListenerEvent): Unit = { - event match { - case e: GlutenPlanFallbackEvent => events.append(e) - case _ => - } - } - } - spark.sparkContext.addSparkListener(listener) - withSQLConf(GlutenConfig.EXPRESSION_BLACK_LIST.key -> "add") { - try { - val df = spark.sql("select sum(id + 1) from range(10)") - df.collect() - spark.sparkContext.listenerBus.waitUntilEmpty() - val project = find(df.queryExecution.executedPlan) { - _.isInstanceOf[ProjectExec] - } - assert(project.isDefined) - assert( - events.exists(_.fallbackNodeToReason.values.toSet - .exists(_.contains("Not supported to map spark function name")))) - } finally { - spark.sparkContext.removeSparkListener(listener) - } - } - } - - test("ExpandFallbackPolicy should propagate fallback reason to vanilla SparkPlan") { - val events = new ArrayBuffer[GlutenPlanFallbackEvent] - val listener = new SparkListener { - override def onOtherEvent(event: SparkListenerEvent): Unit = { - event match { - case e: GlutenPlanFallbackEvent => events.append(e) - case _ => - } - } - } - spark.sparkContext.addSparkListener(listener) - spark.range(10).selectExpr("id as c1", "id as c2").write.format("parquet").saveAsTable("t") - withTable("t") { - withSQLConf( - GlutenConfig.EXPRESSION_BLACK_LIST.key -> "max", - GlutenConfig.COLUMNAR_WHOLESTAGE_FALLBACK_THRESHOLD.key -> "1") { - try { - val df = spark.sql("select c2, max(c1) as id from t group by c2") - df.collect() - spark.sparkContext.listenerBus.waitUntilEmpty() - val agg = collect(df.queryExecution.executedPlan) { case a: HashAggregateExec => a } - assert(agg.size == 2) - assert( - events.count( - _.fallbackNodeToReason.values.toSet.exists(_.contains( - "Could not find a valid substrait mapping name for max" - ))) == 3) - } finally { - spark.sparkContext.removeSparkListener(listener) - } - } - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenAggregationQuerySuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenAggregationQuerySuite.scala deleted file mode 100644 index e8494d2c075..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenAggregationQuerySuite.scala +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait -import org.apache.spark.tags.SlowHiveTest - -@SlowHiveTest -class GlutenHashAggregationQuerySuite - extends HashAggregationQuerySuite - with GlutenTestSetWithSystemPropertyTrait {} - -@SlowHiveTest -class GlutenHashAggregationQueryWithControlledFallbackSuite - extends HashAggregationQueryWithControlledFallbackSuite - with GlutenTestSetWithSystemPropertyTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveCommandSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveCommandSuite.scala deleted file mode 100644 index 9ce9db42874..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveCommandSuite.scala +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.spark.sql.{AnalysisException, GlutenTestSetWithSystemPropertyTrait, Row} -import org.apache.spark.tags.SlowHiveTest - -import java.io.File -import java.nio.file.Files -import java.nio.file.StandardCopyOption - -@SlowHiveTest -class GlutenHiveCommandSuite extends HiveCommandSuite with GlutenTestSetWithSystemPropertyTrait { - - override def testNameBlackList: Seq[String] = super.testNameBlackList ++ Seq( - // Rewritten with a workspace-backed file because TestHive.getHiveFile resolves this resource - // from the spark-hive tests jar in gluten-ut. - "LOAD DATA LOCAL", - "LOAD DATA" - ) - - Seq(true, false).foreach { - local => - val loadQuery = if (local) "LOAD DATA LOCAL" else "LOAD DATA" - testGluten(loadQuery) { - testLoadData(loadQuery, local) - } - } - - private def testLoadData(loadQuery: String, local: Boolean): Unit = { - // employee.dat has two columns separated by '|', the first is an int, the second is a string. - // Its content looks like: - // 16|john - // 17|robert - val testData = getWorkspaceFilePath( - "sql", - "hive", - "src", - "test", - "resources", - "data", - "files", - "employee.dat").toFile.getCanonicalFile - - def withInputFile(fn: File => Unit): Unit = { - if (local) { - fn(testData) - } else { - val tmp = File.createTempFile(testData.getName(), ".tmp") - Files.copy(testData.toPath, tmp.toPath, StandardCopyOption.REPLACE_EXISTING) - try { - fn(tmp) - } finally { - tmp.delete() - } - } - } - - withTable("non_part_table", "part_table") { - sql(""" - |CREATE TABLE non_part_table (employeeID INT, employeeName STRING) - |ROW FORMAT DELIMITED - |FIELDS TERMINATED BY '|' - |LINES TERMINATED BY '\n' - """.stripMargin) - - // LOAD DATA INTO non-partitioned table can't specify partition - intercept[AnalysisException] { - sql( - s"""$loadQuery INPATH "${testData.toURI}" INTO TABLE non_part_table PARTITION(ds="1")""") - } - - withInputFile { - path => - sql(s"""$loadQuery INPATH "${path.toURI}" INTO TABLE non_part_table""") - - // Non-local mode is expected to move the file, while local mode is expected to copy it. - // Check once here that the behavior is the expected. - assert(local === path.exists()) - } - - checkAnswer(sql("SELECT * FROM non_part_table WHERE employeeID = 16"), Row(16, "john") :: Nil) - - // Incorrect URI. - // file://path/to/data/files/employee.dat - // - // TODO: need a similar test for non-local mode. - if (local) { - val incorrectUri = "file://path/to/data/files/employee.dat" - intercept[AnalysisException] { - sql(s"""LOAD DATA LOCAL INPATH "$incorrectUri" INTO TABLE non_part_table""") - } - } - - // Use URI as inpath: - // file:/path/to/data/files/employee.dat - withInputFile { - path => sql(s"""$loadQuery INPATH "${path.toURI}" INTO TABLE non_part_table""") - } - - checkAnswer( - sql("SELECT * FROM non_part_table WHERE employeeID = 16"), - Row(16, "john") :: Row(16, "john") :: Nil) - - // Overwrite existing data. - withInputFile { - path => sql(s"""$loadQuery INPATH "${path.toURI}" OVERWRITE INTO TABLE non_part_table""") - } - - checkAnswer(sql("SELECT * FROM non_part_table WHERE employeeID = 16"), Row(16, "john") :: Nil) - - sql(""" - |CREATE TABLE part_table (employeeID INT, employeeName STRING) - |PARTITIONED BY (c STRING, d STRING) - |ROW FORMAT DELIMITED - |FIELDS TERMINATED BY '|' - |LINES TERMINATED BY '\n' - """.stripMargin) - - // LOAD DATA INTO partitioned table must specify partition - withInputFile { - f => - val path = f.toURI - intercept[AnalysisException] { - sql(s"""$loadQuery INPATH "$path" INTO TABLE part_table""") - } - - intercept[AnalysisException] { - sql(s"""$loadQuery INPATH "$path" INTO TABLE part_table PARTITION(c="1")""") - } - intercept[AnalysisException] { - sql(s"""$loadQuery INPATH "$path" INTO TABLE part_table PARTITION(d="1")""") - } - intercept[AnalysisException] { - sql(s"""$loadQuery INPATH "$path" INTO TABLE part_table PARTITION(c="1", k="2")""") - } - } - - withInputFile { - f => - sql(s"""$loadQuery INPATH "${f.toURI}" INTO TABLE part_table PARTITION(c="1", d="2")""") - } - checkAnswer( - sql("SELECT employeeID, employeeName FROM part_table WHERE c = '1' AND d = '2'"), - sql("SELECT * FROM non_part_table").collect()) - - // Different order of partition columns. - withInputFile { - f => - sql(s"""$loadQuery INPATH "${f.toURI}" INTO TABLE part_table PARTITION(d="1", c="2")""") - } - checkAnswer( - sql("SELECT employeeID, employeeName FROM part_table WHERE c = '2' AND d = '1'"), - sql("SELECT * FROM non_part_table")) - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveComparisonTestSupport.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveComparisonTestSupport.scala deleted file mode 100644 index d96078f2d55..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveComparisonTestSupport.scala +++ /dev/null @@ -1,324 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.spark.sql.Dataset -import org.apache.spark.sql.catalyst.plans.logical._ -import org.apache.spark.sql.catalyst.util._ -import org.apache.spark.sql.execution.HiveResult.hiveResultString -import org.apache.spark.sql.execution.SQLExecution -import org.apache.spark.sql.execution.command._ -import org.apache.spark.sql.hive.test.{TestHive, TestHiveQueryExecution} - -import java.io.{File, FileOutputStream} -import java.util -import java.util.Locale - -import scala.util.control.NonFatal - -/** - * Mostly copied from Spark's [[HiveComparisonTest]]. This support reuses [[GlutenTestHiveTables]] - * for TestHive table registrations and customizes [[createQueryTest]] so `../../data` resolves to - * workspace-backed Hive test resources. - */ -trait GlutenHiveComparisonTestSupport extends GlutenHiveResourcePathSupport { - this: HiveComparisonTest => - - private val testDataPath: String = { - hiveResourcePath("data").toAbsolutePath.normalize.toUri.getPath - .stripSuffix("/") - } - - private val glutenAnswerCache: File = hiveResourcePath("golden").toFile - - override def createQueryTest( - testCaseName: String, - sql: String, - reset: Boolean = true, - tryWithoutResettingFirst: Boolean = false, - skip: Boolean = false): Unit = { - // testCaseName must not contain ':', which is not allowed to appear in a filename of Windows - assert(!testCaseName.contains(":")) - - // If test sharding is enable, skip tests that are not in the correct shard. - shardInfo.foreach { - case (shardId, numShards) if testCaseName.hashCode % numShards != shardId => return - case (shardId, _) => logDebug(s"Shard $shardId includes test '$testCaseName'") - } - - // Skip tests found in directories specified by user. - skipDirectories - .map(new File(_, testCaseName)) - .filter(_.exists) - .foreach(_ => return) - - // If runonlytests is set, skip this test unless we find a file in one of the specified - // directories. - val runIndicators = - runOnlyDirectories - .map(new File(_, testCaseName)) - .filter(_.exists) - if (runOnlyDirectories.nonEmpty && runIndicators.isEmpty) { - logDebug( - s"Skipping test '$testCaseName' not found in ${runOnlyDirectories.map(_.getCanonicalPath)}") - return - } - - test(testCaseName) { - assume(!skip) - logDebug(s"=== HIVE TEST: $testCaseName ===") - - val sqlWithoutComment = - sql.split("\n").filterNot(l => l.matches("--.*(?<=[^\\\\]);")).mkString("\n") - val allQueries = - sqlWithoutComment.split("(?<=[^\\\\]);").map(_.trim).filterNot(q => q == "").toSeq - - // TODO: DOCUMENT UNSUPPORTED - val queryList = - allQueries - // In hive, setting the hive.outerjoin.supports.filters flag to "false" essentially tells - // the system to return the wrong answer. Since we have no intention of mirroring their - // previously broken behavior we simply filter out changes to this setting. - .filterNot(_.contains("hive.outerjoin.supports.filters")) - .filterNot(_.contains("hive.exec.post.hooks")) - - if (allQueries != queryList) { - logWarning(s"Simplifications made on unsupported operations for test $testCaseName") - } - - lazy val consoleTestCase = { - val quotes = "\"\"\"" - queryList.zipWithIndex - .map { - case (query, i) => - s"""val q$i = sql($quotes$query$quotes); q$i.collect()""" - } - .mkString("\n== Console version of this test ==\n", "\n", "\n") - } - - def doTest(reset: Boolean, isSpeculative: Boolean = false): Unit = { - // Clear old output for this testcase. - outputDirectories.map(new File(_, testCaseName)).filter(_.exists()).foreach(_.delete()) - - if (reset) { - TestHive.reset() - } - - // Register workspace-backed table definitions before lazy TestHive auto-loading kicks in. - GlutenTestHiveTables.registerHiveQTestUtilsTables(hiveTestResourceDir) - - // Many tests drop indexes on src and srcpart at the beginning, so we need to load those - // tables here. Since DROP INDEX DDL is just passed to Hive, it bypasses the analyzer and - // thus the tables referenced in those DDL commands cannot be extracted for use by our - // test table auto-loading mechanism. In addition, the tests which use the SHOW TABLES - // command expect these tables to exist. - val hasShowTableCommand = - queryList.exists(_.toLowerCase(Locale.ROOT).contains("show tables")) - for (table <- Seq("src", "srcpart")) { - val hasMatchingQuery = queryList.exists { - query => - val normalizedQuery = query.toLowerCase(Locale.ROOT).stripSuffix(";") - normalizedQuery.endsWith(table) || - normalizedQuery.contains(s"from $table") || - normalizedQuery.contains(s"from default.$table") - } - if (hasShowTableCommand || hasMatchingQuery) { - TestHive.loadTestTable(table) - } - } - - val hiveCacheFiles = queryList.zipWithIndex.map { - case (queryString, i) => - val cachedAnswerName = s"$testCaseName-$i-${getMd5(queryString)}" - new File(glutenAnswerCache, cachedAnswerName) - } - - val hiveCachedResults = hiveCacheFiles - .flatMap { - cachedAnswerFile => - logDebug(s"Looking for cached answer file $cachedAnswerFile.") - if (cachedAnswerFile.exists) { - Some(fileToString(cachedAnswerFile)) - } else { - logDebug(s"File $cachedAnswerFile not found") - None - } - } - .map { - case "" => Nil - case "\n" => Seq("") - case other => other.split("\n").toSeq - } - - val hiveResults: Seq[Seq[String]] = - if (hiveCachedResults.size == queryList.size) { - logInfo(s"Using answer cache for test: $testCaseName") - hiveCachedResults - } else { - throw new UnsupportedOperationException( - "Cannot find result file for test case: " + testCaseName) - } - - // Run w/ catalyst - val catalystResults = queryList.zip(hiveResults).map { - case (queryString, hive) => - val query = new TestHiveQueryExecution(queryString.replace("../../data", testDataPath)) - def getResult(): Seq[String] = { - SQLExecution.withNewExecutionId(query)(hiveResultString(query.executedPlan)) - } - try { (query, prepareAnswer(query, getResult())) } - catch { - case e: Throwable => - val errorMessage = - s""" - |Failed to execute query using catalyst: - |Error: ${e.getMessage} - |${stackTraceToString(e)} - |$queryString - |$query - |== HIVE - ${hive.size} row(s) == - |${hive.mkString("\n")} - """.stripMargin - stringToFile( - new File(failedDirectory, testCaseName), - errorMessage + consoleTestCase) - fail(errorMessage) - } - } - - queryList.zip(hiveResults).zip(catalystResults).foreach { - case ((query, hive), (hiveQuery, catalyst)) => - // Check that the results match unless its an EXPLAIN query. - val preparedHive = prepareAnswer(hiveQuery, hive) - - // We will ignore the ExplainCommand, ShowFunctions, DescribeFunction - if ( - (!hiveQuery.logical.isInstanceOf[ExplainCommand]) && - (!hiveQuery.logical.isInstanceOf[ShowFunctions]) && - (!hiveQuery.logical.isInstanceOf[DescribeFunction]) && - (!hiveQuery.logical.isInstanceOf[DescribeCommandBase]) && - (!hiveQuery.logical.isInstanceOf[DescribeRelation]) && - (!hiveQuery.logical.isInstanceOf[DescribeColumn]) && - preparedHive != catalyst - ) { - - val hivePrintOut = s"== HIVE - ${preparedHive.size} row(s) ==" +: preparedHive - val catalystPrintOut = s"== CATALYST - ${catalyst.size} row(s) ==" +: catalyst - - val resultComparison = sideBySide(hivePrintOut, catalystPrintOut).mkString("\n") - - if (recomputeCache) { - logWarning(s"Clearing cache files for failed test $testCaseName") - hiveCacheFiles.foreach(_.delete()) - } - - // If this query is reading other tables that were created during this test run - // also print out the query plans and results for those. - val computedTablesMessages: String = - try { - val tablesRead = - new TestHiveQueryExecution(query).executedPlan.collect { - case ts: HiveTableScanExec => ts.relation.tableMeta.identifier - }.toSet - - TestHive.reset() - val executions = queryList.map(new TestHiveQueryExecution(_)) - executions.foreach(_.toRdd) - val tablesGenerated = queryList.zip(executions).flatMap { - case (q, e) => - e.analyzed.collect { - case i: InsertIntoHiveTable if tablesRead.contains(i.table.identifier) => - (q, e, i) - } - } - - tablesGenerated - .map { - case (hiveql, execution, insert) => - val rdd = - Dataset.ofRows(TestHive.sparkSession, insert.query).queryExecution.toRdd - s""" - |=== Generated Table === - |$hiveql - |$execution - |== Results == - |${rdd.collect().mkString("\n")} - """.stripMargin - } - .mkString("\n") - - } catch { - case NonFatal(e) => - logError("Failed to compute generated tables", e) - s"Couldn't compute dependent tables: $e" - } - - val errorMessage = - s""" - |Results do not match for $testCaseName: - |$hiveQuery\n${hiveQuery.analyzed.output.map(_.name).mkString("\t")} - |$resultComparison - |$computedTablesMessages - """.stripMargin - - stringToFile(new File(wrongDirectory, testCaseName), errorMessage + consoleTestCase) - if (isSpeculative && !reset) { - fail("Failed on first run; retrying") - } else { - fail(errorMessage) - } - } - } - - // Touch passed file. - new FileOutputStream(new File(passedDirectory, testCaseName)).close() - } - - val canSpeculativelyTryWithoutReset: Boolean = { - val excludedSubstrings = Seq("into table", "create table", "drop index") - !queryList.map(_.toLowerCase(Locale.ROOT)).exists { - query => excludedSubstrings.exists(s => query.contains(s)) - } - } - - val savedSettings = new util.HashMap[String, String] - savedSettings.putAll(TestHive.conf.settings) - try { - try { - if (tryWithoutResettingFirst && canSpeculativelyTryWithoutReset) { - doTest(reset = false, isSpeculative = true) - } else { - doTest(reset) - } - } catch { - case tf: org.scalatest.exceptions.TestFailedException => - if (tryWithoutResettingFirst && canSpeculativelyTryWithoutReset) { - logWarning("Test failed without reset(); retrying with reset()") - doTest(reset = true) - } else { - throw tf - } - } - } catch { - case tf: org.scalatest.exceptions.TestFailedException => throw tf - } finally { - TestHive.conf.settings.clear() - TestHive.conf.settings.putAll(savedSettings) - } - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveDDLSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveDDLSuite.scala deleted file mode 100644 index 049c3b97f77..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveDDLSuite.scala +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.spark.sql.{AnalysisException, GlutenTestSetWithSystemPropertyTrait, Row} -import org.apache.spark.tags.SlowHiveTest - -@SlowHiveTest -class GlutenHiveDDLSuite - extends HiveDDLSuite - with GlutenTestSetWithSystemPropertyTrait - with GlutenHiveResourcePathSupport { - - override def testNameBlackList: Seq[String] = Seq( - // Rewritten with workspace-backed schema URLs because TestHive.getHiveFile resolves - // these resources from the spark-hive tests jar in gluten-ut. - "SPARK-34370: support Avro schema evolution (add column with avro.schema.url)", - "SPARK-34370: support Avro schema evolution (remove column with avro.schema.url)", - "SPARK-34261: Avoid side effect if create exists temporary function" - ) - - testGluten("SPARK-34370: support Avro schema evolution (add column with avro.schema.url)") { - checkAvroSchemaEvolutionAddColumn( - avroSchemaUrlProperty("schemaWithOneField.avsc"), - avroSchemaUrlProperty("schemaWithTwoFields.avsc")) - } - - testGluten("SPARK-34370: support Avro schema evolution (remove column with avro.schema.url)") { - checkAvroSchemaEvolutionRemoveColumn( - avroSchemaUrlProperty("schemaWithTwoFields.avsc"), - avroSchemaUrlProperty("schemaWithOneField.avsc")) - } - - testGluten("SPARK-34261: Avoid side effect if create exists temporary function") { - withUserDefinedFunction("f1" -> true) { - sql("CREATE TEMPORARY FUNCTION f1 AS 'org.apache.hadoop.hive.ql.udf.UDFUUID'") - - val jarName = "TestUDTF.jar" - val jar = hiveResourcePath(jarName).toUri.toString - spark.sparkContext.addedJars.keys - .find(_.contains(jarName)) - .foreach(spark.sparkContext.addedJars.remove) - assert(!spark.sparkContext.listJars().exists(_.contains(jarName))) - val msg = intercept[AnalysisException] { - sql( - "CREATE TEMPORARY FUNCTION f1 AS " + - s"'org.apache.hadoop.hive.ql.udf.UDFUUID' USING JAR '$jar'") - }.getMessage - assert(msg.contains("Function f1 already exists")) - assert(!spark.sparkContext.listJars().exists(_.contains(jarName))) - - sql( - "CREATE OR REPLACE TEMPORARY FUNCTION f1 AS " + - s"'org.apache.hadoop.hive.ql.udf.UDFUUID' USING JAR '$jar'") - assert(spark.sparkContext.listJars().exists(_.contains(jarName))) - } - } - - private def avroSchemaUrlProperty(fileName: String): String = { - val schemaPath = hiveResourcePath(fileName) - s"'avro.schema.url'='${schemaPath.toUri.toString}'" - } - - private def checkAvroSchemaEvolutionAddColumn( - originalSerdeProperties: String, - evolvedSerdeProperties: String): Unit = { - withTable("t") { - sql(s""" - |CREATE TABLE t PARTITIONED BY (ds string) - |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' - |WITH SERDEPROPERTIES ($originalSerdeProperties) - |STORED AS - |INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' - |OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' - |""".stripMargin) - sql("INSERT INTO t partition (ds='1981-01-07') VALUES ('col2_value')") - sql(s"ALTER TABLE t SET SERDEPROPERTIES ($evolvedSerdeProperties)") - sql("INSERT INTO t partition (ds='1983-04-27') VALUES ('col1_value', 'col2_value')") - checkAnswer( - spark.table("t"), - Row("col1_default", "col2_value", "1981-01-07") :: - Row("col1_value", "col2_value", "1983-04-27") :: Nil) - } - } - - private def checkAvroSchemaEvolutionRemoveColumn( - originalSerdeProperties: String, - evolvedSerdeProperties: String): Unit = { - withTable("t") { - sql(s""" - |CREATE TABLE t PARTITIONED BY (ds string) - |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' - |WITH SERDEPROPERTIES ($originalSerdeProperties) - |STORED AS - |INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' - |OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' - |""".stripMargin) - sql("INSERT INTO t partition (ds='1983-04-27') VALUES ('col1_value', 'col2_value')") - sql(s"ALTER TABLE t SET SERDEPROPERTIES ($evolvedSerdeProperties)") - sql("INSERT INTO t partition (ds='1981-01-07') VALUES ('col2_value')") - checkAnswer( - spark.table("t"), - Row("col2_value", "1981-01-07") :: - Row("col2_value", "1983-04-27") :: Nil) - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveExplainSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveExplainSuite.scala deleted file mode 100644 index 0f6b3701669..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveExplainSuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait -import org.apache.spark.tags.SlowHiveTest - -@SlowHiveTest -class GlutenHiveExplainSuite extends HiveExplainSuite with GlutenTestSetWithSystemPropertyTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHivePlanTest.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHivePlanTest.scala deleted file mode 100644 index 6cdf5f5845d..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHivePlanTest.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait -import org.apache.spark.tags.SlowHiveTest - -@SlowHiveTest -class GlutenHivePlanTest extends HivePlanTest with GlutenTestSetWithSystemPropertyTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveQuerySuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveQuerySuite.scala deleted file mode 100644 index f547953c58a..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveQuerySuite.scala +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.spark.SparkFiles -import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait -import org.apache.spark.sql.hive.HiveUtils.{builtinHiveVersion => hiveVersion} -import org.apache.spark.sql.hive.test.HiveTestJars -import org.apache.spark.tags.SlowHiveTest - -import java.io.File - -@SlowHiveTest -class GlutenHiveQuerySuite - extends HiveQuerySuite - with GlutenTestSetWithSystemPropertyTrait - with GlutenHiveComparisonTestSupport { - - override def testNameBlackList: Seq[String] = Seq( - "ADD FILE command", - "ADD JAR command 2", - "CREATE TEMPORARY FUNCTION", - "SPARK-33084: Add jar support Ivy URI in SQL" - ) - - testGluten("ADD FILE command") { - val testFile = hiveResourcePath("data/files/v1.txt").toFile.toURI - sql(s"ADD FILE $testFile") - - val checkAddFileRDD = sparkContext.parallelize(1 to 2, 1).mapPartitions { - _ => Iterator.single(new File(SparkFiles.get("v1.txt")).canRead) - } - - assert(checkAddFileRDD.first()) - assert(sql("list files").filter(_.getString(0).contains("data/files/v1.txt")).count() > 0) - assert(sql("list file").filter(_.getString(0).contains("data/files/v1.txt")).count() > 0) - assert(sql(s"list file $testFile").count() == 1) - } - - testGluten("ADD JAR command 2") { - val testJar = HiveTestJars.getHiveHcatalogCoreJar().toURI - val testData = hiveResourcePath("data/files/sample.json").toUri - sql(s"ADD JAR $testJar") - withTable("t1") { - sql("""CREATE TABLE t1(a string, b string) - |ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'""".stripMargin) - sql(s"""LOAD DATA LOCAL INPATH "$testData" INTO TABLE t1""") - sql("select * from src join t1 on src.key = t1.a") - } - assert( - sql("list jars") - .filter(_.getString(0).contains(HiveTestJars.getHiveHcatalogCoreJar().getName)) - .count() > 0) - assert( - sql("list jar") - .filter(_.getString(0).contains(HiveTestJars.getHiveHcatalogCoreJar().getName)) - .count() > 0) - val testJar2 = hiveResourcePath("TestUDTF.jar").toFile.getCanonicalPath - sql(s"ADD JAR $testJar2") - assert(sql(s"list jar $testJar").count() == 1) - } - - testGluten("SPARK-33084: Add jar support Ivy URI in SQL") { - val testData = hiveResourcePath("data/files/sample.json").toUri - withTable("t") { - // Use transitive=false as it should be good enough to test the Ivy support in Hive ADD JAR. - sql( - s"ADD JAR ivy://org.apache.hive.hcatalog:hive-hcatalog-core:$hiveVersion" + - "?transitive=false") - sql("""CREATE TABLE t(a string, b string) - |ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'""".stripMargin) - sql(s"""LOAD DATA LOCAL INPATH "$testData" INTO TABLE t""") - sql("SELECT * FROM src JOIN t on src.key = t.a") - assert( - sql("LIST JARS") - .filter(_.getString(0).contains( - s"org.apache.hive.hcatalog_hive-hcatalog-core-$hiveVersion.jar")) - .count() > 0) - assert( - sql("LIST JAR") - .filter(_.getString(0).contains( - s"org.apache.hive.hcatalog_hive-hcatalog-core-$hiveVersion.jar")) - .count() > 0) - } - } - - testGluten("CREATE TEMPORARY FUNCTION") { - val jarURL = hiveResourcePath("TestUDTF.jar").toUri.toURL - sql(s"ADD JAR $jarURL") - withUserDefinedFunction("udtf_count2" -> true) { - sql("""CREATE TEMPORARY FUNCTION udtf_count2 AS - |'org.apache.spark.sql.hive.execution.GenericUDTFCount2' - |""".stripMargin) - assert(sql("DESCRIBE FUNCTION udtf_count2").count() > 1) - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveResolutionSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveResolutionSuite.scala deleted file mode 100644 index d46c2eed7ad..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveResolutionSuite.scala +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait -import org.apache.spark.tags.SlowHiveTest - -@SlowHiveTest -class GlutenHiveResolutionSuite - extends HiveResolutionSuite - with GlutenTestSetWithSystemPropertyTrait - with GlutenHiveComparisonTestSupport {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveResourcePathSupport.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveResourcePathSupport.scala deleted file mode 100644 index 41a20e8ae6d..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveResourcePathSupport.scala +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.spark.SparkFunSuite - -import java.nio.file.Path - -/** Shared helpers for resolving workspace-backed Hive test resources. */ -trait GlutenHiveResourcePathSupport { - this: SparkFunSuite => - - protected lazy val hiveTestResourceDir: Path = - getWorkspaceFilePath("sql", "hive", "src", "test", "resources") - - final protected def hiveResourcePath(relativePath: String): Path = { - GlutenTestHiveTables.hiveResourcePath(hiveTestResourceDir, relativePath) - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala deleted file mode 100644 index 1020294d883..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.gluten.config.GlutenConfig - -import org.apache.spark.{DebugFilesystem, SparkConf} -import org.apache.spark.sql.Row -import org.apache.spark.sql.catalyst.TableIdentifier - -class GlutenHiveSQLQueryCHSuite extends GlutenHiveSQLQuerySuiteBase { - - override def sparkConf: SparkConf = { - defaultSparkConf - .set("spark.plugins", "org.apache.gluten.GlutenPlugin") - .set(GlutenConfig.NATIVE_VALIDATION_ENABLED.key, "false") - .set(GlutenConfig.NATIVE_WRITER_ENABLED.key, "true") - .set("spark.sql.storeAssignmentPolicy", "legacy") - .set("spark.default.parallelism", "1") - .set("spark.memory.offHeap.enabled", "true") - .set("spark.memory.offHeap.size", "1024MB") - .set("spark.hadoop.fs.file.impl", classOf[DebugFilesystem].getName) - } - - testGluten("5182: Fix failed to parse post join filters") { - withSQLConf( - "spark.sql.hive.convertMetastoreParquet" -> "false") { - sql("DROP TABLE IF EXISTS test_5182_0;") - sql("DROP TABLE IF EXISTS test_5182_1;") - sql( - "CREATE TABLE test_5182_0 (from_uid STRING, vgift_typeid int, vm_count int, " + - "status bigint, ts bigint, vm_typeid int) " + - "USING hive OPTIONS(fileFormat 'parquet') PARTITIONED BY (`day` STRING);") - sql( - "CREATE TABLE test_5182_1 (typeid int, groupid int, ss_id bigint, " + - "ss_start_time bigint, ss_end_time bigint) " + - "USING hive OPTIONS(fileFormat 'parquet');") - sql( - "INSERT INTO test_5182_0 partition(day='2024-03-31') " + - "VALUES('uid_1', 2, 10, 1, 11111111111, 2);") - sql("INSERT INTO test_5182_1 VALUES(2, 1, 1, 1000000000, 2111111111);") - val df = spark.sql( - "select ee.from_uid as uid,day, vgift_typeid, money from " + - "(select t_a.day, if(cast(substr(t_a.ts,1,10) as bigint) between " + - "t_b.ss_start_time and t_b.ss_end_time, t_b.ss_id, 0) ss_id, " + - "t_a.vgift_typeid, t_a.from_uid, vm_count money from " + - "(select from_uid,day,vgift_typeid,vm_count,ts from test_5182_0 " + - "where day between '2024-03-30' and '2024-03-31' and status=1 and vm_typeid=2) t_a " + - "left join test_5182_1 t_b on t_a.vgift_typeid=t_b.typeid " + - "where t_b.groupid in (1,2)) ee where ss_id=1;") - checkAnswer(df, Seq(Row("uid_1", "2024-03-31", 2, 10))) - } - spark.sessionState.catalog.dropTable( - TableIdentifier("test_5182_0"), - ignoreIfNotExists = true, - purge = false) - spark.sessionState.catalog.dropTable( - TableIdentifier("test_5182_1"), - ignoreIfNotExists = true, - purge = false) - } - - testGluten("5249: Reading csv may throw Unexpected empty column") { - sql("DROP TABLE IF EXISTS test_5249;") - sql( - "CREATE TABLE test_5249 (name STRING, uid STRING) " + - "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' " + - "STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' " + - "OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';") - sql("INSERT INTO test_5249 VALUES('name_1', 'id_1');") - val df = spark.sql( - "SELECT name, uid, count(distinct uid) total_uid_num from test_5249 " + - "group by name, uid with cube;") - checkAnswer( - df, - Seq( - Row("name_1", "id_1", 1), - Row("name_1", null, 1), - Row(null, "id_1", 1), - Row(null, null, 1))) - spark.sessionState.catalog.dropTable( - TableIdentifier("test_5249"), - ignoreIfNotExists = true, - purge = false) - } - - testGluten("GLUTEN-7116: Support outer explode") { - sql("create table if not exists test_7116 (id int, name string)") - sql("insert into test_7116 values (1, 'a,b'), (2, null), (null, 'c,d'), (3, '')") - val query = - """ - |select id, col_name - |from test_7116 lateral view outer explode(split(name, ',')) as col_name - |""".stripMargin - val df = sql(query) - checkAnswer( - df, - Seq(Row(1, "a"), Row(1, "b"), Row(2, null), Row(null, "c"), Row(null, "d"), Row(3, ""))) - spark.sessionState.catalog.dropTable( - TableIdentifier("test_7116"), - ignoreIfNotExists = true, - purge = false) - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuite.scala deleted file mode 100644 index d7d6c57c5b0..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuite.scala +++ /dev/null @@ -1,261 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.execution.FileSourceScanExecTransformer - -import org.apache.spark.SparkConf -import org.apache.spark.sql.Row -import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.execution.SparkPlan -import org.apache.spark.sql.hive.{HiveExternalCatalog, HiveTableScanExecTransformer} -import org.apache.spark.sql.hive.client.HiveClient - -class GlutenHiveSQLQuerySuite extends GlutenHiveSQLQuerySuiteBase { - - override def sparkConf: SparkConf = { - defaultSparkConf - .set("spark.plugins", "org.apache.gluten.GlutenPlugin") - .set("spark.default.parallelism", "1") - .set("spark.memory.offHeap.enabled", "true") - .set("spark.memory.offHeap.size", "1024MB") - } - - testGluten("hive orc scan") { - withSQLConf("spark.sql.hive.convertMetastoreOrc" -> "false") { - sql("DROP TABLE IF EXISTS test_orc") - sql( - "CREATE TABLE test_orc (name STRING, favorite_color STRING)" + - " USING hive OPTIONS(fileFormat 'orc')") - sql("INSERT INTO test_orc VALUES('test_1', 'red')") - val df = spark.sql("select * from test_orc") - checkAnswer(df, Seq(Row("test_1", "red"))) - checkOperatorMatch[HiveTableScanExecTransformer](df) - } - spark.sessionState.catalog.dropTable( - TableIdentifier("test_orc"), - ignoreIfNotExists = true, - purge = false) - } - - testGluten("Add orc char type validation") { - withSQLConf("spark.sql.hive.convertMetastoreOrc" -> "false") { - sql("DROP TABLE IF EXISTS test_orc") - sql( - "CREATE TABLE test_orc (name char(10), id int)" + - " USING hive OPTIONS(fileFormat 'orc')") - sql("INSERT INTO test_orc VALUES('test', 1)") - } - - def testExecPlan( - convertMetastoreOrc: String, - charTypeFallbackEnabled: String, - shouldFindTransformer: Boolean, - transformerClass: Class[_ <: SparkPlan] - ): Unit = { - - withSQLConf( - "spark.sql.hive.convertMetastoreOrc" -> convertMetastoreOrc, - GlutenConfig.VELOX_FORCE_ORC_CHAR_TYPE_SCAN_FALLBACK.key -> charTypeFallbackEnabled - ) { - val queries = Seq("select id from test_orc", "select name, id from test_orc") - - queries.foreach { - query => - val executedPlan = getExecutedPlan(spark.sql(query)) - val planCondition = executedPlan.exists(_.find(transformerClass.isInstance).isDefined) - - if (shouldFindTransformer) { - assert(planCondition) - } else { - assert(!planCondition) - } - } - } - } - - testExecPlan( - "false", - "true", - shouldFindTransformer = false, - classOf[HiveTableScanExecTransformer]) - testExecPlan( - "false", - "false", - shouldFindTransformer = true, - classOf[HiveTableScanExecTransformer]) - - testExecPlan( - "true", - "true", - shouldFindTransformer = false, - classOf[FileSourceScanExecTransformer]) - testExecPlan( - "true", - "false", - shouldFindTransformer = true, - classOf[FileSourceScanExecTransformer]) - spark.sessionState.catalog.dropTable( - TableIdentifier("test_orc"), - ignoreIfNotExists = true, - purge = false) - } - - testGluten("avoid unnecessary filter binding for subfield during scan") { - withSQLConf( - "spark.sql.hive.convertMetastoreParquet" -> "false") { - sql("DROP TABLE IF EXISTS test_subfield") - sql( - "CREATE TABLE test_subfield (name STRING, favorite_color STRING," + - " label STRUCT) USING hive OPTIONS(fileFormat 'parquet')") - sql( - "INSERT INTO test_subfield VALUES('test_1', 'red', named_struct('label_1', 'label-a'," + - "'label_2', 'label-b'))") - val df = spark.sql("select * from test_subfield where name='test_1'") - checkAnswer(df, Seq(Row("test_1", "red", Row("label-a", "label-b")))) - checkOperatorMatch[HiveTableScanExecTransformer](df) - } - spark.sessionState.catalog.dropTable( - TableIdentifier("test_subfield"), - ignoreIfNotExists = true, - purge = false) - } - - testGluten("orc.force.positional.evolution maps Hive ORC columns by position") { - val hiveClient: HiveClient = - spark.sharedState.externalCatalog.unwrapped.asInstanceOf[HiveExternalCatalog].client - - withSQLConf("spark.sql.hive.convertMetastoreOrc" -> "false") { - withTempDir { - dir => - val orcLoc = s"file:///$dir/test_orc_pos" - withTable("test_orc_pos", "test_orc_pos_renamed") { - // Write ORC files whose physical column names are c1, c2 (c1 = 1, c2 = 2). - hiveClient.runSqlHive( - s"create table test_orc_pos(c1 int, c2 int) stored as orc location '$orcLoc'") - hiveClient.runSqlHive("insert into test_orc_pos select 1, 2") - - // A second table over the SAME files but with mismatched column names (x, y). - // By name, x/y are not present in the files; only position mapping can read them. - hiveClient.runSqlHive( - s"create table test_orc_pos_renamed(x int, y int) stored as orc location '$orcLoc'") - - // orc.force.positional.evolution=true => read by position: x -> c1 (=1), y -> c2 (=2). - withSQLConf("spark.hadoop.orc.force.positional.evolution" -> "true") { - val df = sql("select x, y from test_orc_pos_renamed") - checkAnswer(df, Seq(Row(1, 2))) - checkOperatorMatch[HiveTableScanExecTransformer](df) - } - } - } - } - } - - testGluten( - "GLUTEN: Hive ORC files with _col* names read by position without positional flag") { - // Regression for the case where two ORC tables must use OPPOSITE column - // mapping modes in the same query: one with real column names (by name) and - // one written by old Hive with placeholder _col* names (by position). The - // native reader must decide the mode per file (matching vanilla Spark's - // OrcUtils.requestedColumnIds), so a _col* file reads correctly even though - // orc.force.positional.evolution is NOT set (ORC is read by name by - // default). Without the fix the _col* columns would read back as NULL. - val hiveClient: HiveClient = - spark.sharedState.externalCatalog.unwrapped.asInstanceOf[HiveExternalCatalog].client - - withSQLConf("spark.sql.hive.convertMetastoreOrc" -> "false") { - withTempDir { - dir => - val colStarLoc = s"file:///$dir/test_orc_colstar" - val namedLoc = s"file:///$dir/test_orc_named" - withTable("test_orc_colstar", "test_orc_colstar_renamed", "test_orc_named") { - // Naming the columns literally _col0/_col1 guarantees the physical - // ORC field names are placeholders, independent of the Hive - // version (mirrors Spark's SPARK-34897 setup). - hiveClient.runSqlHive( - s"create table test_orc_colstar(`_col0` int, `_col1` string) " + - s"stored as orc location '$colStarLoc'") - hiveClient.runSqlHive("insert into test_orc_colstar select 7, 'a'") - - // A second table over the SAME files but with real names. By name, - // id/name are absent from the _col* files; only position mapping - // can read them -- and it must happen WITHOUT the positional flag. - hiveClient.runSqlHive( - s"create table test_orc_colstar_renamed(id int, name string) " + - s"stored as orc location '$colStarLoc'") - - // A table with real physical column names, read by name. - hiveClient.runSqlHive( - s"create table test_orc_named(uid int, label string) " + - s"stored as orc location '$namedLoc'") - hiveClient.runSqlHive("insert into test_orc_named select 7, 'b'") - - // No positional flag set. The _col* table read via real names must - // still return the values (positional fallback). - val colStar = sql("select id, name from test_orc_colstar_renamed") - checkAnswer(colStar, Seq(Row(7, "a"))) - checkOperatorMatch[HiveTableScanExecTransformer](colStar) - - // The real-name table still reads correctly by name in the same - // session (opposite mapping mode). - val named = sql("select uid, label from test_orc_named") - checkAnswer(named, Seq(Row(7, "b"))) - checkOperatorMatch[HiveTableScanExecTransformer](named) - - // Both in one query (the original failure folded the join to an - // empty LocalTableScan). The join must return a non-empty result. - val joined = sql( - "select c.name, n.label from test_orc_colstar_renamed c " + - "join test_orc_named n on c.id = n.uid") - checkAnswer(joined, Seq(Row("a", "b"))) - } - } - } - } - - test("GLUTEN-11062: Supports mixed input format for partitioned Hive table") { - val hiveClient: HiveClient = - spark.sharedState.externalCatalog.unwrapped.asInstanceOf[HiveExternalCatalog].client - - withSQLConf("spark.sql.hive.convertMetastoreParquet" -> "false") { - withTempDir { - dir => - val parquetLoc = s"file:///$dir/test_parquet" - val orcLoc = s"file:///$dir/test_orc" - withTable("test_parquet", "test_orc") { - hiveClient.runSqlHive(s"""create table test_parquet(id int) - partitioned by(pid int) - stored as parquet location '$parquetLoc' - """.stripMargin) - hiveClient.runSqlHive("insert into test_parquet partition(pid=1) select 2") - hiveClient.runSqlHive(s"""create table test_orc(id int) - partitioned by(pid int) - stored as orc location '$orcLoc' - """.stripMargin) - hiveClient.runSqlHive("insert into test_orc partition(pid=2) select 2") - hiveClient.runSqlHive( - s"alter table test_parquet add partition (pid=2) location '$orcLoc/pid=2'") - hiveClient.runSqlHive("alter table test_parquet partition(pid=2) SET FILEFORMAT orc") - val df = sql("select pid, id from test_parquet order by pid") - checkAnswer(df, Seq(Row(1, 2), Row(2, 2))) - checkOperatorMatch[HiveTableScanExecTransformer](df) - } - } - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuiteBase.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuiteBase.scala deleted file mode 100644 index c8540647d3f..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuiteBase.scala +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.gluten.execution.TransformSupport - -import org.apache.spark.SparkConf -import org.apache.spark.internal.config -import org.apache.spark.internal.config.UI.UI_ENABLED -import org.apache.spark.sql.{DataFrame, GlutenSQLTestsTrait, SparkSession} -import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode -import org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation -import org.apache.spark.sql.hive.HiveUtils -import org.apache.spark.sql.internal.SQLConf - -import scala.reflect.ClassTag - -abstract class GlutenHiveSQLQuerySuiteBase extends GlutenSQLTestsTrait { - private var _spark: SparkSession = null - - override def beforeAll(): Unit = { - prepareWorkDir() - if (_spark == null) { - _spark = SparkSession.builder().config(sparkConf).enableHiveSupport().getOrCreate() - } - - _spark.sparkContext.setLogLevel("warn") - } - - override protected def spark: SparkSession = _spark - - override def afterAll(): Unit = { - try { - super.afterAll() - if (_spark != null) { - try { - _spark.sessionState.catalog.reset() - } finally { - _spark.stop() - _spark = null - } - } - } finally { - SparkSession.clearActiveSession() - SparkSession.clearDefaultSession() - doThreadPostAudit() - } - } - - protected def defaultSparkConf: SparkConf = { - val conf = new SparkConf() - .set("spark.master", "local[1]") - .set("spark.sql.test", "") - .set("spark.sql.testkey", "true") - .set(SQLConf.CODEGEN_FALLBACK.key, "false") - .set(SQLConf.CODEGEN_FACTORY_MODE.key, CodegenObjectFactoryMode.CODEGEN_ONLY.toString) - .set( - HiveUtils.HIVE_METASTORE_BARRIER_PREFIXES.key, - "org.apache.spark.sql.hive.execution.PairSerDe") - // SPARK-8910 - .set(UI_ENABLED, false) - .set(config.UNSAFE_EXCEPTION_ON_MEMORY_LEAK, true) - // Hive changed the default of hive.metastore.disallow.incompatible.col.type.changes - // from false to true. For details, see the JIRA HIVE-12320 and HIVE-17764. - .set("spark.hadoop.hive.metastore.disallow.incompatible.col.type.changes", "false") - // Disable ConvertToLocalRelation for better test coverage. Test cases built on - // LocalRelation will exercise the optimization rules better by disabling it as - // this rule may potentially block testing of other optimization rules such as - // ConstantPropagation etc. - .set(SQLConf.OPTIMIZER_EXCLUDED_RULES.key, ConvertToLocalRelation.ruleName) - - conf.set( - "spark.sql.warehouse.dir", - getClass.getResource("/").getPath + "/tests-working-home/spark-warehouse") - val metastore = getClass.getResource("/").getPath + getClass.getCanonicalName + "/metastore_db" - conf.set("javax.jdo.option.ConnectionURL", s"jdbc:derby:;databaseName=$metastore;create=true") - } - - def checkOperatorMatch[T <: TransformSupport](df: DataFrame)(implicit tag: ClassTag[T]): Unit = { - val executedPlan = getExecutedPlan(df) - assert(executedPlan.exists(plan => plan.getClass == tag.runtimeClass)) - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLViewSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLViewSuite.scala deleted file mode 100644 index d537a9aae6f..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLViewSuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait -import org.apache.spark.tags.SlowHiveTest - -@SlowHiveTest -class GlutenHiveSQLViewSuite extends HiveSQLViewSuite with GlutenTestSetWithSystemPropertyTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveScriptTransformationSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveScriptTransformationSuite.scala deleted file mode 100644 index 158ad4f46e6..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveScriptTransformationSuite.scala +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait -import org.apache.spark.tags.SlowHiveTest - -@SlowHiveTest -class GlutenHiveScriptTransformationSuite - extends HiveScriptTransformationSuite - with GlutenTestSetWithSystemPropertyTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSerDeReadWriteSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSerDeReadWriteSuite.scala deleted file mode 100644 index 92703dea371..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSerDeReadWriteSuite.scala +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait -import org.apache.spark.tags.SlowHiveTest - -@SlowHiveTest -class GlutenHiveSerDeReadWriteSuite - extends HiveSerDeReadWriteSuite - with GlutenTestSetWithSystemPropertyTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSerDeSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSerDeSuite.scala deleted file mode 100644 index 3687714a592..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSerDeSuite.scala +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait -import org.apache.spark.sql.hive.test.TestHive -import org.apache.spark.tags.SlowHiveTest - -@SlowHiveTest -class GlutenHiveSerDeSuite - extends HiveSerDeSuite - with GlutenTestSetWithSystemPropertyTrait - with GlutenHiveComparisonTestSupport { - - /** - * Mostly copied from Spark's [[HiveSerDeSuite]] and [[GlutenTestSetWithSystemPropertyTrait]], and - * customized so `sales.txt` is loaded from the workspace-backed Hive test resources instead of - * via [[TestHive.getHiveFile]]. - */ - override def beforeAll(): Unit = { - System.setProperty("spark.plugins", "org.apache.gluten.GlutenPlugin") - System.setProperty("spark.memory.offHeap.enabled", "true") - System.setProperty("spark.memory.offHeap.size", "1024MB") - System.setProperty( - "spark.shuffle.manager", - "org.apache.spark.shuffle.sort.ColumnarShuffleManager") - - import TestHive._ - import org.apache.hadoop.hive.serde2.RegexSerDe - - TestHive.setCacheTables(false) - sql(s"""CREATE TABLE IF NOT EXISTS sales (key STRING, value INT) - |ROW FORMAT SERDE '${classOf[RegexSerDe].getCanonicalName}' - |WITH SERDEPROPERTIES ("input.regex" = "([^ ]*)\t([^ ]*)") - """.stripMargin) - sql(s"""LOAD DATA LOCAL INPATH '${hiveResourcePath("data/files/sales.txt").toFile.toURI}' - |INTO TABLE sales""".stripMargin) - } - - override def afterAll(): Unit = { - try { - super.afterAll() - } finally { - System.clearProperty("spark.plugins") - System.clearProperty("spark.memory.offHeap.enabled") - System.clearProperty("spark.memory.offHeap.size") - System.clearProperty("spark.shuffle.manager") - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveTableScanSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveTableScanSuite.scala deleted file mode 100644 index ac4903218fc..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveTableScanSuite.scala +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait -import org.apache.spark.sql.Row -import org.apache.spark.tags.SlowHiveTest - -@SlowHiveTest -class GlutenHiveTableScanSuite - extends HiveTableScanSuite - with GlutenTestSetWithSystemPropertyTrait - with GlutenHiveComparisonTestSupport { - - override def testNameBlackList: Seq[String] = Seq( - // Rewritten with a workspace-backed file because the upstream test resolves this resource - // from the spark-hive tests jar in gluten-ut. - "Spark-4077: timestamp query for null value" - ) - - testGluten("Spark-4077: timestamp query for null value") { - withTable("timestamp_query_null") { - sql(""" - |CREATE TABLE timestamp_query_null (time TIMESTAMP,id INT) - |ROW FORMAT DELIMITED - |FIELDS TERMINATED BY ',' - |LINES TERMINATED BY '\n' - """.stripMargin) - val location = hiveResourcePath("data/files/issue-4077-data.txt").toFile.toURI - - sql(s"LOAD DATA LOCAL INPATH '$location' INTO TABLE timestamp_query_null") - assert( - sql("SELECT time FROM timestamp_query_null LIMIT 2").collect() === - Array(Row(java.sql.Timestamp.valueOf("2014-12-11 00:00:00")), Row(null))) - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveTypeCoercionSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveTypeCoercionSuite.scala deleted file mode 100644 index d77f2c7d578..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveTypeCoercionSuite.scala +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait -import org.apache.spark.tags.SlowHiveTest - -@SlowHiveTest -class GlutenHiveTypeCoercionSuite - extends HiveTypeCoercionSuite - with GlutenTestSetWithSystemPropertyTrait - with GlutenHiveComparisonTestSupport {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveUDAFSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveUDAFSuite.scala deleted file mode 100644 index 975fc9fb899..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveUDAFSuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait -import org.apache.spark.tags.SlowHiveTest - -@SlowHiveTest -class GlutenHiveUDAFSuite extends HiveUDAFSuite with GlutenTestSetWithSystemPropertyTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveUDFSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveUDFSuite.scala deleted file mode 100644 index a196da6ab4f..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveUDFSuite.scala +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.spark.sql.{GlutenTestSetWithSystemPropertyTrait, Row} -import org.apache.spark.tags.SlowHiveTest - -@SlowHiveTest -class GlutenHiveUDFSuite - extends HiveUDFSuite - with GlutenTestSetWithSystemPropertyTrait - with GlutenHiveResourcePathSupport { - - override def testNameBlackList: Seq[String] = Seq( - "UDTF", - "permanent UDTF" - ) - - override def beforeAll(): Unit = { - super.beforeAll() - GlutenTestHiveTables.registerHiveQTestUtilsTables(hiveTestResourceDir) - } - - testGluten("UDTF") { - withUserDefinedFunction("udtf_count2" -> true) { - sql(s"ADD JAR ${hiveResourcePath("TestUDTF.jar").toFile.getCanonicalPath}") - sql(""" - |CREATE TEMPORARY FUNCTION udtf_count2 - |AS 'org.apache.spark.sql.hive.execution.GenericUDTFCount2' - """.stripMargin) - - checkAnswer( - sql("SELECT key, cc FROM src LATERAL VIEW udtf_count2(value) dd AS cc"), - Row(97, 500) :: Row(97, 500) :: Nil) - - checkAnswer( - sql("SELECT udtf_count2(a) FROM (SELECT 1 AS a FROM src LIMIT 3) t"), - Row(3) :: Row(3) :: Nil) - } - } - - testGluten("permanent UDTF") { - withUserDefinedFunction("udtf_count_temp" -> false) { - sql(s""" - |CREATE FUNCTION udtf_count_temp - |AS 'org.apache.spark.sql.hive.execution.GenericUDTFCount2' - |USING JAR '${hiveResourcePath("TestUDTF.jar").toUri}' - """.stripMargin) - - checkAnswer( - sql("SELECT key, cc FROM src LATERAL VIEW udtf_count_temp(value) dd AS cc"), - Row(97, 500) :: Row(97, 500) :: Nil) - - checkAnswer( - sql("SELECT udtf_count_temp(a) FROM (SELECT 1 AS a FROM src LIMIT 3) t"), - Row(3) :: Row(3) :: Nil) - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenObjectHashAggregateSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenObjectHashAggregateSuite.scala deleted file mode 100644 index 327b633b1f0..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenObjectHashAggregateSuite.scala +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait -import org.apache.spark.tags.SlowHiveTest - -@SlowHiveTest -class GlutenObjectHashAggregateSuite - extends ObjectHashAggregateSuite - with GlutenTestSetWithSystemPropertyTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenPruneHiveTablePartitionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenPruneHiveTablePartitionsSuite.scala deleted file mode 100644 index 14ffda794dc..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenPruneHiveTablePartitionsSuite.scala +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait -import org.apache.spark.tags.SlowHiveTest - -@SlowHiveTest -class GlutenPruneHiveTablePartitionsSuite - extends PruneHiveTablePartitionsSuite - with GlutenTestSetWithSystemPropertyTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenPruningSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenPruningSuite.scala deleted file mode 100644 index aeb6e85b6a0..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenPruningSuite.scala +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait -import org.apache.spark.tags.SlowHiveTest - -@SlowHiveTest -class GlutenPruningSuite - extends PruningSuite - with GlutenTestSetWithSystemPropertyTrait - with GlutenHiveComparisonTestSupport { - override def beforeAll(): Unit = { - super.beforeAll() - GlutenTestHiveTables.registerHiveQTestUtilsTables(hiveTestResourceDir) - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenSQLMetricsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenSQLMetricsSuite.scala deleted file mode 100644 index df750c09c49..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenSQLMetricsSuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait -import org.apache.spark.tags.SlowHiveTest - -@SlowHiveTest -class GlutenSQLMetricsSuite extends SQLMetricsSuite with GlutenTestSetWithSystemPropertyTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenSQLQuerySuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenSQLQuerySuite.scala deleted file mode 100644 index 2b9272c7063..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenSQLQuerySuite.scala +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.spark.TestUtils -import org.apache.spark.sql.{GlutenTestSetWithSystemPropertyTrait, Row} -import org.apache.spark.tags.SlowHiveTest - -import java.sql.Date - -trait GlutenSQLQuerySuiteBase - extends SQLQuerySuiteBase - with GlutenTestSetWithSystemPropertyTrait - with GlutenHiveResourcePathSupport { - - override def testNameBlackList: Seq[String] = Seq( - // Rewritten with workspace-backed jar paths because TestHive.getHiveFile resolves - // these jars from the Spark test jars in gluten-ut. - "script", - "describe functions - user defined functions", - "describe functions - temporary user defined functions", - "SPARK-32668: HiveGenericUDTF initialize UDTF should use StructObjectInspector method" - ) - - override def beforeAll(): Unit = { - super.beforeAll() - GlutenTestHiveTables.registerHiveQTestUtilsTables(hiveTestResourceDir) - } - - testGluten("script") { - withTempView("script_table") { - import spark.implicits._ - - assume(TestUtils.testCommandAvailable("/bin/bash")) - assume(TestUtils.testCommandAvailable("echo")) - assume(TestUtils.testCommandAvailable("sed")) - val scriptFilePath = hiveResourcePath("test_script.sh").toFile.getCanonicalPath - val df = Seq(("x1", "y1", "z1"), ("x2", "y2", "z2")).toDF("c1", "c2", "c3") - df.createOrReplaceTempView("script_table") - val query = sql(s""" - |SELECT col1 FROM (from(SELECT c1, c2, c3 FROM script_table) tempt_table - |REDUCE c1, c2, c3 USING 'bash $scriptFilePath' AS - |(col1 STRING, col2 STRING)) script_test_table""".stripMargin) - checkAnswer(query, Row("x1_y1") :: Row("x2_y2") :: Nil) - } - } - - testGluten("describe functions - user defined functions") { - withUserDefinedFunction("udtf_count" -> false) { - sql(s""" - |CREATE FUNCTION udtf_count - |AS 'org.apache.spark.sql.hive.execution.GenericUDTFCount2' - |USING JAR '${hiveResourcePath("TestUDTF.jar").toUri}' - """.stripMargin) - - checkKeywordsExist( - sql("describe function udtf_count"), - s"Function: default.udtf_count", - s"Class: org.apache.spark.sql.hive.execution.GenericUDTFCount2", - "Usage: N/A" - ) - checkAnswer( - sql("SELECT udtf_count(a) FROM (SELECT 1 AS a FROM src LIMIT 3) t"), - Row(3) :: Row(3) :: Nil) - checkKeywordsExist( - sql("describe function udtf_count"), - s"Function: default.udtf_count", - s"Class: org.apache.spark.sql.hive.execution.GenericUDTFCount2", - "Usage: N/A" - ) - } - } - - testGluten("describe functions - temporary user defined functions") { - withUserDefinedFunction("udtf_count_temp" -> true) { - sql(s""" - |CREATE TEMPORARY FUNCTION udtf_count_temp - |AS 'org.apache.spark.sql.hive.execution.GenericUDTFCount2' - |USING JAR '${hiveResourcePath("TestUDTF.jar").toUri}' - """.stripMargin) - - checkKeywordsExist( - sql("describe function udtf_count_temp"), - "Function: udtf_count_temp", - s"Class: org.apache.spark.sql.hive.execution.GenericUDTFCount2", - "Usage: N/A") - checkAnswer( - sql("SELECT udtf_count_temp(a) FROM (SELECT 1 AS a FROM src LIMIT 3) t"), - Row(3) :: Row(3) :: Nil) - checkKeywordsExist( - sql("describe function udtf_count_temp"), - "Function: udtf_count_temp", - s"Class: org.apache.spark.sql.hive.execution.GenericUDTFCount2", - "Usage: N/A") - } - } - - testGluten( - "SPARK-32668: HiveGenericUDTF initialize UDTF should use StructObjectInspector method") { - withUserDefinedFunction("udtf_stack1" -> true, "udtf_stack2" -> true) { - sql(s""" - |CREATE TEMPORARY FUNCTION udtf_stack1 - |AS 'org.apache.spark.sql.hive.execution.UDTFStack' - |USING JAR '${hiveResourcePath("SPARK-21101-1.0.jar").toUri}' - """.stripMargin) - sql(s""" - |CREATE TEMPORARY FUNCTION udtf_stack2 - |AS 'org.apache.spark.sql.hive.execution.UDTFStack2' - |USING JAR '${hiveResourcePath("SPARK-21101-1.0.jar").toUri}' - """.stripMargin) - - Seq("udtf_stack1", "udtf_stack2").foreach { - udf => - checkAnswer( - sql(s"SELECT $udf(2, 'A', 10, date '2015-01-01', 'B', 20, date '2016-01-01')"), - Seq(Row("A", 10, Date.valueOf("2015-01-01")), Row("B", 20, Date.valueOf("2016-01-01"))) - ) - } - } - } -} - -@SlowHiveTest -class GlutenSQLQuerySuite extends SQLQuerySuite with GlutenSQLQuerySuiteBase {} - -@SlowHiveTest -class GlutenSQLQuerySuiteAE extends SQLQuerySuiteAE with GlutenSQLQuerySuiteBase {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenTestHiveTables.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenTestHiveTables.scala deleted file mode 100644 index 0eaa597a868..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenTestHiveTables.scala +++ /dev/null @@ -1,235 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.spark.sql.hive.test.TestHive.sparkSession -import org.apache.spark.sql.hive.test.TestHive.sparkSession.TestTable -import org.apache.spark.sql.hive.test.TestHiveQueryExecution - -import org.apache.hadoop.hive.serde2.`lazy`.LazySimpleSerDe - -import java.nio.file.Path - -/** Wrappers around [[TestHive]] lazy-loaded table registrations. */ -object GlutenTestHiveTables { - def hiveResourcePath(resourceDir: Path, relativePath: String): Path = { - relativePath.split('/').foldLeft(resourceDir) { case (path, child) => path.resolve(child) } - } - - private def hiveDataFile(resourceDir: Path, fileName: String): String = { - hiveResourcePath(resourceDir, fileName).toAbsolutePath.normalize.toString - } - - implicit private class SqlCmd(sql: String) { - def cmd: () => Unit = { - () => new TestHiveQueryExecution(sql).executedPlan.executeCollect(): Unit - } - } - - def registerTestTable(testTable: TestTable): Unit = { - sparkSession.registerTestTable(testTable) - } - - def registerHiveQTestUtilsTables(resourceDir: Path): Unit = { - def createTableSQL(tblName: String): String = { - s"CREATE TABLE $tblName (key INT, value STRING) STORED AS textfile" - } - // The test tables that are defined in the Hive QTestUtil. - // /itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java - // https://github.com/apache/hive/blob/branch-0.13/data/scripts/q_test_init.sql - @transient - val hiveQTestUtilTables: Seq[TestTable] = Seq( - TestTable( - "src", - createTableSQL("src").cmd, - s"""LOAD DATA LOCAL INPATH '${hiveDataFile(resourceDir, "data/files/kv1.txt")}' - |INTO TABLE src""".stripMargin.cmd - ), - TestTable( - "src1", - createTableSQL("src1").cmd, - s"""LOAD DATA LOCAL INPATH '${hiveDataFile(resourceDir, "data/files/kv3.txt")}' - |INTO TABLE src1""".stripMargin.cmd - ), - TestTable( - "srcpart", - () => { - s"${createTableSQL("srcpart")} PARTITIONED BY (ds STRING, hr STRING)".cmd.apply() - for (ds <- Seq("2008-04-08", "2008-04-09"); hr <- Seq("11", "12")) { - s""" - |LOAD DATA LOCAL INPATH '${hiveDataFile(resourceDir, "data/files/kv1.txt")}' - |OVERWRITE INTO TABLE srcpart PARTITION (ds='$ds',hr='$hr') - """.stripMargin.cmd.apply() - } - } - ), - TestTable( - "srcpart1", - () => { - s"${createTableSQL("srcpart1")} PARTITIONED BY (ds STRING, hr INT)".cmd.apply() - for (ds <- Seq("2008-04-08", "2008-04-09"); hr <- 11 to 12) { - s""" - |LOAD DATA LOCAL INPATH '${hiveDataFile(resourceDir, "data/files/kv1.txt")}' - |OVERWRITE INTO TABLE srcpart1 PARTITION (ds='$ds',hr='$hr') - """.stripMargin.cmd.apply() - } - } - ), - TestTable( - "src_thrift", - () => { - import org.apache.hadoop.hive.serde2.thrift.ThriftDeserializer - import org.apache.hadoop.mapred.{SequenceFileInputFormat, SequenceFileOutputFormat} - import org.apache.thrift.protocol.TBinaryProtocol - - s""" - |CREATE TABLE src_thrift(fake INT) - |ROW FORMAT SERDE '${classOf[ThriftDeserializer].getName}' - |WITH SERDEPROPERTIES( - | 'serialization.class'='org.apache.spark.sql.hive.test.Complex', - | 'serialization.format'='${classOf[TBinaryProtocol].getName}' - |) - |STORED AS - |INPUTFORMAT '${classOf[SequenceFileInputFormat[_, _]].getName}' - |OUTPUTFORMAT '${classOf[SequenceFileOutputFormat[_, _]].getName}' - """.stripMargin.cmd.apply() - - s""" - |LOAD DATA LOCAL INPATH '${hiveDataFile(resourceDir, "data/files/complex.seq")}' - |INTO TABLE src_thrift - """.stripMargin.cmd.apply() - } - ), - TestTable( - "serdeins", - s"""CREATE TABLE serdeins (key INT, value STRING) - |ROW FORMAT SERDE '${classOf[LazySimpleSerDe].getCanonicalName}' - |WITH SERDEPROPERTIES ('field.delim'='\\t') - """.stripMargin.cmd, - "INSERT OVERWRITE TABLE serdeins SELECT * FROM src".cmd - ), - TestTable( - "episodes", - s"""CREATE TABLE episodes (title STRING, air_date STRING, doctor INT) - |STORED AS avro - |TBLPROPERTIES ( - | 'avro.schema.literal'='{ - | "type": "record", - | "name": "episodes", - | "namespace": "testing.hive.avro.serde", - | "fields": [ - | { - | "name": "title", - | "type": "string", - | "doc": "episode title" - | }, - | { - | "name": "air_date", - | "type": "string", - | "doc": "initial date" - | }, - | { - | "name": "doctor", - | "type": "int", - | "doc": "main actor playing the Doctor in episode" - | } - | ] - | }' - |) - """.stripMargin.cmd, - s""" - |LOAD DATA LOCAL INPATH '${hiveDataFile(resourceDir, "data/files/episodes.avro")}' - |INTO TABLE episodes - """.stripMargin.cmd - ), - // THIS TABLE IS NOT THE SAME AS THE HIVE TEST TABLE episodes_partitioned AS DYNAMIC - // PARTITIONING IS NOT YET SUPPORTED - TestTable( - "episodes_part", - s"""CREATE TABLE episodes_part (title STRING, air_date STRING, doctor INT) - |PARTITIONED BY (doctor_pt INT) - |STORED AS avro - |TBLPROPERTIES ( - | 'avro.schema.literal'='{ - | "type": "record", - | "name": "episodes", - | "namespace": "testing.hive.avro.serde", - | "fields": [ - | { - | "name": "title", - | "type": "string", - | "doc": "episode title" - | }, - | { - | "name": "air_date", - | "type": "string", - | "doc": "initial date" - | }, - | { - | "name": "doctor", - | "type": "int", - | "doc": "main actor playing the Doctor in episode" - | } - | ] - | }' - |) - """.stripMargin.cmd, - // WORKAROUND: Required to pass schema to SerDe for partitioned tables. - // TODO: Pass this automatically from the table to partitions. - s""" - |ALTER TABLE episodes_part SET SERDEPROPERTIES ( - | 'avro.schema.literal'='{ - | "type": "record", - | "name": "episodes", - | "namespace": "testing.hive.avro.serde", - | "fields": [ - | { - | "name": "title", - | "type": "string", - | "doc": "episode title" - | }, - | { - | "name": "air_date", - | "type": "string", - | "doc": "initial date" - | }, - | { - | "name": "doctor", - | "type": "int", - | "doc": "main actor playing the Doctor in episode" - | } - | ] - | }' - |) - """.stripMargin.cmd, - s""" - INSERT OVERWRITE TABLE episodes_part PARTITION (doctor_pt=1) - SELECT title, air_date, doctor FROM episodes - """.cmd - ), - TestTable( - "src_json", - s"""CREATE TABLE src_json (json STRING) STORED AS TEXTFILE - """.stripMargin.cmd, - s"""LOAD DATA LOCAL INPATH '${hiveDataFile(resourceDir, "data/files/json.txt")}' - |INTO TABLE src_json""".stripMargin.cmd - ) - ) - - hiveQTestUtilTables.foreach(registerTestTable) - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenUDAQuerySuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenUDAQuerySuite.scala deleted file mode 100644 index 0d279899558..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenUDAQuerySuite.scala +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait -import org.apache.spark.tags.SlowHiveTest - -@SlowHiveTest -class GlutenHashUDAQuerySuite extends HashUDAQuerySuite with GlutenTestSetWithSystemPropertyTrait {} - -@SlowHiveTest -class GlutenHashUDAQueryWithControlledFallbackSuite - extends HashUDAQueryWithControlledFallbackSuite - with GlutenTestSetWithSystemPropertyTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenWindowQuerySuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenWindowQuerySuite.scala deleted file mode 100644 index 701fbb8467a..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenWindowQuerySuite.scala +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait -import org.apache.spark.tags.SlowHiveTest - -@SlowHiveTest -class GlutenWindowQuerySuite - extends WindowQuerySuite - with GlutenTestSetWithSystemPropertyTrait - with GlutenHiveResourcePathSupport { - - /** - * Mostly copied from Spark's [[WindowQuerySuite]] and [[GlutenTestSetWithSystemPropertyTrait]], - * and customized so `part_tiny.txt` is loaded from the workspace-backed Hive test resources - * instead of via [[TestHive.getHiveFile]]. - */ - override def beforeAll(): Unit = { - System.setProperty("spark.plugins", "org.apache.gluten.GlutenPlugin") - System.setProperty("spark.memory.offHeap.enabled", "true") - System.setProperty("spark.memory.offHeap.size", "1024MB") - System.setProperty( - "spark.shuffle.manager", - "org.apache.spark.shuffle.sort.ColumnarShuffleManager") - - sql("DROP TABLE IF EXISTS part") - sql(""" - |CREATE TABLE part( - | p_partkey INT, - | p_name STRING, - | p_mfgr STRING, - | p_brand STRING, - | p_type STRING, - | p_size INT, - | p_container STRING, - | p_retailprice DOUBLE, - | p_comment STRING) USING hive - """.stripMargin) - val testData1 = hiveResourcePath("data/files/part_tiny.txt").toFile.toURI - sql(s""" - |LOAD DATA LOCAL INPATH '$testData1' overwrite into table part - """.stripMargin) - } - - override def afterAll(): Unit = { - try { - super.afterAll() - } finally { - System.clearProperty("spark.plugins") - System.clearProperty("spark.memory.offHeap.enabled") - System.clearProperty("spark.memory.offHeap.size") - System.clearProperty("spark.shuffle.manager") - } - } -} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenBucketedReadSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenBucketedReadSuite.scala deleted file mode 100644 index 9a9f06e02c5..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenBucketedReadSuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.sources - -import org.apache.spark.sql._ - -class GlutenBucketedReadWithoutHiveSupportSuite - extends BucketedReadWithoutHiveSupportSuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenBucketedWriteSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenBucketedWriteSuite.scala deleted file mode 100644 index 2083a993676..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenBucketedWriteSuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.sources - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenBucketedWriteWithoutHiveSupportSuite - extends BucketedWriteWithoutHiveSupportSuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenCreateTableAsSelectSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenCreateTableAsSelectSuite.scala deleted file mode 100644 index 7f31d62f74b..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenCreateTableAsSelectSuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.sources - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenCreateTableAsSelectSuite - extends CreateTableAsSelectSuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenDDLSourceLoadSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenDDLSourceLoadSuite.scala deleted file mode 100644 index 03775cab391..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenDDLSourceLoadSuite.scala +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.sources - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -// please note that the META-INF/services had to be modified for the test directory for this to work -class GlutenDDLSourceLoadSuite extends DDLSourceLoadSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenDisableUnnecessaryBucketedScanSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenDisableUnnecessaryBucketedScanSuite.scala deleted file mode 100644 index fd77663985b..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenDisableUnnecessaryBucketedScanSuite.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.sources - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenDisableUnnecessaryBucketedScanWithoutHiveSupportSuite - extends DisableUnnecessaryBucketedScanWithoutHiveSupportSuite - with GlutenSQLTestsBaseTrait {} - -class GlutenDisableUnnecessaryBucketedScanWithoutHiveSupportSuiteAE - extends DisableUnnecessaryBucketedScanWithoutHiveSupportSuiteAE - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenExternalCommandRunnerSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenExternalCommandRunnerSuite.scala deleted file mode 100644 index 84ba336099a..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenExternalCommandRunnerSuite.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.sources - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenExternalCommandRunnerSuite - extends ExternalCommandRunnerSuite - with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenFilteredScanSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenFilteredScanSuite.scala deleted file mode 100644 index d751f20ae3f..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenFilteredScanSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.sources - -import org.apache.spark.sql._ - -class GlutenFilteredScanSuite extends FilteredScanSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenFiltersSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenFiltersSuite.scala deleted file mode 100644 index ad91b92aae2..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenFiltersSuite.scala +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.sources - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -/** Unit test suites for data source filters. */ -class GlutenFiltersSuite extends FiltersSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala deleted file mode 100644 index 165d5173130..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.sources - -import org.apache.spark.sql._ - -class GlutenInsertSuite extends InsertSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenPartitionedWriteSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenPartitionedWriteSuite.scala deleted file mode 100644 index 26c847ff232..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenPartitionedWriteSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.sources - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenPartitionedWriteSuite extends PartitionedWriteSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenPathOptionSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenPathOptionSuite.scala deleted file mode 100644 index 94171f44cec..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenPathOptionSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.sources - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenPathOptionSuite extends PathOptionSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenPrunedScanSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenPrunedScanSuite.scala deleted file mode 100644 index 920d4f3af64..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenPrunedScanSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.sources - -import org.apache.spark.sql._ - -class GlutenPrunedScanSuite extends PrunedScanSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenResolvedDataSourceSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenResolvedDataSourceSuite.scala deleted file mode 100644 index ddd06bb3fd8..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenResolvedDataSourceSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.sources - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenResolvedDataSourceSuite extends ResolvedDataSourceSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenSaveLoadSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenSaveLoadSuite.scala deleted file mode 100644 index 5ae0204b835..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenSaveLoadSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.sources - -import org.apache.spark.sql.GlutenSQLTestsBaseTrait - -class GlutenSaveLoadSuite extends SaveLoadSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenTableScanSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenTableScanSuite.scala deleted file mode 100644 index ebd17781ff2..00000000000 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/sources/GlutenTableScanSuite.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.sources - -import org.apache.spark.sql._ - -class GlutenTableScanSuite extends TableScanSuite with GlutenSQLTestsBaseTrait {} diff --git a/package/pom.xml b/package/pom.xml index cee31a8ecf5..9024671290d 100644 --- a/package/pom.xml +++ b/package/pom.xml @@ -1,17 +1,20 @@ + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> 4.0.0 @@ -233,7 +236,6 @@ target - *spark3.3* *spark3.4* *spark3.5* *spark4.0* diff --git a/pom.xml b/pom.xml index c7d33eda615..5b1a91cd99c 100644 --- a/pom.xml +++ b/pom.xml @@ -979,7 +979,7 @@ - spark-3.3,spark-3.4,spark-3.5,spark-4.0,spark-4.1 + spark-3.4,spark-3.5,spark-4.0,spark-4.1 false Missing spark version profile: -Pspark-<version> @@ -1212,34 +1212,6 @@ true - - spark-3.3 - - 3.3 - 33 - spark-sql-columnar-shims-spark33 - 3.3.1 - 1.5.0 - 5 - delta-core - 2.3.0 - 23 - 2.15.0 - 4.8 - 3.3.2 - 0.15.0 - 1.7.32 - 2.17.2 - - - - org.apache.logging.log4j - log4j-core - ${log4j.version} - provided - - - spark-3.4 diff --git a/shims/pom.xml b/shims/pom.xml index faf6c15362f..50ffa892b56 100644 --- a/shims/pom.xml +++ b/shims/pom.xml @@ -1,17 +1,20 @@ + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> 4.0.0 @@ -68,12 +71,6 @@ - - spark-3.3 - - spark33 - - spark-3.4 diff --git a/shims/spark33/pom.xml b/shims/spark33/pom.xml deleted file mode 100644 index 5074d33ce1e..00000000000 --- a/shims/spark33/pom.xml +++ /dev/null @@ -1,118 +0,0 @@ - - - - 4.0.0 - - - org.apache.gluten - spark-sql-columnar-shims - 1.8.0-SNAPSHOT - ../pom.xml - - - spark-sql-columnar-shims-spark33 - jar - Gluten Shims for Spark 3.3 - - - - org.apache.gluten - ${project.prefix}-shims-common - ${project.version} - compile - - - org.apache.spark - spark-sql_${scala.binary.version} - provided - true - - - org.apache.spark - spark-catalyst_${scala.binary.version} - provided - true - - - org.apache.spark - spark-core_${scala.binary.version} - provided - true - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - provided - - - - - org.scalatest - scalatest_${scala.binary.version} - test - - - org.apache.spark - spark-core_${scala.binary.version} - test-jar - - - org.apache.spark - spark-sql_${scala.binary.version} - test-jar - - - org.apache.spark - spark-catalyst_${scala.binary.version} - test-jar - - - org.apache.spark - spark-hive_${scala.binary.version} - provided - - - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - org.scalastyle - scalastyle-maven-plugin - - - com.diffplug.spotless - spotless-maven-plugin - - - net.alchim31.maven - scala-maven-plugin - - - org.scalatest - scalatest-maven-plugin - - - org.apache.maven.plugins - maven-compiler-plugin - - - - diff --git a/shims/spark33/src/main/java/org/apache/spark/sql/execution/vectorized/WritableColumnVectorShim.java b/shims/spark33/src/main/java/org/apache/spark/sql/execution/vectorized/WritableColumnVectorShim.java deleted file mode 100644 index 1b77cb666c7..00000000000 --- a/shims/spark33/src/main/java/org/apache/spark/sql/execution/vectorized/WritableColumnVectorShim.java +++ /dev/null @@ -1,223 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.vectorized; - -import org.apache.spark.sql.types.DataType; -import org.apache.spark.unsafe.types.UTF8String; - -import java.nio.ByteBuffer; - -/** - * because spark33 add new function abstract method 'putBooleans(int, byte)' in - * 'WritableColumnVector' And function getByteBuffer() - */ -public class WritableColumnVectorShim extends WritableColumnVector { - /** - * Sets up the common state and also handles creating the child columns if this is a nested type. - * - * @param capacity - * @param type - */ - protected WritableColumnVectorShim(int capacity, DataType type) { - super(capacity, type); - } - - @Override - public int getDictId(int rowId) { - return 0; - } - - @Override - protected void reserveInternal(int capacity) {} - - @Override - public void putNotNull(int rowId) {} - - @Override - public void putNull(int rowId) {} - - @Override - public void putNulls(int rowId, int count) {} - - @Override - public void putNotNulls(int rowId, int count) {} - - @Override - public void putBoolean(int rowId, boolean value) {} - - @Override - public void putBooleans(int rowId, int count, boolean value) {} - - @Override - public void putBooleans(int rowId, byte src) { - throw new UnsupportedOperationException("Unsupported function"); - } - - @Override - public void putByte(int rowId, byte value) {} - - @Override - public void putBytes(int rowId, int count, byte value) {} - - @Override - public void putBytes(int rowId, int count, byte[] src, int srcIndex) {} - - @Override - public void putShort(int rowId, short value) {} - - @Override - public void putShorts(int rowId, int count, short value) {} - - @Override - public void putShorts(int rowId, int count, short[] src, int srcIndex) {} - - @Override - public void putShorts(int rowId, int count, byte[] src, int srcIndex) {} - - @Override - public void putInt(int rowId, int value) {} - - @Override - public void putInts(int rowId, int count, int value) {} - - @Override - public void putInts(int rowId, int count, int[] src, int srcIndex) {} - - @Override - public void putInts(int rowId, int count, byte[] src, int srcIndex) {} - - @Override - public void putIntsLittleEndian(int rowId, int count, byte[] src, int srcIndex) {} - - @Override - public void putLong(int rowId, long value) {} - - @Override - public void putLongs(int rowId, int count, long value) {} - - @Override - public void putLongs(int rowId, int count, long[] src, int srcIndex) {} - - @Override - public void putLongs(int rowId, int count, byte[] src, int srcIndex) {} - - @Override - public void putLongsLittleEndian(int rowId, int count, byte[] src, int srcIndex) {} - - @Override - public void putFloat(int rowId, float value) {} - - @Override - public void putFloats(int rowId, int count, float value) {} - - @Override - public void putFloats(int rowId, int count, float[] src, int srcIndex) {} - - @Override - public void putFloats(int rowId, int count, byte[] src, int srcIndex) {} - - @Override - public void putFloatsLittleEndian(int rowId, int count, byte[] src, int srcIndex) {} - - @Override - public void putDouble(int rowId, double value) {} - - @Override - public void putDoubles(int rowId, int count, double value) {} - - @Override - public void putDoubles(int rowId, int count, double[] src, int srcIndex) {} - - @Override - public void putDoubles(int rowId, int count, byte[] src, int srcIndex) {} - - @Override - public void putDoublesLittleEndian(int rowId, int count, byte[] src, int srcIndex) {} - - @Override - public void putArray(int rowId, int offset, int length) {} - - @Override - public int putByteArray(int rowId, byte[] value, int offset, int count) { - return 0; - } - - @Override - protected UTF8String getBytesAsUTF8String(int rowId, int count) { - return null; - } - - @Override - public ByteBuffer getByteBuffer(int rowId, int count) { - throw new UnsupportedOperationException("Unsupported this function"); - } - - @Override - public int getArrayLength(int rowId) { - return 0; - } - - @Override - public int getArrayOffset(int rowId) { - return 0; - } - - @Override - protected WritableColumnVector reserveNewColumn(int capacity, DataType type) { - return null; - } - - @Override - public boolean isNullAt(int rowId) { - return false; - } - - @Override - public boolean getBoolean(int rowId) { - return false; - } - - @Override - public byte getByte(int rowId) { - return 0; - } - - @Override - public short getShort(int rowId) { - return 0; - } - - @Override - public int getInt(int rowId) { - return 0; - } - - @Override - public long getLong(int rowId) { - return 0; - } - - @Override - public float getFloat(int rowId) { - return 0; - } - - @Override - public double getDouble(int rowId) { - return 0; - } -} diff --git a/shims/spark33/src/main/java/org/apache/spark/sql/vectorized/ArrowColumnarArray.java b/shims/spark33/src/main/java/org/apache/spark/sql/vectorized/ArrowColumnarArray.java deleted file mode 100644 index cb0c11e5e40..00000000000 --- a/shims/spark33/src/main/java/org/apache/spark/sql/vectorized/ArrowColumnarArray.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.vectorized; - -public final class ArrowColumnarArray extends AbstractColumnarArray { - - public ArrowColumnarArray(ColumnVector data, int offset, int length) { - super(data, offset, length); - } -} diff --git a/shims/spark33/src/main/resources/META-INF/services/org.apache.gluten.sql.shims.SparkShimProvider b/shims/spark33/src/main/resources/META-INF/services/org.apache.gluten.sql.shims.SparkShimProvider deleted file mode 100644 index 9b06666c2bf..00000000000 --- a/shims/spark33/src/main/resources/META-INF/services/org.apache.gluten.sql.shims.SparkShimProvider +++ /dev/null @@ -1 +0,0 @@ -org.apache.gluten.sql.shims.spark33.SparkShimProvider \ No newline at end of file diff --git a/shims/spark33/src/main/scala/org/apache/gluten/execution/GenerateTreeStringShim.scala b/shims/spark33/src/main/scala/org/apache/gluten/execution/GenerateTreeStringShim.scala deleted file mode 100644 index 8936e6ca635..00000000000 --- a/shims/spark33/src/main/scala/org/apache/gluten/execution/GenerateTreeStringShim.scala +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.execution - -import org.apache.spark.sql.execution.UnaryExecNode - -/** - * Spark 3.5 has changed the parameter type of the generateTreeString API in TreeNode. In order to - * support multiple versions of Spark, we cannot directly override the generateTreeString method in - * WhostageTransformer. Therefore, we have defined the GenerateTreeStringShim trait in the shim to - * allow different Spark versions to override their own generateTreeString. - */ - -trait WholeStageTransformerGenerateTreeStringShim extends UnaryExecNode { - - def stageId: Int - - def substraitPlanJson: String - - def wholeStageTransformerContextDefined: Boolean - - override def generateTreeString( - depth: Int, - lastChildren: Seq[Boolean], - append: String => Unit, - verbose: Boolean, - prefix: String = "", - addSuffix: Boolean = false, - maxFields: Int, - printNodeId: Boolean, - indent: Int = 0): Unit = { - val prefix = if (printNodeId) "^ " else s"^($stageId) " - child.generateTreeString( - depth, - lastChildren, - append, - verbose, - prefix, - addSuffix = false, - maxFields, - printNodeId = printNodeId, - indent) - - if (verbose && wholeStageTransformerContextDefined) { - append(prefix + "Substrait plan:\n") - append(substraitPlanJson) - append("\n") - } - } -} - -trait InputAdapterGenerateTreeStringShim extends UnaryExecNode { - - override def generateTreeString( - depth: Int, - lastChildren: Seq[Boolean], - append: String => Unit, - verbose: Boolean, - prefix: String = "", - addSuffix: Boolean = false, - maxFields: Int, - printNodeId: Boolean, - indent: Int = 0): Unit = { - child.generateTreeString( - depth, - lastChildren, - append, - verbose, - prefix = "", - addSuffix = false, - maxFields, - printNodeId, - indent) - } -} diff --git a/shims/spark33/src/main/scala/org/apache/gluten/execution/PartitionedFileUtilShim.scala b/shims/spark33/src/main/scala/org/apache/gluten/execution/PartitionedFileUtilShim.scala deleted file mode 100644 index a6fdd6de2f9..00000000000 --- a/shims/spark33/src/main/scala/org/apache/gluten/execution/PartitionedFileUtilShim.scala +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.execution - -import org.apache.spark.sql.execution.datasources.PartitionedFile - -object PartitionedFileUtilShim { - // Helper method to create PartitionedFile from path and length. - def makePartitionedFileFromPath(path: String, length: Long): PartitionedFile = { - PartitionedFile(null, path, 0, length, Array.empty) - } -} diff --git a/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala b/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala deleted file mode 100644 index 1208049c853..00000000000 --- a/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala +++ /dev/null @@ -1,319 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.sql.shims.spark33 - -import org.apache.gluten.execution.datasource.GlutenFormatFactory -import org.apache.gluten.expression.{ExpressionNames, Sig} -import org.apache.gluten.sql.shims.SparkShims -import org.apache.gluten.utils.ExceptionUtils - -import org.apache.spark._ -import org.apache.spark.sql.{AnalysisException, SparkSession} -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.analysis.DecimalPrecision -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.plans.QueryPlan -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.catalyst.util.RebaseDateTime.RebaseSpec -import org.apache.spark.sql.catalyst.util.TimestampFormatter -import org.apache.spark.sql.connector.catalog.Table -import org.apache.spark.sql.execution.{FileSourceScanExec, PartitionedFileUtil, QueryExecution, SparkPlan, SparkPlanner} -import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec -import org.apache.spark.sql.execution.datasources._ -import org.apache.spark.sql.execution.datasources.FileFormatWriter.Empty2Null -import org.apache.spark.sql.execution.datasources.parquet.ParquetFilters -import org.apache.spark.sql.execution.datasources.v2.BatchScanExec -import org.apache.spark.sql.execution.exchange.BroadcastExchangeLike -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy -import org.apache.spark.sql.types.{DecimalType, StructField, StructType} -import org.apache.spark.storage.{GlutenShuffleBlockFetcherIterator, GlutenShuffleBlockFetcherIteratorBase, ShuffleBlockFetcherIteratorParams} - -import org.apache.hadoop.fs.{FileStatus, Path} -import org.apache.parquet.crypto.ParquetCryptoRuntimeException -import org.apache.parquet.hadoop.metadata.ParquetMetadata -import org.apache.parquet.schema.MessageType - -import java.time.ZoneOffset - -import scala.collection.mutable - -class Spark33Shims extends SparkShims { - - override def scalarExpressionMappings: Seq[Sig] = { - Seq(Sig[Empty2Null](ExpressionNames.EMPTY2NULL)) - } - - override def aggregateExpressionMappings: Seq[Sig] = Seq() - - override def runtimeReplaceableExpressionMappings: Seq[Sig] = Seq() - - override def isNullIntolerant(expr: Expression): Boolean = expr.isInstanceOf[NullIntolerant] - - override def generateFileScanRDD( - sparkSession: SparkSession, - readFunction: PartitionedFile => Iterator[InternalRow], - filePartitions: Seq[FilePartition], - fileSourceScanExec: FileSourceScanExec): FileScanRDD = { - new FileScanRDD( - sparkSession, - readFunction, - filePartitions, - new StructType( - fileSourceScanExec.requiredSchema.fields ++ - fileSourceScanExec.relation.partitionSchema.fields), - fileSourceScanExec.metadataColumns - ) - } - - override def filesGroupedToBuckets( - selectedPartitions: Array[PartitionDirectory]): Map[Int, Array[PartitionedFile]] = { - selectedPartitions - .flatMap { - p => p.files.map(f => PartitionedFileUtil.getPartitionedFile(f, f.getPath, p.values)) - } - .groupBy { - f => - BucketingUtils - .getBucketId(new Path(f.filePath).getName) - .getOrElse(throw invalidBucketFile(f.filePath)) - } - } - - override def getBatchScanExecTable(batchScan: BatchScanExec): Table = null - - override def generatePartitionedFile( - partitionValues: InternalRow, - filePath: String, - start: Long, - length: Long, - @transient locations: Array[String] = Array.empty): PartitionedFile = - PartitionedFile(partitionValues, filePath, start, length, locations) - - override def generateMetadataColumns( - file: PartitionedFile, - metadataColumnNames: Seq[String]): Map[String, String] = { - val originMetadataColumn = super.generateMetadataColumns(file, metadataColumnNames) - val metadataColumn: mutable.Map[String, String] = mutable.Map(originMetadataColumn.toSeq: _*) - val path = new Path(file.filePath) - for (columnName <- metadataColumnNames) { - columnName match { - case FileFormat.FILE_PATH => metadataColumn += (FileFormat.FILE_PATH -> path.toString) - case FileFormat.FILE_NAME => metadataColumn += (FileFormat.FILE_NAME -> path.getName) - case FileFormat.FILE_SIZE => - metadataColumn += (FileFormat.FILE_SIZE -> file.fileSize.toString) - case FileFormat.FILE_MODIFICATION_TIME => - val fileModifyTime = TimestampFormatter - .getFractionFormatter(ZoneOffset.UTC) - .format(file.modificationTime * 1000L) - metadataColumn += (FileFormat.FILE_MODIFICATION_TIME -> fileModifyTime) - case _ => - } - } - metadataColumn.toMap - } - - private def invalidBucketFile(path: String): Throwable = { - new SparkException( - errorClass = "INVALID_BUCKET_FILE", - messageParameters = Array(path), - cause = null) - } - - override def getExtendedColumnarPostRules(): List[SparkSession => Rule[SparkPlan]] = { - List(session => GlutenFormatFactory.getExtendedColumnarPostRule(session)) - } - - def setJobDescriptionOrTagForBroadcastExchange( - sc: SparkContext, - broadcastExchange: BroadcastExchangeLike): Unit = { - // Setup a job group here so later it may get cancelled by groupId if necessary. - sc.setJobGroup( - broadcastExchange.runId.toString, - s"broadcast exchange (runId ${broadcastExchange.runId})", - interruptOnCancel = true) - } - - def cancelJobGroupForBroadcastExchange( - sc: SparkContext, - broadcastExchange: BroadcastExchangeLike): Unit = { - sc.cancelJobGroup(broadcastExchange.runId.toString) - } - - def getFileStatus(partition: PartitionDirectory): Seq[(FileStatus, Map[String, Any])] = - partition.files.map(f => (f, Map.empty[String, Any])) - - def isFileSplittable( - relation: HadoopFsRelation, - filePath: Path, - sparkSchema: StructType): Boolean = true - - def isRowIndexMetadataColumn(name: String): Boolean = false - - def findRowIndexColumnIndexInSchema(sparkSchema: StructType): Int = -1 - - def splitFiles( - sparkSession: SparkSession, - file: FileStatus, - filePath: Path, - isSplitable: Boolean, - maxSplitBytes: Long, - partitionValues: InternalRow, - metadata: Map[String, Any] = Map.empty): Seq[PartitionedFile] = { - PartitionedFileUtil.splitFiles( - sparkSession, - file, - filePath, - isSplitable, - maxSplitBytes, - partitionValues) - } - - def structFromAttributes(attrs: Seq[Attribute]): StructType = { - StructType(attrs.map(a => StructField(a.name, a.dataType, a.nullable, a.metadata))) - } - - def attributesFromStruct(structType: StructType): Seq[Attribute] = { - structType.fields.map { - field => AttributeReference(field.name, field.dataType, field.nullable, field.metadata)() - } - } - - def getAnalysisExceptionPlan(ae: AnalysisException): Option[LogicalPlan] = { - ae.plan - } - - override def getKeyGroupedPartitioning(batchScan: BatchScanExec): Option[Seq[Expression]] = { - batchScan.keyGroupedPartitioning - } - - override def extractExpressionTimestampAddUnit(exp: Expression): Option[Seq[String]] = { - exp match { - case timestampAdd: TimestampAdd => - Option.apply(Seq(timestampAdd.unit, timestampAdd.timeZoneId.getOrElse(""))) - case _ => Option.empty - } - } - - override def withAnsiEvalMode(expr: Expression): Boolean = { - expr match { - // Use the cast's own flag rather than the session conf: store-assignment casts - // can carry ansiEnabled = false even when the session runs in ANSI mode. - case c: Cast => c.ansiEnabled - case _ => false - } - } - - override def createParquetFilters( - conf: SQLConf, - schema: MessageType, - caseSensitive: Option[Boolean] = None): ParquetFilters = { - new ParquetFilters( - schema, - conf.parquetFilterPushDownDate, - conf.parquetFilterPushDownTimestamp, - conf.parquetFilterPushDownDecimal, - conf.parquetFilterPushDownStringStartWith, - conf.parquetFilterPushDownInFilterThreshold, - caseSensitive.getOrElse(conf.caseSensitiveAnalysis), - RebaseSpec(LegacyBehaviorPolicy.CORRECTED) - ) - } - - override def getOperatorId(plan: QueryPlan[_]): Option[Int] = { - plan.getTagValue(QueryPlan.OP_ID_TAG) - } - - override def setOperatorId(plan: QueryPlan[_], opId: Int): Unit = { - plan.setTagValue(QueryPlan.OP_ID_TAG, opId) - } - - override def unsetOperatorId(plan: QueryPlan[_]): Unit = { - plan.unsetTagValue(QueryPlan.OP_ID_TAG) - } - override def isParquetFileEncrypted(footer: ParquetMetadata): Boolean = { - try { - footer.toString - false - } catch { - case e: Exception if ExceptionUtils.hasCause(e, classOf[ParquetCryptoRuntimeException]) => - true - case e: Throwable => - e.printStackTrace() - false - } - } - - override def extractExpressionTimestampDiffUnit(exp: Expression): Option[String] = { - exp match { - case timestampDiff: TimestampDiff => - Some(timestampDiff.unit) - case _ => Option.empty - } - } - - override def widerDecimalType(d1: DecimalType, d2: DecimalType): DecimalType = { - DecimalPrecision.widerDecimalType(d1, d2) - } - - override def getErrorMessage(raiseError: RaiseError): Option[Expression] = { - Some(raiseError.child) - } - - /** - * Shim layer for QueryExecution to maintain compatibility across different Spark versions. - * - * @since Spark - * 4.1 - */ - override def createSparkPlan( - sparkSession: SparkSession, - planner: SparkPlanner, - plan: LogicalPlan): SparkPlan = - QueryExecution.createSparkPlan(sparkSession, planner, plan) - - override def isFinalAdaptivePlan(p: AdaptiveSparkPlanExec): Boolean = { - val args = p.argString(Int.MaxValue) - val index = args.indexOf("isFinalPlan=") - assert(index >= 0) - args.substring(index + "isFinalPlan=".length).trim.toBoolean - } - - override def getShuffleBlockFetcherIterator(params: ShuffleBlockFetcherIteratorParams) - : GlutenShuffleBlockFetcherIteratorBase = { - new GlutenShuffleBlockFetcherIterator( - params.context, - params.shuffleClient, - params.blockManager, - params.mapOutputTracker, - params.blocksByAddress, - params.streamWrapper, - params.maxBytesInFlight, - params.maxReqsInFlight, - params.maxBlocksInFlightPerAddress, - params.maxReqSizeShuffleToMem, - params.maxAttemptsOnNettyOOM, - params.detectCorrupt, - params.detectCorruptUseExtraMemory, - params.checksumEnabled, - params.checksumAlgorithm, - params.shuffleMetrics, - params.doBatchFetch - ) - } -} diff --git a/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/SparkShimProvider.scala b/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/SparkShimProvider.scala deleted file mode 100644 index 8a60cf06ce0..00000000000 --- a/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/SparkShimProvider.scala +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.sql.shims.spark33 - -import org.apache.gluten.sql.shims.SparkShims - -class SparkShimProvider extends org.apache.gluten.sql.shims.SparkShimProvider { - def createShim: SparkShims = { - new Spark33Shims() - } -} diff --git a/shims/spark33/src/main/scala/org/apache/gluten/utils/InternalRowUtil.scala b/shims/spark33/src/main/scala/org/apache/gluten/utils/InternalRowUtil.scala deleted file mode 100644 index 7c9ffc70cab..00000000000 --- a/shims/spark33/src/main/scala/org/apache/gluten/utils/InternalRowUtil.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.utils - -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.encoders.RowEncoder -import org.apache.spark.sql.types.StructType - -object InternalRowUtil { - def toString(struct: StructType, rows: Iterator[InternalRow]): String = { - val encoder = RowEncoder(struct).resolveAndBind() - val deserializer = encoder.createDeserializer() - rows.map(deserializer).mkString(System.lineSeparator()) - } - - def toString(struct: StructType, rows: Iterator[InternalRow], start: Int, length: Int): String = { - toString(struct, rows.slice(start, start + length)) - } - -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/ShuffleUtils.scala b/shims/spark33/src/main/scala/org/apache/spark/ShuffleUtils.scala deleted file mode 100644 index d2b58e67cd7..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/ShuffleUtils.scala +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark - -import org.apache.spark.shuffle.{BaseShuffleHandle, ShuffleHandle} -import org.apache.spark.storage.{BlockId, BlockManagerId} - -object ShuffleUtils { - def getReaderParam[K, C]( - handle: ShuffleHandle, - startMapIndex: Int, - endMapIndex: Int, - startPartition: Int, - endPartition: Int): Tuple2[Iterator[(BlockManagerId, Seq[(BlockId, Long, Int)])], Boolean] = { - val baseShuffleHandle = handle.asInstanceOf[BaseShuffleHandle[K, _, C]] - if (baseShuffleHandle.dependency.isShuffleMergeFinalizedMarked) { - val res = SparkEnv.get.mapOutputTracker.getPushBasedShuffleMapSizesByExecutorId( - handle.shuffleId, - startMapIndex, - endMapIndex, - startPartition, - endPartition) - (res.iter, res.enableBatchFetch) - } else { - val address = SparkEnv.get.mapOutputTracker.getMapSizesByExecutorId( - handle.shuffleId, - startMapIndex, - endMapIndex, - startPartition, - endPartition) - (address, true) - } - } -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/SparkContextUtils.scala b/shims/spark33/src/main/scala/org/apache/spark/SparkContextUtils.scala deleted file mode 100644 index 4e9e308dca7..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/SparkContextUtils.scala +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark - -import org.apache.spark.rdd.RDD -import org.apache.spark.sql.vectorized.ColumnarBatch - -object SparkContextUtils { - def createPartitioningAwareUnionRDD( - sc: SparkContext, - rdds: Seq[RDD[ColumnarBatch]], - numPartitions: Int): RDD[ColumnarBatch] = { - throw new UnsupportedOperationException( - "SQLPartitioningAwareUnionRDD is only available in Spark 4.1+") - } -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/shuffle/GlutenMapStatusUtil.scala b/shims/spark33/src/main/scala/org/apache/spark/shuffle/GlutenMapStatusUtil.scala deleted file mode 100644 index 9b5000946a2..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/shuffle/GlutenMapStatusUtil.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.shuffle - -import org.apache.spark.scheduler.MapStatus -import org.apache.spark.storage.BlockManagerId - -object GlutenMapStatusUtil { - def isRowBasedChecksumEnabled: Boolean = false - - def createMapStatus( - loc: BlockManagerId, - uncompressedSizes: Array[Long], - mapTaskId: Long, - checksumValue: Long): MapStatus = { - MapStatus(loc, uncompressedSizes, mapTaskId) - } -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/shuffle/SparkSortShuffleWriterUtil.scala b/shims/spark33/src/main/scala/org/apache/spark/shuffle/SparkSortShuffleWriterUtil.scala deleted file mode 100644 index 9e684c2afdd..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/shuffle/SparkSortShuffleWriterUtil.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.shuffle - -import org.apache.spark.TaskContext -import org.apache.spark.shuffle.api.ShuffleExecutorComponents -import org.apache.spark.shuffle.sort.SortShuffleWriter - -object SparkSortShuffleWriterUtil { - def create[K, V, C]( - handle: BaseShuffleHandle[K, V, C], - mapId: Long, - context: TaskContext, - writeMetrics: ShuffleWriteMetricsReporter, - shuffleExecutorComponents: ShuffleExecutorComponents): ShuffleWriter[K, V] = { - new SortShuffleWriter(handle, mapId, context, shuffleExecutorComponents) - } -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/catalyst/expressions/EvalMode.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/catalyst/expressions/EvalMode.scala deleted file mode 100644 index 0a3c63ccd8b..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/catalyst/expressions/EvalMode.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.internal.SQLConf - -/** For compatibility with Spark version <= 3.3. The class was added in vanilla Spark since 3.4. */ -object EvalMode extends Enumeration { - val LEGACY, ANSI, TRY = Value - - def fromSQLConf(conf: SQLConf): Value = if (conf.ansiEnabled) { - ANSI - } else { - LEGACY - } - - def fromBoolean(ansiEnabled: Boolean): Value = if (ansiEnabled) { - ANSI - } else { - LEGACY - } -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpressionsEvaluator.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpressionsEvaluator.scala deleted file mode 100644 index 1469f57e64d..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpressionsEvaluator.scala +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions - -import org.apache.spark.sql.internal.SQLConf - -// A helper class to evaluate expressions. -trait ExpressionsEvaluator { - protected lazy val runtime = - new SubExprEvaluationRuntime(SQLConf.get.subexpressionEliminationCacheMaxEntries) - - protected def prepareExpressions( - exprs: Seq[Expression], - subExprEliminationEnabled: Boolean): Seq[Expression] = { - // We need to make sure that we do not reuse stateful expressions. - // Different with Spark 3.4 above, without cleanedExpression for stateful expression. - if (subExprEliminationEnabled) { - runtime.proxyExpressions(exprs) - } else { - exprs - } - } - - /** - * Initializes internal states given the current partition index. This is used by nondeterministic - * expressions to set initial states. The default implementation does nothing. - */ - def initialize(partitionIndex: Int): Unit = {} - - protected def initializeExprs(exprs: Seq[Expression], partitionIndex: Int): Unit = { - exprs.foreach(_.foreach { - case n: Nondeterministic => n.initialize(partitionIndex) - case _ => - }) - } -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/InvokeExtractors.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/InvokeExtractors.scala deleted file mode 100644 index 95372c082cb..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/InvokeExtractors.scala +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.expressions.objects - -import org.apache.spark.sql.catalyst.expressions.Expression - -/** - * Extractors for Invoke expressions to ensure compatibility across different Spark versions. - * - * For Spark 3.3, StructsToJson is not replaced with Invoke expressions, so this extractor returns - * None to maintain API compatibility with other versions. - */ -object StructsToJsonInvoke { - def unapply(expr: Expression): Option[(Map[String, String], Expression, Option[String])] = { - None - } -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/catalyst/optimizer/CollapseProjectShim.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/catalyst/optimizer/CollapseProjectShim.scala deleted file mode 100644 index 1df1456f401..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/catalyst/optimizer/CollapseProjectShim.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.optimizer - -import org.apache.spark.sql.catalyst.expressions.{Expression, NamedExpression} - -object CollapseProjectShim { - def canCollapseExpressions( - consumers: Seq[Expression], - producers: Seq[NamedExpression], - alwaysInline: Boolean): Boolean = { - CollapseProject.canCollapseExpressions(consumers, producers, alwaysInline) - } - - def buildCleanedProjectList( - upper: Seq[NamedExpression], - lower: Seq[NamedExpression]): Seq[NamedExpression] = { - CollapseProject.buildCleanedProjectList(upper, lower) - } -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/catalyst/types/DataTypeUtils.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/catalyst/types/DataTypeUtils.scala deleted file mode 100644 index 597b5936f2d..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/catalyst/types/DataTypeUtils.scala +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.catalyst.types - -import org.apache.spark.sql.types.DataType - -object DataTypeUtils { - - /** - * Check if `this` and `other` are the same data type when ignoring nullability - * (`StructField.nullable`, `ArrayType.containsNull`, and `MapType.valueContainsNull`). - */ - def sameType(left: DataType, right: DataType): Boolean = left.sameType(right) -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/classic/ClassicColumn.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/classic/ClassicColumn.scala deleted file mode 100644 index bc0fbfcfc3f..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/classic/ClassicColumn.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.classic - -import org.apache.spark.sql.Column -import org.apache.spark.sql.catalyst.expressions.Expression - -/** Ensures compatibility with Spark 4.0. */ -object ClassicColumn { - def apply(e: Expression): Column = { - Column(e) - } -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/classic/ClassicDataset.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/classic/ClassicDataset.scala deleted file mode 100644 index 40b1ab3c543..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/classic/ClassicDataset.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.classic - -import org.apache.spark.sql.{DataFrame, Dataset, SparkSession} -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan - -/** Since Spark 4.0, the method ofRows cannot be invoked directly from sql.Dataset. */ -object ClassicDataset { - def ofRows(sparkSession: SparkSession, logicalPlan: LogicalPlan): DataFrame = { - Dataset.ofRows(sparkSession, logicalPlan) - } -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/classic/ClassicTypes.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/classic/ClassicTypes.scala deleted file mode 100644 index 7235c0b9b9d..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/classic/ClassicTypes.scala +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.classic - -import org.apache.spark.sql - -/** - * Prior to Spark 4.0, `ClassicSparkSession` refers to `sql.SparkSession`. Since Spark 4.0, it - * refers to `sql.classic.SparkSession`. - */ -object ClassicTypes { - - type ClassicSparkSession = sql.SparkSession -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/classic/conversions.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/classic/conversions.scala deleted file mode 100644 index 5a54519121c..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/classic/conversions.scala +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.classic - -import org.apache.spark.sql.Column -import org.apache.spark.sql.SparkSession - -/** - * Just to ensure the code below works for Spark versions earlier than 4.0. - * - * import org.apache.spark.sql.classic.ClassicConversions._ - */ -trait ClassicConversions { - - implicit class ColumnConstructorExt(val c: Column.type) {} -} - -object ClassicConversions extends ClassicConversions - -/** - * Just to ensure the code below works for Spark versions earlier than 4.0. - * - * import org.apache.spark.sql.classic.ExtendedClassicConversions._ - */ -object ExtendedClassicConversions { - - implicit class RichSqlSparkSession(sqlSparkSession: SparkSession.type) {} -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala deleted file mode 100644 index 0c43289ab8a..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/AbstractFileSourceScanExec.scala +++ /dev/null @@ -1,614 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution - -import org.apache.spark.rdd.RDD -import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier} -import org.apache.spark.sql.catalyst.catalog.BucketSpec -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, Partitioning, UnknownPartitioning} -import org.apache.spark.sql.errors.QueryExecutionErrors -import org.apache.spark.sql.execution.datasources._ -import org.apache.spark.sql.execution.datasources.parquet.{ParquetFileFormat => ParquetSource} -import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} -import org.apache.spark.sql.execution.vectorized.ConstantColumnVector -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.vectorized.ColumnarBatch -import org.apache.spark.util.Utils -import org.apache.spark.util.collection.BitSet - -import org.apache.hadoop.fs.Path - -import java.util.concurrent.TimeUnit._ - -import scala.collection.mutable.HashMap - -/** - * Physical plan node for scanning data from HadoopFsRelations. - * - * @param relation - * The file-based relation to scan. - * @param output - * Output attributes of the scan, including data attributes and partition attributes. - * @param requiredSchema - * Required schema of the underlying relation, excluding partition columns. - * @param partitionFilters - * Predicates to use for partition pruning. - * @param optionalBucketSet - * Bucket ids for bucket pruning. - * @param optionalNumCoalescedBuckets - * Number of coalesced buckets. - * @param dataFilters - * Filters on non-partition columns. - * @param tableIdentifier - * Identifier for the table in the metastore. - * @param disableBucketedScan - * Disable bucketed scan based on physical query plan, see rule [[DisableUnnecessaryBucketedScan]] - * for details. - */ -abstract class AbstractFileSourceScanExec( - @transient override val relation: HadoopFsRelation, - override val output: Seq[Attribute], - requiredSchema: StructType, - partitionFilters: Seq[Expression], - optionalBucketSet: Option[BitSet], - optionalNumCoalescedBuckets: Option[Int], - dataFilters: Seq[Expression], - override val tableIdentifier: Option[TableIdentifier], - disableBucketedScan: Boolean = false) - extends DataSourceScanExec { - - lazy val metadataColumns: Seq[AttributeReference] = - output.collect { case FileSourceMetadataAttribute(attr) => attr } - - override def supportsColumnar: Boolean = { - // The value should be defined in GlutenPlan. - throw new UnsupportedOperationException( - "Unreachable code from org.apache.spark.sql.execution.AbstractFileSourceScanExec" + - ".supportsColumnar") - } - - private lazy val needsUnsafeRowConversion: Boolean = { - if (relation.fileFormat.isInstanceOf[ParquetSource]) { - conf.parquetVectorizedReaderEnabled - } else { - false - } - } - - override def vectorTypes: Option[Seq[String]] = - relation.fileFormat - .vectorTypes( - requiredSchema = requiredSchema, - partitionSchema = relation.partitionSchema, - relation.sparkSession.sessionState.conf) - .map { - vectorTypes => - // for column-based file format, append metadata column's vector type classes if any - vectorTypes ++ Seq.fill(metadataColumns.size)(classOf[ConstantColumnVector].getName) - } - - private lazy val driverMetrics: HashMap[String, Long] = HashMap.empty - - /** - * Send the driver-side metrics. Before calling this function, selectedPartitions has been - * initialized. See SPARK-26327 for more details. - */ - private def sendDriverMetrics(): Unit = { - driverMetrics.foreach(e => metrics(e._1).add(e._2)) - val executionId = sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) - SQLMetrics.postDriverMetricUpdates( - sparkContext, - executionId, - metrics.filter(e => driverMetrics.contains(e._1)).values.toSeq) - } - - private def isDynamicPruningFilter(e: Expression): Boolean = - e.exists(_.isInstanceOf[PlanExpression[_]]) - - @transient lazy val selectedPartitions: Array[PartitionDirectory] = { - val optimizerMetadataTimeNs = relation.location.metadataOpsTimeNs.getOrElse(0L) - val startTime = System.nanoTime() - val ret = - relation.location.listFiles(partitionFilters.filterNot(isDynamicPruningFilter), dataFilters) - setFilesNumAndSizeMetric(ret, true) - val timeTakenMs = - NANOSECONDS.toMillis((System.nanoTime() - startTime) + optimizerMetadataTimeNs) - driverMetrics("metadataTime") = timeTakenMs - ret - }.toArray - - // We can only determine the actual partitions at runtime when a dynamic partition filter is - // present. This is because such a filter relies on information that is only available at run - // time (for instance the keys used in the other side of a join). - @transient private lazy val dynamicallySelectedPartitions: Array[PartitionDirectory] = { - val dynamicPartitionFilters = partitionFilters.filter(isDynamicPruningFilter) - - if (dynamicPartitionFilters.nonEmpty) { - val startTime = System.nanoTime() - // call the file index for the files matching all filters except dynamic partition filters - val predicate = dynamicPartitionFilters.reduce(And) - val partitionColumns = relation.partitionSchema - val boundPredicate = Predicate.create( - predicate.transform { - case a: AttributeReference => - val index = partitionColumns.indexWhere(a.name == _.name) - BoundReference(index, partitionColumns(index).dataType, nullable = true) - }, - Nil - ) - val ret = selectedPartitions.filter(p => boundPredicate.eval(p.values)) - setFilesNumAndSizeMetric(ret, false) - val timeTakenMs = (System.nanoTime() - startTime) / 1000 / 1000 - driverMetrics("pruningTime") = timeTakenMs - ret - } else { - selectedPartitions - } - } - - /** - * [[partitionFilters]] can contain subqueries whose results are available only at runtime so - * accessing [[selectedPartitions]] should be guarded by this method during planning - */ - private def hasPartitionsAvailableAtRunTime: Boolean = { - partitionFilters.exists(ExecSubqueryExpression.hasSubquery) - } - - private def toAttribute(colName: String): Option[Attribute] = - output.find(_.name == colName) - - // exposed for testing - lazy val bucketedScan: Boolean = { - if ( - relation.sparkSession.sessionState.conf.bucketingEnabled && relation.bucketSpec.isDefined - && !disableBucketedScan - ) { - val spec = relation.bucketSpec.get - val bucketColumns = spec.bucketColumnNames.flatMap(n => toAttribute(n)) - bucketColumns.size == spec.bucketColumnNames.size - } else { - false - } - } - - override lazy val (outputPartitioning, outputOrdering): (Partitioning, Seq[SortOrder]) = { - if (bucketedScan) { - // For bucketed columns: - // ----------------------- - // `HashPartitioning` would be used only when: - // 1. ALL the bucketing columns are being read from the table - // - // For sorted columns: - // --------------------- - // Sort ordering should be used when ALL these criteria's match: - // 1. `HashPartitioning` is being used - // 2. A prefix (or all) of the sort columns are being read from the table. - // - // Sort ordering would be over the prefix subset of `sort columns` being read - // from the table. - // e.g. - // Assume (col0, col2, col3) are the columns read from the table - // If sort columns are (col0, col1), then sort ordering would be considered as (col0) - // If sort columns are (col1, col0), then sort ordering would be empty as per rule #2 - // above - val spec = relation.bucketSpec.get - val bucketColumns = spec.bucketColumnNames.flatMap(n => toAttribute(n)) - val numPartitions = optionalNumCoalescedBuckets.getOrElse(spec.numBuckets) - val partitioning = HashPartitioning(bucketColumns, numPartitions) - val sortColumns = - spec.sortColumnNames.map(x => toAttribute(x)).takeWhile(x => x.isDefined).map(_.get) - val shouldCalculateSortOrder = - conf.getConf(SQLConf.LEGACY_BUCKETED_TABLE_SCAN_OUTPUT_ORDERING) && - sortColumns.nonEmpty && - !hasPartitionsAvailableAtRunTime - - val sortOrder = if (shouldCalculateSortOrder) { - // In case of bucketing, its possible to have multiple files belonging to the - // same bucket in a given relation. Each of these files are locally sorted - // but those files combined together are not globally sorted. Given that, - // the RDD partition will not be sorted even if the relation has sort columns set - // Current solution is to check if all the buckets have a single file in it - - val files = selectedPartitions.flatMap(partition => partition.files) - val bucketToFilesGrouping = - files.map(_.getPath.getName).groupBy(file => BucketingUtils.getBucketId(file)) - val singleFilePartitions = bucketToFilesGrouping.forall(p => p._2.length <= 1) - - // TODO SPARK-24528 Sort order is currently ignored if buckets are coalesced. - if (singleFilePartitions && optionalNumCoalescedBuckets.isEmpty) { - // TODO Currently Spark does not support writing columns sorting in descending order - // so using Ascending order. This can be fixed in future - sortColumns.map(attribute => SortOrder(attribute, Ascending)) - } else { - Nil - } - } else { - Nil - } - (partitioning, sortOrder) - } else { - (UnknownPartitioning(0), Nil) - } - } - - @transient - private lazy val pushedDownFilters = { - val supportNestedPredicatePushdown = DataSourceUtils.supportNestedPredicatePushdown(relation) - // `dataFilters` should not include any metadata col filters - // because the metadata struct has been flatted in FileSourceStrategy - // and thus metadata col filters are invalid to be pushed down - dataFilters - .filterNot(_.references.exists { - case FileSourceMetadataAttribute(_) => true - case _ => false - }) - .flatMap(DataSourceStrategy.translateFilter(_, supportNestedPredicatePushdown)) - } - - override lazy val metadata: Map[String, String] = { - def seqToString(seq: Seq[Any]) = seq.mkString("[", ", ", "]") - val location = relation.location - val locationDesc = - location.getClass.getSimpleName + - Utils.buildLocationMetadata(location.rootPaths, maxMetadataValueLength) - val metadata = - Map( - "Format" -> relation.fileFormat.toString, - "ReadSchema" -> requiredSchema.catalogString, - "Batched" -> supportsColumnar.toString, - "PartitionFilters" -> seqToString(partitionFilters), - "PushedFilters" -> seqToString(pushedDownFilters), - "DataFilters" -> seqToString(dataFilters), - "Location" -> locationDesc - ) - - relation.bucketSpec - .map { - spec => - val bucketedKey = "Bucketed" - if (bucketedScan) { - val numSelectedBuckets = optionalBucketSet.map(b => b.cardinality()).getOrElse { - spec.numBuckets - } - metadata ++ Map( - bucketedKey -> "true", - "SelectedBucketsCount" -> (s"$numSelectedBuckets out of ${spec.numBuckets}" + - optionalNumCoalescedBuckets.map(b => s" (Coalesced to $b)").getOrElse("")) - ) - } else if (!relation.sparkSession.sessionState.conf.bucketingEnabled) { - metadata + (bucketedKey -> "false (disabled by configuration)") - } else if (disableBucketedScan) { - metadata + (bucketedKey -> "false (disabled by query planner)") - } else { - metadata + (bucketedKey -> "false (bucket column(s) not read)") - } - } - .getOrElse { - metadata - } - } - - override def verboseStringWithOperatorId(): String = { - val metadataStr = metadata.toSeq.sorted - .filterNot { - case (_, value) if (value.isEmpty || value.equals("[]")) => true - case (key, _) if (key.equals("DataFilters") || key.equals("Format")) => true - case (_, _) => false - } - .map { - case (key, _) if (key.equals("Location")) => - val location = relation.location - val numPaths = location.rootPaths.length - val abbreviatedLocation = if (numPaths <= 1) { - location.rootPaths.mkString("[", ", ", "]") - } else { - "[" + location.rootPaths.head + s", ... ${numPaths - 1} entries]" - } - s"$key: ${location.getClass.getSimpleName} ${redact(abbreviatedLocation)}" - case (key, value) => s"$key: ${redact(value)}" - } - - s""" - |$formattedNodeName - |${ExplainUtils.generateFieldString("Output", output)} - |${metadataStr.mkString("\n")} - |""".stripMargin - } - - lazy val inputRDD: RDD[InternalRow] = { - val readFile: (PartitionedFile) => Iterator[InternalRow] = - relation.fileFormat.buildReaderWithPartitionValues( - sparkSession = relation.sparkSession, - dataSchema = relation.dataSchema, - partitionSchema = relation.partitionSchema, - requiredSchema = requiredSchema, - filters = pushedDownFilters, - options = relation.options, - hadoopConf = relation.sparkSession.sessionState.newHadoopConfWithOptions(relation.options) - ) - - val readRDD = if (bucketedScan) { - createBucketedReadRDD( - relation.bucketSpec.get, - readFile, - dynamicallySelectedPartitions, - relation) - } else { - createReadRDD(readFile, dynamicallySelectedPartitions, relation) - } - sendDriverMetrics() - readRDD - } - - override def inputRDDs(): Seq[RDD[InternalRow]] = { - inputRDD :: Nil - } - - /** SQL metrics generated only for scans using dynamic partition pruning. */ - private lazy val staticMetrics = if (partitionFilters.exists(isDynamicPruningFilter)) { - Map( - "staticFilesNum" -> SQLMetrics.createMetric(sparkContext, "static number of files read"), - "staticFilesSize" -> SQLMetrics.createSizeMetric(sparkContext, "static size of files read") - ) - } else { - Map.empty[String, SQLMetric] - } - - /** Helper for computing total number and size of files in selected partitions. */ - private def setFilesNumAndSizeMetric( - partitions: Seq[PartitionDirectory], - static: Boolean): Unit = { - val filesNum = partitions.map(_.files.size.toLong).sum - val filesSize = partitions.map(_.files.map(_.getLen).sum).sum - if (!static || !partitionFilters.exists(isDynamicPruningFilter)) { - driverMetrics("numFiles") = filesNum - driverMetrics("filesSize") = filesSize - } else { - driverMetrics("staticFilesNum") = filesNum - driverMetrics("staticFilesSize") = filesSize - } - if (relation.partitionSchema.nonEmpty) { - driverMetrics("numPartitions") = partitions.length - } - } - - override lazy val metrics = Map( - "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"), - "numFiles" -> SQLMetrics.createMetric(sparkContext, "number of files read"), - "metadataTime" -> SQLMetrics.createTimingMetric(sparkContext, "metadata time"), - "filesSize" -> SQLMetrics.createSizeMetric(sparkContext, "size of files read") - ) ++ { - // Tracking scan time has overhead, we can't afford to do it for each row, and can only do - // it for each batch. - if (supportsColumnar) { - Some("scanTime" -> SQLMetrics.createTimingMetric(sparkContext, "scan time")) - } else { - None - } - } ++ { - if (relation.partitionSchema.nonEmpty) { - Map( - "numPartitions" -> SQLMetrics.createMetric(sparkContext, "number of partitions read"), - "pruningTime" -> - SQLMetrics.createTimingMetric(sparkContext, "dynamic partition pruning time") - ) - } else { - Map.empty[String, SQLMetric] - } - } ++ staticMetrics - - override protected def doExecute(): RDD[InternalRow] = { - val numOutputRows = longMetric("numOutputRows") - if (needsUnsafeRowConversion) { - inputRDD.mapPartitionsWithIndexInternal { - (index, iter) => - val toUnsafe = UnsafeProjection.create(schema) - toUnsafe.initialize(index) - iter.map { - row => - numOutputRows += 1 - toUnsafe(row) - } - } - } else { - inputRDD.mapPartitionsInternal { - iter => - iter.map { - row => - numOutputRows += 1 - row - } - } - } - } - - override protected def doExecuteColumnar(): RDD[ColumnarBatch] = { - val numOutputRows = longMetric("numOutputRows") - val scanTime = longMetric("scanTime") - inputRDD.asInstanceOf[RDD[ColumnarBatch]].mapPartitionsInternal { - batches => - new Iterator[ColumnarBatch] { - - override def hasNext: Boolean = { - // The `FileScanRDD` returns an iterator which scans the file during the `hasNext` call. - val startNs = System.nanoTime() - val res = batches.hasNext - scanTime += NANOSECONDS.toMillis(System.nanoTime() - startNs) - res - } - - override def next(): ColumnarBatch = { - val batch = batches.next() - numOutputRows += batch.numRows() - batch - } - } - } - } - - override val nodeNamePrefix: String = "File" - - /** - * Create an RDD for bucketed reads. The non-bucketed variant of this function is - * [[createReadRDD]]. - * - * The algorithm is pretty simple: each RDD partition being returned should include all the files - * with the same bucket id from all the given Hive partitions. - * - * @param bucketSpec - * the bucketing spec. - * @param readFile - * a function to read each (part of a) file. - * @param selectedPartitions - * Hive-style partition that are part of the read. - * @param fsRelation - * [[HadoopFsRelation]] associated with the read. - */ - private def createBucketedReadRDD( - bucketSpec: BucketSpec, - readFile: (PartitionedFile) => Iterator[InternalRow], - selectedPartitions: Array[PartitionDirectory], - fsRelation: HadoopFsRelation): RDD[InternalRow] = { - logInfo(s"Planning with ${bucketSpec.numBuckets} buckets") - val filesGroupedToBuckets = - selectedPartitions - .flatMap { - p => p.files.map(f => PartitionedFileUtil.getPartitionedFile(f, f.getPath, p.values)) - } - .groupBy { - f => - BucketingUtils - .getBucketId(new Path(f.filePath).getName) - .getOrElse(throw QueryExecutionErrors.invalidBucketFile(f.filePath)) - } - - val prunedFilesGroupedToBuckets = if (optionalBucketSet.isDefined) { - val bucketSet = optionalBucketSet.get - filesGroupedToBuckets.filter(f => bucketSet.get(f._1)) - } else { - filesGroupedToBuckets - } - - val filePartitions = optionalNumCoalescedBuckets - .map { - numCoalescedBuckets => - logInfo(s"Coalescing to $numCoalescedBuckets buckets") - val coalescedBuckets = prunedFilesGroupedToBuckets.groupBy(_._1 % numCoalescedBuckets) - Seq.tabulate(numCoalescedBuckets) { - bucketId => - val partitionedFiles = coalescedBuckets - .get(bucketId) - .map { - _.values.flatten.toArray - } - .getOrElse(Array.empty) - FilePartition(bucketId, partitionedFiles) - } - } - .getOrElse { - Seq.tabulate(bucketSpec.numBuckets) { - bucketId => - FilePartition(bucketId, prunedFilesGroupedToBuckets.getOrElse(bucketId, Array.empty)) - } - } - - new FileScanRDD( - fsRelation.sparkSession, - readFile, - filePartitions, - new StructType(requiredSchema.fields ++ fsRelation.partitionSchema.fields), - metadataColumns) - } - - /** - * Create an RDD for non-bucketed reads. The bucketed variant of this function is - * [[createBucketedReadRDD]]. - * - * @param readFile - * a function to read each (part of a) file. - * @param selectedPartitions - * Hive-style partition that are part of the read. - * @param fsRelation - * [[HadoopFsRelation]] associated with the read. - */ - private def createReadRDD( - readFile: (PartitionedFile) => Iterator[InternalRow], - selectedPartitions: Array[PartitionDirectory], - fsRelation: HadoopFsRelation): RDD[InternalRow] = { - val openCostInBytes = fsRelation.sparkSession.sessionState.conf.filesOpenCostInBytes - val maxSplitBytes = - FilePartition.maxSplitBytes(fsRelation.sparkSession, selectedPartitions) - logInfo( - s"Planning scan with bin packing, max size: $maxSplitBytes bytes, " + - s"open cost is considered as scanning $openCostInBytes bytes.") - - // Filter files with bucket pruning if possible - val bucketingEnabled = fsRelation.sparkSession.sessionState.conf.bucketingEnabled - val shouldProcess: Path => Boolean = optionalBucketSet match { - case Some(bucketSet) if bucketingEnabled => - // Do not prune the file if bucket file name is invalid - filePath => BucketingUtils.getBucketId(filePath.getName).forall(bucketSet.get) - case _ => - _ => true - } - - val splitFiles = selectedPartitions - .flatMap { - partition => - partition.files.flatMap { - file => - // getPath() is very expensive so we only want to call it once in this block: - val filePath = file.getPath - - if (shouldProcess(filePath)) { - val isSplitable = - relation.fileFormat.isSplitable(relation.sparkSession, relation.options, filePath) - PartitionedFileUtil.splitFiles( - sparkSession = relation.sparkSession, - file = file, - filePath = filePath, - isSplitable = isSplitable, - maxSplitBytes = maxSplitBytes, - partitionValues = partition.values - ) - } else { - Seq.empty - } - } - } - .sortBy(_.length)(implicitly[Ordering[Long]].reverse) - - val partitions = - FilePartition.getFilePartitions(relation.sparkSession, splitFiles, maxSplitBytes) - - new FileScanRDD( - fsRelation.sparkSession, - readFile, - partitions, - new StructType(requiredSchema.fields ++ fsRelation.partitionSchema.fields), - metadataColumns) - } - - // Filters unused DynamicPruningExpression expressions - one which has been replaced - // with DynamicPruningExpression(Literal.TrueLiteral) during Physical Planning - protected def filterUnusedDynamicPruningExpressions( - predicates: Seq[Expression]): Seq[Expression] = { - predicates.filterNot(_ == DynamicPruningExpression(Literal.TrueLiteral)) - } -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/ExpandOutputPartitioningShim.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/ExpandOutputPartitioningShim.scala deleted file mode 100644 index 7dbad48dafa..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/ExpandOutputPartitioningShim.scala +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution - -import org.apache.spark.sql.catalyst.expressions.Expression -import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, Partitioning, PartitioningCollection} - -import scala.collection.mutable - -// https://issues.apache.org/jira/browse/SPARK-31869 -class ExpandOutputPartitioningShim( - streamedKeyExprs: Seq[Expression], - buildKeyExprs: Seq[Expression], - expandLimit: Int) { - // An one-to-many mapping from a streamed key to build keys. - private lazy val streamedKeyToBuildKeyMapping = { - val mapping = mutable.Map.empty[Expression, Seq[Expression]] - streamedKeyExprs.zip(buildKeyExprs).foreach { - case (streamedKey, buildKey) => - val key = streamedKey.canonicalized - mapping.get(key) match { - case Some(v) => mapping.put(key, v :+ buildKey) - case None => mapping.put(key, Seq(buildKey)) - } - } - mapping.toMap - } - - def expandPartitioning(partitioning: Partitioning): Partitioning = { - partitioning match { - case h: HashPartitioning => expandOutputPartitioning(h) - case c: PartitioningCollection => expandOutputPartitioning(c) - case _ => partitioning - } - } - - // Expands the given partitioning collection recursively. - private def expandOutputPartitioning( - partitioning: PartitioningCollection): PartitioningCollection = { - PartitioningCollection(partitioning.partitionings.flatMap { - case h: HashPartitioning => expandOutputPartitioning(h).partitionings - case c: PartitioningCollection => Seq(expandOutputPartitioning(c)) - case other => Seq(other) - }) - } - - // Expands the given hash partitioning by substituting streamed keys with build keys. - // For example, if the expressions for the given partitioning are Seq("a", "b", "c") - // where the streamed keys are Seq("b", "c") and the build keys are Seq("x", "y"), - // the expanded partitioning will have the following expressions: - // Seq("a", "b", "c"), Seq("a", "b", "y"), Seq("a", "x", "c"), Seq("a", "x", "y"). - // The expanded expressions are returned as PartitioningCollection. - private def expandOutputPartitioning(partitioning: HashPartitioning): PartitioningCollection = { - val maxNumCombinations = expandLimit - var currentNumCombinations = 0 - - def generateExprCombinations( - current: Seq[Expression], - accumulated: Seq[Expression]): Seq[Seq[Expression]] = { - if (currentNumCombinations >= maxNumCombinations) { - Nil - } else if (current.isEmpty) { - currentNumCombinations += 1 - Seq(accumulated) - } else { - val buildKeysOpt = streamedKeyToBuildKeyMapping.get(current.head.canonicalized) - generateExprCombinations(current.tail, accumulated :+ current.head) ++ - buildKeysOpt - .map(_.flatMap(b => generateExprCombinations(current.tail, accumulated :+ b))) - .getOrElse(Nil) - } - } - - PartitioningCollection( - generateExprCombinations(partitioning.expressions, Nil) - .map(exprs => partitioning.withNewChildren(exprs).asInstanceOf[HashPartitioning])) - } -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala deleted file mode 100644 index 240b377d453..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution - -import org.apache.gluten.metrics.GlutenTimeMetric - -import org.apache.spark.Partition -import org.apache.spark.rdd.RDD -import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier} -import org.apache.spark.sql.catalyst.expressions.{And, Attribute, AttributeReference, BoundReference, Expression, FileSourceMetadataAttribute, PlanExpression, Predicate} -import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, PartitionDirectory} -import org.apache.spark.sql.execution.datasources.parquet.ParquetUtils -import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} -import org.apache.spark.sql.types.StructType -import org.apache.spark.util.collection.BitSet - -import java.util.concurrent.TimeUnit.NANOSECONDS - -import scala.collection.mutable - -abstract class FileSourceScanExecShim( - @transient override val relation: HadoopFsRelation, - override val output: Seq[Attribute], - val requiredSchema: StructType, - val partitionFilters: Seq[Expression], - val optionalBucketSet: Option[BitSet], - val optionalNumCoalescedBuckets: Option[Int], - val dataFilters: Seq[Expression], - override val tableIdentifier: Option[TableIdentifier], - val disableBucketedScan: Boolean = false) - extends AbstractFileSourceScanExec( - relation, - output, - requiredSchema, - partitionFilters, - optionalBucketSet, - optionalNumCoalescedBuckets, - dataFilters, - tableIdentifier, - disableBucketedScan) { - - // Note: "metrics" is made transient to avoid sending driver-side metrics to tasks. - @transient override lazy val metrics: Map[String, SQLMetric] = Map() - - def dataFiltersInScan: Seq[Expression] = dataFilters.filterNot(_.references.exists { - case FileSourceMetadataAttribute(attr) if attr.name == "_metadata" => true - case _ => false - }) - - def hasUnsupportedColumns: Boolean = { - // TODO, fallback if user define same name column due to we can't right now - // detect which column is metadata column which is user defined column. - val metadataColumnsNames = metadataColumns.map(_.name) - output - .filterNot(metadataColumns.toSet) - .exists(v => metadataColumnsNames.contains(v.name)) - } - - def isMetadataColumn(attr: Attribute): Boolean = metadataColumns.contains(attr) - - def hasFieldIds: Boolean = ParquetUtils.hasFieldIds(requiredSchema) - - // The codes below are copied from FileSourceScanExec in Spark, - // all of them are private. - protected lazy val driverMetrics: mutable.HashMap[String, Long] = mutable.HashMap.empty - - protected lazy val driverMetricsAlias = { - if (partitionFilters.exists(isDynamicPruningFilter)) { - Map( - "staticFilesNum" -> SQLMetrics.createMetric(sparkContext, "static number of files read"), - "staticFilesSize" -> SQLMetrics.createSizeMetric(sparkContext, "static size of files read") - ) - } else { - Map.empty[String, SQLMetric] - } - } - - /** - * Send the driver-side metrics. Before calling this function, selectedPartitions has been - * initialized. See SPARK-26327 for more details. - */ - protected def sendDriverMetrics(): Unit = { - driverMetrics.foreach(e => metrics(e._1).add(e._2)) - val executionId = sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) - SQLMetrics.postDriverMetricUpdates( - sparkContext, - executionId, - metrics.filter(e => driverMetrics.contains(e._1)).values.toSeq) - } - - protected def setFilesNumAndSizeMetric( - partitions: Seq[PartitionDirectory], - static: Boolean): Unit = { - val filesNum = partitions.map(_.files.size.toLong).sum - val filesSize = partitions.map(_.files.map(_.getLen).sum).sum - if (!static || !partitionFilters.exists(isDynamicPruningFilter)) { - driverMetrics("numFiles") = filesNum - driverMetrics("filesSize") = filesSize - } else { - driverMetrics("staticFilesNum") = filesNum - driverMetrics("staticFilesSize") = filesSize - } - if (relation.partitionSchema.nonEmpty) { - driverMetrics("numPartitions") = partitions.length - } - } - - @transient override lazy val selectedPartitions: Array[PartitionDirectory] = { - val optimizerMetadataTimeNs = relation.location.metadataOpsTimeNs.getOrElse(0L) - GlutenTimeMetric.withNanoTime { - val ret = - relation.location.listFiles(partitionFilters.filterNot(isDynamicPruningFilter), dataFilters) - setFilesNumAndSizeMetric(ret, static = true) - ret - }(t => driverMetrics("metadataTime") = NANOSECONDS.toMillis(t + optimizerMetadataTimeNs)) - }.toArray - - protected def isDynamicPruningFilter(e: Expression): Boolean = - e.exists(_.isInstanceOf[PlanExpression[_]]) - - // We can only determine the actual partitions at runtime when a dynamic partition filter is - // present. This is because such a filter relies on information that is only available at run - // time (for instance the keys used in the other side of a join). - @transient private lazy val dynamicallySelectedPartitions: Array[PartitionDirectory] = { - val dynamicPartitionFilters = - partitionFilters.filter(isDynamicPruningFilter) - val selected = if (dynamicPartitionFilters.nonEmpty) { - GlutenTimeMetric.withMillisTime { - // call the file index for the files matching all filters except dynamic partition filters - val boundedFilters = dynamicPartitionFilters.map { - dynamicPartitionFilter => - dynamicPartitionFilter.transform { - case a: AttributeReference => - val index = relation.partitionSchema.indexWhere(a.name == _.name) - BoundReference(index, relation.partitionSchema(index).dataType, nullable = true) - } - } - val boundPredicate = Predicate.create(boundedFilters.reduce(And), Nil) - val ret = selectedPartitions.filter(p => boundPredicate.eval(p.values)) - setFilesNumAndSizeMetric(ret, static = false) - ret - }(t => driverMetrics("pruningTime") = t) - } else { - selectedPartitions - } - sendDriverMetrics() - selected - } - - def getPartitionArray: Array[PartitionDirectory] = { - dynamicallySelectedPartitions - } - - def getPartitionsSeq(): Seq[Partition] = { - Seq() - } -} - -abstract class ArrowFileSourceScanLikeShim(original: FileSourceScanExec) - extends DataSourceScanExec { - override val nodeNamePrefix: String = "ArrowFile" - - override lazy val metrics = original.metrics - - override def tableIdentifier: Option[TableIdentifier] = original.tableIdentifier - - override def inputRDDs(): Seq[RDD[InternalRow]] = original.inputRDDs() - - override def relation: HadoopFsRelation = original.relation - - override protected def metadata: Map[String, String] = original.metadata -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/PartitioningAndOrderingPreservingNodeShim.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/PartitioningAndOrderingPreservingNodeShim.scala deleted file mode 100644 index c2b1a8291fa..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/PartitioningAndOrderingPreservingNodeShim.scala +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution - -trait OrderPreservingNodeShim extends AliasAwareOutputOrdering -trait PartitioningPreservingNodeShim extends AliasAwareOutputPartitioning diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatDataWriter.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatDataWriter.scala deleted file mode 100644 index 799c1dad659..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatDataWriter.scala +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources - -import org.apache.gluten.execution.{BatchCarrierRow, PlaceholderRow, TerminalRow} -import org.apache.gluten.execution.datasource.GlutenFormatFactory - -import org.apache.spark.internal.io.FileCommitProtocol -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.execution.metric.SQLMetric - -import org.apache.hadoop.mapreduce.TaskAttemptContext - -/** - * Dynamic partition writer with single writer, meaning only one writer is opened at any time for - * writing. The records to be written are required to be sorted on partition and/or bucket column(s) - * before writing. - */ -class DynamicPartitionDataSingleWriter( - description: WriteJobDescription, - taskAttemptContext: TaskAttemptContext, - committer: FileCommitProtocol, - customMetrics: Map[String, SQLMetric] = Map.empty) - extends BaseDynamicPartitionDataWriter( - description, - taskAttemptContext, - committer, - customMetrics) { - - private var currentPartitionValues: Option[UnsafeRow] = None - private var currentBucketId: Option[Int] = None - - private val partitionColIndice: Array[Int] = - description.partitionColumns.flatMap { - pcol => - description.allColumns.zipWithIndex.collect { - case (acol, index) if acol.name == pcol.name && acol.exprId == pcol.exprId => index - } - }.toArray - - private def beforeWrite(record: InternalRow): Unit = { - val nextPartitionValues = if (isPartitioned) Some(getPartitionValues(record)) else None - val nextBucketId = if (isBucketed) Some(getBucketId(record)) else None - - if (currentPartitionValues != nextPartitionValues || currentBucketId != nextBucketId) { - // See a new partition or bucket - write to a new partition dir (or a new bucket file). - if (isPartitioned && currentPartitionValues != nextPartitionValues) { - currentPartitionValues = Some(nextPartitionValues.get.copy()) - statsTrackers.foreach(_.newPartition(currentPartitionValues.get)) - } - if (isBucketed) { - currentBucketId = nextBucketId - } - - fileCounter = 0 - renewCurrentWriter(currentPartitionValues, currentBucketId, closeCurrentWriter = true) - } else if ( - description.maxRecordsPerFile > 0 && - recordsInFile >= description.maxRecordsPerFile - ) { - renewCurrentWriterIfTooManyRecords(currentPartitionValues, currentBucketId) - } - } - - override def write(record: InternalRow): Unit = { - record match { - case carrierRow: BatchCarrierRow => - carrierRow match { - case placeholderRow: PlaceholderRow => - // Do nothing. - case terminalRow: TerminalRow => - val numRows = terminalRow.batch().numRows() - if (numRows > 0) { - val blockStripes = GlutenFormatFactory.rowSplitter - .splitBlockByPartitionAndBucket(terminalRow.batch(), partitionColIndice, isBucketed) - val iter = blockStripes.iterator() - while (iter.hasNext) { - val blockStripe = iter.next() - val headingRow = blockStripe.getHeadingRow - beforeWrite(headingRow) - val columnBatch = blockStripe.getColumnarBatch - currentWriter.write(terminalRow.withNewBatch(columnBatch)) - columnBatch.close() - } - blockStripes.release() - for (_ <- 0 until numRows) { - statsTrackers.foreach(_.newRow(currentWriter.path, record)) - } - recordsInFile += numRows - } - } - case _ => - beforeWrite(record) - writeRecord(record) - } - } -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatWriter.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatWriter.scala deleted file mode 100644 index 3e2eb6c4df4..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatWriter.scala +++ /dev/null @@ -1,452 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources - -import org.apache.gluten.execution.datasource.GlutenFormatFactory - -import org.apache.spark._ -import org.apache.spark.internal.Logging -import org.apache.spark.internal.io.{FileCommitProtocol, SparkHadoopWriterUtils} -import org.apache.spark.rdd.RDD -import org.apache.spark.shuffle.FetchFailedException -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.catalog.BucketSpec -import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.expressions.BindReferences.bindReferences -import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode} -import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning -import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DateTimeUtils} -import org.apache.spark.sql.errors.QueryExecutionErrors -import org.apache.spark.sql.execution._ -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.StringType -import org.apache.spark.unsafe.types.UTF8String -import org.apache.spark.util.{SerializableConfiguration, Utils} - -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.{FileAlreadyExistsException, Path} -import org.apache.hadoop.mapreduce._ -import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat -import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl - -import java.util.{Date, UUID} - -/** A helper object for writing FileFormat data out to a location. */ -object FileFormatWriter extends Logging { - - var executeWriterWrappedSparkPlan: SparkPlan => RDD[InternalRow] = _ - - /** Describes how output files should be placed in the filesystem. */ - case class OutputSpec( - outputPath: String, - customPartitionLocations: Map[TablePartitionSpec, String], - outputColumns: Seq[Attribute]) - - /** A function that converts the empty string to null for partition values. */ - case class Empty2Null(child: Expression) extends UnaryExpression with String2StringExpression { - override def convert(v: UTF8String): UTF8String = if (v.numBytes() == 0) null else v - override def nullable: Boolean = true - override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { - nullSafeCodeGen( - ctx, - ev, - c => { - s"""if ($c.numBytes() == 0) { - | ${ev.isNull} = true; - | ${ev.value} = null; - |} else { - | ${ev.value} = $c; - |}""".stripMargin - } - ) - } - - override protected def withNewChildInternal(newChild: Expression): Empty2Null = - copy(child = newChild) - } - - /** Describes how concurrent output writers should be executed. */ - case class ConcurrentOutputWriterSpec( - maxWriters: Int, - createSorter: () => UnsafeExternalRowSorter) - - /** - * Basic work flow of this command is: - * 1. Driver side setup, including output committer initialization and data source specific - * preparation work for the write job to be issued. 2. Issues a write job consists of one or - * more executor side tasks, each of which writes all rows within an RDD partition. 3. If no - * exception is thrown in a task, commits that task, otherwise aborts that task; If any - * exception is thrown during task commitment, also aborts that task. 4. If all tasks are - * committed, commit the job, otherwise aborts the job; If any exception is thrown during job - * commitment, also aborts the job. 5. If the job is successfully committed, perform - * post-commit operations such as processing statistics. - * @return - * The set of all partition paths that were updated during this write job. - */ - def write( - sparkSession: SparkSession, - plan: SparkPlan, - fileFormat: FileFormat, - committer: FileCommitProtocol, - outputSpec: OutputSpec, - hadoopConf: Configuration, - partitionColumns: Seq[Attribute], - bucketSpec: Option[BucketSpec], - statsTrackers: Seq[WriteJobStatsTracker], - options: Map[String, String]): Set[String] = { - - val nativeEnabled = - "true" == sparkSession.sparkContext.getLocalProperty("isNativeApplicable") - val numStaticPartitionCols = - Option(sparkSession.sparkContext.getLocalProperty("numStaticPartitionCols")) - .map(_.toInt) - .getOrElse(0) - - if (nativeEnabled) { - logInfo( - s"Writing data with Gluten's native writer. The topmost node of the query plan to " + - s"write is: ${plan.nodeName}") - assert(plan.isInstanceOf[ColumnarToRowTransition]) - } - - val job = Job.getInstance(hadoopConf) - job.setOutputKeyClass(classOf[Void]) - job.setOutputValueClass(classOf[InternalRow]) - FileOutputFormat.setOutputPath(job, new Path(outputSpec.outputPath)) - - val partitionSet = AttributeSet(partitionColumns) - // cleanup the internal metadata information of - // the file source metadata attribute if any before write out - val finalOutputSpec = outputSpec.copy(outputColumns = outputSpec.outputColumns - .map(FileSourceMetadataAttribute.cleanupFileSourceMetadataInformation)) - val dataColumns = finalOutputSpec.outputColumns.filterNot(partitionSet.contains) - - var needConvert = false - val projectList: Seq[NamedExpression] = plan.output.map { - case p if partitionSet.contains(p) && p.dataType == StringType && p.nullable => - needConvert = true - Alias(Empty2Null(p), p.name)() - case attr => attr - } - - val empty2NullPlan = if (needConvert) { - ProjectExec(projectList, plan) - } else { - plan - } - - val writerBucketSpec = bucketSpec.map { - spec => - val bucketColumns = spec.bucketColumnNames.map(c => dataColumns.find(_.name == c).get) - - if ( - options.getOrElse(BucketingUtils.optionForHiveCompatibleBucketWrite, "false") == - "true" - ) { - // Hive bucketed table: use `HiveHash` and bitwise-and as bucket id expression. - // Without the extra bitwise-and operation, we can get wrong bucket id when hash value of - // columns is negative. See Hive implementation in - // `org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils#getBucketNumber()`. - val hashId = BitwiseAnd(HiveHash(bucketColumns), Literal(Int.MaxValue)) - val bucketIdExpression = Pmod(hashId, Literal(spec.numBuckets)) - - // The bucket file name prefix is following Hive, Presto and Trino conversion, so this - // makes sure Hive bucketed table written by Spark, can be read by other SQL engines. - // - // Hive: `org.apache.hadoop.hive.ql.exec.Utilities#getBucketIdFromFile()`. - // Trino: `io.trino.plugin.hive.BackgroundHiveSplitLoader#BUCKET_PATTERNS`. - val fileNamePrefix = (bucketId: Int) => f"$bucketId%05d_0_" - WriterBucketSpec(bucketIdExpression, fileNamePrefix) - } else { - // Spark bucketed table: use `HashPartitioning.partitionIdExpression` as bucket id - // expression, so that we can guarantee the data distribution is same between shuffle and - // bucketed data source, which enables us to only shuffle one side when join a bucketed - // table and a normal one. - val bucketIdExpression = - HashPartitioning(bucketColumns, spec.numBuckets).partitionIdExpression - WriterBucketSpec(bucketIdExpression, (_: Int) => "") - } - } - val sortColumns = bucketSpec.toSeq.flatMap { - spec => spec.sortColumnNames.map(c => dataColumns.find(_.name == c).get) - } - - val caseInsensitiveOptions = CaseInsensitiveMap(options) - - val dataSchema = dataColumns.toStructType - DataSourceUtils.verifySchema(fileFormat, dataSchema) - // Note: prepareWrite has side effect. It sets "job". - val outputWriterFactory = - fileFormat.prepareWrite(sparkSession, job, caseInsensitiveOptions, dataSchema) - - val description = new WriteJobDescription( - uuid = UUID.randomUUID.toString, - serializableHadoopConf = new SerializableConfiguration(job.getConfiguration), - outputWriterFactory = outputWriterFactory, - allColumns = finalOutputSpec.outputColumns, - dataColumns = dataColumns, - partitionColumns = partitionColumns, - bucketSpec = writerBucketSpec, - path = finalOutputSpec.outputPath, - customPartitionLocations = finalOutputSpec.customPartitionLocations, - maxRecordsPerFile = caseInsensitiveOptions - .get("maxRecordsPerFile") - .map(_.toLong) - .getOrElse(sparkSession.sessionState.conf.maxRecordsPerFile), - timeZoneId = caseInsensitiveOptions - .get(DateTimeUtils.TIMEZONE_OPTION) - .getOrElse(sparkSession.sessionState.conf.sessionLocalTimeZone), - statsTrackers = statsTrackers - ) - - // We should first sort by partition columns, then bucket id, and finally sorting columns. - val requiredOrdering = partitionColumns.drop(numStaticPartitionCols) ++ - writerBucketSpec.map(_.bucketIdExpression) ++ sortColumns - // the sort order doesn't matter - val actualOrdering = empty2NullPlan.outputOrdering.map(_.child) - val orderingMatched = if (requiredOrdering.length > actualOrdering.length) { - false - } else { - requiredOrdering.zip(actualOrdering).forall { - case (requiredOrder, childOutputOrder) => - requiredOrder.semanticEquals(childOutputOrder) - } - } - - SQLExecution.checkSQLExecutionId(sparkSession) - - // propagate the description UUID into the jobs, so that committers - // get an ID guaranteed to be unique. - job.getConfiguration.set("spark.sql.sources.writeJobUUID", description.uuid) - - // This call shouldn't be put into the `try` block below because it only initializes and - // prepares the job, any exception thrown from here shouldn't cause abortJob() to be called. - committer.setupJob(job) - - def nativeWrap(plan: SparkPlan) = { - var wrapped: SparkPlan = plan - if (writerBucketSpec.isDefined) { - // We need to add the bucket id expression to the output of the sort plan, - // so that we can use backend to calculate the bucket id for each row. - wrapped = ProjectExec( - wrapped.output :+ Alias(writerBucketSpec.get.bucketIdExpression, "__bucket_value__")(), - wrapped) - // TODO: to optimize, bucket value is computed twice here - } - - val nativeFormat = sparkSession.sparkContext.getLocalProperty("nativeFormat") - (GlutenFormatFactory(nativeFormat).getWriterWrappedSparkPlan(wrapped), None) - } - - try { - val (finalPlan, concurrentOutputWriterSpec) = if (orderingMatched) { - if (!nativeEnabled) { - (empty2NullPlan, None) - } else { - nativeWrap(empty2NullPlan) - } - } else { - // SPARK-21165: the `requiredOrdering` is based on the attributes from analyzed plan, and - // the physical plan may have different attribute ids due to optimizer removing some - // aliases. Here we bind the expression ahead to avoid potential attribute ids mismatch. - val orderingExpr = bindReferences( - requiredOrdering.map(SortOrder(_, Ascending)), - finalOutputSpec.outputColumns) - val sortPlan = SortExec(orderingExpr, global = false, child = empty2NullPlan) - - val maxWriters = sparkSession.sessionState.conf.maxConcurrentOutputFileWriters - var concurrentWritersEnabled = maxWriters > 0 && sortColumns.isEmpty - if (nativeEnabled && concurrentWritersEnabled) { - log.warn( - s"spark.sql.maxConcurrentOutputFileWriters(being set to $maxWriters) will be " + - "ignored when native writer is being active. No concurrent Writers.") - concurrentWritersEnabled = false - } - - if (concurrentWritersEnabled) { - ( - empty2NullPlan, - Some(ConcurrentOutputWriterSpec(maxWriters, () => sortPlan.createSorter()))) - } else { - if (!nativeEnabled) { - (sortPlan, None) - } else { - nativeWrap(sortPlan) - } - } - } - - val rdd = finalPlan.execute() - - // SPARK-23271 If we are attempting to write a zero partition rdd, create a dummy single - // partition rdd to make sure we at least set up one write task to write the metadata. - val rddWithNonEmptyPartitions = if (rdd.partitions.length == 0) { - sparkSession.sparkContext.parallelize(Array.empty[InternalRow], 1) - } else { - rdd - } - - val jobIdInstant = new Date().getTime - val ret = new Array[WriteTaskResult](rddWithNonEmptyPartitions.partitions.length) - sparkSession.sparkContext.runJob( - rddWithNonEmptyPartitions, - (taskContext: TaskContext, iter: Iterator[InternalRow]) => { - executeTask( - description = description, - jobIdInstant = jobIdInstant, - sparkStageId = taskContext.stageId(), - sparkPartitionId = taskContext.partitionId(), - sparkAttemptNumber = taskContext.taskAttemptId().toInt & Integer.MAX_VALUE, - committer, - iterator = iter, - concurrentOutputWriterSpec = concurrentOutputWriterSpec - ) - }, - rddWithNonEmptyPartitions.partitions.indices, - (index, res: WriteTaskResult) => { - committer.onTaskCommit(res.commitMsg) - ret(index) = res - } - ) - - val commitMsgs = ret.map(_.commitMsg) - - logInfo(s"Start to commit write Job ${description.uuid}.") - val (_, duration) = Utils.timeTakenMs(committer.commitJob(job, commitMsgs)) - logInfo(s"Write Job ${description.uuid} committed. Elapsed time: $duration ms.") - - processStats(description.statsTrackers, ret.map(_.summary.stats), duration) - logInfo(s"Finished processing stats for write job ${description.uuid}.") - - // return a set of all the partition paths that were updated during this job - ret.map(_.summary.updatedPartitions).reduceOption(_ ++ _).getOrElse(Set.empty) - } catch { - case cause: Throwable => - logError(s"Aborting job ${description.uuid}.", cause) - committer.abortJob(job) - throw QueryExecutionErrors.jobAbortedError(cause) - } - } - // scalastyle:on argcount - - /** Writes data out in a single Spark task. */ - private def executeTask( - description: WriteJobDescription, - jobIdInstant: Long, - sparkStageId: Int, - sparkPartitionId: Int, - sparkAttemptNumber: Int, - committer: FileCommitProtocol, - iterator: Iterator[InternalRow], - concurrentOutputWriterSpec: Option[ConcurrentOutputWriterSpec]): WriteTaskResult = { - - val jobId = SparkHadoopWriterUtils.createJobID(new Date(jobIdInstant), sparkStageId) - val taskId = new TaskID(jobId, TaskType.MAP, sparkPartitionId) - val taskAttemptId = new TaskAttemptID(taskId, sparkAttemptNumber) - - // Set up the attempt context required to use in the output committer. - val taskAttemptContext: TaskAttemptContext = { - // Set up the configuration object - val hadoopConf = description.serializableHadoopConf.value - hadoopConf.set("mapreduce.job.id", jobId.toString) - hadoopConf.set("mapreduce.task.id", taskAttemptId.getTaskID.toString) - hadoopConf.set("mapreduce.task.attempt.id", taskAttemptId.toString) - hadoopConf.setBoolean("mapreduce.task.ismap", true) - hadoopConf.setInt("mapreduce.task.partition", 0) - - new TaskAttemptContextImpl(hadoopConf, taskAttemptId) - } - - committer.setupTask(taskAttemptContext) - - val dataWriter = - if (sparkPartitionId != 0 && !iterator.hasNext) { - // In case of empty job, - // leave first partition to save meta for file format like parquet/orc. - new EmptyDirectoryDataWriter(description, taskAttemptContext, committer) - } else if (description.partitionColumns.isEmpty && description.bucketSpec.isEmpty) { - new SingleDirectoryDataWriter(description, taskAttemptContext, committer) - } else { - concurrentOutputWriterSpec match { - case Some(spec) => - new DynamicPartitionDataConcurrentWriter( - description, - taskAttemptContext, - committer, - spec) - case _ => - new DynamicPartitionDataSingleWriter(description, taskAttemptContext, committer) - } - } - - try { - Utils.tryWithSafeFinallyAndFailureCallbacks(block = { - // Execute the task to write rows out and commit the task. - dataWriter.writeWithIterator(iterator) - dataWriter.commit() - })( - catchBlock = { - // If there is an error, abort the task - dataWriter.abort() - logError(s"Job $jobId aborted.") - }, - finallyBlock = { - dataWriter.close() - }) - } catch { - case e: FetchFailedException => - throw e - case f: FileAlreadyExistsException if SQLConf.get.fastFailFileFormatOutput => - // If any output file to write already exists, it does not make sense to re-run this task. - // We throw the exception and let Executor throw ExceptionFailure to abort the job. - throw new TaskOutputFileAlreadyExistException(f) - case t: Throwable => - throw QueryExecutionErrors.taskFailedWhileWritingRowsError(t) - } - } - - /** - * For every registered [[WriteJobStatsTracker]], call `processStats()` on it, passing it the - * corresponding [[WriteTaskStats]] from all executors. - */ - private[datasources] def processStats( - statsTrackers: Seq[WriteJobStatsTracker], - statsPerTask: Seq[Seq[WriteTaskStats]], - jobCommitDuration: Long): Unit = { - - val numStatsTrackers = statsTrackers.length - assert( - statsPerTask.forall(_.length == numStatsTrackers), - s"""Every WriteTask should have produced one `WriteTaskStats` object for every tracker. - |There are $numStatsTrackers statsTrackers, but some task returned - |${statsPerTask.find(_.length != numStatsTrackers).get.length} results instead. - """.stripMargin - ) - - val statsPerTracker = if (statsPerTask.nonEmpty) { - statsPerTask.transpose - } else { - statsTrackers.map(_ => Seq.empty) - } - - statsTrackers.zip(statsPerTracker).foreach { - case (statsTracker, stats) => statsTracker.processStats(stats, jobCommitDuration) - } - } -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/WriteFiles.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/WriteFiles.scala deleted file mode 100644 index 356a16942bb..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/WriteFiles.scala +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources - -import org.apache.spark.internal.io.FileCommitProtocol -import org.apache.spark.rdd.RDD -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.catalog.BucketSpec -import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec -import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, UnaryNode} -import org.apache.spark.sql.connector.write.WriterCommitMessage -import org.apache.spark.sql.execution.{SparkPlan, UnaryExecNode} -import org.apache.spark.sql.execution.datasources.FileFormatWriter.ConcurrentOutputWriterSpec - -/** - * This class is copied from Spark 3.4 and modified for Gluten. Spark 3.4 introduced a new operator, - * WriteFiles. In order to support the WriteTransformer in Spark 3.4, we need to copy the WriteFiles - * file into versions 3.2 and 3.3 to resolve compilation issues. Within WriteFiles, the - * doExecuteWrite method overrides the same method in SparkPlan. To avoid modifying SparkPlan, we - * have changed the doExecuteWrite method to a non-overridden method. - */ - -/** - * The write files spec holds all information of [[V1WriteCommand]] if its provider is - * [[FileFormat]]. - */ -case class WriteFilesSpec( - description: WriteJobDescription, - committer: FileCommitProtocol, - concurrentOutputWriterSpecFunc: SparkPlan => Option[ConcurrentOutputWriterSpec]) - -/** - * During Optimizer, [[V1Writes]] injects the [[WriteFiles]] between [[V1WriteCommand]] and query. - * [[WriteFiles]] must be the root plan as the child of [[V1WriteCommand]]. - */ -case class WriteFiles( - child: LogicalPlan, - fileFormat: FileFormat, - partitionColumns: Seq[Attribute], - bucketSpec: Option[BucketSpec], - options: Map[String, String], - staticPartitions: TablePartitionSpec) - extends UnaryNode { - override def output: Seq[Attribute] = child.output - override protected def stringArgs: Iterator[Any] = Iterator(child) - override protected def withNewChildInternal(newChild: LogicalPlan): WriteFiles = - copy(child = newChild) -} - -/** Responsible for writing files. */ -case class WriteFilesExec( - child: SparkPlan, - fileFormat: FileFormat, - partitionColumns: Seq[Attribute], - bucketSpec: Option[BucketSpec], - options: Map[String, String], - staticPartitions: TablePartitionSpec) - extends UnaryExecNode { - override def output: Seq[Attribute] = Seq.empty - - def doExecuteWrite(writeFilesSpec: WriteFilesSpec): RDD[WriterCommitMessage] = { - throw new UnsupportedOperationException(s"$nodeName does not support doExecuteWrite") - } - - override protected def doExecute(): RDD[InternalRow] = { - throw new UnsupportedOperationException(s"$nodeName does not support doExecute") - } - - override protected def stringArgs: Iterator[Any] = Iterator(child) - - override protected def withNewChildInternal(newChild: SparkPlan): WriteFilesExec = - copy(child = newChild) -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala deleted file mode 100644 index bb1eca0ec92..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala +++ /dev/null @@ -1,265 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.orc - -import org.apache.gluten.execution.datasource.GlutenFormatFactory - -import org.apache.spark.TaskContext -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection -import org.apache.spark.sql.execution.WholeStageCodegenExec -import org.apache.spark.sql.execution.datasources._ -import org.apache.spark.sql.sources._ -import org.apache.spark.sql.types._ -import org.apache.spark.util.{SerializableConfiguration, Utils} - -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.{FileStatus, Path} -import org.apache.hadoop.mapred.JobConf -import org.apache.hadoop.mapreduce._ -import org.apache.hadoop.mapreduce.lib.input.FileSplit -import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -import org.apache.orc.{OrcUtils => _, _} -import org.apache.orc.OrcConf.COMPRESS -import org.apache.orc.mapred.OrcStruct -import org.apache.orc.mapreduce._ - -import java.io._ -import java.net.URI - -/** - * This class is copied from Spark and modified for Gluten. Gluten should make sure this class is - * loaded before the original class. If a future Spark release introduces breaking changes to this - * class, we can move this file into a version-specific shim (e.g. shims/spark34) so it is only used - * by the Spark versions that need it. - */ -class OrcFileFormat extends FileFormat with DataSourceRegister with Serializable { - - override def shortName(): String = "orc" - - override def toString: String = "ORC" - - override def hashCode(): Int = getClass.hashCode() - - override def equals(other: Any): Boolean = other.isInstanceOf[OrcFileFormat] - - override def inferSchema( - sparkSession: SparkSession, - options: Map[String, String], - files: Seq[FileStatus]): Option[StructType] = { - OrcUtils.inferSchema(sparkSession, files, options) - } - - override def prepareWrite( - sparkSession: SparkSession, - job: Job, - options: Map[String, String], - dataSchema: StructType): OutputWriterFactory = { - val orcOptions = new OrcOptions(options, sparkSession.sessionState.conf) - - val conf = job.getConfiguration - - // Pass compression to job conf so that the file extension can be aware of it. - conf.set(COMPRESS.getAttribute, orcOptions.compressionCodec) - - conf - .asInstanceOf[JobConf] - .setOutputFormat(classOf[org.apache.orc.mapred.OrcOutputFormat[OrcStruct]]) - - if (sparkSession.sparkContext.getLocalProperty("isNativeApplicable") == "true") { - val nativeConf = - GlutenFormatFactory(shortName()).nativeConf(options, orcOptions.compressionCodec) - - new OutputWriterFactory { - override def getFileExtension(context: TaskAttemptContext): String = { - val compressionExtension: String = { - val name = context.getConfiguration.get(COMPRESS.getAttribute) - OrcUtils.extensionsForCompressionCodecNames.getOrElse(name, "") - } - compressionExtension + ".orc" - } - - override def newInstance( - path: String, - dataSchema: StructType, - context: TaskAttemptContext): OutputWriter = { - GlutenFormatFactory(shortName()) - .createOutputWriter(path, dataSchema, context, nativeConf) - } - } - } else { - new OutputWriterFactory { - override def newInstance( - path: String, - dataSchema: StructType, - context: TaskAttemptContext): OutputWriter = { - new OrcOutputWriter(path, dataSchema, context) - } - - override def getFileExtension(context: TaskAttemptContext): String = { - val compressionExtension: String = { - val name = context.getConfiguration.get(COMPRESS.getAttribute) - OrcUtils.extensionsForCompressionCodecNames.getOrElse(name, "") - } - compressionExtension + ".orc" - } - } - } - } - - override def supportBatch(sparkSession: SparkSession, schema: StructType): Boolean = { - val conf = sparkSession.sessionState.conf - conf.orcVectorizedReaderEnabled && conf.wholeStageEnabled && - !WholeStageCodegenExec.isTooManyFields(conf, schema) && - schema.forall( - s => - OrcUtils.supportColumnarReads( - s.dataType, - sparkSession.sessionState.conf.orcVectorizedReaderNestedColumnEnabled)) - } - - override def isSplitable( - sparkSession: SparkSession, - options: Map[String, String], - path: Path): Boolean = { - true - } - - override def buildReaderWithPartitionValues( - sparkSession: SparkSession, - dataSchema: StructType, - partitionSchema: StructType, - requiredSchema: StructType, - filters: Seq[Filter], - options: Map[String, String], - hadoopConf: Configuration): PartitionedFile => Iterator[InternalRow] = { - - val resultSchema = StructType(requiredSchema.fields ++ partitionSchema.fields) - val sqlConf = sparkSession.sessionState.conf - val enableVectorizedReader = supportBatch(sparkSession, resultSchema) - val capacity = sqlConf.orcVectorizedReaderBatchSize - - OrcConf.IS_SCHEMA_EVOLUTION_CASE_SENSITIVE.setBoolean(hadoopConf, sqlConf.caseSensitiveAnalysis) - - val broadcastedConf = - sparkSession.sparkContext.broadcast(new SerializableConfiguration(hadoopConf)) - val isCaseSensitive = sparkSession.sessionState.conf.caseSensitiveAnalysis - val orcFilterPushDown = sparkSession.sessionState.conf.orcFilterPushDown - val ignoreCorruptFiles = sparkSession.sessionState.conf.ignoreCorruptFiles - - (file: PartitionedFile) => { - val conf = broadcastedConf.value.value - - val filePath = new Path(new URI(file.filePath)) - - val fs = filePath.getFileSystem(conf) - val readerOptions = OrcFile.readerOptions(conf).filesystem(fs) - val orcSchema = - Utils.tryWithResource(OrcFile.createReader(filePath, readerOptions))(_.getSchema) - val resultedColPruneInfo = - OrcUtils.requestedColumnIds(isCaseSensitive, dataSchema, requiredSchema, orcSchema, conf) - - if (resultedColPruneInfo.isEmpty) { - Iterator.empty - } else { - // ORC predicate pushdown - if (orcFilterPushDown && filters.nonEmpty) { - OrcUtils.readCatalystSchema(filePath, conf, ignoreCorruptFiles).foreach { - fileSchema => - OrcFilters.createFilter(fileSchema, filters).foreach { - f => OrcInputFormat.setSearchArgument(conf, f, fileSchema.fieldNames) - } - } - } - - val (requestedColIds, canPruneCols) = resultedColPruneInfo.get - val resultSchemaString = OrcUtils.orcResultSchemaString( - canPruneCols, - dataSchema, - resultSchema, - partitionSchema, - conf) - assert( - requestedColIds.length == requiredSchema.length, - "[BUG] requested column IDs do not match required schema") - val taskConf = new Configuration(conf) - - val includeColumns = requestedColIds.filter(_ != -1).sorted.mkString(",") - taskConf.set(OrcConf.INCLUDE_COLUMNS.getAttribute, includeColumns) - val fileSplit = new FileSplit(filePath, file.start, file.length, Array.empty) - val attemptId = new TaskAttemptID(new TaskID(new JobID(), TaskType.MAP, 0), 0) - val taskAttemptContext = new TaskAttemptContextImpl(taskConf, attemptId) - - if (enableVectorizedReader) { - val batchReader = new OrcColumnarBatchReader(capacity) - // SPARK-23399 Register a task completion listener first to call `close()` in all cases. - // There is a possibility that `initialize` and `initBatch` hit some errors (like OOM) - // after opening a file. - val iter = new RecordReaderIterator(batchReader) - Option(TaskContext.get()).foreach(_.addTaskCompletionListener[Unit](_ => iter.close())) - val requestedDataColIds = requestedColIds ++ Array.fill(partitionSchema.length)(-1) - val requestedPartitionColIds = - Array.fill(requiredSchema.length)(-1) ++ Range(0, partitionSchema.length) - batchReader.initialize(fileSplit, taskAttemptContext) - batchReader.initBatch( - TypeDescription.fromString(resultSchemaString), - resultSchema.fields, - requestedDataColIds, - requestedPartitionColIds, - file.partitionValues) - - iter.asInstanceOf[Iterator[InternalRow]] - } else { - val orcRecordReader = new OrcInputFormat[OrcStruct] - .createRecordReader(fileSplit, taskAttemptContext) - val iter = new RecordReaderIterator[OrcStruct](orcRecordReader) - Option(TaskContext.get()).foreach(_.addTaskCompletionListener[Unit](_ => iter.close())) - - val fullSchema = requiredSchema.toAttributes ++ partitionSchema.toAttributes - val unsafeProjection = GenerateUnsafeProjection.generate(fullSchema, fullSchema) - val deserializer = new OrcDeserializer(requiredSchema, requestedColIds) - - if (partitionSchema.length == 0) { - iter.map(value => unsafeProjection(deserializer.deserialize(value))) - } else { - val joinedRow = new JoinedRow() - iter.map( - value => - unsafeProjection(joinedRow(deserializer.deserialize(value), file.partitionValues))) - } - } - } - } - } - - override def supportDataType(dataType: DataType): Boolean = dataType match { - case _: AtomicType => true - - case st: StructType => st.forall(f => supportDataType(f.dataType)) - - case ArrayType(elementType, _) => supportDataType(elementType) - - case MapType(keyType, valueType, _) => - supportDataType(keyType) && supportDataType(valueType) - - case udt: UserDefinedType[_] => supportDataType(udt.sqlType) - - case _ => false - } -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala deleted file mode 100644 index a534f880811..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala +++ /dev/null @@ -1,604 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.parquet - -import org.apache.gluten.execution.datasource.GlutenFormatFactory - -import org.apache.spark.TaskContext -import org.apache.spark.internal.Logging -import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection -import org.apache.spark.sql.catalyst.parser.LegacyTypeStringParser -import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DateTimeUtils} -import org.apache.spark.sql.errors.QueryExecutionErrors -import org.apache.spark.sql.execution.WholeStageCodegenExec -import org.apache.spark.sql.execution.datasources._ -import org.apache.spark.sql.execution.vectorized.{OffHeapColumnVector, OnHeapColumnVector} -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.sources._ -import org.apache.spark.sql.types._ -import org.apache.spark.util.{SerializableConfiguration, ThreadUtils} - -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.{FileStatus, Path} -import org.apache.hadoop.mapred.FileSplit -import org.apache.hadoop.mapreduce._ -import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -import org.apache.parquet.filter2.compat.FilterCompat -import org.apache.parquet.filter2.predicate.FilterApi -import org.apache.parquet.format.converter.ParquetMetadataConverter.SKIP_ROW_GROUPS -import org.apache.parquet.hadoop._ -import org.apache.parquet.hadoop.ParquetOutputFormat.JobSummaryLevel -import org.apache.parquet.hadoop.codec.CodecConfig -import org.apache.parquet.hadoop.util.ContextUtil - -import java.net.URI - -import scala.collection.JavaConverters._ -import scala.collection.mutable -import scala.util.{Failure, Try} - -class ParquetFileFormat extends FileFormat with DataSourceRegister with Logging with Serializable { - // Hold a reference to the (serializable) singleton instance of ParquetLogRedirector. This - // ensures the ParquetLogRedirector class is initialized whether an instance of ParquetFileFormat - // is constructed or deserialized. Do not heed the Scala compiler's warning about an unused field - // here. - private val parquetLogRedirector = ParquetLogRedirector.INSTANCE - - override def shortName(): String = "parquet" - - override def toString: String = "Parquet" - - override def hashCode(): Int = getClass.hashCode() - - override def equals(other: Any): Boolean = other.isInstanceOf[ParquetFileFormat] - - override def prepareWrite( - sparkSession: SparkSession, - job: Job, - options: Map[String, String], - dataSchema: StructType): OutputWriterFactory = { - if (sparkSession.sparkContext.getLocalProperty("isNativeApplicable") == "true") { - // Pass compression to job conf so that the file extension can be aware of it. - val conf = ContextUtil.getConfiguration(job) - val writeOptions = CaseInsensitiveMap( - sparkSession.sessionState - .newHadoopConfWithOptions(options) - .iterator() - .asScala - .map(entry => entry.getKey -> entry.getValue) - .toMap) - val parquetOptions = new ParquetOptions(writeOptions, sparkSession.sessionState.conf) - conf.set(ParquetOutputFormat.COMPRESSION, parquetOptions.compressionCodecClassName) - val nativeConf = - GlutenFormatFactory(shortName()) - .nativeConf(writeOptions, parquetOptions.compressionCodecClassName) - - new OutputWriterFactory { - override def getFileExtension(context: TaskAttemptContext): String = { - CodecConfig.from(context).getCodec.getExtension + ".parquet" - } - - override def newInstance( - path: String, - dataSchema: StructType, - context: TaskAttemptContext): OutputWriter = { - GlutenFormatFactory(shortName()) - .createOutputWriter(path, dataSchema, context, nativeConf) - - } - } - } else { - val parquetOptions = new ParquetOptions(options, sparkSession.sessionState.conf) - - val conf = ContextUtil.getConfiguration(job) - - val committerClass = - conf.getClass( - SQLConf.PARQUET_OUTPUT_COMMITTER_CLASS.key, - classOf[ParquetOutputCommitter], - classOf[OutputCommitter]) - - if (conf.get(SQLConf.PARQUET_OUTPUT_COMMITTER_CLASS.key) == null) { - logInfo( - "Using default output committer for Parquet: " + - classOf[ParquetOutputCommitter].getCanonicalName) - } else { - logInfo( - "Using user defined output committer for Parquet: " + committerClass.getCanonicalName) - } - - conf.setClass(SQLConf.OUTPUT_COMMITTER_CLASS.key, committerClass, classOf[OutputCommitter]) - - // We're not really using `ParquetOutputFormat[Row]` for writing data here, - // because we override - // it in `ParquetOutputWriter` to support appending and dynamic partitioning. The reason why - // we set it here is to setup the output committer class to `ParquetOutputCommitter`, which is - // bundled with `ParquetOutputFormat[Row]`. - job.setOutputFormatClass(classOf[ParquetOutputFormat[Row]]) - - ParquetOutputFormat.setWriteSupportClass(job, classOf[ParquetWriteSupport]) - - // This metadata is useful for keeping UDTs like Vector/Matrix. - ParquetWriteSupport.setSchema(dataSchema, conf) - - // Sets flags for `ParquetWriteSupport`, which converts Catalyst schema to Parquet - // schema and writes actual rows to Parquet files. - conf.set( - SQLConf.PARQUET_WRITE_LEGACY_FORMAT.key, - sparkSession.sessionState.conf.writeLegacyParquetFormat.toString) - - conf.set( - SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key, - sparkSession.sessionState.conf.parquetOutputTimestampType.toString) - - conf.set( - SQLConf.PARQUET_FIELD_ID_WRITE_ENABLED.key, - sparkSession.sessionState.conf.parquetFieldIdWriteEnabled.toString) - - // Sets compression scheme - conf.set(ParquetOutputFormat.COMPRESSION, parquetOptions.compressionCodecClassName) - - // SPARK-15719: Disables writing Parquet summary files by default. - if ( - conf.get(ParquetOutputFormat.JOB_SUMMARY_LEVEL) == null - && conf.get(ParquetOutputFormat.ENABLE_JOB_SUMMARY) == null - ) { - conf.setEnum(ParquetOutputFormat.JOB_SUMMARY_LEVEL, JobSummaryLevel.NONE) - } - - if ( - ParquetOutputFormat.getJobSummaryLevel(conf) != JobSummaryLevel.NONE - && !classOf[ParquetOutputCommitter].isAssignableFrom(committerClass) - ) { - // output summary is requested, but the class is not a Parquet Committer - logWarning( - s"Committer $committerClass is not a ParquetOutputCommitter and cannot" + - s" create job summaries. " + - s"Set Parquet option ${ParquetOutputFormat.JOB_SUMMARY_LEVEL} to NONE.") - } - - new OutputWriterFactory { - // This OutputWriterFactory instance is deserialized when writing Parquet files on the - // executor side without constructing or deserializing ParquetFileFormat. Therefore, we hold - // another reference to ParquetLogRedirector.INSTANCE here to ensure the latter class is - // initialized. - private val parquetLogRedirector = ParquetLogRedirector.INSTANCE - - override def newInstance( - path: String, - dataSchema: StructType, - context: TaskAttemptContext): OutputWriter = { - new ParquetOutputWriter(path, context) - } - - override def getFileExtension(context: TaskAttemptContext): String = { - CodecConfig.from(context).getCodec.getExtension + ".parquet" - } - } - } - } - - override def inferSchema( - sparkSession: SparkSession, - parameters: Map[String, String], - files: Seq[FileStatus]): Option[StructType] = { - ParquetUtils.inferSchema(sparkSession, parameters, files) - } - - /** Returns whether the reader will return the rows as batch or not. */ - override def supportBatch(sparkSession: SparkSession, schema: StructType): Boolean = { - val conf = sparkSession.sessionState.conf - ParquetUtils.isBatchReadSupportedForSchema(conf, schema) && conf.wholeStageEnabled && - !WholeStageCodegenExec.isTooManyFields(conf, schema) - } - - override def vectorTypes( - requiredSchema: StructType, - partitionSchema: StructType, - sqlConf: SQLConf): Option[Seq[String]] = { - Option( - Seq.fill(requiredSchema.fields.length + partitionSchema.fields.length)( - if (!sqlConf.offHeapColumnVectorEnabled) { - classOf[OnHeapColumnVector].getName - } else { - classOf[OffHeapColumnVector].getName - } - )) - } - - override def isSplitable( - sparkSession: SparkSession, - options: Map[String, String], - path: Path): Boolean = { - true - } - - override def buildReaderWithPartitionValues( - sparkSession: SparkSession, - dataSchema: StructType, - partitionSchema: StructType, - requiredSchema: StructType, - filters: Seq[Filter], - options: Map[String, String], - hadoopConf: Configuration): PartitionedFile => Iterator[InternalRow] = { - hadoopConf.set(ParquetInputFormat.READ_SUPPORT_CLASS, classOf[ParquetReadSupport].getName) - hadoopConf.set(ParquetReadSupport.SPARK_ROW_REQUESTED_SCHEMA, requiredSchema.json) - hadoopConf.set(ParquetWriteSupport.SPARK_ROW_SCHEMA, requiredSchema.json) - hadoopConf.set( - SQLConf.SESSION_LOCAL_TIMEZONE.key, - sparkSession.sessionState.conf.sessionLocalTimeZone) - hadoopConf.setBoolean( - SQLConf.NESTED_SCHEMA_PRUNING_ENABLED.key, - sparkSession.sessionState.conf.nestedSchemaPruningEnabled) - hadoopConf.setBoolean( - SQLConf.CASE_SENSITIVE.key, - sparkSession.sessionState.conf.caseSensitiveAnalysis) - - // Sets flags for `ParquetToSparkSchemaConverter` - hadoopConf.setBoolean( - SQLConf.PARQUET_BINARY_AS_STRING.key, - sparkSession.sessionState.conf.isParquetBinaryAsString) - hadoopConf.setBoolean( - SQLConf.PARQUET_INT96_AS_TIMESTAMP.key, - sparkSession.sessionState.conf.isParquetINT96AsTimestamp) - - val broadcastedHadoopConf = - sparkSession.sparkContext.broadcast(new SerializableConfiguration(hadoopConf)) - - // TODO: if you move this into the closure it reverts to the default values. - // If true, enable using the custom RecordReader for parquet. This only works for - // a subset of the types (no complex types). - val resultSchema = StructType(partitionSchema.fields ++ requiredSchema.fields) - val sqlConf = sparkSession.sessionState.conf - val enableOffHeapColumnVector = sqlConf.offHeapColumnVectorEnabled - val enableVectorizedReader: Boolean = - ParquetUtils.isBatchReadSupportedForSchema(sqlConf, resultSchema) - val enableRecordFilter: Boolean = sqlConf.parquetRecordFilterEnabled - val timestampConversion: Boolean = sqlConf.isParquetINT96TimestampConversion - val capacity = sqlConf.parquetVectorizedReaderBatchSize - val enableParquetFilterPushDown: Boolean = sqlConf.parquetFilterPushDown - // Whole stage codegen (PhysicalRDD) is able to deal with batches directly - val returningBatch = supportBatch(sparkSession, resultSchema) - val pushDownDate = sqlConf.parquetFilterPushDownDate - val pushDownTimestamp = sqlConf.parquetFilterPushDownTimestamp - val pushDownDecimal = sqlConf.parquetFilterPushDownDecimal - val pushDownStringStartWith = sqlConf.parquetFilterPushDownStringStartWith - val pushDownInFilterThreshold = sqlConf.parquetFilterPushDownInFilterThreshold - val isCaseSensitive = sqlConf.caseSensitiveAnalysis - val parquetOptions = new ParquetOptions(options, sparkSession.sessionState.conf) - val datetimeRebaseModeInRead = parquetOptions.datetimeRebaseModeInRead - val int96RebaseModeInRead = parquetOptions.int96RebaseModeInRead - - (file: PartitionedFile) => { - assert(file.partitionValues.numFields == partitionSchema.size) - - val filePath = new Path(new URI(file.filePath)) - val split = new FileSplit(filePath, file.start, file.length, Array.empty[String]) - - val sharedConf = broadcastedHadoopConf.value.value - - lazy val footerFileMetaData = - ParquetFooterReader.readFooter(sharedConf, filePath, SKIP_ROW_GROUPS).getFileMetaData - val datetimeRebaseSpec = DataSourceUtils.datetimeRebaseSpec( - footerFileMetaData.getKeyValueMetaData.get, - datetimeRebaseModeInRead) - // Try to push down filters when filter push-down is enabled. - val pushed = if (enableParquetFilterPushDown) { - val parquetSchema = footerFileMetaData.getSchema - val parquetFilters = new ParquetFilters( - parquetSchema, - pushDownDate, - pushDownTimestamp, - pushDownDecimal, - pushDownStringStartWith, - pushDownInFilterThreshold, - isCaseSensitive, - datetimeRebaseSpec) - filters - // Collects all converted Parquet filter predicates. Notice that not all predicates can be - // converted (`ParquetFilters.createFilter` returns an `Option`). That's why a `flatMap` - // is used here. - .flatMap(parquetFilters.createFilter(_)) - .reduceOption(FilterApi.and) - } else { - None - } - - // PARQUET_INT96_TIMESTAMP_CONVERSION says to apply timezone conversions to int96 timestamps' - // *only* if the file was created by something other than "parquet-mr", so check the actual - // writer here for this file. We have to do this per-file, as each file in the table may - // have different writers. - // Define isCreatedByParquetMr as function to avoid unnecessary parquet footer reads. - def isCreatedByParquetMr: Boolean = - footerFileMetaData.getCreatedBy.startsWith("parquet-mr") - - val convertTz = - if (timestampConversion && !isCreatedByParquetMr) { - Some(DateTimeUtils.getZoneId(sharedConf.get(SQLConf.SESSION_LOCAL_TIMEZONE.key))) - } else { - None - } - - val int96RebaseSpec = DataSourceUtils.int96RebaseSpec( - footerFileMetaData.getKeyValueMetaData.get, - int96RebaseModeInRead) - - val attemptId = new TaskAttemptID(new TaskID(new JobID(), TaskType.MAP, 0), 0) - val hadoopAttemptContext = - new TaskAttemptContextImpl(broadcastedHadoopConf.value.value, attemptId) - - // Try to push down filters when filter push-down is enabled. - // Notice: This push-down is RowGroups level, not individual records. - if (pushed.isDefined) { - ParquetInputFormat.setFilterPredicate(hadoopAttemptContext.getConfiguration, pushed.get) - } - val taskContext = Option(TaskContext.get()) - if (enableVectorizedReader) { - val vectorizedReader = new VectorizedParquetRecordReader( - convertTz.orNull, - datetimeRebaseSpec.mode.toString, - datetimeRebaseSpec.timeZone, - int96RebaseSpec.mode.toString, - int96RebaseSpec.timeZone, - enableOffHeapColumnVector && taskContext.isDefined, - capacity - ) - // SPARK-37089: We cannot register a task completion listener to close this iterator here - // because downstream exec nodes have already registered their listeners. Since listeners - // are executed in reverse order of registration, a listener registered here would close the - // iterator while downstream exec nodes are still running. When off-heap column vectors are - // enabled, this can cause a use-after-free bug leading to a segfault. - // - // Instead, we use FileScanRDD's task completion listener to close this iterator. - val iter = new RecordReaderIterator(vectorizedReader) - try { - vectorizedReader.initialize(split, hadoopAttemptContext) - logDebug(s"Appending $partitionSchema ${file.partitionValues}") - vectorizedReader.initBatch(partitionSchema, file.partitionValues) - if (returningBatch) { - vectorizedReader.enableReturningBatches() - } - - // UnsafeRowParquetRecordReader appends the columns internally to avoid another copy. - iter.asInstanceOf[Iterator[InternalRow]] - } catch { - case e: Throwable => - // SPARK-23457: In case there is an exception in initialization, close the iterator to - // avoid leaking resources. - iter.close() - throw e - } - } else { - logDebug(s"Falling back to parquet-mr") - // ParquetRecordReader returns InternalRow - val readSupport = new ParquetReadSupport( - convertTz, - enableVectorizedReader = false, - datetimeRebaseSpec, - int96RebaseSpec) - val reader = if (pushed.isDefined && enableRecordFilter) { - val parquetFilter = FilterCompat.get(pushed.get, null) - new ParquetRecordReader[InternalRow](readSupport, parquetFilter) - } else { - new ParquetRecordReader[InternalRow](readSupport) - } - val iter = new RecordReaderIterator[InternalRow](reader) - try { - reader.initialize(split, hadoopAttemptContext) - - val fullSchema = requiredSchema.toAttributes ++ partitionSchema.toAttributes - val unsafeProjection = GenerateUnsafeProjection.generate(fullSchema, fullSchema) - - if (partitionSchema.length == 0) { - // There is no partition columns - iter.map(unsafeProjection) - } else { - val joinedRow = new JoinedRow() - iter.map(d => unsafeProjection(joinedRow(d, file.partitionValues))) - } - } catch { - case e: Throwable => - // SPARK-23457: In case there is an exception in initialization, close the iterator to - // avoid leaking resources. - iter.close() - throw e - } - } - } - } - - override def supportDataType(dataType: DataType): Boolean = dataType match { - case _: AtomicType => true - - case st: StructType => st.forall(f => supportDataType(f.dataType)) - - case ArrayType(elementType, _) => supportDataType(elementType) - - case MapType(keyType, valueType, _) => - supportDataType(keyType) && supportDataType(valueType) - - case udt: UserDefinedType[_] => supportDataType(udt.sqlType) - - case _ => false - } -} - -object ParquetFileFormat extends Logging { - private[parquet] def readSchema( - footers: Seq[Footer], - sparkSession: SparkSession): Option[StructType] = { - - val converter = new ParquetToSparkSchemaConverter( - sparkSession.sessionState.conf.isParquetBinaryAsString, - sparkSession.sessionState.conf.isParquetINT96AsTimestamp) - - val seen = mutable.HashSet[String]() - val finalSchemas: Seq[StructType] = footers.flatMap { - footer => - val metadata = footer.getParquetMetadata.getFileMetaData - val serializedSchema = metadata.getKeyValueMetaData.asScala.toMap - .get(ParquetReadSupport.SPARK_METADATA_KEY) - if (serializedSchema.isEmpty) { - // Falls back to Parquet schema if no Spark SQL schema found. - Some(converter.convert(metadata.getSchema)) - } else if (!seen.contains(serializedSchema.get)) { - seen += serializedSchema.get - - // Don't throw even if we failed to parse the serialized Spark schema. Just fallback to - // whatever is available. - Some(Try(DataType.fromJson(serializedSchema.get)) - .recover { - case _: Throwable => - logInfo( - "Serialized Spark schema in Parquet key-value metadata is not in JSON format, " + - "falling back to the deprecated DataType.fromCaseClassString parser.") - LegacyTypeStringParser.parseString(serializedSchema.get) - } - .recover { - case cause: Throwable => - logWarning( - s"""Failed to parse serialized Spark schema in Parquet key-value metadata: - |\t$serializedSchema - """.stripMargin, - cause - ) - } - .map(_.asInstanceOf[StructType]) - .getOrElse { - // Falls back to Parquet schema if Spark SQL schema can't be parsed. - converter.convert(metadata.getSchema) - }) - } else { - None - } - } - - finalSchemas.reduceOption { - (left, right) => - try left.merge(right) - catch { - case e: Throwable => - throw QueryExecutionErrors.failedToMergeIncompatibleSchemasError(left, right, e) - } - } - } - - /** - * Reads Parquet footers in multi-threaded manner. If the config - * "spark.sql.files.ignoreCorruptFiles" is set to true, we will ignore the corrupted files when - * reading footers. - */ - private[parquet] def readParquetFootersInParallel( - conf: Configuration, - partFiles: Seq[FileStatus], - ignoreCorruptFiles: Boolean): Seq[Footer] = { - ThreadUtils - .parmap(partFiles, "readingParquetFooters", 8) { - currentFile => - try { - // Skips row group information since we only need the schema. - // ParquetFileReader.readFooter throws RuntimeException, instead of IOException, - // when it can't read the footer. - Some( - new Footer( - currentFile.getPath, - ParquetFooterReader.readFooter(conf, currentFile, SKIP_ROW_GROUPS))) - } catch { - case e: RuntimeException => - if (ignoreCorruptFiles) { - logWarning(s"Skipped the footer in the corrupted file: $currentFile", e) - None - } else { - throw QueryExecutionErrors.cannotReadFooterForFileError(currentFile, e) - } - } - } - .flatten - } - - /** - * Figures out a merged Parquet schema with a distributed Spark job. - * - * Note that locality is not taken into consideration here because: - * - * 1. For a single Parquet part-file, in most cases the footer only resides in the last block of - * that file. Thus we only need to retrieve the location of the last block. However, Hadoop - * `FileSystem` only provides API to retrieve locations of all blocks, which can be - * potentially expensive. - * 2. This optimization is mainly useful for S3, where file metadata operations can be pretty - * slow. And basically locality is not available when using S3 (you can't run computation on - * S3 nodes). - */ - def mergeSchemasInParallel( - parameters: Map[String, String], - filesToTouch: Seq[FileStatus], - sparkSession: SparkSession): Option[StructType] = { - val assumeBinaryIsString = sparkSession.sessionState.conf.isParquetBinaryAsString - val assumeInt96IsTimestamp = sparkSession.sessionState.conf.isParquetINT96AsTimestamp - - val reader = (files: Seq[FileStatus], conf: Configuration, ignoreCorruptFiles: Boolean) => { - // Converter used to convert Parquet `MessageType` to Spark SQL `StructType` - val converter = new ParquetToSparkSchemaConverter( - assumeBinaryIsString = assumeBinaryIsString, - assumeInt96IsTimestamp = assumeInt96IsTimestamp) - - readParquetFootersInParallel(conf, files, ignoreCorruptFiles) - .map(ParquetFileFormat.readSchemaFromFooter(_, converter)) - } - - SchemaMergeUtils.mergeSchemasInParallel(sparkSession, parameters, filesToTouch, reader) - } - - /** - * Reads Spark SQL schema from a Parquet footer. If a valid serialized Spark SQL schema string can - * be found in the file metadata, returns the deserialized [[StructType]], otherwise, returns a - * [[StructType]] converted from the [[org.apache.parquet.schema.MessageType]] stored in this - * footer. - */ - def readSchemaFromFooter(footer: Footer, converter: ParquetToSparkSchemaConverter): StructType = { - val fileMetaData = footer.getParquetMetadata.getFileMetaData - fileMetaData.getKeyValueMetaData.asScala.toMap - .get(ParquetReadSupport.SPARK_METADATA_KEY) - .flatMap(deserializeSchemaString) - .getOrElse(converter.convert(fileMetaData.getSchema)) - } - - private def deserializeSchemaString(schemaString: String): Option[StructType] = { - // Tries to deserialize the schema string as JSON first, then falls back to the case class - // string parser (data generated by older versions of Spark SQL uses this format). - Try(DataType.fromJson(schemaString).asInstanceOf[StructType]) - .recover { - case _: Throwable => - logInfo( - "Serialized Spark schema in Parquet key-value metadata is not in JSON format, " + - "falling back to the deprecated DataType.fromCaseClassString parser.") - LegacyTypeStringParser.parseString(schemaString).asInstanceOf[StructType] - } - .recoverWith { - case cause: Throwable => - logWarning( - "Failed to parse and ignored serialized Spark schema in " + - s"Parquet key-value metadata:\n\t$schemaString", - cause) - Failure(cause) - } - .toOption - } -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFooterReaderShim.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFooterReaderShim.scala deleted file mode 100644 index b1419e5e623..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFooterReaderShim.scala +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.parquet - -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.{FileStatus, Path} -import org.apache.parquet.format.converter.ParquetMetadataConverter -import org.apache.parquet.hadoop.metadata.ParquetMetadata - -/** Shim layer for ParquetFooterReader to maintain compatibility across different Spark versions. */ -object ParquetFooterReaderShim { - - /** @since Spark 4.1 */ - def readFooter( - configuration: Configuration, - fileStatus: FileStatus, - filter: ParquetMetadataConverter.MetadataFilter): ParquetMetadata = { - ParquetFooterReader.readFooter(configuration, fileStatus, filter) - } - - /** @since Spark 4.1 */ - def readFooter( - configuration: Configuration, - file: Path, - filter: ParquetMetadataConverter.MetadataFilter): ParquetMetadata = { - ParquetFooterReader.readFooter(configuration, file, filter) - } -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AbstractBatchScanExec.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AbstractBatchScanExec.scala deleted file mode 100644 index b12a257a0c1..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AbstractBatchScanExec.scala +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.v2 - -import org.apache.spark.SparkException -import org.apache.spark.rdd.RDD -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.plans.physical.{KeyGroupedPartitioning, SinglePartition} -import org.apache.spark.sql.catalyst.util.{truncatedString, InternalRowSet} -import org.apache.spark.sql.connector.read._ -import org.apache.spark.sql.execution.datasources.DataSourceStrategy - -import com.google.common.base.Objects - -/** Physical plan node for scanning a batch of data from a data source v2. */ -abstract class AbstractBatchScanExec( - output: Seq[AttributeReference], - @transient scan: Scan, - val runtimeFilters: Seq[Expression], - keyGroupedPartitioning: Option[Seq[Expression]] = None) - extends DataSourceV2ScanExecBase { - - @transient lazy val batch = scan.toBatch - - // TODO: unify the equal/hashCode implementation for all data source v2 query plans. - override def equals(other: Any): Boolean = other match { - case other: AbstractBatchScanExec => - this.batch == other.batch && this.runtimeFilters == other.runtimeFilters - case _ => - false - } - - override def hashCode(): Int = Objects.hashCode(batch, runtimeFilters) - - @transient override lazy val inputPartitions: Seq[InputPartition] = inputPartitionsShim - - @transient protected lazy val inputPartitionsShim: Seq[InputPartition] = - batch.planInputPartitions() - - @transient private lazy val filteredPartitions: Seq[Seq[InputPartition]] = { - val dataSourceFilters = runtimeFilters.flatMap { - case DynamicPruningExpression(e) => DataSourceStrategy.translateRuntimeFilter(e) - case _ => None - } - - if (dataSourceFilters.nonEmpty) { - val originalPartitioning = outputPartitioning - - // the cast is safe as runtime filters are only assigned if the scan can be filtered - val filterableScan = scan.asInstanceOf[SupportsRuntimeFiltering] - filterableScan.filter(dataSourceFilters.toArray) - - // call toBatch again to get filtered partitions - val newPartitions = scan.toBatch.planInputPartitions() - - originalPartitioning match { - case p: KeyGroupedPartitioning => - if (newPartitions.exists(!_.isInstanceOf[HasPartitionKey])) { - throw new SparkException( - "Data source must have preserved the original partitioning " + - "during runtime filtering: not all partitions implement HasPartitionKey after " + - "filtering") - } - - val newRows = new InternalRowSet(p.expressions.map(_.dataType)) - newRows ++= newPartitions.map(_.asInstanceOf[HasPartitionKey].partitionKey()) - val oldRows = p.partitionValuesOpt.get - - if (oldRows.size != newRows.size) { - throw new SparkException( - "Data source must have preserved the original partitioning " + - "during runtime filtering: the number of unique partition values obtained " + - s"through HasPartitionKey changed: before ${oldRows.size}, after ${newRows.size}") - } - - if (!oldRows.forall(newRows.contains)) { - throw new SparkException( - "Data source must have preserved the original partitioning " + - "during runtime filtering: the number of unique partition values obtained " + - s"through HasPartitionKey remain the same but do not exactly match") - } - - groupPartitions(newPartitions).get.map(_._2) - - case _ => - // no validation is needed as the data source did not report any specific partitioning - newPartitions.map(Seq(_)) - } - - } else { - partitions - } - } - - override lazy val readerFactory: PartitionReaderFactory = batch.createReaderFactory() - - override lazy val inputRDD: RDD[InternalRow] = { - if (filteredPartitions.isEmpty && outputPartitioning == SinglePartition) { - // return an empty RDD with 1 partition if dynamic filtering removed the only split - sparkContext.parallelize(Array.empty[InternalRow], 1) - } else { - new DataSourceRDD( - sparkContext, - filteredPartitions, - readerFactory, - supportsColumnar, - customMetrics) - } - } - - override def simpleString(maxFields: Int): String = { - val truncatedOutputString = truncatedString(output, "[", ", ", "]", maxFields) - val runtimeFiltersString = s"RuntimeFilters: ${runtimeFilters.mkString("[", ",", "]")}" - val result = s"$nodeName$truncatedOutputString ${scan.description()} $runtimeFiltersString" - redact(result) - } -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala deleted file mode 100644 index 05e1b88f761..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.v2 - -import org.apache.spark.SparkException -import org.apache.spark.rdd.RDD -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.plans.physical.KeyGroupedPartitioning -import org.apache.spark.sql.catalyst.util.InternalRowSet -import org.apache.spark.sql.connector.catalog.Table -import org.apache.spark.sql.connector.expressions.aggregate.Aggregation -import org.apache.spark.sql.connector.read.{HasPartitionKey, InputPartition, Scan, SupportsRuntimeFiltering} -import org.apache.spark.sql.execution.datasources.DataSourceStrategy -import org.apache.spark.sql.execution.datasources.v2.orc.OrcScan -import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetScan -import org.apache.spark.sql.execution.metric.SQLMetric -import org.apache.spark.sql.vectorized.ColumnarBatch - -abstract class BatchScanExecShim( - override val output: Seq[AttributeReference], - @transient override val scan: Scan, - override val runtimeFilters: Seq[Expression], - val keyGroupedPartitioning: Option[Seq[Expression]] = None, - val ordering: Option[Seq[SortOrder]] = None, - @transient val table: Table, - val commonPartitionValues: Option[Seq[(InternalRow, Int)]] = None, - val applyPartialClustering: Boolean = false, - val replicatePartitions: Boolean = false) - extends AbstractBatchScanExec(output, scan, runtimeFilters) { - - // Note: "metrics" is made transient to avoid sending driver-side metrics to tasks. - @transient override lazy val metrics: Map[String, SQLMetric] = Map() - - lazy val metadataColumns: Seq[AttributeReference] = output.collect { - case FileSourceMetadataAttribute(attr) => attr - } - - def hasUnsupportedColumns: Boolean = { - // TODO, fallback if user define same name column due to we can't right now - // detect which column is metadata column which is user defined column. - val metadataColumnsNames = metadataColumns.map(_.name) - output - .filterNot(metadataColumns.toSet) - .exists(v => metadataColumnsNames.contains(v.name)) - } - - def postDriverMetrics(): Unit = {} - - override def doExecuteColumnar(): RDD[ColumnarBatch] = { - throw new UnsupportedOperationException("Need to implement this method") - } - - @transient protected lazy val filteredPartitions: Seq[Seq[InputPartition]] = { - val dataSourceFilters = runtimeFilters.flatMap { - case DynamicPruningExpression(e) => DataSourceStrategy.translateRuntimeFilter(e) - case _ => None - } - - if (dataSourceFilters.nonEmpty) { - val originalPartitioning = outputPartitioning - - // the cast is safe as runtime filters are only assigned if the scan can be filtered - val filterableScan = scan.asInstanceOf[SupportsRuntimeFiltering] - filterableScan.filter(dataSourceFilters.toArray) - - // call toBatch again to get filtered partitions - val newPartitions = scan.toBatch.planInputPartitions() - - originalPartitioning match { - case p: KeyGroupedPartitioning => - if (newPartitions.exists(!_.isInstanceOf[HasPartitionKey])) { - throw new SparkException( - "Data source must have preserved the original partitioning " + - "during runtime filtering: not all partitions implement HasPartitionKey after " + - "filtering") - } - - val newRows = new InternalRowSet(p.expressions.map(_.dataType)) - newRows ++= newPartitions.map(_.asInstanceOf[HasPartitionKey].partitionKey()) - val oldRows = p.partitionValuesOpt.get - - if (oldRows.size != newRows.size) { - throw new SparkException( - "Data source must have preserved the original partitioning " + - "during runtime filtering: the number of unique partition values obtained " + - s"through HasPartitionKey changed: before ${oldRows.size}, after ${newRows.size}") - } - - if (!oldRows.forall(newRows.contains)) { - throw new SparkException( - "Data source must have preserved the original partitioning " + - "during runtime filtering: the number of unique partition values obtained " + - s"through HasPartitionKey remain the same but do not exactly match") - } - - groupPartitions(newPartitions).get.map(_._2) - - case _ => - // no validation is needed as the data source did not report any specific partitioning - newPartitions.map(Seq(_)) - } - - } else { - partitions - } - } - - @transient lazy val pushedAggregate: Option[Aggregation] = { - scan match { - case s: ParquetScan => s.pushedAggregate - case o: OrcScan => o.pushedAggregate - case _ => None - } - } - - final override protected def otherCopyArgs: Seq[AnyRef] = { - Seq( - ordering, - table, - commonPartitionValues, - // Box boolean to match `AnyRef` - Boolean.box(applyPartialClustering), - Boolean.box(replicatePartitions)) - } -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/v2/utils/CatalogUtil.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/v2/utils/CatalogUtil.scala deleted file mode 100644 index 517b69b7f8b..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/v2/utils/CatalogUtil.scala +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.v2.utils - -import org.apache.spark.sql.catalyst.catalog.BucketSpec -import org.apache.spark.sql.connector.expressions.Transform - -object CatalogUtil { - - def convertPartitionTransforms(partitions: Seq[Transform]): (Seq[String], Option[BucketSpec]) = { - import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.TransformHelper - partitions.convertTransforms - } -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/python/BasePythonRunnerShim.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/python/BasePythonRunnerShim.scala deleted file mode 100644 index 82d971d5d6a..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/python/BasePythonRunnerShim.scala +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.python - -import org.apache.spark.SparkEnv -import org.apache.spark.TaskContext -import org.apache.spark.api.python.{BasePythonRunner, ChainedPythonFunctions} -import org.apache.spark.sql.execution.metric.SQLMetric -import org.apache.spark.sql.vectorized.ColumnarBatch - -import java.io.DataOutputStream -import java.net.Socket - -abstract class BasePythonRunnerShim( - funcs: Seq[(ChainedPythonFunctions, Long)], - evalType: Int, - argMetas: Array[Array[(Int, Option[String])]], - pythonMetrics: Map[String, SQLMetric]) - extends BasePythonRunner[ColumnarBatch, ColumnarBatch]( - funcs.map(_._1), - evalType, - argMetas.map(_.map(_._1))) { - // The type aliases below provide consistent type names in child classes, - // ensuring code compatibility with both Spark 4.0 and earlier versions. - type Writer = WriterThread - type PythonWorker = Socket - - protected def createNewWriter( - env: SparkEnv, - worker: PythonWorker, - inputIterator: Iterator[ColumnarBatch], - partitionIndex: Int, - context: TaskContext): Writer - - protected def writeUdf( - dataOut: DataOutputStream, - argMetas: Array[Array[(Int, Option[String])]]): Unit = { - PythonUDFRunner.writeUDFs(dataOut, funcs.map(_._1), argMetas.map(_.map(_._1))) - } - - override protected def newWriterThread( - env: SparkEnv, - worker: PythonWorker, - inputIterator: Iterator[ColumnarBatch], - partitionIndex: Int, - context: TaskContext): Writer = { - createNewWriter(env, worker, inputIterator, partitionIndex, context) - } -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExecBase.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExecBase.scala deleted file mode 100644 index 7221e330a7d..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExecBase.scala +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.python - -import org.apache.spark.TaskContext -import org.apache.spark.api.python.ChainedPythonFunctions -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.Expression -import org.apache.spark.sql.types.StructType - -abstract class EvalPythonExecBase extends EvalPythonExec { - - override protected def evaluate( - funcs: Seq[ChainedPythonFunctions], - argOffsets: Array[Array[Int]], - iter: Iterator[InternalRow], - schema: StructType, - context: TaskContext): Iterator[InternalRow] = { - throw new IllegalStateException("EvalPythonExecTransformer doesn't support evaluate") - } -} - -object EvalPythonExecBase { - object NamedArgumentExpressionShim { - def unapply(expr: Expression): Option[(String, Expression)] = None - } -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/stat/StatFunctions.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/stat/StatFunctions.scala deleted file mode 100644 index 08ba7680ca7..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/stat/StatFunctions.scala +++ /dev/null @@ -1,364 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.stat - -import org.apache.spark.internal.Logging -import org.apache.spark.sql.{Column, DataFrame, Dataset, Row} -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Cast, Expression, GenericInternalRow, GetArrayItem, Literal, TryCast} -import org.apache.spark.sql.catalyst.expressions.aggregate._ -import org.apache.spark.sql.catalyst.plans.logical.LocalRelation -import org.apache.spark.sql.catalyst.util.{GenericArrayData, QuantileSummaries} -import org.apache.spark.sql.errors.QueryExecutionErrors -import org.apache.spark.sql.functions.count -import org.apache.spark.sql.types._ -import org.apache.spark.unsafe.types.UTF8String - -import java.util.Locale - -/** - * This file is copied from Spark - * - * The df.describe() and df.summary() issues are fixed by - * https://github.com/apache/spark/pull/40914. We picked it into Gluten to fix the describe and - * summary issue. And this file can be removed after upgrading spark version to 3.4 or higher - * version. - */ -object StatFunctions extends Logging { - - /** - * Calculates the approximate quantiles of multiple numerical columns of a DataFrame in one pass. - * - * The result of this algorithm has the following deterministic bound: If the DataFrame has N - * elements and if we request the quantile at probability `p` up to error `err`, then the - * algorithm will return a sample `x` from the DataFrame so that the *exact* rank of `x` is close - * to (p * N). More precisely, - * - * floor((p - err) * N) <= rank(x) <= ceil((p + err) * N). - * - * This method implements a variation of the Greenwald-Khanna algorithm (with some speed - * optimizations). The algorithm was first present in Space-efficient Online Computation of Quantile - * Summaries by Greenwald and Khanna. - * - * @param df - * the dataframe - * @param cols - * numerical columns of the dataframe - * @param probabilities - * a list of quantile probabilities Each number must belong to [0, 1]. For example 0 is the - * minimum, 0.5 is the median, 1 is the maximum. - * @param relativeError - * The relative target precision to achieve (greater than or equal 0). If set to zero, the exact - * quantiles are computed, which could be very expensive. Note that values greater than 1 are - * accepted but give the same result as 1. - * @return - * for each column, returns the requested approximations - * @note - * null and NaN values will be ignored in numerical columns before calculation. For a column - * only containing null or NaN values, an empty array is returned. - */ - def multipleApproxQuantiles( - df: DataFrame, - cols: Seq[String], - probabilities: Seq[Double], - relativeError: Double): Seq[Seq[Double]] = { - require(relativeError >= 0, s"Relative Error must be non-negative but got $relativeError") - val columns: Seq[Column] = cols.map { - colName => - val field = df.resolve(colName) - require( - field.dataType.isInstanceOf[NumericType], - s"Quantile calculation for column $colName with data type ${field.dataType}" + - " is not supported.") - Column(Cast(Column(colName).expr, DoubleType)) - } - val emptySummaries = Array.fill(cols.size)( - new QuantileSummaries(QuantileSummaries.defaultCompressThreshold, relativeError)) - - // Note that it works more or less by accident as `rdd.aggregate` is not a pure function: - // this function returns the same array as given in the input (because `aggregate` reuses - // the same argument). - def apply(summaries: Array[QuantileSummaries], row: Row): Array[QuantileSummaries] = { - var i = 0 - while (i < summaries.length) { - if (!row.isNullAt(i)) { - val v = row.getDouble(i) - if (!v.isNaN) summaries(i) = summaries(i).insert(v) - } - i += 1 - } - summaries - } - - def merge( - sum1: Array[QuantileSummaries], - sum2: Array[QuantileSummaries]): Array[QuantileSummaries] = { - sum1.zip(sum2).map { case (s1, s2) => s1.compress().merge(s2.compress()) } - } - - val summaries = df.select(columns: _*).rdd.treeAggregate(emptySummaries)(apply, merge) - - summaries.map { - summary => - summary.query(probabilities) match { - case Some(q) => q - case None => Seq() - } - } - } - - /** Calculate the Pearson Correlation Coefficient for the given columns */ - def pearsonCorrelation(df: DataFrame, cols: Seq[String]): Double = { - val counts = collectStatisticalData(df, cols, "correlation") - counts.Ck / math.sqrt(counts.MkX * counts.MkY) - } - - /** Helper class to simplify tracking and merging counts. */ - private class CovarianceCounter extends Serializable { - var xAvg = 0.0 // the mean of all examples seen so far in col1 - var yAvg = 0.0 // the mean of all examples seen so far in col2 - var Ck = 0.0 // the co-moment after k examples - var MkX = 0.0 // sum of squares of differences from the (current) mean for col1 - var MkY = 0.0 // sum of squares of differences from the (current) mean for col2 - var count = 0L // count of observed examples - - // add an example to the calculation - def add(x: Double, y: Double): this.type = { - val deltaX = x - xAvg - val deltaY = y - yAvg - count += 1 - xAvg += deltaX / count - yAvg += deltaY / count - Ck += deltaX * (y - yAvg) - MkX += deltaX * (x - xAvg) - MkY += deltaY * (y - yAvg) - this - } - - // merge counters from other partitions. Formula can be found at: - // http://en.wikipedia.org/wiki/Algorithms_for_calculating_variance - def merge(other: CovarianceCounter): this.type = { - if (other.count > 0) { - val totalCount = count + other.count - val deltaX = xAvg - other.xAvg - val deltaY = yAvg - other.yAvg - Ck += other.Ck + deltaX * deltaY * count / totalCount * other.count - xAvg = (xAvg * count + other.xAvg * other.count) / totalCount - yAvg = (yAvg * count + other.yAvg * other.count) / totalCount - MkX += other.MkX + deltaX * deltaX * count / totalCount * other.count - MkY += other.MkY + deltaY * deltaY * count / totalCount * other.count - count = totalCount - } - this - } - - // return the sample covariance for the observed examples - def cov: Double = Ck / (count - 1) - } - - private def collectStatisticalData( - df: DataFrame, - cols: Seq[String], - functionName: String): CovarianceCounter = { - require( - cols.length == 2, - s"Currently $functionName calculation is supported " + - "between two columns.") - cols.map(name => (name, df.resolve(name))).foreach { - case (name, data) => - require( - data.dataType.isInstanceOf[NumericType], - s"Currently $functionName calculation " + - s"for columns with dataType ${data.dataType.catalogString} not supported." - ) - } - val columns = cols.map(n => Column(Cast(Column(n).expr, DoubleType))) - df.select(columns: _*) - .queryExecution - .toRdd - .treeAggregate(new CovarianceCounter)( - seqOp = (counter, row) => { - counter.add(row.getDouble(0), row.getDouble(1)) - }, - combOp = (baseCounter, other) => { - baseCounter.merge(other) - }) - } - - /** - * Calculate the covariance of two numerical columns of a DataFrame. - * - * @param df - * The DataFrame - * @param cols - * the column names - * @return - * the covariance of the two columns. - */ - def calculateCov(df: DataFrame, cols: Seq[String]): Double = { - val counts = collectStatisticalData(df, cols, "covariance") - counts.cov - } - - /** Generate a table of frequencies for the elements of two columns. */ - def crossTabulate(df: DataFrame, col1: String, col2: String): DataFrame = { - val tableName = s"${col1}_$col2" - val counts = df.groupBy(col1, col2).agg(count("*")).take(1e6.toInt) - if (counts.length == 1e6.toInt) { - logWarning( - "The maximum limit of 1e6 pairs have been collected, which may not be all of " + - "the pairs. Please try reducing the amount of distinct items in your columns.") - } - - def cleanElement(element: Any): String = { - if (element == null) "null" else element.toString - } - - // get the distinct sorted values of column 2, so that we can make them the column names - val distinctCol2: Map[Any, Int] = - counts.map(e => cleanElement(e.get(1))).distinct.sorted.zipWithIndex.toMap - val columnSize = distinctCol2.size - require( - columnSize < 1e4, - s"The number of distinct values for $col2, can't " + - s"exceed 1e4. Currently $columnSize") - val table = counts - .groupBy(_.get(0)) - .map { - case (col1Item, rows) => - val countsRow = new GenericInternalRow(columnSize + 1) - rows.foreach { - (row: Row) => - // row.get(0) is column 1 - // row.get(1) is column 2 - // row.get(2) is the frequency - val columnIndex = distinctCol2(cleanElement(row.get(1))) - countsRow.setLong(columnIndex + 1, row.getLong(2)) - } - // the value of col1 is the first value, the rest are the counts - countsRow.update(0, UTF8String.fromString(cleanElement(col1Item))) - countsRow - } - .toSeq - - // Back ticks can't exist in DataFrame column names, therefore drop them. To be able to accept - // special keywords and `.`, wrap the column names in ``. - def cleanColumnName(name: String): String = { - name.replace("`", "") - } - - // In the map, the column names (._1) are not ordered by the index (._2). This was the bug in - // SPARK-8681. We need to explicitly sort by the column index and assign the column names. - val headerNames = distinctCol2.toSeq.sortBy(_._2).map { - r => StructField(cleanColumnName(r._1.toString), LongType) - } - val schema = StructType(StructField(tableName, StringType) +: headerNames) - - Dataset.ofRows(df.sparkSession, LocalRelation(schema.toAttributes, table)).na.fill(0.0) - } - - /** Calculate selected summary statistics for a dataset */ - def summary(ds: Dataset[_], statistics: Seq[String]): DataFrame = { - - val defaultStatistics = Seq("count", "mean", "stddev", "min", "25%", "50%", "75%", "max") - val selectedStatistics = if (statistics.nonEmpty) statistics else defaultStatistics - - val percentiles = selectedStatistics.filter(a => a.endsWith("%")).map { - p => - try { - p.stripSuffix("%").toDouble / 100.0 - } catch { - case e: NumberFormatException => - throw QueryExecutionErrors.cannotParseStatisticAsPercentileError(p, e) - } - } - require(percentiles.forall(p => p >= 0 && p <= 1), "Percentiles must be in the range [0, 1]") - - def castAsDoubleIfNecessary(e: Expression): Expression = if (e.dataType == StringType) { - TryCast(e, DoubleType) - } else { - e - } - - var percentileIndex = 0 - val statisticFns = selectedStatistics.map { - stats => - if (stats.endsWith("%")) { - val index = percentileIndex - percentileIndex += 1 - (child: Expression) => - GetArrayItem( - new ApproximatePercentile( - castAsDoubleIfNecessary(child), - Literal(new GenericArrayData(percentiles), ArrayType(DoubleType, false))) - .toAggregateExpression(), - Literal(index) - ) - } else { - stats.toLowerCase(Locale.ROOT) match { - case "count" => (child: Expression) => Count(child).toAggregateExpression() - case "count_distinct" => - (child: Expression) => Count(child).toAggregateExpression(isDistinct = true) - case "approx_count_distinct" => - (child: Expression) => HyperLogLogPlusPlus(child).toAggregateExpression() - case "mean" => - (child: Expression) => Average(castAsDoubleIfNecessary(child)).toAggregateExpression() - case "stddev" => - (child: Expression) => - StddevSamp(castAsDoubleIfNecessary(child)).toAggregateExpression() - case "min" => (child: Expression) => Min(child).toAggregateExpression() - case "max" => (child: Expression) => Max(child).toAggregateExpression() - case _ => throw QueryExecutionErrors.statisticNotRecognizedError(stats) - } - } - } - - val selectedCols = ds.logicalPlan.output - .filter(a => a.dataType.isInstanceOf[NumericType] || a.dataType.isInstanceOf[StringType]) - - val aggExprs = statisticFns.flatMap { - func => selectedCols.map(c => Column(Cast(func(c), StringType)).as(c.name)) - } - - // If there is no selected columns, we don't need to run this aggregate, so make it a lazy val. - lazy val aggResult = ds.select(aggExprs: _*).queryExecution.toRdd.map(_.copy()).collect().head - - // We will have one row for each selected statistic in the result. - val result = Array.fill[InternalRow](selectedStatistics.length) { - // each row has the statistic name, and statistic values of each selected column. - new GenericInternalRow(selectedCols.length + 1) - } - - var rowIndex = 0 - while (rowIndex < result.length) { - val statsName = selectedStatistics(rowIndex) - result(rowIndex).update(0, UTF8String.fromString(statsName)) - for (colIndex <- selectedCols.indices) { - val statsValue = aggResult.getUTF8String(rowIndex * selectedCols.length + colIndex) - result(rowIndex).update(colIndex + 1, statsValue) - } - rowIndex += 1 - } - - // All columns are string type - val output = AttributeReference("summary", StringType)() +: - selectedCols.map(c => AttributeReference(c.name, StringType)()) - - Dataset.ofRows(ds.sparkSession, LocalRelation(output, result)) - } -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/ui/TypeAlias.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/ui/TypeAlias.scala deleted file mode 100644 index 5abb70bed04..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/ui/TypeAlias.scala +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.ui - -/** - * Ensures compatibility for the type HttpServletRequest across Spark 4.0 and earlier versions. - * Starting from Spark 4.0, `jakarta.servlet.http.HttpServletRequest` is used. - */ -object TypeAlias { - type HttpServletRequest = javax.servlet.http.HttpServletRequest -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/hive/execution/AbstractHiveTableScanExec.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/hive/execution/AbstractHiveTableScanExec.scala deleted file mode 100644 index 8b2a54a0649..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/hive/execution/AbstractHiveTableScanExec.scala +++ /dev/null @@ -1,259 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.spark.rdd.RDD -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.analysis.CastSupport -import org.apache.spark.sql.catalyst.catalog.HiveTableRelation -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.expressions.SchemaPruning.RootField -import org.apache.spark.sql.execution._ -import org.apache.spark.sql.execution.metric.SQLMetrics -import org.apache.spark.sql.hive._ -import org.apache.spark.sql.hive.client.HiveClientImpl -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.{BooleanType, DataType, StructType} -import org.apache.spark.util.Utils - -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.hive.ql.metadata.{Partition => HivePartition} -import org.apache.hadoop.hive.ql.plan.TableDesc -import org.apache.hadoop.hive.serde.serdeConstants -import org.apache.hadoop.hive.serde2.objectinspector._ -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils - -import scala.collection.JavaConverters._ - -/** - * The Hive table scan operator. Column and partition pruning are both handled. - * - * @param requestedAttributes - * Attributes to be fetched from the Hive table. - * @param relation - * The Hive table be scanned. - * @param partitionPruningPred - * An optional partition pruning predicate for partitioned table. - * @param prunedOutput - * The pruned output. - */ -abstract private[hive] class AbstractHiveTableScanExec( - requestedAttributes: Seq[Attribute], - relation: HiveTableRelation, - partitionPruningPred: Seq[Expression], - prunedOutput: Seq[Attribute] = Seq.empty[Attribute])( - @transient protected val sparkSession: SparkSession) - extends LeafExecNode - with CastSupport { - - require( - partitionPruningPred.isEmpty || relation.isPartitioned, - "Partition pruning predicates only supported for partitioned tables.") - - override def conf: SQLConf = sparkSession.sessionState.conf - - override def nodeName: String = s"Scan hive ${relation.tableMeta.qualifiedName}" - - override lazy val metrics = Map( - "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows")) - - override def producedAttributes: AttributeSet = outputSet ++ - AttributeSet(partitionPruningPred.flatMap(_.references)) - - private val originalAttributes = AttributeMap(relation.output.map(a => a -> a)) - - override def output: Seq[Attribute] = { - if (prunedOutput.nonEmpty) { - prunedOutput - } else { - // Retrieve the original attributes based on expression ID so that capitalization matches. - requestedAttributes.map(attr => originalAttributes.getOrElse(attr, attr)).distinct - } - } - - // Bind all partition key attribute references in the partition pruning predicate for later - // evaluation. - private lazy val boundPruningPred = partitionPruningPred.reduceLeftOption(And).map { - pred => - require( - pred.dataType == BooleanType, - s"Data type of predicate $pred must be ${BooleanType.catalogString} rather than " + - s"${pred.dataType.catalogString}.") - - BindReferences.bindReference(pred, relation.partitionCols) - } - - @transient private lazy val hiveQlTable = HiveClientImpl.toHiveTable(relation.tableMeta) - @transient private lazy val tableDesc = new TableDesc( - hiveQlTable.getInputFormatClass, - hiveQlTable.getOutputFormatClass, - hiveQlTable.getMetadata) - - // Create a local copy of hadoopConf,so that scan specific modifications should not impact - // other queries - @transient private lazy val hadoopConf = { - val c = sparkSession.sessionState.newHadoopConf() - // append columns ids and names before broadcast - addColumnMetadataToConf(c) - c - } - - @transient private lazy val hadoopReader = - new HadoopTableReader(output, relation.partitionCols, tableDesc, sparkSession, hadoopConf) - - private def castFromString(value: String, dataType: DataType) = { - cast(Literal(value), dataType).eval(null) - } - - private def addColumnMetadataToConf(hiveConf: Configuration): Unit = { - // Specifies needed column IDs for those non-partitioning columns. - val columnOrdinals = AttributeMap(relation.dataCols.zipWithIndex) - val neededColumnIDs = output.flatMap(columnOrdinals.get).map(o => o: Integer) - val neededColumnNames = output.filter(columnOrdinals.contains).map(_.name) - - HiveShim.appendReadColumns(hiveConf, neededColumnIDs, neededColumnNames) - - val deserializer = tableDesc.getDeserializerClass.getConstructor().newInstance() - deserializer.initialize(hiveConf, tableDesc.getProperties) - - // Specifies types and object inspectors of columns to be scanned. - val structOI = ObjectInspectorUtils - .getStandardObjectInspector(deserializer.getObjectInspector, ObjectInspectorCopyOption.JAVA) - .asInstanceOf[StructObjectInspector] - - val columnTypeNames = structOI.getAllStructFieldRefs.asScala - .map(_.getFieldObjectInspector) - .map(TypeInfoUtils.getTypeInfoFromObjectInspector(_).getTypeName) - .mkString(",") - - hiveConf.set(serdeConstants.LIST_COLUMN_TYPES, columnTypeNames) - hiveConf.set(serdeConstants.LIST_COLUMNS, relation.dataCols.map(_.name).mkString(",")) - } - - /** - * Prunes partitions not involve the query plan. - * - * @param partitions - * All partitions of the relation. - * @return - * Partitions that are involved in the query plan. - */ - private[hive] def prunePartitions(partitions: Seq[HivePartition]): Seq[HivePartition] = { - boundPruningPred match { - case None => partitions - case Some(shouldKeep) => - partitions.filter { - part => - val dataTypes = relation.partitionCols.map(_.dataType) - val castedValues = part.getValues.asScala - .zip(dataTypes) - .map { case (value, dataType) => castFromString(value, dataType) } - - // Only partitioned values are needed here, since the predicate has - // already been bound to partition key attribute references. - val row = InternalRow.fromSeq(castedValues.toSeq) - shouldKeep.eval(row).asInstanceOf[Boolean] - } - } - } - - // This is used on the driver side, so it is important to avoid executing subqueries - @transient lazy val basePrunedPartitions: Seq[HivePartition] = { - if (relation.prunedPartitions.nonEmpty) { - relation.prunedPartitions.get.map(HiveClientImpl.toHivePartition(_, hiveQlTable)) - } else { - rawPartitions - } - } - - @transient lazy val prunedPartitions: Seq[HivePartition] = - if (relation.prunedPartitions.nonEmpty) { - if (partitionPruningPred.forall(!ExecSubqueryExpression.hasSubquery(_))) { - basePrunedPartitions - } else { - prunePartitions(basePrunedPartitions) - } - } else if ( - sparkSession.sessionState.conf.metastorePartitionPruning && - partitionPruningPred.nonEmpty - ) { - basePrunedPartitions - } else { - prunePartitions(basePrunedPartitions) - } - - // exposed for tests - @transient lazy val rawPartitions: Seq[HivePartition] = { - val prunedPartitions = - if ( - sparkSession.sessionState.conf.metastorePartitionPruning && - partitionPruningPred.nonEmpty - ) { - // Retrieve the original attributes based on expression ID so that capitalization matches. - val normalizedFilters = partitionPruningPred.map(_.transform { - case a: AttributeReference => originalAttributes(a) - }) - sparkSession.sessionState.catalog - .listPartitionsByFilter(relation.tableMeta.identifier, normalizedFilters) - } else { - sparkSession.sessionState.catalog.listPartitions(relation.tableMeta.identifier) - } - prunedPartitions.map(HiveClientImpl.toHivePartition(_, hiveQlTable)) - } - - override protected def doExecute(): RDD[InternalRow] = { - // Using dummyCallSite, as getCallSite can turn out to be expensive with - // multiple partitions. - val rdd = if (!relation.isPartitioned) { - Utils.withDummyCallSite(sparkContext) { - hadoopReader.makeRDDForTable(hiveQlTable) - } - } else { - Utils.withDummyCallSite(sparkContext) { - hadoopReader.makeRDDForPartitionedTable(prunedPartitions) - } - } - val numOutputRows = longMetric("numOutputRows") - // Avoid to serialize MetastoreRelation because schema is lazy. (see SPARK-15649) - val outputSchema = schema - rdd.mapPartitionsWithIndexInternal { - (index, iter) => - val proj = UnsafeProjection.create(outputSchema) - proj.initialize(index) - iter.map { - r => - numOutputRows += 1 - proj(r) - } - } - } - - // Filters unused DynamicPruningExpression expressions - one which has been replaced - // with DynamicPruningExpression(Literal.TrueLiteral) during Physical Planning - private def filterUnusedDynamicPruningExpressions( - predicates: Seq[Expression]): Seq[Expression] = { - predicates.filterNot(_ == DynamicPruningExpression(Literal.TrueLiteral)) - } - - override def otherCopyArgs: Seq[AnyRef] = Seq(sparkSession) - - def pruneSchema(schema: StructType, requestedFields: Seq[RootField]): StructType = { - SchemaPruning.pruneSchema(schema, requestedFields) - } -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/hive/execution/HiveFileFormat.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/hive/execution/HiveFileFormat.scala deleted file mode 100644 index 23a752c87f6..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/hive/execution/HiveFileFormat.scala +++ /dev/null @@ -1,235 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.hive.execution - -import org.apache.gluten.execution.datasource.GlutenFormatFactory - -import org.apache.spark.internal.Logging -import org.apache.spark.internal.config.SPECULATION_ENABLED -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.errors.QueryExecutionErrors -import org.apache.spark.sql.execution.datasources.{FileFormat, OutputWriter, OutputWriterFactory} -import org.apache.spark.sql.execution.datasources.orc.OrcOptions -import org.apache.spark.sql.execution.datasources.parquet.ParquetOptions -import org.apache.spark.sql.hive.{HiveInspectors, HiveTableUtil} -import org.apache.spark.sql.hive.HiveShim.{ShimFileSinkDesc => FileSinkDesc} -import org.apache.spark.sql.sources.DataSourceRegister -import org.apache.spark.sql.types.StructType -import org.apache.spark.util.SerializableJobConf - -import org.apache.hadoop.fs.{FileStatus, Path} -import org.apache.hadoop.hive.ql.exec.Utilities -import org.apache.hadoop.hive.ql.io.{HiveFileFormatUtils, HiveOutputFormat} -import org.apache.hadoop.hive.serde2.Serializer -import org.apache.hadoop.hive.serde2.objectinspector.{ObjectInspectorUtils, StructObjectInspector} -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils -import org.apache.hadoop.io.Writable -import org.apache.hadoop.mapred.{JobConf, Reporter} -import org.apache.hadoop.mapreduce.{Job, TaskAttemptContext} -import org.apache.parquet.hadoop.ParquetOutputFormat -import org.apache.parquet.hadoop.metadata.CompressionCodecName - -import scala.collection.JavaConverters._ - -/** - * `FileFormat` for writing Hive tables. - * - * TODO: implement the read logic. - */ -class HiveFileFormat(fileSinkConf: FileSinkDesc) - extends FileFormat - with DataSourceRegister - with Logging { - - def this() = this(null) - - override def shortName(): String = "hive" - - override def inferSchema( - sparkSession: SparkSession, - options: Map[String, String], - files: Seq[FileStatus]): Option[StructType] = { - throw QueryExecutionErrors.inferSchemaUnsupportedForHiveError() - } - - override def prepareWrite( - sparkSession: SparkSession, - job: Job, - options: Map[String, String], - dataSchema: StructType): OutputWriterFactory = { - val conf = job.getConfiguration - val tableDesc = fileSinkConf.getTableInfo - conf.set("mapred.output.format.class", tableDesc.getOutputFileFormatClassName) - - // When speculation is on and output committer class name contains "Direct", we should warn - // users that they may loss data if they are using a direct output committer. - val speculationEnabled = sparkSession.sparkContext.conf.get(SPECULATION_ENABLED) - val outputCommitterClass = conf.get("mapred.output.committer.class", "") - if (speculationEnabled && outputCommitterClass.contains("Direct")) { - val warningMessage = - s"$outputCommitterClass may be an output committer that writes data directly to " + - "the final location. Because speculation is enabled, this output committer may " + - "cause data loss (see the case in SPARK-10063). If possible, please use an output " + - "committer that does not have this behavior (e.g. FileOutputCommitter)." - logWarning(warningMessage) - } - - // Add table properties from storage handler to hadoopConf, so any custom storage - // handler settings can be set to hadoopConf - HiveTableUtil.configureJobPropertiesForStorageHandler(tableDesc, conf, false) - Utilities.copyTableJobPropertiesToConf(tableDesc, conf) - - // Avoid referencing the outer object. - val fileSinkConfSer = fileSinkConf - val outputFormat = fileSinkConf.tableInfo.getOutputFileFormatClassName - if ("true" == sparkSession.sparkContext.getLocalProperty("isNativeApplicable")) { - val nativeFormat = sparkSession.sparkContext.getLocalProperty("nativeFormat") - val tableOptions = tableDesc.getProperties.asScala.toMap - val compressionCodec = nativeFormat match { - case "parquet" if fileSinkConf.compressed => - // MapredParquetOutputFormat use the `ParquetOutputFormat.COMPRESSION` as - // the compression codec. - tableOptions.getOrElse( - ParquetOutputFormat.COMPRESSION, - conf.get(ParquetOutputFormat.COMPRESSION, CompressionCodecName.UNCOMPRESSED.name)) - case "parquet" => - val parquetOptions = - new ParquetOptions(tableOptions, sparkSession.sessionState.conf) - parquetOptions.compressionCodecClassName - case _ => - if (fileSinkConf.compressed) { - fileSinkConf.compressCodec - } else { - val orcOptions = new OrcOptions(tableOptions, sparkSession.sessionState.conf) - orcOptions.compressionCodec - } - } - - val nativeConf = - GlutenFormatFactory(nativeFormat).nativeConf(tableOptions, compressionCodec) - - new OutputWriterFactory { - private val jobConf = new SerializableJobConf(new JobConf(conf)) - @transient private lazy val outputFormat = - jobConf.value.getOutputFormat.asInstanceOf[HiveOutputFormat[AnyRef, Writable]] - - override def getFileExtension(context: TaskAttemptContext): String = { - Utilities.getFileExtension(jobConf.value, fileSinkConfSer.getCompressed, outputFormat) - } - - override def newInstance( - path: String, - dataSchema: StructType, - context: TaskAttemptContext): OutputWriter = { - GlutenFormatFactory(nativeFormat) - .createOutputWriter(path, dataSchema, context, nativeConf) - } - } - } else { - new OutputWriterFactory { - private val jobConf = new SerializableJobConf(new JobConf(conf)) - @transient private lazy val outputFormat = - jobConf.value.getOutputFormat.asInstanceOf[HiveOutputFormat[AnyRef, Writable]] - - override def getFileExtension(context: TaskAttemptContext): String = { - Utilities.getFileExtension(jobConf.value, fileSinkConfSer.getCompressed, outputFormat) - } - - override def newInstance( - path: String, - dataSchema: StructType, - context: TaskAttemptContext): OutputWriter = { - new HiveOutputWriter(path, fileSinkConfSer, jobConf.value, dataSchema) - } - } - } - } - - override def supportFieldName(name: String): Boolean = { - fileSinkConf.getTableInfo.getOutputFileFormatClassName match { - case "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat" => - !name.matches(".*[ ,;{}()\n\t=].*") - case "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat" => - try { - TypeInfoUtils.getTypeInfoFromTypeString(s"struct<$name:int>") - true - } catch { - case _: IllegalArgumentException => false - } - case _ => true - } - } -} - -class HiveOutputWriter( - val path: String, - fileSinkConf: FileSinkDesc, - jobConf: JobConf, - dataSchema: StructType) - extends OutputWriter - with HiveInspectors { - - private def tableDesc = fileSinkConf.getTableInfo - - private val serializer = { - val serializer = - tableDesc.getDeserializerClass.getConstructor().newInstance().asInstanceOf[Serializer] - serializer.initialize(jobConf, tableDesc.getProperties) - serializer - } - - private val hiveWriter = HiveFileFormatUtils.getHiveRecordWriter( - jobConf, - tableDesc, - serializer.getSerializedClass, - fileSinkConf, - new Path(path), - Reporter.NULL) - - /** - * Since SPARK-30201 ObjectInspectorCopyOption.JAVA change to ObjectInspectorCopyOption.DEFAULT. - * The reason is DEFAULT option can convert `UTF8String` to `Text` with bytes and we can - * compatible with non UTF-8 code bytes during write. - */ - private val standardOI = ObjectInspectorUtils - .getStandardObjectInspector( - tableDesc.getDeserializer(jobConf).getObjectInspector, - ObjectInspectorCopyOption.DEFAULT) - .asInstanceOf[StructObjectInspector] - - private val fieldOIs = - standardOI.getAllStructFieldRefs.asScala.map(_.getFieldObjectInspector).toArray - private val dataTypes = dataSchema.map(_.dataType).toArray - private val wrappers = fieldOIs.zip(dataTypes).map { case (f, dt) => wrapperFor(f, dt) } - private val outputData = new Array[Any](fieldOIs.length) - - override def write(row: InternalRow): Unit = { - var i = 0 - while (i < fieldOIs.length) { - outputData(i) = if (row.isNullAt(i)) null else wrappers(i)(row.get(i, dataTypes(i))) - i += 1 - } - hiveWriter.write(serializer.serialize(outputData, standardOI)) - } - - override def close(): Unit = { - // Seems the boolean value passed into close does not matter. - hiveWriter.close(false) - } -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/storage/GlutenPushBasedFetchHelper.scala b/shims/spark33/src/main/scala/org/apache/spark/storage/GlutenPushBasedFetchHelper.scala deleted file mode 100644 index 1d07721726d..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/storage/GlutenPushBasedFetchHelper.scala +++ /dev/null @@ -1,384 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.storage - -import org.apache.spark.MapOutputTracker -import org.apache.spark.MapOutputTracker.SHUFFLE_PUSH_MAP_ID -import org.apache.spark.internal.Logging -import org.apache.spark.network.shuffle.{BlockStoreClient, MergedBlockMeta, MergedBlocksMetaListener} -import org.apache.spark.storage.BlockManagerId.SHUFFLE_MERGER_IDENTIFIER -import org.apache.spark.storage.ShuffleBlockFetcherIterator._ - -import org.roaringbitmap.RoaringBitmap - -import java.util.concurrent.TimeUnit - -import scala.collection.mutable -import scala.collection.mutable.ArrayBuffer -import scala.util.{Failure, Success} - -/** - * Helper class for [[ShuffleBlockFetcherIterator]] that encapsulates all the push-based - * functionality to fetch push-merged block meta and shuffle chunks. A push-merged block contains - * multiple shuffle chunks where each shuffle chunk contains multiple shuffle blocks that belong to - * the common reduce partition and were merged by the external shuffle service to that chunk. - */ -private class GlutenPushBasedFetchHelper( - private val iterator: GlutenShuffleBlockFetcherIterator, - private val shuffleClient: BlockStoreClient, - private val blockManager: BlockManager, - private val mapOutputTracker: MapOutputTracker) extends Logging { - - private[this] val startTimeNs = System.nanoTime() - - private[storage] val localShuffleMergerBlockMgrId = BlockManagerId( - SHUFFLE_MERGER_IDENTIFIER, - blockManager.blockManagerId.host, - blockManager.blockManagerId.port, - blockManager.blockManagerId.topologyInfo) - - /** - * A map for storing shuffle chunk bitmap. - */ - private[this] val chunksMetaMap = new mutable.HashMap[ShuffleBlockChunkId, RoaringBitmap]() - - /** - * Returns true if the address is for a push-merged block. - */ - def isPushMergedShuffleBlockAddress(address: BlockManagerId): Boolean = { - SHUFFLE_MERGER_IDENTIFIER == address.executorId - } - - /** - * Returns true if the address is of a remote push-merged block. false otherwise. - */ - def isRemotePushMergedBlockAddress(address: BlockManagerId): Boolean = { - isPushMergedShuffleBlockAddress(address) && address.host != blockManager.blockManagerId.host - } - - /** - * Returns true if the address is of a push-merged-local block. false otherwise. - */ - def isLocalPushMergedBlockAddress(address: BlockManagerId): Boolean = { - isPushMergedShuffleBlockAddress(address) && address.host == blockManager.blockManagerId.host - } - - /** - * This is executed by the task thread when the `iterator.next()` is invoked and the iterator - * processes a response of type [[ShuffleBlockFetcherIterator.SuccessFetchResult]]. - * - * @param blockId - * shuffle chunk id. - */ - def removeChunk(blockId: ShuffleBlockChunkId): Unit = { - chunksMetaMap.remove(blockId) - } - - /** - * This is executed by the task thread when the `iterator.next()` is invoked and the iterator - * processes a response of type [[ShuffleBlockFetcherIterator.PushMergedLocalMetaFetchResult]]. - * - * @param blockId - * shuffle chunk id. - */ - def addChunk(blockId: ShuffleBlockChunkId, chunkMeta: RoaringBitmap): Unit = { - chunksMetaMap(blockId) = chunkMeta - } - - /** - * This is executed by the task thread when the `iterator.next()` is invoked and the iterator - * processes a response of type [[ShuffleBlockFetcherIterator.PushMergedRemoteMetaFetchResult]]. - * - * @param shuffleId - * shuffle id. - * @param reduceId - * reduce id. - * @param blockSize - * size of the push-merged block. - * @param bitmaps - * chunk bitmaps, where each bitmap contains all the mapIds that were merged to that chunk. - * @return - * shuffle chunks to fetch. - */ - def createChunkBlockInfosFromMetaResponse( - shuffleId: Int, - shuffleMergeId: Int, - reduceId: Int, - blockSize: Long, - bitmaps: Array[RoaringBitmap]): ArrayBuffer[(BlockId, Long, Int)] = { - val approxChunkSize = blockSize / bitmaps.length - val blocksToFetch = new ArrayBuffer[(BlockId, Long, Int)]() - for (i <- bitmaps.indices) { - val blockChunkId = ShuffleBlockChunkId(shuffleId, shuffleMergeId, reduceId, i) - chunksMetaMap.put(blockChunkId, bitmaps(i)) - logDebug(s"adding block chunk $blockChunkId of size $approxChunkSize") - blocksToFetch += ((blockChunkId, approxChunkSize, SHUFFLE_PUSH_MAP_ID)) - } - blocksToFetch - } - - /** - * This is executed by the task thread when the iterator is initialized and only if it has - * push-merged blocks for which it needs to fetch the metadata. - * - * @param req - * [[ShuffleBlockFetcherIterator.FetchRequest]] that only contains requests to fetch metadata of - * push-merged blocks. - */ - def sendFetchMergedStatusRequest(req: FetchRequest): Unit = { - val sizeMap = req.blocks.map { - case FetchBlockInfo(blockId, size, _) => - val shuffleBlockId = blockId.asInstanceOf[ShuffleMergedBlockId] - ((shuffleBlockId.shuffleId, shuffleBlockId.reduceId), size) - }.toMap - val address = req.address - val mergedBlocksMetaListener = new MergedBlocksMetaListener { - override def onSuccess( - shuffleId: Int, - shuffleMergeId: Int, - reduceId: Int, - meta: MergedBlockMeta): Unit = { - logDebug(s"Received the meta of push-merged block for ($shuffleId, $shuffleMergeId," + - s" $reduceId) from ${req.address.host}:${req.address.port}") - try { - iterator.addToResultsQueue(PushMergedRemoteMetaFetchResult( - shuffleId, - shuffleMergeId, - reduceId, - sizeMap((shuffleId, reduceId)), - meta.readChunkBitmaps(), - address)) - } catch { - case exception: Exception => - logError( - s"Failed to parse the meta of push-merged block for ($shuffleId, " + - s"$shuffleMergeId, $reduceId) from" + - s" ${req.address.host}:${req.address.port}", - exception - ) - iterator.addToResultsQueue( - PushMergedRemoteMetaFailedFetchResult( - shuffleId, - shuffleMergeId, - reduceId, - address)) - } - } - - override def onFailure( - shuffleId: Int, - shuffleMergeId: Int, - reduceId: Int, - exception: Throwable): Unit = { - logError( - s"Failed to get the meta of push-merged block for ($shuffleId, $reduceId) " + - s"from ${req.address.host}:${req.address.port}", - exception) - iterator.addToResultsQueue( - PushMergedRemoteMetaFailedFetchResult(shuffleId, shuffleMergeId, reduceId, address)) - } - } - req.blocks.foreach { - block => - val shuffleBlockId = block.blockId.asInstanceOf[ShuffleMergedBlockId] - shuffleClient.getMergedBlockMeta( - address.host, - address.port, - shuffleBlockId.shuffleId, - shuffleBlockId.shuffleMergeId, - shuffleBlockId.reduceId, - mergedBlocksMetaListener) - } - } - - /** - * This is executed by the task thread when the iterator is initialized. It fetches all the - * outstanding push-merged local blocks. - * @param pushMergedLocalBlocks - * set of identified merged local blocks and their sizes. - */ - def fetchAllPushMergedLocalBlocks( - pushMergedLocalBlocks: mutable.LinkedHashSet[BlockId]): Unit = { - if (pushMergedLocalBlocks.nonEmpty) { - blockManager.hostLocalDirManager.foreach(fetchPushMergedLocalBlocks(_, pushMergedLocalBlocks)) - } - } - - /** - * Fetch the push-merged blocks dirs if they are not in the cache and eventually fetch push-merged - * local blocks. - */ - private def fetchPushMergedLocalBlocks( - hostLocalDirManager: HostLocalDirManager, - pushMergedLocalBlocks: mutable.LinkedHashSet[BlockId]): Unit = { - val cachedPushedMergedDirs = hostLocalDirManager.getCachedHostLocalDirsFor( - SHUFFLE_MERGER_IDENTIFIER) - if (cachedPushedMergedDirs.isDefined) { - logDebug(s"Fetch the push-merged-local blocks with cached merged dirs: " + - s"${cachedPushedMergedDirs.get.mkString(", ")}") - pushMergedLocalBlocks.foreach { - blockId => - fetchPushMergedLocalBlock( - blockId, - cachedPushedMergedDirs.get, - localShuffleMergerBlockMgrId) - } - } else { - // Push-based shuffle is only enabled when the external shuffle service is enabled. If the - // external shuffle service is not enabled, then there will not be any push-merged blocks - // for the iterator to fetch. - logDebug(s"Asynchronous fetch the push-merged-local blocks without cached merged " + - s"dirs from the external shuffle service") - hostLocalDirManager.getHostLocalDirs( - blockManager.blockManagerId.host, - blockManager.externalShuffleServicePort, - Array(SHUFFLE_MERGER_IDENTIFIER)) { - case Success(dirs) => - logDebug(s"Fetched merged dirs in " + - s"${TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNs)} ms") - pushMergedLocalBlocks.foreach { - blockId => - logDebug(s"Successfully fetched local dirs: " + - s"${dirs.get(SHUFFLE_MERGER_IDENTIFIER).mkString(", ")}") - fetchPushMergedLocalBlock( - blockId, - dirs(SHUFFLE_MERGER_IDENTIFIER), - localShuffleMergerBlockMgrId) - } - case Failure(throwable) => - // If we see an exception with getting the local dirs for push-merged-local blocks, - // we fallback to fetch the original blocks. We do not report block fetch failure. - logWarning( - s"Error while fetching the merged dirs for push-merged-local " + - s"blocks: ${pushMergedLocalBlocks.mkString(", ")}. Fetch the original blocks instead", - throwable - ) - pushMergedLocalBlocks.foreach { - blockId => - iterator.addToResultsQueue(FallbackOnPushMergedFailureResult( - blockId, - localShuffleMergerBlockMgrId, - 0, - isNetworkReqDone = false)) - } - } - } - } - - /** - * Fetch a single push-merged-local block generated. This can also be executed by the task thread - * as well as the netty thread. - * @param blockId - * ShuffleBlockId to be fetched - * @param localDirs - * Local directories where the push-merged shuffle files are stored - * @param blockManagerId - * BlockManagerId - */ - private[this] def fetchPushMergedLocalBlock( - blockId: BlockId, - localDirs: Array[String], - blockManagerId: BlockManagerId): Unit = { - try { - val shuffleBlockId = blockId.asInstanceOf[ShuffleMergedBlockId] - val chunksMeta = blockManager.getLocalMergedBlockMeta(shuffleBlockId, localDirs) - iterator.addToResultsQueue(PushMergedLocalMetaFetchResult( - shuffleBlockId.shuffleId, - shuffleBlockId.shuffleMergeId, - shuffleBlockId.reduceId, - chunksMeta.readChunkBitmaps(), - localDirs)) - } catch { - case e: Exception => - // If we see an exception with reading a push-merged-local meta, we fallback to - // fetch the original blocks. We do not report block fetch failure - // and will continue with the remaining local block read. - logWarning( - s"Error occurred while fetching push-merged-local meta, " + - s"prepare to fetch the original blocks", - e) - iterator.addToResultsQueue( - FallbackOnPushMergedFailureResult(blockId, blockManagerId, 0, isNetworkReqDone = false)) - } - } - - /** - * This is executed by the task thread when the `iterator.next()` is invoked and the iterator - * processes a response of type: 1) [[ShuffleBlockFetcherIterator.SuccessFetchResult]] 2) - * [[ShuffleBlockFetcherIterator.FallbackOnPushMergedFailureResult]] 3) - * [[ShuffleBlockFetcherIterator.PushMergedRemoteMetaFailedFetchResult]] - * - * This initiates fetching fallback blocks for a push-merged block or a shuffle chunk that failed - * to fetch. It makes a call to the map output tracker to get the list of original blocks for the - * given push-merged block/shuffle chunk, split them into remote and local blocks, and process - * them accordingly. It also updates the numberOfBlocksToFetch in the iterator as it processes - * failed response and finds more push-merged requests to remote and again updates it with - * additional requests for original blocks. The fallback happens when: - * 1. There is an exception while creating shuffle chunks from push-merged-local shuffle block. - * See fetchLocalBlock. - * 2. There is a failure when fetching remote shuffle chunks. - * 3. There is a failure when processing SuccessFetchResult which is for a shuffle chunk (local - * or remote). - * 4. There is a zero-size buffer when processing SuccessFetchResult for a shuffle chunk (local - * or remote). - */ - def initiateFallbackFetchForPushMergedBlock( - blockId: BlockId, - address: BlockManagerId): Unit = { - assert(blockId.isInstanceOf[ShuffleMergedBlockId] || blockId.isInstanceOf[ShuffleBlockChunkId]) - logWarning(s"Falling back to fetch the original blocks for push-merged block $blockId") - // Increase the blocks processed since we will process another block in the next iteration of - // the while loop in ShuffleBlockFetcherIterator.next(). - val fallbackBlocksByAddr: Iterator[(BlockManagerId, Seq[(BlockId, Long, Int)])] = - blockId match { - case shuffleBlockId: ShuffleMergedBlockId => - iterator.decreaseNumBlocksToFetch(1) - mapOutputTracker.getMapSizesForMergeResult( - shuffleBlockId.shuffleId, - shuffleBlockId.reduceId) - case _ => - val shuffleChunkId = blockId.asInstanceOf[ShuffleBlockChunkId] - val chunkBitmap: RoaringBitmap = chunksMetaMap.remove(shuffleChunkId).get - var blocksProcessed = 1 - // When there is a failure to fetch a remote shuffle chunk, then we try to - // fallback not only for that particular remote shuffle chunk but also for all the - // pending chunks that belong to the same host. The reason for doing so is that it - // is very likely that the subsequent requests for shuffle chunks from this host will - // fail as well. Since, push-based shuffle is best effort and we try not to increase the - // delay of the fetches, we immediately fallback for all the pending shuffle chunks in the - // fetchRequests queue. - if (isRemotePushMergedBlockAddress(address)) { - // Fallback for all the pending fetch requests - val pendingShuffleChunks = iterator.removePendingChunks(shuffleChunkId, address) - pendingShuffleChunks.foreach { - pendingBlockId => - logInfo(s"Falling back immediately for shuffle chunk $pendingBlockId") - val bitmapOfPendingChunk: RoaringBitmap = chunksMetaMap.remove(pendingBlockId).get - chunkBitmap.or(bitmapOfPendingChunk) - } - // These blocks were added to numBlocksToFetch so we increment numBlocksProcessed - blocksProcessed += pendingShuffleChunks.size - } - iterator.decreaseNumBlocksToFetch(blocksProcessed) - mapOutputTracker.getMapSizesForMergeResult( - shuffleChunkId.shuffleId, - shuffleChunkId.reduceId, - chunkBitmap) - } - iterator.fallbackFetch(fallbackBlocksByAddr) - } -} diff --git a/shims/spark33/src/main/scala/org/apache/spark/storage/GlutenShuffleBlockFetcherIterator.scala b/shims/spark33/src/main/scala/org/apache/spark/storage/GlutenShuffleBlockFetcherIterator.scala deleted file mode 100644 index 982414cc152..00000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/storage/GlutenShuffleBlockFetcherIterator.scala +++ /dev/null @@ -1,1509 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.storage - -import org.apache.spark.{MapOutputTracker, TaskContext} -import org.apache.spark.MapOutputTracker.SHUFFLE_PUSH_MAP_ID -import org.apache.spark.errors.SparkCoreErrors -import org.apache.spark.internal.Logging -import org.apache.spark.network.buffer.{FileSegmentManagedBuffer, ManagedBuffer} -import org.apache.spark.network.shuffle._ -import org.apache.spark.network.shuffle.checksum.{Cause, ShuffleChecksumHelper} -import org.apache.spark.network.util.TransportConf -import org.apache.spark.shuffle.ShuffleReadMetricsReporter -import org.apache.spark.util.{TaskCompletionListener, Utils} - -import io.netty.util.internal.OutOfDirectMemoryError -import org.apache.commons.io.IOUtils - -import javax.annotation.concurrent.GuardedBy - -import java.io.{InputStream, IOException} -import java.nio.channels.ClosedByInterruptException -import java.util.concurrent.{ConcurrentHashMap, LinkedBlockingQueue, TimeUnit} -import java.util.zip.CheckedInputStream - -import scala.collection.mutable -import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet, Queue} -import scala.util.{Failure, Success} - -/** - * An iterator that fetches multiple blocks. For local blocks, it fetches from the local block - * manager. For remote blocks, it fetches them using the provided BlockTransferService. - * - * This creates an iterator of (BlockID, InputStream) tuples so the caller can handle blocks in a - * pipelined fashion as they are received. - * - * The implementation throttles the remote fetches so they don't exceed maxBytesInFlight to avoid - * using too much memory. - * - * @param context - * [[TaskContext]], used for metrics update - * @param shuffleClient - * [[BlockStoreClient]] for fetching remote blocks - * @param blockManager - * [[BlockManager]] for reading local blocks - * @param blocksByAddress - * list of blocks to fetch grouped by the [[BlockManagerId]]. For each block we also require two - * info: 1. the size (in bytes as a long field) in order to throttle the memory usage; 2. the - * mapIndex for this block, which indicate the index in the map stage. Note that zero-sized blocks - * are already excluded, which happened in - * [[org.apache.spark.MapOutputTracker.convertMapStatuses]]. - * @param mapOutputTracker - * [[MapOutputTracker]] for falling back to fetching the original blocks if we fail to fetch - * shuffle chunks when push based shuffle is enabled. - * @param streamWrapper - * A function to wrap the returned input stream. - * @param maxBytesInFlight - * max size (in bytes) of remote blocks to fetch at any given point. - * @param maxReqsInFlight - * max number of remote requests to fetch blocks at any given point. - * @param maxBlocksInFlightPerAddress - * max number of shuffle blocks being fetched at any given point for a given remote host:port. - * @param maxReqSizeShuffleToMem - * max size (in bytes) of a request that can be shuffled to memory. - * @param maxAttemptsOnNettyOOM - * The max number of a block could retry due to Netty OOM before throwing the fetch failure. - * @param detectCorrupt - * whether to detect any corruption in fetched blocks. - * @param checksumEnabled - * whether the shuffle checksum is enabled. When enabled, Spark will try to diagnose the cause of - * the block corruption. - * @param checksumAlgorithm - * the checksum algorithm that is used when calculating the checksum value for the block data. - * @param shuffleMetrics - * used to report shuffle metrics. - * @param doBatchFetch - * fetch continuous shuffle blocks from same executor in batch if the server side supports. - */ -final class GlutenShuffleBlockFetcherIterator( - context: TaskContext, - shuffleClient: BlockStoreClient, - blockManager: BlockManager, - mapOutputTracker: MapOutputTracker, - blocksByAddress: Iterator[(BlockManagerId, Seq[(BlockId, Long, Int)])], - streamWrapper: (BlockId, InputStream) => InputStream, - maxBytesInFlight: Long, - maxReqsInFlight: Int, - maxBlocksInFlightPerAddress: Int, - val maxReqSizeShuffleToMem: Long, - maxAttemptsOnNettyOOM: Int, - detectCorrupt: Boolean, - detectCorruptUseExtraMemory: Boolean, - checksumEnabled: Boolean, - checksumAlgorithm: String, - shuffleMetrics: ShuffleReadMetricsReporter, - doBatchFetch: Boolean) - extends GlutenShuffleBlockFetcherIteratorBase - with DownloadFileManager - with Logging { - - import ShuffleBlockFetcherIterator._ - - // Make remote requests at most maxBytesInFlight / 5 in length; the reason to keep them - // smaller than maxBytesInFlight is to allow multiple, parallel fetches from up to 5 - // nodes, rather than blocking on reading output from one node. - private val targetRemoteRequestSize = math.max(maxBytesInFlight / 5, 1L) - - /** - * Total number of blocks to fetch. - */ - private[this] var numBlocksToFetch = 0 - - /** - * The number of blocks processed by the caller. The iterator is exhausted when - * [[numBlocksProcessed]] == [[numBlocksToFetch]]. - */ - private[this] var numBlocksProcessed = 0 - - private[this] val startTimeNs = System.nanoTime() - - /** Host local blocks to fetch, excluding zero-sized blocks. */ - private[this] val hostLocalBlocks = scala.collection.mutable.LinkedHashSet[(BlockId, Int)]() - - /** - * A queue to hold our results. This turns the asynchronous model provided by - * [[org.apache.spark.network.BlockTransferService]] into a synchronous model (iterator). - */ - private[this] val results = new LinkedBlockingQueue[FetchResult] - - /** - * Current [[FetchResult]] being processed per thread. We track this so we can release the current - * buffer in case of a runtime exception when processing the current buffer. Using - * ConcurrentHashMap to support concurrent access from multiple threads while allowing cleanup - * from any thread. - */ - private[this] val currentResults = ConcurrentHashMap.newKeySet[SuccessFetchResult]() - - /** - * Queue of fetch requests to issue; we'll pull requests off this gradually to make sure that the - * number of bytes in flight is limited to maxBytesInFlight. - */ - private[this] val fetchRequests = new Queue[FetchRequest] - - /** - * Queue of fetch requests which could not be issued the first time they were dequeued. These - * requests are tried again when the fetch constraints are satisfied. - */ - private[this] val deferredFetchRequests = new HashMap[BlockManagerId, Queue[FetchRequest]]() - - /** Current bytes in flight from our requests */ - private[this] var bytesInFlight = 0L - - /** Current number of requests in flight */ - private[this] var reqsInFlight = 0 - - /** Current number of blocks in flight per host:port */ - private[this] val numBlocksInFlightPerAddress = new HashMap[BlockManagerId, Int]() - - /** - * Count the retry times for the blocks due to Netty OOM. The block will stop retry if retry times - * has exceeded the [[maxAttemptsOnNettyOOM]]. - */ - private[this] val blockOOMRetryCounts = new HashMap[String, Int] - - /** - * The blocks that can't be decompressed successfully, it is used to guarantee that we retry at - * most once for those corrupted blocks. - */ - private[this] val corruptedBlocks = mutable.HashSet[BlockId]() - - /** - * Whether the iterator is still active. If isZombie is true, the callback interface will no - * longer place fetched blocks into [[results]]. - */ - @GuardedBy("this") - private[this] var isZombie = false - - /** - * A set to store the files used for shuffling remote huge blocks. Files in this set will be - * deleted when cleanup. This is a layer of defensiveness against disk file leaks. - */ - @GuardedBy("this") - private[this] val shuffleFilesSet = mutable.HashSet[DownloadFile]() - - private[this] val onCompleteCallback = new GlutenShuffleFetchCompletionListener(this) - - private[this] val pushBasedFetchHelper = new GlutenPushBasedFetchHelper( - this, - shuffleClient, - blockManager, - mapOutputTracker) - - initialize() - - override def createTempFile(transportConf: TransportConf): DownloadFile = { - // we never need to do any encryption or decryption here, regardless of configs, because that - // is handled at another layer in the code. When encryption is enabled, shuffle data is written - // to disk encrypted in the first place, and sent over the network still encrypted. - new SimpleDownloadFile( - blockManager.diskBlockManager.createTempLocalBlock()._2, - transportConf) - } - - override def registerTempFileToClean(file: DownloadFile): Boolean = synchronized { - if (isZombie) { - false - } else { - shuffleFilesSet += file - true - } - } - - /** - * Mark the iterator as zombie, and release all buffers that haven't been deserialized yet. - */ - private[storage] def cleanup(): Unit = { - synchronized { - isZombie = true - } - // Release all current result buffers from all threads - while (!currentResults.isEmpty) { - currentResults.toArray(new Array[SuccessFetchResult](0)).foreach { - result => - if (currentResults.remove(result)) { - result.buf.release() - } - } - } - - // Release buffers in the results queue - val iter = results.iterator() - while (iter.hasNext) { - val result = iter.next() - result match { - case SuccessFetchResult(blockId, mapIndex, address, _, buf, _) => - if (address != blockManager.blockManagerId) { - if (hostLocalBlocks.contains(blockId -> mapIndex)) { - shuffleMetrics.incLocalBlocksFetched(1) - shuffleMetrics.incLocalBytesRead(buf.size) - } else { - shuffleMetrics.incRemoteBytesRead(buf.size) - if (buf.isInstanceOf[FileSegmentManagedBuffer]) { - shuffleMetrics.incRemoteBytesReadToDisk(buf.size) - } - shuffleMetrics.incRemoteBlocksFetched(1) - } - } - buf.release() - case _ => - } - } - shuffleFilesSet.foreach { - file => - if (!file.delete()) { - logWarning("Failed to cleanup shuffle fetch temp file " + file.path()) - } - } - } - - private[this] def sendRequest(req: FetchRequest): Unit = { - logDebug("Sending request for %d blocks (%s) from %s".format( - req.blocks.size, - Utils.bytesToString(req.size), - req.address.hostPort)) - bytesInFlight += req.size - reqsInFlight += 1 - - // so we can look up the block info of each blockID - val infoMap = req.blocks.map { - case FetchBlockInfo(blockId, size, mapIndex) => (blockId.toString, (size, mapIndex)) - }.toMap - val remainingBlocks = new HashSet[String]() ++= infoMap.keys - val deferredBlocks = new ArrayBuffer[String]() - val blockIds = req.blocks.map(_.blockId.toString) - val address = req.address - - @inline def enqueueDeferredFetchRequestIfNecessary(): Unit = { - if (remainingBlocks.isEmpty && deferredBlocks.nonEmpty) { - val blocks = deferredBlocks.map { - blockId => - val (size, mapIndex) = infoMap(blockId) - FetchBlockInfo(BlockId(blockId), size, mapIndex) - } - results.put(DeferFetchRequestResult(FetchRequest(address, blocks.toSeq))) - deferredBlocks.clear() - } - } - - val blockFetchingListener = new BlockFetchingListener { - override def onBlockFetchSuccess(blockId: String, buf: ManagedBuffer): Unit = { - // Only add the buffer to results queue if the iterator is not zombie, - // i.e. cleanup() has not been called yet. - GlutenShuffleBlockFetcherIterator.this.synchronized { - if (!isZombie) { - // Increment the ref count because we need to pass this to a different thread. - // This needs to be released after use. - buf.retain() - remainingBlocks -= blockId - blockOOMRetryCounts.remove(blockId) - results.put(new SuccessFetchResult( - BlockId(blockId), - infoMap(blockId)._2, - address, - infoMap(blockId)._1, - buf, - remainingBlocks.isEmpty)) - logDebug("remainingBlocks: " + remainingBlocks) - enqueueDeferredFetchRequestIfNecessary() - } - } - logTrace(s"Got remote block $blockId after ${Utils.getUsedTimeNs(startTimeNs)}") - } - - override def onBlockFetchFailure(blockId: String, e: Throwable): Unit = { - GlutenShuffleBlockFetcherIterator.this.synchronized { - logError(s"Failed to get block(s) from ${req.address.host}:${req.address.port}", e) - e match { - // SPARK-27991: Catch the Netty OOM and set the flag `isNettyOOMOnShuffle` (shared among - // tasks) to true as early as possible. The pending fetch requests won't be sent - // afterwards until the flag is set to false on: - // 1) the Netty free memory >= maxReqSizeShuffleToMem - // - we'll check this whenever there's a fetch request succeeds. - // 2) the number of in-flight requests becomes 0 - // - we'll check this in `fetchUpToMaxBytes` whenever it's invoked. - // Although Netty memory is shared across multiple modules, e.g., shuffle, rpc, the flag - // only takes effect for the shuffle due to the implementation simplicity concern. - // And we'll buffer the consecutive block failures caused by the OOM error until there's - // no remaining blocks in the current request. Then, we'll package these blocks into - // a same fetch request for the retry later. In this way, instead of creating the fetch - // request per block, it would help reduce the concurrent connections and data loads - // pressure at remote server. - // Note that catching OOM and do something based on it is only a workaround for - // handling the Netty OOM issue, which is not the best way towards memory management. - // We can get rid of it when we find a way to manage Netty's memory precisely. - case _: OutOfDirectMemoryError - if blockOOMRetryCounts.getOrElseUpdate(blockId, 0) < maxAttemptsOnNettyOOM => - if (!isZombie) { - val failureTimes = blockOOMRetryCounts(blockId) - blockOOMRetryCounts(blockId) += 1 - if (isNettyOOMOnShuffle.compareAndSet(false, true)) { - // The fetcher can fail remaining blocks in batch for the same error. So we only - // log the warning once to avoid flooding the logs. - logInfo(s"Block $blockId has failed $failureTimes times " + - s"due to Netty OOM, will retry") - } - remainingBlocks -= blockId - deferredBlocks += blockId - enqueueDeferredFetchRequestIfNecessary() - } - - case _ => - val block = BlockId(blockId) - if (block.isShuffleChunk) { - remainingBlocks -= blockId - results.put(FallbackOnPushMergedFailureResult( - block, - address, - infoMap(blockId)._1, - remainingBlocks.isEmpty)) - } else { - results.put(FailureFetchResult(block, infoMap(blockId)._2, address, e)) - } - } - } - } - } - - // Fetch remote shuffle blocks to disk when the request is too large. Since the shuffle data is - // already encrypted and compressed over the wire(w.r.t. the related configs), we can just fetch - // the data and write it to file directly. - if (req.size > maxReqSizeShuffleToMem) { - shuffleClient.fetchBlocks( - address.host, - address.port, - address.executorId, - blockIds.toArray, - blockFetchingListener, - this) - } else { - shuffleClient.fetchBlocks( - address.host, - address.port, - address.executorId, - blockIds.toArray, - blockFetchingListener, - null) - } - } - - /** - * This is called from initialize and also from the fallback which is triggered from - * [[PushBasedFetchHelper]]. - */ - private[this] def partitionBlocksByFetchMode( - blocksByAddress: Iterator[(BlockManagerId, Seq[(BlockId, Long, Int)])], - localBlocks: mutable.LinkedHashSet[(BlockId, Int)], - hostLocalBlocksByExecutor: mutable.LinkedHashMap[BlockManagerId, Seq[(BlockId, Long, Int)]], - pushMergedLocalBlocks: mutable.LinkedHashSet[BlockId]): ArrayBuffer[FetchRequest] = { - logDebug(s"maxBytesInFlight: $maxBytesInFlight, targetRemoteRequestSize: " - + s"$targetRemoteRequestSize, maxBlocksInFlightPerAddress: $maxBlocksInFlightPerAddress") - - // Partition to local, host-local, push-merged-local, remote (includes push-merged-remote) - // blocks.Remote blocks are further split into FetchRequests of size at most maxBytesInFlight - // in order to limit the amount of data in flight - val collectedRemoteRequests = new ArrayBuffer[FetchRequest] - var localBlockBytes = 0L - var hostLocalBlockBytes = 0L - var numHostLocalBlocks = 0 - var pushMergedLocalBlockBytes = 0L - val prevNumBlocksToFetch = numBlocksToFetch - - val fallback = FallbackStorage.FALLBACK_BLOCK_MANAGER_ID.executorId - val localExecIds = Set(blockManager.blockManagerId.executorId, fallback) - for ((address, blockInfos) <- blocksByAddress) { - checkBlockSizes(blockInfos) - if (pushBasedFetchHelper.isPushMergedShuffleBlockAddress(address)) { - // These are push-merged blocks or shuffle chunks of these blocks. - if (address.host == blockManager.blockManagerId.host) { - numBlocksToFetch += blockInfos.size - pushMergedLocalBlocks ++= blockInfos.map(_._1) - pushMergedLocalBlockBytes += blockInfos.map(_._2).sum - } else { - collectFetchRequests(address, blockInfos, collectedRemoteRequests) - } - } else if (localExecIds.contains(address.executorId)) { - val mergedBlockInfos = mergeContinuousShuffleBlockIdsIfNeeded( - blockInfos.map(info => FetchBlockInfo(info._1, info._2, info._3)), - doBatchFetch) - numBlocksToFetch += mergedBlockInfos.size - localBlocks ++= mergedBlockInfos.map(info => (info.blockId, info.mapIndex)) - localBlockBytes += mergedBlockInfos.map(_.size).sum - } else if ( - blockManager.hostLocalDirManager.isDefined && - address.host == blockManager.blockManagerId.host - ) { - val mergedBlockInfos = mergeContinuousShuffleBlockIdsIfNeeded( - blockInfos.map(info => FetchBlockInfo(info._1, info._2, info._3)), - doBatchFetch) - numBlocksToFetch += mergedBlockInfos.size - val blocksForAddress = - mergedBlockInfos.map(info => (info.blockId, info.size, info.mapIndex)) - hostLocalBlocksByExecutor += address -> blocksForAddress - numHostLocalBlocks += blocksForAddress.size - hostLocalBlockBytes += mergedBlockInfos.map(_.size).sum - } else { - val (_, timeCost) = Utils.timeTakenMs[Unit] { - collectFetchRequests(address, blockInfos, collectedRemoteRequests) - } - logDebug(s"Collected remote fetch requests for $address in $timeCost ms") - } - } - val (remoteBlockBytes, numRemoteBlocks) = - collectedRemoteRequests.foldLeft((0L, 0))((x, y) => (x._1 + y.size, x._2 + y.blocks.size)) - val totalBytes = localBlockBytes + remoteBlockBytes + hostLocalBlockBytes + - pushMergedLocalBlockBytes - val blocksToFetchCurrentIteration = numBlocksToFetch - prevNumBlocksToFetch - assert( - blocksToFetchCurrentIteration == localBlocks.size + - numHostLocalBlocks + numRemoteBlocks + pushMergedLocalBlocks.size, - s"The number of non-empty blocks $blocksToFetchCurrentIteration doesn't equal to the sum " + - s"of the number of local blocks ${localBlocks.size} + " + - s"the number of host-local blocks $numHostLocalBlocks " + - s"the number of push-merged-local blocks ${pushMergedLocalBlocks.size} " + - s"+ the number of remote blocks $numRemoteBlocks " - ) - logInfo(s"Getting $blocksToFetchCurrentIteration " + - s"(${Utils.bytesToString(totalBytes)}) non-empty blocks including " + - s"${localBlocks.size} (${Utils.bytesToString(localBlockBytes)}) local and " + - s"$numHostLocalBlocks (${Utils.bytesToString(hostLocalBlockBytes)}) " + - s"host-local and ${pushMergedLocalBlocks.size} " + - s"(${Utils.bytesToString(pushMergedLocalBlockBytes)}) " + - s"push-merged-local and $numRemoteBlocks (${Utils.bytesToString(remoteBlockBytes)}) " + - s"remote blocks") - this.hostLocalBlocks ++= hostLocalBlocksByExecutor.values - .flatMap(infos => infos.map(info => (info._1, info._3))) - collectedRemoteRequests - } - - private def createFetchRequest( - blocks: Seq[FetchBlockInfo], - address: BlockManagerId, - forMergedMetas: Boolean): FetchRequest = { - logDebug(s"Creating fetch request of ${blocks.map(_.size).sum} at $address " - + s"with ${blocks.size} blocks") - FetchRequest(address, blocks, forMergedMetas) - } - - private def createFetchRequests( - curBlocks: Seq[FetchBlockInfo], - address: BlockManagerId, - isLast: Boolean, - collectedRemoteRequests: ArrayBuffer[FetchRequest], - enableBatchFetch: Boolean, - forMergedMetas: Boolean = false): ArrayBuffer[FetchBlockInfo] = { - val mergedBlocks = mergeContinuousShuffleBlockIdsIfNeeded(curBlocks, enableBatchFetch) - numBlocksToFetch += mergedBlocks.size - val retBlocks = new ArrayBuffer[FetchBlockInfo] - if (mergedBlocks.length <= maxBlocksInFlightPerAddress) { - collectedRemoteRequests += createFetchRequest(mergedBlocks, address, forMergedMetas) - } else { - mergedBlocks.grouped(maxBlocksInFlightPerAddress).foreach { - blocks => - if (blocks.length == maxBlocksInFlightPerAddress || isLast) { - collectedRemoteRequests += createFetchRequest(blocks, address, forMergedMetas) - } else { - // The last group does not exceed `maxBlocksInFlightPerAddress`. Put it back - // to `curBlocks`. - retBlocks ++= blocks - numBlocksToFetch -= blocks.size - } - } - } - retBlocks - } - - private def collectFetchRequests( - address: BlockManagerId, - blockInfos: Seq[(BlockId, Long, Int)], - collectedRemoteRequests: ArrayBuffer[FetchRequest]): Unit = { - val iterator = blockInfos.iterator - var curRequestSize = 0L - var curBlocks = new ArrayBuffer[FetchBlockInfo]() - - while (iterator.hasNext) { - val (blockId, size, mapIndex) = iterator.next() - curBlocks += FetchBlockInfo(blockId, size, mapIndex) - curRequestSize += size - blockId match { - // Either all blocks are push-merged blocks, shuffle chunks, or original blocks. - // Based on these types, we decide to do batch fetch and create FetchRequests with - // forMergedMetas set. - case ShuffleBlockChunkId(_, _, _, _) => - if ( - curRequestSize >= targetRemoteRequestSize || - curBlocks.size >= maxBlocksInFlightPerAddress - ) { - curBlocks = createFetchRequests( - curBlocks.toSeq, - address, - isLast = false, - collectedRemoteRequests, - enableBatchFetch = false) - curRequestSize = curBlocks.map(_.size).sum - } - case ShuffleMergedBlockId(_, _, _) => - if (curBlocks.size >= maxBlocksInFlightPerAddress) { - curBlocks = createFetchRequests( - curBlocks.toSeq, - address, - isLast = false, - collectedRemoteRequests, - enableBatchFetch = false, - forMergedMetas = true) - } - case _ => - // For batch fetch, the actual block in flight should count for merged block. - val mayExceedsMaxBlocks = !doBatchFetch && curBlocks.size >= maxBlocksInFlightPerAddress - if (curRequestSize >= targetRemoteRequestSize || mayExceedsMaxBlocks) { - curBlocks = createFetchRequests( - curBlocks.toSeq, - address, - isLast = false, - collectedRemoteRequests, - doBatchFetch) - curRequestSize = curBlocks.map(_.size).sum - } - } - } - // Add in the final request - if (curBlocks.nonEmpty) { - val (enableBatchFetch, forMergedMetas) = { - curBlocks.head.blockId match { - case ShuffleBlockChunkId(_, _, _, _) => (false, false) - case ShuffleMergedBlockId(_, _, _) => (false, true) - case _ => (doBatchFetch, false) - } - } - createFetchRequests( - curBlocks.toSeq, - address, - isLast = true, - collectedRemoteRequests, - enableBatchFetch = enableBatchFetch, - forMergedMetas = forMergedMetas) - } - } - - private def assertPositiveBlockSize(blockId: BlockId, blockSize: Long): Unit = { - if (blockSize < 0) { - throw BlockException(blockId, "Negative block size " + blockSize) - } else if (blockSize == 0) { - throw BlockException(blockId, "Zero-sized blocks should be excluded.") - } - } - - private def checkBlockSizes(blockInfos: Seq[(BlockId, Long, Int)]): Unit = { - blockInfos.foreach { case (blockId, size, _) => assertPositiveBlockSize(blockId, size) } - } - - /** - * Fetch the local blocks while we are fetching remote blocks. This is ok because - * `ManagedBuffer`'s memory is allocated lazily when we create the input stream, so all we track - * in-memory are the ManagedBuffer references themselves. - */ - private[this] def fetchLocalBlocks( - localBlocks: mutable.LinkedHashSet[(BlockId, Int)]): Unit = { - logDebug(s"Start fetching local blocks: ${localBlocks.mkString(", ")}") - val iter = localBlocks.iterator - while (iter.hasNext) { - val (blockId, mapIndex) = iter.next() - try { - val buf = blockManager.getLocalBlockData(blockId) - shuffleMetrics.incLocalBlocksFetched(1) - shuffleMetrics.incLocalBytesRead(buf.size) - buf.retain() - results.put(new SuccessFetchResult( - blockId, - mapIndex, - blockManager.blockManagerId, - buf.size(), - buf, - false)) - } catch { - // If we see an exception, stop immediately. - case e: Exception => - e match { - // ClosedByInterruptException is an excepted exception when kill task, - // don't log the exception stack trace to avoid confusing users. - // See: SPARK-28340 - case ce: ClosedByInterruptException => - logError("Error occurred while fetching local blocks, " + ce.getMessage) - case ex: Exception => logError("Error occurred while fetching local blocks", ex) - } - results.put(new FailureFetchResult(blockId, mapIndex, blockManager.blockManagerId, e)) - return - } - } - } - - private[this] def fetchHostLocalBlock( - blockId: BlockId, - mapIndex: Int, - localDirs: Array[String], - blockManagerId: BlockManagerId): Boolean = { - try { - val buf = blockManager.getHostLocalShuffleData(blockId, localDirs) - buf.retain() - results.put(SuccessFetchResult( - blockId, - mapIndex, - blockManagerId, - buf.size(), - buf, - isNetworkReqDone = false)) - true - } catch { - case e: Exception => - // If we see an exception, stop immediately. - logError(s"Error occurred while fetching local blocks", e) - results.put(FailureFetchResult(blockId, mapIndex, blockManagerId, e)) - false - } - } - - /** - * Fetch the host-local blocks while we are fetching remote blocks. This is ok because - * `ManagedBuffer`'s memory is allocated lazily when we create the input stream, so all we track - * in-memory are the ManagedBuffer references themselves. - */ - private[this] def fetchHostLocalBlocks( - hostLocalDirManager: HostLocalDirManager, - hostLocalBlocksByExecutor: mutable.LinkedHashMap[BlockManagerId, Seq[(BlockId, Long, Int)]]) - : Unit = { - val cachedDirsByExec = hostLocalDirManager.getCachedHostLocalDirs - val (hostLocalBlocksWithCachedDirs, hostLocalBlocksWithMissingDirs) = { - val (hasCache, noCache) = hostLocalBlocksByExecutor.partition { - case (hostLocalBmId, _) => - cachedDirsByExec.contains(hostLocalBmId.executorId) - } - (hasCache.toMap, noCache.toMap) - } - - if (hostLocalBlocksWithMissingDirs.nonEmpty) { - logDebug(s"Asynchronous fetching host-local blocks without cached executors' dir: " + - s"${hostLocalBlocksWithMissingDirs.mkString(", ")}") - - // If the external shuffle service is enabled, we'll fetch the local directories for - // multiple executors from the external shuffle service, which located at the same host - // with the executors, in once. Otherwise, we'll fetch the local directories from those - // executors directly one by one. The fetch requests won't be too much since one host is - // almost impossible to have many executors at the same time practically. - val dirFetchRequests = if (blockManager.externalShuffleServiceEnabled) { - val host = blockManager.blockManagerId.host - val port = blockManager.externalShuffleServicePort - Seq((host, port, hostLocalBlocksWithMissingDirs.keys.toArray)) - } else { - hostLocalBlocksWithMissingDirs.keys.map(bmId => (bmId.host, bmId.port, Array(bmId))).toSeq - } - - dirFetchRequests.foreach { - case (host, port, bmIds) => - hostLocalDirManager.getHostLocalDirs(host, port, bmIds.map(_.executorId)) { - case Success(dirsByExecId) => - fetchMultipleHostLocalBlocks( - hostLocalBlocksWithMissingDirs.filterKeys(bmIds.contains).toMap, - dirsByExecId, - cached = false) - - case Failure(throwable) => - logError("Error occurred while fetching host local blocks", throwable) - val bmId = bmIds.head - val blockInfoSeq = hostLocalBlocksWithMissingDirs(bmId) - val (blockId, _, mapIndex) = blockInfoSeq.head - results.put(FailureFetchResult(blockId, mapIndex, bmId, throwable)) - } - } - } - - if (hostLocalBlocksWithCachedDirs.nonEmpty) { - logDebug(s"Synchronous fetching host-local blocks with cached executors' dir: " + - s"${hostLocalBlocksWithCachedDirs.mkString(", ")}") - fetchMultipleHostLocalBlocks(hostLocalBlocksWithCachedDirs, cachedDirsByExec, cached = true) - } - } - - private def fetchMultipleHostLocalBlocks( - bmIdToBlocks: Map[BlockManagerId, Seq[(BlockId, Long, Int)]], - localDirsByExecId: Map[String, Array[String]], - cached: Boolean): Unit = { - // We use `forall` because once there's a failed block fetch, `fetchHostLocalBlock` will put - // a `FailureFetchResult` immediately to the `results`. So there's no reason to fetch the - // remaining blocks. - val allFetchSucceeded = bmIdToBlocks.forall { - case (bmId, blockInfos) => - blockInfos.forall { - case (blockId, _, mapIndex) => - fetchHostLocalBlock(blockId, mapIndex, localDirsByExecId(bmId.executorId), bmId) - } - } - if (allFetchSucceeded) { - logDebug(s"Got host-local blocks from ${bmIdToBlocks.keys.mkString(", ")} " + - s"(${if (cached) "with" else "without"} cached executors' dir) " + - s"in ${Utils.getUsedTimeNs(startTimeNs)}") - } - } - - private[this] def initialize(): Unit = { - // Add a task completion callback (called in both success case and failure case) to cleanup. - context.addTaskCompletionListener(onCompleteCallback) - // Local blocks to fetch, excluding zero-sized blocks. - val localBlocks = mutable.LinkedHashSet[(BlockId, Int)]() - val hostLocalBlocksByExecutor = - mutable.LinkedHashMap[BlockManagerId, Seq[(BlockId, Long, Int)]]() - val pushMergedLocalBlocks = mutable.LinkedHashSet[BlockId]() - // Partition blocks by the different fetch modes: local, host-local, push-merged-local and - // remote blocks. - val remoteRequests = partitionBlocksByFetchMode( - blocksByAddress, - localBlocks, - hostLocalBlocksByExecutor, - pushMergedLocalBlocks) - // Add the remote requests into our queue in a random order - fetchRequests ++= Utils.randomize(remoteRequests) - assert( - (0 == reqsInFlight) == (0 == bytesInFlight), - "expected reqsInFlight = 0 but found reqsInFlight = " + reqsInFlight + - ", expected bytesInFlight = 0 but found bytesInFlight = " + bytesInFlight - ) - - // Send out initial requests for blocks, up to our maxBytesInFlight - fetchUpToMaxBytes() - - val numDeferredRequest = deferredFetchRequests.values.map(_.size).sum - val numFetches = remoteRequests.size - fetchRequests.size - numDeferredRequest - logInfo(s"Started $numFetches remote fetches in ${Utils.getUsedTimeNs(startTimeNs)}" + - (if (numDeferredRequest > 0) s", deferred $numDeferredRequest requests" else "")) - - // Get Local Blocks - fetchLocalBlocks(localBlocks) - logDebug(s"Got local blocks in ${Utils.getUsedTimeNs(startTimeNs)}") - // Get host local blocks if any - fetchAllHostLocalBlocks(hostLocalBlocksByExecutor) - pushBasedFetchHelper.fetchAllPushMergedLocalBlocks(pushMergedLocalBlocks) - } - - private def fetchAllHostLocalBlocks( - hostLocalBlocksByExecutor: mutable.LinkedHashMap[BlockManagerId, Seq[(BlockId, Long, Int)]]) - : Unit = { - if (hostLocalBlocksByExecutor.nonEmpty) { - blockManager.hostLocalDirManager.foreach(fetchHostLocalBlocks(_, hostLocalBlocksByExecutor)) - } - } - - override def hasNext: Boolean = numBlocksProcessed < numBlocksToFetch - - /** - * Fetches the next (BlockId, InputStream). If a task fails, the ManagedBuffers underlying each - * InputStream will be freed by the cleanup() method registered with the TaskCompletionListener. - * However, callers should close() these InputStreams as soon as they are no longer needed, in - * order to release memory as early as possible. - * - * Throws a FetchFailedException if the next block could not be fetched. - */ - override def next(): (BlockId, InputStream) = { - if (!hasNext) { - throw SparkCoreErrors.noSuchElementError() - } - - numBlocksProcessed += 1 - - var result: FetchResult = null - var input: InputStream = null - // This's only initialized when shuffle checksum is enabled. - var checkedIn: CheckedInputStream = null - var streamCompressedOrEncrypted: Boolean = false - // Take the next fetched result and try to decompress it to detect data corruption, - // then fetch it one more time if it's corrupt, throw FailureFetchResult if the second fetch - // is also corrupt, so the previous stage could be retried. - // For local shuffle block, throw FailureFetchResult for the first IOException. - while (result == null) { - val startFetchWait = System.nanoTime() - result = results.take() - val fetchWaitTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startFetchWait) - shuffleMetrics.incFetchWaitTime(fetchWaitTime) - - result match { - case r @ SuccessFetchResult(blockId, mapIndex, address, size, buf, isNetworkReqDone) => - if (address != blockManager.blockManagerId) { - if ( - hostLocalBlocks.contains(blockId -> mapIndex) || - pushBasedFetchHelper.isLocalPushMergedBlockAddress(address) - ) { - // It is a host local block or a local shuffle chunk - shuffleMetrics.incLocalBlocksFetched(1) - shuffleMetrics.incLocalBytesRead(buf.size) - } else { - numBlocksInFlightPerAddress(address) = numBlocksInFlightPerAddress(address) - 1 - shuffleMetrics.incRemoteBytesRead(buf.size) - if (buf.isInstanceOf[FileSegmentManagedBuffer]) { - shuffleMetrics.incRemoteBytesReadToDisk(buf.size) - } - shuffleMetrics.incRemoteBlocksFetched(1) - bytesInFlight -= size - } - } - if (isNetworkReqDone) { - reqsInFlight -= 1 - resetNettyOOMFlagIfPossible(maxReqSizeShuffleToMem) - logDebug("Number of requests in flight " + reqsInFlight) - } - - val in = if (buf.size == 0) { - // We will never legitimately receive a zero-size block. All blocks with zero records - // have zero size and all zero-size blocks have no records (and hence should never - // have been requested in the first place). This statement relies on behaviors of the - // shuffle writers, which are guaranteed by the following test cases: - // - // - BypassMergeSortShuffleWriterSuite: "write with some empty partitions" - // - UnsafeShuffleWriterSuite: "writeEmptyIterator" - // - DiskBlockObjectWriterSuite: "commit() and close() without ever opening or writing" - // - // There is not an explicit test for SortShuffleWriter but the underlying APIs that - // uses are shared by the UnsafeShuffleWriter (both writers use DiskBlockObjectWriter - // which returns a zero-size from commitAndGet() in case no records were written - // since the last call. - val msg = s"Received a zero-size buffer for block $blockId from $address " + - s"(expectedApproxSize = $size, isNetworkReqDone=$isNetworkReqDone)" - if (blockId.isShuffleChunk) { - // Zero-size block may come from nodes with hardware failures, For shuffle chunks, - // the original shuffle blocks that belong to that zero-size shuffle chunk is - // available and we can opt to fallback immediately. - logWarning(msg) - pushBasedFetchHelper.initiateFallbackFetchForPushMergedBlock(blockId, address) - // Set result to null to trigger another iteration of the while loop to get either. - result = null - null - } else { - throwFetchFailedException(blockId, mapIndex, address, new IOException(msg)) - } - } else { - try { - val bufIn = buf.createInputStream() - if (checksumEnabled) { - val checksum = ShuffleChecksumHelper.getChecksumByAlgorithm(checksumAlgorithm) - checkedIn = new CheckedInputStream(bufIn, checksum) - checkedIn - } else { - bufIn - } - } catch { - // The exception could only be throwed by local shuffle block - case e: IOException => - assert(buf.isInstanceOf[FileSegmentManagedBuffer]) - e match { - case ce: ClosedByInterruptException => - logError("Failed to create input stream from local block, " + - ce.getMessage) - case e: IOException => - logError("Failed to create input stream from local block", e) - } - buf.release() - if (blockId.isShuffleChunk) { - pushBasedFetchHelper.initiateFallbackFetchForPushMergedBlock(blockId, address) - // Set result to null to trigger another iteration of the while loop to get - // either. - result = null - null - } else { - throwFetchFailedException(blockId, mapIndex, address, e) - } - } - } - - if (in != null) { - try { - input = streamWrapper(blockId, in) - // If the stream is compressed or wrapped, then we optionally decompress/unwrap the - // first maxBytesInFlight/3 bytes into memory, to check for corruption in that portion - // of the data. But even if 'detectCorruptUseExtraMemory' configuration is off, or if - // the corruption is later, we'll still detect the corruption later in the stream. - streamCompressedOrEncrypted = !input.eq(in) - if (streamCompressedOrEncrypted && detectCorruptUseExtraMemory) { - // TODO: manage the memory used here, and spill it into disk in case of OOM. - input = Utils.copyStreamUpTo(input, maxBytesInFlight / 3) - } - } catch { - case e: IOException => - // When shuffle checksum is enabled, for a block that is corrupted twice, - // we'd calculate the checksum of the block by consuming the remaining data - // in the buf. So, we should release the buf later. - if (!(checksumEnabled && corruptedBlocks.contains(blockId))) { - buf.release() - } - - if (blockId.isShuffleChunk) { - // TODO (SPARK-36284): Add shuffle checksum support for push-based shuffle - // Retrying a corrupt block may result again in a corrupt block. For shuffle - // chunks, we opt to fallback on the original shuffle blocks that belong to that - // corrupt shuffle chunk immediately instead of retrying to fetch the corrupt - // chunk. This also makes the code simpler because the chunkMeta corresponding to - // a shuffle chunk is always removed from chunksMetaMap whenever a shuffle chunk - // gets processed. If we try to re-fetch a corrupt shuffle chunk, then it has to - // be added back to the chunksMetaMap. - pushBasedFetchHelper.initiateFallbackFetchForPushMergedBlock(blockId, address) - // Set result to null to trigger another iteration of the while loop. - result = null - } else if (buf.isInstanceOf[FileSegmentManagedBuffer]) { - throwFetchFailedException(blockId, mapIndex, address, e) - } else if (corruptedBlocks.contains(blockId)) { - // It's the second time this block is detected corrupted - if (checksumEnabled) { - // Diagnose the cause of data corruption if shuffle checksum is enabled - val diagnosisResponse = diagnoseCorruption(checkedIn, address, blockId) - buf.release() - logError(diagnosisResponse) - throwFetchFailedException( - blockId, - mapIndex, - address, - e, - Some(diagnosisResponse)) - } else { - throwFetchFailedException(blockId, mapIndex, address, e) - } - } else { - // It's the first time this block is detected corrupted - logWarning(s"got an corrupted block $blockId from $address, fetch again", e) - corruptedBlocks += blockId - fetchRequests += FetchRequest( - address, - Array(FetchBlockInfo(blockId, size, mapIndex))) - result = null - } - } finally { - if (blockId.isShuffleChunk) { - pushBasedFetchHelper.removeChunk(blockId.asInstanceOf[ShuffleBlockChunkId]) - } - // TODO: release the buf here to free memory earlier - if (input == null) { - // Close the underlying stream if there was an issue in wrapping the stream using - // streamWrapper - in.close() - } - } - } - - case FailureFetchResult(blockId, mapIndex, address, e) => - var errorMsg: String = null - if (e.isInstanceOf[OutOfDirectMemoryError]) { - errorMsg = s"Block $blockId fetch failed after $maxAttemptsOnNettyOOM " + - s"retries due to Netty OOM" - logError(errorMsg) - } - throwFetchFailedException(blockId, mapIndex, address, e, Some(errorMsg)) - - case DeferFetchRequestResult(request) => - val address = request.address - numBlocksInFlightPerAddress(address) = - numBlocksInFlightPerAddress(address) - request.blocks.size - bytesInFlight -= request.size - reqsInFlight -= 1 - logDebug("Number of requests in flight " + reqsInFlight) - val defReqQueue = - deferredFetchRequests.getOrElseUpdate(address, new Queue[FetchRequest]()) - defReqQueue.enqueue(request) - result = null - - case FallbackOnPushMergedFailureResult(blockId, address, size, isNetworkReqDone) => - // We get this result in 3 cases: - // 1. Failure to fetch the data of a remote shuffle chunk. In this case, the - // blockId is a ShuffleBlockChunkId. - // 2. Failure to read the push-merged-local meta. In this case, the blockId is - // ShuffleBlockId. - // 3. Failure to get the push-merged-local directories from the external shuffle service. - // In this case, the blockId is ShuffleBlockId. - if (pushBasedFetchHelper.isRemotePushMergedBlockAddress(address)) { - numBlocksInFlightPerAddress(address) = numBlocksInFlightPerAddress(address) - 1 - bytesInFlight -= size - } - if (isNetworkReqDone) { - reqsInFlight -= 1 - logDebug("Number of requests in flight " + reqsInFlight) - } - pushBasedFetchHelper.initiateFallbackFetchForPushMergedBlock(blockId, address) - // Set result to null to trigger another iteration of the while loop to get either - // a SuccessFetchResult or a FailureFetchResult. - result = null - - case PushMergedLocalMetaFetchResult( - shuffleId, - shuffleMergeId, - reduceId, - bitmaps, - localDirs) => - // Fetch push-merged-local shuffle block data as multiple shuffle chunks - val shuffleBlockId = ShuffleMergedBlockId(shuffleId, shuffleMergeId, reduceId) - try { - val bufs: Seq[ManagedBuffer] = blockManager.getLocalMergedBlockData( - shuffleBlockId, - localDirs) - // Since the request for local block meta completed successfully, numBlocksToFetch - // is decremented. - numBlocksToFetch -= 1 - // Update total number of blocks to fetch, reflecting the multiple local shuffle - // chunks. - numBlocksToFetch += bufs.size - bufs.zipWithIndex.foreach { - case (buf, chunkId) => - buf.retain() - val shuffleChunkId = ShuffleBlockChunkId( - shuffleId, - shuffleMergeId, - reduceId, - chunkId) - pushBasedFetchHelper.addChunk(shuffleChunkId, bitmaps(chunkId)) - results.put(SuccessFetchResult( - shuffleChunkId, - SHUFFLE_PUSH_MAP_ID, - pushBasedFetchHelper.localShuffleMergerBlockMgrId, - buf.size(), - buf, - isNetworkReqDone = false)) - } - } catch { - case e: Exception => - // If we see an exception with reading push-merged-local index file, we fallback - // to fetch the original blocks. We do not report block fetch failure - // and will continue with the remaining local block read. - logWarning( - s"Error occurred while reading push-merged-local index, " + - s"prepare to fetch the original blocks", - e) - pushBasedFetchHelper.initiateFallbackFetchForPushMergedBlock( - shuffleBlockId, - pushBasedFetchHelper.localShuffleMergerBlockMgrId) - } - result = null - - case PushMergedRemoteMetaFetchResult( - shuffleId, - shuffleMergeId, - reduceId, - blockSize, - bitmaps, - address) => - // The original meta request is processed so we decrease numBlocksToFetch and - // numBlocksInFlightPerAddress by 1. We will collect new shuffle chunks request and the - // count of this is added to numBlocksToFetch in collectFetchReqsFromMergedBlocks. - numBlocksInFlightPerAddress(address) = numBlocksInFlightPerAddress(address) - 1 - numBlocksToFetch -= 1 - val blocksToFetch = pushBasedFetchHelper.createChunkBlockInfosFromMetaResponse( - shuffleId, - shuffleMergeId, - reduceId, - blockSize, - bitmaps) - val additionalRemoteReqs = new ArrayBuffer[FetchRequest] - collectFetchRequests(address, blocksToFetch.toSeq, additionalRemoteReqs) - fetchRequests ++= additionalRemoteReqs - // Set result to null to force another iteration. - result = null - - case PushMergedRemoteMetaFailedFetchResult( - shuffleId, - shuffleMergeId, - reduceId, - address) => - // The original meta request failed so we decrease numBlocksInFlightPerAddress by 1. - numBlocksInFlightPerAddress(address) = numBlocksInFlightPerAddress(address) - 1 - // If we fail to fetch the meta of a push-merged block, we fall back to fetching the - // original blocks. - pushBasedFetchHelper.initiateFallbackFetchForPushMergedBlock( - ShuffleMergedBlockId(shuffleId, shuffleMergeId, reduceId), - address) - // Set result to null to force another iteration. - result = null - } - - // Send fetch requests up to maxBytesInFlight - fetchUpToMaxBytes() - } - - val successResult = result.asInstanceOf[SuccessFetchResult] - currentResults.add(successResult) - ( - successResult.blockId, - new GlutenBufferReleasingInputStream( - input, - this, - () => { - val result = successResult - if (currentResults.remove(result)) { - result.buf.release() - } - }, - successResult.blockId, - successResult.mapIndex, - successResult.address, - detectCorrupt && streamCompressedOrEncrypted, - successResult.isNetworkReqDone, - Option(checkedIn) - )) - } - - /** - * Get the suspect corruption cause for the corrupted block. It should be only invoked when - * checksum is enabled and corruption was detected at least once. - * - * This will firstly consume the rest of stream of the corrupted block to calculate the checksum - * of the block. Then, it will raise a synchronized RPC call along with the checksum to ask the - * server(where the corrupted block is fetched from) to diagnose the cause of corruption and - * return it. - * - * Any exception raised during the process will result in the [[Cause.UNKNOWN_ISSUE]] of the - * corruption cause since corruption diagnosis is only a best effort. - * - * @param checkedIn - * the [[CheckedInputStream]] which is used to calculate the checksum. - * @param address - * the address where the corrupted block is fetched from. - * @param blockId - * the blockId of the corrupted block. - * @return - * The corruption diagnosis response for different causes. - */ - private[storage] def diagnoseCorruption( - checkedIn: CheckedInputStream, - address: BlockManagerId, - blockId: BlockId): String = { - logInfo("Start corruption diagnosis.") - blockId match { - case shuffleBlock: ShuffleBlockId => - val startTimeNs = System.nanoTime() - val buffer = new Array[Byte](ShuffleChecksumHelper.CHECKSUM_CALCULATION_BUFFER) - // consume the remaining data to calculate the checksum - var cause: Cause = null - try { - while (checkedIn.read(buffer) != -1) {} - val checksum = checkedIn.getChecksum.getValue - cause = shuffleClient.diagnoseCorruption( - address.host, - address.port, - address.executorId, - shuffleBlock.shuffleId, - shuffleBlock.mapId, - shuffleBlock.reduceId, - checksum, - checksumAlgorithm) - } catch { - case e: Exception => - logWarning("Unable to diagnose the corruption cause of the corrupted block", e) - cause = Cause.UNKNOWN_ISSUE - } - val duration = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNs) - val diagnosisResponse = cause match { - case Cause.UNSUPPORTED_CHECKSUM_ALGORITHM => - s"Block $blockId is corrupted but corruption diagnosis failed due to " + - s"unsupported checksum algorithm: $checksumAlgorithm" - - case Cause.CHECKSUM_VERIFY_PASS => - s"Block $blockId is corrupted but checksum verification passed" - - case Cause.UNKNOWN_ISSUE => - s"Block $blockId is corrupted but the cause is unknown" - - case otherCause => - s"Block $blockId is corrupted due to $otherCause" - } - logInfo(s"Finished corruption diagnosis in $duration ms. $diagnosisResponse") - diagnosisResponse - case shuffleBlockChunk: ShuffleBlockChunkId => - // TODO SPARK-36284 Add shuffle checksum support for push-based shuffle - val diagnosisResponse = s"BlockChunk $shuffleBlockChunk is corrupted but corruption " + - s"diagnosis is skipped due to lack of shuffle checksum support for push-based shuffle." - logWarning(diagnosisResponse) - diagnosisResponse - case unexpected: BlockId => - throw new IllegalArgumentException(s"Unexpected type of BlockId, $unexpected") - } - } - - override def onComplete(): Unit = { - onCompleteCallback.onComplete(context) - } - - private def fetchUpToMaxBytes(): Unit = { - if (isNettyOOMOnShuffle.get()) { - if (reqsInFlight > 0) { - // Return immediately if Netty is still OOMed and there're ongoing fetch requests - return - } else { - resetNettyOOMFlagIfPossible(0) - } - } - - // Send fetch requests up to maxBytesInFlight. If you cannot fetch from a remote host - // immediately, defer the request until the next time it can be processed. - - // Process any outstanding deferred fetch requests if possible. - if (deferredFetchRequests.nonEmpty) { - for ((remoteAddress, defReqQueue) <- deferredFetchRequests) { - while ( - isRemoteBlockFetchable(defReqQueue) && - !isRemoteAddressMaxedOut(remoteAddress, defReqQueue.front) - ) { - val request = defReqQueue.dequeue() - logDebug(s"Processing deferred fetch request for $remoteAddress with " - + s"${request.blocks.length} blocks") - send(remoteAddress, request) - if (defReqQueue.isEmpty) { - deferredFetchRequests -= remoteAddress - } - } - } - } - - // Process any regular fetch requests if possible. - while (isRemoteBlockFetchable(fetchRequests)) { - val request = fetchRequests.dequeue() - val remoteAddress = request.address - if (isRemoteAddressMaxedOut(remoteAddress, request)) { - logDebug(s"Deferring fetch request for $remoteAddress with ${request.blocks.size} blocks") - val defReqQueue = deferredFetchRequests.getOrElse(remoteAddress, new Queue[FetchRequest]()) - defReqQueue.enqueue(request) - deferredFetchRequests(remoteAddress) = defReqQueue - } else { - send(remoteAddress, request) - } - } - - def send(remoteAddress: BlockManagerId, request: FetchRequest): Unit = { - if (request.forMergedMetas) { - pushBasedFetchHelper.sendFetchMergedStatusRequest(request) - } else { - sendRequest(request) - } - numBlocksInFlightPerAddress(remoteAddress) = - numBlocksInFlightPerAddress.getOrElse(remoteAddress, 0) + request.blocks.size - } - - def isRemoteBlockFetchable(fetchReqQueue: Queue[FetchRequest]): Boolean = { - fetchReqQueue.nonEmpty && - (bytesInFlight == 0 || - (reqsInFlight + 1 <= maxReqsInFlight && - bytesInFlight + fetchReqQueue.front.size <= maxBytesInFlight)) - } - - // Checks if sending a new fetch request will exceed the max no. of blocks being fetched from a - // given remote address. - def isRemoteAddressMaxedOut(remoteAddress: BlockManagerId, request: FetchRequest): Boolean = { - numBlocksInFlightPerAddress.getOrElse(remoteAddress, 0) + request.blocks.size > - maxBlocksInFlightPerAddress - } - } - - private[storage] def throwFetchFailedException( - blockId: BlockId, - mapIndex: Int, - address: BlockManagerId, - e: Throwable, - message: Option[String] = None) = { - val msg = message.getOrElse(e.getMessage) - blockId match { - case ShuffleBlockId(shufId, mapId, reduceId) => - throw SparkCoreErrors.fetchFailedError(address, shufId, mapId, mapIndex, reduceId, msg, e) - case ShuffleBlockBatchId(shuffleId, mapId, startReduceId, _) => - throw SparkCoreErrors.fetchFailedError( - address, - shuffleId, - mapId, - mapIndex, - startReduceId, - msg, - e) - case _ => throw SparkCoreErrors.failToGetNonShuffleBlockError(blockId, e) - } - } - - /** - * All the below methods are used by [[PushBasedFetchHelper]] to communicate with the iterator - */ - private[storage] def addToResultsQueue(result: FetchResult): Unit = { - results.put(result) - } - - private[storage] def decreaseNumBlocksToFetch(blocksFetched: Int): Unit = { - numBlocksToFetch -= blocksFetched - } - - /** - * Currently used by [[PushBasedFetchHelper]] to fetch fallback blocks when there is a fetch - * failure related to a push-merged block or shuffle chunk. This is executed by the task thread - * when the `iterator.next()` is invoked and if that initiates fallback. - */ - private[storage] def fallbackFetch( - originalBlocksByAddr: Iterator[(BlockManagerId, Seq[(BlockId, Long, Int)])]): Unit = { - val originalLocalBlocks = mutable.LinkedHashSet[(BlockId, Int)]() - val originalHostLocalBlocksByExecutor = - mutable.LinkedHashMap[BlockManagerId, Seq[(BlockId, Long, Int)]]() - val originalMergedLocalBlocks = mutable.LinkedHashSet[BlockId]() - val originalRemoteReqs = partitionBlocksByFetchMode( - originalBlocksByAddr, - originalLocalBlocks, - originalHostLocalBlocksByExecutor, - originalMergedLocalBlocks) - // Add the remote requests into our queue in a random order - fetchRequests ++= Utils.randomize(originalRemoteReqs) - logInfo(s"Created ${originalRemoteReqs.size} fallback remote requests for push-merged") - // fetch all the fallback blocks that are local. - fetchLocalBlocks(originalLocalBlocks) - // Merged local blocks should be empty during fallback - assert( - originalMergedLocalBlocks.isEmpty, - "There should be zero push-merged blocks during fallback") - // Some of the fallback local blocks could be host local blocks - fetchAllHostLocalBlocks(originalHostLocalBlocksByExecutor) - } - - /** - * Removes all the pending shuffle chunks that are on the same host and have the same reduceId as - * the current chunk that had a fetch failure. This is executed by the task thread when the - * `iterator.next()` is invoked and if that initiates fallback. - * - * @return - * set of all the removed shuffle chunk Ids. - */ - private[storage] def removePendingChunks( - failedBlockId: ShuffleBlockChunkId, - address: BlockManagerId): mutable.HashSet[ShuffleBlockChunkId] = { - val removedChunkIds = new mutable.HashSet[ShuffleBlockChunkId]() - - def sameShuffleReducePartition(block: BlockId): Boolean = { - val chunkId = block.asInstanceOf[ShuffleBlockChunkId] - chunkId.shuffleId == failedBlockId.shuffleId && chunkId.reduceId == failedBlockId.reduceId - } - - def filterRequests(queue: mutable.Queue[FetchRequest]): Unit = { - val fetchRequestsToRemove = new mutable.Queue[FetchRequest]() - fetchRequestsToRemove ++= queue.dequeueAll { - req => - val firstBlock = req.blocks.head - firstBlock.blockId.isShuffleChunk && req.address.equals(address) && - sameShuffleReducePartition(firstBlock.blockId) - } - fetchRequestsToRemove.foreach { - _ => - removedChunkIds ++= - fetchRequestsToRemove.flatMap(_.blocks.map(_.blockId.asInstanceOf[ShuffleBlockChunkId])) - } - } - - filterRequests(fetchRequests) - deferredFetchRequests.get(address).foreach { - defRequests => - filterRequests(defRequests) - if (defRequests.isEmpty) deferredFetchRequests.remove(address) - } - removedChunkIds - } -} - -/** - * Helper class that ensures a ManagedBuffer is released upon InputStream.close() and also detects - * stream corruption if streamCompressedOrEncrypted is true - */ -class GlutenBufferReleasingInputStream( - // This is visible for testing - val delegate: InputStream, - private val iterator: GlutenShuffleBlockFetcherIterator, - private val releaseCallback: () => Unit, - private val blockId: BlockId, - private val mapIndex: Int, - private val address: BlockManagerId, - private val detectCorruption: Boolean, - private val isNetworkReqDone: Boolean, - private val checkedInOpt: Option[CheckedInputStream]) - extends InputStream { - private[this] var closed = false - - override def read(): Int = - tryOrFetchFailedException(delegate.read()) - - override def close(): Unit = { - if (!closed) { - try { - delegate.close() - releaseCallback() - } finally { - // Unset the flag when a remote request finished and free memory is fairly enough. - if (isNetworkReqDone) { - ShuffleBlockFetcherIterator.resetNettyOOMFlagIfPossible(iterator.maxReqSizeShuffleToMem) - } - closed = true - } - } - } - - override def available(): Int = delegate.available() - - override def mark(readlimit: Int): Unit = delegate.mark(readlimit) - - override def skip(n: Long): Long = - tryOrFetchFailedException(delegate.skip(n)) - - override def markSupported(): Boolean = delegate.markSupported() - - override def read(b: Array[Byte]): Int = - tryOrFetchFailedException(delegate.read(b)) - - override def read(b: Array[Byte], off: Int, len: Int): Int = - tryOrFetchFailedException(delegate.read(b, off, len)) - - override def reset(): Unit = delegate.reset() - - /** - * Execute a block of code that returns a value, close this stream quietly and re-throwing - * IOException as FetchFailedException when detectCorruption is true. This method is only used by - * the `read` and `skip` methods inside `BufferReleasingInputStream` currently. - */ - private def tryOrFetchFailedException[T](block: => T): T = { - try { - block - } catch { - case e: IOException if detectCorruption => - val diagnosisResponse = - checkedInOpt.map(checkedIn => iterator.diagnoseCorruption(checkedIn, address, blockId)) - IOUtils.closeQuietly(this) - // We'd never retry the block whatever the cause is since the block has been - // partially consumed by downstream RDDs. - iterator.throwFetchFailedException(blockId, mapIndex, address, e, diagnosisResponse) - } - } -} - -/** - * A listener to be called at the completion of the ShuffleBlockFetcherIterator - * @param data - * the ShuffleBlockFetcherIterator to process - */ -private class GlutenShuffleFetchCompletionListener(var data: GlutenShuffleBlockFetcherIterator) - extends TaskCompletionListener { - - override def onTaskCompletion(context: TaskContext): Unit = { - if (data != null) { - data.cleanup() - // Null out the referent here to make sure we don't keep a reference to this - // ShuffleBlockFetcherIterator, after we're done reading from it, to let it be - // collected during GC. Otherwise we can hold metadata on block locations(blocksByAddress) - data = null - } - } - - // Just an alias for onTaskCompletion to avoid confusing - def onComplete(context: TaskContext): Unit = this.onTaskCompletion(context) -} diff --git a/tools/gluten-it/README.md b/tools/gluten-it/README.md index d2963fcc6d2..d42da331fc3 100644 --- a/tools/gluten-it/README.md +++ b/tools/gluten-it/README.md @@ -22,7 +22,7 @@ mvn clean package -P{Spark-Version} sbin/gluten-it.sh ``` -Note: **Spark-Version** can only be **spark-3.3**, **spark-3.4** or **spark-3.5**. +Note: **Spark-Version** can only be **spark-3.4**, **spark-3.5**, **spark-4.0** or **spark-4.1**. ## Usage diff --git a/tools/gluten-it/pom.xml b/tools/gluten-it/pom.xml index f909b68f39b..f38ace4f9f5 100644 --- a/tools/gluten-it/pom.xml +++ b/tools/gluten-it/pom.xml @@ -287,16 +287,6 @@ true - - spark-3.3 - - 3.3.1 - 2.12.15 - 2.12 - delta-core - 2.3.0 - - spark-3.4 diff --git a/tools/workload/tpcds-delta/run_tpcds/run-tpcds.sh b/tools/workload/tpcds-delta/run_tpcds/run-tpcds.sh index 048e793d990..a488e8134bc 100755 --- a/tools/workload/tpcds-delta/run_tpcds/run-tpcds.sh +++ b/tools/workload/tpcds-delta/run_tpcds/run-tpcds.sh @@ -44,4 +44,4 @@ cat tpcds_delta.scala | ${SPARK_HOME}/bin/spark-shell \ # e.g. # --conf spark.gluten.loadLibFromJar=true \ # --jars /PATH_TO_GLUTEN_HOME/package/target/thirdparty-lib/gluten-thirdparty-lib-ubuntu-22.04-x86_64.jar, - # /PATH_TO_GLUTEN_HOME/package/target/gluten-velox-bundle-spark3.3_2.12-ubuntu_22.04_x86_64-1.x.x-SNAPSHOT.jar + # /PATH_TO_GLUTEN_HOME/package/target/gluten-velox-bundle-spark3.5_2.12-ubuntu_22.04_x86_64-1.x.x-SNAPSHOT.jar diff --git a/tools/workload/tpcds/run_tpcds/run-tpcds.sh b/tools/workload/tpcds/run_tpcds/run-tpcds.sh index cc7aec34f21..071683eb872 100755 --- a/tools/workload/tpcds/run_tpcds/run-tpcds.sh +++ b/tools/workload/tpcds/run_tpcds/run-tpcds.sh @@ -37,4 +37,4 @@ cat tpcds_parquet.scala | ${SPARK_HOME}/bin/spark-shell \ # e.g. # --conf spark.gluten.loadLibFromJar=true \ # --jars /PATH_TO_GLUTEN_HOME/package/target/thirdparty-lib/gluten-thirdparty-lib-ubuntu-22.04-x86_64.jar, - # /PATH_TO_GLUTEN_HOME/package/target/gluten-velox-bundle-spark3.3_2.12-ubuntu_22.04_x86_64-1.x.x-SNAPSHOT.jar + # /PATH_TO_GLUTEN_HOME/package/target/gluten-velox-bundle-spark3.5_2.12-ubuntu_22.04_x86_64-1.x.x-SNAPSHOT.jar