What about hot s3 storage support?

[YuriyGavrilov]

There is path and api option for today. What about s3?

[jackye1995]

Checkout config example here: https://lance.org/integrations/trino/config/#directory-namespace

[YuriyGavrilov]

Checkout config example here: https://lance.org/integrations/trino/config/#directory-namespace

This is wonderful 🙏 thx

[YuriyGavrilov]

@jackye1995 one more question about rest catalog, is it possible to use same rest catalog as for iceberg for example lakekeeper?

[jackye1995]

You can specify the impl=iceberg, and you will also need to include jars related to the additional Lance Namespace implementation for Iceberg REST catalog: https://github.com/lance-format/lance-namespace-impls/tree/main/java/lance-namespace-iceberg

(great point I should also add to documentation)

It should work with lakekeeper, my integration test is actually written against it: https://github.com/lance-format/lance-namespace-impls/blob/main/docker/iceberg/docker-compose.yml

[YuriyGavrilov]

You can specify the impl=iceberg, and you will also need to include jars related to the additional Lance Namespace implementation for Iceberg REST catalog: https://github.com/lance-format/lance-namespace-impls/tree/main/java/lance-namespace-iceberg

(great point I should also add to documentation)

It should work with lakekeeper, my integration test is actually written against it: https://github.com/lance-format/lance-namespace-impls/blob/main/docker/iceberg/docker-compose.yml

thx @jackye1995 main point was to use already existed rest catalog - iceberg for one part of users lance for other ML side users and production workload. so need to investigate and try. hope there will be no Keycloak oauth bottleneck (we use it in lakekeeper). thx again for this information you was "just in time" with this information for me 🙏🏻

[jackye1995]

Let me know if you see any issues, I'm quickly iterating the Trino connector these few weeks, should be able to fix it quickly

[YuriyGavrilov]

some try @jackye1995

with trino

services:

  trino:
    build: ./trino
    environment:
      - CATALOG_MANAGEMENT=dynamic
    healthcheck:
      test: ["CMD", "curl", "-I", "http://localhost:8080/v1/status"]
      interval: 2s
      timeout: 10s
      retries: 2
      start_period: 10s
    ports:
      - "9999:8080"
    networks:
      - lakekeeper-network

  jupyter:
    image: quay.io/jupyter/pyspark-notebook:2024-10-14
    depends_on:
      lakekeeper:
        condition: service_healthy

      lakekeeper-setup:
        condition: service_completed_successfully
      trino:
        condition: service_healthy

    command: start-notebook.sh --NotebookApp.token=''
    volumes:
      - ./notebooks:/home/jovyan/examples/
    networks:
      - lakekeeper-network
    ports:
      - "8888:8888"

  postgres-lakekeeper:
    image: postgres:17
    container_name: postgres-lakekeeper
    environment:
      POSTGRES_USER: lakekeeper
      POSTGRES_PASSWORD: lakekeeper
      POSTGRES_DB: lakekeeper
    ports:
      - "5435:5432"
    volumes:
      - lakekeeper-postgres-data:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U lakekeeper -d lakekeeper"]
      interval: 2s
      timeout: 10s
      retries: 5
    networks:
      - lakekeeper-network

  minio:
    image: minio/minio:latest
    container_name: minio-lakekeeper
    environment:
      MINIO_ROOT_USER: minio-root-user
      MINIO_ROOT_PASSWORD: minio-root-password
      MINIO_DOMAIN: minio
    command: server /data --console-address ":9001"
    ports:
      - "19000:9000"
      - "19001:9001"
    volumes:
      - lakekeeper-minio-data:/data
    healthcheck:
      test: ["CMD", "mc", "ready", "local"]
      interval: 2s
      timeout: 10s
      retries: 5
    networks:
      - lakekeeper-network

  minio-setup:
    image: minio/mc:latest
    container_name: minio-setup
    depends_on:
      minio:
        condition: service_healthy
    entrypoint: >
      /bin/sh -c "
        mc alias set myminio http://minio:9000 minio-root-user minio-root-password &&
        mc mb myminio/warehouse --ignore-existing &&
        echo 'MinIO bucket created'
      "
    networks:
      - lakekeeper-network

  lakekeeper-migrate:
    image: quay.io/lakekeeper/catalog:latest-main
    container_name: lakekeeper-migrate
    depends_on:
      postgres-lakekeeper:
        condition: service_healthy
    environment:
      - LAKEKEEPER__PG_ENCRYPTION_KEY=test-encryption-key-not-secure
      - LAKEKEEPER__PG_DATABASE_URL_READ=postgresql://lakekeeper:lakekeeper@postgres-lakekeeper:5432/lakekeeper
      - LAKEKEEPER__PG_DATABASE_URL_WRITE=postgresql://lakekeeper:lakekeeper@postgres-lakekeeper:5432/lakekeeper
    restart: "no"
    command: ["migrate"]
    networks:
      - lakekeeper-network

  lakekeeper:
    image: quay.io/lakekeeper/catalog:latest-main
    container_name: lakekeeper
    depends_on:
      lakekeeper-migrate:
        condition: service_completed_successfully
      minio-setup:
        condition: service_completed_successfully
    environment:
      - LAKEKEEPER__PG_ENCRYPTION_KEY=test-encryption-key-not-secure
      - LAKEKEEPER__PG_DATABASE_URL_READ=postgresql://lakekeeper:lakekeeper@postgres-lakekeeper:5432/lakekeeper
      - LAKEKEEPER__PG_DATABASE_URL_WRITE=postgresql://lakekeeper:lakekeeper@postgres-lakekeeper:5432/lakekeeper
      - LAKEKEEPER__AUTHZ_BACKEND=allowall
      - RUST_LOG=info
    command: ["serve"]
    healthcheck:
      test: ["CMD", "/home/nonroot/lakekeeper", "healthcheck"]
      interval: 2s
      timeout: 10s
      retries: 5
      start_period: 5s
    ports:
      - "8282:8181"
    networks:
      - lakekeeper-network

  lakekeeper-bootstrap:
    image: curlimages/curl
    container_name: lakekeeper-bootstrap
    depends_on:
      lakekeeper:
        condition: service_healthy
    restart: "no"
    command:
      - -w
      - "%{http_code}"
      - "-X"
      - "POST"
      - "-v"
      - "http://lakekeeper:8181/management/v1/bootstrap"
      - "-H"
      - "Content-Type: application/json"
      - "--data"
      - '{"accept-terms-of-use": true}'
      - "-o"
      - "/dev/null"
    networks:
      - lakekeeper-network

  lakekeeper-setup:
    image: curlimages/curl
    container_name: lakekeeper-setup
    depends_on:
      lakekeeper-bootstrap:
        condition: service_completed_successfully
    restart: "no"
    entrypoint: ["/bin/sh", "-c"]
    command:
      - |
        echo "Creating test_warehouse..."
        curl -sf -X POST "http://lakekeeper:8181/management/v1/warehouse" \
          -H "Content-Type: application/json" \
          -d '{
            "warehouse-name": "test_warehouse",
            "project-id": "00000000-0000-0000-0000-000000000000",
            "storage-profile": {
              "type": "s3",
              "bucket": "warehouse",
              "endpoint": "http://minio:9000",
              "region": "us-east-1",
              "path-style-access": true,
              "flavor": "minio",
              "sts-enabled": false
            },
            "storage-credential": {
              "type": "s3",
              "credential-type": "access-key",
              "aws-access-key-id": "minio-root-user",
              "aws-secret-access-key": "minio-root-password"
            }
          }' && echo "Warehouse created successfully" || echo "Failed to create warehouse"
    networks:
      - lakekeeper-network

volumes:
  lakekeeper-postgres-data:
  lakekeeper-minio-data:

networks:
  lakekeeper-network:
    driver: bridge

docker file

FROM alpine:3.19 AS builder

ARG VERSION=0.1.0-beta.1
ARG TRINO_VERSION=476

RUN apk add --no-cache curl tar gzip

WORKDIR /build

RUN curl -fsSL -o plugin.tar.gz "https://github.com/lancedb/lance-trino/releases/download/v${VERSION}/trino-lance-${VERSION}-trino${TRINO_VERSION}.tar.gz"

RUN mkdir lance_plugin

RUN tar -xzf plugin.tar.gz -C lance_plugin --strip-components=1

FROM trinodb/trino:476

COPY --from=builder --chown=trino:trino /build/lance_plugin /usr/lib/trino/plugin/lance

build fine and run

create catalog with trino

CREATE CATALOG lance1 USING lance
WITH (
    "lance.impl" = 'dir',
    "lance.root" = 's3://warehouse/lance',
    "lance.storage.endpoint" = 'http://minio:9000',
    "lance.storage.region" = 'us-east-1',
    "lance.storage.path-style-access" = 'true',
    "lance.storage.access_key_id" = 'minio-root-user',
    "lance.storage.secret_access_key" = 'minio-root-password'
);

fine

create schema

create schema lance1.test

SQL Error [65536]: Query failed (#20260216_213844_00030_w7zyk): Namespace operation failed: Not supported: Child namespaces are only supported when manifest mode is enabled, /workspace/rust/lance-namespace-impls/src/dir.rs:902:23

java.lang.RuntimeException: Namespace operation failed: Not supported: Child namespaces are only supported when manifest mode is enabled, /workspace/rust/lance-namespace-impls/src/dir.rs:902:23
	at org.lance.namespace.DirectoryNamespace.createNamespaceNative(Native Method)
	at org.lance.namespace.DirectoryNamespace.createNamespace(DirectoryNamespace.java:220)
	at io.trino.plugin.lance.LanceMetadata.createSchema(LanceMetadata.java:183)
	at io.trino.tracing.TracingConnectorMetadata.createSchema(TracingConnectorMetadata.java:342)
	at io.trino.metadata.MetadataManager.createSchema(MetadataManager.java:826)
	at io.trino.tracing.TracingMetadata.createSchema(TracingMetadata.java:376)
	at io.trino.execution.CreateSchemaTask.internalExecute(CreateSchemaTask.java:128)
	at io.trino.execution.CreateSchemaTask.execute(CreateSchemaTask.java:82)
	at io.trino.execution.CreateSchemaTask.execute(CreateSchemaTask.java:54)
	at io.trino.execution.DataDefinitionExecution.start(DataDefinitionExecution.java:152)
	at io.trino.execution.SqlQueryManager.createQuery(SqlQueryManager.java:272)
	at io.trino.dispatcher.LocalDispatchQuery.startExecution(LocalDispatchQuery.java:150)
	at io.trino.dispatcher.LocalDispatchQuery.lambda$waitForMinimumWorkers$1(LocalDispatchQuery.java:134)
	at io.airlift.concurrent.MoreFutures.lambda$addSuccessCallback$12(MoreFutures.java:570)
	at io.airlift.concurrent.MoreFutures$3.onSuccess(MoreFutures.java:545)
	at com.google.common.util.concurrent.Futures$CallbackListener.run(Futures.java:1132)
	at io.trino.$gen.Trino_476____20260216_212811_2.run(Unknown Source)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1095)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:619)
	at java.base/java.lang.Thread.run(Thread.java:1447)

[YuriyGavrilov]

Some another logs

service

  trino:
    build: ./trino
    environment:
      - CATALOG_MANAGEMENT=dynamic
      - LANCE_ALLOW_HTTP=true
      - AWS_ALLOW_HTTP=true
      - AWS_ACCESS_KEY_ID=minio-root-user
      - AWS_SECRET_ACCESS_KEY=minio-root-password
      - AWS_REGION=us-east-1
      - AWS_ENDPOINT_URL=http://minio:9000
      
    healthcheck:
      test: ["CMD", "curl", "-I", "http://localhost:8080/v1/status"]
      interval: 2s
      timeout: 10s
      retries: 2
      start_period: 10s
    ports:
      - "9999:8080"
    networks:
      - lakekeeper-network

CREATE CATALOG lance6 USING lance
WITH (
    "lance.impl" = 'dir',
    "lance.root" = 's3://warehouse/lance-warehouse',
    "lance.single_level_ns" = 'true',
    "lance.storage.endpoint" = 'http://minio:9000',
    "lance.storage.region" = 'us-east-1',
    "lance.storage.path-style-access" = 'true',
    "lance.storage.access_key_id" = 'minio-root-user',
    "lance.storage.secret_access_key" = 'minio-root-password',
    "lance.storage.allow_http" = 'true'
);

CREATE TABLE lance6.default.users (id BIGINT, name VARCHAR);

SQL Error [65536]: Query failed (#20260217_204131_00003_9tadn): Internal error

java.lang.ExceptionInInitializerError: undefined
at org.apache.arrow.c.ArrowSchema.directBuffer(ArrowSchema.java:143)
at org.apache.arrow.c.ArrowSchema.markReleased(ArrowSchema.java:119)
at org.apache.arrow.c.ArrowSchema.allocateNew(ArrowSchema.java:108)
at org.lance.Dataset.create(Dataset.java:136)
at io.trino.plugin.lance.LanceMetadata.createEmptyDataset(LanceMetadata.java:1112)
at io.trino.plugin.lance.LanceMetadata.createTable(LanceMetadata.java:642)
at io.trino.tracing.TracingConnectorMetadata.createTable(TracingConnectorMetadata.java:379)
at io.trino.metadata.MetadataManager.createTable(MetadataManager.java:862)
at io.trino.tracing.TracingMetadata.createTable(TracingMetadata.java:403)
at io.trino.execution.CreateTableTask.internalExecute(CreateTableTask.java:297)
at io.trino.execution.CreateTableTask.execute(CreateTableTask.java:126)
at io.trino.execution.CreateTableTask.execute(CreateTableTask.java:92)
at io.trino.execution.DataDefinitionExecution.start(DataDefinitionExecution.java:152)
at io.trino.execution.SqlQueryManager.createQuery(SqlQueryManager.java:272)
at io.trino.dispatcher.LocalDispatchQuery.startExecution(LocalDispatchQuery.java:150)
at io.trino.dispatcher.LocalDispatchQuery.lambda$waitForMinimumWorkers$1(LocalDispatchQuery.java:134)
at io.airlift.concurrent.MoreFutures.lambda$addSuccessCallback$12(MoreFutures.java:570)
at io.airlift.concurrent.MoreFutures$3.onSuccess(MoreFutures.java:545)
at com.google.common.util.concurrent.Futures$CallbackListener.run(Futures.java:1132)
at io.trino.$gen.Trino_476____20260217_204003_2.run(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1095)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:619)
at java.base/java.lang.Thread.run(Thread.java:1447)
Caused by: java.lang.RuntimeException: Failed to initialize MemoryUtil. You must start Java with --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED (See https://arrow.apache.org/docs/java/install.html)
at org.apache.arrow.memory.util.MemoryUtil.(MemoryUtil.java:140)
... 23 more
Caused by: java.lang.reflect.InaccessibleObjectException: Unable to make field long java.nio.Buffer.address accessible: module java.base does not "opens java.nio" to unnamed module @5c15e5f7
at java.base/java.lang.reflect.AccessibleObject.throwInaccessibleObjectException(AccessibleObject.java:353)
at java.base/java.lang.reflect.AccessibleObject.checkCanSetAccessible(AccessibleObject.java:329)
at java.base/java.lang.reflect.AccessibleObject.checkCanSetAccessible(AccessibleObject.java:277)
at java.base/java.lang.reflect.Field.checkCanSetAccessible(Field.java:179)
at java.base/java.lang.reflect.Field.setAccessible(Field.java:173)
at org.apache.arrow.memory.util.MemoryUtil.(MemoryUtil.java:84)
... 23 more

insert into lance6."default".users values( '111', 'Mike')

SQL Error [65536]: Query failed (#20260217_204422_00008_9tadn): Cannot invoke "io.trino.spi.connector.ConnectorTableMetadata.getTableSchema()" because the return value of "io.trino.spi.connector.ConnectorMetadata.getTableMetadata(io.trino.spi.connector.ConnectorSession, io.trino.spi.connector.ConnectorTableHandle)" is null

CREATE TABLE lance6.default.users1 (id VARCHAR, name VARCHAR);
SQL Error [65536]: Query failed (#20260217_204550_00012_9tadn): Could not initialize class org.apache.arrow.memory.util.MemoryUtil

java.lang.NoClassDefFoundError: Could not initialize class org.apache.arrow.memory.util.MemoryUtil
at org.apache.arrow.c.ArrowSchema.directBuffer(ArrowSchema.java:143)
at org.apache.arrow.c.ArrowSchema.markReleased(ArrowSchema.java:119)
at org.apache.arrow.c.ArrowSchema.allocateNew(ArrowSchema.java:108)
at org.lance.Dataset.create(Dataset.java:136)
at io.trino.plugin.lance.LanceMetadata.createEmptyDataset(LanceMetadata.java:1112)
at io.trino.plugin.lance.LanceMetadata.createTable(LanceMetadata.java:642)
at io.trino.tracing.TracingConnectorMetadata.createTable(TracingConnectorMetadata.java:379)
at io.trino.metadata.MetadataManager.createTable(MetadataManager.java:862)
at io.trino.tracing.TracingMetadata.createTable(TracingMetadata.java:403)
at io.trino.execution.CreateTableTask.internalExecute(CreateTableTask.java:297)
at io.trino.execution.CreateTableTask.execute(CreateTableTask.java:126)
at io.trino.execution.CreateTableTask.execute(CreateTableTask.java:92)
at io.trino.execution.DataDefinitionExecution.start(DataDefinitionExecution.java:152)
at io.trino.execution.SqlQueryManager.createQuery(SqlQueryManager.java:272)
at io.trino.dispatcher.LocalDispatchQuery.startExecution(LocalDispatchQuery.java:150)
at io.trino.dispatcher.LocalDispatchQuery.lambda$waitForMinimumWorkers$1(LocalDispatchQuery.java:134)
at io.airlift.concurrent.MoreFutures.lambda$addSuccessCallback$12(MoreFutures.java:570)
at io.airlift.concurrent.MoreFutures$3.onSuccess(MoreFutures.java:545)
at com.google.common.util.concurrent.Futures$CallbackListener.run(Futures.java:1132)
at io.trino.$gen.Trino_476____20260217_204003_2.run(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1095)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:619)
at java.base/java.lang.Thread.run(Thread.java:1447)
Caused by: java.lang.ExceptionInInitializerError: Exception java.lang.RuntimeException: Failed to initialize MemoryUtil. You must start Java with --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED (See https://arrow.apache.org/docs/java/install.html) [in thread "Query-20260217_204131_00003_9tadn-185"]
at org.apache.arrow.memory.util.MemoryUtil.(MemoryUtil.java:140)
... 23 more

select * from lance6.default.users1

SQL Error [65536]: Query failed (#20260217_204739_00021_9tadn): Cannot invoke "io.trino.spi.connector.ConnectorTableMetadata.getTableSchema()" because the return value of "io.trino.spi.connector.ConnectorMetadata.getTableMetadata(io.trino.spi.connector.ConnectorSession, io.trino.spi.connector.ConnectorTableHandle)" is null

[YuriyGavrilov]

One more finding - lance.impl=dir working but not the rest

lance connector with

connector.name=lance
lance.impl=dir
lance.root=s3://warehouse/test_warehouse
lance.storage.endpoint=http://minio:9000
lance.storage.region=us-east-1
lance.storage.path-style-access=true
lance.storage.allow-http=true
lance.storage.aws_access_key_id=minio-root-user
lance.storage.aws_secret_access_key=minio-root-password

rest

connector.name=lance
lance.impl=rest
lance.uri=http://lakekeeper:8181/catalog/v1/test_warehouse
lance.warehouse=test_warehouse
lance.manifest.cache-ttl=0s
lance.storage.endpoint=http://minio:9000
lance.storage.region=us-east-1
lance.storage.path-style-access=true
lance.storage.allow-http=true
lance.storage.aws_access_key_id=minio-root-user
lance.storage.aws_secret_access_key=minio-root-password
lance.storage.access_key_id=minio-root-user
lance.storage.secret_access_key=minio-root-password

also add java opt

  trino:
    build: ./trino
    environment:
      - CATALOG_MANAGEMENT=dynamic
      - LANCE_ALLOW_HTTP=true
      - AWS_ALLOW_HTTP=true
      - AWS_ACCESS_KEY_ID=minio-root-user
      - AWS_SECRET_ACCESS_KEY=minio-root-password
      - AWS_REGION=us-east-1
      - AWS_ENDPOINT_URL=http://minio:9000
      - JDK_JAVA_OPTIONS=--add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED
    healthcheck:
      test: ["CMD", "curl", "-I", "http://localhost:8080/v1/status"]
      interval: 2s
      timeout: 10s
      retries: 2
      start_period: 10s
    ports:
      - "9999:8080"
    volumes:
      - ./lance5.properties:/etc/trino/catalog/lance5.properties
      - ./lance_rest.properties:/etc/trino/catalog/lance_rest.properties
      - ./lance_ice.properties:/etc/trino/catalog/lance_ice.properties
    networks:
      - lakekeeper-network

try to test iceberg table and lance

create table lance5.test_schema5.users60 as select orderpriority, totalprice, orderkey, orderdate FROM tpch.sf10.orders;

create table demo1.test_schema5.users60 as select orderpriority, totalprice, orderkey, orderdate FROM tpch.sf10.orders;

select orderpriority, sum(totalprice), sum(orderkey) from lance5.test_schema5.users60 where orderkey in (3000007,1638,1639,1664,1665,1666,1502407,1502432,1502433,1502434,1502435,3940,3941,3942,3943,3968,3) group by orderpriority 

5-LOW	1117939.68	4510327
2-HIGH	593450.5499999999	3012084
4-NOT SPECIFIED	575390.4500000001	1511984
1-URGENT	448686.9	1505762

select orderpriority, sum(totalprice), sum(orderkey) from  demo1.test_schema5.users60 where orderkey in (3000007,1638,1639,1664,1665,1666,1502407,1502432,1502433,1502434,1502435,3940,3941,3942,3943,3968,3) group by orderpriority 

4-NOT SPECIFIED	548607.98	1511984
1-URGENT	459873.15	1505762
2-HIGH	629562.52	3012084
5-LOW	1060084.04	4510327