feat: Add C ABI for embedded seekdb (multi‑language SDKs)

.github/workflows/build-libseekdb.yml

@@ -0,0 +1,290 @@
+# Build, pack and upload libseekdb for multiple platforms (linux x64/arm64, macos arm64) to S3
+#
+# Reference build environments (use these systems/env as the standard):
+#   linux-x64:   runner ubuntu-22.04, container quay.io/pypa/manylinux2014_x86_64 (glibc 2.17, CentOS 7+), zip libseekdb-linux-x64.zip
+#   linux-arm64: runner ubuntu-22.04-arm, container quay.io/pypa/manylinux2014_aarch64 (glibc 2.17), zip libseekdb-linux-arm64.zip
+#   darwin-arm64: runner macos-14, native, zip libseekdb-darwin-arm64.zip (min macOS 11.0)
+#
+# macOS builds use runner macos-14 and set CMAKE_OSX_DEPLOYMENT_TARGET=11.0 so the dylib runs on macOS 11+ (Big Sur and later).
+# Artifacts: three platform zips (linux-x64, linux-arm64, darwin-arm64); combined artifact libseekdb-all-platforms; optional S3 upload when DESTINATION_TARGET_PATH or AWS_S3_BUCKET and credentials are set.
+name: Build libseekdb
+run-name: Build libseekdb for ${{ env.COMMIT_SHA }}
+
+on:
+  push:
+    branches:
+      - main
+      - master
+      - develop
+      - "*.*.x"
+      - "integration/*"
+    paths-ignore:
+      - "*.md"
+      - "LICENSE"
+      - "CODEOWNERS"
+      - "docs/**"
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: 'Branch, tag or commit SHA to build (empty = use default branch)'
+        required: false
+        type: string
+        default: ''
+  pull_request:
+    paths-ignore:
+      - "*.md"
+      - "LICENSE"
+      - "CODEOWNERS"
+      - "docs/**"
+
+env:
+  # Commit/ref actually built: workflow_dispatch input ref, PR head, or event commit (push)
+  COMMIT_SHA: ${{ github.event_name == 'workflow_dispatch' && inputs.ref != '' && inputs.ref || (github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha) }}
+  AWS_REGION: ${{ vars.AWS_REGION || 'ap-southeast-1' }}
+  # S3 upload defaults (override via Variables: DESTINATION_TARGET_PATH or AWS_S3_BUCKET)
+  BUCKET_NAME: ${{ vars.AWS_S3_BUCKET || 'oceanbase-seekdb-builds' }}
+  DESTINATION_TARGET_PATH: ${{ vars.DESTINATION_TARGET_PATH || format('s3://oceanbase-seekdb-builds/libseekdb/all_commits/{0}', env.COMMIT_SHA) }}
+  S3_BUCKET: ${{ vars.AWS_S3_BUCKET || 'oceanbase-seekdb-builds' }}
+  S3_PREFIX: libseekdb/all_commits/${{ env.COMMIT_SHA }}
+
+jobs:
+  # ---------- Build libseekdb on Linux / macOS ----------
+  build:
+    name: Build libseekdb (${{ matrix.platform }})
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux-x64
+            runner: ubuntu-22.04
+            artifact_name: libseekdb-linux-x64
+            container_image: quay.io/pypa/manylinux2014_x86_64
+            deps_file: oceanbase.el7.x86_64.deps
+          - platform: linux-arm64
+            runner: ubuntu-22.04-arm
+            artifact_name: libseekdb-linux-arm64
+            container_image: quay.io/pypa/manylinux2014_aarch64
+            deps_file: oceanbase.el7.aarch64.deps
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # workflow_dispatch: use inputs.ref if set; PR: head sha; push: event sha
+          ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref != '' && inputs.ref || (github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha) }}
+
+      - name: Cache deps (Linux, el7)
+        uses: actions/cache@v4
+        with:
+          path: deps/3rd
+          key: ${{ runner.os }}-libseekdb-deps-${{ matrix.platform }}-el7-${{ hashFiles(format('deps/init/{0}', matrix.deps_file)) }}
+          restore-keys: |
+            ${{ runner.os }}-libseekdb-deps-${{ matrix.platform }}-el7-
+
+      - name: Cache ccache (Linux)
+        uses: actions/cache@v4
+        with:
+          path: .ccache
+          # Static key per platform so second run always hits; ccache content is keyed by source hash internally
+          key: ${{ runner.os }}-ccache-libseekdb-${{ matrix.platform }}
+          restore-keys: |
+            ${{ runner.os }}-ccache-libseekdb-${{ matrix.platform }}
+
+      # Run build inside manylinux2014 so Node/actions run on host (glibc 2.28+), build runs in CentOS 7 (glibc 2.17) for compatibility
+      - name: Build libseekdb (Linux, manylinux2014)
+        env:
+          BUILD_TYPE: release
+        run: |
+          docker run --rm -u root \
+            -v "$GITHUB_WORKSPACE:$GITHUB_WORKSPACE" -w "$GITHUB_WORKSPACE" \
+            -e BUILD_TYPE -e GITHUB_WORKSPACE \
+            ${{ matrix.container_image }} \
+            bash -c '
+              set -e
+              # Install deps in two steps: base first (wget required for dep_create.sh), then ccache if available
+              yum install -y git wget rpm cpio make glibc-devel glibc-headers binutils m4 python3 python3-devel libtool libaio ncurses-devel which zlib-devel
+              yum install -y epel-release 2>/dev/null || true
+              yum install -y ccache 2>/dev/null || true
+              git config --global --add safe.directory "$GITHUB_WORKSPACE"
+              export CCACHE_DIR="$GITHUB_WORKSPACE/.ccache"
+              export CCACHE_COMPILERCHECK=content
+              export CCACHE_NOHASHDIR=1
+              mkdir -p deps/3rd/usr/local/oceanbase/devtools/bin
+              CCACHE_SRC=$(command -v ccache 2>/dev/null || true)
+              if [ -n "$CCACHE_SRC" ] && [ -x "$CCACHE_SRC" ]; then
+                ln -sf "$CCACHE_SRC" deps/3rd/usr/local/oceanbase/devtools/bin/ccache
+                USE_CCACHE="-DOB_USE_CCACHE=ON"
+              else
+                USE_CCACHE="-DOB_USE_CCACHE=OFF"
+              fi
+              export PATH="$GITHUB_WORKSPACE/deps/3rd/usr/local/oceanbase/devtools/bin:$PATH"
+              bash build.sh init
+              if [ -x /opt/python/cp39-cp39/bin/python3.9 ]; then
+                export PATH="/opt/python/cp39-cp39/bin:$PATH"
+                PYVER=3.9
+              else
+                PYVER=$(python3 -c "import sys; print(f\"{sys.version_info.major}.{sys.version_info.minor}\")")
+              fi
+              bash build.sh release --init $USE_CCACHE -DBUILD_EMBED_MODE=ON -DPYTHON_VERSION=$PYVER
+              cd build_release && make -j$(nproc) libseekdb
+              [ -n "$USE_CCACHE" ] && ccache -s || true
+              cd "$GITHUB_WORKSPACE/package/libseekdb" && bash libseekdb-build.sh "$GITHUB_WORKSPACE/build_release/src/include"
+            '
+      - name: Fix ownership (container writes as root)
+        run: sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE"
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.artifact_name }}
+          path: package/libseekdb/libseekdb-*.zip
+
+      # Save caches even on failure so next run can resume (deps/ccache)
+      - name: Save Cache deps (Linux)
+        if: always()
+        uses: actions/cache/save@v4
+        with:
+          path: deps/3rd
+          key: ${{ runner.os }}-libseekdb-deps-${{ matrix.platform }}-el7-${{ hashFiles(format('deps/init/{0}', matrix.deps_file)) }}
+      - name: Save Cache ccache (Linux)
+        if: always()
+        uses: actions/cache/save@v4
+        with:
+          path: .ccache
+          key: ${{ runner.os }}-ccache-libseekdb-${{ matrix.platform }}
+
+  # ---------- Build on macOS (no container) ----------
+  build-macos:
+    name: Build libseekdb (${{ matrix.platform }})
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: darwin-arm64
+            runner: macos-14
+            artifact_name: libseekdb-darwin-arm64
+            arch: arm64
+            cmake_arch: arm64
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref != '' && inputs.ref || (github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha) }}
+
+      - name: Install macOS dependencies
+        run: brew install cmake dylibbundler googletest ccache pybind11 utf8proc thrift re2 brotli bzip2 || true
+
+      - name: Cache deps (macOS)
+        uses: actions/cache@v4
+        with:
+          path: deps/3rd
+          key: ${{ runner.os }}-libseekdb-deps-${{ matrix.platform }}-${{ hashFiles('deps/init/oceanbase.macos.arm64.deps') }}
+          restore-keys: |
+            ${{ runner.os }}-libseekdb-deps-${{ matrix.platform }}-
+
+      - name: Cache ccache (macOS)
+        uses: actions/cache@v4
+        with:
+          path: .ccache
+          # Static key per platform so second run always hits
+          key: ${{ runner.os }}-ccache-libseekdb-${{ matrix.platform }}
+          restore-keys: |
+            ${{ runner.os }}-ccache-libseekdb-${{ matrix.platform }}
+
+      - name: Build init (macOS)
+        run: bash build.sh init
+
+      - name: Build libseekdb (macOS)
+        env:
+          BUILD_TYPE: release
+          ARCH: ${{ matrix.arch }}
+          CMAKE_OSX_ARCHITECTURES: ${{ matrix.cmake_arch }}
+          CCACHE_DIR: ${{ github.workspace }}/.ccache
+          CCACHE_COMPILERCHECK: content
+          CCACHE_NOHASHDIR: 1
+        run: |
+          # Env.cmake looks for ccache in deps/3rd/.../devtools/bin; put it there so no Env.cmake change is needed
+          mkdir -p deps/3rd/usr/local/oceanbase/devtools/bin
+          ln -sf "$(which ccache)" deps/3rd/usr/local/oceanbase/devtools/bin/ccache
+          # Use runner's Python (macOS has 3.x, not 3.8 by default) so embed CMake finds it
+          PYVER=$(python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')")
+          bash build.sh release --init -DOB_USE_CCACHE=ON -DBUILD_EMBED_MODE=ON -DPYTHON_VERSION=$PYVER -DCMAKE_OSX_ARCHITECTURES=${{ matrix.cmake_arch }} -DCMAKE_OSX_DEPLOYMENT_TARGET=11.0
+          cd build_release && make -j$(sysctl -n hw.ncpu) libseekdb
+          ccache -s
+
+      - name: Pack libseekdb (macOS)
+        env:
+          ARCH: ${{ matrix.arch }}
+        run: cd package/libseekdb && bash libseekdb-build.sh
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.artifact_name }}
+          path: package/libseekdb/libseekdb-*.zip
+
+      # Save caches even on failure so next run can resume (deps/ccache)
+      - name: Save Cache deps (macOS)
+        if: always()
+        uses: actions/cache/save@v4
+        with:
+          path: deps/3rd
+          key: ${{ runner.os }}-libseekdb-deps-${{ matrix.platform }}-${{ hashFiles('deps/init/oceanbase.macos.arm64.deps') }}
+      - name: Save Cache ccache (macOS)
+        if: always()
+        uses: actions/cache/save@v4
+        with:
+          path: .ccache
+          key: ${{ runner.os }}-ccache-libseekdb-${{ matrix.platform }}
+
+  # ---------- Collect libseekdb artifacts and upload to S3 ----------
+  release-artifacts:
+    name: Collect artifacts and upload to S3
+    runs-on: ubuntu-22.04
+    needs:
+      - build
+      - build-macos
+    steps:
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: release-artifacts
+          merge-multiple: true
+
+      - name: List all artifacts
+        run: |
+          echo "=== All artifacts ==="
+          ls -la release-artifacts/
+
+      - name: Upload combined artifact (for workflow download)
+        uses: actions/upload-artifact@v4
+        with:
+          name: libseekdb-all-platforms
+          path: release-artifacts/
+
+      - name: Configure AWS credentials
+        if: env.DESTINATION_TARGET_PATH != '' || env.S3_BUCKET != ''
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Upload to S3
+        if: env.DESTINATION_TARGET_PATH != '' || env.S3_BUCKET != ''
+        run: |
+          set -e
+          if [ -n "${{ env.DESTINATION_TARGET_PATH }}" ]; then
+            S3_TARGET="${{ env.DESTINATION_TARGET_PATH }}"
+          else
+            S3_TARGET="s3://${{ env.S3_BUCKET }}/${{ env.S3_PREFIX }}/"
+          fi
+          [ "${S3_TARGET: -1}" != "/" ] && S3_TARGET="${S3_TARGET}/"
+          echo "Uploading to $S3_TARGET"
+          aws s3 cp release-artifacts/ "$S3_TARGET" --recursive --exclude "*" --include "*.zip" --no-progress
+          echo "Uploaded:"
+          aws s3 ls "$S3_TARGET" --recursive
+          echo "Done."
+        continue-on-error: true

.github/workflows/buildbase/action.yml

@@ -25,11 +25,58 @@ runs:
         max-size: 800M
         save: ${{inputs.save_cache}}
         key: ${{inputs.os}}
+        # With append-timestamp (default true), saved key becomes os-<timestamp>; restore-keys enables prefix match on restore
+        restore-keys: |
+          ${{ inputs.os }}
 
     - name: Build project
       shell: bash
       run: |
         bash build.sh debug -DOB_USE_CCACHE=ON
         ccache -z
-        cd build_debug && make -j4
+        cd build_release && make -j4
         ccache -s
+
+    # Use Python already installed by caller (e.g. compile.yml: python3 on Ubuntu). Pass version so embed CMake finds it.
+    - name: Build libseekdb
+      shell: bash
+      run: |
+        PYVER=$(python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')")
+        bash build.sh release -DOB_USE_CCACHE=ON -DBUILD_EMBED_MODE=ON -DPYTHON_VERSION=$PYVER
+        ccache -z
+        cd build_release && make -j4 libseekdb
+        ccache -s
+
+    - name: Test Node.js FFI binding
+      shell: bash
+      run: |
+        cd unittest/include/nodejs
+        npm install
+        bash test.sh
+
+    # Use system compiler/headers for node-gyp; deps/3rd GCC has different sysroot and breaks bits/libc-header-start.h
+    - name: Test Node.js N-API binding
+      shell: bash
+      run: |
+        PATH=$(echo "$PATH" | tr ':' '\n' | grep -v 'deps/3rd' | tr '\n' ':' | sed 's/:$//')
+        cd unittest/include/nodejs_napi
+        npm install
+        bash test.sh
+
+    - name: Test Python binding
+      shell: bash
+      run: |
+        cd unittest/include/python
+        bash test.sh
+
+    - name: Test Rust binding
+      shell: bash
+      run: |
+        cd unittest/include/rust
+        bash test.sh
+
+    - name: Test Go binding
+      shell: bash
+      run: |
+        cd unittest/include/go
+        bash test.sh

.github/workflows/compile.yml

@@ -40,7 +40,22 @@ jobs:
       run: |
         export DEBIAN_FRONTEND=noninteractive
         sudo apt-get update
-        sudo apt-get install -y git wget rpm rpm2cpio cpio make build-essential binutils m4 libtool-bin libncurses5 python3
+        sudo apt-get install -y git wget rpm rpm2cpio cpio make build-essential binutils m4 libtool-bin libncurses5 python3 python3-pip
+
+    - name: Setup Node.js
+      uses: actions/setup-node@v3
+      with:
+        node-version: '18'
+
+    - name: Setup Rust
+      uses: dtolnay/rust-toolchain@stable
+      with:
+        toolchain: stable
+
+    - name: Setup Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: '1.21'
 
     - name: Cache deps
       id: cache-deps

.gitignore

@@ -384,6 +384,9 @@ tools/ob-configserver/bin/*
 tools/ob-configserver/tests/*.log
 tools/ob-configserver/tests/*.out
 
+############# package/libseekdb #############
+package/libseekdb/libseekdb-*.zip
+
 ############# deps #############
 deps/oblib/src/common/storage/ob_io_selector.h
 deps/3rd