ingest.yml

name: Build ChromaDB index
# This creates a index to the GSAS-II documents and places it as
# downloadable release file under the name 
# https://github.com/<owner>/<repo>/releases/download/latest-chroma-db/chroma_db_latest.zip
# (as well as a artifact)

on:
  schedule:
    - cron: "0 4 * * 1"   # every Monday 04:00 UTC — full rebuild with book+manual
  workflow_dispatch:
    inputs:
      include_book:
        description: "Include Book HTML contents"
        type: boolean
        default: true
      include_manual:
        description: "Include Programmers' Manual"
        type: boolean
        default: true

jobs:
  ingest:
    runs-on: ubuntu-latest
    timeout-minutes: 90
    permissions:
      #contents: read
      contents: write   # changed: needed to create/update release/tag

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Cache pip
        uses: actions/cache@v4
        with:
          path: ~/.cache/pip
          key: pip-ingest-${{ hashFiles('pyproject.toml') }}

      - name: Cache bge-base ONNX model
        uses: actions/cache@v4
        with:
          path: ${{ github.workspace }}/ci_data/models
          key: bge-base-en-v1.5-onnx-quantized

      - name: Install gsas_query package
        run: pip install -e ".[dev]"

      - name: Warm up embedding model (downloads to cache if cold)
        env:
          GSAS_QUERY_DATA_DIR: ${{ github.workspace }}/ci_data
        run: python -c "from gsas_query._embed import get_embedding_function; get_embedding_function()"

      - name: Build ChromaDB index (HTML only, contents by options)
        env:
          TOKENIZERS_PARALLELISM: "false"
          OMP_NUM_THREADS: "1"
          GSAS_QUERY_DATA_DIR: ${{ github.workspace }}/ci_data
        run: |
          cmd="python ingest.py --html-only"
          # Scheduled runs (no inputs) always include book+manual for a full rebuild.
          # Manual dispatch respects the checkboxes.
          include_book="${{ inputs.include_book }}"
          include_manual="${{ inputs.include_manual }}"
          if [ "${{ github.event_name }}" = "schedule" ] || [ "${include_book:-true}" = "true" ]; then
            cmd="$cmd --book"
          fi
          if [ "${{ github.event_name }}" = "schedule" ] || [ "${include_manual:-true}" = "true" ]; then
            cmd="$cmd --manual"
          fi
          echo "Running: $cmd"
          eval "$cmd"
          
      - name: Show index stats
        env:
          GSAS_QUERY_DATA_DIR: ${{ github.workspace }}/ci_data
        run: |
          python -c "
          import chromadb, os
          path = os.path.join(os.environ['GSAS_QUERY_DATA_DIR'], 'chroma_db')
          c = chromadb.PersistentClient(path=path)
          col = c.get_collection('gsasii_docs')
          print(f'Chunks in index: {col.count()}')
          "

      - name: Upload chroma_db artifact
        uses: actions/upload-artifact@v4
        with:
          name: chroma_db
          path: ci_data/chroma_db/
          retention-days: 90

      - name: Package chroma_db as zip
        run: |
          cd ci_data
          zip -r ../chroma_db_latest.zip chroma_db

      - name: Create/Update stable release and upload asset
        uses: softprops/action-gh-release@v2
        with:
          tag_name: latest-chroma-db
          name: Latest ChromaDB Index
          body: Auto-updated ChromaDB index artifact from ingest workflow.
          files: chroma_db_latest.zip
          overwrite_files: true
          make_latest: false
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}