refactor(translator): move into src layout and use uv properly (#213)

Now translator is actually a proper python package. We could publish it
in pypy if we were so inclined, but for now, at least it's using uv
workspaces and actually is setup to not be so intermingled with
scram-django. I also updated scheduler to now use the root lockfile so
that we can use one lockfile to rule them all. We also have a pre-commit
step to check for an out of date lockfile to make sure we don't change
pyproject but forget to update the lock. Having the lock file committed
allows us to have our builds actually be reproducible, which is nice.

Also updates django-simple-history as `pkg_resources` seems to have been
broken and 3.4+ removes that dependency (and is still compatible with
django 4.2)

Author: crankynetman

.github/workflows/behave.yml

@@ -73,6 +73,7 @@ jobs:
         with:
             parallel: true
             flag-name: behave
+            fail-on-error: false
 
       - name: Upload Coverage to GitHub
         if: matrix.python-version == '3.12'

.github/workflows/coverage-finish.yml

@@ -18,3 +18,4 @@ jobs:
         uses: coverallsapp/github-action@v2
         with:
           parallel-finished: true
+          fail-on-error: false

.github/workflows/pytest.yml

@@ -98,6 +98,7 @@ jobs:
           file: ./scheduler/coverage.xml
           parallel: true
           flag-name: scheduler
+          fail-on-error: false
 
       - name: Display Scheduler Coverage Metrics for scheduler tests
         if: matrix.python-version == '3.12'

.pre-commit-config.yaml

@@ -15,3 +15,9 @@ repos:
       - id: ruff-format
       - id: ruff-check
         args: [--fix, --exit-non-zero-on-fix]
+
+  - repo: https://github.com/astral-sh/uv-pre-commit
+    rev: 0.10.7
+    hooks:
+      - id: uv-lock
+        args: [--check]

.vscode/launch.json

@@ -44,7 +44,7 @@
             },
             "pathMappings": [
                 {
-                    "localRoot": "${workspaceFolder}/translator",
+                    "localRoot": "${workspaceFolder}/translator/src/translator",
                     "remoteRoot": "/app"
                 }
             ]
@@ -59,7 +59,7 @@
           },
           "pathMappings": [
               {
-                  "localRoot": "${workspaceFolder}/translator",
+                  "localRoot": "${workspaceFolder}/translator/src/translator",
                   "remoteRoot": "/app"
               }
           ]

Makefile

@@ -41,7 +41,7 @@ integration-tests: run
 ## behave-translator
 .Phony: behave-translator
 behave-translator: compose.override.yml
-	@docker compose exec -T translator /usr/local/bin/behave /app/tests/acceptance/features
+	@docker compose exec -T translator behave /app/tests/acceptance/features
 
 ## build: rebuilds all your containers or a single one if CONTAINER is specified
 .Phony: build

compose.override.local.yml

@@ -105,6 +105,9 @@ services:
       replicas: 1
 
   translator:
+    build:
+      context: .
+      dockerfile: ./compose/local/translator/Dockerfile
     volumes:
       - ./translator/tests/:/app/tests/
     env_file:
@@ -117,6 +120,9 @@ services:
       - DEBUG=${DEBUG:-}
 
   translator-secondary:
+    build:
+      context: .
+      dockerfile: ./compose/local/translator/Dockerfile
     volumes:
       - ./translator/tests/:/app/tests/
     env_file:

compose.yml

@@ -101,7 +101,7 @@ services:
   translator:
     build:
       context: .
-      dockerfile: ./compose/local/translator/Dockerfile
+      dockerfile: ./compose/production/translator/Dockerfile
     depends_on:
       redis:
         condition: service_healthy
@@ -119,7 +119,7 @@ services:
   translator-secondary:
     build:
       context: .
-      dockerfile: ./compose/local/translator/Dockerfile
+      dockerfile: ./compose/production/translator/Dockerfile
     depends_on:
       redis:
         condition: service_healthy

compose/local/translator/Dockerfile

@@ -1,7 +1,8 @@
-FROM python:3.13-slim-bookworm
+FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim
 
-ENV PYTHONUNBUFFERED 1
-ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED=1
+ENV UV_COMPILE_BYTECODE=1
+ENV UV_LINK_MODE=copy
 
 RUN apt-get update \
   # dependencies for building Python packages
@@ -10,24 +11,39 @@ RUN apt-get update \
   && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
   && rm -rf /var/lib/apt/lists/*
 
-# Requirements are installed here to ensure they will be cached.
-COPY ./translator/requirements /requirements
-RUN pip install uv
-RUN uv pip install --system -r /requirements/base.txt
+WORKDIR /app
 
-RUN mkdir /app \
-  && cd /app \
-  && git clone -b v3.33.0 https://github.com/osrg/gobgp.git \
-  && cd gobgp/api \
-  && python3 -m grpc_tools.protoc -I./ --python_out=/app/ --pyi_out=/app/ --grpc_python_out=/app/ *.proto
+# At first we install  the deps only (as protobuf stuff doesn't exist yet but we need deps to setup protobufs). We also
+# mount instead of copying to have a small build cache gain. Not that useful but is modern practice so why not.
+RUN --mount=type=cache,target=/root/.cache/uv \
+  --mount=type=bind,source=uv.lock,target=uv.lock \
+  --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+  --mount=type=bind,source=translator/pyproject.toml,target=translator/pyproject.toml \
+  uv sync --frozen --no-install-project --package translator
 
-COPY ./translator/translator.py /app
-COPY ./translator/gobgp.py /app
-COPY ./translator/exceptions.py /app
-COPY ./translator/shared.py /app
+# put the venv into PATH
+ENV PATH="/app/.venv/bin:$PATH"
 
-RUN chmod +x /app/translator.py
+COPY ./translator/src ./translator/src/
 
-WORKDIR /app
-
-ENTRYPOINT ["/app/translator.py"]
+# Generate protobuf files
+# TODO: Make this less jank and don't do the git clone crap?
+# Protos don't work well in packages so we generate them in /app/ outside the package so that bare
+# `import gobgp_pb2` style imports work.
+RUN git clone -b v3.33.0 https://github.com/osrg/gobgp.git gobgp \
+  && cd gobgp/api \
+  && python3 -m grpc_tools.protoc -I./ \
+  --python_out=/app/ \
+  --pyi_out=/app/ \
+  --grpc_python_out=/app/ \
+  *.proto \
+  && cd /app && rm -rf gobgp
+
+# Install the project itself (including the protobuff stuff) again mounting in the project files.
+RUN --mount=type=cache,target=/root/.cache/uv \
+  --mount=type=bind,source=uv.lock,target=uv.lock \
+  --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+  --mount=type=bind,source=translator/pyproject.toml,target=translator/pyproject.toml \
+  uv sync --frozen --package translator
+
+ENTRYPOINT ["python", "-m", "translator.translator"]

compose/production/celery/Dockerfile

@@ -21,19 +21,23 @@ ENV UV_NO_DEV=1
 ENV UV_TOOL_BIN_DIR=/usr/local/bin
 
 COPY ./scheduler/pyproject.toml ./scheduler/uv.lock* ./
-
-# Install the project's dependencies using the lockfile and settings
+# Install the project's dependencies only
 RUN --mount=type=cache,target=/root/.cache/uv \
-    --mount=type=bind,source=scheduler/uv.lock,target=uv.lock \
-    --mount=type=bind,source=scheduler/pyproject.toml,target=pyproject.toml \
-    uv sync --locked --no-install-project
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    --mount=type=bind,source=scheduler/pyproject.toml,target=scheduler/pyproject.toml \
+    uv sync --frozen --no-install-project --package scheduler
 
 # Then, add the rest of the project source code and install it
-# Installing separately from its dependencies allows optimal layer caching
-COPY ./scheduler/src /app/
+# Installing separately from its dependencies allows optimal layer caching. We use --frozen since we
+# have differing python versions across the stack.
+COPY ./scheduler/src ./scheduler/src/
 
 RUN --mount=type=cache,target=/root/.cache/uv \
-    uv sync --locked
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    --mount=type=bind,source=scheduler/pyproject.toml,target=scheduler/pyproject.toml \
+    uv sync --frozen --package scheduler
 
 # Place executables in the environment at the front of the path
 ENV PATH="/app/.venv/bin:$PATH"