Added LiteLLM to the stack

Development/litellm/docker/.env.example

@@ -0,0 +1,22 @@
+############
+# Secrets
+# YOU MUST CHANGE THESE BEFORE GOING INTO PRODUCTION
+############
+
+LITELLM_MASTER_KEY="sk-1234"
+
+############
+# Database - You can change these to any PostgreSQL database that has logical replication enabled.
+############
+
+DATABASE_URL="your-postgres-db-url" 
+
+
+############
+# User Auth - SMTP server details for email-based auth for users to create keys 
+############
+
+# SMTP_HOST = "fake-mail-host"
+# SMTP_USERNAME = "fake-mail-user"
+# SMTP_PASSWORD="fake-mail-password"
+# SMTP_SENDER_EMAIL="fake-sender-email"

Development/litellm/docker/Dockerfile.alpine

@@ -0,0 +1,56 @@
+# Base image for building
+ARG LITELLM_BUILD_IMAGE=python:3.11-alpine
+
+# Runtime image
+ARG LITELLM_RUNTIME_IMAGE=python:3.11-alpine
+
+# Builder stage
+FROM $LITELLM_BUILD_IMAGE AS builder
+
+# Set the working directory to /app
+WORKDIR /app
+
+# Install build dependencies
+RUN apk add --no-cache gcc python3-dev musl-dev
+
+RUN pip install --upgrade pip && \
+    pip install build
+
+# Copy the current directory contents into the container at /app
+COPY . .
+
+# Build the package
+RUN rm -rf dist/* && python -m build
+
+# There should be only one wheel file now, assume the build only creates one
+RUN ls -1 dist/*.whl | head -1
+
+# Install the package
+RUN pip install dist/*.whl
+
+# install dependencies as wheels
+RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
+
+# Runtime stage
+FROM $LITELLM_RUNTIME_IMAGE AS runtime
+
+# Update dependencies and clean up
+RUN apk upgrade --no-cache
+
+WORKDIR /app
+
+# Copy the built wheel from the builder stage to the runtime stage; assumes only one wheel file is present
+COPY --from=builder /app/dist/*.whl .
+COPY --from=builder /wheels/ /wheels/
+
+# Install the built wheel using pip; again using a wildcard if it's the only file
+RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
+
+RUN chmod +x docker/entrypoint.sh
+RUN chmod +x docker/prod_entrypoint.sh
+
+EXPOSE 4000/tcp
+
+# Set your entrypoint and command
+ENTRYPOINT ["docker/prod_entrypoint.sh"]
+CMD ["--port", "4000"]

Development/litellm/docker/Dockerfile.custom_ui

@@ -0,0 +1,42 @@
+# Use the provided base image
+FROM ghcr.io/berriai/litellm:litellm_fwd_server_root_path-dev
+
+# Set the working directory to /app
+WORKDIR /app
+
+# Install Node.js and npm (adjust version as needed)
+RUN apt-get update && apt-get install -y nodejs npm
+
+# Copy the UI source into the container
+COPY ./ui/litellm-dashboard /app/ui/litellm-dashboard
+
+# Set an environment variable for UI_BASE_PATH
+# This can be overridden at build time
+# set UI_BASE_PATH to "<your server root path>/ui"
+ENV UI_BASE_PATH="/prod/ui"
+
+# Build the UI with the specified UI_BASE_PATH
+WORKDIR /app/ui/litellm-dashboard
+RUN npm install
+RUN UI_BASE_PATH=$UI_BASE_PATH npm run build
+
+# Create the destination directory
+RUN mkdir -p /app/litellm/proxy/_experimental/out
+
+# Move the built files to the appropriate location
+# Assuming the build output is in ./out directory
+RUN rm -rf /app/litellm/proxy/_experimental/out/* && \
+    mv ./out/* /app/litellm/proxy/_experimental/out/
+
+# Switch back to the main app directory
+WORKDIR /app
+
+# Make sure your docker/entrypoint.sh is executable
+RUN chmod +x docker/entrypoint.sh
+RUN chmod +x docker/prod_entrypoint.sh
+
+# Expose the necessary port
+EXPOSE 4000/tcp
+
+# Override the CMD instruction with your desired command and arguments
+CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug"]

Development/litellm/docker/Dockerfile.database

@@ -0,0 +1,87 @@
+# Base image for building
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/python:latest-dev
+
+# Runtime image
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/python:latest-dev
+# Builder stage
+FROM $LITELLM_BUILD_IMAGE AS builder
+
+# Set the working directory to /app
+WORKDIR /app
+
+USER root
+
+# Install build dependencies
+RUN apk add --no-cache gcc python3-dev openssl openssl-dev
+
+
+RUN pip install --upgrade pip && \
+    pip install build
+
+# Copy the current directory contents into the container at /app
+COPY . .
+
+# Build Admin UI
+RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
+
+# Build the package
+RUN rm -rf dist/* && python -m build
+
+# There should be only one wheel file now, assume the build only creates one
+RUN ls -1 dist/*.whl | head -1
+
+# Install the package
+RUN pip install dist/*.whl
+
+# install dependencies as wheels
+RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
+
+# Runtime stage
+FROM $LITELLM_RUNTIME_IMAGE AS runtime
+
+# Ensure runtime stage runs as root
+USER root
+
+# Install runtime dependencies
+RUN apk add --no-cache openssl
+
+WORKDIR /app
+# Copy the current directory contents into the container at /app
+COPY . .
+RUN ls -la /app
+
+# Copy the built wheel from the builder stage to the runtime stage; assumes only one wheel file is present
+COPY --from=builder /app/dist/*.whl .
+COPY --from=builder /wheels/ /wheels/
+
+# Install the built wheel using pip; again using a wildcard if it's the only file
+RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
+
+# Install semantic_router without dependencies
+RUN pip install semantic_router --no-deps
+
+# ensure pyjwt is used, not jwt
+RUN pip uninstall jwt -y
+RUN pip uninstall PyJWT -y
+RUN pip install PyJWT==2.9.0 --no-cache-dir
+
+# Build Admin UI
+RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
+
+# Generate prisma client
+RUN prisma generate
+RUN chmod +x docker/entrypoint.sh
+RUN chmod +x docker/prod_entrypoint.sh
+EXPOSE 4000/tcp
+
+RUN apk add --no-cache supervisor
+COPY docker/supervisord.conf /etc/supervisord.conf
+
+# # Set your entrypoint and command
+
+
+ENTRYPOINT ["docker/prod_entrypoint.sh"]
+
+# Append "--detailed_debug" to the end of CMD to view detailed debug logs 
+# CMD ["--port", "4000", "--detailed_debug"]
+CMD ["--port", "4000"]

Development/litellm/docker/Dockerfile.dev

@@ -0,0 +1,87 @@
+# Base image for building
+ARG LITELLM_BUILD_IMAGE=python:3.11-slim
+
+# Runtime image
+ARG LITELLM_RUNTIME_IMAGE=python:3.11-slim
+
+# Builder stage
+FROM $LITELLM_BUILD_IMAGE AS builder
+
+# Set the working directory to /app
+WORKDIR /app
+
+USER root
+
+# Install build dependencies in one layer
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    gcc \
+    python3-dev \
+    libssl-dev \
+    pkg-config \
+    && rm -rf /var/lib/apt/lists/* \
+    && pip install --upgrade pip build
+
+# Copy requirements first for better layer caching
+COPY requirements.txt .
+
+# Install Python dependencies with cache mount for faster rebuilds
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
+
+# Fix JWT dependency conflicts early
+RUN pip uninstall jwt -y || true && \
+    pip uninstall PyJWT -y || true && \
+    pip install PyJWT==2.9.0 --no-cache-dir
+
+# Copy only necessary files for build
+COPY pyproject.toml README.md schema.prisma poetry.lock ./
+COPY litellm/ ./litellm/
+COPY enterprise/ ./enterprise/
+COPY docker/ ./docker/
+
+# Build Admin UI once
+RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
+
+# Build the package
+RUN rm -rf dist/* && python -m build
+
+# Install the built package
+RUN pip install dist/*.whl
+
+# Runtime stage
+FROM $LITELLM_RUNTIME_IMAGE AS runtime
+
+# Ensure runtime stage runs as root
+USER root
+
+# Install only runtime dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libssl3 \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Copy only necessary runtime files
+COPY docker/entrypoint.sh docker/prod_entrypoint.sh ./docker/
+COPY litellm/ ./litellm/
+COPY pyproject.toml README.md schema.prisma poetry.lock ./
+
+# Copy pre-built wheels and install everything at once
+COPY --from=builder /wheels/ /wheels/
+COPY --from=builder /app/dist/*.whl .
+
+# Install all dependencies in one step with no-cache for smaller image
+RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/ && \
+    rm -f *.whl && \
+    rm -rf /wheels
+
+# Generate prisma client and set permissions
+RUN prisma generate && \
+    chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh
+
+EXPOSE 4000/tcp
+
+ENTRYPOINT ["docker/prod_entrypoint.sh"]
+
+# Append "--detailed_debug" to the end of CMD to view detailed debug logs 
+CMD ["--port", "4000"]

Development/litellm/docker/Dockerfile.non_root

@@ -0,0 +1,104 @@
+# Base images
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/python:latest-dev
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/python:latest-dev
+
+# -----------------
+# Builder Stage
+# -----------------
+FROM $LITELLM_BUILD_IMAGE AS builder
+WORKDIR /app
+
+# Install build dependencies
+USER root
+RUN apk add --no-cache build-base bash \
+  && pip install --no-cache-dir --upgrade pip build
+
+# Copy project files
+COPY . .
+
+# Build Admin UI
+RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
+
+# Build package and wheel dependencies
+RUN rm -rf dist/* && python -m build && \
+  pip install dist/*.whl && \
+  pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
+
+# -----------------
+# Runtime Stage
+# -----------------
+FROM $LITELLM_RUNTIME_IMAGE AS runtime
+WORKDIR /app
+
+# Install runtime dependencies
+USER root
+RUN apk upgrade --no-cache && \
+  apk add --no-cache bash libstdc++ ca-certificates openssl
+
+# Copy only necessary artifacts from builder stage for runtime
+COPY . .
+COPY --from=builder /app/docker/entrypoint.sh /app/docker/prod_entrypoint.sh /app/docker/
+COPY --from=builder /app/schema.prisma /app/schema.prisma
+COPY --from=builder /app/dist/*.whl .
+COPY --from=builder /wheels/ /wheels/
+
+# Install package from wheel and dependencies
+RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ \
+  && rm -f *.whl \
+  && rm -rf /wheels
+
+# Install semantic_router without dependencies
+RUN pip install semantic_router --no-deps
+
+# Ensure correct JWT library is used (pyjwt not jwt)
+RUN pip uninstall jwt -y && \
+  pip uninstall PyJWT -y && \
+  pip install PyJWT==2.9.0 --no-cache-dir
+
+# --- Prisma Handling for Non-Root User ---
+# Set Prisma cache directories
+ENV PRISMA_BINARY_CACHE_DIR=/nonexistent
+ENV NPM_CONFIG_CACHE=/.npm
+
+# Install prisma and make entrypoints executable
+RUN pip install --no-cache-dir prisma && \
+  chmod +x docker/entrypoint.sh && \
+  chmod +x docker/prod_entrypoint.sh
+
+# Create directories and set permissions for non-root user
+RUN mkdir -p /nonexistent /.npm && \
+  chown -R nobody:nogroup /app && \
+  chown -R nobody:nogroup /nonexistent /.npm && \
+  PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
+  chown -R nobody:nogroup $PRISMA_PATH
+
+# --- OpenShift Compatibility: Apply Red Hat recommended pattern ---
+# Get paths for directories that need write access at runtime
+RUN PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
+  LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \
+  # Set group ownership to 0 (root group) for OpenShift compatibility && \
+  chgrp -R 0 $PRISMA_PATH && \
+  [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 $LITELLM_PROXY_EXTRAS_PATH || true && \
+  # Mirror owner permissions to group (g=u) as recommended by Red Hat && \
+  chmod -R g=u $PRISMA_PATH && \
+  [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u $LITELLM_PROXY_EXTRAS_PATH || true && \
+  # Ensure directories are writable by group && \
+  chmod -R g+w $PRISMA_PATH && \
+  [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true
+
+# Switch to non-root user
+USER nobody
+
+# Set HOME for prisma generate to have a writable directory
+ENV HOME=/app
+RUN prisma generate
+# --- End of Prisma Handling ---
+
+EXPOSE 4000/tcp
+
+# Set entrypoint and command
+ENTRYPOINT ["/app/docker/prod_entrypoint.sh"]
+
+# Append "--detailed_debug" to the end of CMD to view detailed debug logs
+# CMD ["--port", "4000", "--detailed_debug"]
+CMD ["--port", "4000"]

Development/litellm/docker/README.md

@@ -0,0 +1,65 @@
+# Docker Development Guide
+
+This guide provides instructions for building and running the LiteLLM application using Docker and Docker Compose.
+
+## Prerequisites
+
+- Docker
+- Docker Compose
+
+## Building and Running the Application
+
+To build and run the application, you will use the `docker-compose.yml` file located in the root of the project. This file is configured to use the `Dockerfile.non_root` for a secure, non-root container environment.
+
+### 1. Set the Master Key
+
+The application requires a `MASTER_KEY` for signing and validating tokens. You must set this key as an environment variable before running the application.
+
+Create a `.env` file in the root of the project and add the following line:
+
+```
+MASTER_KEY=your-secret-key
+```
+
+Replace `your-secret-key` with a strong, randomly generated secret.
+
+### 2. Build and Run the Containers
+
+Once you have set the `MASTER_KEY`, you can build and run the containers using the following command:
+
+```bash
+docker-compose up -d --build
+```
+
+This command will:
+
+-   Build the Docker image using `Dockerfile.non_root`.
+-   Start the `litellm`, `litellm_db`, and `prometheus` services in detached mode (`-d`).
+-   The `--build` flag ensures that the image is rebuilt if there are any changes to the Dockerfile or the application code.
+
+### 3. Verifying the Application is Running
+
+You can check the status of the running containers with the following command:
+
+```bash
+docker-compose ps
+```
+
+To view the logs of the `litellm` container, run:
+
+```bash
+docker-compose logs -f litellm
+```
+
+### 4. Stopping the Application
+
+To stop the running containers, use the following command:
+
+```bash
+docker-compose down
+```
+
+## Troubleshooting
+
+-   **`build_admin_ui.sh: not found`**: This error can occur if the Docker build context is not set correctly. Ensure that you are running the `docker-compose` command from the root of the project.
+-   **`Master key is not initialized`**: This error means the `MASTER_key` environment variable is not set. Make sure you have created a `.env` file in the project root with the `MASTER_KEY` defined.

Development/litellm/docker/build_admin_ui.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+# # try except this script
+# set -e
+
+# print current dir 
+echo
+pwd
+
+
+# only run this step for litellm enterprise, we run this if enterprise/enterprise_ui/_enterprise.json exists
+if [ ! -f "enterprise/enterprise_ui/enterprise_colors.json" ]; then
+    echo "Admin UI - using default LiteLLM UI"
+    exit 0
+fi
+
+echo "Building Custom Admin UI..."
+
+# Install dependencies
+# Check if we are on macOS
+if [[ "$(uname)" == "Darwin" ]]; then
+    # Install dependencies using Homebrew
+    if ! command -v brew &> /dev/null; then
+        echo "Error: Homebrew not found. Please install Homebrew and try again."
+        exit 1
+    fi
+    brew update
+    brew install curl
+else
+    # Assume Linux, try using apt-get
+    if command -v apt-get &> /dev/null; then
+        apt-get update
+        apt-get install -y curl
+    elif command -v apk &> /dev/null; then
+        # Try using apk if apt-get is not available
+        apk update
+        apk add curl
+    else
+        echo "Error: Unsupported package manager. Cannot install dependencies."
+        exit 1
+    fi
+fi
+curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash
+source ~/.nvm/nvm.sh
+nvm install v18.17.0
+nvm use v18.17.0
+npm install -g npm
+
+# copy _enterprise.json from this directory to /ui/litellm-dashboard, and rename it to ui_colors.json
+cp enterprise/enterprise_ui/enterprise_colors.json ui/litellm-dashboard/ui_colors.json
+
+# cd in to /ui/litellm-dashboard
+cd ui/litellm-dashboard
+
+# ensure have access to build_ui.sh
+chmod +x ./build_ui.sh
+
+# run ./build_ui.sh
+./build_ui.sh
+
+# return to root directory
+cd ../..

Development/litellm/docker/build_from_pip/Dockerfile.build_from_pip

@@ -0,0 +1,29 @@
+FROM cgr.dev/chainguard/python:latest-dev
+
+USER root
+WORKDIR /app
+
+ENV HOME=/home/litellm
+ENV PATH="${HOME}/venv/bin:$PATH"
+
+# Install runtime dependencies
+RUN apk update && \
+    apk add --no-cache gcc python3-dev openssl openssl-dev
+
+RUN python -m venv ${HOME}/venv
+RUN ${HOME}/venv/bin/pip install --no-cache-dir --upgrade pip
+
+COPY docker/build_from_pip/requirements.txt .
+RUN --mount=type=cache,target=${HOME}/.cache/pip \
+    ${HOME}/venv/bin/pip install -r requirements.txt
+
+# Copy Prisma schema file
+COPY schema.prisma .
+
+# Generate prisma client
+RUN prisma generate
+
+EXPOSE 4000/tcp
+
+ENTRYPOINT ["litellm"]
+CMD ["--port", "4000"]

Development/litellm/docker/build_from_pip/Readme.md

@@ -0,0 +1,9 @@
+# Docker to build LiteLLM Proxy from litellm pip package
+
+### When to use this ?
+
+If you need to build LiteLLM Proxy from litellm pip package, you can use this Dockerfile as a reference.
+
+### Why build from pip package ?
+
+- If your company has a strict requirement around security / building images you can follow steps outlined here 

Development/litellm/docker/build_from_pip/requirements.txt

@@ -0,0 +1,5 @@
+litellm[proxy]==1.67.4.dev1 # Specify the litellm version you want to use
+prometheus_client
+langfuse
+prisma
+ddtrace==2.19.0 # for advanced DD tracing / profiling

Development/litellm/docker/entrypoint.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+echo $(pwd)
+
+# Run the Python migration script
+python3 litellm/proxy/prisma_migration.py
+
+# Check if the Python script executed successfully
+if [ $? -eq 0 ]; then
+    echo "Migration script ran successfully!"
+else
+    echo "Migration script failed!"
+    exit 1
+fi

Development/litellm/docker/prod_entrypoint.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+if [ "$SEPARATE_HEALTH_APP" = "1" ]; then
+    export LITELLM_ARGS="$@"
+    exec supervisord -c /etc/supervisord.conf
+fi
+
+if [ "$USE_DDTRACE" = "true" ]; then
+    export DD_TRACE_OPENAI_ENABLED="False"
+    exec ddtrace-run litellm "$@"
+else
+    exec litellm "$@"
+fi

Development/litellm/docker/supervisord.conf

@@ -0,0 +1,42 @@
+[supervisord]
+nodaemon=true
+loglevel=info
+
+[group:litellm]
+programs=main,health
+
+[program:main]
+command=sh -c 'if [ "$USE_DDTRACE" = "true" ]; then export DD_TRACE_OPENAI_ENABLED="False"; exec ddtrace-run python -m litellm.proxy.proxy_cli --host 0.0.0.0 --port=4000 $LITELLM_ARGS; else exec python -m litellm.proxy.proxy_cli --host 0.0.0.0 --port=4000 $LITELLM_ARGS; fi'
+autostart=true
+autorestart=true
+startretries=3
+priority=1
+exitcodes=0
+stopasgroup=true
+killasgroup=true
+stdout_logfile=/dev/stdout
+stderr_logfile=/dev/stderr
+stdout_logfile_maxbytes = 0
+stderr_logfile_maxbytes = 0
+environment=PYTHONUNBUFFERED=true
+
+[program:health]
+command=sh -c '[ "$SEPARATE_HEALTH_APP" = "1" ] && exec uvicorn litellm.proxy.health_endpoints.health_app_factory:build_health_app --factory --host 0.0.0.0 --port=${SEPARATE_HEALTH_PORT:-4001} || exit 0'
+autostart=true
+autorestart=true
+startretries=3
+priority=2
+exitcodes=0
+stopasgroup=true
+killasgroup=true
+stdout_logfile=/dev/stdout
+stderr_logfile=/dev/stderr
+stdout_logfile_maxbytes = 0
+stderr_logfile_maxbytes = 0
+environment=PYTHONUNBUFFERED=true
+
+[eventlistener:process_monitor]
+command=python -c "from supervisor import childutils; import os, signal; [os.kill(os.getppid(), signal.SIGTERM) for h,p in iter(lambda: childutils.listener.wait(), None) if h['eventname'] in ['PROCESS_STATE_FATAL', 'PROCESS_STATE_EXITED'] and dict([x.split(':') for x in p.split(' ')])['processname'] in ['main', 'health'] or childutils.listener.ok()]"
+events=PROCESS_STATE_EXITED,PROCESS_STATE_FATAL
+autostart=true
+autorestart=true

Development/litellm/docker/tests/nonroot.yaml

@@ -0,0 +1,18 @@
+schemaVersion: 2.0.0
+
+metadataTest:
+  entrypoint: ["docker/prod_entrypoint.sh"]
+  user: "nobody"
+  workdir: "/app"
+
+fileExistenceTests:
+  - name: "Prisma Folder"
+    path: "/usr/local/lib/python3.13/site-packages/prisma/"
+    shouldExist: true
+    uid: 65534
+    gid: 65534
+  - name: "Prisma Schema"
+    path: "/usr/local/lib/python3.13/site-packages/prisma/schema.prisma"
+    shouldExist: true
+    uid: 65534
+    gid: 65534