Added LiteLLM to the stack

2025-08-18 09:40:50 +00:00
parent 0648c1968c
commit d220b04e32
2682 changed files with 533609 additions and 1 deletions
--- a/Development/litellm/tests/proxy_unit_tests/test_configs/custom_auth.py
+++ b/Development/litellm/tests/proxy_unit_tests/test_configs/custom_auth.py
@@ -0,0 +1,22 @@
+from litellm.proxy._types import UserAPIKeyAuth
+from fastapi import Request
+from dotenv import load_dotenv
+import os
+
+load_dotenv()
+
+
+async def user_api_key_auth(request: Request, api_key: str) -> UserAPIKeyAuth:
+    try:
+        print(f"api_key: {api_key}")
+        if api_key == "":
+            raise Exception(
+                f"CustomAuth - Malformed API Key passed in. Ensure Key has `Bearer` prefix"
+            )
+        if api_key == f"{os.getenv('PROXY_MASTER_KEY')}-1234":
+            return UserAPIKeyAuth(api_key=api_key)
+        raise Exception
+    except Exception as e:
+        if len(str(e)) > 0:
+            raise e
+        raise Exception("Failed custom auth")
--- a/Development/litellm/tests/proxy_unit_tests/test_configs/custom_callbacks.py
+++ b/Development/litellm/tests/proxy_unit_tests/test_configs/custom_callbacks.py
@@ -0,0 +1,121 @@
+from litellm.integrations.custom_logger import CustomLogger
+import inspect
+import litellm
+
+
+class testCustomCallbackProxy(CustomLogger):
+    def __init__(self):
+        self.success: bool = False  # type: ignore
+        self.failure: bool = False  # type: ignore
+        self.async_success: bool = False  # type: ignore
+        self.async_success_embedding: bool = False  # type: ignore
+        self.async_failure: bool = False  # type: ignore
+        self.async_failure_embedding: bool = False  # type: ignore
+
+        self.async_completion_kwargs = None  # type: ignore
+        self.async_embedding_kwargs = None  # type: ignore
+        self.async_embedding_response = None  # type: ignore
+
+        self.async_completion_kwargs_fail = None  # type: ignore
+        self.async_embedding_kwargs_fail = None  # type: ignore
+
+        self.streaming_response_obj = None  # type: ignore
+        blue_color_code = "\033[94m"
+        reset_color_code = "\033[0m"
+        print(f"{blue_color_code}Initialized LiteLLM custom logger")
+        try:
+            print(f"Logger Initialized with following methods:")
+            methods = [
+                method
+                for method in dir(self)
+                if inspect.ismethod(getattr(self, method))
+            ]
+
+            # Pretty print the methods
+            for method in methods:
+                print(f" - {method}")
+            print(f"{reset_color_code}")
+        except Exception:
+            pass
+
+    def log_pre_api_call(self, model, messages, kwargs):
+        print(f"Pre-API Call")
+
+    def log_post_api_call(self, kwargs, response_obj, start_time, end_time):
+        print(f"Post-API Call")
+
+    def log_stream_event(self, kwargs, response_obj, start_time, end_time):
+        print(f"On Stream")
+
+    def log_success_event(self, kwargs, response_obj, start_time, end_time):
+        print(f"On Success")
+        self.success = True
+
+    def log_failure_event(self, kwargs, response_obj, start_time, end_time):
+        print(f"On Failure")
+        self.failure = True
+
+    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
+        print(f"On Async success")
+        self.async_success = True
+        print("Value of async success: ", self.async_success)
+        print("\n kwargs: ", kwargs)
+        if (
+            kwargs.get("model") == "azure-embedding-model"
+            or kwargs.get("model") == "ada"
+        ):
+            print("Got an embedding model", kwargs.get("model"))
+            print("Setting embedding success to True")
+            self.async_success_embedding = True
+            print("Value of async success embedding: ", self.async_success_embedding)
+            self.async_embedding_kwargs = kwargs
+            self.async_embedding_response = response_obj
+        if kwargs.get("stream") == True:
+            self.streaming_response_obj = response_obj
+
+        self.async_completion_kwargs = kwargs
+
+        model = kwargs.get("model", None)
+        messages = kwargs.get("messages", None)
+        user = kwargs.get("user", None)
+
+        # Access litellm_params passed to litellm.completion(), example access `metadata`
+        litellm_params = kwargs.get("litellm_params", {})
+        metadata = litellm_params.get(
+            "metadata", {}
+        )  # headers passed to LiteLLM proxy, can be found here
+
+        # Calculate cost using  litellm.completion_cost()
+        cost = litellm.completion_cost(completion_response=response_obj)
+        response = response_obj
+        # tokens used in response
+        usage = response_obj["usage"]
+
+        print("\n\n in custom callback vars my custom logger, ", vars(my_custom_logger))
+
+        print(
+            f"""
+                Model: {model},
+                Messages: {messages},
+                User: {user},
+                Usage: {usage},
+                Cost: {cost},
+                Response: {response}
+                Proxy Metadata: {metadata}
+            """
+        )
+        return
+
+    async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
+        print(f"On Async Failure")
+        self.async_failure = True
+        print("Value of async failure: ", self.async_failure)
+        print("\n kwargs: ", kwargs)
+        if kwargs.get("model") == "text-embedding-ada-002":
+            self.async_failure_embedding = True
+            self.async_embedding_kwargs_fail = kwargs
+
+        self.async_completion_kwargs_fail = kwargs
+
+
+my_custom_logger = testCustomCallbackProxy()
--- a/Development/litellm/tests/proxy_unit_tests/test_configs/test_bad_config.yaml
+++ b/Development/litellm/tests/proxy_unit_tests/test_configs/test_bad_config.yaml
@@ -0,0 +1,21 @@
+model_list:
+  - model_name: gpt-3.5-turbo
+    litellm_params:
+      api_key: bad-key
+      model: gpt-3.5-turbo
+  - model_name: working-azure-gpt-3.5-turbo
+    litellm_params:
+      model: azure/chatgpt-v-3
+      api_base: os.environ/AZURE_API_BASE
+      api_key: os.environ/AZURE_API_KEY
+  - model_name: azure-gpt-3.5-turbo
+    litellm_params:
+      model: azure/chatgpt-v-3
+      api_base: os.environ/AZURE_API_BASE
+      api_key: bad-key
+  - model_name: azure-embedding
+    litellm_params:
+      model: azure/azure-embedding-model
+      api_base: os.environ/AZURE_API_BASE
+      api_key: bad-key
+    
--- a/Development/litellm/tests/proxy_unit_tests/test_configs/test_cloudflare_azure_with_cache_config.yaml
+++ b/Development/litellm/tests/proxy_unit_tests/test_configs/test_cloudflare_azure_with_cache_config.yaml
@@ -0,0 +1,17 @@
+model_list:
+  - model_name: azure-cloudflare
+    litellm_params:
+      model: azure/chatgpt-v-3
+      api_base: https://gateway.ai.cloudflare.com/v1/0399b10e77ac6668c80404a5ff49eb37/litellm-test/azure-openai/openai-gpt-4-test-v-1
+      api_key: os.environ/AZURE_API_KEY
+      api_version: 2023-07-01-preview
+
+litellm_settings:
+  set_verbose: True
+  cache: True          # set cache responses to True
+  cache_params:        # set cache params for s3
+    type: s3
+    s3_bucket_name: litellm-my-test-bucket-2   # AWS Bucket Name for S3
+    s3_region_name: us-east-1              # AWS Region Name for S3
+    s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID  # AWS Access Key ID for S3
+    s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY  # AWS Secret Access Key for S3
--- a/Development/litellm/tests/proxy_unit_tests/test_configs/test_config.yaml
+++ b/Development/litellm/tests/proxy_unit_tests/test_configs/test_config.yaml
@@ -0,0 +1,28 @@
+general_settings:
+  database_url: os.environ/DATABASE_URL
+  master_key: os.environ/PROXY_MASTER_KEY
+litellm_settings:
+  drop_params: true
+  success_callback: ["langfuse"]
+  
+model_list:
+- litellm_params:
+    api_base: https://my-endpoint-europe-berri-992.openai.azure.com/
+    api_key: os.environ/AZURE_EUROPE_API_KEY
+    model: azure/gpt-35-turbo
+  model_name: azure-model
+- litellm_params:
+    api_base: https://my-endpoint-canada-berri992.openai.azure.com
+    api_key: os.environ/AZURE_CANADA_API_KEY
+    model: azure/gpt-35-turbo
+  model_name: azure-model
+- litellm_params:
+    api_base: https://openai-france-1234.openai.azure.com
+    api_key: os.environ/AZURE_FRANCE_API_KEY
+    model: azure/gpt-turbo
+  model_name: azure-model
+- litellm_params:
+    model: gpt-3.5-turbo
+  model_info:
+    description: this is a test openai model
+  model_name: test_openai_models
--- a/Development/litellm/tests/proxy_unit_tests/test_configs/test_config_custom_auth.yaml
+++ b/Development/litellm/tests/proxy_unit_tests/test_configs/test_config_custom_auth.yaml
@@ -0,0 +1,11 @@
+model_list: 
+  - model_name: "openai-model"
+    litellm_params: 
+      model: "gpt-3.5-turbo"
+
+litellm_settings:
+  drop_params: True
+  set_verbose: True
+
+general_settings:
+  custom_auth: custom_auth.user_api_key_auth
--- a/Development/litellm/tests/proxy_unit_tests/test_configs/test_config_no_auth.yaml
+++ b/Development/litellm/tests/proxy_unit_tests/test_configs/test_config_no_auth.yaml
@@ -0,0 +1,127 @@
+model_list:
+- litellm_params:
+    api_base: https://my-endpoint-europe-berri-992.openai.azure.com/
+    api_key: os.environ/AZURE_EUROPE_API_KEY
+    model: azure/gpt-35-turbo
+  model_name: azure-model
+- litellm_params:
+    api_base: https://my-endpoint-canada-berri992.openai.azure.com
+    api_key: os.environ/AZURE_CANADA_API_KEY
+    model: azure/gpt-35-turbo
+  model_name: azure-model
+- litellm_params:
+    api_base: https://gateway.ai.cloudflare.com/v1/0399b10e77ac6668c80404a5ff49eb37/litellm-test/azure-openai/openai-gpt-4-test-v-1
+    api_key: os.environ/AZURE_API_KEY
+    model: azure/chatgpt-v-3
+  model_name: azure-cloudflare-model
+- litellm_params:
+    api_base: https://openai-france-1234.openai.azure.com
+    api_key: os.environ/AZURE_FRANCE_API_KEY
+    model: azure/gpt-turbo
+  model_name: azure-model
+- litellm_params:
+    model: gpt-3.5-turbo
+  model_info:
+    description: this is a test openai model
+  model_name: test_openai_models
+- litellm_params:
+    model: gpt-3.5-turbo
+  model_info:
+    description: this is a test openai model
+    id: 56f1bd94-3b54-4b67-9ea2-7c70e9a3a709
+  model_name: test_openai_models
+- litellm_params:
+    model: gpt-3.5-turbo
+  model_info:
+    description: this is a test openai model
+    id: 4d1ee26c-abca-450c-8744-8e87fd6755e9
+  model_name: test_openai_models
+- litellm_params:
+    model: gpt-3.5-turbo
+  model_info:
+    description: this is a test openai model
+    id: 00e19c0f-b63d-42bb-88e9-016fb0c60764
+  model_name: test_openai_models
+- litellm_params:
+    model: gpt-3.5-turbo
+  model_info:
+    description: this is a test openai model
+    id: 79fc75bf-8e1b-47d5-8d24-9365a854af03
+  model_name: test_openai_models
+- litellm_params:
+    api_base: os.environ/AZURE_API_BASE
+    api_key: os.environ/AZURE_API_KEY
+    api_version: 2023-07-01-preview
+    model: azure/azure-embedding-model
+  model_info:
+    mode: embedding
+  model_name: azure-embedding-model
+- litellm_params:
+    model: gpt-3.5-turbo
+  model_info:
+    description: this is a test openai model
+    id: 55848c55-4162-40f9-a6e2-9a722b9ef404
+  model_name: test_openai_models
+- litellm_params:
+    model: gpt-3.5-turbo
+  model_info:
+    description: this is a test openai model
+    id: 34339b1e-e030-4bcc-a531-c48559f10ce4
+  model_name: test_openai_models
+- litellm_params:
+    model: gpt-3.5-turbo
+  model_info:
+    description: this is a test openai model
+    id: f6f74e14-ac64-4403-9365-319e584dcdc5
+  model_name: test_openai_models
+- litellm_params:
+    model: gpt-3.5-turbo
+  model_info:
+    description: this is a test openai model
+    id: 9b1ef341-322c-410a-8992-903987fef439
+  model_name: test_openai_models
+- litellm_params:
+    model: dall-e-3
+  model_info:
+    mode: image_generation
+  model_name: dall-e-3
+- litellm_params:
+    api_base: os.environ/AZURE_SWEDEN_API_BASE
+    api_key: os.environ/AZURE_SWEDEN_API_KEY
+    api_version: 2023-12-01-preview
+    model: azure/dall-e-3-test
+  model_info:
+    mode: image_generation
+  model_name: dall-e-3
+- litellm_params:
+    api_base: os.environ/AZURE_API_BASE
+    api_key: os.environ/AZURE_API_KEY
+    api_version: 2023-06-01-preview
+    model: azure/
+  model_info:
+    mode: image_generation
+  model_name: dall-e-2
+- litellm_params:
+    api_base: os.environ/AZURE_API_BASE
+    api_key: os.environ/AZURE_API_KEY
+    api_version: 2023-07-01-preview
+    model: azure/azure-embedding-model
+  model_info:
+    base_model: text-embedding-ada-002
+    mode: embedding
+  model_name: text-embedding-ada-002
+- litellm_params:
+    model: gpt-3.5-turbo
+  model_info:
+    description: this is a test openai model
+    id: 34cb2419-7c63-44ae-a189-53f1d1ce5953
+  model_name: test_openai_models
+- litellm_params:
+    model: amazon.titan-embed-text-v1
+  model_name: amazon-embeddings
+- litellm_params:
+    model: gpt-3.5-turbo
+  model_info:
+    description: this is a test openai model
+    id: 753dca9a-898d-4ff7-9961-5acf7cdf38cf
+  model_name: test_openai_models
--- a/Development/litellm/tests/proxy_unit_tests/test_configs/test_custom_logger.yaml
+++ b/Development/litellm/tests/proxy_unit_tests/test_configs/test_custom_logger.yaml
@@ -0,0 +1,26 @@
+model_list: 
+  - model_name: Azure OpenAI GPT-4 Canada
+    litellm_params:
+      model: azure/chatgpt-v-3
+      api_base: os.environ/AZURE_API_BASE
+      api_key: os.environ/AZURE_API_KEY
+      api_version: "2023-07-01-preview"
+    model_info:
+      mode: chat
+      input_cost_per_token: 0.0002
+      id: gm
+  - model_name: azure-embedding-model
+    litellm_params:
+      model: azure/azure-embedding-model
+      api_base: os.environ/AZURE_API_BASE
+      api_key: os.environ/AZURE_API_KEY
+      api_version: "2023-07-01-preview"
+    model_info:
+      mode: embedding
+      input_cost_per_token: 0.002
+      id: hello
+
+litellm_settings:
+  drop_params: True
+  set_verbose: True
+  callbacks: custom_callbacks.my_custom_logger
--- a/Development/litellm/tests/proxy_unit_tests/test_configs/test_guardrails_config.yaml
+++ b/Development/litellm/tests/proxy_unit_tests/test_configs/test_guardrails_config.yaml
@@ -0,0 +1,32 @@
+
+
+model_list:
+- litellm_params:
+    api_base: https://my-endpoint-europe-berri-992.openai.azure.com/
+    api_key: os.environ/AZURE_EUROPE_API_KEY
+    model: azure/gpt-35-turbo
+  model_name: azure-model
+- litellm_params:
+    api_base: https://my-endpoint-canada-berri992.openai.azure.com
+    api_key: os.environ/AZURE_CANADA_API_KEY
+    model: azure/gpt-35-turbo
+  model_name: azure-model
+- litellm_params:
+    api_base: https://openai-france-1234.openai.azure.com
+    api_key: os.environ/AZURE_FRANCE_API_KEY
+    model: azure/gpt-turbo
+  model_name: azure-model
+
+
+
+litellm_settings:
+  guardrails:
+    - prompt_injection:
+        callbacks: [lakera_prompt_injection, detect_prompt_injection]
+        default_on: true
+    - hide_secrets:
+        callbacks: [hide_secrets]
+        default_on: true
+    - moderations:
+        callbacks: [openai_moderations]
+        default_on: false