Added LiteLLM to the stack
This commit is contained in:
@@ -0,0 +1,199 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
sys.path.insert(
|
||||
0, os.path.abspath("../../../../..")
|
||||
) # Adds the parent directory to the system path
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"api_base", ["https://api.openai.com/v1", "https://api.openai.com"]
|
||||
)
|
||||
def test_openai_realtime_handler_url_construction(api_base):
|
||||
from litellm.llms.openai.realtime.handler import OpenAIRealtime
|
||||
|
||||
handler = OpenAIRealtime()
|
||||
url = handler._construct_url(
|
||||
api_base=api_base,
|
||||
query_params={
|
||||
"model": "gpt-4o-realtime-preview-2024-10-01",
|
||||
}
|
||||
)
|
||||
# Model parameter should be included in the URL
|
||||
assert url.startswith("wss://api.openai.com/v1/realtime?")
|
||||
assert "model=gpt-4o-realtime-preview-2024-10-01" in url
|
||||
|
||||
|
||||
def test_openai_realtime_handler_url_with_extra_params():
|
||||
from litellm.llms.openai.realtime.handler import OpenAIRealtime
|
||||
from litellm.types.realtime import RealtimeQueryParams
|
||||
|
||||
handler = OpenAIRealtime()
|
||||
api_base = "https://api.openai.com/v1"
|
||||
query_params: RealtimeQueryParams = {
|
||||
"model": "gpt-4o-realtime-preview-2024-10-01",
|
||||
"intent": "chat"
|
||||
}
|
||||
url = handler._construct_url(api_base=api_base, query_params=query_params)
|
||||
# Both 'model' and other params should be included in the query string
|
||||
assert url.startswith("wss://api.openai.com/v1/realtime?")
|
||||
assert "model=gpt-4o-realtime-preview-2024-10-01" in url
|
||||
assert "intent=chat" in url
|
||||
|
||||
|
||||
def test_openai_realtime_handler_model_parameter_inclusion():
|
||||
"""
|
||||
Test that the model parameter is properly included in the WebSocket URL
|
||||
to prevent 'missing_model' errors from OpenAI.
|
||||
|
||||
This test specifically verifies the fix for the issue where model parameter
|
||||
was being excluded from the query string, causing OpenAI to return
|
||||
invalid_request_error.missing_model errors.
|
||||
"""
|
||||
from litellm.llms.openai.realtime.handler import OpenAIRealtime
|
||||
from litellm.types.realtime import RealtimeQueryParams
|
||||
|
||||
handler = OpenAIRealtime()
|
||||
api_base = "https://api.openai.com/"
|
||||
|
||||
# Test with just model parameter
|
||||
query_params_model_only: RealtimeQueryParams = {
|
||||
"model": "gpt-4o-mini-realtime-preview"
|
||||
}
|
||||
url = handler._construct_url(api_base=api_base, query_params=query_params_model_only)
|
||||
|
||||
# Verify the URL structure
|
||||
assert url.startswith("wss://api.openai.com/v1/realtime?")
|
||||
assert "model=gpt-4o-mini-realtime-preview" in url
|
||||
|
||||
# Test with model + additional parameters
|
||||
query_params_with_extras: RealtimeQueryParams = {
|
||||
"model": "gpt-4o-mini-realtime-preview",
|
||||
"intent": "chat"
|
||||
}
|
||||
url_with_extras = handler._construct_url(api_base=api_base, query_params=query_params_with_extras)
|
||||
|
||||
# Verify both parameters are included
|
||||
assert url_with_extras.startswith("wss://api.openai.com/v1/realtime?")
|
||||
assert "model=gpt-4o-mini-realtime-preview" in url_with_extras
|
||||
assert "intent=chat" in url_with_extras
|
||||
|
||||
# Verify the URL is properly formatted for OpenAI
|
||||
# Should match the pattern: wss://api.openai.com/v1/realtime?model=MODEL_NAME
|
||||
expected_pattern = "wss://api.openai.com/v1/realtime?model="
|
||||
assert expected_pattern in url
|
||||
assert expected_pattern in url_with_extras
|
||||
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_realtime_success():
|
||||
from litellm.llms.openai.realtime.handler import OpenAIRealtime
|
||||
from litellm.types.realtime import RealtimeQueryParams
|
||||
|
||||
handler = OpenAIRealtime()
|
||||
api_base = "https://api.openai.com/v1"
|
||||
api_key = "test-key"
|
||||
model = "gpt-4o-realtime-preview-2024-10-01"
|
||||
query_params: RealtimeQueryParams = {"model": model, "intent": "chat"}
|
||||
|
||||
dummy_websocket = AsyncMock()
|
||||
dummy_logging_obj = MagicMock()
|
||||
mock_backend_ws = AsyncMock()
|
||||
|
||||
class DummyAsyncContextManager:
|
||||
def __init__(self, value):
|
||||
self.value = value
|
||||
async def __aenter__(self):
|
||||
return self.value
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
return None
|
||||
|
||||
with patch("websockets.connect", return_value=DummyAsyncContextManager(mock_backend_ws)) as mock_ws_connect, \
|
||||
patch("litellm.llms.openai.realtime.handler.RealTimeStreaming") as mock_realtime_streaming:
|
||||
mock_streaming_instance = MagicMock()
|
||||
mock_realtime_streaming.return_value = mock_streaming_instance
|
||||
mock_streaming_instance.bidirectional_forward = AsyncMock()
|
||||
|
||||
await handler.async_realtime(
|
||||
model=model,
|
||||
websocket=dummy_websocket,
|
||||
logging_obj=dummy_logging_obj,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
query_params=query_params,
|
||||
)
|
||||
|
||||
mock_realtime_streaming.assert_called_once()
|
||||
mock_streaming_instance.bidirectional_forward.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_realtime_url_contains_model():
|
||||
"""
|
||||
Test that the async_realtime method properly constructs a URL with the model parameter
|
||||
when connecting to OpenAI, preventing 'missing_model' errors.
|
||||
"""
|
||||
from litellm.llms.openai.realtime.handler import OpenAIRealtime
|
||||
from litellm.types.realtime import RealtimeQueryParams
|
||||
|
||||
handler = OpenAIRealtime()
|
||||
api_base = "https://api.openai.com/"
|
||||
api_key = "test-key"
|
||||
model = "gpt-4o-mini-realtime-preview"
|
||||
query_params: RealtimeQueryParams = {"model": model}
|
||||
|
||||
dummy_websocket = AsyncMock()
|
||||
dummy_logging_obj = MagicMock()
|
||||
mock_backend_ws = AsyncMock()
|
||||
|
||||
class DummyAsyncContextManager:
|
||||
def __init__(self, value):
|
||||
self.value = value
|
||||
async def __aenter__(self):
|
||||
return self.value
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
return None
|
||||
|
||||
with patch("websockets.connect", return_value=DummyAsyncContextManager(mock_backend_ws)) as mock_ws_connect, \
|
||||
patch("litellm.llms.openai.realtime.handler.RealTimeStreaming") as mock_realtime_streaming:
|
||||
|
||||
mock_streaming_instance = MagicMock()
|
||||
mock_realtime_streaming.return_value = mock_streaming_instance
|
||||
mock_streaming_instance.bidirectional_forward = AsyncMock()
|
||||
|
||||
await handler.async_realtime(
|
||||
model=model,
|
||||
websocket=dummy_websocket,
|
||||
logging_obj=dummy_logging_obj,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
query_params=query_params,
|
||||
)
|
||||
|
||||
# Verify websockets.connect was called with the correct URL
|
||||
mock_ws_connect.assert_called_once()
|
||||
called_url = mock_ws_connect.call_args[0][0]
|
||||
|
||||
# Verify the URL contains the model parameter
|
||||
assert called_url.startswith("wss://api.openai.com/v1/realtime?")
|
||||
assert f"model={model}" in called_url
|
||||
|
||||
# Verify proper headers were set
|
||||
called_kwargs = mock_ws_connect.call_args[1]
|
||||
assert "extra_headers" in called_kwargs
|
||||
extra_headers = called_kwargs["extra_headers"]
|
||||
assert extra_headers["Authorization"] == f"Bearer {api_key}"
|
||||
assert extra_headers["OpenAI-Beta"] == "realtime=v1"
|
||||
|
||||
mock_realtime_streaming.assert_called_once()
|
||||
mock_streaming_instance.bidirectional_forward.assert_awaited_once()
|
||||
@@ -0,0 +1,652 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import AsyncMock, MagicMock, Mock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
sys.path.insert(
|
||||
0, os.path.abspath("../../../../..")
|
||||
) # Adds the parent directory to the system path
|
||||
|
||||
import litellm
|
||||
from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.types.llms.openai import (
|
||||
OutputTextDeltaEvent,
|
||||
ResponseCompletedEvent,
|
||||
ResponsesAPIRequestParams,
|
||||
ResponsesAPIResponse,
|
||||
ResponsesAPIStreamEvents,
|
||||
)
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
|
||||
class TestOpenAIResponsesAPIConfig:
|
||||
def setup_method(self):
|
||||
self.config = OpenAIResponsesAPIConfig()
|
||||
self.model = "gpt-4o"
|
||||
self.logging_obj = MagicMock()
|
||||
|
||||
def test_map_openai_params(self):
|
||||
"""Test that parameters are correctly mapped"""
|
||||
test_params = {"input": "Hello world", "temperature": 0.7, "stream": True}
|
||||
|
||||
result = self.config.map_openai_params(
|
||||
response_api_optional_params=test_params,
|
||||
model=self.model,
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
# The function should return the params unchanged
|
||||
assert result == test_params
|
||||
|
||||
def validate_responses_api_request_params(self, params, expected_fields):
|
||||
"""
|
||||
Validate that the params dict has the expected structure of ResponsesAPIRequestParams
|
||||
|
||||
Args:
|
||||
params: The dict to validate
|
||||
expected_fields: Dict of field names and their expected values
|
||||
"""
|
||||
# Check that it's a dict
|
||||
assert isinstance(params, dict), "Result should be a dict"
|
||||
|
||||
# Check expected fields have correct values
|
||||
for field, value in expected_fields.items():
|
||||
assert field in params, f"Missing expected field: {field}"
|
||||
assert (
|
||||
params[field] == value
|
||||
), f"Field {field} has value {params[field]}, expected {value}"
|
||||
|
||||
def test_transform_responses_api_request(self):
|
||||
"""Test request transformation"""
|
||||
input_text = "What is the capital of France?"
|
||||
optional_params = {"temperature": 0.7, "stream": True, "background": True}
|
||||
|
||||
result = self.config.transform_responses_api_request(
|
||||
model=self.model,
|
||||
input=input_text,
|
||||
response_api_optional_request_params=optional_params,
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
# Validate the result has the expected structure and values
|
||||
expected_fields = {
|
||||
"model": self.model,
|
||||
"input": input_text,
|
||||
"temperature": 0.7,
|
||||
"stream": True,
|
||||
"background": True,
|
||||
}
|
||||
|
||||
self.validate_responses_api_request_params(result, expected_fields)
|
||||
|
||||
def test_transform_streaming_response(self):
|
||||
"""Test streaming response transformation"""
|
||||
# Test with a text delta event
|
||||
chunk = {
|
||||
"type": "response.output_text.delta",
|
||||
"item_id": "item_123",
|
||||
"output_index": 0,
|
||||
"content_index": 0,
|
||||
"delta": "Hello",
|
||||
}
|
||||
|
||||
result = self.config.transform_streaming_response(
|
||||
model=self.model, parsed_chunk=chunk, logging_obj=self.logging_obj
|
||||
)
|
||||
|
||||
assert isinstance(result, OutputTextDeltaEvent)
|
||||
assert result.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA
|
||||
assert result.delta == "Hello"
|
||||
assert result.item_id == "item_123"
|
||||
|
||||
# Test with a completed event - providing all required fields
|
||||
completed_chunk = {
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "resp_123",
|
||||
"created_at": 1234567890,
|
||||
"model": "gpt-4o",
|
||||
"object": "response",
|
||||
"output": [],
|
||||
"parallel_tool_calls": False,
|
||||
"error": None,
|
||||
"incomplete_details": None,
|
||||
"instructions": None,
|
||||
"metadata": None,
|
||||
"temperature": 0.7,
|
||||
"tool_choice": "auto",
|
||||
"tools": [],
|
||||
"top_p": 1.0,
|
||||
"max_output_tokens": None,
|
||||
"previous_response_id": None,
|
||||
"reasoning": None,
|
||||
"status": "completed",
|
||||
"text": None,
|
||||
"truncation": "auto",
|
||||
"usage": None,
|
||||
"user": None,
|
||||
},
|
||||
}
|
||||
|
||||
# Mock the get_event_model_class to avoid validation issues in tests
|
||||
with patch.object(
|
||||
OpenAIResponsesAPIConfig, "get_event_model_class"
|
||||
) as mock_get_class:
|
||||
mock_get_class.return_value = ResponseCompletedEvent
|
||||
|
||||
result = self.config.transform_streaming_response(
|
||||
model=self.model,
|
||||
parsed_chunk=completed_chunk,
|
||||
logging_obj=self.logging_obj,
|
||||
)
|
||||
|
||||
assert result.type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED
|
||||
assert result.response.id == "resp_123"
|
||||
|
||||
@pytest.mark.serial
|
||||
def test_validate_environment(self):
|
||||
"""Test that validate_environment correctly sets the Authorization header"""
|
||||
# Test with provided API key
|
||||
headers = {}
|
||||
api_key = "test_api_key"
|
||||
litellm_params = GenericLiteLLMParams(api_key=api_key)
|
||||
result = self.config.validate_environment(
|
||||
headers=headers, model=self.model, litellm_params=litellm_params
|
||||
)
|
||||
|
||||
assert "Authorization" in result
|
||||
assert result["Authorization"] == f"Bearer {api_key}"
|
||||
|
||||
# Test with empty headers
|
||||
headers = {}
|
||||
|
||||
with patch("litellm.api_key", "litellm_api_key"):
|
||||
litellm_params = GenericLiteLLMParams()
|
||||
result = self.config.validate_environment(
|
||||
headers=headers, model=self.model, litellm_params=litellm_params
|
||||
)
|
||||
|
||||
assert "Authorization" in result
|
||||
assert result["Authorization"] == "Bearer litellm_api_key"
|
||||
|
||||
# Test with existing headers
|
||||
headers = {"Content-Type": "application/json"}
|
||||
|
||||
with patch("litellm.openai_key", "openai_key"):
|
||||
with patch("litellm.api_key", None):
|
||||
litellm_params = GenericLiteLLMParams()
|
||||
result = self.config.validate_environment(
|
||||
headers=headers, model=self.model, litellm_params=litellm_params
|
||||
)
|
||||
|
||||
assert "Authorization" in result
|
||||
assert result["Authorization"] == "Bearer openai_key"
|
||||
assert "Content-Type" in result
|
||||
assert result["Content-Type"] == "application/json"
|
||||
|
||||
# Test with environment variable
|
||||
headers = {}
|
||||
|
||||
with patch("litellm.api_key", None):
|
||||
with patch("litellm.openai_key", None):
|
||||
with patch(
|
||||
"litellm.llms.openai.responses.transformation.get_secret_str",
|
||||
return_value="env_api_key",
|
||||
):
|
||||
litellm_params = GenericLiteLLMParams()
|
||||
result = self.config.validate_environment(
|
||||
headers=headers, model=self.model, litellm_params=litellm_params
|
||||
)
|
||||
|
||||
assert "Authorization" in result
|
||||
assert result["Authorization"] == "Bearer env_api_key"
|
||||
|
||||
def test_get_complete_url(self):
|
||||
"""Test that get_complete_url returns the correct URL"""
|
||||
# Test with provided API base
|
||||
api_base = "https://custom-openai.example.com/v1"
|
||||
|
||||
result = self.config.get_complete_url(
|
||||
api_base=api_base,
|
||||
litellm_params={},
|
||||
)
|
||||
|
||||
assert result == "https://custom-openai.example.com/v1/responses"
|
||||
|
||||
# Test with litellm.api_base
|
||||
with patch("litellm.api_base", "https://litellm-api-base.example.com/v1"):
|
||||
result = self.config.get_complete_url(
|
||||
api_base=None,
|
||||
litellm_params={},
|
||||
)
|
||||
|
||||
assert result == "https://litellm-api-base.example.com/v1/responses"
|
||||
|
||||
# Test with environment variable
|
||||
with patch("litellm.api_base", None):
|
||||
with patch(
|
||||
"litellm.llms.openai.responses.transformation.get_secret_str",
|
||||
return_value="https://env-api-base.example.com/v1",
|
||||
):
|
||||
result = self.config.get_complete_url(
|
||||
api_base=None,
|
||||
litellm_params={},
|
||||
)
|
||||
|
||||
assert result == "https://env-api-base.example.com/v1/responses"
|
||||
|
||||
# Test with default API base
|
||||
with patch("litellm.api_base", None):
|
||||
with patch(
|
||||
"litellm.llms.openai.responses.transformation.get_secret_str",
|
||||
return_value=None,
|
||||
):
|
||||
result = self.config.get_complete_url(
|
||||
api_base=None,
|
||||
litellm_params={},
|
||||
)
|
||||
|
||||
assert result == "https://api.openai.com/v1/responses"
|
||||
|
||||
# Test with trailing slash in API base
|
||||
api_base = "https://custom-openai.example.com/v1/"
|
||||
|
||||
result = self.config.get_complete_url(
|
||||
api_base=api_base,
|
||||
litellm_params={},
|
||||
)
|
||||
|
||||
assert result == "https://custom-openai.example.com/v1/responses"
|
||||
|
||||
def test_get_event_model_class_generic_event(self):
|
||||
"""Test that get_event_model_class returns the correct event model class"""
|
||||
from litellm.types.llms.openai import GenericEvent
|
||||
|
||||
event_type = "test"
|
||||
result = self.config.get_event_model_class(event_type)
|
||||
assert result == GenericEvent
|
||||
|
||||
def test_transform_streaming_response_generic_event(self):
|
||||
"""Test that transform_streaming_response returns the correct event model class"""
|
||||
from litellm.types.llms.openai import GenericEvent
|
||||
|
||||
chunk = {"type": "test", "test": "test"}
|
||||
result = self.config.transform_streaming_response(
|
||||
model=self.model, parsed_chunk=chunk, logging_obj=self.logging_obj
|
||||
)
|
||||
assert isinstance(result, GenericEvent)
|
||||
assert result.type == "test"
|
||||
|
||||
|
||||
class TestAzureResponsesAPIConfig:
|
||||
def setup_method(self):
|
||||
self.config = AzureOpenAIResponsesAPIConfig()
|
||||
self.model = "gpt-4o"
|
||||
self.logging_obj = MagicMock()
|
||||
|
||||
def test_azure_get_complete_url_with_version_types(self):
|
||||
"""Test Azure get_complete_url with different API version types"""
|
||||
base_url = "https://litellm8397336933.openai.azure.com"
|
||||
|
||||
# Test with preview version - should use openai/v1/responses
|
||||
result_preview = self.config.get_complete_url(
|
||||
api_base=base_url,
|
||||
litellm_params={"api_version": "preview"},
|
||||
)
|
||||
assert result_preview == "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=preview"
|
||||
|
||||
# Test with latest version - should use openai/v1/responses
|
||||
result_latest = self.config.get_complete_url(
|
||||
api_base=base_url,
|
||||
litellm_params={"api_version": "latest"},
|
||||
)
|
||||
assert result_latest == "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=latest"
|
||||
|
||||
# Test with date-based version - should use openai/responses
|
||||
result_date = self.config.get_complete_url(
|
||||
api_base=base_url,
|
||||
litellm_params={"api_version": "2025-01-01"},
|
||||
)
|
||||
assert result_date == "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-01-01"
|
||||
|
||||
|
||||
class TestTransformListInputItemsRequest:
|
||||
"""Test suite for transform_list_input_items_request function"""
|
||||
|
||||
def setup_method(self):
|
||||
"""Setup test fixtures"""
|
||||
self.openai_config = OpenAIResponsesAPIConfig()
|
||||
self.azure_config = AzureOpenAIResponsesAPIConfig()
|
||||
self.response_id = "resp_abc123"
|
||||
self.api_base = "https://api.openai.com/v1/responses"
|
||||
self.litellm_params = GenericLiteLLMParams()
|
||||
self.headers = {"Authorization": "Bearer test-key"}
|
||||
|
||||
def test_openai_transform_list_input_items_request_minimal(self):
|
||||
"""Test OpenAI implementation with minimal parameters"""
|
||||
# Execute
|
||||
url, params = self.openai_config.transform_list_input_items_request(
|
||||
response_id=self.response_id,
|
||||
api_base=self.api_base,
|
||||
litellm_params=self.litellm_params,
|
||||
headers=self.headers,
|
||||
)
|
||||
|
||||
# Assert
|
||||
expected_url = f"{self.api_base}/{self.response_id}/input_items"
|
||||
assert url == expected_url
|
||||
assert params == {"limit": 20, "order": "desc"}
|
||||
|
||||
def test_openai_transform_list_input_items_request_all_params(self):
|
||||
"""Test OpenAI implementation with all optional parameters"""
|
||||
# Execute
|
||||
url, params = self.openai_config.transform_list_input_items_request(
|
||||
response_id=self.response_id,
|
||||
api_base=self.api_base,
|
||||
litellm_params=self.litellm_params,
|
||||
headers=self.headers,
|
||||
after="cursor_after_123",
|
||||
before="cursor_before_456",
|
||||
include=["metadata", "content"],
|
||||
limit=50,
|
||||
order="asc",
|
||||
)
|
||||
|
||||
# Assert
|
||||
expected_url = f"{self.api_base}/{self.response_id}/input_items"
|
||||
expected_params = {
|
||||
"after": "cursor_after_123",
|
||||
"before": "cursor_before_456",
|
||||
"include": "metadata,content", # Should be comma-separated string
|
||||
"limit": 50,
|
||||
"order": "asc",
|
||||
}
|
||||
assert url == expected_url
|
||||
assert params == expected_params
|
||||
|
||||
def test_openai_transform_list_input_items_request_include_list_formatting(self):
|
||||
"""Test that include list is properly formatted as comma-separated string"""
|
||||
# Execute
|
||||
url, params = self.openai_config.transform_list_input_items_request(
|
||||
response_id=self.response_id,
|
||||
api_base=self.api_base,
|
||||
litellm_params=self.litellm_params,
|
||||
headers=self.headers,
|
||||
include=["metadata", "content", "annotations"],
|
||||
)
|
||||
|
||||
# Assert
|
||||
assert params["include"] == "metadata,content,annotations"
|
||||
|
||||
def test_openai_transform_list_input_items_request_none_values(self):
|
||||
"""Test OpenAI implementation with None values for optional parameters"""
|
||||
# Execute - pass only required parameters and explicit None for truly optional params
|
||||
url, params = self.openai_config.transform_list_input_items_request(
|
||||
response_id=self.response_id,
|
||||
api_base=self.api_base,
|
||||
litellm_params=self.litellm_params,
|
||||
headers=self.headers,
|
||||
after=None,
|
||||
before=None,
|
||||
include=None,
|
||||
)
|
||||
|
||||
# Assert
|
||||
expected_url = f"{self.api_base}/{self.response_id}/input_items"
|
||||
expected_params = {
|
||||
"limit": 20,
|
||||
"order": "desc",
|
||||
} # Default values should be present
|
||||
assert url == expected_url
|
||||
assert params == expected_params
|
||||
|
||||
def test_openai_transform_list_input_items_request_empty_include_list(self):
|
||||
"""Test OpenAI implementation with empty include list"""
|
||||
# Execute
|
||||
url, params = self.openai_config.transform_list_input_items_request(
|
||||
response_id=self.response_id,
|
||||
api_base=self.api_base,
|
||||
litellm_params=self.litellm_params,
|
||||
headers=self.headers,
|
||||
include=[],
|
||||
)
|
||||
|
||||
# Assert
|
||||
assert "include" not in params # Empty list should not be included
|
||||
|
||||
def test_azure_transform_list_input_items_request_minimal(self):
|
||||
"""Test Azure implementation with minimal parameters"""
|
||||
# Setup
|
||||
azure_api_base = "https://test.openai.azure.com/openai/responses?api-version=2024-05-01-preview"
|
||||
|
||||
# Execute
|
||||
url, params = self.azure_config.transform_list_input_items_request(
|
||||
response_id=self.response_id,
|
||||
api_base=azure_api_base,
|
||||
litellm_params=self.litellm_params,
|
||||
headers=self.headers,
|
||||
)
|
||||
|
||||
# Assert
|
||||
assert self.response_id in url
|
||||
assert "/input_items" in url
|
||||
assert params == {"limit": 20, "order": "desc"}
|
||||
|
||||
def test_azure_transform_list_input_items_request_url_construction(self):
|
||||
"""Test Azure implementation URL construction with response_id in path"""
|
||||
# Setup
|
||||
azure_api_base = "https://test.openai.azure.com/openai/responses?api-version=2024-05-01-preview"
|
||||
|
||||
# Execute
|
||||
url, params = self.azure_config.transform_list_input_items_request(
|
||||
response_id=self.response_id,
|
||||
api_base=azure_api_base,
|
||||
litellm_params=self.litellm_params,
|
||||
headers=self.headers,
|
||||
)
|
||||
|
||||
# Assert
|
||||
# The Azure implementation should construct URL with response_id in path
|
||||
assert self.response_id in url
|
||||
assert "/input_items" in url
|
||||
assert "api-version=2024-05-01-preview" in url
|
||||
|
||||
def test_azure_transform_list_input_items_request_with_all_params(self):
|
||||
"""Test Azure implementation with all optional parameters"""
|
||||
# Setup
|
||||
azure_api_base = "https://test.openai.azure.com/openai/responses?api-version=2024-05-01-preview"
|
||||
|
||||
# Execute
|
||||
url, params = self.azure_config.transform_list_input_items_request(
|
||||
response_id=self.response_id,
|
||||
api_base=azure_api_base,
|
||||
litellm_params=self.litellm_params,
|
||||
headers=self.headers,
|
||||
after="cursor_after_123",
|
||||
before="cursor_before_456",
|
||||
include=["metadata", "content"],
|
||||
limit=100,
|
||||
order="asc",
|
||||
)
|
||||
|
||||
# Assert
|
||||
expected_params = {
|
||||
"after": "cursor_after_123",
|
||||
"before": "cursor_before_456",
|
||||
"include": "metadata,content",
|
||||
"limit": 100,
|
||||
"order": "asc",
|
||||
}
|
||||
assert params == expected_params
|
||||
|
||||
@patch("litellm.router.Router")
|
||||
def test_mock_litellm_router_with_transform_list_input_items_request(
|
||||
self, mock_router
|
||||
):
|
||||
"""Mock test using litellm.router for transform_list_input_items_request"""
|
||||
# Setup mock router
|
||||
mock_router_instance = Mock()
|
||||
mock_router.return_value = mock_router_instance
|
||||
|
||||
# Mock the provider config
|
||||
mock_provider_config = Mock(spec=OpenAIResponsesAPIConfig)
|
||||
mock_provider_config.transform_list_input_items_request.return_value = (
|
||||
"https://api.openai.com/v1/responses/resp_123/input_items",
|
||||
{"limit": 20, "order": "desc"},
|
||||
)
|
||||
|
||||
# Setup router mock
|
||||
mock_router_instance.get_provider_responses_api_config.return_value = (
|
||||
mock_provider_config
|
||||
)
|
||||
|
||||
# Test parameters
|
||||
response_id = "resp_test123"
|
||||
|
||||
# Execute
|
||||
url, params = mock_provider_config.transform_list_input_items_request(
|
||||
response_id=response_id,
|
||||
api_base="https://api.openai.com/v1/responses",
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={"Authorization": "Bearer test"},
|
||||
after="cursor_123",
|
||||
include=["metadata"],
|
||||
limit=30,
|
||||
)
|
||||
|
||||
# Assert
|
||||
mock_provider_config.transform_list_input_items_request.assert_called_once_with(
|
||||
response_id=response_id,
|
||||
api_base="https://api.openai.com/v1/responses",
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={"Authorization": "Bearer test"},
|
||||
after="cursor_123",
|
||||
include=["metadata"],
|
||||
limit=30,
|
||||
)
|
||||
assert url == "https://api.openai.com/v1/responses/resp_123/input_items"
|
||||
assert params == {"limit": 20, "order": "desc"}
|
||||
|
||||
@patch("litellm.list_input_items")
|
||||
def test_mock_litellm_list_input_items_integration(self, mock_list_input_items):
|
||||
"""Test integration with litellm.list_input_items function"""
|
||||
# Setup mock response
|
||||
mock_response = {
|
||||
"object": "list",
|
||||
"data": [
|
||||
{
|
||||
"id": "input_item_123",
|
||||
"object": "input_item",
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": "Test message",
|
||||
}
|
||||
],
|
||||
"has_more": False,
|
||||
"first_id": "input_item_123",
|
||||
"last_id": "input_item_123",
|
||||
}
|
||||
mock_list_input_items.return_value = mock_response
|
||||
|
||||
# Execute
|
||||
result = mock_list_input_items(
|
||||
response_id="resp_test123",
|
||||
after="cursor_after",
|
||||
limit=10,
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
|
||||
# Assert
|
||||
mock_list_input_items.assert_called_once_with(
|
||||
response_id="resp_test123",
|
||||
after="cursor_after",
|
||||
limit=10,
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
assert result["object"] == "list"
|
||||
assert len(result["data"]) == 1
|
||||
|
||||
def test_parameter_validation_edge_cases(self):
|
||||
"""Test edge cases for parameter validation"""
|
||||
# Test with limit=0
|
||||
url, params = self.openai_config.transform_list_input_items_request(
|
||||
response_id=self.response_id,
|
||||
api_base=self.api_base,
|
||||
litellm_params=self.litellm_params,
|
||||
headers=self.headers,
|
||||
limit=0,
|
||||
)
|
||||
assert params["limit"] == 0
|
||||
|
||||
# Test with very large limit
|
||||
url, params = self.openai_config.transform_list_input_items_request(
|
||||
response_id=self.response_id,
|
||||
api_base=self.api_base,
|
||||
litellm_params=self.litellm_params,
|
||||
headers=self.headers,
|
||||
limit=1000,
|
||||
)
|
||||
assert params["limit"] == 1000
|
||||
|
||||
# Test with single item in include list
|
||||
url, params = self.openai_config.transform_list_input_items_request(
|
||||
response_id=self.response_id,
|
||||
api_base=self.api_base,
|
||||
litellm_params=self.litellm_params,
|
||||
headers=self.headers,
|
||||
include=["metadata"],
|
||||
)
|
||||
assert params["include"] == "metadata"
|
||||
|
||||
def test_url_construction_with_different_api_bases(self):
|
||||
"""Test URL construction with different API base formats"""
|
||||
test_cases = [
|
||||
{
|
||||
"api_base": "https://api.openai.com/v1/responses",
|
||||
"expected_suffix": "/resp_abc123/input_items",
|
||||
},
|
||||
{
|
||||
"api_base": "https://api.openai.com/v1/responses/", # with trailing slash
|
||||
"expected_suffix": "/resp_abc123/input_items",
|
||||
},
|
||||
{
|
||||
"api_base": "https://custom-api.example.com/v1/responses",
|
||||
"expected_suffix": "/resp_abc123/input_items",
|
||||
},
|
||||
]
|
||||
|
||||
for case in test_cases:
|
||||
url, params = self.openai_config.transform_list_input_items_request(
|
||||
response_id=self.response_id,
|
||||
api_base=case["api_base"],
|
||||
litellm_params=self.litellm_params,
|
||||
headers=self.headers,
|
||||
)
|
||||
assert url.endswith(case["expected_suffix"])
|
||||
|
||||
def test_return_type_validation(self):
|
||||
"""Test that function returns correct types"""
|
||||
url, params = self.openai_config.transform_list_input_items_request(
|
||||
response_id=self.response_id,
|
||||
api_base=self.api_base,
|
||||
litellm_params=self.litellm_params,
|
||||
headers=self.headers,
|
||||
)
|
||||
|
||||
# Assert return types
|
||||
assert isinstance(url, str)
|
||||
assert isinstance(params, dict)
|
||||
|
||||
# Assert URL is properly formatted
|
||||
assert url.startswith("http")
|
||||
assert "input_items" in url
|
||||
|
||||
# Assert params contains expected keys with correct types
|
||||
for key, value in params.items():
|
||||
assert isinstance(key, str)
|
||||
assert value is not None
|
||||
@@ -0,0 +1,43 @@
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.llms.openai.openai import OpenAIConfig
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def config() -> OpenAIConfig:
|
||||
return OpenAIConfig()
|
||||
|
||||
def test_gpt5_supports_reasoning_effort(config: OpenAIConfig):
|
||||
assert "reasoning_effort" in config.get_supported_openai_params(model="gpt-5")
|
||||
assert "reasoning_effort" in config.get_supported_openai_params(model="gpt-5-mini")
|
||||
|
||||
def test_gpt5_maps_max_tokens(config: OpenAIConfig):
|
||||
params = config.map_openai_params(
|
||||
non_default_params={"max_tokens": 10},
|
||||
optional_params={},
|
||||
model="gpt-5",
|
||||
drop_params=False,
|
||||
)
|
||||
assert params["max_completion_tokens"] == 10
|
||||
assert "max_tokens" not in params
|
||||
|
||||
|
||||
def test_gpt5_temperature_drop(config: OpenAIConfig):
|
||||
params = config.map_openai_params(
|
||||
non_default_params={"temperature": 0.2},
|
||||
optional_params={},
|
||||
model="gpt-5",
|
||||
drop_params=True,
|
||||
)
|
||||
assert "temperature" not in params
|
||||
|
||||
|
||||
def test_gpt5_temperature_error(config: OpenAIConfig):
|
||||
with pytest.raises(litellm.utils.UnsupportedParamsError):
|
||||
config.map_openai_params(
|
||||
non_default_params={"temperature": 0.2},
|
||||
optional_params={},
|
||||
model="gpt-5",
|
||||
drop_params=False,
|
||||
)
|
||||
@@ -0,0 +1,45 @@
|
||||
import pytest
|
||||
|
||||
from litellm.llms.openai.chat.o_series_transformation import OpenAIOSeriesConfig
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_name,expected",
|
||||
[
|
||||
# Valid O-series models
|
||||
("o1", True),
|
||||
("o3", True),
|
||||
("o4-mini", True),
|
||||
("o3-mini", True),
|
||||
# Valid O-series models with provider prefix
|
||||
("openai/o1", True),
|
||||
("openai/o3", True),
|
||||
("openai/o4-mini", True),
|
||||
("openai/o3-mini", True),
|
||||
# Non-O-series models
|
||||
("gpt-4", False),
|
||||
("gpt-3.5-turbo", False),
|
||||
("claude-3-opus", False),
|
||||
# Non-O-series models with provider prefix
|
||||
("openai/gpt-4", False),
|
||||
("openai/gpt-3.5-turbo", False),
|
||||
("anthropic/claude-3-opus", False),
|
||||
# Edge cases
|
||||
("o", False), # Too short
|
||||
("o5", False), # Not a valid O-series model
|
||||
("o1-", False), # Invalid suffix
|
||||
("o3_", False), # Invalid suffix
|
||||
],
|
||||
)
|
||||
def test_is_model_o_series_model(model_name: str, expected: bool):
|
||||
"""
|
||||
Test that is_model_o_series_model correctly identifies O-series models.
|
||||
|
||||
Args:
|
||||
model_name: The model name to test
|
||||
expected: The expected result (True if it should be identified as an O-series model)
|
||||
"""
|
||||
config = OpenAIOSeriesConfig()
|
||||
assert (
|
||||
config.is_model_o_series_model(model_name) == expected
|
||||
), f"Expected {model_name} to be {'an O-series model' if expected else 'not an O-series model'}"
|
||||
@@ -0,0 +1,131 @@
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import MagicMock, call, patch
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(
|
||||
0, os.path.abspath("../../..")
|
||||
) # Adds the parent directory to the system path
|
||||
|
||||
import litellm
|
||||
from litellm.llms.openai.common_utils import BaseOpenAILLM
|
||||
|
||||
# Test parameters for different API functions
|
||||
API_FUNCTION_PARAMS = [
|
||||
# (function_name, is_async, args)
|
||||
(
|
||||
"completion",
|
||||
False,
|
||||
{
|
||||
"model": "gpt-4o-mini",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"max_tokens": 10,
|
||||
},
|
||||
),
|
||||
(
|
||||
"completion",
|
||||
True,
|
||||
{
|
||||
"model": "gpt-4o-mini",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"max_tokens": 10,
|
||||
},
|
||||
),
|
||||
(
|
||||
"completion",
|
||||
True,
|
||||
{
|
||||
"model": "gpt-4o-mini",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"max_tokens": 10,
|
||||
"stream": True,
|
||||
},
|
||||
),
|
||||
("embedding", False, {"model": "text-embedding-ada-002", "input": "Hello world"}),
|
||||
("embedding", True, {"model": "text-embedding-ada-002", "input": "Hello world"}),
|
||||
(
|
||||
"image_generation",
|
||||
False,
|
||||
{"model": "dall-e-3", "prompt": "A beautiful sunset over mountains"},
|
||||
),
|
||||
(
|
||||
"image_generation",
|
||||
True,
|
||||
{"model": "dall-e-3", "prompt": "A beautiful sunset over mountains"},
|
||||
),
|
||||
(
|
||||
"speech",
|
||||
False,
|
||||
{
|
||||
"model": "tts-1",
|
||||
"input": "Hello, this is a test of text to speech",
|
||||
"voice": "alloy",
|
||||
},
|
||||
),
|
||||
(
|
||||
"speech",
|
||||
True,
|
||||
{
|
||||
"model": "tts-1",
|
||||
"input": "Hello, this is a test of text to speech",
|
||||
"voice": "alloy",
|
||||
},
|
||||
),
|
||||
("transcription", False, {"model": "whisper-1", "file": MagicMock()}),
|
||||
("transcription", True, {"model": "whisper-1", "file": MagicMock()}),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("function_name,is_async,args", API_FUNCTION_PARAMS)
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_client_reuse(function_name, is_async, args):
|
||||
"""
|
||||
Test that multiple API calls reuse the same OpenAI client
|
||||
"""
|
||||
litellm.set_verbose = True
|
||||
|
||||
# Determine which client class to mock based on whether the test is async
|
||||
client_path = (
|
||||
"litellm.llms.openai.openai.AsyncOpenAI"
|
||||
if is_async
|
||||
else "litellm.llms.openai.openai.OpenAI"
|
||||
)
|
||||
|
||||
# Create the appropriate patches
|
||||
with patch(client_path) as mock_client_class, patch.object(
|
||||
BaseOpenAILLM, "set_cached_openai_client"
|
||||
) as mock_set_cache, patch.object(
|
||||
BaseOpenAILLM, "get_cached_openai_client"
|
||||
) as mock_get_cache:
|
||||
# Setup the mock to return None first time (cache miss) then a client for subsequent calls
|
||||
mock_client = MagicMock()
|
||||
mock_get_cache.side_effect = [None] + [
|
||||
mock_client
|
||||
] * 9 # First call returns None, rest return the mock client
|
||||
|
||||
# Make 10 API calls
|
||||
for _ in range(10):
|
||||
try:
|
||||
# Call the appropriate function based on parameters
|
||||
if is_async:
|
||||
# Add 'a' prefix for async functions
|
||||
func = getattr(litellm, f"a{function_name}")
|
||||
await func(**args)
|
||||
else:
|
||||
func = getattr(litellm, function_name)
|
||||
func(**args)
|
||||
except Exception:
|
||||
# We expect exceptions since we're mocking the client
|
||||
pass
|
||||
|
||||
# Verify client was created only once
|
||||
assert (
|
||||
mock_client_class.call_count == 1
|
||||
), f"{'Async' if is_async else ''}OpenAI client should be created only once"
|
||||
|
||||
# Verify the client was cached
|
||||
assert mock_set_cache.call_count == 1, "Client should be cached once"
|
||||
|
||||
# Verify we tried to get from cache 10 times (once per request)
|
||||
assert mock_get_cache.call_count == 10, "Should check cache for each request"
|
||||
@@ -0,0 +1,67 @@
|
||||
import pytest
|
||||
|
||||
from litellm.llms.openai.vector_stores.transformation import OpenAIVectorStoreConfig
|
||||
from litellm.types.vector_stores import (
|
||||
VectorStoreCreateOptionalRequestParams,
|
||||
)
|
||||
|
||||
|
||||
class TestOpenAIVectorStoreAPIConfig:
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"metadata", [{}, None]
|
||||
)
|
||||
def test_transform_create_vector_store_request_with_metadata_empty_or_none(self, metadata):
|
||||
"""
|
||||
Test transform_create_vector_store_request when metadata is None or empty dict.
|
||||
"""
|
||||
config = OpenAIVectorStoreConfig()
|
||||
api_base = "https://api.openai.com/v1/vector_stores"
|
||||
|
||||
vector_store_create_params: VectorStoreCreateOptionalRequestParams = {
|
||||
"name": "test-vector-store",
|
||||
"file_ids": ["file-123", "file-456"],
|
||||
"metadata": metadata,
|
||||
}
|
||||
|
||||
url, request_body = config.transform_create_vector_store_request(
|
||||
vector_store_create_params, api_base
|
||||
)
|
||||
|
||||
assert url == api_base
|
||||
assert request_body["name"] == "test-vector-store"
|
||||
assert request_body["file_ids"] == ["file-123", "file-456"]
|
||||
assert request_body["metadata"] == metadata
|
||||
|
||||
|
||||
def test_transform_create_vector_store_request_with_large_metadata(self):
|
||||
"""
|
||||
Test transform_create_vector_store_request with metadata exceeding 16 keys.
|
||||
|
||||
OpenAI limits metadata to 16 keys maximum.
|
||||
"""
|
||||
config = OpenAIVectorStoreConfig()
|
||||
api_base = "https://api.openai.com/v1/vector_stores"
|
||||
|
||||
# Create metadata with more than 16 keys
|
||||
large_metadata = {f"key_{i}": f"value_{i}" for i in range(20)}
|
||||
|
||||
vector_store_create_params: VectorStoreCreateOptionalRequestParams = {
|
||||
"name": "test-vector-store",
|
||||
"metadata": large_metadata,
|
||||
}
|
||||
|
||||
url, request_body = config.transform_create_vector_store_request(
|
||||
vector_store_create_params, api_base
|
||||
)
|
||||
|
||||
assert url == api_base
|
||||
assert request_body["name"] == "test-vector-store"
|
||||
|
||||
# Should be trimmed to 16 keys
|
||||
assert len(request_body["metadata"]) == 16
|
||||
|
||||
# Should contain the first 16 keys (as per add_openai_metadata implementation)
|
||||
for i in range(16):
|
||||
assert f"key_{i}" in request_body["metadata"]
|
||||
assert request_body["metadata"][f"key_{i}"] == f"value_{i}"
|
||||
Reference in New Issue
Block a user