Added LiteLLM to the stack

Development/litellm/tests/proxy_admin_ui_tests/conftest.py

@@ -0,0 +1,54 @@
+# conftest.py
+
+import importlib
+import os
+import sys
+
+import pytest
+
+sys.path.insert(
+    0, os.path.abspath("../..")
+)  # Adds the parent directory to the system path
+import litellm
+
+
+@pytest.fixture(scope="function", autouse=True)
+def setup_and_teardown():
+    """
+    This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained.
+    """
+    curr_dir = os.getcwd()  # Get the current working directory
+    sys.path.insert(
+        0, os.path.abspath("../..")
+    )  # Adds the project directory to the system path
+
+    import litellm
+    from litellm import Router
+
+    importlib.reload(litellm)
+    import asyncio
+
+    loop = asyncio.get_event_loop_policy().new_event_loop()
+    asyncio.set_event_loop(loop)
+    print(litellm)
+    # from litellm import Router, completion, aembedding, acompletion, embedding
+    yield
+
+    # Teardown code (executes after the yield point)
+    loop.close()  # Close the loop created earlier
+    asyncio.set_event_loop(None)  # Remove the reference to the loop
+
+
+def pytest_collection_modifyitems(config, items):
+    # Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests
+    custom_logger_tests = [
+        item for item in items if "custom_logger" in item.parent.name
+    ]
+    other_tests = [item for item in items if "custom_logger" not in item.parent.name]
+
+    # Sort tests based on their names
+    custom_logger_tests.sort(key=lambda x: x.name)
+    other_tests.sort(key=lambda x: x.name)
+
+    # Reorder the items list
+    items[:] = custom_logger_tests + other_tests

Development/litellm/tests/proxy_admin_ui_tests/e2e_ui_tests/login_to_ui.spec.ts

@@ -0,0 +1,44 @@
+/*
+
+Login to Admin UI
+Basic UI Test
+
+Click on all the tabs ensure nothing is broken
+*/
+
+import { test, expect } from '@playwright/test';
+
+test('admin login test', async ({ page }) => {
+  // Go to the specified URL
+  await page.goto('http://localhost:4000/ui');
+
+  // Enter "admin" in the username input field
+  await page.fill('input[name="username"]', 'admin');
+
+  // Enter "gm" in the password input field
+  await page.fill('input[name="password"]', 'gm');
+
+  // Optionally, you can add an assertion to verify the login button is enabled
+  const loginButton = page.locator('input[type="submit"]');
+  await expect(loginButton).toBeEnabled();
+
+  // Optionally, you can click the login button to submit the form
+  await loginButton.click();
+  const tabs = [
+    'Virtual Keys',
+    'Test Key',
+    'Models',
+    'Usage',
+    'Teams',
+    'Internal User',
+    'Settings',
+    'Experimental',
+    'API Reference',
+    'Model Hub'
+  ];
+
+  for (const tab of tabs) {
+    const tabElement = page.locator('span.ant-menu-title-content', { hasText: tab });
+    await tabElement.click();
+  }
+});

Development/litellm/tests/proxy_admin_ui_tests/e2e_ui_tests/require_auth_for_dashboard.spec.ts

@@ -0,0 +1,31 @@
+// tests/auth.spec.ts
+import { test, expect } from "@playwright/test";
+
+test.describe("Authentication Checks", () => {
+  test("should redirect unauthenticated user from a protected page", async ({ page }) => {
+    test.setTimeout(30000);
+
+    page.on("console", (msg) => console.log("PAGE LOG:", msg.text()));
+
+    const protectedPageUrl = "http://localhost:4000/ui?page=llm-playground";
+    const expectedRedirectUrl = "http://localhost:4000/sso/key/generate";
+
+    console.log(`Attempting to navigate to protected page: ${protectedPageUrl}`);
+
+    await page.goto(protectedPageUrl);
+
+    console.log(`Navigation initiated. Current URL: ${page.url()}`);
+
+    try {
+      await page.waitForURL(expectedRedirectUrl, { timeout: 10000 });
+      console.log(`Waited for URL. Current URL is now: ${page.url()}`);
+    } catch (error) {
+      console.error(`Timeout waiting for URL: ${expectedRedirectUrl}. Current URL: ${page.url()}`);
+      await page.screenshot({ path: "redirect-fail-screenshot.png" });
+      throw error;
+    }
+
+    await expect(page).toHaveURL(expectedRedirectUrl);
+    console.log(`Assertion passed: Page URL is ${expectedRedirectUrl}`);
+  });
+});

Development/litellm/tests/proxy_admin_ui_tests/e2e_ui_tests/search_users.spec.ts

@@ -0,0 +1,214 @@
+/*
+Search Users in Admin UI
+E2E Test for user search functionality
+
+Tests:
+1. Navigate to Internal Users tab
+2. Verify search input exists
+3. Test search functionality
+4. Verify results update
+5. Test filtering by email, user ID, and SSO user ID
+*/
+
+import { test, expect } from "@playwright/test";
+
+test("user search test", async ({ page }) => {
+  // Set a longer timeout for the entire test
+  test.setTimeout(60000);
+
+  // Enable console logging
+  page.on("console", (msg) => console.log("PAGE LOG:", msg.text()));
+
+  // Login first
+  await page.goto("http://localhost:4000/ui");
+  console.log("Navigated to login page");
+
+  // Wait for login form to be visible
+  await page.waitForSelector('input[name="username"]', { timeout: 10000 });
+  console.log("Login form is visible");
+
+  await page.fill('input[name="username"]', "admin");
+  await page.fill('input[name="password"]', "gm");
+  console.log("Filled login credentials");
+
+  const loginButton = page.locator('input[type="submit"]');
+  await expect(loginButton).toBeEnabled();
+  await loginButton.click();
+  console.log("Clicked login button");
+
+  // Wait for navigation to complete and dashboard to load
+  await page.waitForLoadState("networkidle");
+  console.log("Page loaded after login");
+
+  // Take a screenshot for debugging
+  await page.screenshot({ path: "after-login.png" });
+  console.log("Took screenshot after login");
+
+  // Try to find the Internal User tab with more debugging
+  console.log("Looking for Internal User tab...");
+  const internalUserTab = page.locator("span.ant-menu-title-content", {
+    hasText: "Internal User",
+  });
+
+  // Wait for the tab to be visible
+  await internalUserTab.waitFor({ state: "visible", timeout: 10000 });
+  console.log("Internal User tab is visible");
+
+  // Take another screenshot before clicking
+  await page.screenshot({ path: "before-tab-click.png" });
+  console.log("Took screenshot before tab click");
+
+  await internalUserTab.click();
+  console.log("Clicked Internal User tab");
+
+  // Wait for the page to load and table to be visible
+  await page.waitForSelector("tbody tr", { timeout: 30000 });
+  await page.waitForTimeout(2000); // Additional wait for table to stabilize
+  console.log("Table is visible");
+
+  // Take a final screenshot
+  await page.screenshot({ path: "after-tab-click.png" });
+  console.log("Took screenshot after tab click");
+
+  // Verify search input exists
+  const searchInput = page.locator('input[placeholder="Search by email..."]');
+  await expect(searchInput).toBeVisible();
+  console.log("Search input is visible");
+
+  // Test search functionality
+  const initialUserCount = await page.locator("tbody tr").count();
+  console.log(`Initial user count: ${initialUserCount}`);
+
+  // Perform a search
+  const testEmail = "test@";
+  await searchInput.fill(testEmail);
+  console.log("Filled search input");
+
+  // Wait for the debounced search to complete
+  await page.waitForTimeout(500);
+  console.log("Waited for debounce");
+
+  // Wait for the results count to update
+  await page.waitForFunction((initialCount) => {
+    const currentCount = document.querySelectorAll("tbody tr").length;
+    return currentCount !== initialCount;
+  }, initialUserCount);
+  console.log("Results updated");
+
+  const filteredUserCount = await page.locator("tbody tr").count();
+  console.log(`Filtered user count: ${filteredUserCount}`);
+
+  expect(filteredUserCount).toBeDefined();
+
+  // Clear the search
+  await searchInput.clear();
+  console.log("Cleared search");
+
+  await page.waitForTimeout(500);
+  console.log("Waited for debounce after clear");
+
+  await page.waitForFunction((initialCount) => {
+    const currentCount = document.querySelectorAll("tbody tr").length;
+    return currentCount === initialCount;
+  }, initialUserCount);
+  console.log("Results reset");
+
+  const resetUserCount = await page.locator("tbody tr").count();
+  console.log(`Reset user count: ${resetUserCount}`);
+
+  expect(resetUserCount).toBe(initialUserCount);
+});
+
+test("user filter test", async ({ page }) => {
+  // Set a longer timeout for the entire test
+  test.setTimeout(60000);
+
+  // Enable console logging
+  page.on("console", (msg) => console.log("PAGE LOG:", msg.text()));
+
+  // Login first
+  await page.goto("http://localhost:4000/ui");
+  console.log("Navigated to login page");
+
+  // Wait for login form to be visible
+  await page.waitForSelector('input[name="username"]', { timeout: 10000 });
+  console.log("Login form is visible");
+
+  await page.fill('input[name="username"]', "admin");
+  await page.fill('input[name="password"]', "gm");
+  console.log("Filled login credentials");
+
+  const loginButton = page.locator('input[type="submit"]');
+  await expect(loginButton).toBeEnabled();
+  await loginButton.click();
+  console.log("Clicked login button");
+
+  // Wait for navigation to complete and dashboard to load
+  await page.waitForLoadState("networkidle");
+  console.log("Page loaded after login");
+
+  // Navigate to Internal Users tab
+  const internalUserTab = page.locator("span.ant-menu-title-content", {
+    hasText: "Internal User",
+  });
+  await internalUserTab.waitFor({ state: "visible", timeout: 10000 });
+  await internalUserTab.click();
+  console.log("Clicked Internal User tab");
+
+  // Wait for the page to load and table to be visible
+  await page.waitForSelector("tbody tr", { timeout: 30000 });
+  await page.waitForTimeout(2000); // Additional wait for table to stabilize
+  console.log("Table is visible");
+
+  // Get initial user count
+  const initialUserCount = await page.locator("tbody tr").count();
+  console.log(`Initial user count: ${initialUserCount}`);
+
+  // Click the filter button to show additional filters
+  const filterButton = page.getByRole("button", {
+    name: "Filters",
+    exact: true,
+  });
+  await filterButton.click();
+  console.log("Clicked filter button");
+  await page.waitForTimeout(500); // Wait for filters to appear
+
+  // Test user ID filter
+  const userIdInput = page.locator('input[placeholder="Filter by User ID"]');
+  await expect(userIdInput).toBeVisible();
+  console.log("User ID filter is visible");
+
+  await userIdInput.fill("user");
+  console.log("Filled user ID filter");
+  await page.waitForTimeout(1000);
+  const userIdFilteredCount = await page.locator("tbody tr").count();
+  console.log(`User ID filtered count: ${userIdFilteredCount}`);
+  expect(userIdFilteredCount).toBeLessThan(initialUserCount);
+
+  // Clear user ID filter
+  await userIdInput.clear();
+  await page.waitForTimeout(1000);
+  console.log("Cleared user ID filter");
+
+  // Test SSO user ID filter
+  const ssoUserIdInput = page.locator('input[placeholder="Filter by SSO ID"]');
+  await expect(ssoUserIdInput).toBeVisible();
+  console.log("SSO user ID filter is visible");
+
+  await ssoUserIdInput.fill("sso");
+  console.log("Filled SSO user ID filter");
+  await page.waitForTimeout(1000);
+  const ssoUserIdFilteredCount = await page.locator("tbody tr").count();
+  console.log(`SSO user ID filtered count: ${ssoUserIdFilteredCount}`);
+  expect(ssoUserIdFilteredCount).toBeLessThan(initialUserCount);
+
+  // Clear SSO user ID filter
+  await ssoUserIdInput.clear();
+  await page.waitForTimeout(5000);
+  console.log("Cleared SSO user ID filter");
+
+  // Verify count returns to initial after clearing all filters
+  const finalUserCount = await page.locator("tbody tr").count();
+  console.log(`Final user count: ${finalUserCount}`);
+  expect(finalUserCount).toBe(initialUserCount);
+});

Development/litellm/tests/proxy_admin_ui_tests/e2e_ui_tests/team_admin.spec.ts

@@ -0,0 +1,250 @@
+import { test, expect } from "@playwright/test";
+import { loginToUI } from "../utils/login";
+
+// test.describe("Invite User, Set Password, and Login", () => {
+//   let testEmail: string;
+//   const testPassword = "Password123!"; // Define a password
+//   const teamName1 = `team-invite-test-1-${Date.now()}`;
+//   const teamName2 = `team-invite-test-2-${Date.now()}`;
+//   const keyName1 = `key-${teamName1}`;
+//   const keyName2 = `key-${teamName2}`;
+
+//   test.beforeEach(async ({ page }) => {
+//     await loginToUI(page); // Login as admin first
+//     await page.goto("http://localhost:4000/ui?page=teams");
+
+//     // --- Create Team 1 ---
+//     await page.getByRole("button", { name: "+ Create New Team" }).click();
+//     await page
+//       .getByLabel("Team Name")
+//       .waitFor({ state: "visible", timeout: 5000 }); // Wait for label
+//     await page.getByLabel("Team Name").click();
+//     await page.getByLabel("Team Name").fill(teamName1);
+//     await page.getByRole("button", { name: "Create Team" }).click();
+//     // Wait for the modal to close or for a success message if applicable
+//     await expect(
+//       page.locator(".ant-modal-wrap").filter({ hasText: "Create New Team" })
+//     ).not.toBeVisible({ timeout: 10000 });
+//     console.log(`Created Team 1: ${teamName1}`);
+
+//     // --- Create Team 2 ---
+//     await page.getByRole("button", { name: "+ Create New Team" }).click();
+//     await page
+//       .getByLabel("Team Name")
+//       .waitFor({ state: "visible", timeout: 5000 }); // Wait for label
+//     await page.getByLabel("Team Name").click();
+//     await page.getByLabel("Team Name").fill(teamName2);
+//     await page.getByRole("button", { name: "Create Team" }).click();
+//     // Wait for the modal to close or for a success message if applicable
+//     await expect(
+//       page.locator(".ant-modal-wrap").filter({ hasText: "Create New Team" })
+//     ).not.toBeVisible({ timeout: 10000 });
+//     console.log(`Created Team 2: ${teamName2}`);
+
+//     // // Verify both teams are listed
+//     // await page.goto("http://localhost:4000/ui?page=teams"); // Refresh or ensure on teams page
+//     // await page.waitForTimeout(3000);
+//     await expect(page.getByText(teamName1)).toBeVisible({ timeout: 10000 });
+//     await expect(page.getByText(teamName2)).toBeVisible({ timeout: 10000 });
+
+//     // --- Navigate to Keys Page ---
+//     await page.goto("http://localhost:4000/ui?page=api-keys");
+//     await page.waitForTimeout(3000);
+//     await expect(
+//       page.getByRole("button", { name: "+ Create New Key" })
+//     ).toBeVisible(); // Wait for page load
+
+//     // --- Create Key for Team 1 ---
+//     await page.getByRole("button", { name: "+ Create New Key" }).click();
+//     const createKeyModal1 = page
+//       .locator(".ant-modal-wrap")
+//       .filter({ hasText: "Key Ownership" });
+//     await expect(createKeyModal1).toBeVisible();
+
+//     // Select Team 1
+//     await createKeyModal1
+//       .locator(".ant-select-selector >> input")
+//       .first()
+//       .click(); // Click to open team dropdown
+//     await createKeyModal1
+//       .locator(".ant-select-selector >> input")
+//       .first()
+//       .fill(teamName1);
+
+//     await page
+//       .locator(".ant-select-item-option")
+//       .filter({ hasText: teamName1 })
+//       .first()
+//       .click(); // Click specific team name
+
+//     // Enter Key Name 1
+//     await page.fill('input[id="key_alias"]', keyName1);
+
+//     // Click on models dropdown
+//     await page.locator("input#models").click();
+//     await page.waitForSelector(
+//       '.ant-select-item-option[title="All Team Models"]'
+//     );
+//     await page
+//       .locator('.ant-select-item-option[title="All Team Models"]')
+//       .click();
+
+//     // Click Create Key
+//     await createKeyModal1.getByRole("button", { name: "Create Key" }).click();
+
+//     // Close the Key Generated modal (which appears after successful creation)
+//     const keyGeneratedModal1 = page
+//       .locator(".ant-modal-wrap")
+//       .filter({ hasText: "Save your Key" });
+//     await expect(keyGeneratedModal1).toBeVisible({ timeout: 10000 });
+//     await keyGeneratedModal1.locator('button[aria-label="Close"]').click();
+//     await expect(keyGeneratedModal1).not.toBeVisible(); // Wait for close
+//     console.log(`Created Key 1: ${keyName1} for Team: ${teamName1}`);
+
+//     // --- Create Key for Team 2 ---
+//     await page.getByRole("button", { name: "+ Create New Key" }).click();
+//     const createKeyModal2 = page
+//       .locator(".ant-modal-wrap")
+//       .filter({ hasText: "Key Ownership" });
+//     await expect(createKeyModal2).toBeVisible();
+
+//     // Select Team 2
+//     await createKeyModal2
+//       .locator(".ant-select-selector >> input")
+//       .first()
+//       .click(); // Click to open team dropdown
+//     await page
+//       .locator(".ant-select-item-option")
+//       .filter({ hasText: teamName2 })
+//       .click(); // Click specific team name
+
+//     // Enter Key Name 2
+//     await page.fill('input[id="key_alias"]', keyName2);
+
+//     // Click on models dropdown
+//     await page.locator("input#models").click();
+//     await page.waitForSelector(
+//       '.ant-select-item-option[title="All Team Models"]'
+//     );
+//     await page
+//       .locator('.ant-select-item-option[title="All Team Models"]')
+//       .click();
+
+//     // Click Create Key
+//     await createKeyModal2.getByRole("button", { name: "Create Key" }).click();
+
+//     // Close the Key Generated modal
+//     const keyGeneratedModal2 = page
+//       .locator(".ant-modal-wrap")
+//       .filter({ hasText: "Save your Key" });
+//     await expect(keyGeneratedModal2).toBeVisible({ timeout: 10000 });
+//     await keyGeneratedModal2.locator('button[aria-label="Close"]').click();
+//     await expect(keyGeneratedModal2).not.toBeVisible(); // Wait for close
+//     console.log(`Created Key 2: ${keyName2} for Team: ${teamName2}`);
+//   });
+
+//   test("Invite user, set password via link, and login", async ({ page }) => {
+//     // Navigate to Users page
+//     await page.goto("http://localhost:4000/ui?page=users");
+
+//     // Go to Internal User tab
+//     const internalUserTab = page.locator("span.ant-menu-title-content", {
+//       hasText: "Internal User",
+//     });
+//     await internalUserTab.waitFor({ state: "visible", timeout: 10000 });
+//     await internalUserTab.click();
+
+//     // --- Invite User Flow ---
+//     await page.getByRole("button", { name: "+ Invite User" }).click();
+
+//     // Wait for the invite user modal to be visible
+//     const inviteModal = page
+//       .locator(".ant-modal-wrap")
+//       .filter({ hasText: "Invite User" });
+//     await expect(inviteModal).toBeVisible();
+
+//     testEmail = `test-${Date.now()}@litellm.ai`; // Use a unique email
+//     // Assuming the email input is the first one with 'base-input' test id inside the modal
+//     await inviteModal.getByTestId("base-input").first().fill(testEmail);
+
+//     // Select Global Admin Role (or another appropriate role)
+//     const globalRoleLabel = inviteModal.getByLabel("Global Proxy Role");
+//     await globalRoleLabel.click();
+//     // Wait for the dropdown option to be visible before clicking
+//     const adminRoleOption = page.getByTitle("Admin (All Permissions)", {
+//       exact: true,
+//     });
+//     await adminRoleOption.waitFor({ state: "visible", timeout: 5000 });
+//     await adminRoleOption.click();
+
+//     // Select Team - Add explicit wait before clicking
+//     const teamIdLabel = inviteModal.getByLabel("Team ID");
+//     // Wait for the label associated with the Team ID select to be visible
+//     await teamIdLabel.waitFor({ state: "visible", timeout: 10000 }); // Increased timeout for safety
+//     await teamIdLabel.click();
+
+//     // Wait for the team name option to be visible in the dropdown
+//     const teamNameOption = page.getByText(teamName1, { exact: true });
+//     await teamNameOption.waitFor({ state: "visible", timeout: 5000 });
+//     await teamNameOption.click();
+
+//     // Create User
+//     await inviteModal.getByRole("button", { name: "Create User" }).click();
+
+//     // --- Capture Invitation Link ---
+//     const invitationModal = page
+//       .locator(".ant-modal-wrap")
+//       .filter({ hasText: "Invitation Link" });
+//     await expect(invitationModal).toBeVisible({ timeout: 15000 }); // Wait longer for modal
+
+//     // Locate the text element containing the URL more reliably
+//     const invitationUrl = await page
+//       .locator("div.flex.justify-between.pt-5.pb-2") // find the correct div
+//       .filter({ hasText: "Invitation Link" }) // find the div that has text "Invitation Link"
+//       .locator("p") // find all <p> inside that div
+//       .nth(1) // pick the second <p> (index 1)
+//       .innerText();
+
+//     // Close Invitation Link Modal
+//     await page
+//       .locator(".ant-modal-wrap")
+//       .filter({ hasText: "Invitation Link" })
+//       .locator('button[aria-label="Close"]')
+//       .click();
+
+//     // Close Invite User Modal
+//     await page
+//       .locator(".ant-modal-wrap")
+//       .filter({ hasText: "Invite User" })
+//       .locator('button[aria-label="Close"]')
+//       .click();
+
+//     // Open invite link as new page (simulate invited user)
+//     const context = await page.context()?.browser()?.newContext();
+//     const invitedUserPage = await context?.newPage();
+//     if (!invitedUserPage) {
+//       throw new Error("invitedUserPage is undefined");
+//     }
+//     await invitedUserPage?.goto(invitationUrl || "");
+
+//     //Insert new password
+//     await invitedUserPage?.fill("input#password", testPassword);
+
+//     //Click on submit
+//     await invitedUserPage?.getByRole("button", { name: "Sign Up" }).click();
+
+//     // // --- Verify Keys Created ---
+//     // await invitedUserPage?.waitForSelector("table");
+
+//     // // Verify keyName1 (associated with user's team) IS visible in the table
+//     // const keyTable = invitedUserPage.locator('table'); // Locate the table element
+//     // await expect(keyTable).toBeVisible({ timeout: 10000 }); // Ensure table exists
+//     // // Use getByText within the table scope to find the key name
+//     // await expect(keyTable.getByText(keyName1, { exact: true })).toBeVisible({ timeout: 10000 });
+//     // console.log(`Verified key ${keyName1} is visible for user ${testEmail}`);
+
+//     // // Verify keyName2 (associated with the *other* team) IS NOT visible
+//     // await expect(keyTable.getByText(keyName2, { exact: true })).not.toBeVisible();
+//     // console.log(`Verified key ${keyName2} is NOT visible for user ${testEmail}`);
+//   });
+// });

Development/litellm/tests/proxy_admin_ui_tests/e2e_ui_tests/view_internal_user.spec.ts

@@ -0,0 +1,75 @@
+/*
+Test view internal user page
+*/
+
+import { test, expect } from "@playwright/test";
+
+test("view internal user page", async ({ page }) => {
+  // Go to the specified URL
+  await page.goto("http://localhost:4000/ui");
+
+  // Enter "admin" in the username input field
+  await page.fill('input[name="username"]', "admin");
+
+  // Enter "gm" in the password input field
+  await page.fill('input[name="password"]', "gm");
+
+  // Click the login button
+  const loginButton = page.locator('input[type="submit"]');
+  await expect(loginButton).toBeEnabled();
+  await loginButton.click();
+
+  // Wait for the Internal User tab and click it
+  const tabElement = page.locator("span.ant-menu-title-content", {
+    hasText: "Internal User",
+  });
+  await tabElement.click();
+
+  // Wait for the table to load
+  await page.waitForSelector("tbody tr", { timeout: 10000 });
+  await page.waitForTimeout(2000); // Additional wait for table to stabilize
+
+  // Test all expected fields are present
+  // number of keys owned by user
+  const keysBadges = page.locator(
+    "p.tremor-Badge-text.text-sm.whitespace-nowrap",
+    { hasText: "Keys" }
+  );
+  const keysCountArray = await keysBadges.evaluateAll((elements) =>
+    elements.map((el) => {
+      const text = el.textContent;
+      return text ? parseInt(text.split(" ")[0], 10) : 0;
+    })
+  );
+
+  const hasNonZeroKeys = keysCountArray.some((count) => count > 0);
+  expect(hasNonZeroKeys).toBe(true);
+
+  // test pagination
+  // Wait for pagination controls to be visible
+  await page.waitForSelector(".flex.justify-between.items-center", {
+    timeout: 5000,
+  });
+
+  // Check if we're on the first page by looking at the results count
+  const resultsText =
+    (await page.locator(".text-sm.text-gray-700").textContent()) || "";
+  const isFirstPage = resultsText.includes("1 -");
+
+  if (isFirstPage) {
+    // On first page, previous button should be disabled
+    const prevButton = page.locator("button", { hasText: "Previous" });
+    await expect(prevButton).toBeDisabled();
+  }
+
+  // Next button should be enabled if there are more pages
+  const nextButton = page.locator("button", { hasText: "Next" });
+  const totalResults =
+    (await page.locator(".text-sm.text-gray-700").textContent()) || "";
+  const hasMorePages =
+    totalResults.includes("of") && !totalResults.includes("1 - 25 of 25");
+
+  if (hasMorePages) {
+    await expect(nextButton).toBeEnabled();
+  }
+});

Development/litellm/tests/proxy_admin_ui_tests/e2e_ui_tests/view_user_info.spec.ts

@@ -0,0 +1,81 @@
+import { test, expect } from "@playwright/test";
+import { loginToUI } from "../utils/login";
+
+test.describe("User Info View", () => {
+  test.beforeEach(async ({ page }) => {
+    await loginToUI(page);
+    // Navigate to users page
+    await page.goto("http://localhost:4000/ui?page=users");
+  });
+
+  test("should display user info when clicking on user ID", async ({
+    page,
+  }) => {
+    // Wait for users table to load
+    await page.waitForSelector("table");
+
+    // Get the first user ID cell
+    const firstUserIdCell = page.locator(
+      "table tbody tr:first-child td:first-child"
+    );
+    const userId = await firstUserIdCell.textContent();
+    console.log("Found user ID:", userId);
+
+    // Click on the user ID
+    await firstUserIdCell.click();
+
+    // Check for tabs
+    await expect(page.locator('button:has-text("Overview")')).toBeVisible();
+    await expect(page.locator('button:has-text("Details")')).toBeVisible();
+
+    // Switch to details tab
+    await page.locator('button:has-text("Details")').click();
+
+    // Check details section
+    await expect(page.locator("text=User ID")).toBeVisible();
+    await expect(page.locator("text=Email")).toBeVisible();
+
+    // Go back to users list
+    await page.locator('button:has-text("Back to Users")').click();
+
+    // Verify we're back on the users page
+    await expect(page.locator("table")).toBeVisible();
+    await expect(
+      page.locator('input[placeholder="Search by email..."]')
+    ).toBeVisible();
+  });
+
+  // test("should handle user deletion", async ({ page }) => {
+  //   // Wait for users table to load
+  //   await page.waitForSelector("table");
+
+  //   // Get the first user ID cell
+  //   const firstUserIdCell = page.locator(
+  //     "table tbody tr:first-child td:first-child"
+  //   );
+  //   const userId = await firstUserIdCell.textContent();
+
+  //   // Click on the user ID
+  //   await firstUserIdCell.click();
+
+  //   // Wait for user info view to load
+  //   await page.waitForSelector('h1:has-text("User")');
+
+  //   // Click delete button
+  //   await page.locator('button:has-text("Delete User")').click();
+
+  //   // Confirm deletion in modal
+  //   await page.locator('button:has-text("Delete")').click();
+
+  //   // Verify success message
+  //   await expect(page.locator("text=User deleted successfully")).toBeVisible();
+
+  //   // Verify we're back on the users page
+  //   await expect(page.locator('h1:has-text("Users")')).toBeVisible();
+
+  //   // Verify user is no longer in the table
+  //   if (userId) {
+  //     await expect(page.locator(`text=${userId}`)).not.toBeVisible();
+  //   }
+  // });
+});

Development/litellm/tests/proxy_admin_ui_tests/package-lock.json

@@ -0,0 +1,91 @@
+{
+  "name": "proxy_admin_ui_tests",
+  "version": "1.0.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "proxy_admin_ui_tests",
+      "version": "1.0.0",
+      "license": "ISC",
+      "devDependencies": {
+        "@playwright/test": "^1.47.2",
+        "@types/node": "^22.5.5"
+      }
+    },
+    "node_modules/@playwright/test": {
+      "version": "1.47.2",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.47.2.tgz",
+      "integrity": "sha512-jTXRsoSPONAs8Za9QEQdyjFn+0ZQFjCiIztAIF6bi1HqhBzG9Ma7g1WotyiGqFSBRZjIEqMdT8RUlbk1QVhzCQ==",
+      "dev": true,
+      "dependencies": {
+        "playwright": "1.47.2"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@types/node": {
+      "version": "22.5.5",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-22.5.5.tgz",
+      "integrity": "sha512-Xjs4y5UPO/CLdzpgR6GirZJx36yScjh73+2NlLlkFRSoQN8B0DpfXPdZGnvVmLRLOsqDpOfTNv7D9trgGhmOIA==",
+      "dev": true,
+      "dependencies": {
+        "undici-types": "~6.19.2"
+      }
+    },
+    "node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "dev": true,
+      "hasInstallScript": true,
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
+    "node_modules/playwright": {
+      "version": "1.47.2",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.47.2.tgz",
+      "integrity": "sha512-nx1cLMmQWqmA3UsnjaaokyoUpdVaaDhJhMoxX2qj3McpjnsqFHs516QAKYhqHAgOP+oCFTEOCOAaD1RgD/RQfA==",
+      "dev": true,
+      "dependencies": {
+        "playwright-core": "1.47.2"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "optionalDependencies": {
+        "fsevents": "2.3.2"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.47.2",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.47.2.tgz",
+      "integrity": "sha512-3JvMfF+9LJfe16l7AbSmU555PaTl2tPyQsVInqm3id16pdDfvZ8TTZ/pyzmkbDrZTQefyzU7AIHlZqQnxpqHVQ==",
+      "dev": true,
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/undici-types": {
+      "version": "6.19.8",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz",
+      "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==",
+      "dev": true
+    }
+  }
+}

Development/litellm/tests/proxy_admin_ui_tests/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "proxy_admin_ui_tests",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {},
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "@playwright/test": "^1.47.2",
+    "@types/node": "^22.5.5"
+  }
+}

Development/litellm/tests/proxy_admin_ui_tests/playwright.config.ts

@@ -0,0 +1,84 @@
+import { defineConfig, devices } from '@playwright/test';
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// import dotenv from 'dotenv';
+// import path from 'path';
+// dotenv.config({ path: path.resolve(__dirname, '.env') });
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: './e2e_ui_tests',
+  testIgnore: ['**/tests/pass_through_tests/**', '../pass_through_tests/**/*'],
+  testMatch: '**/*.spec.ts',  // Only run files ending in .spec.ts
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: 'html',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    // baseURL: 'http://127.0.0.1:3000',
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+
+    {
+      name: 'firefox',
+      use: { ...devices['Desktop Firefox'] },
+    },
+
+    {
+      name: 'webkit',
+      use: { ...devices['Desktop Safari'] },
+    },
+
+    /* Test against mobile viewports. */
+    // {
+    //   name: 'Mobile Chrome',
+    //   use: { ...devices['Pixel 5'] },
+    // },
+    // {
+    //   name: 'Mobile Safari',
+    //   use: { ...devices['iPhone 12'] },
+    // },
+
+    /* Test against branded browsers. */
+    // {
+    //   name: 'Microsoft Edge',
+    //   use: { ...devices['Desktop Edge'], channel: 'msedge' },
+    // },
+    // {
+    //   name: 'Google Chrome',
+    //   use: { ...devices['Desktop Chrome'], channel: 'chrome' },
+    // },
+  ],
+  timeout: 4*60*1000,
+  expect: {
+    timeout: 10 * 1000
+  }
+  /* Run your local dev server before starting the tests */
+  // webServer: {
+  //   command: 'npm run start',
+  //   url: 'http://127.0.0.1:3000',
+  //   reuseExistingServer: !process.env.CI,
+  // },
+});

Development/litellm/tests/proxy_admin_ui_tests/test-results/.last-run.json

@@ -0,0 +1,4 @@
+{
+  "status": "passed",
+  "failedTests": []
+}

Development/litellm/tests/proxy_admin_ui_tests/test_role_based_access.py

@@ -0,0 +1,536 @@
+"""
+RBAC tests
+"""
+
+import os
+import sys
+import traceback
+import uuid
+from datetime import datetime
+
+from dotenv import load_dotenv
+from fastapi import Request
+from fastapi.routing import APIRoute
+
+load_dotenv()
+import io
+import os
+import time
+
+# this file is to test litellm/proxy
+
+sys.path.insert(
+    0, os.path.abspath("../..")
+)  # Adds the parent directory to the system path
+import asyncio
+import logging
+from unittest.mock import MagicMock
+import pytest
+
+import litellm
+from litellm._logging import verbose_proxy_logger
+from litellm.proxy.auth.auth_checks import get_user_object
+from litellm.proxy.management_endpoints.key_management_endpoints import (
+    delete_key_fn,
+    generate_key_fn,
+    generate_key_helper_fn,
+    info_key_fn,
+    regenerate_key_fn,
+    update_key_fn,
+)
+from litellm.proxy.management_endpoints.internal_user_endpoints import new_user
+from litellm.proxy.management_endpoints.organization_endpoints import (
+    new_organization,
+    organization_member_add,
+)
+
+from litellm.proxy.management_endpoints.team_endpoints import (
+    new_team,
+    team_info,
+    update_team,
+)
+from litellm.proxy.proxy_server import (
+    LitellmUserRoles,
+    audio_transcriptions,
+    chat_completion,
+    completion,
+    embeddings,
+    model_list,
+    moderations,
+    user_api_key_auth,
+)
+from litellm.proxy.management_endpoints.customer_endpoints import (
+    new_end_user,
+)
+from litellm.proxy.spend_tracking.spend_management_endpoints import (
+    global_spend,
+    global_spend_logs,
+    global_spend_models,
+    global_spend_keys,
+    spend_key_fn,
+    spend_user_fn,
+    view_spend_logs,
+)
+from starlette.datastructures import URL
+
+from litellm.proxy.utils import PrismaClient, ProxyLogging, hash_token, update_spend
+
+verbose_proxy_logger.setLevel(level=logging.DEBUG)
+
+from starlette.datastructures import URL
+
+from litellm.caching.caching import DualCache
+from litellm.proxy._types import *
+
+proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache())
+
+
+@pytest.fixture
+def prisma_client():
+    from litellm.proxy.proxy_cli import append_query_params
+
+    ### add connection pool + pool timeout args
+    params = {"connection_limit": 100, "pool_timeout": 60}
+    database_url = os.getenv("DATABASE_URL")
+    modified_url = append_query_params(database_url, params)
+    os.environ["DATABASE_URL"] = modified_url
+
+    # Assuming PrismaClient is a class that needs to be instantiated
+    prisma_client = PrismaClient(
+        database_url=os.environ["DATABASE_URL"], proxy_logging_obj=proxy_logging_obj
+    )
+
+    # Reset litellm.proxy.proxy_server.prisma_client to None
+    litellm.proxy.proxy_server.litellm_proxy_budget_name = (
+        f"litellm-proxy-budget-{time.time()}"
+    )
+    litellm.proxy.proxy_server.user_custom_key_generate = None
+
+    return prisma_client
+
+
+"""
+RBAC Tests
+
+1. Add a user to an organization
+    - test 1 - if organization_id does exist expect to create a new user and user, organization relation
+
+2. org admin creates team in his org → success 
+
+3. org admin adds new internal user to his org → success 
+
+4. org admin creates team and internal user not in his org → fail both
+"""
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "user_role",
+    [
+        LitellmUserRoles.ORG_ADMIN,
+        LitellmUserRoles.INTERNAL_USER,
+        LitellmUserRoles.INTERNAL_USER_VIEW_ONLY,
+    ],
+)
+async def test_create_new_user_in_organization(prisma_client, user_role):
+    """
+
+    Add a member to an organization and assert the user object is created with the correct organization memberships / roles
+    """
+    master_key = "sk-1234"
+    setattr(litellm.proxy.proxy_server, "prisma_client", prisma_client)
+    setattr(litellm.proxy.proxy_server, "master_key", master_key)
+    setattr(litellm.proxy.proxy_server, "llm_router", MagicMock())
+
+    await litellm.proxy.proxy_server.prisma_client.connect()
+
+    created_user_id = f"new-user-{uuid.uuid4()}"
+
+    response = await new_organization(
+        data=NewOrganizationRequest(
+            organization_alias=f"new-org-{uuid.uuid4()}",
+        ),
+        user_api_key_dict=UserAPIKeyAuth(
+            user_id=created_user_id,
+            user_role=LitellmUserRoles.PROXY_ADMIN,
+        ),
+    )
+
+    org_id = response.organization_id
+
+    response = await organization_member_add(
+        data=OrganizationMemberAddRequest(
+            organization_id=org_id,
+            member=OrgMember(role=user_role, user_id=created_user_id),
+        ),
+        http_request=None,
+    )
+
+    print("new user response", response)
+
+    # call get_user_object
+
+    user_object = await get_user_object(
+        user_id=created_user_id,
+        prisma_client=prisma_client,
+        user_api_key_cache=DualCache(),
+        user_id_upsert=False,
+    )
+
+    print("user object", user_object)
+
+    assert user_object.organization_memberships is not None
+
+    _membership = user_object.organization_memberships[0]
+
+    assert _membership.user_id == created_user_id
+    assert _membership.organization_id == org_id
+
+    if user_role != None:
+        assert _membership.user_role == user_role
+    else:
+        assert _membership.user_role == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY
+
+
+@pytest.mark.asyncio
+async def test_org_admin_create_team_permissions(prisma_client):
+    """
+    Create a new org admin
+
+    org admin creates a new team in their org -> success
+    """
+    import json
+
+    master_key = "sk-1234"
+    setattr(litellm.proxy.proxy_server, "prisma_client", prisma_client)
+    setattr(litellm.proxy.proxy_server, "master_key", master_key)
+    setattr(litellm.proxy.proxy_server, "llm_router", MagicMock())
+
+    await litellm.proxy.proxy_server.prisma_client.connect()
+
+    response = await new_organization(
+        data=NewOrganizationRequest(
+            organization_alias=f"new-org-{uuid.uuid4()}",
+        ),
+        user_api_key_dict=UserAPIKeyAuth(
+            user_role=LitellmUserRoles.PROXY_ADMIN,
+        ),
+    )
+
+    org_id = response.organization_id
+    created_user_id = f"new-user-{uuid.uuid4()}"
+    response = await organization_member_add(
+        data=OrganizationMemberAddRequest(
+            organization_id=org_id,
+            member=OrgMember(role=LitellmUserRoles.ORG_ADMIN, user_id=created_user_id),
+        ),
+        http_request=None,
+    )
+
+    # create key with the response["user_id"]
+    # proxy admin will generate key for org admin
+    _new_key = await generate_key_fn(
+        data=GenerateKeyRequest(user_id=created_user_id),
+        user_api_key_dict=UserAPIKeyAuth(user_id=created_user_id),
+    )
+
+    new_key = _new_key.key
+
+    print("user api key auth response", response)
+
+    # Create /team/new request -> expect auth to pass
+    request = Request(scope={"type": "http"})
+    request._url = URL(url="/team/new")
+
+    async def return_body():
+        body = {"organization_id": org_id}
+        return bytes(json.dumps(body), "utf-8")
+
+    request.body = return_body
+    response = await user_api_key_auth(request=request, api_key="Bearer " + new_key)
+
+    # after auth - actually create team now
+    response = await new_team(
+        data=NewTeamRequest(
+            organization_id=org_id,
+        ),
+        http_request=request,
+        user_api_key_dict=UserAPIKeyAuth(
+            user_id=response.user_id,
+        ),
+    )
+
+    print("response from new team")
+
+
+@pytest.mark.asyncio
+async def test_org_admin_create_user_permissions(prisma_client):
+    """
+    1. Create a new org admin
+
+    2. org admin adds a new member to their org -> success (using using /organization/member_add)
+
+    """
+    import json
+
+    master_key = "sk-1234"
+    setattr(litellm.proxy.proxy_server, "prisma_client", prisma_client)
+    setattr(litellm.proxy.proxy_server, "master_key", master_key)
+    setattr(litellm.proxy.proxy_server, "llm_router", MagicMock())
+
+    await litellm.proxy.proxy_server.prisma_client.connect()
+
+    # create new org
+    response = await new_organization(
+        data=NewOrganizationRequest(
+            organization_alias=f"new-org-{uuid.uuid4()}",
+        ),
+        user_api_key_dict=UserAPIKeyAuth(
+            user_role=LitellmUserRoles.PROXY_ADMIN,
+        ),
+    )
+    # Create Org Admin
+    org_id = response.organization_id
+    created_user_id = f"new-user-{uuid.uuid4()}"
+    response = await organization_member_add(
+        data=OrganizationMemberAddRequest(
+            organization_id=org_id,
+            member=OrgMember(role=LitellmUserRoles.ORG_ADMIN, user_id=created_user_id),
+        ),
+        http_request=None,
+    )
+
+    # create key with for Org Admin
+    _new_key = await generate_key_fn(
+        data=GenerateKeyRequest(user_id=created_user_id),
+        user_api_key_dict=UserAPIKeyAuth(user_id=created_user_id),
+    )
+
+    new_key = _new_key.key
+
+    print("user api key auth response", response)
+
+    # Create /organization/member_add request -> expect auth to pass
+    request = Request(scope={"type": "http"})
+    request._url = URL(url="/organization/member_add")
+
+    async def return_body():
+        body = {"organization_id": org_id}
+        return bytes(json.dumps(body), "utf-8")
+
+    request.body = return_body
+    response = await user_api_key_auth(request=request, api_key="Bearer " + new_key)
+
+    # after auth - actually actually add new user to organization
+    new_internal_user_for_org = f"new-org-user-{uuid.uuid4()}"
+    response = await organization_member_add(
+        data=OrganizationMemberAddRequest(
+            organization_id=org_id,
+            member=OrgMember(
+                role=LitellmUserRoles.INTERNAL_USER, user_id=new_internal_user_for_org
+            ),
+        ),
+        http_request=request,
+    )
+
+    print("response from new team")
+
+
+@pytest.mark.asyncio
+async def test_org_admin_create_user_team_wrong_org_permissions(prisma_client):
+    """
+    Create a new org admin
+
+    org admin creates a new user and new team in orgs they are not part of -> expect error
+    """
+    import json
+
+    master_key = "sk-1234"
+    setattr(litellm.proxy.proxy_server, "prisma_client", prisma_client)
+    setattr(litellm.proxy.proxy_server, "master_key", master_key)
+    setattr(litellm.proxy.proxy_server, "llm_router", MagicMock())
+
+    await litellm.proxy.proxy_server.prisma_client.connect()
+    created_user_id = f"new-user-{uuid.uuid4()}"
+    response = await new_organization(
+        data=NewOrganizationRequest(
+            organization_alias=f"new-org-{uuid.uuid4()}",
+        ),
+        user_api_key_dict=UserAPIKeyAuth(
+            user_role=LitellmUserRoles.PROXY_ADMIN,
+        ),
+    )
+
+    response2 = await new_organization(
+        data=NewOrganizationRequest(
+            organization_alias=f"new-org-{uuid.uuid4()}",
+        ),
+        user_api_key_dict=UserAPIKeyAuth(
+            user_role=LitellmUserRoles.PROXY_ADMIN,
+        ),
+    )
+
+    org1_id = response.organization_id  # has an admin
+
+    org2_id = response2.organization_id  # does not have an org admin
+
+    # Create Org Admin for Org1
+    created_user_id = f"new-user-{uuid.uuid4()}"
+    response = await organization_member_add(
+        data=OrganizationMemberAddRequest(
+            organization_id=org1_id,
+            member=OrgMember(role=LitellmUserRoles.ORG_ADMIN, user_id=created_user_id),
+        ),
+        http_request=None,
+    )
+
+    _new_key = await generate_key_fn(
+        data=GenerateKeyRequest(
+            user_id=created_user_id,
+        ),
+        user_api_key_dict=UserAPIKeyAuth(
+            user_role=LitellmUserRoles.ORG_ADMIN,
+            user_id=created_user_id,
+        ),
+    )
+
+    new_key = _new_key.key
+
+    print("user api key auth response", response)
+
+    # Add a new request in organization=org_without_admins -> expect fail (organization/member_add)
+    request = Request(scope={"type": "http"})
+    request._url = URL(url="/organization/member_add")
+
+    async def return_body():
+        body = {"organization_id": org2_id}
+        return bytes(json.dumps(body), "utf-8")
+
+    request.body = return_body
+
+    try:
+        response = await user_api_key_auth(request=request, api_key="Bearer " + new_key)
+        pytest.fail(
+            f"This should have failed!. creating a user in an org without admins"
+        )
+    except Exception as e:
+        print("got exception", e)
+        print("exception.message", e.message)
+        assert (
+            "You do not have a role within the selected organization. Passed organization_id"
+            in e.message
+        )
+
+    # Create /team/new request in organization=org_without_admins -> expect fail
+    request = Request(scope={"type": "http"})
+    request._url = URL(url="/team/new")
+
+    async def return_body():
+        body = {"organization_id": org2_id}
+        return bytes(json.dumps(body), "utf-8")
+
+    request.body = return_body
+
+    try:
+        response = await user_api_key_auth(request=request, api_key="Bearer " + new_key)
+        pytest.fail(
+            f"This should have failed!. Org Admin creating a team in an org where they are not an admin"
+        )
+    except Exception as e:
+        print("got exception", e)
+        print("exception.message", e.message)
+        assert (
+            "You do not have the required role to call" in e.message
+            and org2_id in e.message
+        )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "route, user_role, expected_result",
+    [
+        # Proxy Admin checks
+        ("/global/spend/logs", LitellmUserRoles.PROXY_ADMIN, True),
+        ("/key/delete", LitellmUserRoles.PROXY_ADMIN, True),
+        ("/key/generate", LitellmUserRoles.PROXY_ADMIN, True),
+        ("/key/regenerate", LitellmUserRoles.PROXY_ADMIN, True),
+        # # Internal User checks - allowed routes
+        ("/global/spend/logs", LitellmUserRoles.INTERNAL_USER, True),
+        ("/key/delete", LitellmUserRoles.INTERNAL_USER, True),
+        ("/key/generate", LitellmUserRoles.INTERNAL_USER, True),
+        ("/key/82akk800000000jjsk/regenerate", LitellmUserRoles.INTERNAL_USER, True),
+        # Internal User Viewer
+        ("/key/generate", LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, False),
+        (
+            "/key/82akk800000000jjsk/regenerate",
+            LitellmUserRoles.INTERNAL_USER_VIEW_ONLY,
+            False,
+        ),
+        ("/key/delete", LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, False),
+        ("/team/new", LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, False),
+        ("/team/delete", LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, False),
+        ("/team/update", LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, False),
+        # Proxy Admin Viewer
+        ("/global/spend/logs", LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, True),
+        ("/key/delete", LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, False),
+        ("/key/generate", LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, False),
+        (
+            "/key/82akk800000000jjsk/regenerate",
+            LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
+            False,
+        ),
+        ("/team/new", LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, False),
+        ("/team/delete", LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, False),
+        ("/team/update", LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, False),
+        # Internal User checks - disallowed routes
+        ("/organization/member_add", LitellmUserRoles.INTERNAL_USER, False),
+    ],
+)
+async def test_user_role_permissions(prisma_client, route, user_role, expected_result):
+    """Test user role based permissions for different routes"""
+    try:
+        # Setup
+        setattr(litellm.proxy.proxy_server, "prisma_client", prisma_client)
+        setattr(litellm.proxy.proxy_server, "master_key", "sk-1234")
+        await litellm.proxy.proxy_server.prisma_client.connect()
+
+        # Admin - admin creates a new user
+        user_api_key_dict = UserAPIKeyAuth(
+            user_role=LitellmUserRoles.PROXY_ADMIN,
+            api_key="sk-1234",
+            user_id="1234",
+        )
+
+        request = NewUserRequest(user_role=user_role)
+        new_user_response = await new_user(request, user_api_key_dict=user_api_key_dict)
+        user_id = new_user_response.user_id
+
+        # Generate key for new user with team_id="litellm-dashboard"
+        key_response = await generate_key_fn(
+            data=GenerateKeyRequest(user_id=user_id, team_id="litellm-dashboard"),
+            user_api_key_dict=user_api_key_dict,
+        )
+        generated_key = key_response.key
+        bearer_token = "Bearer " + generated_key
+
+        # Create request with route
+        request = Request(scope={"type": "http"})
+        request._url = URL(url=route)
+
+        # Test authorization
+        if expected_result is True:
+            # Should pass without error
+            result = await user_api_key_auth(request=request, api_key=bearer_token)
+            print(f"Auth passed as expected for {route} with role {user_role}")
+        else:
+            # Should raise an error
+            with pytest.raises(Exception) as exc_info:
+                await user_api_key_auth(request=request, api_key=bearer_token)
+            print(f"Auth failed as expected for {route} with role {user_role}")
+            print(f"Error message: {str(exc_info.value)}")
+
+    except Exception as e:
+        if expected_result:
+            pytest.fail(f"Expected success but got exception: {str(e)}")
+        else:
+            print(f"Got expected exception: {str(e)}")

Development/litellm/tests/proxy_admin_ui_tests/test_route_check_unit_tests.py

@@ -0,0 +1,226 @@
+import os
+import sys
+import traceback
+import uuid
+import datetime as dt
+from datetime import datetime
+
+from dotenv import load_dotenv
+from fastapi import Request
+from fastapi.routing import APIRoute
+
+load_dotenv()
+import io
+import os
+import time
+
+
+# this file is to test litellm/proxy
+
+sys.path.insert(
+    0, os.path.abspath("../..")
+)  # Adds the parent directory to the system path
+import asyncio
+import logging
+
+from fastapi import HTTPException, Request
+import pytest
+from litellm.proxy.auth.route_checks import RouteChecks
+from litellm.proxy._types import LiteLLM_UserTable, LitellmUserRoles, UserAPIKeyAuth
+from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
+    router as llm_passthrough_router,
+)
+
+# Replace the actual hash_token function with our mock
+import litellm.proxy.auth.route_checks
+
+
+# Mock objects and functions
+class MockRequest:
+    def __init__(self, query_params=None):
+        self.query_params = query_params or {}
+
+
+def mock_hash_token(token):
+    return token
+
+
+litellm.proxy.auth.route_checks.hash_token = mock_hash_token
+
+
+# Test is_llm_api_route
+def test_is_llm_api_route():
+    assert RouteChecks.is_llm_api_route("/v1/chat/completions") is True
+    assert RouteChecks.is_llm_api_route("/v1/completions") is True
+    assert RouteChecks.is_llm_api_route("/v1/embeddings") is True
+    assert RouteChecks.is_llm_api_route("/v1/images/generations") is True
+    assert RouteChecks.is_llm_api_route("/v1/threads/thread_12345") is True
+    assert RouteChecks.is_llm_api_route("/bedrock/model/invoke") is True
+    assert RouteChecks.is_llm_api_route("/vertex-ai/text") is True
+    assert RouteChecks.is_llm_api_route("/gemini/generate") is True
+    assert RouteChecks.is_llm_api_route("/cohere/generate") is True
+    assert RouteChecks.is_llm_api_route("/anthropic/messages") is True
+    assert RouteChecks.is_llm_api_route("/anthropic/v1/messages") is True
+    assert RouteChecks.is_llm_api_route("/azure/endpoint") is True
+    assert (
+        RouteChecks.is_llm_api_route("/v1/realtime?model=gpt-4o-realtime-preview")
+        is True
+    )
+    assert (
+        RouteChecks.is_llm_api_route("/realtime?model=gpt-4o-realtime-preview") is True
+    )
+    assert (
+        RouteChecks.is_llm_api_route(
+            "/openai/deployments/vertex_ai/gemini-1.5-flash/chat/completions"
+        )
+        is True
+    )
+    assert (
+        RouteChecks.is_llm_api_route(
+            "/openai/deployments/gemini/gemini-1.5-flash/chat/completions"
+        )
+        is True
+    )
+    assert (
+        RouteChecks.is_llm_api_route(
+            "/openai/deployments/anthropic/claude-3-5-sonnet-20240620/chat/completions"
+        )
+        is True
+    )
+
+    # MCP routes
+    assert RouteChecks.is_llm_api_route("/mcp") is True
+    assert RouteChecks.is_llm_api_route("/mcp/") is True
+    assert RouteChecks.is_llm_api_route("/mcp/tools") is True
+    assert RouteChecks.is_llm_api_route("/mcp/tools/call") is True
+    assert RouteChecks.is_llm_api_route("/mcp/tools/list") is True
+
+    
+    # check non-matching routes
+    assert RouteChecks.is_llm_api_route("/some/random/route") is False
+    assert RouteChecks.is_llm_api_route("/key/regenerate/82akk800000000jjsk") is False
+    assert RouteChecks.is_llm_api_route("/key/82akk800000000jjsk/delete") is False
+
+    all_llm_api_routes = llm_passthrough_router.routes
+
+    # check all routes in llm_passthrough_router, ensure they are considered llm api routes
+    for route in all_llm_api_routes:
+        print("route", route)
+        route_path = str(route.path)
+        print("route_path", route_path)
+        assert RouteChecks.is_llm_api_route(route_path) is True
+
+
+# Test _route_matches_pattern
+def test_route_matches_pattern():
+    # check matching routes
+    assert (
+        RouteChecks._route_matches_pattern(
+            "/threads/thread_12345", "/threads/{thread_id}"
+        )
+        is True
+    )
+    assert (
+        RouteChecks._route_matches_pattern(
+            "/key/regenerate/82akk800000000jjsk", "/key/{token_id}/regenerate"
+        )
+        is False
+    )
+    assert (
+        RouteChecks._route_matches_pattern(
+            "/v1/chat/completions", "/v1/chat/completions"
+        )
+        is True
+    )
+    assert (
+        RouteChecks._route_matches_pattern(
+            "/v1/models/gpt-4", "/v1/models/{model_name}"
+        )
+        is True
+    )
+
+    # check non-matching routes
+    assert (
+        RouteChecks._route_matches_pattern(
+            "/v1/chat/completionz/thread_12345", "/v1/chat/completions/{thread_id}"
+        )
+        is False
+    )
+    assert (
+        RouteChecks._route_matches_pattern(
+            "/v1/{thread_id}/messages", "/v1/messages/thread_2345"
+        )
+        is False
+    )
+
+
+@pytest.fixture
+def route_checks():
+    return RouteChecks()
+
+
+def test_llm_api_route(route_checks):
+    """
+    Internal User is allowed to access all LLM API routes
+    """
+    assert (
+        route_checks.non_proxy_admin_allowed_routes_check(
+            user_obj=None,
+            _user_role=LitellmUserRoles.INTERNAL_USER.value,
+            route="/v1/chat/completions",
+            request=MockRequest(),
+            valid_token=UserAPIKeyAuth(api_key="test_key"),
+            request_data={},
+        )
+        is None
+    )
+
+
+def test_key_info_route_allowed(route_checks):
+    """
+    Internal User is allowed to access /key/info route
+    """
+    assert (
+        route_checks.non_proxy_admin_allowed_routes_check(
+            user_obj=None,
+            _user_role=LitellmUserRoles.INTERNAL_USER.value,
+            route="/key/info",
+            request=MockRequest(query_params={"key": "test_key"}),
+            valid_token=UserAPIKeyAuth(api_key="test_key"),
+            request_data={},
+        )
+        is None
+    )
+
+
+def test_user_info_route_allowed(route_checks):
+    """
+    Internal User is allowed to access /user/info route for their own user_id
+    """
+    assert (
+        route_checks.non_proxy_admin_allowed_routes_check(
+            user_obj=None,
+            _user_role=LitellmUserRoles.INTERNAL_USER.value,
+            route="/user/info",
+            request=MockRequest(query_params={"user_id": "test_user"}),
+            valid_token=UserAPIKeyAuth(api_key="test_key", user_id="test_user"),
+            request_data={},
+        )
+        is None
+    )
+
+
+def test_user_info_route_forbidden(route_checks):
+    """
+    Internal User is not allowed to access /user/info route for a different user_id
+    """
+    with pytest.raises(HTTPException) as exc_info:
+        route_checks.non_proxy_admin_allowed_routes_check(
+            user_obj=None,
+            _user_role=LitellmUserRoles.INTERNAL_USER.value,
+            route="/user/info",
+            request=MockRequest(query_params={"user_id": "wrong_user"}),
+            valid_token=UserAPIKeyAuth(api_key="test_key", user_id="test_user"),
+            request_data={},
+        )
+    assert exc_info.value.status_code == 403

Development/litellm/tests/proxy_admin_ui_tests/test_sso_sign_in.py

@@ -0,0 +1,196 @@
+import pytest
+from fastapi.testclient import TestClient
+from fastapi import Request, Header
+from unittest.mock import patch, MagicMock, AsyncMock
+
+import sys
+import os
+
+sys.path.insert(
+    0, os.path.abspath("../..")
+)  # Adds the parent directory to the system path
+import litellm
+from litellm.proxy.proxy_server import app
+from litellm.proxy.utils import PrismaClient, ProxyLogging
+from litellm.proxy.management_endpoints.ui_sso import auth_callback
+from litellm.proxy._types import LitellmUserRoles
+import os
+import jwt
+import time
+from litellm.caching.caching import DualCache
+
+proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache())
+
+
+@pytest.fixture
+def mock_env_vars(monkeypatch):
+    monkeypatch.setenv("GOOGLE_CLIENT_ID", "mock_google_client_id")
+    monkeypatch.setenv("GOOGLE_CLIENT_SECRET", "mock_google_client_secret")
+    monkeypatch.setenv("PROXY_BASE_URL", "http://testserver")
+    monkeypatch.setenv("LITELLM_MASTER_KEY", "mock_master_key")
+
+
+@pytest.fixture
+def prisma_client():
+    from litellm.proxy.proxy_cli import append_query_params
+
+    ### add connection pool + pool timeout args
+    params = {"connection_limit": 100, "pool_timeout": 60}
+    database_url = os.getenv("DATABASE_URL")
+    modified_url = append_query_params(database_url, params)
+    os.environ["DATABASE_URL"] = modified_url
+
+    # Assuming PrismaClient is a class that needs to be instantiated
+    prisma_client = PrismaClient(
+        database_url=os.environ["DATABASE_URL"], proxy_logging_obj=proxy_logging_obj
+    )
+
+    # Reset litellm.proxy.proxy_server.prisma_client to None
+    litellm.proxy.proxy_server.litellm_proxy_budget_name = (
+        f"litellm-proxy-budget-{time.time()}"
+    )
+    litellm.proxy.proxy_server.user_custom_key_generate = None
+
+    return prisma_client
+
+
+@patch("fastapi_sso.sso.google.GoogleSSO")
+@pytest.mark.asyncio
+async def test_auth_callback_new_user(mock_google_sso, mock_env_vars, prisma_client):
+    """
+    Tests that a new SSO Sign In user is by default given an 'INTERNAL_USER_VIEW_ONLY' role
+    """
+    import uuid
+    import litellm
+
+    litellm._turn_on_debug()
+
+    # Generate a unique user ID
+    unique_user_id = str(uuid.uuid4())
+    unique_user_email = f"newuser{unique_user_id}@example.com"
+
+    try:
+        # Set up the prisma client
+        setattr(litellm.proxy.proxy_server, "prisma_client", prisma_client)
+        await litellm.proxy.proxy_server.prisma_client.connect()
+
+        # Set up the master key
+        litellm.proxy.proxy_server.master_key = "mock_master_key"
+
+        # Mock the GoogleSSO verify_and_process method
+        mock_sso_result = MagicMock()
+        mock_sso_result.email = unique_user_email
+        mock_sso_result.id = unique_user_id
+        mock_sso_result.provider = "google"
+        mock_google_sso.return_value.verify_and_process = AsyncMock(
+            return_value=mock_sso_result
+        )
+
+        # Create a mock Request object
+        mock_request = Request(
+            scope={
+                "type": "http",
+                "method": "GET",
+                "scheme": "http",
+                "server": ("testserver", 80),
+                "path": "/sso/callback",
+                "query_string": b"",
+                "headers": {},
+            }
+        )
+
+        # Call the auth_callback function directly
+        response = await auth_callback(request=mock_request)
+
+        # Assert the response
+        assert response.status_code == 303
+        assert response.headers["location"].startswith(f"http://testserver/ui/?login=success")
+
+        # Verify that the user was added to the database
+        user = await prisma_client.db.litellm_usertable.find_first(
+            where={"user_id": unique_user_id}
+        )
+        print("inserted user from SSO", user)
+        assert user is not None
+        assert user.user_email == unique_user_email
+        assert user.user_role == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY
+        assert user.metadata == {"auth_provider": "google"}
+
+    finally:
+        # Clean up: Delete the user from the database
+        await prisma_client.db.litellm_usertable.delete(
+            where={"user_id": unique_user_id}
+        )
+
+
+@patch("fastapi_sso.sso.google.GoogleSSO")
+@pytest.mark.asyncio
+async def test_auth_callback_new_user_with_sso_default(
+    mock_google_sso, mock_env_vars, prisma_client
+):
+    """
+    When litellm_settings.default_internal_user_params.user_role = 'INTERNAL_USER'
+
+    Tests that a new SSO Sign In user is by default given an 'INTERNAL_USER' role
+    """
+    import uuid
+
+    # Generate a unique user ID
+    unique_user_id = str(uuid.uuid4())
+    unique_user_email = f"newuser{unique_user_id}@example.com"
+
+    try:
+        # Set up the prisma client
+        setattr(litellm.proxy.proxy_server, "prisma_client", prisma_client)
+        litellm.default_internal_user_params = {
+            "user_role": LitellmUserRoles.INTERNAL_USER.value
+        }
+        await litellm.proxy.proxy_server.prisma_client.connect()
+
+        # Set up the master key
+        litellm.proxy.proxy_server.master_key = "mock_master_key"
+
+        # Mock the GoogleSSO verify_and_process method
+        mock_sso_result = MagicMock()
+        mock_sso_result.email = unique_user_email
+        mock_sso_result.id = unique_user_id
+        mock_sso_result.provider = "google"
+        mock_google_sso.return_value.verify_and_process = AsyncMock(
+            return_value=mock_sso_result
+        )
+
+        # Create a mock Request object
+        mock_request = Request(
+            scope={
+                "type": "http",
+                "method": "GET",
+                "scheme": "http",
+                "server": ("testserver", 80),
+                "path": "/sso/callback",
+                "query_string": b"",
+                "headers": {},
+            }
+        )
+
+        # Call the auth_callback function directly
+        response = await auth_callback(request=mock_request)
+
+        # Assert the response
+        assert response.status_code == 303
+        assert response.headers["location"].startswith(f"http://testserver/ui/?login=success")
+
+        # Verify that the user was added to the database
+        user = await prisma_client.db.litellm_usertable.find_first(
+            where={"user_id": unique_user_id}
+        )
+        print("inserted user from SSO", user)
+        assert user is not None
+        assert user.user_email == unique_user_email
+        assert user.user_role == LitellmUserRoles.INTERNAL_USER
+
+    finally:
+        # Clean up: Delete the user from the database
+        await prisma_client.db.litellm_usertable.delete(
+            where={"user_id": unique_user_id}
+        )
+        litellm.default_internal_user_params = None

Development/litellm/tests/proxy_admin_ui_tests/test_usage_endpoints.py

@@ -0,0 +1,321 @@
+"""
+Tests the following endpoints used by the UI 
+
+/global/spend/logs
+/global/spend/keys
+/global/spend/models
+/global/activity
+/global/activity/model
+
+
+For all tests - test the following:
+- Response is valid 
+- Response for Admin User is different from response from Internal User
+"""
+
+import os
+import sys
+import traceback
+import uuid
+from datetime import datetime
+
+from dotenv import load_dotenv
+from fastapi import Request
+from fastapi.routing import APIRoute
+
+load_dotenv()
+import io
+import os
+import time
+
+# this file is to test litellm/proxy
+
+sys.path.insert(
+    0, os.path.abspath("../..")
+)  # Adds the parent directory to the system path
+import asyncio
+import logging
+
+import pytest
+
+import litellm
+from litellm._logging import verbose_proxy_logger
+from litellm.proxy.management_endpoints.internal_user_endpoints import (
+    new_user,
+    user_info,
+    user_update,
+)
+from litellm.proxy.management_endpoints.key_management_endpoints import (
+    delete_key_fn,
+    generate_key_fn,
+    generate_key_helper_fn,
+    info_key_fn,
+    regenerate_key_fn,
+    update_key_fn,
+)
+from litellm.proxy.management_endpoints.team_endpoints import (
+    new_team,
+    team_info,
+    update_team,
+)
+from litellm.proxy.proxy_server import (
+    LitellmUserRoles,
+    audio_transcriptions,
+    chat_completion,
+    completion,
+    embeddings,
+    model_list,
+    moderations,
+    user_api_key_auth,
+)
+from litellm.proxy.management_endpoints.customer_endpoints import (
+    new_end_user,
+)
+from litellm.proxy.spend_tracking.spend_management_endpoints import (
+    global_spend,
+    global_spend_logs,
+    global_spend_models,
+    global_spend_keys,
+    spend_key_fn,
+    spend_user_fn,
+    view_spend_logs,
+)
+from litellm.proxy.utils import PrismaClient, ProxyLogging, hash_token, update_spend
+
+verbose_proxy_logger.setLevel(level=logging.DEBUG)
+
+from starlette.datastructures import URL
+
+from litellm.caching.caching import DualCache
+from litellm.types.proxy.management_endpoints.ui_sso import LiteLLM_UpperboundKeyGenerateParams
+from litellm.proxy._types import (
+    DynamoDBArgs,
+    GenerateKeyRequest,
+    RegenerateKeyRequest,
+    KeyRequest,
+    NewCustomerRequest,
+    NewTeamRequest,
+    NewUserRequest,
+    ProxyErrorTypes,
+    ProxyException,
+    UpdateKeyRequest,
+    UpdateTeamRequest,
+    UpdateUserRequest,
+    UserAPIKeyAuth,
+)
+
+proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache())
+
+
+@pytest.fixture
+def prisma_client():
+    from litellm.proxy.proxy_cli import append_query_params
+
+    ### add connection pool + pool timeout args
+    params = {"connection_limit": 100, "pool_timeout": 60}
+    database_url = os.getenv("DATABASE_URL")
+    modified_url = append_query_params(database_url, params)
+    os.environ["DATABASE_URL"] = modified_url
+
+    # Assuming PrismaClient is a class that needs to be instantiated
+    prisma_client = PrismaClient(
+        database_url=os.environ["DATABASE_URL"], proxy_logging_obj=proxy_logging_obj
+    )
+
+    # Reset litellm.proxy.proxy_server.prisma_client to None
+    litellm.proxy.proxy_server.litellm_proxy_budget_name = (
+        f"litellm-proxy-budget-{time.time()}"
+    )
+    litellm.proxy.proxy_server.user_custom_key_generate = None
+
+    return prisma_client
+
+
+@pytest.mark.asyncio()
+async def test_view_daily_spend_ui(prisma_client):
+    print("prisma client=", prisma_client)
+    setattr(litellm.proxy.proxy_server, "prisma_client", prisma_client)
+    setattr(litellm.proxy.proxy_server, "master_key", "sk-1234")
+
+    await litellm.proxy.proxy_server.prisma_client.connect()
+    from litellm.proxy.proxy_server import user_api_key_cache
+
+    spend_logs_for_admin = await global_spend_logs(
+        user_api_key_dict=UserAPIKeyAuth(
+            api_key="sk-1234",
+            user_role=LitellmUserRoles.PROXY_ADMIN,
+        ),
+        api_key=None,
+    )
+
+    print("spend_logs_for_admin=", spend_logs_for_admin)
+
+    spend_logs_for_internal_user = await global_spend_logs(
+        user_api_key_dict=UserAPIKeyAuth(
+            api_key="sk-1234", user_role=LitellmUserRoles.INTERNAL_USER, user_id="1234"
+        ),
+        api_key=None,
+    )
+
+    print("spend_logs_for_internal_user=", spend_logs_for_internal_user)
+
+    # Calculate total spend for admin
+    admin_total_spend = sum(log.get("spend", 0) for log in spend_logs_for_admin)
+
+    # Calculate total spend for internal user (0 in this case, but we'll keep it generic)
+    internal_user_total_spend = sum(
+        log.get("spend", 0) for log in spend_logs_for_internal_user
+    )
+
+    print("total_spend_for_admin=", admin_total_spend)
+    print("total_spend_for_internal_user=", internal_user_total_spend)
+
+    assert (
+        admin_total_spend > internal_user_total_spend
+    ), "Admin should have more spend than internal user"
+
+
+@pytest.mark.asyncio
+async def test_global_spend_models(prisma_client):
+    print("prisma client=", prisma_client)
+    setattr(litellm.proxy.proxy_server, "prisma_client", prisma_client)
+    setattr(litellm.proxy.proxy_server, "master_key", "sk-1234")
+
+    await litellm.proxy.proxy_server.prisma_client.connect()
+
+    # Test for admin user
+    models_spend_for_admin = await global_spend_models(
+        limit=10,
+        user_api_key_dict=UserAPIKeyAuth(
+            api_key="sk-1234",
+            user_role=LitellmUserRoles.PROXY_ADMIN,
+        ),
+    )
+
+    print("models_spend_for_admin=", models_spend_for_admin)
+
+    # Test for internal user
+    models_spend_for_internal_user = await global_spend_models(
+        limit=10,
+        user_api_key_dict=UserAPIKeyAuth(
+            api_key="sk-1234", user_role=LitellmUserRoles.INTERNAL_USER, user_id="1234"
+        ),
+    )
+
+    print("models_spend_for_internal_user=", models_spend_for_internal_user)
+
+    # Assertions
+    assert isinstance(models_spend_for_admin, list), "Admin response should be a list"
+    assert isinstance(
+        models_spend_for_internal_user, list
+    ), "Internal user response should be a list"
+
+    # Check if the response has the expected shape for both admin and internal user
+    expected_keys = ["model", "total_spend"]
+
+    if len(models_spend_for_admin) > 0:
+        assert all(
+            key in models_spend_for_admin[0] for key in expected_keys
+        ), f"Admin response should contain keys: {expected_keys}"
+        assert isinstance(
+            models_spend_for_admin[0]["model"], str
+        ), "Model should be a string"
+        assert isinstance(
+            models_spend_for_admin[0]["total_spend"], (int, float)
+        ), "Total spend should be a number"
+
+    if len(models_spend_for_internal_user) > 0:
+        assert all(
+            key in models_spend_for_internal_user[0] for key in expected_keys
+        ), f"Internal user response should contain keys: {expected_keys}"
+        assert isinstance(
+            models_spend_for_internal_user[0]["model"], str
+        ), "Model should be a string"
+        assert isinstance(
+            models_spend_for_internal_user[0]["total_spend"], (int, float)
+        ), "Total spend should be a number"
+
+    # Check if the lists are sorted by total_spend in descending order
+    if len(models_spend_for_admin) > 1:
+        assert all(
+            models_spend_for_admin[i]["total_spend"]
+            >= models_spend_for_admin[i + 1]["total_spend"]
+            for i in range(len(models_spend_for_admin) - 1)
+        ), "Admin response should be sorted by total_spend in descending order"
+
+    if len(models_spend_for_internal_user) > 1:
+        assert all(
+            models_spend_for_internal_user[i]["total_spend"]
+            >= models_spend_for_internal_user[i + 1]["total_spend"]
+            for i in range(len(models_spend_for_internal_user) - 1)
+        ), "Internal user response should be sorted by total_spend in descending order"
+
+    # Check if admin has access to more or equal models compared to internal user
+    assert len(models_spend_for_admin) >= len(
+        models_spend_for_internal_user
+    ), "Admin should have access to at least as many models as internal user"
+
+    # Check if the response contains expected fields
+    if len(models_spend_for_admin) > 0:
+        assert all(
+            key in models_spend_for_admin[0] for key in ["model", "total_spend"]
+        ), "Admin response should contain model, total_spend, and total_tokens"
+
+    if len(models_spend_for_internal_user) > 0:
+        assert all(
+            key in models_spend_for_internal_user[0] for key in ["model", "total_spend"]
+        ), "Internal user response should contain model, total_spend, and total_tokens"
+
+
+@pytest.mark.asyncio
+async def test_global_spend_keys(prisma_client):
+    print("prisma client=", prisma_client)
+    setattr(litellm.proxy.proxy_server, "prisma_client", prisma_client)
+    setattr(litellm.proxy.proxy_server, "master_key", "sk-1234")
+
+    await litellm.proxy.proxy_server.prisma_client.connect()
+
+    # Test for admin user
+    keys_spend_for_admin = await global_spend_keys(
+        limit=10,
+        user_api_key_dict=UserAPIKeyAuth(
+            api_key="sk-1234",
+            user_role=LitellmUserRoles.PROXY_ADMIN,
+        ),
+    )
+
+    print("keys_spend_for_admin=", keys_spend_for_admin)
+
+    # Test for internal user
+    keys_spend_for_internal_user = await global_spend_keys(
+        limit=10,
+        user_api_key_dict=UserAPIKeyAuth(
+            api_key="sk-1234", user_role=LitellmUserRoles.INTERNAL_USER, user_id="1234"
+        ),
+    )
+
+    print("keys_spend_for_internal_user=", keys_spend_for_internal_user)
+
+    # Assertions
+    assert isinstance(keys_spend_for_admin, list), "Admin response should be a list"
+    assert isinstance(
+        keys_spend_for_internal_user, list
+    ), "Internal user response should be a list"
+
+    # Check if admin has access to more or equal keys compared to internal user
+    assert len(keys_spend_for_admin) >= len(
+        keys_spend_for_internal_user
+    ), "Admin should have access to at least as many keys as internal user"
+
+    # Check if the response contains expected fields
+    if len(keys_spend_for_admin) > 0:
+        assert all(
+            key in keys_spend_for_admin[0]
+            for key in ["api_key", "total_spend", "key_alias", "key_name"]
+        ), "Admin response should contain api_key, total_spend, key_alias, and key_name"
+
+    if len(keys_spend_for_internal_user) > 0:
+        assert all(
+            key in keys_spend_for_internal_user[0]
+            for key in ["api_key", "total_spend", "key_alias", "key_name"]
+        ), "Internal user response should contain api_key, total_spend, key_alias, and key_name"

Development/litellm/tests/proxy_admin_ui_tests/ui_unit_tests/fileMock.js

@@ -0,0 +1 @@
+module.exports = 'test-file-stub';

Development/litellm/tests/proxy_admin_ui_tests/ui_unit_tests/handle_add_model_submit_test.tsx

@@ -0,0 +1,83 @@
+import { handleAddModelSubmit } from '../../../ui/litellm-dashboard/src/components/add_model/handle_add_model_submit';
+import { modelCreateCall } from '../../../ui/litellm-dashboard/src/components/networking';
+
+// Mock the dependencies
+const mockModelCreateCall = jest.fn().mockResolvedValue({ data: 'success' });
+jest.mock('../../../ui/litellm-dashboard/src/components/networking', () => ({
+  modelCreateCall: async (accessToken: string, formValues: any) => mockModelCreateCall(formValues)
+}));
+
+// Also need to mock provider_map
+jest.mock('../../../ui/litellm-dashboard/src/components/provider_info_helpers', () => ({
+  provider_map: {
+    'openai': 'openai'
+  }
+}));
+
+jest.mock('antd', () => ({
+  message: {
+    error: jest.fn()
+  }
+}));
+
+describe('handleAddModelSubmit', () => {
+  const mockForm = {
+    resetFields: jest.fn()
+  };
+  const mockAccessToken = 'test-token';
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    mockModelCreateCall.mockClear();
+  });
+
+  it('should not modify model name when all-wildcard is not selected', async () => {
+    const formValues = {
+      model: 'gpt-4',
+      custom_llm_provider: 'openai',
+      model_name: 'my-gpt4-deployment'
+    };
+
+    await handleAddModelSubmit(formValues, mockAccessToken, mockForm);
+    
+    console.log('Expected call:', {
+      model_name: 'my-gpt4-deployment',
+      litellm_params: {
+        model: 'gpt-4',
+        custom_llm_provider: 'openai'
+      },
+      model_info: {}
+    });
+    console.log('Actual calls:', mockModelCreateCall.mock.calls);
+
+    expect(mockModelCreateCall).toHaveBeenCalledWith({
+      model_name: 'my-gpt4-deployment',
+      litellm_params: {
+        model: 'gpt-4',
+        custom_llm_provider: 'openai'
+      },
+      model_info: {}
+    });
+    expect(mockForm.resetFields).toHaveBeenCalled();
+  });
+
+  it('should handle all-wildcard model correctly', async () => {
+    const formValues = {
+      model: 'all-wildcard',
+      custom_llm_provider: 'openai',
+      model_name: 'my-deployment'
+    };
+
+    await handleAddModelSubmit(formValues, mockAccessToken, mockForm);
+
+    expect(mockModelCreateCall).toHaveBeenCalledWith({
+      model_name: 'openai/*',
+      litellm_params: {
+        model: 'openai/*',
+        custom_llm_provider: 'openai'
+      },
+      model_info: {}
+    });
+    expect(mockForm.resetFields).toHaveBeenCalled();
+  });
+});

Development/litellm/tests/proxy_admin_ui_tests/ui_unit_tests/jest.config.js

@@ -0,0 +1,18 @@
+module.exports = {
+    preset: 'ts-jest',
+    testEnvironment: 'jsdom',
+    moduleNameMapper: {
+      '\\.(css|less|scss|sass)$': 'identity-obj-proxy',
+      '\\.(jpg|jpeg|png|gif|webp|svg)$': '<rootDir>/__mocks__/fileMock.js'
+    },
+    setupFilesAfterEnv: ['<rootDir>/jest.setup.js'],
+    testMatch: [
+      '<rootDir>/**/*.test.tsx',
+      '<rootDir>/**/*_test.tsx'  // Added this to match your file naming
+    ],
+    moduleDirectories: ['node_modules'],
+    testPathIgnorePatterns: ['/node_modules/'],
+    transform: {
+      '^.+\\.(ts|tsx)$': 'ts-jest'
+    }
+  }

Development/litellm/tests/proxy_admin_ui_tests/ui_unit_tests/jest.setup.js

@@ -0,0 +1 @@
+// Add any global setup here

Development/litellm/tests/proxy_admin_ui_tests/ui_unit_tests/package.json

@@ -0,0 +1,26 @@
+{
+    "name": "ui-unit-tests",
+    "version": "1.0.0",
+    "scripts": {
+        "test": "jest",
+        "test:watch": "jest --watch"
+    },
+    "devDependencies": {
+        "@testing-library/react": "^14.0.0",
+        "@testing-library/jest-dom": "^6.0.0",
+        "@types/jest": "^29.5.0",
+        "@types/react": "^18.2.0",
+        "@types/react-dom": "^18.2.0",
+        "identity-obj-proxy": "^3.0.0",
+        "jest": "^29.5.0",
+        "jest-environment-jsdom": "^29.5.0",
+        "ts-jest": "^29.1.0",
+        "typescript": "^5.0.0"
+    },
+    "dependencies": {
+        "antd": "^5.12.5",
+        "@ant-design/icons": "^5.0.0",
+        "react": "^18.2.0",
+        "react-dom": "^18.2.0"
+    }
+}

Development/litellm/tests/proxy_admin_ui_tests/ui_unit_tests/tsconfig.json

@@ -0,0 +1,30 @@
+{
+  "compilerOptions": {
+      "target": "es5",
+      "lib": ["dom", "dom.iterable", "esnext"],
+      "allowJs": true,
+      "skipLibCheck": true,
+      "esModuleInterop": true,
+      "allowSyntheticDefaultImports": true,
+      "strict": true,
+      "forceConsistentCasingInFileNames": true,
+      "noFallthroughCasesInSwitch": true,
+      "module": "esnext",
+      "moduleResolution": "node",
+      "resolveJsonModule": true,
+      "isolatedModules": true,
+      "noEmit": true,
+      "jsx": "react-jsx",
+      "baseUrl": ".",
+      "paths": {
+          "*": ["*", "node_modules/*"]
+      }
+  },
+  "include": [
+      "**/*.ts",
+      "**/*.tsx"
+  ],
+  "exclude": [
+      "node_modules"
+  ]
+}

Development/litellm/tests/proxy_admin_ui_tests/utils/login.ts

@@ -0,0 +1,23 @@
+import { Page, expect } from "@playwright/test";
+
+export async function loginToUI(page: Page) {
+  // Login first
+  await page.goto("http://localhost:4000/ui");
+  console.log("Navigated to login page");
+
+  // Wait for login form to be visible
+  await page.waitForSelector('input[name="username"]', { timeout: 10000 });
+  console.log("Login form is visible");
+
+  await page.fill('input[name="username"]', "admin");
+  await page.fill('input[name="password"]', "gm");
+  console.log("Filled login credentials");
+
+  const loginButton = page.locator('input[type="submit"]');
+  await expect(loginButton).toBeEnabled();
+  await loginButton.click();
+  console.log("Clicked login button");
+
+  // Wait for navigation to complete
+  await page.waitForURL("**/*");
+}