diff --git a/apps/litellm/README.md b/apps/litellm/README.md index ba36fd3..903c0c1 100644 --- a/apps/litellm/README.md +++ b/apps/litellm/README.md @@ -1,14 +1,19 @@ # LiteLLM LiteLLM is the cluster-local LLM proxy for OpenWebUI. It routes requests to the -external llama.cpp OpenAI-compatible endpoint at `http://framework-llm:8080/v1`. +external llama.cpp OpenAI-compatible endpoint at +`http://framework-llm.open-webui.svc.cluster.local:8080/v1`. The public dashboard is exposed at `https://litellm.noxxos.nl` through LiteLLM's native Authentik SSO. API clients should use LiteLLM bearer keys. ## Secrets -Create the LiteLLM secret before syncing: +Create the LiteLLM secrets before syncing. The Helm chart references these +secrets directly, so the LiteLLM pod will fail with `secret "litellm-sso" not +found` until the SSO secret exists in the `litellm` namespace. + +Create the LiteLLM master key and salt: ```bash kubectl create namespace litellm @@ -58,7 +63,7 @@ if ak_is_group_member(request.user, name="admins"): return {"litellm_role": "internal_user"} ``` -Create the LiteLLM SSO secret: +Then create the LiteLLM SSO secret before syncing the app: ```bash kubectl -n litellm create secret generic litellm-sso \ diff --git a/apps/litellm/values.yaml b/apps/litellm/values.yaml index c18bb9e..d8a2137 100644 --- a/apps/litellm/values.yaml +++ b/apps/litellm/values.yaml @@ -51,27 +51,40 @@ litellm: proxy_config: model_list: - - model_name: qwen3-27b + - model_name: qwen27 litellm_params: - model: openai/qwen3.6-27B - api_base: http://framework-llm:8080/v1 + model: openai/qwen27 + api_base: http://framework-llm.open-webui.svc.cluster.local:8080/v1 api_key: not-needed model_info: mode: chat - - model_name: qwen3-122b-a10b + max_input_tokens: 131072 + - model_name: qwen122 litellm_params: - model: openai/qwen3.5-122B-A10B - api_base: http://framework-llm:8080/v1 + model: openai/qwen122 + api_base: http://framework-llm.open-webui.svc.cluster.local:8080/v1 api_key: not-needed model_info: mode: chat + max_input_tokens: 262144 + max_output_tokens: 16384 + - model_name: qwen122-abliterated + litellm_params: + model: openai/qwen122-abliterated + api_base: http://framework-llm.open-webui.svc.cluster.local:8080/v1 + api_key: not-needed + model_info: + mode: chat + max_input_tokens: 262144 + max_output_tokens: 16384 - model_name: qwen35-4b litellm_params: model: openai/qwen35-4b - api_base: http://framework-llm:8080/v1 + api_base: http://framework-llm.open-webui.svc.cluster.local:8080/v1 api_key: not-needed model_info: mode: chat + max_input_tokens: 32768 general_settings: master_key: os.environ/PROXY_MASTER_KEY litellm_settings: diff --git a/apps/open-webui/README.md b/apps/open-webui/README.md index 64cb2cc..7cebb91 100644 --- a/apps/open-webui/README.md +++ b/apps/open-webui/README.md @@ -4,7 +4,7 @@ OpenWebUI is exposed at `https://openwebui.noxxos.nl` and connects to LiteLLM at `http://litellm.litellm.svc.cluster.local:4000/v1`. LiteLLM then routes requests to the OpenAI-compatible llama.cpp endpoint at -`http://framework-llm:8080/v1`. +`http://framework-llm.open-webui.svc.cluster.local:8080/v1`. ## Authentik