diff --git a/apps/litellm/Chart.yaml b/apps/litellm/Chart.yaml new file mode 100644 index 0000000..aadef7f --- /dev/null +++ b/apps/litellm/Chart.yaml @@ -0,0 +1,15 @@ +apiVersion: v2 +name: litellm +description: LiteLLM proxy wrapper chart +type: application +version: 1.0.0 +appVersion: "1.84.0" + +dependencies: + - name: litellm-helm + alias: litellm + version: 1.84.0 + repository: oci://ghcr.io/berriai + - name: oauth2-proxy + version: 10.4.3 + repository: https://oauth2-proxy.github.io/manifests diff --git a/apps/litellm/README.md b/apps/litellm/README.md new file mode 100644 index 0000000..1c8ea61 --- /dev/null +++ b/apps/litellm/README.md @@ -0,0 +1,58 @@ +# LiteLLM + +LiteLLM is the cluster-local LLM proxy for OpenWebUI. It routes requests to the +external llama.cpp OpenAI-compatible endpoint at `http://framework-llm:8080/v1`. + +The public dashboard is exposed at `https://litellm.noxxos.nl` through +OAuth2 Proxy and Authentik. API clients should use LiteLLM bearer keys. + +## Secrets + +Create the LiteLLM secret before syncing: + +```bash +kubectl create namespace litellm +kubectl -n litellm create secret generic litellm-secrets \ + --from-literal=master-key='sk-' \ + --from-literal=salt-key='' +``` + +OpenWebUI needs a copy of the LiteLLM key in its own namespace: + +```bash +kubectl -n open-webui create secret generic litellm-secrets \ + --from-literal=openwebui-api-keys='0p3n-w3bu!;sk-' +``` + +`openwebui-api-keys` is semicolon-separated because OpenWebUI is configured with +two OpenAI-compatible endpoints: its internal Pipelines service first, then +LiteLLM. + +Create an Authentik OAuth2/OIDC provider and application for OAuth2 Proxy. + +External URL: + +```text +https://litellm.noxxos.nl +``` + +Allowed redirect URI: + +```text +https://litellm.noxxos.nl/oauth2/callback +``` + +Create the OAuth2 Proxy secret: + +```bash +kubectl -n litellm create secret generic litellm-oauth2-proxy \ + --from-literal=client-id='' \ + --from-literal=client-secret='' \ + --from-literal=cookie-secret='<32-byte-base64-cookie-secret>' +``` + +Generate a cookie secret with: + +```bash +openssl rand -base64 32 | head -c 32 | base64 +``` diff --git a/apps/litellm/application.yaml b/apps/litellm/application.yaml new file mode 100644 index 0000000..be3fc30 --- /dev/null +++ b/apps/litellm/application.yaml @@ -0,0 +1,41 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: litellm + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "2" + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + repoURL: https://git.mvzijl.nl/marco/veda.git + targetRevision: applicationset-rewrite + path: apps/litellm + helm: + releaseName: litellm + valueFiles: + - values.yaml + destination: + server: https://kubernetes.default.svc + namespace: litellm + ignoreDifferences: + - group: gateway.networking.k8s.io + kind: HTTPRoute + jqPathExpressions: + - .spec.parentRefs[] | .group, .kind + - .spec.rules[].backendRefs[] | .group, .kind, .weight + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - SkipDryRunOnMissingResource=true + managedNamespaceMetadata: + labels: + pod-security.kubernetes.io/enforce: baseline + pod-security.kubernetes.io/audit: baseline + pod-security.kubernetes.io/warn: baseline diff --git a/apps/litellm/values.yaml b/apps/litellm/values.yaml new file mode 100644 index 0000000..e212cca --- /dev/null +++ b/apps/litellm/values.yaml @@ -0,0 +1,208 @@ +litellm: + nameOverride: litellm + replicaCount: 1 + + image: + pullPolicy: IfNotPresent + + service: + type: ClusterIP + port: 4000 + + resources: + requests: + cpu: 100m + memory: 512Mi + limits: + memory: 2Gi + + masterkeySecretName: litellm-secrets + masterkeySecretKey: master-key + environmentConfigMaps: + - litellm-env + + db: + useExisting: true + deployStandalone: false + endpoint: litellm-pg-cluster-rw.litellm.svc.cluster.local + database: litellm + secret: + name: litellm-pg-cluster-app + usernameKey: username + passwordKey: password + endpointKey: host + + migrationJob: + enabled: true + hooks: + argocd: + enabled: true + helm: + enabled: false + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 512Mi + + proxy_config: + model_list: + - model_name: qwen3-27b + litellm_params: + model: openai/qwen3.6-27B + api_base: http://framework-llm:8080/v1 + api_key: not-needed + - model_name: qwen3-122b-a10b + litellm_params: + model: openai/qwen3.5-122B-A10B + api_base: http://framework-llm:8080/v1 + api_key: not-needed + - model_name: qwen35-4b + litellm_params: + model: openai/qwen35-4b + api_base: http://framework-llm:8080/v1 + api_key: not-needed + general_settings: + master_key: os.environ/PROXY_MASTER_KEY + litellm_settings: + drop_params: true + + envVars: + DD_ENV: production + DD_SERVICE: litellm + USE_DDTRACE: "false" + + extraEnvVars: + - name: LITELLM_SALT_KEY + valueFrom: + secretKeyRef: + name: litellm-secrets + key: salt-key + + extraResources: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: litellm-env + namespace: litellm + data: + LITELLM_ENV: production + - apiVersion: postgresql.cnpg.io/v1 + kind: Cluster + metadata: + name: litellm-pg-cluster + namespace: litellm + spec: + instances: 2 + bootstrap: + initdb: + database: litellm + owner: litellm + storage: + storageClass: local-path + size: 10Gi + resources: + requests: + cpu: 100m + memory: 100Mi + limits: + memory: 512Mi + plugins: + - enabled: true + name: barman-cloud.cloudnative-pg.io + isWALArchiver: true + parameters: + barmanObjectName: litellm-backup-store + - apiVersion: barmancloud.cnpg.io/v1 + kind: ObjectStore + metadata: + name: litellm-backup-store + namespace: litellm + spec: + retentionPolicy: "30d" + configuration: + destinationPath: s3://cnpg-backup-litellm/ + endpointURL: http://rook-ceph-rgw-ceph-objectstore.rook-ceph.svc:80 + s3Credentials: + accessKeyId: + name: litellm-pg-backups + key: AWS_ACCESS_KEY_ID + secretAccessKey: + name: litellm-pg-backups + key: AWS_SECRET_ACCESS_KEY + wal: + compression: bzip2 + data: + compression: bzip2 + immediateCheckpoint: true + - apiVersion: postgresql.cnpg.io/v1 + kind: ScheduledBackup + metadata: + name: litellm-pg-backup + namespace: litellm + spec: + method: plugin + immediate: true + schedule: "0 15 3 * * *" + backupOwnerReference: self + cluster: + name: litellm-pg-cluster + pluginConfiguration: + name: barman-cloud.cloudnative-pg.io + - apiVersion: objectbucket.io/v1alpha1 + kind: ObjectBucketClaim + metadata: + name: litellm-pg-backups + namespace: litellm + spec: + bucketName: cnpg-backup-litellm + storageClassName: ceph-bucket + additionalConfig: + maxSize: "50Gi" + +oauth2-proxy: + config: + existingSecret: litellm-oauth2-proxy + emailDomains: + - "*" + configFile: |- + provider = "oidc" + oidc_issuer_url = "https://sso.noxxos.nl/application/o/litellm/" + redirect_url = "https://litellm.noxxos.nl/oauth2/callback" + upstreams = [ "http://litellm:4000" ] + email_domains = [ "*" ] + reverse_proxy = true + pass_authorization_header = true + pass_access_token = true + set_xauthrequest = true + skip_provider_button = true + cookie_secure = true + cookie_samesite = "lax" + cookie_domains = [ ".noxxos.nl" ] + whitelist_domains = [ ".noxxos.nl" ] + + service: + type: ClusterIP + portNumber: 80 + + gatewayApi: + enabled: true + gatewayRef: + name: traefik-gateway + namespace: traefik + sectionName: websecure + hostnames: + - litellm.noxxos.nl + rules: + - matches: + - path: + type: PathPrefix + value: / + + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + memory: 256Mi diff --git a/apps/open-webui/README.md b/apps/open-webui/README.md index 0720337..64cb2cc 100644 --- a/apps/open-webui/README.md +++ b/apps/open-webui/README.md @@ -1,7 +1,10 @@ # OpenWebUI -OpenWebUI is exposed at `https://openwebui.noxxos.nl` and connects to the -OpenAI-compatible llama.cpp endpoint at `http://framework-llm:8080/v1`. +OpenWebUI is exposed at `https://openwebui.noxxos.nl` and connects to LiteLLM at +`http://litellm.litellm.svc.cluster.local:4000/v1`. + +LiteLLM then routes requests to the OpenAI-compatible llama.cpp endpoint at +`http://framework-llm:8080/v1`. ## Authentik @@ -22,3 +25,12 @@ kubectl -n open-webui create secret generic open-webui-oidc \ --from-literal=client-id='' \ --from-literal=client-secret='' ``` + +OpenWebUI also reads its OpenAI-compatible API keys from the `litellm-secrets` +Secret in the `open-webui` namespace. Keep it in sync with the `litellm` +namespace copy: + +```bash +kubectl -n open-webui create secret generic litellm-secrets \ + --from-literal=openwebui-api-keys='0p3n-w3bu!;sk-' +``` diff --git a/apps/open-webui/application.yaml b/apps/open-webui/application.yaml index 557b810..52408f7 100644 --- a/apps/open-webui/application.yaml +++ b/apps/open-webui/application.yaml @@ -4,7 +4,7 @@ metadata: name: open-webui namespace: argocd annotations: - argocd.argoproj.io/sync-wave: "2" + argocd.argoproj.io/sync-wave: "3" finalizers: - resources-finalizer.argocd.argoproj.io spec: diff --git a/apps/open-webui/values.yaml b/apps/open-webui/values.yaml index c20961a..a2986d6 100644 --- a/apps/open-webui/values.yaml +++ b/apps/open-webui/values.yaml @@ -64,10 +64,9 @@ open-webui: enableOpenaiApi: true openaiBaseApiUrls: - - http://framework-llm:8080/v1 - openaiApiKeys: - - 0p3n-w3bu! - - not-needed + - http://litellm.litellm.svc.cluster.local:4000/v1 + openaiApiKeysExistingSecret: litellm-secrets + openaiApiKeysExistingSecretKey: openwebui-api-keys extraEnvVars: - name: WEBUI_URL