Add LiteLLM proxy

apps/litellm/Chart.yaml

@@ -0,0 +1,15 @@
+apiVersion: v2
+name: litellm
+description: LiteLLM proxy wrapper chart
+type: application
+version: 1.0.0
+appVersion: "1.84.0"
+
+dependencies:
+  - name: litellm-helm
+    alias: litellm
+    version: 1.84.0
+    repository: oci://ghcr.io/berriai
+  - name: oauth2-proxy
+    version: 10.4.3
+    repository: https://oauth2-proxy.github.io/manifests

apps/litellm/README.md

@@ -0,0 +1,58 @@
+# LiteLLM
+
+LiteLLM is the cluster-local LLM proxy for OpenWebUI. It routes requests to the
+external llama.cpp OpenAI-compatible endpoint at `http://framework-llm:8080/v1`.
+
+The public dashboard is exposed at `https://litellm.noxxos.nl` through
+OAuth2 Proxy and Authentik. API clients should use LiteLLM bearer keys.
+
+## Secrets
+
+Create the LiteLLM secret before syncing:
+
+```bash
+kubectl create namespace litellm
+kubectl -n litellm create secret generic litellm-secrets \
+  --from-literal=master-key='sk-<litellm-master-key>' \
+  --from-literal=salt-key='<random-high-entropy-salt>'
+```
+
+OpenWebUI needs a copy of the LiteLLM key in its own namespace:
+
+```bash
+kubectl -n open-webui create secret generic litellm-secrets \
+  --from-literal=openwebui-api-keys='0p3n-w3bu!;sk-<litellm-master-key>'
+```
+
+`openwebui-api-keys` is semicolon-separated because OpenWebUI is configured with
+two OpenAI-compatible endpoints: its internal Pipelines service first, then
+LiteLLM.
+
+Create an Authentik OAuth2/OIDC provider and application for OAuth2 Proxy.
+
+External URL:
+
+```text
+https://litellm.noxxos.nl
+```
+
+Allowed redirect URI:
+
+```text
+https://litellm.noxxos.nl/oauth2/callback
+```
+
+Create the OAuth2 Proxy secret:
+
+```bash
+kubectl -n litellm create secret generic litellm-oauth2-proxy \
+  --from-literal=client-id='<authentik-client-id>' \
+  --from-literal=client-secret='<authentik-client-secret>' \
+  --from-literal=cookie-secret='<32-byte-base64-cookie-secret>'
+```
+
+Generate a cookie secret with:
+
+```bash
+openssl rand -base64 32 | head -c 32 | base64
+```

apps/litellm/application.yaml

@@ -0,0 +1,41 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: litellm
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  source:
+    repoURL: https://git.mvzijl.nl/marco/veda.git
+    targetRevision: applicationset-rewrite
+    path: apps/litellm
+    helm:
+      releaseName: litellm
+      valueFiles:
+        - values.yaml
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: litellm
+  ignoreDifferences:
+    - group: gateway.networking.k8s.io
+      kind: HTTPRoute
+      jqPathExpressions:
+        - .spec.parentRefs[] | .group, .kind
+        - .spec.rules[].backendRefs[] | .group, .kind, .weight
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true
+      - SkipDryRunOnMissingResource=true
+    managedNamespaceMetadata:
+      labels:
+        pod-security.kubernetes.io/enforce: baseline
+        pod-security.kubernetes.io/audit: baseline
+        pod-security.kubernetes.io/warn: baseline

apps/litellm/values.yaml

@@ -0,0 +1,208 @@
+litellm:
+  nameOverride: litellm
+  replicaCount: 1
+
+  image:
+    pullPolicy: IfNotPresent
+
+  service:
+    type: ClusterIP
+    port: 4000
+
+  resources:
+    requests:
+      cpu: 100m
+      memory: 512Mi
+    limits:
+      memory: 2Gi
+
+  masterkeySecretName: litellm-secrets
+  masterkeySecretKey: master-key
+  environmentConfigMaps:
+    - litellm-env
+
+  db:
+    useExisting: true
+    deployStandalone: false
+    endpoint: litellm-pg-cluster-rw.litellm.svc.cluster.local
+    database: litellm
+    secret:
+      name: litellm-pg-cluster-app
+      usernameKey: username
+      passwordKey: password
+      endpointKey: host
+
+  migrationJob:
+    enabled: true
+    hooks:
+      argocd:
+        enabled: true
+      helm:
+        enabled: false
+    resources:
+      requests:
+        cpu: 100m
+        memory: 256Mi
+      limits:
+        memory: 512Mi
+
+  proxy_config:
+    model_list:
+      - model_name: qwen3-27b
+        litellm_params:
+          model: openai/qwen3.6-27B
+          api_base: http://framework-llm:8080/v1
+          api_key: not-needed
+      - model_name: qwen3-122b-a10b
+        litellm_params:
+          model: openai/qwen3.5-122B-A10B
+          api_base: http://framework-llm:8080/v1
+          api_key: not-needed
+      - model_name: qwen35-4b
+        litellm_params:
+          model: openai/qwen35-4b
+          api_base: http://framework-llm:8080/v1
+          api_key: not-needed
+    general_settings:
+      master_key: os.environ/PROXY_MASTER_KEY
+    litellm_settings:
+      drop_params: true
+
+  envVars:
+    DD_ENV: production
+    DD_SERVICE: litellm
+    USE_DDTRACE: "false"
+
+  extraEnvVars:
+    - name: LITELLM_SALT_KEY
+      valueFrom:
+        secretKeyRef:
+          name: litellm-secrets
+          key: salt-key
+
+  extraResources:
+    - apiVersion: v1
+      kind: ConfigMap
+      metadata:
+        name: litellm-env
+        namespace: litellm
+      data:
+        LITELLM_ENV: production
+    - apiVersion: postgresql.cnpg.io/v1
+      kind: Cluster
+      metadata:
+        name: litellm-pg-cluster
+        namespace: litellm
+      spec:
+        instances: 2
+        bootstrap:
+          initdb:
+            database: litellm
+            owner: litellm
+        storage:
+          storageClass: local-path
+          size: 10Gi
+        resources:
+          requests:
+            cpu: 100m
+            memory: 100Mi
+          limits:
+            memory: 512Mi
+        plugins:
+          - enabled: true
+            name: barman-cloud.cloudnative-pg.io
+            isWALArchiver: true
+            parameters:
+              barmanObjectName: litellm-backup-store
+    - apiVersion: barmancloud.cnpg.io/v1
+      kind: ObjectStore
+      metadata:
+        name: litellm-backup-store
+        namespace: litellm
+      spec:
+        retentionPolicy: "30d"
+        configuration:
+          destinationPath: s3://cnpg-backup-litellm/
+          endpointURL: http://rook-ceph-rgw-ceph-objectstore.rook-ceph.svc:80
+          s3Credentials:
+            accessKeyId:
+              name: litellm-pg-backups
+              key: AWS_ACCESS_KEY_ID
+            secretAccessKey:
+              name: litellm-pg-backups
+              key: AWS_SECRET_ACCESS_KEY
+          wal:
+            compression: bzip2
+          data:
+            compression: bzip2
+            immediateCheckpoint: true
+    - apiVersion: postgresql.cnpg.io/v1
+      kind: ScheduledBackup
+      metadata:
+        name: litellm-pg-backup
+        namespace: litellm
+      spec:
+        method: plugin
+        immediate: true
+        schedule: "0 15 3 * * *"
+        backupOwnerReference: self
+        cluster:
+          name: litellm-pg-cluster
+        pluginConfiguration:
+          name: barman-cloud.cloudnative-pg.io
+    - apiVersion: objectbucket.io/v1alpha1
+      kind: ObjectBucketClaim
+      metadata:
+        name: litellm-pg-backups
+        namespace: litellm
+      spec:
+        bucketName: cnpg-backup-litellm
+        storageClassName: ceph-bucket
+        additionalConfig:
+          maxSize: "50Gi"
+
+oauth2-proxy:
+  config:
+    existingSecret: litellm-oauth2-proxy
+    emailDomains:
+      - "*"
+    configFile: |-
+      provider = "oidc"
+      oidc_issuer_url = "https://sso.noxxos.nl/application/o/litellm/"
+      redirect_url = "https://litellm.noxxos.nl/oauth2/callback"
+      upstreams = [ "http://litellm:4000" ]
+      email_domains = [ "*" ]
+      reverse_proxy = true
+      pass_authorization_header = true
+      pass_access_token = true
+      set_xauthrequest = true
+      skip_provider_button = true
+      cookie_secure = true
+      cookie_samesite = "lax"
+      cookie_domains = [ ".noxxos.nl" ]
+      whitelist_domains = [ ".noxxos.nl" ]
+
+  service:
+    type: ClusterIP
+    portNumber: 80
+
+  gatewayApi:
+    enabled: true
+    gatewayRef:
+      name: traefik-gateway
+      namespace: traefik
+      sectionName: websecure
+    hostnames:
+      - litellm.noxxos.nl
+    rules:
+      - matches:
+          - path:
+              type: PathPrefix
+              value: /
+
+  resources:
+    requests:
+      cpu: 50m
+      memory: 128Mi
+    limits:
+      memory: 256Mi

apps/open-webui/README.md

@@ -1,7 +1,10 @@
 # OpenWebUI
 
-OpenWebUI is exposed at `https://openwebui.noxxos.nl` and connects to the
+OpenWebUI is exposed at `https://openwebui.noxxos.nl` and connects to LiteLLM at
-OpenAI-compatible llama.cpp endpoint at `http://framework-llm:8080/v1`.
+`http://litellm.litellm.svc.cluster.local:4000/v1`.
+
+LiteLLM then routes requests to the OpenAI-compatible llama.cpp endpoint at
+`http://framework-llm:8080/v1`.
 
 ## Authentik
 
@@ -22,3 +25,12 @@ kubectl -n open-webui create secret generic open-webui-oidc \
   --from-literal=client-id='<authentik-client-id>' \
   --from-literal=client-secret='<authentik-client-secret>'
 ```
+
+OpenWebUI also reads its OpenAI-compatible API keys from the `litellm-secrets`
+Secret in the `open-webui` namespace. Keep it in sync with the `litellm`
+namespace copy:
+
+```bash
+kubectl -n open-webui create secret generic litellm-secrets \
+  --from-literal=openwebui-api-keys='0p3n-w3bu!;sk-<litellm-master-key>'
+```

apps/open-webui/application.yaml

@@ -4,7 +4,7 @@ metadata:
   name: open-webui
   namespace: argocd
   annotations:
-    argocd.argoproj.io/sync-wave: "2"
+    argocd.argoproj.io/sync-wave: "3"
   finalizers:
     - resources-finalizer.argocd.argoproj.io
 spec:

apps/open-webui/values.yaml

@@ -64,10 +64,9 @@ open-webui:
 
   enableOpenaiApi: true
   openaiBaseApiUrls:
-    - http://framework-llm:8080/v1
+    - http://litellm.litellm.svc.cluster.local:4000/v1
-  openaiApiKeys:
+  openaiApiKeysExistingSecret: litellm-secrets
-    - 0p3n-w3bu!
+  openaiApiKeysExistingSecretKey: openwebui-api-keys
-    - not-needed
 
   extraEnvVars:
     - name: WEBUI_URL