feat: Deploy ollama and open-webui

home/programs/aichat.nix

@@ -15,21 +15,10 @@
       model = "openai:gpt-4o";
       rag_embedding_model = "ollama:RobinBially/nomic-embed-text-8k";
       clients = [
-        {
-          type = "openai-compatible";
-          name = "llama";
-          api_base = "https://llama.darksailor.dev/api/v1";
-          api_key_cmd = "cat ${config.sops.secrets."llama/api_key".path}";
-          models = [
-            {
-              name = "qwen_2_5_1";
-            }
-          ];
-        }
         {
           type = "openai-compatible";
           name = "ollama";
-          api_base = "https://llama.darksailor.dev/api/ollama/v1";
+          api_base = "https://ollama.darksailor.dev/v1";
           api_key_cmd = "cat ${config.sops.secrets."llama/api_key".path}";
           models = [
             {
@@ -38,7 +27,12 @@
               default_chunk_size = 8000;
             }
             {
-              name = "mistral";
+              name = "deepseek-r1:7b";
+              type = "chat";
+            }
+            {
+              name = "deepseek-r1:14b";
+              type = "chat";
             }
           ];
         }

nixos/mirai/services/llama.nix

@@ -20,7 +20,7 @@
   services = {
     ollama = {
       enable = true;
-      loadModels = ["deepseek-r1:7b"];
+      loadModels = ["deepseek-r1:7b" "deepseek-r1:14b" "RobinBially/nomic-embed-text-8k"];
       port = 11434;
       host = "0.0.0.0";
       environmentVariables = {
@@ -31,81 +31,28 @@
       enable = true;
       port = 7070;
       environment = {
+        SCARF_NO_ANALYTICS = "True";
+        DO_NOT_TRACK = "True";
+        ANONYMIZED_TELEMETRY = "False";
         WEBUI_AUTH = "False";
-        WEBUI_URL = "https://llama.darksailor.dev";
         ENABLE_LOGIN_FORM = "False";
-        OLLAMA_BASE_URL = "https://llama.darksailor.dev/api/ollama";
+        WEBUI_URL = "https://llama.darksailor.dev";
-        # OPENAI_BASE_URLS = "https://api.openai.com/v1;https://llama.darksailor.dev/api/v1";
+        OLLAMA_BASE_URL = "https://ollama.darksailor.dev";
-        OPENAI_BASE_URLS = "https://api.openai.com/v1";
+        OPENAI_BASE_URL = "https://api.openai.com/v1";
       };
-      environmentFile = "${config.sops.templates."OPENAI_API_KEY.env".path}";
+      # environmentFile = "${config.sops.templates."OPENAI_API_KEY.env".path}";
     };
-    # llama-cpp = let
-    #   deepseek_r1 = map (part: "https://huggingface.co/unsloth/DeepSeek-R1-GGUF/resolve/main/DeepSeek-R1-UD-IQ1_M/DeepSeek-R1-UD-IQ1_M-0000${toString part}-of-00004.gguf?download=true") [1 2 3 4];
-    # in {
-    #   enable = true;
-    #   host = "127.0.0.1";
-    #   port = 3000;
-    #   # model = builtins.fetchurl {
-    #   #   name = "qwen_2.5.1_coder_7b_instruct_gguf";
-    #   #   sha256 = "61834b88c1a1ce5c277028a98c4a0c94a564210290992a7ba301bbef96ef8eba";
-    #   #   url = "https://huggingface.co/bartowski/Qwen2.5.1-Coder-7B-Instruct-GGUF/resolve/main/Qwen2.5.1-Coder-7B-Instruct-Q8_0.gguf?download=true";
-    #   # };
-    #   model = deepseek_r1;
-    # };
 
     caddy = {
-      # handle /api/ollama/* {
-      #     uri strip_prefix /api/ollama
-      #     reverse_proxy localhost:11434
-      #
-      #     @apikey {
-      #         header Authorization "Bearer {env.LLAMA_API_KEY}"
-      #     }
-      #
-      #     handle @apikey {
-      #         header {
-      #             # Set response headers or proxy to a different service if API key is valid
-      #             Access-Control-Allow-Origin *
-      #             -Authorization "Bearer {env.LLAMA_API_KEY}"  # Remove the header after validation
-      #         }
-      #         reverse_proxy localhost:11434
-      #     }
-      #
-      #     handle {
-      #         respond "Unauthorized" 403
-      #     }
-      # }
       virtualHosts."llama.darksailor.dev".extraConfig = ''
-        handle /api/v1/* {
+        forward_auth localhost:5555 {
-          uri strip_prefix /api/v1
+            uri /api/authz/forward-auth
-          reverse_proxy localhost:3000
+            copy_headers Remote-User Remote-Groups Remote-Email Remote-Name
-
-          @apikey {
-              header Authorization "Bearer {env.LLAMA_API_KEY}"
-          }
-
-          handle @apikey {
-              header {
-                  # Set response headers or proxy to a different service if API key is valid
-                  Access-Control-Allow-Origin *
-                  -Authorization "Bearer {env.LLAMA_API_KEY}"  # Remove the header after validation
-              }
-              reverse_proxy localhost:11434
-          }
-
-          handle {
-              respond "Unauthorized" 403
-          }
-        }
-
-        handle {
-            forward_auth localhost:5555 {
-                uri /api/authz/forward-auth
-                copy_headers Remote-User Remote-Groups Remote-Email Remote-Name
-            }
-            reverse_proxy localhost:7070
         }
+        reverse_proxy localhost:7070
+      '';
+      virtualHosts."ollama.darksailor.dev".extraConfig = ''
+        reverse_proxy localhost:11434
       '';
     };
   };

nixos/ryu/services/ollama.nix

@@ -1,16 +1,17 @@
 {pkgs, ...}: {
   services = {
     ollama = {
-      enable = true;
+      enable = false;
       host = "127.0.0.1";
+      loadModels = ["deepseek-r1:7b" "deepseek-r1:14b"];
       port = 11434;
       package = pkgs.ollama-cuda;
+      # acceleration = "cuda";
     };
     open-webui = {
-      enable = true;
+      enable = false;
       environment = {
-        "OLLAMA_API_BASE_URL" = "http://127.0.0.1:11434/api";
+        OLLAMA_BASE_URL = "http://127.0.0.1:11434";
-        "OLLAMA_BASE_URL" = "http://127.0.0.1:11434";
         WEBUI_AUTH = "False";
         ENABLE_LOGIN_FORM = "False";
       };