From 7427d8802633c84a10fa62e818f36186371ad79d Mon Sep 17 00:00:00 2001 From: uttarayan21 Date: Thu, 6 Feb 2025 01:43:54 +0530 Subject: [PATCH] feat: Deploy ollama and open-webui --- home/programs/aichat.nix | 20 +++----- nixos/mirai/services/llama.nix | 83 ++++++---------------------------- nixos/ryu/services/ollama.nix | 9 ++-- 3 files changed, 27 insertions(+), 85 deletions(-) diff --git a/home/programs/aichat.nix b/home/programs/aichat.nix index 9d979351..3defe3b9 100644 --- a/home/programs/aichat.nix +++ b/home/programs/aichat.nix @@ -15,21 +15,10 @@ model = "openai:gpt-4o"; rag_embedding_model = "ollama:RobinBially/nomic-embed-text-8k"; clients = [ - { - type = "openai-compatible"; - name = "llama"; - api_base = "https://llama.darksailor.dev/api/v1"; - api_key_cmd = "cat ${config.sops.secrets."llama/api_key".path}"; - models = [ - { - name = "qwen_2_5_1"; - } - ]; - } { type = "openai-compatible"; name = "ollama"; - api_base = "https://llama.darksailor.dev/api/ollama/v1"; + api_base = "https://ollama.darksailor.dev/v1"; api_key_cmd = "cat ${config.sops.secrets."llama/api_key".path}"; models = [ { @@ -38,7 +27,12 @@ default_chunk_size = 8000; } { - name = "mistral"; + name = "deepseek-r1:7b"; + type = "chat"; + } + { + name = "deepseek-r1:14b"; + type = "chat"; } ]; } diff --git a/nixos/mirai/services/llama.nix b/nixos/mirai/services/llama.nix index 3f95c80b..fe41392b 100644 --- a/nixos/mirai/services/llama.nix +++ b/nixos/mirai/services/llama.nix @@ -20,7 +20,7 @@ services = { ollama = { enable = true; - loadModels = ["deepseek-r1:7b"]; + loadModels = ["deepseek-r1:7b" "deepseek-r1:14b" "RobinBially/nomic-embed-text-8k"]; port = 11434; host = "0.0.0.0"; environmentVariables = { @@ -31,81 +31,28 @@ enable = true; port = 7070; environment = { + SCARF_NO_ANALYTICS = "True"; + DO_NOT_TRACK = "True"; + ANONYMIZED_TELEMETRY = "False"; WEBUI_AUTH = "False"; - WEBUI_URL = "https://llama.darksailor.dev"; ENABLE_LOGIN_FORM = "False"; - OLLAMA_BASE_URL = "https://llama.darksailor.dev/api/ollama"; - # OPENAI_BASE_URLS = "https://api.openai.com/v1;https://llama.darksailor.dev/api/v1"; - OPENAI_BASE_URLS = "https://api.openai.com/v1"; + WEBUI_URL = "https://llama.darksailor.dev"; + OLLAMA_BASE_URL = "https://ollama.darksailor.dev"; + OPENAI_BASE_URL = "https://api.openai.com/v1"; }; - environmentFile = "${config.sops.templates."OPENAI_API_KEY.env".path}"; + # environmentFile = "${config.sops.templates."OPENAI_API_KEY.env".path}"; }; - # llama-cpp = let - # deepseek_r1 = map (part: "https://huggingface.co/unsloth/DeepSeek-R1-GGUF/resolve/main/DeepSeek-R1-UD-IQ1_M/DeepSeek-R1-UD-IQ1_M-0000${toString part}-of-00004.gguf?download=true") [1 2 3 4]; - # in { - # enable = true; - # host = "127.0.0.1"; - # port = 3000; - # # model = builtins.fetchurl { - # # name = "qwen_2.5.1_coder_7b_instruct_gguf"; - # # sha256 = "61834b88c1a1ce5c277028a98c4a0c94a564210290992a7ba301bbef96ef8eba"; - # # url = "https://huggingface.co/bartowski/Qwen2.5.1-Coder-7B-Instruct-GGUF/resolve/main/Qwen2.5.1-Coder-7B-Instruct-Q8_0.gguf?download=true"; - # # }; - # model = deepseek_r1; - # }; caddy = { - # handle /api/ollama/* { - # uri strip_prefix /api/ollama - # reverse_proxy localhost:11434 - # - # @apikey { - # header Authorization "Bearer {env.LLAMA_API_KEY}" - # } - # - # handle @apikey { - # header { - # # Set response headers or proxy to a different service if API key is valid - # Access-Control-Allow-Origin * - # -Authorization "Bearer {env.LLAMA_API_KEY}" # Remove the header after validation - # } - # reverse_proxy localhost:11434 - # } - # - # handle { - # respond "Unauthorized" 403 - # } - # } virtualHosts."llama.darksailor.dev".extraConfig = '' - handle /api/v1/* { - uri strip_prefix /api/v1 - reverse_proxy localhost:3000 - - @apikey { - header Authorization "Bearer {env.LLAMA_API_KEY}" - } - - handle @apikey { - header { - # Set response headers or proxy to a different service if API key is valid - Access-Control-Allow-Origin * - -Authorization "Bearer {env.LLAMA_API_KEY}" # Remove the header after validation - } - reverse_proxy localhost:11434 - } - - handle { - respond "Unauthorized" 403 - } - } - - handle { - forward_auth localhost:5555 { - uri /api/authz/forward-auth - copy_headers Remote-User Remote-Groups Remote-Email Remote-Name - } - reverse_proxy localhost:7070 + forward_auth localhost:5555 { + uri /api/authz/forward-auth + copy_headers Remote-User Remote-Groups Remote-Email Remote-Name } + reverse_proxy localhost:7070 + ''; + virtualHosts."ollama.darksailor.dev".extraConfig = '' + reverse_proxy localhost:11434 ''; }; }; diff --git a/nixos/ryu/services/ollama.nix b/nixos/ryu/services/ollama.nix index 8e652ad4..5504f2f6 100644 --- a/nixos/ryu/services/ollama.nix +++ b/nixos/ryu/services/ollama.nix @@ -1,16 +1,17 @@ {pkgs, ...}: { services = { ollama = { - enable = true; + enable = false; host = "127.0.0.1"; + loadModels = ["deepseek-r1:7b" "deepseek-r1:14b"]; port = 11434; package = pkgs.ollama-cuda; + # acceleration = "cuda"; }; open-webui = { - enable = true; + enable = false; environment = { - "OLLAMA_API_BASE_URL" = "http://127.0.0.1:11434/api"; - "OLLAMA_BASE_URL" = "http://127.0.0.1:11434"; + OLLAMA_BASE_URL = "http://127.0.0.1:11434"; WEBUI_AUTH = "False"; ENABLE_LOGIN_FORM = "False"; };