From c94ca8bc6dbd5c8cc2fac141c10b13879b6a66af Mon Sep 17 00:00:00 2001 From: uttarayan21 Date: Fri, 29 Nov 2024 16:56:39 +0530 Subject: [PATCH] feat: Secure endpoints for llama and ollama with api keys --- common/home.nix | 7 ++-- nixos/mirai/configuration.nix | 7 ++++ nixos/mirai/docker.nix | 36 ++++++++++++++++ nixos/mirai/services.nix | 79 ++++++++++++++++++----------------- 4 files changed, 87 insertions(+), 42 deletions(-) diff --git a/common/home.nix b/common/home.nix index 1f5b81b3..0ec0f9a3 100644 --- a/common/home.nix +++ b/common/home.nix @@ -344,6 +344,9 @@ in { enable = true; enableFishIntegration = true; settings = { + save_session = true; + model = "openai:gpt-4o"; + rag_embedding_model = "ollama:RobinBially/nomic-embed-text-8k"; clients = [ { type = "openai-compatible"; @@ -368,7 +371,7 @@ in { default_chunk_size = 8000; } { - name = "codellama:minstral"; + name = "mistral"; } ]; } @@ -387,8 +390,6 @@ in { ]; } ]; - rag_embedding_model = "ollama:RobinBially/nomic-embed-text-8k"; - model = "openai:gpt-4o"; }; }; }; diff --git a/nixos/mirai/configuration.nix b/nixos/mirai/configuration.nix index 823b807b..0b7feb61 100644 --- a/nixos/mirai/configuration.nix +++ b/nixos/mirai/configuration.nix @@ -18,11 +18,18 @@ secrets."nextcloud/adminpass".owner = config.users.users.nextcloud.name; secrets."llama/user".owner = config.services.caddy.user; secrets."builder/mirai/cache/private" = {}; + secrets."llama/api_key".owner = config.services.caddy.user; secrets.users = { sopsFile = ../../secrets/users.yaml; format = "yaml"; key = ""; }; + templates = { + "LLAMA_API_KEY.env".content = '' + LLAMA_API_KEY=${config.sops.placeholder."llama/api_key"} + ''; + api_key_env.owner = config.services.caddy.user; + }; }; # Use the systemd-boot EFI boot loader. diff --git a/nixos/mirai/docker.nix b/nixos/mirai/docker.nix index 1b620ecd..3a386d90 100644 --- a/nixos/mirai/docker.nix +++ b/nixos/mirai/docker.nix @@ -30,4 +30,40 @@ # reverse_proxy localhost:8123 # ''; # }; + # containers.llama = { + # autoStart = true; + # privateNetwork = true; + # hostAddress = "192.168.100.10"; + # localAddress = "192.168.100.11"; + # hostAddress6 = "fc00::1"; + # localAddress6 = "fc00::2"; + # config = { + # config, + # pkgs, + # libs, + # ... + # }: { + # system.stateVersion = "24.11"; + # networking = { + # firewall = { + # enable = true; + # allowedTCPPorts = [4000]; + # }; + # # Use systemd-resolved inside the container + # # Workaround for bug https://github.com/NixOS/nixpkgs/issues/162686 + # useHostResolvConf = lib.mkForce false; + # }; + # services.resolved.enable = true; + # services.llama-cpp = { + # enable = true; + # host = "127.0.0.1"; + # port = 4000; + # model = builtins.fetchurl { + # name = "qwen_2.5.1_coder_7b_instruct_gguf"; + # sha256 = "61834b88c1a1ce5c277028a98c4a0c94a564210290992a7ba301bbef96ef8eba"; + # url = "https://huggingface.co/bartowski/Qwen2.5.1-Coder-7B-Instruct-GGUF/resolve/main/Qwen2.5.1-Coder-7B-Instruct-Q8_0.gguf?download=true"; + # }; + # }; + # }; + # }; } diff --git a/nixos/mirai/services.nix b/nixos/mirai/services.nix index 213958f7..5625f183 100644 --- a/nixos/mirai/services.nix +++ b/nixos/mirai/services.nix @@ -11,7 +11,6 @@ "authelia/servers/darksailor/sessionSecret".owner = config.systemd.services.authelia-darksailor.serviceConfig.User; "authelia/users/servius".owner = config.systemd.services.authelia-darksailor.serviceConfig.User; users.owner = config.systemd.services.authelia-darksailor.serviceConfig.User; - "llama/api_key".owner = config.services.caddy.user; }; }; services = { @@ -142,7 +141,7 @@ }; ollama = { enable = true; - loadModels = ["RobinBially/nomic-embed-text-8k" "minstral"]; + loadModels = ["RobinBially/nomic-embed-text-8k" "mistral"]; port = 11434; host = "0.0.0.0"; environmentVariables = { @@ -181,11 +180,45 @@ handle /api/v1/* { uri strip_prefix /api/v1 reverse_proxy localhost:3000 + + @apikey { + header Authorization "Bearer {env.LLAMA_API_KEY}" + } + + handle @apikey { + header { + # Set response headers or proxy to a different service if API key is valid + Access-Control-Allow-Origin * + -Authorization "Bearer {env.LLAMA_API_KEY}" # Remove the header after validation + } + reverse_proxy localhost:11434 + } + + handle { + respond "Unauthorized" 403 + } } handle /api/ollama/* { uri strip_prefix /api/ollama reverse_proxy localhost:11434 + + @apikey { + header Authorization "Bearer {env.LLAMA_API_KEY}" + } + + handle @apikey { + header { + # Set response headers or proxy to a different service if API key is valid + Access-Control-Allow-Origin * + -Authorization "Bearer {env.LLAMA_API_KEY}" # Remove the header after validation + } + reverse_proxy localhost:11434 + } + + handle { + respond "Unauthorized" 403 + } } handle { @@ -212,41 +245,9 @@ ''; }; }; - - # containers.llama = { - # autoStart = true; - # privateNetwork = true; - # hostAddress = "192.168.100.10"; - # localAddress = "192.168.100.11"; - # hostAddress6 = "fc00::1"; - # localAddress6 = "fc00::2"; - # config = { - # config, - # pkgs, - # libs, - # ... - # }: { - # system.stateVersion = "24.11"; - # networking = { - # firewall = { - # enable = true; - # allowedTCPPorts = [4000]; - # }; - # # Use systemd-resolved inside the container - # # Workaround for bug https://github.com/NixOS/nixpkgs/issues/162686 - # useHostResolvConf = lib.mkForce false; - # }; - # services.resolved.enable = true; - # services.llama-cpp = { - # enable = true; - # host = "127.0.0.1"; - # port = 4000; - # model = builtins.fetchurl { - # name = "qwen_2.5.1_coder_7b_instruct_gguf"; - # sha256 = "61834b88c1a1ce5c277028a98c4a0c94a564210290992a7ba301bbef96ef8eba"; - # url = "https://huggingface.co/bartowski/Qwen2.5.1-Coder-7B-Instruct-GGUF/resolve/main/Qwen2.5.1-Coder-7B-Instruct-Q8_0.gguf?download=true"; - # }; - # }; - # }; - # }; + systemd.services.caddy = { + serviceConfig = { + EnvironmentFile = config.sops.templates."LLAMA_API_KEY.env".path; + }; + }; }