From c8a823d40b62f095c54341dd3476ca32f9beecee Mon Sep 17 00:00:00 2001 From: Pol Dellaiera Date: Sun, 16 Mar 2025 10:37:16 +0100 Subject: [PATCH] nixos/litellm: init --- .../manual/release-notes/rl-2505.section.md | 2 + nixos/modules/module-list.nix | 1 + nixos/modules/services/misc/litellm.nix | 182 ++++++++++++++++++ nixos/tests/all-tests.nix | 1 + nixos/tests/litellm.nix | 27 +++ .../python-modules/litellm/default.nix | 11 +- 6 files changed, 221 insertions(+), 3 deletions(-) create mode 100644 nixos/modules/services/misc/litellm.nix create mode 100644 nixos/tests/litellm.nix diff --git a/nixos/doc/manual/release-notes/rl-2505.section.md b/nixos/doc/manual/release-notes/rl-2505.section.md index 0516d4410361..09e2e8396cb0 100644 --- a/nixos/doc/manual/release-notes/rl-2505.section.md +++ b/nixos/doc/manual/release-notes/rl-2505.section.md @@ -178,6 +178,8 @@ - [echoip](https://github.com/mpolden/echoip), a simple service for looking up your IP address. Available as [services.echoip](#opt-services.echoip.enable). +- [LiteLLM](https://github.com/BerriAI/litellm), a LLM Gateway to provide model access, fallbacks and spend tracking across 100+ LLMs. All in the OpenAI format. Available as [services.litellm](#opt-services.litellm.enable). + - [Buffyboard](https://gitlab.postmarketos.org/postmarketOS/buffybox/-/tree/master/buffyboard), a framebuffer on-screen keyboard. Available as [services.buffyboard](option.html#opt-services.buffyboard). - [KanBoard](https://github.com/kanboard/kanboard), a project management tool that focuses on the Kanban methodology. Available as [services.kanboard](#opt-services.kanboard.enable). diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 4b3d75a1a8a7..c46cd9d4b6e4 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -825,6 +825,7 @@ ./services/misc/languagetool.nix ./services/misc/leaps.nix ./services/misc/lifecycled.nix + ./services/misc/litellm.nix ./services/misc/llama-cpp.nix ./services/misc/logkeys.nix ./services/misc/mame.nix diff --git a/nixos/modules/services/misc/litellm.nix b/nixos/modules/services/misc/litellm.nix new file mode 100644 index 000000000000..621d7e9542cf --- /dev/null +++ b/nixos/modules/services/misc/litellm.nix @@ -0,0 +1,182 @@ +{ + config, + lib, + pkgs, + ... +}: +let + inherit (lib) types; + + cfg = config.services.litellm; + settingsFormat = pkgs.formats.yaml { }; +in +{ + options = { + services.litellm = { + enable = lib.mkEnableOption "LiteLLM server"; + package = lib.mkPackageOption pkgs "litellm" { }; + + stateDir = lib.mkOption { + type = types.path; + default = "/var/lib/litellm"; + example = "/home/foo"; + description = "State directory of LiteLLM."; + }; + + host = lib.mkOption { + type = types.str; + default = "127.0.0.1"; + example = "0.0.0.0"; + description = '' + The host address which the LiteLLM server HTTP interface listens to. + ''; + }; + + port = lib.mkOption { + type = types.port; + default = 8080; + example = 11111; + description = '' + Which port the LiteLLM server listens to. + ''; + }; + + settings = lib.mkOption { + type = types.submodule { + freeformType = settingsFormat.type; + options = { + model_list = lib.mkOption { + type = settingsFormat.type; + description = '' + List of supported models on the server, with model-specific configs. + ''; + default = [ ]; + }; + router_settings = lib.mkOption { + type = settingsFormat.type; + description = '' + LiteLLM Router settings + ''; + default = { }; + }; + + litellm_settings = lib.mkOption { + type = settingsFormat.type; + description = '' + LiteLLM Module settings + ''; + default = { }; + }; + + general_settings = lib.mkOption { + type = settingsFormat.type; + description = '' + LiteLLM Server settings + ''; + default = { }; + }; + + environment_variables = lib.mkOption { + type = settingsFormat.type; + description = '' + Environment variables to pass to the Lite + ''; + default = { }; + }; + }; + }; + default = { }; + description = '' + Configuration for LiteLLM. + See for more. + ''; + }; + + environment = lib.mkOption { + type = types.attrsOf types.str; + default = { + SCARF_NO_ANALYTICS = "True"; + DO_NOT_TRACK = "True"; + ANONYMIZED_TELEMETRY = "False"; + }; + example = '' + { + NO_DOCS="True"; + } + ''; + description = '' + Extra environment variables for LiteLLM. + ''; + }; + + environmentFile = lib.mkOption { + description = '' + Environment file to be passed to the systemd service. + Useful for passing secrets to the service to prevent them from being + world-readable in the Nix store. + ''; + type = lib.types.nullOr lib.types.path; + default = null; + example = "/var/lib/secrets/liteLLMSecrets"; + }; + + openFirewall = lib.mkOption { + type = types.bool; + default = false; + description = '' + Whether to open the firewall for LiteLLM. + This adds `services.litellm.port` to `networking.firewall.allowedTCPPorts`. + ''; + }; + }; + }; + + config = lib.mkIf cfg.enable { + systemd.services.litellm = { + description = "LLM Gateway to provide model access, fallbacks and spend tracking across 100+ LLMs."; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + + environment = cfg.environment; + + serviceConfig = + let + configFile = settingsFormat.generate "config.yaml" cfg.settings; + in + { + ExecStart = "${lib.getExe cfg.package} --host \"${cfg.host}\" --port ${toString cfg.port} --config ${configFile}"; + EnvironmentFile = lib.optional (cfg.environmentFile != null) cfg.environmentFile; + WorkingDirectory = cfg.stateDir; + StateDirectory = "litellm"; + RuntimeDirectory = "litellm"; + RuntimeDirectoryMode = "0755"; + PrivateTmp = true; + DynamicUser = true; + DevicePolicy = "closed"; + LockPersonality = true; + PrivateUsers = true; + ProtectHome = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectControlGroups = true; + RestrictNamespaces = true; + RestrictRealtime = true; + SystemCallArchitectures = "native"; + UMask = "0077"; + RestrictAddressFamilies = [ + "AF_INET" + "AF_INET6" + "AF_UNIX" + ]; + ProtectClock = true; + ProtectProc = "invisible"; + }; + }; + + networking.firewall = lib.mkIf cfg.openFirewall { allowedTCPPorts = [ cfg.port ]; }; + }; + + meta.maintainers = with lib.maintainers; [ drupol ]; +} diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index a2a69f2375e9..55c0cbd8ce1a 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -634,6 +634,7 @@ in { limesurvey = handleTest ./limesurvey.nix {}; limine = import ./limine { inherit runTest; }; listmonk = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./listmonk.nix {}; + litellm = runTest ./litellm.nix; litestream = handleTest ./litestream.nix {}; lldap = handleTest ./lldap.nix {}; localsend = handleTest ./localsend.nix {}; diff --git a/nixos/tests/litellm.nix b/nixos/tests/litellm.nix new file mode 100644 index 000000000000..96d67f9521a0 --- /dev/null +++ b/nixos/tests/litellm.nix @@ -0,0 +1,27 @@ +{ lib, ... }: +let + mainPort = "8080"; +in +{ + name = "litellm"; + + nodes = { + machine = + { ... }: + { + services.litellm = { + enable = true; + }; + }; + }; + + testScript = '' + machine.start() + machine.wait_for_unit("litellm.service") + machine.wait_for_open_port(${mainPort}) + ''; + + meta = with lib.maintainers; { + maintainers = [ drupol ]; + }; +} diff --git a/pkgs/development/python-modules/litellm/default.nix b/pkgs/development/python-modules/litellm/default.nix index e69090bbca02..5b7baadcf1d7 100644 --- a/pkgs/development/python-modules/litellm/default.nix +++ b/pkgs/development/python-modules/litellm/default.nix @@ -35,6 +35,7 @@ tokenizers, uvloop, uvicorn, + nixosTests, }: buildPythonPackage rec { @@ -99,12 +100,16 @@ buildPythonPackage rec { # access network doCheck = false; - meta = with lib; { + passthru.tests = { + inherit (nixosTests) litellm; + }; + + meta = { description = "Use any LLM as a drop in replacement for gpt-3.5-turbo. Use Azure, OpenAI, Cohere, Anthropic, Ollama, VLLM, Sagemaker, HuggingFace, Replicate (100+ LLMs)"; mainProgram = "litellm"; homepage = "https://github.com/BerriAI/litellm"; changelog = "https://github.com/BerriAI/litellm/releases/tag/${src.tag}"; - license = licenses.mit; - maintainers = with maintainers; [ happysalada ]; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ happysalada ]; }; }