Merge pull request #1066 from offlinehacker/nixos/logstash/update

nixos/logstash: update and simplify to be fully compatible with new version

nixos/modules/services/logging/logstash.nix

@@ -3,72 +3,8 @@
 with pkgs.lib;
 
 let
-
   cfg = config.services.logstash;
 
-  listToConfig = list: "[ " + (concatStringsSep ", " (map exprToConfig list)) + " ]";
-
-  hashToConfig = attrs:
-    let
-      attrNameToConfigList = name:
-        [ (exprToConfig name)  (exprToConfig (getAttr name attrs)) ];
-    in
-      "[ " +
-      (concatStringsSep ", " (map attrNameToConfigList (attrNames attrs))) +
-      " ]";
-
-  valueToConfig = nvpair: let name = nvpair.name; value = nvpair.value; in
-    if (isAttrs value) && ((!(value ? __type)) || value.__type == "repeated")
-      then ''
-        ${name} {
-          ${exprToConfig value}
-        }
-      ''
-      else "${name} => ${exprToConfig value}";
-
-  repeatedAttrsToConfig = values:
-      concatStringsSep "\n" (map valueToConfig values);
-
-  attrsToConfig = attrs:
-    let
-      attrToConfig = name: valueToConfig {
-        inherit name;
-        value = (getAttr name attrs);
-      };
-    in
-      concatStringsSep "\n" (map attrToConfig (attrNames attrs));
-
-  exprToConfig = expr:
-    let
-      isCustomType = expr: (isAttrs expr) && (expr ? __type);
-
-      isFloat = expr: (isCustomType expr) && (expr.__type == "float");
-
-      isHash = expr: (isCustomType expr) && (expr.__type == "hash");
-
-      isRepeatedAttrs = expr: (isCustomType expr) && (expr.__type == "repeated");
-    in
-      if builtins.isBool expr then (if expr then "true" else "false") else
-      if builtins.isString expr then ''"${expr}"'' else
-      if builtins.isInt expr then toString expr else
-      if isFloat expr then expr.value else
-      if isList expr then listToConfig expr else
-      if isHash expr then hashToConfig expr.value else
-      if isRepeatedAttrs expr then repeatedAttrsToConfig expr.values
-      else attrsToConfig expr;
-
-  mergeConfigs = configs:
-    let
-      op = attrs: newAttrs:
-        let
-          isRepeated = newAttrs ? __type && newAttrs.__type == "repeated";
-        in {
-            values = attrs.values ++ (if isRepeated then newAttrs.values else
-              map (name: { inherit name; value = getAttr name newAttrs; })
-              (attrNames newAttrs));
-          };
-    in (foldl op { values = []; } configs) // { __type = "repeated"; };
-
 in
 
 {
@@ -78,48 +14,45 @@ in
     services.logstash = {
       enable = mkOption {
         default = false;
-        description = ''
+        description = "Enable logstash";
-          Enable logstash.
-        '';
       };
 
       inputConfig = mkOption {
-        default = {};
+        default = ''stdin { type => "example" }'';
-        description = ''
+        description = "Logstash input configuration";
-          An attribute set (or an expression generated by mkNameValuePairs)
+        example = ''
-          representing a logstash configuration's input section.
+          # Read from journal
-          Logstash configs are name-value pairs, where values can be bools,
+          pipe {
-          strings, numbers, arrays, hashes, or other name-value pairs,
+            command => "${pkgs.systemd}/bin/journalctl -f -o json"
-          and names are strings that can be repeated. Name-value pairs with no
+            type => "syslog" codec => json {}
-          repeats are represented by attr sets. Bools, strings, ints, and
+          }
-          arrays are mapped directly. Name-value pairs with repeats can be
-          generated by the config.lib.logstash.mkNameValuePairs function, which
-          takes a list of attrsets and combines them while preserving attribute
-          name duplicates if they occur. Similarly, there are the mkFloat and
-          mkHash functions, which take a string representation of a float and an
-          attrset, respectively.
         '';
-        apply = mergeConfigs;
       };
 
       filterConfig = mkOption {
-        default = {};
+        default = ''noop {}'';
-        description = ''
+        description = "logstash filter configuration";
-          An attribute set (or an expression generated by mkNameValuePairs)
+        example = ''
-          representing a logstash configuration's filter section.
+          if [type] == "syslog" {
-          See inputConfig description for details.
+            # Keep only relevant systemd fields
+            # http://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html
+            prune {
+              whitelist_names => [
+                "type", "@timestamp", "@version",
+                "MESSAGE", "PRIORITY", "SYSLOG_FACILITY",
+              ]
+            }
+          }
         '';
-        apply = mergeConfigs;
       };
 
       outputConfig = mkOption {
-        default = {};
+        default = ''stdout { debug => true debug_format => "json"}'';
-        description = ''
+        description = "Logstash output configuration";
-          An attribute set (or an expression generated by mkNameValuePairs)
+        example = ''
-          representing a logstash configuration's output section.
+          redis { host => "localhost" data_type => "list" key => "logstash" codec => json }
-          See inputConfig description for details.
+          elasticsearch { embedded => true }
         '';
-        apply = mergeConfigs;
       };
     };
   };
@@ -127,35 +60,26 @@ in
 
   ###### implementation
 
-  config = mkMerge [ {
+  config = mkIf cfg.enable {
-    lib.logstash = {
-      mkFloat = stringRep: { __type = "float"; value = stringRep; };
-
-      mkHash = attrs: { __type = "hash"; value = attrs; };
-
-      mkNameValuePairs = mergeConfigs;
-    };
-  } ( mkIf cfg.enable {
     systemd.services.logstash = with pkgs; {
       description = "Logstash daemon";
-
       wantedBy = [ "multi-user.target" ];
 
-      path = [ jre ];
+      serviceConfig = {
+        ExecStart = "${jre}/bin/java -jar ${logstash} agent -f ${writeText "logstash.conf" ''
+          input {
+            ${cfg.inputConfig}
+          }
 
-      script = "cd /tmp && exec java -jar ${logstash} agent -f ${writeText "logstash.conf" ''
+          filter {
-        input {
+            ${cfg.filterConfig}
-          ${exprToConfig cfg.inputConfig}
+          }
-        }
 
-        filter {
+          output {
-          ${exprToConfig cfg.filterConfig}
+            ${cfg.outputConfig}
-        }
+          }
-
+        ''}";
-        output {
+      };
-          ${exprToConfig cfg.outputConfig}
-        }
-      ''} &> /var/log/logstash.log";
     };
-  })];
+  };
 }

nixos/tests/default.nix

@@ -16,6 +16,7 @@ with import ../lib/testing.nix { inherit system minimal; };
   kde4 = makeTest (import ./kde4.nix);
   #kexec = makeTest (import ./kexec.nix);
   login = makeTest (import ./login.nix {});
+  logstash = makeTest (import ./logstash.nix);
   latestKernel.login = makeTest (import ./login.nix ({ config, pkgs, ... }: { boot.kernelPackages = pkgs.linuxPackages_latest; }));
   misc = makeTest (import ./misc.nix);
   #mpich = makeTest (import ./mpich.nix);

nixos/tests/logstash.nix

@@ -0,0 +1,40 @@
+{ pkgs, ... }:
+
+# This test runs logstash and checks if messages flows and elasticsearch is
+# started
+
+{
+  nodes = {
+    one =
+      { config, pkgs, ... }:
+        {
+          services = {
+            logstash = {
+              enable = true;
+              inputConfig = ''
+                exec { command => "echo flowers" interval => 1 type => "test" }
+                exec { command => "echo dragons" interval => 1 type => "test" }
+              '';
+              filterConfig = ''
+                if [type] == "test" {
+                  grep { match => ["message", "flowers"] drop => true }
+                }
+              '';
+              outputConfig = ''
+                stdout { codec => rubydebug }
+                elasticsearch { embedded => true }
+              '';
+            };
+          };
+        };
+    };
+  
+  testScript = ''
+    startAll;
+  
+    $one->waitForUnit("logstash.service");
+    $one->waitUntilSucceeds("journalctl -n 20 _SYSTEMD_UNIT=logstash.service | grep flowers");
+    $one->fail("journalctl -n 20 _SYSTEMD_UNIT=logstash.service | grep dragons");
+    $one->waitUntilSucceeds("curl -s http://127.0.0.1:9200/_status?pretty=true | grep logstash");
+  '';
+}