movefasta/nixpkgs
0
0
Fork 0
mirror of https://github.com/NixOS/nixpkgs.git synced 2025-07-13 21:50:33 +03:00
Code Activity

nixos/services.hadoop: remove with lib;

Browse source
This commit is contained in:
Felix Buehler 2024-12-08 13:18:22 +01:00
parent ddcd8d565e
commit e5c3196d17
1 changed files with 41 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

83
nixos/modules/services/cluster/hadoop/default.nix
View file

@ -3,15 +3,14 @@ let
cfg = config.services.hadoop; cfg = config.services.hadoop;
opt = options.services.hadoop; opt = options.services.hadoop;
in in
with lib;
{ {
imports = [ ./yarn.nix ./hdfs.nix ./hbase.nix ]; imports = [ ./yarn.nix ./hdfs.nix ./hbase.nix ];
options.services.hadoop = { options.services.hadoop = {
coreSite = mkOption { coreSite = lib.mkOption {
default = {}; default = {};
type = types.attrsOf types.anything; type = lib.types.attrsOf lib.types.anything;
example = literalExpression '' example = lib.literalExpression ''
{ {
"fs.defaultFS" = "hdfs://localhost"; "fs.defaultFS" = "hdfs://localhost";
} }
@ -21,31 +20,31 @@ with lib;
<https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/core-default.xml> <https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/core-default.xml>
''; '';
}; };
coreSiteInternal = mkOption { coreSiteInternal = lib.mkOption {
default = {}; default = {};
type = types.attrsOf types.anything; type = lib.types.attrsOf lib.types.anything;
internal = true; internal = true;
description = '' description = ''
Internal option to add configs to core-site.xml based on module options Internal option to add configs to core-site.xml based on module options
''; '';
}; };
hdfsSiteDefault = mkOption { hdfsSiteDefault = lib.mkOption {
default = { default = {
"dfs.namenode.rpc-bind-host" = "0.0.0.0"; "dfs.namenode.rpc-bind-host" = "0.0.0.0";
"dfs.namenode.http-address" = "0.0.0.0:9870"; "dfs.namenode.http-address" = "0.0.0.0:9870";
"dfs.namenode.servicerpc-bind-host" = "0.0.0.0"; "dfs.namenode.servicerpc-bind-host" = "0.0.0.0";
"dfs.namenode.http-bind-host" = "0.0.0.0"; "dfs.namenode.http-bind-host" = "0.0.0.0";
}; };
type = types.attrsOf types.anything; type = lib.types.attrsOf lib.types.anything;
description = '' description = ''
Default options for hdfs-site.xml Default options for hdfs-site.xml
''; '';
}; };
hdfsSite = mkOption { hdfsSite = lib.mkOption {
default = {}; default = {};
type = types.attrsOf types.anything; type = lib.types.attrsOf lib.types.anything;
example = literalExpression '' example = lib.literalExpression ''
{ {
"dfs.nameservices" = "namenode1"; "dfs.nameservices" = "namenode1";
} }
@ -55,23 +54,23 @@ with lib;
<https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/hdfs-default.xml> <https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/hdfs-default.xml>
''; '';
}; };
hdfsSiteInternal = mkOption { hdfsSiteInternal = lib.mkOption {
default = {}; default = {};
type = types.attrsOf types.anything; type = lib.types.attrsOf lib.types.anything;
internal = true; internal = true;
description = '' description = ''
Internal option to add configs to hdfs-site.xml based on module options Internal option to add configs to hdfs-site.xml based on module options
''; '';
}; };
mapredSiteDefault = mkOption { mapredSiteDefault = lib.mkOption {
default = { default = {
"mapreduce.framework.name" = "yarn"; "mapreduce.framework.name" = "yarn";
"yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=${cfg.package}"; "yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=${cfg.package}";
"mapreduce.map.env" = "HADOOP_MAPRED_HOME=${cfg.package}"; "mapreduce.map.env" = "HADOOP_MAPRED_HOME=${cfg.package}";
"mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=${cfg.package}"; "mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=${cfg.package}";
}; };
defaultText = literalExpression '' defaultText = lib.literalExpression ''
{ {
"mapreduce.framework.name" = "yarn"; "mapreduce.framework.name" = "yarn";
"yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}"; "yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}";
@ -79,15 +78,15 @@ with lib;
"mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}"; "mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}";
} }
''; '';
type = types.attrsOf types.anything; type = lib.types.attrsOf lib.types.anything;
description = '' description = ''
Default options for mapred-site.xml Default options for mapred-site.xml
''; '';
}; };
mapredSite = mkOption { mapredSite = lib.mkOption {
default = {}; default = {};
type = types.attrsOf types.anything; type = lib.types.attrsOf lib.types.anything;
example = literalExpression '' example = lib.literalExpression ''
{ {
"mapreduce.map.java.opts" = "-Xmx900m -XX:+UseParallelGC"; "mapreduce.map.java.opts" = "-Xmx900m -XX:+UseParallelGC";
} }
@ -98,7 +97,7 @@ with lib;
''; '';
}; };
yarnSiteDefault = mkOption { yarnSiteDefault = lib.mkOption {
default = { default = {
"yarn.nodemanager.admin-env" = "PATH=$PATH"; "yarn.nodemanager.admin-env" = "PATH=$PATH";
"yarn.nodemanager.aux-services" = "mapreduce_shuffle"; "yarn.nodemanager.aux-services" = "mapreduce_shuffle";
@ -112,15 +111,15 @@ with lib;
"yarn.resourcemanager.bind-host" = "0.0.0.0"; "yarn.resourcemanager.bind-host" = "0.0.0.0";
"yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler"; "yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler";
}; };
type = types.attrsOf types.anything; type = lib.types.attrsOf lib.types.anything;
description = '' description = ''
Default options for yarn-site.xml Default options for yarn-site.xml
''; '';
}; };
yarnSite = mkOption { yarnSite = lib.mkOption {
default = {}; default = {};
type = types.attrsOf types.anything; type = lib.types.attrsOf lib.types.anything;
example = literalExpression '' example = lib.literalExpression ''
{ {
"yarn.resourcemanager.hostname" = "''${config.networking.hostName}"; "yarn.resourcemanager.hostname" = "''${config.networking.hostName}";
} }
@ -130,19 +129,19 @@ with lib;
<https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-common/yarn-default.xml> <https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-common/yarn-default.xml>
''; '';
}; };
yarnSiteInternal = mkOption { yarnSiteInternal = lib.mkOption {
default = {}; default = {};
type = types.attrsOf types.anything; type = lib.types.attrsOf lib.types.anything;
internal = true; internal = true;
description = '' description = ''
Internal option to add configs to yarn-site.xml based on module options Internal option to add configs to yarn-site.xml based on module options
''; '';
}; };
httpfsSite = mkOption { httpfsSite = lib.mkOption {
default = { }; default = { };
type = types.attrsOf types.anything; type = lib.types.attrsOf lib.types.anything;
example = literalExpression '' example = lib.literalExpression ''
{ {
"hadoop.http.max.threads" = 500; "hadoop.http.max.threads" = 500;
} }
@ -153,19 +152,19 @@ with lib;
''; '';
}; };
log4jProperties = mkOption { log4jProperties = lib.mkOption {
default = "${cfg.package}/etc/hadoop/log4j.properties"; default = "${cfg.package}/etc/hadoop/log4j.properties";
defaultText = literalExpression '' defaultText = lib.literalExpression ''
"''${config.${opt.package}}/etc/hadoop/log4j.properties" "''${config.${opt.package}}/etc/hadoop/log4j.properties"
''; '';
type = types.path; type = lib.types.path;
example = literalExpression '' example = lib.literalExpression ''
"''${pkgs.hadoop}/etc/hadoop/log4j.properties"; "''${pkgs.hadoop}/etc/hadoop/log4j.properties";
''; '';
description = "log4j.properties file added to HADOOP_CONF_DIR"; description = "log4j.properties file added to HADOOP_CONF_DIR";
}; };
containerExecutorCfg = mkOption { containerExecutorCfg = lib.mkOption {
default = { default = {
# must be the same as yarn.nodemanager.linux-container-executor.group in yarnSite # must be the same as yarn.nodemanager.linux-container-executor.group in yarnSite
"yarn.nodemanager.linux-container-executor.group"="hadoop"; "yarn.nodemanager.linux-container-executor.group"="hadoop";
@ -173,8 +172,8 @@ with lib;
"feature.terminal.enabled"=1; "feature.terminal.enabled"=1;
"feature.mount-cgroup.enabled" = 1; "feature.mount-cgroup.enabled" = 1;
}; };
type = types.attrsOf types.anything; type = lib.types.attrsOf lib.types.anything;
example = literalExpression '' example = lib.literalExpression ''
options.services.hadoop.containerExecutorCfg.default // { options.services.hadoop.containerExecutorCfg.default // {
"feature.terminal.enabled" = 0; "feature.terminal.enabled" = 0;
} }
@ -185,10 +184,10 @@ with lib;
''; '';
}; };
extraConfDirs = mkOption { extraConfDirs = lib.mkOption {
default = []; default = [];
type = types.listOf types.path; type = lib.types.listOf lib.types.path;
example = literalExpression '' example = lib.literalExpression ''
[ [
./extraHDFSConfs ./extraHDFSConfs
./extraYARNConfs ./extraYARNConfs
@ -197,13 +196,13 @@ with lib;
description = "Directories containing additional config files to be added to HADOOP_CONF_DIR"; description = "Directories containing additional config files to be added to HADOOP_CONF_DIR";
}; };
gatewayRole.enable = mkEnableOption "gateway role for deploying hadoop configs"; gatewayRole.enable = lib.mkEnableOption "gateway role for deploying hadoop configs";
package = mkPackageOption pkgs "hadoop" { }; package = lib.mkPackageOption pkgs "hadoop" { };
}; };
config = mkIf cfg.gatewayRole.enable { config = lib.mkIf cfg.gatewayRole.enable {
users.groups.hadoop = { users.groups.hadoop = {
gid = config.ids.gids.hadoop; gid = config.ids.gids.hadoop;
}; };