From 624602ef7562e2c5b7e4f8826a59a8fd55d56183 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Thu, 9 Jun 2016 02:17:18 +0200 Subject: [PATCH 1/3] python-lti: 0.4.0 -> 0.4.1 --- pkgs/top-level/python-packages.nix | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index d52e2ddeec0e..783de4def10f 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -6952,19 +6952,22 @@ in modules // { }; }; - lti = buildPythonPackage rec { - version = "0.4.0"; + lti = let self' = (self.override {self = self';}) // {pytest = self.pytest_27;}; + in buildPythonPackage rec { + version = "0.4.1"; name = "PyLTI-${version}"; + disabled = !isPy27; + propagatedBuildInputs = with self; [ httplib2 oauth oauth2 semantic-version ]; - buildInputs = with self; [ + buildInputs = with self'; [ flask httpretty oauthlib pyflakes pytest pytestcache pytestcov covCore pytestflakes pytestpep8 sphinx mock ]; src = pkgs.fetchurl { url = "mirror://pypi/P/PyLTI/${name}.tar.gz"; - sha256 = "1lkk6qx8yfx1h0rhi4abnd44x0wakggi6zs0nvi572lajf6ydmdh"; + sha256 = "076llj10j85zw3zq2gygx2pcfqi9rgcld5m4vq1iai1fk15x60fz"; }; meta = { From a48796ab831f564cd83fff2a3ce6e46114a6cb80 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Thu, 9 Jun 2016 02:18:59 +0200 Subject: [PATCH 2/3] inginious: turn lib into a python app. --- pkgs/servers/inginious/default.nix | 71 ++++++++++++++++++++++++++++++ pkgs/top-level/all-packages.nix | 2 + pkgs/top-level/python-packages.nix | 64 --------------------------- 3 files changed, 73 insertions(+), 64 deletions(-) create mode 100644 pkgs/servers/inginious/default.nix diff --git a/pkgs/servers/inginious/default.nix b/pkgs/servers/inginious/default.nix new file mode 100644 index 000000000000..ba6a54fc973d --- /dev/null +++ b/pkgs/servers/inginious/default.nix @@ -0,0 +1,71 @@ +{ pkgs, lib, pythonPackages }: +with lib; + +let + docker_1_7_2 = pythonPackages.docker.override rec { + name = "docker-py-1.7.2"; + + src = pkgs.fetchurl { + url = "mirror://pypi/d/docker-py/${name}.tar.gz"; + sha256 = "0k6hm3vmqh1d3wr9rryyif5n4rzvcffdlb1k4jvzp7g4996d3ccm"; + }; + }; + + webpy-custom = pythonPackages.web.override { + name = "web.py-INGI"; + src = pkgs.fetchFromGitHub { + owner = "UCL-INGI"; + repo = "webpy-INGI"; + # tip of branch "ingi" + rev = "f487e78d65d6569eb70003e588d5c6ace54c384f"; + sha256 = "159vwmb8554xk98rw380p3ah170r6gm861r1nqf2l452vvdfxscd"; + }; + }; + +in pythonPackages.buildPythonApplication rec { + version = "0.3a2.dev0"; + name = "inginious-${version}"; + + disabled = pythonPackages.isPy3k; + + patchPhase = '' + # transient failures + substituteInPlace inginious/backend/tests/TestRemoteAgent.py \ + --replace "test_update_task_directory" "noop" + ''; + + propagatedBuildInputs = with pythonPackages; [ + requests2 + cgroup-utils docker_1_7_2 docutils lti mock pygments + pymongo pyyaml rpyc sh simpleldap sphinx_rtd_theme tidylib + websocket_client watchdog webpy-custom flup + ]; + + buildInputs = with pythonPackages; [ nose selenium virtual-display ]; + + /* Hydra fix exists only on github for now. + src = pkgs.fetchurl { + url = "mirror://pypi/I/INGInious/INGInious-${version}.tar.gz"; + }; + */ + src = pkgs.fetchFromGitHub { + owner = "UCL-INGI"; + repo = "INGInious"; + rev = "07d111c0a3045c7cc4e464d4adb8aa28b75a6948"; + sha256 = "0kldbkc9yw1mgg5w5q5v8k2hz089c5c4rvxb5xhbagkzgm2gn230"; + }; + + # Only patch shebangs in /bin, other scripts are run within docker + # containers and will fail if patched. + dontPatchShebangs = true; + preFixup = '' + patchShebangs $prefix/bin + ''; + + meta = { + description = "An intelligent grader that allows secured and automated testing of code made by students"; + homepage = "https://github.com/UCL-INGI/INGInious"; + license = licenses.agpl3; + maintainers = with maintainers; [ layus ]; + }; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 3097ac832654..7145aa06ba26 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -13094,6 +13094,8 @@ in inferno = callPackage_i686 ../applications/inferno { }; + inginious = callPackage ../servers/inginious {}; + inkscape = callPackage ../applications/graphics/inkscape { inherit (pythonPackages) python pyxml lxml numpy; lcms = lcms2; diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 783de4def10f..7346435f3c49 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -11084,70 +11084,6 @@ in modules // { }; }; - inginious = let - # patched version of docker bindings. - docker-custom = self.docker.override { - name = "docker-1.3.0-dirty"; - src = pkgs.fetchFromGitHub { - owner = "GuillaumeDerval"; - repo = "docker-py"; - # tip of branch "master" - rev = "966becd0af514e67de5afbf885257a5005e49626"; - sha256 = "09k41dh86cbb7z4b8926fi5b2qq670mm6agl5py3giacakrap66c"; - }; - }; - - webpy-custom = self.web.override { - name = "web.py-INGI"; - src = pkgs.fetchFromGitHub { - owner = "UCL-INGI"; - repo = "webpy-INGI"; - # tip of branch "ingi" - rev = "f487e78d65d6569eb70003e588d5c6ace54c384f"; - sha256 = "159vwmb8554xk98rw380p3ah170r6gm861r1nqf2l452vvdfxscd"; - }; - }; - in buildPythonPackage rec { - version = "0.3a2.dev0"; - name = "inginious-${version}"; - - disabled = isPy3k; - - patchPhase = '' - # transient failures - substituteInPlace inginious/backend/tests/TestRemoteAgent.py \ - --replace "test_update_task_directory" "noop" - ''; - - propagatedBuildInputs = with self; [ - requests2 - cgroup-utils docker-custom docutils lti mock pygments - pymongo pyyaml rpyc sh simpleldap sphinx_rtd_theme tidylib - websocket_client watchdog webpy-custom - ]; - - buildInputs = with self; [ nose selenium virtual-display ]; - - /* Hydra fix exists only on github for now. - src = pkgs.fetchurl { - url = "mirror://pypi/I/INGInious/INGInious-${version}.tar.gz"; - md5 = "40474dd6b6d4fc26e47a1d9c77bcf943"; - }; - */ - src = pkgs.fetchFromGitHub { - owner = "UCL-INGI"; - repo = "INGInious"; - rev = "e019a0e28c442b4201ec4a0be2a816c4ab639683"; - sha256 = "1pwbm7f7xn50rxzwrqpji58n2ami5r3lgbdpb61q0w3dwkxvvvfk"; - }; - - meta = { - description = "An intelligent grader that allows secured and automated testing of code made by students"; - homepage = "https://github.com/UCL-INGI/INGInious"; - license = licenses.agpl3; - maintainers = with maintainers; [ layus ]; - }; - }; interruptingcow = buildPythonPackage rec { name = "interruptingcow-${version}"; From 0fef9ed3ed5443371242c3427fdd6d9baacdc4cf Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Thu, 9 Jun 2016 02:19:50 +0200 Subject: [PATCH 3/3] inginious: init NixOS module --- nixos/modules/module-list.nix | 1 + .../web-servers/lighttpd/inginious.nix | 262 ++++++++++++++++++ 2 files changed, 263 insertions(+) create mode 100644 nixos/modules/services/web-servers/lighttpd/inginious.nix diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 509634cf34c1..2858b05000c9 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -455,6 +455,7 @@ ./services/web-servers/lighttpd/cgit.nix ./services/web-servers/lighttpd/default.nix ./services/web-servers/lighttpd/gitweb.nix + ./services/web-servers/lighttpd/inginious.nix ./services/web-servers/nginx/default.nix ./services/web-servers/phpfpm.nix ./services/web-servers/shellinabox.nix diff --git a/nixos/modules/services/web-servers/lighttpd/inginious.nix b/nixos/modules/services/web-servers/lighttpd/inginious.nix new file mode 100644 index 000000000000..43deccb6aef8 --- /dev/null +++ b/nixos/modules/services/web-servers/lighttpd/inginious.nix @@ -0,0 +1,262 @@ +{ config, lib, pkgs, ... }: +with lib; + +let + cfg = config.services.lighttpd.inginious; + inginious = pkgs.inginious; + execName = "inginious-${if cfg.useLTI then "lti" else "webapp"}"; + + inginiousConfigFile = if cfg.configFile != null then cfg.configFile else pkgs.writeText "inginious.yaml" '' + # Backend; can be: + # - "local" (run containers on the same machine) + # - "remote" (connect to distant docker daemon and auto start agents) (choose this if you use boot2docker) + # - "remote_manual" (connect to distant and manually installed agents) + backend: "${cfg.backendType}" + + ## TODO (maybe): Add an option for the "remote" backend in this NixOS module. + # List of remote docker daemon to which the backend will try + # to connect (backend: remote only) + #docker_daemons: + # - # Host of the docker daemon *from the webapp* + # remote_host: "some.remote.server" + # # Port of the distant docker daemon *from the webapp* + # remote_docker_port: "2375" + # # A mandatory port used by the backend and the agent that will be automatically started. + # # Needs to be available on the remote host, and to be open in the firewall. + # remote_agent_port: "63456" + # # Does the remote docker requires tls? Defaults to false. + # # Parameter can be set to true or path to the certificates + # #use_tls: false + # # Link to the docker daemon *from the host that runs the docker daemon*. Defaults to: + # #local_location: "unix:///var/run/docker.sock" + # # Path to the cgroups "mount" *from the host that runs the docker daemon*. Defaults to: + # #cgroups_location: "/sys/fs/cgroup" + # # Name that will be used to reference the agent + # #"agent_name": "inginious-agent" + + # List of remote agents to which the backend will try + # to connect (backend: remote_manual only) + # Example: + #agents: + # - host: "192.168.59.103" + # port: 5001 + agents: + ${lib.concatMapStrings (agent: + " - host: \"${agent.host}\"\n" + + " port: ${agent.port}\n" + ) cfg.remoteAgents} + + # Location of the task directory + tasks_directory: "${cfg.tasksDirectory}" + + # Super admins: list of user names that can do everything in the backend + superadmins: + ${lib.concatMapStrings (x: " - \"${x}\"\n") cfg.superadmins} + + # Aliases for containers + # Only containers listed here can be used by tasks + containers: + ${lib.concatStrings (lib.mapAttrsToList (name: fullname: + " ${name}: \"${fullname}\"\n" + ) cfg.containers)} + + # Use single minified javascript file (production) or multiple files (dev) ? + use_minified_js: true + + ## TODO (maybe): Add NixOS options for these parameters. + + # MongoDB options + #mongo_opt: + # host: localhost + # database: INGInious + + # Disable INGInious? + #maintenance: false + + #smtp: + # sendername: 'INGInious ' + # host: 'smtp.gmail.com' + # port: 587 + # username: 'configme@gmail.com' + # password: 'secret' + # starttls: True + + ## NixOS extra config + + ${cfg.extraConfig} + ''; +in +{ + options.services.lighttpd.inginious = { + enable = mkEnableOption "INGInious, an automated code testing and grading system."; + + configFile = mkOption { + type = types.nullOr types.path; + default = null; + example = literalExample ''pkgs.writeText "configuration.yaml" "# custom config options ...";''; + description = ''The path to an INGInious configuration file.''; + }; + + extraConfig = mkOption { + type = types.lines; + default = ""; + example = '' + # Load the dummy auth plugin. + plugins: + - plugin_module: inginious.frontend.webapp.plugins.auth.demo_auth + users: + # register the user "test" with the password "someverycomplexpassword" + test: someverycomplexpassword + ''; + description = ''Extra option in YaML format, to be appended to the config file.''; + }; + + tasksDirectory = mkOption { + type = types.path; + default = "${inginious}/lib/python2.7/site-packages/inginious/tasks"; + example = "/var/lib/INGInious/tasks"; + description = '' + Path to the tasks folder. + Defaults to the provided test tasks folder (readonly). + ''; + }; + + useLTI = mkOption { + type = types.bool; + default = false; + description = ''Whether to start the LTI frontend in place of the webapp.''; + }; + + superadmins = mkOption { + type = types.uniq (types.listOf types.str); + default = [ "admin" ]; + example = [ "john" "pepe" "emilia" ]; + description = ''List of user logins allowed to administrate the whole server.''; + }; + + containers = mkOption { + type = types.attrsOf types.str; + default = { + default = "ingi/inginious-c-default"; + }; + example = { + default = "ingi/inginious-c-default"; + sekexe = "ingi/inginious-c-sekexe"; + java = "ingi/inginious-c-java"; + oz = "ingi/inginious-c-oz"; + pythia1compat = "ingi/inginious-c-pythia1compat"; + }; + description = '' + An attrset describing the required containers + These containers will be available in INGInious using their short name (key) + and will be automatically downloaded before INGInious starts. + ''; + }; + + hostPattern = mkOption { + type = types.str; + default = "^inginious."; + example = "^inginious.mydomain.xyz$"; + description = '' + The domain that serves INGInious. + INGInious uses absolute paths which makes it difficult to relocate in its own subdir. + The default configuration will serve INGInious when the server is accessed with a hostname starting with "inginious.". + If left blank, INGInious will take the precedence over all the other lighttpd sites, which is probably not what you want. + ''; + }; + + backendType = mkOption { + type = types.enum [ "local" "remote_manual" ]; # TODO: support backend "remote" + default = "local"; + description = '' + Select how INGINious accesses to grading containers. + The default "local" option ensures that Docker is started and provisioned. + Fore more information, see http://inginious.readthedocs.io/en/latest/install_doc/config_reference.html + Not all backends are supported. Use services.inginious.configFile for full flexibility. + ''; + }; + + remoteAgents = mkOption { + type = types.listOf (types.attrsOf types.str); + default = []; + example = [ { host = "192.0.2.25"; port = "1345"; } ]; + description = ''A list of remote agents, used only when services.inginious.backendType is "remote_manual".''; + }; + }; + + config = mkIf cfg.enable ( + mkMerge [ + # For a local install, we need docker. + (mkIf (cfg.backendType == "local") { + virtualisation.docker = { + enable = true; + # We need docker to listen on port 2375. + extraOptions = "-H tcp://127.0.0.1:2375 -H unix:///var/run/docker.sock"; + storageDriver = mkDefault "overlay"; + socketActivation = false; + }; + + users.extraUsers."lighttpd".extraGroups = [ "docker" ]; + + # Ensure that docker has pulled the required images. + systemd.services.inginious-prefetch = { + script = let + images = lib.unique ( + [ "centos" "ingi/inginious-agent" ] + ++ lib.mapAttrsToList (_: image: image) cfg.containers + ); + in lib.concatMapStrings (image: '' + ${pkgs.docker}/bin/docker pull ${image} + '') images; + + serviceConfig.Type = "oneshot"; + wants = [ "docker.service" ]; + after = [ "docker.service" ]; + wantedBy = [ "lighttpd.service" ]; + before = [ "lighttpd.service" ]; + }; + }) + + # Common + { + # To access inginous tools (like inginious-test-task) + environment.systemPackages = [ inginious ]; + + services.mongodb.enable = true; + + services.lighttpd.enable = true; + services.lighttpd.enableModules = [ "mod_access" "mod_alias" "mod_fastcgi" "mod_redirect" "mod_rewrite" ]; + services.lighttpd.extraConfig = '' + $HTTP["host"] =~ "${cfg.hostPattern}" { + fastcgi.server = ( "/${execName}" => + (( + "socket" => "/run/lighttpd/inginious-fastcgi.socket", + "bin-path" => "${inginious}/bin/${execName} --config=${inginiousConfigFile}", + "max-procs" => 1, + "bin-environment" => ( "REAL_SCRIPT_NAME" => "" ), + "check-local" => "disable" + )) + ) + url.rewrite-once = ( + "^/.well-known/.*" => "$0", + "^/static/.*" => "$0", + "^/.*$" => "/${execName}$0", + "^/favicon.ico$" => "/static/common/favicon.ico", + ) + alias.url += ( + "/static/webapp/" => "${inginious}/lib/python2.7/site-packages/inginious/frontend/webapp/static/", + "/static/common/" => "${inginious}/lib/python2.7/site-packages/inginious/frontend/common/static/" + ) + } + ''; + + systemd.services.lighttpd.preStart = '' + mkdir -p /run/lighttpd + chown lighttpd.lighttpd /run/lighttpd + ''; + + systemd.services.lighttpd.wants = [ "mongodb.service" "docker.service" ]; + systemd.services.lighttpd.after = [ "mongodb.service" "docker.service" ]; + } + ]); +}