Merge pull request #47747 from florianjacob/matomo-archive-processing-service

Matomo archive processing service

nixos/doc/manual/release-notes/rl-1903.xml

@@ -412,6 +412,23 @@
      <option>services.matomo.package</option> which determines the used
      Matomo version.
     </para>
+    <para>
+     The Matomo module now also comes with the systemd service <literal>matomo-archive-processing.service</literal>
+     and a timer that automatically triggers archive processing every hour.
+     This means that you can safely
+     <link xlink:href="https://matomo.org/docs/setup-auto-archiving/#disable-browser-triggers-for-matomo-archiving-and-limit-matomo-reports-to-updating-every-hour">
+      disable browser triggers for Matomo archiving
+     </link> at <literal>Administration > System > General Settings</literal>.
+    </para>
+    <para>
+     Additionally, you can enable to
+     <link xlink:href="https://matomo.org/docs/privacy/#step-2-delete-old-visitors-logs">
+      delete old visitor logs
+     </link> at <literal>Administration > System > Privacy</literal>,
+     but make sure that you run <literal>systemctl start matomo-archive-processing.service</literal>
+     at least once without errors if you have already collected data before,
+     so that the reports get archived before the source data gets deleted.
+    </para>
    </listitem>
    <listitem>
     <para>

nixos/modules/services/web-apps/matomo-doc.xml

@@ -12,15 +12,15 @@
   An automatic setup is not suported by Matomo, so you need to configure Matomo
   itself in the browser-based Matomo setup.
  </para>
+
  <section xml:id="module-services-matomo-database-setup">
   <title>Database Setup</title>
-
   <para>
    You also need to configure a MariaDB or MySQL database and -user for Matomo
    yourself, and enter those credentials in your browser. You can use
    passwordless database authentication via the UNIX_SOCKET authentication
    plugin with the following SQL commands:
-<programlisting>
+   <programlisting>
         # For MariaDB
         INSTALL PLUGIN unix_socket SONAME 'auth_socket';
         CREATE DATABASE matomo;
@@ -32,7 +32,7 @@
         CREATE DATABASE matomo;
         CREATE USER 'matomo'@'localhost' IDENTIFIED WITH auth_socket;
         GRANT ALL PRIVILEGES ON matomo.* TO 'matomo'@'localhost';
-      </programlisting>
+   </programlisting>
    Then fill in <literal>matomo</literal> as database user and database name,
    and leave the password field blank. This authentication works by allowing
    only the <literal>matomo</literal> unix user to authenticate as the
@@ -46,9 +46,30 @@
    database is not on the same host.
   </para>
  </section>
+
+ <section>
+  <title>Archive Processing</title>
+  <para>
+   This module comes with the systemd service <literal>matomo-archive-processing.service</literal>
+   and a timer that automatically triggers archive processing every hour.
+   This means that you can safely
+   <link xlink:href="https://matomo.org/docs/setup-auto-archiving/#disable-browser-triggers-for-matomo-archiving-and-limit-matomo-reports-to-updating-every-hour">
+    disable browser triggers for Matomo archiving
+   </link> at <literal>Administration > System > General Settings</literal>.
+  </para>
+  <para>
+   With automatic archive processing, you can now also enable to
+   <link xlink:href="https://matomo.org/docs/privacy/#step-2-delete-old-visitors-logs">
+    delete old visitor logs
+   </link> at <literal>Administration > System > Privacy</literal>,
+   but make sure that you run <literal>systemctl start matomo-archive-processing.service</literal>
+   at least once without errors if you have already collected data before,
+   so that the reports get archived before the source data gets deleted.
+  </para>
+ </section>
+
  <section xml:id="module-services-matomo-backups">
   <title>Backup</title>
-
   <para>
    You only need to take backups of your MySQL database and the
    <filename>/var/lib/matomo/config/config.ini.php</filename> file. Use a user
@@ -57,9 +78,9 @@
    <link xlink:href="https://matomo.org/faq/how-to-install/faq_138/" />.
   </para>
  </section>
+
  <section xml:id="module-services-matomo-issues">
   <title>Issues</title>
-
   <itemizedlist>
    <listitem>
     <para>
@@ -76,6 +97,7 @@
    </listitem>
   </itemizedlist>
  </section>
+
  <section xml:id="module-services-matomo-other-web-servers">
   <title>Using other Web Servers than nginx</title>
 

nixos/modules/services/web-apps/matomo.nix

@@ -23,20 +23,24 @@ in {
   options = {
     services.matomo = {
       # NixOS PR for database setup: https://github.com/NixOS/nixpkgs/pull/6963
-      # matomo issue for automatic matomo setup: https://github.com/matomo-org/matomo/issues/10257
+      # Matomo issue for automatic Matomo setup: https://github.com/matomo-org/matomo/issues/10257
-      # TODO: find a nice way to do this when more NixOS MySQL and / or matomo automatic setup stuff is implemented.
+      # TODO: find a nice way to do this when more NixOS MySQL and / or Matomo automatic setup stuff is implemented.
       enable = mkOption {
         type = types.bool;
         default = false;
         description = ''
-          Enable matomo web analytics with php-fpm backend.
+          Enable Matomo web analytics with php-fpm backend.
           Either the nginx option or the webServerUser option is mandatory.
         '';
       };
 
       package = mkOption {
         type = types.package;
-        description = "Matomo package to use";
+        description = ''
+          Matomo package for the service to use.
+          This can be used to point to newer releases from nixos-unstable,
+          as they don't get backported if they are not security-relevant.
+        '';
         default = pkgs.matomo;
         defaultText = "pkgs.matomo";
       };
@@ -47,13 +51,27 @@ in {
         example = "lighttpd";
         # TODO: piwik.php might get renamed to matomo.php in future releases
         description = ''
-          Name of the web server user that forwards requests to the ${phpSocket} fastcgi socket for matomo if the nginx
+          Name of the web server user that forwards requests to the ${phpSocket} fastcgi socket for Matomo if the nginx
           option is not used. Either this option or the nginx option is mandatory.
           If you want to use another webserver than nginx, you need to set this to that server's user
           and pass fastcgi requests to `index.php` and `piwik.php` to this socket.
         '';
       };
 
+      periodicArchiveProcessing = mkOption {
+        type = types.bool;
+        default = true;
+        description = ''
+          Enable periodic archive processing, which generates aggregated reports from the visits.
+
+          This means that you can safely disable browser triggers for Matomo archiving,
+          and safely enable to delete old visitor logs.
+          Before deleting visitor logs,
+          make sure though that you run <literal>systemctl start matomo-archive-processing.service</literal>
+          at least once without errors if you have already collected data before.
+        '';
+      };
+
       phpfpmProcessManagerConfig = mkOption {
         type = types.str;
         default = ''
@@ -69,7 +87,7 @@ in {
           catch_workers_output = yes
         '';
         description = ''
-          Settings for phpfpm's process manager. You might need to change this depending on the load for matomo.
+          Settings for phpfpm's process manager. You might need to change this depending on the load for Matomo.
         '';
       };
 
@@ -79,7 +97,7 @@ in {
             (import ../web-servers/nginx/vhost-options.nix { inherit config lib; })
             {
               # enable encryption by default,
-              # as sensitive login and matomo data should not be transmitted in clear text.
+              # as sensitive login and Matomo data should not be transmitted in clear text.
               options.forceSSL.default = true;
               options.enableACME.default = true;
             }
@@ -94,7 +112,7 @@ in {
           enableACME = false;
         };
         description = ''
-            With this option, you can customize an nginx virtualHost which already has sensible defaults for matomo.
+            With this option, you can customize an nginx virtualHost which already has sensible defaults for Matomo.
             Either this option or the webServerUser option is mandatory.
             Set this to {} to just enable the virtualHost if you don't need any customization.
             If enabled, then by default, the <option>serverName</option> is
@@ -124,29 +142,30 @@ in {
     };
     users.groups.${user} = {};
 
-    systemd.services.matomo_setup_update = {
+    systemd.services.matomo-setup-update = {
-      # everything needs to set up and up to date before matomo php files are executed
+      # everything needs to set up and up to date before Matomo php files are executed
       requiredBy = [ "${phpExecutionUnit}.service" ];
       before = [ "${phpExecutionUnit}.service" ];
       # the update part of the script can only work if the database is already up and running
       requires = [ databaseService ];
       after = [ databaseService ];
       path = [ cfg.package ];
+      environment.PIWIK_USER_PATH = dataDir;
       serviceConfig = {
         Type = "oneshot";
         User = user;
         # hide especially config.ini.php from other
         UMask = "0007";
         # TODO: might get renamed to MATOMO_USER_PATH in future versions
-        Environment = "PIWIK_USER_PATH=${dataDir}";
         # chown + chmod in preStart needs root
         PermissionsStartOnly = true;
       };
+
       # correct ownership and permissions in case they're not correct anymore,
       # e.g. after restoring from backup or moving from another system.
       # Note that ${dataDir}/config/config.ini.php might contain the MySQL password.
       preStart = ''
-        # migrate data from piwik to matomo folder
+        # migrate data from piwik to Matomo folder
         if [ -d ${deprecatedDataDir} ]; then
           echo "Migrating from ${deprecatedDataDir} to ${dataDir}"
           mv -T ${deprecatedDataDir} ${dataDir}
@@ -155,7 +174,7 @@ in {
         chmod -R ug+rwX,o-rwx ${dataDir}
         '';
       script = ''
-            # Use User-Private Group scheme to protect matomo data, but allow administration / backup via matomo group
+            # Use User-Private Group scheme to protect Matomo data, but allow administration / backup via 'matomo' group
             # Copy config folder
             chmod g+s "${dataDir}"
             cp -r "${cfg.package}/config" "${dataDir}/"
@@ -169,8 +188,39 @@ in {
       '';
     };
 
+    # If this is run regularly via the timer,
+    # 'Browser trigger archiving' can be disabled in Matomo UI > Settings > General Settings.
+    systemd.services.matomo-archive-processing = {
+      description = "Archive Matomo reports";
+      # the archiving can only work if the database is already up and running
+      requires = [ databaseService ];
+      after = [ databaseService ];
+
+      # TODO: might get renamed to MATOMO_USER_PATH in future versions
+      environment.PIWIK_USER_PATH = dataDir;
+      serviceConfig = {
+        Type = "oneshot";
+        User = user;
+        UMask = "0007";
+        CPUSchedulingPolicy = "idle";
+        IOSchedulingClass = "idle";
+        ExecStart = "${cfg.package}/bin/matomo-console core:archive --url=https://${user}.${fqdn}";
+      };
+    };
+
+    systemd.timers.matomo-archive-processing = mkIf cfg.periodicArchiveProcessing {
+      description = "Automatically archive Matomo reports every hour";
+
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = "hourly";
+        Persistent = "yes";
+        AccuracySec = "10m";
+      };
+    };
+
     systemd.services.${phpExecutionUnit} = {
-      # stop phpfpm on package upgrade, do database upgrade via matomo_setup_update, and then restart
+      # stop phpfpm on package upgrade, do database upgrade via matomo-setup-update, and then restart
       restartTriggers = [ cfg.package ];
       # stop config.ini.php from getting written with read permission for others
       serviceConfig.UMask = "0007";
@@ -200,13 +250,13 @@ in {
       # https://fralef.me/piwik-hardening-with-nginx-and-php-fpm.html
       # https://github.com/perusio/piwik-nginx
       "${user}.${fqdn}" = mkMerge [ cfg.nginx {
-        # don't allow to override the root easily, as it will almost certainly break matomo.
+        # don't allow to override the root easily, as it will almost certainly break Matomo.
         # disadvantage: not shown as default in docs.
         root = mkForce "${cfg.package}/share";
 
         # define locations here instead of as the submodule option's default
         # so that they can easily be extended with additional locations if required
-        # without needing to redefine the matomo ones.
+        # without needing to redefine the Matomo ones.
         # disadvantage: not shown as default in docs.
         locations."/" = {
           index = "index.php";