androidenv: fix autoupdate and custom XMLs (#411999)

pkgs/development/mobile/androidenv/README.md

@@ -1,15 +1,13 @@
 # How to update
 
-1. `./fetchrepo.sh`
-2. `./mkrepo.sh`
-3. Check the `repo.json` diff for new stable versions of `tools`, `platform-tools`, `build-tools`, `emulator` and/or `ndk`
-4. Update the relevant argument defaults in `compose-android-packages.nix`
+`nix-shell maintainers/scripts/update.nix  --argstr package androidenv.test-suite --argstr commit true`
 
 # How to run tests
+
 You may need to make yourself familiar with [package tests](../../../README.md#package-tests), and [Writing larger package tests](../../../README.md#writing-larger-package-tests), then run tests locally with:
 
 ```shell
 $ export NIXPKGS_ALLOW_UNFREE=1
 $ cd path/to/nixpkgs
-$ nix-build -A androidenv.test-suite.tests
+$ nix-build -A androidenv.test-suite
 ```

pkgs/development/mobile/androidenv/compose-android-packages.nix

@@ -31,7 +31,7 @@ in
     # Reads the repo JSON. If repoXmls is provided, will build a repo JSON into the Nix store.
     if repoXmls != null then
       let
-        # Uses mkrepo.rb to create a repo spec.
+        # Uses update.rb to create a repo spec.
         mkRepoJson =
           {
             packages ? [ ],
@@ -43,6 +43,7 @@ in
               ruby.withPackages (
                 pkgs: with pkgs; [
                   slop
+                  curb
                   nokogiri
                 ]
               )
@@ -68,7 +69,7 @@ in
             preferLocalBuild = true;
             unpackPhase = "true";
             buildPhase = ''
-              ruby ${./mkrepo.rb} ${lib.escapeShellArgs mkRepoRubyArguments} > repo.json
+              env ruby -e 'load "${./update.rb}"' -- ${lib.escapeShellArgs mkRepoRubyArguments} --input /dev/null --output repo.json
             '';
             installPhase = ''
               mv repo.json $out

pkgs/development/mobile/androidenv/examples/shell-ifd.nix

@@ -0,0 +1,60 @@
+{
+  # If you want to use the in-tree version of nixpkgs:
+  pkgs ? import ../../../../.. {
+    config.allowUnfree = true;
+  },
+
+  licenseAccepted ? pkgs.callPackage ../license.nix { },
+}:
+
+# Tests IFD with androidenv. Needs a folder of `../xml` in your local tree;
+# use ../fetchrepo.sh to produce it.
+let
+  androidEnv = pkgs.callPackage ./.. {
+    inherit pkgs licenseAccepted;
+  };
+
+  sdkArgs = {
+    repoXmls = {
+      packages = [ ../xml/repository2-3.xml ];
+      images = [
+        ../xml/android-sys-img2-3.xml
+        ../xml/android-tv-sys-img2-3.xml
+        ../xml/google_apis-sys-img2-3.xml
+        ../xml/google_apis_playstore-sys-img2-3.xml
+        ../xml/android-wear-sys-img2-3.xml
+        ../xml/android-wear-cn-sys-img2-3.xml
+        ../xml/android-automotive-sys-img2-3.xml
+      ];
+      addons = [ ../xml/addon2-3.xml ];
+    };
+  };
+
+  androidComposition = androidEnv.composeAndroidPackages sdkArgs;
+  androidSdk = androidComposition.androidsdk;
+  platformTools = androidComposition.platform-tools;
+  jdk = pkgs.jdk;
+in
+pkgs.mkShell {
+  name = "androidenv-example-ifd-demo";
+  packages = [
+    androidSdk
+    platformTools
+    jdk
+  ];
+
+  LANG = "C.UTF-8";
+  LC_ALL = "C.UTF-8";
+  JAVA_HOME = jdk.home;
+
+  # Note: ANDROID_HOME is deprecated. Use ANDROID_SDK_ROOT.
+  ANDROID_SDK_ROOT = "${androidSdk}/libexec/android-sdk";
+
+  shellHook = ''
+    # Write out local.properties for Android Studio.
+    cat <<EOF > local.properties
+    # This file was automatically generated by nix-shell.
+    sdk.dir=$ANDROID_SDK_ROOT
+    EOF
+  '';
+}

pkgs/development/mobile/androidenv/mkrepo.sh

@@ -1,19 +0,0 @@
-#!/usr/bin/env nix-shell
-#!nix-shell -i bash -p "ruby.withPackages (pkgs: with pkgs; [ slop nokogiri moreutils ])"
-
-set -e
-
-pushd "$(dirname "$0")" &>/dev/null || exit 1
-
-echo "Writing repo.json" >&2
-ruby mkrepo.rb \
-    --packages ./xml/repository2-3.xml \
-    --images ./xml/android-sys-img2-3.xml \
-    --images ./xml/android-tv-sys-img2-3.xml \
-    --images ./xml/android-wear-cn-sys-img2-3.xml \
-    --images ./xml/android-wear-sys-img2-3.xml \
-    --images ./xml/android-automotive-sys-img2-3.xml \
-    --images ./xml/google_apis-sys-img2-3.xml \
-    --images ./xml/google_apis_playstore-sys-img2-3.xml \
-    --addons ./xml/addon2-3.xml <./repo.json
-popd &>/dev/null

pkgs/development/mobile/androidenv/test-suite.nix

@@ -18,6 +18,7 @@ let
 in
 stdenv.mkDerivation {
   name = "androidenv-test-suite";
+  version = "1";
   buildInputs = lib.mapAttrsToList (name: value: value) all-tests;
 
   buildCommand = ''
@@ -26,9 +27,8 @@ stdenv.mkDerivation {
 
   passthru.tests = all-tests;
 
-  # This is the toplevel package, so inherit the update script
   passthru.updateScript = {
-    command = [ ./update.sh ];
+    command = [ ./update.rb ];
     attrPath = "androidenv.test-suite";
     supportedFeatures = [ "commit" ];
   };

pkgs/development/mobile/androidenv/update.rb

@@ -1,11 +1,15 @@
-#!/usr/bin/env ruby
+#!/usr/bin/env nix-shell
+#!nix-shell -i ruby -p "ruby.withPackages (ps: with ps; [ slop curb nokogiri ])"
 
 require 'json'
 require 'rubygems'
-require 'nokogiri'
-require 'slop'
 require 'shellwords'
 require 'erb'
+require 'uri'
+require 'stringio'
+require 'slop'
+require 'curb'
+require 'nokogiri'
 
 # Returns a repo URL for a given package name.
 def repo_url value
@@ -32,6 +36,36 @@ def image_url value, dir
   end
 end
 
+# Runs a GET with curl.
+def _curl_get url
+  curl = Curl::Easy.new(url) do |http|
+    http.headers['User-Agent'] = 'nixpkgs androidenv update bot'
+    yield http if block_given?
+  end
+  STDERR.print "GET #{url}"
+  curl.perform
+  STDERR.puts "... #{curl.response_code}"
+
+  StringIO.new(curl.body_str)
+end
+
+# Retrieves a repo from the filesystem or a URL.
+def get location
+  uri = URI.parse(location)
+  case uri.scheme
+  when 'repo'
+    _curl_get repo_url("#{uri.host}#{uri.fragment}.xml")
+  when 'image'
+    _curl_get image_url("sys-img#{uri.fragment}.xml", uri.host)
+  else
+    if File.exist?(uri.path)
+      File.open(uri.path, 'rt')
+    else
+      raise "Repository #{uri} was neither a file nor a repo URL"
+    end
+  end
+end
+
 # Returns a JSON with the data and structure of the input XML
 def to_json_collector doc
   json = {}
@@ -438,9 +472,19 @@ def merge dest, src
 end
 
 opts = Slop.parse do |o|
-  o.array '-p', '--packages', 'packages repo XMLs to parse'
-  o.array '-i', '--images', 'system image repo XMLs to parse'
-  o.array '-a', '--addons', 'addon repo XMLs to parse'
+  o.array '-p', '--packages', 'packages repo XMLs to parse', default: %w[repo://repository#2-3]
+  o.array '-i', '--images', 'system image repo XMLs to parse', default: %w[
+    image://android#2-3
+    image://android-tv#2-3
+    image://android-wear#2-3
+    image://android-wear-cn#2-3
+    image://android-automotive#2-3
+    image://google_apis#2-3
+    image://google_apis_playstore#2-3
+  ]
+  o.array '-a', '--addons', 'addon repo XMLs to parse', default: %w[repo://addon#2-3]
+  o.string '-I', '--input', 'input JSON file for repo', default: File.join(__dir__, 'repo.json')
+  o.string '-O', '--output', 'output JSON file for repo', default: File.join(__dir__, 'repo.json')
 end
 
 result = {}
@@ -451,20 +495,20 @@ result['addons'] = {}
 result['extras'] = {}
 
 opts[:packages].each do |filename|
-  licenses, packages, extras = parse_package_xml(Nokogiri::XML(File.open(filename)) { |conf| conf.noblanks })
+  licenses, packages, extras = parse_package_xml(Nokogiri::XML(get(filename)) { |conf| conf.noblanks })
   merge result['licenses'], licenses
   merge result['packages'], packages
   merge result['extras'], extras
 end
 
 opts[:images].each do |filename|
-  licenses, images = parse_image_xml(Nokogiri::XML(File.open(filename)) { |conf| conf.noblanks })
+  licenses, images = parse_image_xml(Nokogiri::XML(get(filename)) { |conf| conf.noblanks })
   merge result['licenses'], licenses
   merge result['images'], images
 end
 
 opts[:addons].each do |filename|
-  licenses, addons, extras = parse_addon_xml(Nokogiri::XML(File.open(filename)) { |conf| conf.noblanks })
+  licenses, addons, extras = parse_addon_xml(Nokogiri::XML(get(filename)) { |conf| conf.noblanks })
   merge result['licenses'], licenses
   merge result['addons'], addons
   merge result['extras'], extras
@@ -491,7 +535,14 @@ two_years_ago = today - 2 * 365
 input = {}
 prev_latest = {}
 begin
-  input_json = (STDIN.tty?) ? "{}" : STDIN.read
+  input_json = if File.exist?(opts[:input])
+                 STDERR.puts "Reading #{opts[:input]}"
+                 File.read(opts[:input])
+               else
+                 STDERR.puts "Creating new repo"
+                 "{}"
+               end
+
   if input_json != nil && !input_json.empty?
     input = expire_records(JSON.parse(input_json), two_years_ago)
 
@@ -511,40 +562,37 @@ fixup_result = fixup(result)
 # therefore the old packages will work as long as the links are working on the Google servers.
 output = merge input, fixup_result
 
-# See if there are any changes in the latest versions.
-cur_latest = output['latest'] || {}
-
-old_versions = []
-new_versions = []
-changes = []
-changed = false
-
-cur_latest.each do |k, v|
-  prev = prev_latest[k]
-  if prev && prev != v
-    old_versions << "#{k}:#{prev}"
-    new_versions << "#{k}:#{v}"
-    changes << "#{k}: #{prev} -> #{v}"
-    changed = true
-  end
-end
-
 # Write the repository. Append a \n to keep nixpkgs Github Actions happy.
-File.write 'repo.json', (JSON.pretty_generate(sort_recursively(output)) + "\n")
+STDERR.puts "Writing #{opts[:output]}"
+File.write opts[:output], (JSON.pretty_generate(sort_recursively(output)) + "\n")
 
 # Output metadata for the nixpkgs update script.
-changed_paths = []
-if changed
-  if ENV['UPDATE_NIX_ATTR_PATH']
-    # Instantiate it.
-    test_result = `NIXPKGS_ALLOW_UNFREE=1 NIXPKGS_ACCEPT_ANDROID_SDK_LICENSE=1 nix-build ../../../../default.nix -A #{Shellwords.join [ENV['UPDATE_NIX_ATTR_PATH']]} 2>&1`
-    test_status = $?.exitstatus
-    tests_ran = true
-  else
-    tests_ran = false
+if ENV['UPDATE_NIX_ATTR_PATH']
+  # See if there are any changes in the latest versions.
+  cur_latest = output['latest'] || {}
+
+  old_versions = []
+  new_versions = []
+  changes = []
+  changed = false
+
+  cur_latest.each do |k, v|
+    prev = prev_latest[k]
+    if prev && prev != v
+      old_versions << "#{k}:#{prev}"
+      new_versions << "#{k}:#{v}"
+      changes << "#{k}: #{prev} -> #{v}"
+      changed = true
+    end
   end
 
-  template = ERB.new(<<-EOF, trim_mode: '<>-')
+  changed_paths = []
+  if changed
+    # Instantiate it.
+    test_result = `NIXPKGS_ALLOW_UNFREE=1 NIXPKGS_ACCEPT_ANDROID_SDK_LICENSE=1 nix-build #{Shellwords.escape(File.realpath(File.join(__dir__, '..', '..', '..', '..', 'default.nix')))} -A #{Shellwords.join [ENV['UPDATE_NIX_ATTR_PATH']]} 2>&1`
+    test_status = $?.exitstatus
+
+    template = ERB.new(<<-EOF, trim_mode: '<>-')
 androidenv: <%= changes.join('; ') %>
 
 Performed the following automatic androidenv updates:
@@ -553,7 +601,6 @@ Performed the following automatic androidenv updates:
 - <%= change -%>
 <% end  %>
 
-<% if  tests_ran %>
 Tests exited with status: <%= test_status -%>
 
 <% if  !test_result.empty? %>
@@ -562,19 +609,19 @@ Last 100 lines of output:
 <%= test_result.lines.last(100).join -%>
 ```
 <% end  %>
-<% end  %>
 EOF
 
-  changed_paths << {
-    attrPath: 'androidenv.androidPkgs.androidsdk',
-    oldVersion: old_versions.join('; '),
-    newVersion: new_versions.join('; '),
-    files: [
-      File.realpath('repo.json')
-    ],
-    commitMessage: template.result(binding)
-  }
-end
+    changed_paths << {
+      attrPath: 'androidenv.androidPkgs.androidsdk',
+      oldVersion: old_versions.join('; '),
+      newVersion: new_versions.join('; '),
+      files: [
+        opts[:output]
+      ],
+      commitMessage: template.result(binding)
+    }
+  end
 
-# nix-update info is on stderr
-STDOUT.puts JSON.pretty_generate(changed_paths)
+  # nix-update info is on stdout
+  STDOUT.puts JSON.pretty_generate(changed_paths)
+end

pkgs/development/mobile/androidenv/update.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-
-pushd "$(dirname "$0")" &>/dev/null || exit 1
-./fetchrepo.sh && ./mkrepo.sh
-popd &>/dev/null