Add automatic expansion of --requirement list

pdallair · pdallair · commit 687ad6d67642 · 2021-12-27T16:22:23.000-05:00
Related to issue #2529
diff --git a/pythonforandroid/toolchain.py b/pythonforandroid/toolchain.py
@@ -5,7 +5,6 @@
 
 This module defines the entry point for command line and programmatic use.
 """
-
 from os import environ
 from pythonforandroid import __version__
 from pythonforandroid.pythonpackage import get_dep_names_of_package
@@ -651,9 +650,12 @@ def add_parser(subparsers, *args, **kwargs):
                                                 "pyproject.toml"))):
                 have_setup_py_or_similar = True
 
-        # Process requirements and put version in environ
-        if hasattr(args, 'requirements'):
-            requirements = []
+        # Process requirements and put version in environ:
+        if hasattr(args, 'requirements') and args.requirements:
+            all_recipes = [
+                recipe.lower() for recipe in
+                set(Recipe.list_recipes(self.ctx))
+            ]
 
             # Add dependencies from setup.py, but only if they are recipes
             # (because otherwise, setup.py itself will install them later)
@@ -672,10 +674,6 @@ def add_parser(subparsers, *args, **kwargs):
                         )
                     ]
                     info("Dependencies obtained: " + str(dependencies))
-                    all_recipes = [
-                        recipe.lower() for recipe in
-                        set(Recipe.list_recipes(self.ctx))
-                    ]
                     dependencies = set(dependencies).intersection(
                         set(all_recipes)
                     )
@@ -691,7 +689,126 @@ def add_parser(subparsers, *args, **kwargs):
                         "package? Will continue WITHOUT setup.py deps."
                     )
 
-            # Parse --requirements argument list:
+            non_recipe_requirements = []
+            for requirement in args.requirements.split(','):
+                requirement_name = re.sub(r'==\d+(\.\d+)*', '', requirement)
+                if requirement_name not in all_recipes:
+                    non_recipe_requirements.append(requirement)
+                    args.requirements = re.sub(
+                        r',?{}'.format(requirement), '', args.requirements)
+
+            # Compile "non-recipe" requirements' dependencies and add to list.
+            # Otherwise, only recipe requirements' dependencies get installed.
+            # More info https://github.com/kivy/python-for-android/issues/2529
+            if non_recipe_requirements:
+                info("Compiling dependencies for: "
+                     "{}".format(non_recipe_requirements))
+
+                output = shprint(
+                    sh.bash, '-c',
+                    "echo -e '{}' > requirements.in && "
+                    "pip-compile -v --dry-run --annotation-style=line && "
+                    "rm requirements.in".format(
+                        '\n'.join(non_recipe_requirements)))
+
+                # Parse pip-compile output
+                parsed_requirement_info_list = []
+                for line in output.splitlines():
+                    match_data = re.match(
+                        r'^([\w.-]+)==(\d+(\.\d+)*).*'
+                        r'#\s+via\s+([\w\s,.-]+)', line)
+
+                    if match_data:
+                        parent_requirements = match_data.group(4).split(', ')
+                        requirement_name = match_data.group(1)
+                        requirement_version = match_data.group(2)
+
+                        # Requirement is a "non-recipe" one we started with.
+                        if '-r requirements.in' in parent_requirements:
+                            parent_requirements.remove('-r requirements.in')
+
+                        parsed_requirement_info_list.append([
+                            requirement_name,
+                            requirement_version,
+                            parent_requirements])
+
+                info("Requirements obtained from pip-compile: "
+                     "{}".format(["{}=={}".format(x[0], x[1])
+                                  for x in parsed_requirement_info_list]))
+
+                # Remove indirect requirements ultimately installed by a recipe
+                original_parsed_requirement_count = -1
+                while len(parsed_requirement_info_list) != \
+                        original_parsed_requirement_count:
+
+                    original_parsed_requirement_count = \
+                        len(parsed_requirement_info_list)
+
+                    for i, parsed_requirement_info in \
+                            enumerate(reversed(parsed_requirement_info_list)):
+
+                        index = original_parsed_requirement_count - i - 1
+                        requirement_name, requirement_version, \
+                            parent_requirements = parsed_requirement_info
+
+                        # If any parent requirement has a recipe, this
+                        # requirement ought also to be installed by it.
+                        # Hence, it's better not to add this requirement the
+                        # expanded list.
+                        parent_requirements_with_recipe = list(
+                            set(parent_requirements).intersection(
+                                set(all_recipes)))
+
+                        # Any parent requirement removed for the expanded list
+                        # implies that it and its own requirements (including
+                        # this requirement) will be installed by a recipe.
+                        # Hence, it's better not to add this requirement the
+                        # expanded list.
+                        requirement_name_list = \
+                            [x[0] for x in parsed_requirement_info_list]
+                        parent_requirements_still_in_list = list(
+                            set(parent_requirements).intersection(
+                                set(requirement_name_list)))
+
+                        is_ultimately_installed_by_a_recipe = \
+                            len(parent_requirements) and \
+                            (parent_requirements_with_recipe or
+                                len(parent_requirements_still_in_list) !=
+                                len(parent_requirements))
+
+                        if is_ultimately_installed_by_a_recipe:
+                            info(
+                                '{} will be installed by a recipe.  Removing '
+                                'it from requirement list expansion.'.format(
+                                    requirement_name))
+                            del parsed_requirement_info_list[index]
+
+                for parsed_requirement_info in parsed_requirement_info_list:
+                    requirement_name, requirement_version, \
+                        parent_requirements = parsed_requirement_info
+
+                    # If the requirement has a recipe, don't use specific
+                    # version constraints determined by pip-compile.  Some
+                    # recipes may not support the specified version.  Therefor,
+                    # it's probably safer to just let them use their default
+                    # version. User can still force the usage of specific
+                    # version by explicitly declaring it with --requirements.
+                    requirement_has_recipe = requirement_name in all_recipes
+                    requirement_str = \
+                        requirement_name if requirement_has_recipe else \
+                        '{}=={}'.format(requirement_name, requirement_version)
+
+                    requirement_names_arg = re.sub(
+                        r'==\d+(\.\d+)*', '', args.requirements).split(',')
+
+                    # This expansion was carried out based on "non-recipe"
+                    # requirements.  Hence,the counter-part, requirements
+                    # with a recipe, may already be part of list.
+                    if requirement_name not in requirement_names_arg:
+                        args.requirements += ',' + requirement_str
+
+            # Handle specific version requirement constraints (e.g. foo==x.y)
+            requirements = []
             for requirement in split_argument_list(args.requirements):
                 if "==" in requirement:
                     requirement, version = requirement.split(u"==", 1)
@@ -701,6 +818,9 @@ def add_parser(subparsers, *args, **kwargs):
                 requirements.append(requirement)
             args.requirements = u",".join(requirements)
 
+            info('Expanded Requirements List: '
+                 '{}'.format(args.requirements.split(',')))
+
         self.warn_on_deprecated_args(args)
 
         self.storage_dir = args.storage_dir
diff --git a/setup.py b/setup.py
@@ -22,7 +22,7 @@
 install_reqs = [
     'appdirs', 'colorama>=0.3.3', 'jinja2', 'six',
     'enum34; python_version<"3.4"', 'sh>=1.10; sys_platform!="nt"',
-    'pep517<0.7.0', 'toml',
+    'pep517<0.7.0', 'toml', 'pip-tools'
 ]
 # (pep517 and toml are used by pythonpackage.py)
 
diff --git a/tests/test_toolchain.py b/tests/test_toolchain.py
@@ -1,4 +1,5 @@
 import io
+import os
 import sys
 import pytest
 from unittest import mock
@@ -124,6 +125,94 @@ def test_create_no_sdk_dir(self):
         assert ex_info.value.message == (
             'Android SDK dir was not specified, exiting.')
 
+    def test_create_with_complex_requirements(self):
+        requirements = [
+            'python3==3.8.10',  # direct requirement with recipe using version constraint
+            'pandas',           # direct requirement with recipe (no version constraint)
+            'mfpymake==1.2.2',  # direct requirement without recipe using version constraint
+            'telenium',         # direct requirement without recipe (no version constraint)
+            'numpy==1.21.4',    # indirect requirement with recipe using version constraint
+            'mako==1.1.5',      # indirect requirement without recipe using version constraint
+            # There's no reason to specify an indirect requirement unless we want to install a specific version.
+        ]
+        argv = [
+            'toolchain.py',
+            'create',
+            '--sdk-dir=/tmp/android-sdk',
+            '--ndk-dir=/tmp/android-ndk',
+            '--bootstrap=service_only',
+            '--requirements={}'.format(','.join(requirements)),
+            '--dist-name=test_toolchain',
+            '--activity-class-name=abc.myapp.android.CustomPythonActivity',
+            '--service-class-name=xyz.myapp.android.CustomPythonService',
+        ]
+        with patch_sys_argv(argv), mock.patch(
+                'pythonforandroid.build.get_available_apis'
+        ) as m_get_available_apis, mock.patch(
+            'pythonforandroid.build.get_toolchain_versions'
+        ) as m_get_toolchain_versions, mock.patch(
+            'pythonforandroid.build.get_ndk_platform_dir'
+        ) as m_get_ndk_platform_dir, mock.patch(
+            'pythonforandroid.toolchain.build_recipes'
+        ) as m_build_recipes, mock.patch(
+            'pythonforandroid.bootstraps.service_only.'
+            'ServiceOnlyBootstrap.assemble_distribution'
+        ) as m_run_distribute:
+            m_get_available_apis.return_value = [27]
+            m_get_toolchain_versions.return_value = (['4.9'], True)
+            m_get_ndk_platform_dir.return_value = (
+                '/tmp/android-ndk/platforms/android-21/arch-arm', True)
+            tchain = ToolchainCL()
+            assert tchain.ctx.activity_class_name == 'abc.myapp.android.CustomPythonActivity'
+            assert tchain.ctx.service_class_name == 'xyz.myapp.android.CustomPythonService'
+        assert m_get_available_apis.call_args_list in [
+            [mock.call('/tmp/android-sdk')],  # linux case
+            [mock.call('/private/tmp/android-sdk')]  # macos case
+        ]
+        assert m_get_toolchain_versions.call_args_list in [
+            [mock.call('/tmp/android-ndk', mock.ANY)],  # linux case
+            [mock.call('/private/tmp/android-ndk', mock.ANY)],  # macos case
+        ]
+        build_order = [
+            'android', 'cython', 'genericndkbuild', 'hostpython3', 'libbz2',
+            'libffi', 'liblzma', 'numpy', 'openssl', 'pandas', 'pyjnius',
+            'python3', 'pytz', 'setuptools', 'six', 'sqlite3'
+        ]
+        python_modules = [
+            'certifi', 'charset-normalizer', 'cheroot', 'cherrypy',
+            'idna', 'importlib-resources', 'jaraco.classes',
+            'jaraco.collections', 'jaraco.functools', 'jaraco.text',
+            'json-rpc', 'mako', 'markupsafe', 'mfpymake', 'more-itertools',
+            'networkx', 'portend', 'python-dateutil', 'requests', 'telenium',
+            'tempora', 'urllib3', 'werkzeug', 'ws4py', 'zc.lockfile', 'zipp'
+        ]
+        context = mock.ANY
+        project_dir = None
+        # The pip-compile tool used to expanded the list of requirements
+        # doesn't always return results in the same order.
+        # _Call object and its properties are immutable, so we create a
+        # new one with the same values.
+        m_build_recipes_call_args = mock.call(
+            sorted(m_build_recipes.call_args_list[0][0][0]),
+            sorted(m_build_recipes.call_args_list[0][0][1]),
+            m_build_recipes.call_args_list[0][0][2],
+            m_build_recipes.call_args_list[0][0][3],
+            ignore_project_setup_py=m_build_recipes.call_args_list[0][1]['ignore_project_setup_py']
+        )
+        assert m_build_recipes_call_args == mock.call(
+            sorted(build_order),
+            sorted(python_modules),
+            context,
+            project_dir,
+            ignore_project_setup_py=False
+        )
+        assert m_run_distribute.call_args_list == [mock.call()]
+        assert 'VERSION_python3' in os.environ and os.environ['VERSION_python3'] == '3.8.10'
+        assert 'VERSION_mfpymake' in os.environ and os.environ['VERSION_mfpymake'] == '1.2.2'
+        assert 'VERSION_numpy' in os.environ and os.environ['VERSION_numpy'] == '1.21.4'
+        assert 'VERSION_mako' in os.environ and os.environ['VERSION_mako'] == '1.1.5'
+
+
     @pytest.mark.skipif(sys.version_info < (3, 0), reason="requires python3")
     def test_recipes(self):
         """

Original file line number	Diff line number	Diff line change
`@@ -22,7 +22,7 @@`
`22`	`22`	`install_reqs = [`
`23`	`23`	`'appdirs', 'colorama>=0.3.3', 'jinja2', 'six',`
`24`	`24`	`'enum34; python_version<"3.4"', 'sh>=1.10; sys_platform!="nt"',`
`25`		`- 'pep517<0.7.0', 'toml',`
	`25`	`+ 'pep517<0.7.0', 'toml', 'pip-tools'`
`26`	`26`	`]`
`27`	`27`	`# (pep517 and toml are used by pythonpackage.py)`
`28`	`28`