gh-100320: Fix path calculations on Windows when python.exe is moved outside of the normal location (GH-100947)

Author: zooba

Lib/test/test_embed.py

@@ -709,8 +709,10 @@ def check_config(self, configs, expected):
         if MS_WINDOWS:
             value = config.get(key := 'program_name')
             if value and isinstance(value, str):
-                ext = '_d.exe' if debug_build(sys.executable) else '.exe'
-                config[key] = value[:len(value.lower().removesuffix(ext))]
+                value = value[:len(value.lower().removesuffix('.exe'))]
+                if debug_build(sys.executable):
+                    value = value[:len(value.lower().removesuffix('_d'))]
+                config[key] = value
         for key, value in list(expected.items()):
             if value is self.IGNORE_CONFIG:
                 config.pop(key, None)
@@ -1283,7 +1285,7 @@ def test_init_setpythonhome(self):
             stdlib = os.path.join(home, "Lib")
             # Because we are specifying 'home', module search paths
             # are fairly static
-            expected_paths = [paths[0], stdlib, os.path.join(home, 'DLLs')]
+            expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib]
         else:
             version = f'{sys.version_info.major}.{sys.version_info.minor}'
             stdlib = os.path.join(home, sys.platlibdir, f'python{version}')
@@ -1324,7 +1326,7 @@ def test_init_is_python_build_with_home(self):
             stdlib = os.path.join(home, "Lib")
             # Because we are specifying 'home', module search paths
             # are fairly static
-            expected_paths = [paths[0], stdlib, os.path.join(home, 'DLLs')]
+            expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib]
         else:
             version = f'{sys.version_info.major}.{sys.version_info.minor}'
             stdlib = os.path.join(home, sys.platlibdir, f'python{version}')
@@ -1352,7 +1354,7 @@ def test_init_is_python_build_with_home(self):
         config['_is_python_build'] = 1
         exedir = os.path.dirname(sys.executable)
         with open(os.path.join(exedir, 'pybuilddir.txt'), encoding='utf8') as f:
-            expected_paths[2] = os.path.normpath(
+            expected_paths[1 if MS_WINDOWS else 2] = os.path.normpath(
                 os.path.join(exedir, f'{f.read()}\n$'.splitlines()[0]))
         if not MS_WINDOWS:
             # PREFIX (default) is set when running in build directory
@@ -1429,8 +1431,8 @@ def test_init_pybuilddir_win32(self):
 
             module_search_paths = self.module_search_paths()
             module_search_paths[-3] = os.path.join(tmpdir, os.path.basename(module_search_paths[-3]))
-            module_search_paths[-2] = stdlibdir
-            module_search_paths[-1] = tmpdir
+            module_search_paths[-2] = tmpdir
+            module_search_paths[-1] = stdlibdir
 
             executable = self.test_exe
             config = {

Lib/test/test_getpath.py

@@ -38,8 +38,9 @@ def test_normal_win32(self):
             module_search_paths_set=1,
             module_search_paths=[
                 r"C:\Python\python98.zip",
-                r"C:\Python\Lib",
                 r"C:\Python\DLLs",
+                r"C:\Python\Lib",
+                r"C:\Python",
             ],
         )
         actual = getpath(ns, expected)
@@ -64,8 +65,8 @@ def test_buildtree_win32(self):
             module_search_paths_set=1,
             module_search_paths=[
                 r"C:\CPython\PCbuild\amd64\python98.zip",
-                r"C:\CPython\Lib",
                 r"C:\CPython\PCbuild\amd64",
+                r"C:\CPython\Lib",
             ],
         )
         actual = getpath(ns, expected)
@@ -134,8 +135,9 @@ def test_registry_win32(self):
                 r"C:\Python\python98.zip",
                 "path1-dir",
                 # should not contain not-subdirs
-                r"C:\Python\Lib",
                 r"C:\Python\DLLs",
+                r"C:\Python\Lib",
+                r"C:\Python",
             ],
         )
         actual = getpath(ns, expected)
@@ -148,8 +150,9 @@ def test_registry_win32(self):
             module_search_paths_set=1,
             module_search_paths=[
                 r"C:\Python\python98.zip",
-                r"C:\Python\Lib",
                 r"C:\Python\DLLs",
+                r"C:\Python\Lib",
+                r"C:\Python",
             ],
         )
         actual = getpath(ns, expected)
@@ -174,8 +177,9 @@ def test_symlink_normal_win32(self):
             module_search_paths_set=1,
             module_search_paths=[
                 r"C:\Python\python98.zip",
-                r"C:\Python\Lib",
                 r"C:\Python\DLLs",
+                r"C:\Python\Lib",
+                r"C:\Python",
             ],
         )
         actual = getpath(ns, expected)
@@ -202,8 +206,8 @@ def test_symlink_buildtree_win32(self):
             module_search_paths_set=1,
             module_search_paths=[
                 r"C:\CPython\PCbuild\amd64\python98.zip",
-                r"C:\CPython\Lib",
                 r"C:\CPython\PCbuild\amd64",
+                r"C:\CPython\Lib",
             ],
         )
         actual = getpath(ns, expected)
@@ -232,8 +236,8 @@ def test_buildtree_pythonhome_win32(self):
             module_search_paths_set=1,
             module_search_paths=[
                 r"C:\Out\python98.zip",
-                r"C:\CPython\Lib",
                 r"C:\Out",
+                r"C:\CPython\Lib",
             ],
         )
         actual = getpath(ns, expected)
@@ -255,8 +259,8 @@ def test_no_dlls_win32(self):
             module_search_paths_set=1,
             module_search_paths=[
                 r"C:\Python\python98.zip",
-                r"C:\Python\Lib",
                 r"C:\Python",
+                r"C:\Python\Lib",
             ],
         )
         actual = getpath(ns, expected)

Misc/NEWS.d/next/Windows/2023-01-11-16-28-09.gh-issue-100320.2DU2it.rst

@@ -0,0 +1,3 @@
+Ensures the ``PythonPath`` registry key from an install is used when
+launching from a different copy of Python that relies on an existing install
+to provide a copy of its modules and standard library.

Modules/getpath.py

@@ -597,25 +597,27 @@ def search_up(prefix, *landmarks, test=isfile):
 
     # Detect exec_prefix by searching from executable for the platstdlib_dir
     if PLATSTDLIB_LANDMARK and not exec_prefix:
-        if executable_dir:
-            if os_name == 'nt':
-                # QUIRK: For compatibility and security, do not search for DLLs
-                # directory. The fallback below will cover it
-                exec_prefix = executable_dir
-            else:
-                exec_prefix = search_up(executable_dir, PLATSTDLIB_LANDMARK, test=isdir)
+        if os_name == 'nt':
+            # QUIRK: Windows always assumed these were the same
+            # gh-100320: Our PYDs are assumed to be relative to the Lib directory
+            # (that is, prefix) rather than the executable (that is, executable_dir)
+            exec_prefix = prefix
+        if not exec_prefix and executable_dir:
+            exec_prefix = search_up(executable_dir, PLATSTDLIB_LANDMARK, test=isdir)
         if not exec_prefix and EXEC_PREFIX:
             exec_prefix = EXEC_PREFIX
         if not exec_prefix or not isdir(joinpath(exec_prefix, PLATSTDLIB_LANDMARK)):
             if os_name == 'nt':
                 # QUIRK: If DLLs is missing on Windows, don't warn, just assume
-                # that it's all the same as prefix.
-                # gh-98790: We set platstdlib_dir here to avoid adding "DLLs" into
-                # sys.path when it doesn't exist, which would give site-packages
-                # precedence over executable_dir, which is *probably* where our PYDs
-                # live. Ideally, whoever changes our layout will tell us what the
-                # layout is, but in the past this worked, so it should keep working.
-                platstdlib_dir = exec_prefix = prefix
+                # that they're in exec_prefix
+                if not platstdlib_dir:
+                    # gh-98790: We set platstdlib_dir here to avoid adding "DLLs" into
+                    # sys.path when it doesn't exist in the platstdlib place, which
+                    # would give Lib packages precedence over executable_dir where our
+                    # PYDs *probably* live. Ideally, whoever changes our layout will tell
+                    # us what the layout is, but in the past this worked, so it should
+                    # keep working.
+                    platstdlib_dir = exec_prefix
             else:
                 warn('Could not find platform dependent libraries <exec_prefix>')
 
@@ -701,8 +703,14 @@ def search_up(prefix, *landmarks, test=isfile):
                         except OSError:
                             break
                         if isinstance(v, str):
-                            pythonpath.append(v)
+                            pythonpath.extend(v.split(DELIM))
                         i += 1
+                    # Paths from the core key get appended last, but only
+                    # when home was not set and we aren't in a build dir
+                    if not home_was_set and not venv_prefix and not build_prefix:
+                        v = winreg.QueryValue(key, None)
+                        if isinstance(v, str):
+                            pythonpath.extend(v.split(DELIM))
                 finally:
                     winreg.CloseKey(key)
             except OSError:
@@ -714,13 +722,17 @@ def search_up(prefix, *landmarks, test=isfile):
             pythonpath.append(joinpath(prefix, p))
 
     # Then add stdlib_dir and platstdlib_dir
-    if os_name == 'nt' and venv_prefix:
-        # QUIRK: Windows generates paths differently in a venv
+    if os_name == 'nt':
+        # QUIRK: Windows generates paths differently
         if platstdlib_dir:
             pythonpath.append(platstdlib_dir)
         if stdlib_dir:
             pythonpath.append(stdlib_dir)
-        if executable_dir not in pythonpath:
+        if executable_dir and executable_dir not in pythonpath:
+            # QUIRK: the executable directory is on sys.path
+            # We keep it low priority, so that properly installed modules are
+            # found first. It may be earlier in the order if we found some
+            # reason to put it there.
             pythonpath.append(executable_dir)
     else:
         if stdlib_dir: