Skip to content
20 changes: 19 additions & 1 deletion importlib_metadata/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -787,6 +787,24 @@ def find_distributions(self, context=Context()) -> Iterable[Distribution]:
"""


def _clear_lru_cache_after_fork(func):
"""Wrap ``func`` with ``functools.lru_cache`` and clear it after ``fork``.

``FastPath`` caches zip-backed ``pathlib.Path`` objects that keep a
reference to the parent's open ``ZipFile`` handle. Re-using a cached
instance in a forked child can therefore resurrect invalid file pointers
and trigger ``BadZipFile``/``OSError`` failures (python/importlib_metadata#520).
Registering ``cache_clear`` with ``os.register_at_fork`` ensures every
process gets a pristine cache and opens its own archive handles.
"""

cached = functools.lru_cache()(func)
register = getattr(os, 'register_at_fork', None)
if register is not None:
register(after_in_child=cached.cache_clear)
return cached


class FastPath:
"""
Micro-optimized class for searching a root for children.
Expand All @@ -803,7 +821,7 @@ class FastPath:
True
"""

@functools.lru_cache() # type: ignore[misc]
@_clear_lru_cache_after_fork
def __new__(cls, root):
return super().__new__(cls)

Expand Down
1 change: 1 addition & 0 deletions newsfragments/520.bugfix.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Fixed errors in FastPath under fork-multiprocessing.
34 changes: 34 additions & 0 deletions tests/test_zip.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
import multiprocessing
import os
import sys
import unittest

from importlib_metadata import (
FastPath,
PackageNotFoundError,
distribution,
distributions,
Expand Down Expand Up @@ -47,6 +50,37 @@ def test_one_distribution(self):
dists = list(distributions(path=sys.path[:1]))
assert len(dists) == 1

@unittest.skipUnless(
hasattr(os, 'register_at_fork')
and 'fork' in multiprocessing.get_all_start_methods(),
'requires fork-based multiprocessing support',
)
def test_fastpath_cache_cleared_in_forked_child(self):
zip_path = sys.path[0]

FastPath(zip_path)
assert FastPath.__new__.cache_info().currsize >= 1

ctx = multiprocessing.get_context('fork')
parent_conn, child_conn = ctx.Pipe()

def child(conn, root):
try:
before = FastPath.__new__.cache_info().currsize
FastPath(root)
after = FastPath.__new__.cache_info().currsize
conn.send((before, after))
finally:
conn.close()

proc = ctx.Process(target=child, args=(child_conn, zip_path))
proc.start()
child_conn.close()
cache_sizes = parent_conn.recv()
proc.join()

self.assertEqual(cache_sizes, (0, 1))


class TestEgg(TestZip):
def setUp(self):
Expand Down