Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions Lib/importlib/_bootstrap.py
Original file line number Diff line number Diff line change
Expand Up @@ -1280,6 +1280,14 @@ def _find_and_load(name, import_):
# NOTE: because of this, initializing must be set *before*
# putting the new module in sys.modules.
_lock_unlock_module(name)
else:
# Verify the module is still in sys.modules. Another thread may have
# removed it (due to import failure) between our sys.modules.get()
# above and the _initializing check. If removed, we retry the import
# to preserve normal semantics: the caller gets the exception from
# the actual import failure rather than a synthetic error.
if sys.modules.get(name) is not module:
return _find_and_load(name, import_)

if module is None:
message = f'import of {name} halted; None in sys.modules'
Expand Down
65 changes: 65 additions & 0 deletions Lib/test/test_importlib/test_threaded_import.py
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,71 @@ def test_multiprocessing_pool_circular_import(self, size):
'partial', 'pool_in_threads.py')
script_helper.assert_python_ok(fn)

def test_import_failure_race_condition(self):
# Regression test for race condition where a thread could receive
# a partially-initialized module when another thread's import fails.
# The race occurs when:
# 1. Thread 1 starts importing, adds module to sys.modules
# 2. Thread 2 sees the module in sys.modules
# 3. Thread 1's import fails, removes module from sys.modules
# 4. Thread 2 should NOT return the stale module reference
os.mkdir(TESTFN)
self.addCleanup(shutil.rmtree, TESTFN)
sys.path.insert(0, TESTFN)
self.addCleanup(sys.path.remove, TESTFN)

# Create a module that partially initializes then fails
modname = 'failing_import_module'
with open(os.path.join(TESTFN, modname + '.py'), 'w') as f:
f.write('''
import time
PARTIAL_ATTR = 'initialized'
time.sleep(0.05) # Widen race window
raise RuntimeError("Intentional import failure")
''')
self.addCleanup(forget, modname)
importlib.invalidate_caches()

errors = []
results = []

def do_import(delay=0):
time.sleep(delay)
try:
mod = __import__(modname)
# If we got a module, verify it's in sys.modules
if modname not in sys.modules:
errors.append(
f"Got module {mod!r} but {modname!r} not in sys.modules"
)
elif sys.modules[modname] is not mod:
errors.append(
f"Got different module than sys.modules[{modname!r}]"
)
else:
results.append(('success', mod))
except RuntimeError:
results.append(('RuntimeError',))
except Exception as e:
errors.append(f"Unexpected exception: {e}")

# Run multiple iterations to increase chance of hitting the race
for _ in range(10):
errors.clear()
results.clear()
if modname in sys.modules:
del sys.modules[modname]

t1 = threading.Thread(target=do_import, args=(0,))
t2 = threading.Thread(target=do_import, args=(0.01,))
t1.start()
t2.start()
t1.join()
t2.join()

# Neither thread should have errors about stale modules
self.assertEqual(errors, [], f"Race condition detected: {errors}")


def setUpModule():
thread_info = threading_helper.threading_setup()
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fix race condition in :mod:`importlib` where a thread could receive a stale
module reference when another thread's import fails.
44 changes: 43 additions & 1 deletion Python/import.c
Original file line number Diff line number Diff line change
Expand Up @@ -297,12 +297,32 @@ PyImport_GetModule(PyObject *name)
mod = import_get_module(tstate, name);
if (mod != NULL && mod != Py_None) {
if (import_ensure_initialized(tstate->interp, mod, name) < 0) {
goto error;
}
/* Verify the module is still in sys.modules. Another thread may have
removed it (due to import failure) between our import_get_module()
call and the _initializing check in import_ensure_initialized(). */
PyObject *mod_check = import_get_module(tstate, name);
if (mod_check != mod) {
Py_XDECREF(mod_check);
if (_PyErr_Occurred(tstate)) {
goto error;
}
/* The module was removed or replaced. Return NULL to report
"not found" rather than trying to keep up with racing
modifications to sys.modules; returning the new value would
require looping to redo the ensure_initialized check. */
Py_DECREF(mod);
Comment thread
gpshead marked this conversation as resolved.
remove_importlib_frames(tstate);
return NULL;
}
Py_DECREF(mod_check);
}
return mod;

error:
Py_DECREF(mod);
remove_importlib_frames(tstate);
return NULL;
}

/* Get the module object corresponding to a module name.
Expand Down Expand Up @@ -3882,6 +3902,28 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals,
if (import_ensure_initialized(tstate->interp, mod, abs_name) < 0) {
goto error;
}
/* Verify the module is still in sys.modules. Another thread may have
removed it (due to import failure) between our import_get_module()
call and the _initializing check in import_ensure_initialized().
If removed, we retry the import to preserve normal semantics: the
caller gets the exception from the actual import failure rather
than a synthetic error. */
PyObject *mod_check = import_get_module(tstate, abs_name);
if (mod_check != mod) {
Py_XDECREF(mod_check);
if (_PyErr_Occurred(tstate)) {
Py_DECREF(mod);
Comment thread
gpshead marked this conversation as resolved.
Outdated
goto error;
}
Py_DECREF(mod);
mod = import_find_and_load(tstate, abs_name);
if (mod == NULL) {
goto error;
}
}
else {
Py_DECREF(mod_check);
}
}
else {
Py_XDECREF(mod);
Expand Down
Loading