Python 3's testing suite is radically different from 2's. I'm trying

to keep them in the same ballpark, more or less.

There was a VERY embarassing bug in the iter_modules shim.  That's been fixed.
This commit is contained in:
Elf M. Sternberg 2016-07-09 13:54:59 -07:00
parent adc00f0517
commit 34342a0b3b
21 changed files with 374 additions and 232 deletions

View File

@ -115,7 +115,7 @@ class PolyFinder(object):
def getmodulename(cls, path):
filename = os.path.basename(path)
suffixes = ([(-len(suf[0]), suf[0]) for suf in imp.get_suffixes()] +
[(-len(suf[0]), suf[0]) for suf in cls._loader_handlers])
[(-(len(suf[0]) + 1), EXS + suf[0]) for suf in cls._loader_handlers])
suffixes.sort()
for neglen, suffix in suffixes:
if filename[neglen:] == suffix:

View File

@ -1,169 +1,81 @@
import os
import sys
import marshal
import _imp
from importlib import machinery
from importlib.machinery import SOURCE_SUFFIXES as PY_SOURCE_SUFFIXES
from pkgutil import iter_importer_modules
import sys
from importlib._bootstrap import (cache_from_source, SourceFileLoader,
FileFinder, _verbose_message,
_get_supported_file_loaders, _relax_case)
SEP = os.sep
EXS = os.extsep
FLS = [('%s' + SEP + '__init__' + EXS + '%s', True),
('%s' + EXS + '%s', False)]
if sys.version_info[0:2] in [(3,3), (3,4)]:
from importlib._bootstrap import _get_supported_file_loaders
sourcefile_recognizer = 'importlib.SourceFileLoader'
if sys.version_info[0:2] in [(3,5)]:
from importlib._bootstrap_external import _get_supported_file_loaders
sourcefile_recognizer = 'importlib_external.SourceFileLoader'
def _suffixer(loaders):
return [(suffix, loader)
for (loader, suffixes) in loaders
for suffix in suffixes]
def _call_with_frames_removed(f, *args, **kwds):
# Hack. This function name and signature is hard-coded into
# Python's import.c. The name and signature trigger importlib to
# remove itself from any stacktraces. See import.c for details.
return f(*args, **kwds)
class _PolySourceFileLoader(SourceFileLoader):
_compiler = None
class PolySourceFileLoader(machinery.SourceFileLoader):
"""Override the get_code method. Falls back on the SourceFileLoader
if it's a Python file, which will generate pyc files as needed,
or works its way into the Extended version. This method does
not yet address the generation of .pyc/.pyo files from source
files for languages other than Python.
"""
_source_handlers = []
@classmethod
def get_extended_suffixes(cls):
suffixes = []
for compiler, csuffx in cls._source_handlers:
suffixes = suffixes + list(csuffx)
return suffixes
@classmethod
def get_extended_suffixes_inclusive(cls):
return PY_SOURCE_SUFFIXES + cls.get_extended_suffixes()
# TODO: Address the generation of .pyc/.pyo files from source files.
# See importlib/_bootstrap.py for details in SourceFileLoader of
# how that's done.
# All this just to change one line.
def get_code(self, fullname):
source_path = self.get_filename(fullname)
if source_path.endswith(tuple(PY_SOURCE_SUFFIXES)):
return super(ExtendedSourceFileLoader, self).get_code(fullname)
for compiler, suffixes in self._source_handlers:
if source_path.endswith(suffixes):
return _call_with_frames_removed(compiler, source_path, fullname)
else:
raise ImportError("Could not find compiler for %s (%s)" % (fullname, source_path))
# Provide a working namespace for our new FileFinder.
class PolySourceFileFinder(machinery.FileFinder):
# Taken from inspect.py and modified to support alternate suffixes.
@staticmethod
def getmodulename(path):
fname = os.path.basename(path)
suffixes = [(-len(suffix), suffix)
for suffix in (machinery.all_suffixes() +
ExtendedSourceFileLoader.get_extended_suffixes())]
suffixes.sort() # try longest suffixes first, in case they overlap
for neglen, suffix in suffixes:
if fname.endswith(suffix):
return fname[:neglen]
return None
# Taken from pkgutil.py and modified to support alternate suffixes.
@staticmethod
def iter_modules(importer, prefix=''):
if importer.path is None or not os.path.isdir(importer.path):
return
yielded = {}
source_mtime = None
try:
filenames = os.listdir(importer.path)
except OSError:
# ignore unreadable directories like import does
filenames = []
filenames.sort() # handle packages before same-named modules
for fn in filenames:
modname = ExtendedFileFinder.getmodulename(fn)
if modname == '__init__' or modname in yielded:
continue
path = os.path.join(importer.path, fn)
ispkg = False
if not modname and os.path.isdir(path) and '.' not in fn:
modname = fn
try:
dircontents = os.listdir(path)
except OSError:
# ignore unreadable directories like import does
dircontents = []
for fn in dircontents:
subname = ExtendedFileFinder.getmodulename(fn)
if subname == '__init__':
ispkg = True
break
bytecode_path = cache_from_source(source_path)
except NotImplementedError:
bytecode_path = None
else:
continue # not a package
if modname and '.' not in modname:
yielded[modname] = 1
yield prefix + modname, ispkg
try:
st = self.path_stats(source_path)
except NotImplementedError:
pass
def install(compiler, suffixes):
"""Install a specialized version of FileFinder that will search
through alternative extensions first for syntax files and, upon
encountering one, will return a specialized version of
SourceFileLoader for that syntax. By replacing this into
path_hook this makes both import and iter_modules work as
expected.
"""
filefinder = [(f, i) for i, f in enumerate(sys.path_hooks)
if repr(f).find('.path_hook_for_FileFinder') != -1]
if not filefinder:
return
filefinder, fpos = filefinder[0]
supported_loaders = _get_supported_file_loaders()
print([repr(i) for i in supported_loaders])
sourceloader = [(l, i) for i, l in enumerate(supported_loaders)
if repr(l[0]).find(sourcefile_recognizer) != -1]
if not sourceloader:
return
sourceloader, spos = sourceloader[0]
supported_loaders[spos] = (ExtendedSourceFileLoader,
ExtendedSourceFileLoader.get_extended_suffixes_inclusive())
sys.path_hooks[fpos] = ExtendedFileFinder.path_hook(*supported_loaders)
iter_importer_modules.register(ExtendedFileFinder, ExtendedFileFinder.iter_modules)
if sys.path[0] != "":
sys.path.insert(0, "")
class PolySourceFileLoader(FileLoader):
else:
source_mtime = int(st['mtime'])
try:
data = self.get_data(bytecode_path)
except IOError:
pass
else:
try:
bytes_data = self._bytes_from_bytecode(fullname, data,
bytecode_path,
st)
except (ImportError, EOFError):
pass
else:
_verbose_message('{} matches {}', bytecode_path,
source_path)
found = marshal.loads(bytes_data)
if isinstance(found, _code_type):
_imp._fix_co_filename(found, source_path)
_verbose_message('code object from {}',
bytecode_path)
return found
else:
msg = "Non-code object in {}"
raise ImportError(msg.format(bytecode_path),
name=fullname, path=bytecode_path)
source_bytes = self.get_data(source_path)
code_object = self._compiler(source_bytes, source_path, fullname)
_verbose_message('code object from {}', source_path)
if (not sys.dont_write_bytecode and bytecode_path is not None and
source_mtime is not None):
data = bytearray(_MAGIC_BYTES)
data.extend(_w_long(source_mtime))
data.extend(_w_long(len(source_bytes)))
data.extend(marshal.dumps(code_object))
try:
self._cache_bytecode(source_path, bytecode_path, data)
_verbose_message('wrote {!r}', bytecode_path)
except NotImplementedError:
pass
return code_object
class PolyFileFinder(FileFinder):
@ -173,7 +85,6 @@ class PolyFileFinder(FileFinder):
_native_loaders = []
_custom_loaders = []
_installed = False
def __init__(self, path):
# Base (directory) path
@ -184,21 +95,149 @@ class PolyFileFinder(FileFinder):
@property
def _loaders(self):
return cls._native_loaders + cls._custom_loaders
return list(self._native_loaders) + self._custom_loaders
@classmethod
def path_hook(cls):
if not _path_isdir(path):
# By now, we've exhausted every loader except this one, so...
def _install(cls, compiler, suffixes):
if not suffixes:
return
if isinstance(suffixes, str):
suffixes = [suffixes]
suffixset = set(suffixes)
overlap = suffixset.intersection(set([suf[0] for suf in cls._native_loaders]))
if overlap:
raise RuntimeError("Override of native Python extensions is not permitted.")
overlap = suffixset.intersection(
set([loader[0] for loader in cls._custom_loaders]))
if overlap:
# Fail silently
return
newloaderclassname = (suffixes[0].lower().capitalize() +
str(_PolySourceFileLoader).rpartition('.')[2][1:])
newloader = type(newloaderclassname, (_PolySourceFileLoader,),
dict(_compiler = compiler))
cls._custom_loaders += [(suffix, newloader) for suffix in suffixset]
@classmethod
def getmodulename(cls, path):
filename = os.path.basename(path)
suffixes = ([(-len(suf[0]), suf[0]) for suf in cls._native_loaders] +
[(-len(suf[0]), suf[0]) for suf in cls._custom_loaders])
suffixes.sort()
for neglen, suffix in suffixes:
if filename[neglen:] == suffix:
return filename[:neglen]
return None
def find_loader(self, fullname):
"""Try to find a loader for the specified module, or the namespace
package portions. Returns (loader, list-of-portions)."""
is_namespace = False
tail_module = fullname.rpartition('.')[2]
try:
mtime = os.stat(self.path).st_mtime
except OSError:
mtime = -1
if mtime != self._path_mtime:
self._fill_cache()
self._path_mtime = mtime
# tail_module keeps the original casing, for __file__ and friends
if _relax_case():
cache = self._relaxed_path_cache
cache_module = tail_module.lower()
else:
cache = self._path_cache
cache_module = tail_module
# Check if the module is the name of a directory (and thus a package).
if cache_module in cache:
base_path = os.path.join(self.path, tail_module)
if os.path.isdir(base_path):
for suffix, loader in self._loaders:
init_filename = '__init__' + suffix
full_path = os.path.join(base_path, init_filename)
if os.path.isfile(full_path):
return (loader(fullname, full_path), [base_path])
else:
# A namespace package, return the path if we don't also
# find a module in the next section.
is_namespace = True
# Check for a file w/ a proper suffix exists.
for suffix, loader in self._loaders:
print("SL:", suffix, loader)
full_path = os.path.join(self.path, tail_module + suffix)
_verbose_message('trying {}'.format(full_path), verbosity=2)
if cache_module + suffix in cache:
if os.path.isfile(full_path):
return (loader(fullname, full_path), [])
if is_namespace:
_verbose_message('possible namespace for {}'.format(base_path))
return (None, [base_path])
return (None, [])
# In python 3, this was moved OUT of FileFinder and put into
# pkgutils, which is probably correct. I'm leaving it here, as I
# want to trigger the hit before cascading down the singledispatch
# array of iter_importer_modules. That's a hack too far.
def iter_modules(self, prefix=''):
if self.path is None or not os.path.isdir(self.path):
return
yielded = {}
try:
filenames = os.listdir(self.path)
except OSError:
# ignore unreadable directories like import does
filenames = []
filenames.sort()
for fn in filenames:
modname = self.getmodulename(fn)
if modname == '__init__' or modname in yielded:
continue
path = os.path.join(self.path, fn)
ispkg = False
if not modname and os.path.isdir(path) and '.' not in fn:
modname = fn
try:
dircontents = os.listdir(path)
except OSError:
# ignore unreadable directories like import does
dircontents = []
for fn in dircontents:
subname = self.getmodulename(fn)
if subname == '__init__':
ispkg = True
break
else:
continue # not a package
if modname and '.' not in modname:
yielded[modname] = 1
yield prefix + modname, ispkg
@classmethod
def path_hook(cls, *loader_details):
cls._native_loaders = loader_details
def path_hook_for_PolyFileFinder(path):
if not os.path.isdir(path):
raise ImportError("only directories are supported", path=path)
return cls(path)
return PolyFileFinder(path)
return path_hook_for_PolyFileFinder
def install(compiler, suffixes):
filefinder = [(f, i) for i, f in enumerate(sys.path_hooks)
if repr(f).find('.path_hook_for_FileFinder') != -1]
if filefinder:
native_loaders = machinery._get_supported_file_loaders()
filefinder, fpos = filefinder[0]
sys.path_hooks[fpos] = PolyFileFinder.path_hook(*native_loaders)
sys.path_hooks[fpos] = PolyFileFinder.path_hook(*(_suffixer(_get_supported_file_loaders())))
sys.path_importer_cache = {}
PolyFileFinder._install(compiler, suffixes)
def reset():
PolyFileFinder._custom_loaders = []

View File

@ -1,7 +1,11 @@
import UserDict
try:
from UserDict import DictMixin
except ImportError:
from collections import MutableMapping as DictMixin
import os
class EnvironmentVarGuard(UserDict.DictMixin):
class EnvironmentVarGuard(DictMixin):
"""Class to help protect the environment variable properly. Can be used as
a context manager."""

View File

@ -1,7 +1,7 @@
import polyloader
import pytest
import py_compile
import ptutils
from . import ptutils
import stat
import sys
import os
@ -24,6 +24,13 @@ polyloader.install(Compiler("2"), ['2'])
TESTFN = '@test'
if sys.version_info[0:2] >= (2, 6):
VERSION = 2
if sys.version_info[0] >= 3:
VERSION = 3
def clean_tmpfiles(path):
if os.path.exists(path):
os.remove(path)
@ -66,16 +73,14 @@ class Test_Imports(object):
pyc = TESTFN + os.extsep + "pyc"
with open(source, "w") as f:
print >> f, ("# This tests Python's ability to import a", ext,
"file.")
a = random.randrange(1000)
b = random.randrange(1000)
print >> f, "a =", a
print >> f, "b =", b
f.write("# This tests Python's ability to import a" + ext + "file.\n")
f.write("a =" + str(a) + "\n")
f.write("b =" + str(b) + "\n")
try:
mod = __import__(TESTFN)
except ImportError, err:
except ImportError as err:
print("import from %s (%s) failed: %s" % (ext, os.curdir, err))
assert(False)
else:
@ -87,7 +92,7 @@ class Test_Imports(object):
try:
if not sys.dont_write_bytecode:
imp.reload(mod)
except ImportError, err:
except ImportError as err:
print("import from .pyc/.pyo failed: %s" % err)
assert(False)
finally:
@ -99,7 +104,7 @@ class Test_Imports(object):
def test_execute_bit_not_copied(self):
# Issue 6070: under posix .pyc files got their execute bit set if
# the .py file had the execute bit set, but they aren't executable.
oldmask = os.umask(022)
oldmask = os.umask(0o22)
sys.path.insert(0, os.curdir)
try:
fname = TESTFN + os.extsep + "py"
@ -141,39 +146,11 @@ class Test_Imports(object):
assert(orig_path == new_os.path)
assert(orig_getenv != new_os.getenv)
@pytest.mark.skipif(hasattr(sys, 'pypy_version_info'),
reason="PyPy won't load bytecode if source not present.")
def test_module_with_large_stack(self, module='longlist'):
# Regression test for http://bugs.python.org/issue561858.
filename = module + os.extsep + 'py'
# Create a file with a list of 65000 elements.
with open(filename, 'w+') as f:
f.write('d = [\n')
for i in range(65000):
f.write('"",\n')
f.write(']')
# Compile & remove .py file, we only need .pyc (or .pyo).
with open(filename, 'r') as f:
py_compile.compile(filename)
os.remove(filename)
# Need to be able to load from current dir.
sys.path.append('')
# This used to crash.
exec 'import ' + module
# Cleanup.
del sys.path[-1]
clean_tmpfiles(filename)
def test_failing_import_sticks(self):
source = TESTFN + os.extsep + "py"
with open(source, "w") as f:
print >> f, "a = 1 // 0"
f.write("a = 1 // 0\n")
# New in 2.4, we shouldn't be able to import that no matter how often
# we try.

View File

@ -1,7 +1,7 @@
import polyloader
import pytest
import py_compile
import ptutils
from . import ptutils
import stat
import sys
import os

View File

@ -0,0 +1,117 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
test_polyloader
----------------------------------
Tests for `polyloader` module.
"""
import polyloader
import copy
import sys
# Note that these compilers don't actually load much out of the
# source files. That's not the point. The point is to show that the
# correct compiler has been found for a given extension.
class ImportEnvironment(object):
def __init__(self):
pass
def __enter__(self):
polyloader.reset()
self.path = copy.copy(sys.path)
self.path_hooks = copy.copy(sys.path_hooks)
self.meta_path = copy.copy(sys.meta_path)
self.modules = copy.copy(sys.modules)
self.path_importer_cache = copy.copy(sys.path_importer_cache)
return sys
def __exit__(self, type, value, traceback):
sys.path = self.path
sys.path_hooks = self.path_hooks
sys.meta_path = self.meta_path
sys.modules = self.modules
sys.path_importer_cache = self.path_importer_cache
class Compiler:
def __init__(self, pt):
self.pt = pt
def __call__(self, source_text, filename, *extra):
return compile("result='Success for %s: %s'" %
(self.pt, source_text.rstrip()), filename, "exec")
def __repr__(self):
return "Compiler %s" % (self.pt)
def compiler(pt):
return Compiler(pt)
# Functionally, the "From" test and the "Direct" test should be
# indistinguishable. What's interesting about them, though, is that
# in the context of the caller, the "Direct" test now has test and
# test.polytestmix as objects in the calling context. They're fairly
# lightweight, but they do exist, and they do honor the __all__ and
# __path__ cases.
#
# Also:
# See http://python-notes.curiousefficiency.org/en/latest/python_concepts/import_traps.html
# for some 'gotchas' to look forward to. Whee.
class Test_Polymorph_From(object):
def test_import1(self):
with ImportEnvironment() as sys:
polyloader.install(compiler("2"), ['2'])
polyloader.install(compiler("3"), ['3'])
from .polytestmix import test2
from .polytestmix import test3
from .polytestmix import test1
assert(test1.result == "Success for 1: Test One")
assert(test2.result == "Success for 2: Test Two")
assert(test3.result == "Success for 3: Test Three")
class Test_Polymorph_Direct(object):
def test_import2(self):
with ImportEnvironment() as sys:
polyloader.install(compiler("2"), ['2'])
polyloader.install(compiler("3"), ['3'])
import tests_py2.polytestmix.test2
import tests_py2.polytestmix.test3
import tests_py2.polytestmix.test1
assert(tests_py2.polytestmix.test1.result == "Success for 1: Test One")
assert(tests_py2.polytestmix.test2.result == "Success for 2: Test Two")
assert(tests_py2.polytestmix.test3.result == "Success for 3: Test Three")
class Test_Polymorph_Module(object):
def test_import3(self):
with ImportEnvironment() as sys:
polyloader.install(compiler("3"), ['3'])
polyloader.install(compiler("2"), ['2'])
from tests_py2.polytestmix.test3 import result as result3
from tests_py2.polytestmix.test2 import result as result2
from tests_py2.polytestmix.test1 import result as result1
assert(result1 == "Success for 1: Test One")
assert(result2 == "Success for 2: Test Two")
assert(result3 == "Success for 3: Test Three")
class Test_Polymorph_Iterator(object):
''' The Django Compatibility test: Can we load arbitrary modules from a package? '''
def test_iterator(self):
with ImportEnvironment() as sys:
import os
import inspect
polyloader.install(compiler("2"), ['2'])
polyloader.install(compiler("3"), ['3'])
import pkgutil
target_dir = os.path.join(
os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
'polytestmix')
modules = set([name for (_, name, is_pkg) in pkgutil.iter_modules([target_dir])
if not is_pkg and not name.startswith('_')])
assert(modules == set(['test1', 'test2', 'test3']))

View File

@ -1,7 +1,7 @@
import polyloader
import pytest
import py_compile
import ptutils
from . import ptutils
import stat
import sys
import os
@ -42,10 +42,10 @@ def unload(name):
class Test_RelativeImports:
def teardown_class(cls):
unload("tests.relimport")
unload("tests_py2.relimport")
def setup_class(cls):
unload("tests.relimport")
unload("tests_py2.relimport")
def test_relimport_star(self):
# This will import * from .test_import.
@ -54,41 +54,41 @@ class Test_RelativeImports:
def test_issue3221(self):
# Regression test for http://bugs.python.org/issue3221.
def check_absolute():
def check_absolute(ns):
exec "from os import path" in ns
def check_relative():
def check_relative(ns):
exec "from . import relimport" in ns
# Check both OK with __package__ and __name__ correct
ns = dict(__package__='tests', __name__='test.notarealmodule')
check_absolute()
check_relative()
ns = dict(__package__='tests_py2', __name__='test.notarealmodule')
check_absolute(ns)
check_relative(ns)
# Check both OK with only __name__ wrong
ns = dict(__package__='tests', __name__='notarealpkg.notarealmodule')
check_absolute()
check_relative()
ns = dict(__package__='tests_py2', __name__='notarealpkg.notarealmodule')
check_absolute(ns)
check_relative(ns)
# Check relative fails with only __package__ wrong
ns = dict(__package__='foo', __name__='test.notarealmodule')
with pytest.warns(RuntimeWarning) as rw:
check_absolute()
check_absolute(ns)
with pytest.raises(SystemError) as se:
check_relative()
check_relative(ns)
# Check relative fails with __package__ and __name__ wrong
ns = dict(__package__='foo', __name__='notarealpkg.notarealmodule')
with pytest.warns(RuntimeWarning) as se:
check_absolute()
check_absolute(ns)
with pytest.raises(SystemError) as se:
check_relative()
check_relative(ns)
# Check both fail with package set to a non-string
ns = dict(__package__=object())
with pytest.raises(ValueError) as ve:
check_absolute()
check_absolute(ns)
with pytest.raises(ValueError) as ve:
check_relative()
check_relative(ns)
def test_absolute_import_without_future(self):
# If explicit relative import syntax is used, then do not try

1
tests_py3/__init__.py Normal file
View File

@ -0,0 +1 @@
# -*- coding: utf-8 -*-

View File

View File

@ -0,0 +1 @@
result = "Success for 1: Test One"

View File

@ -0,0 +1 @@
Test Two

View File

@ -0,0 +1 @@
Test Three

View File

@ -80,21 +80,21 @@ class Test_Polymorph_Direct(object):
with ImportEnvironment() as sys:
polyloader.install(compiler("2"), ['2'])
polyloader.install(compiler("3"), ['3'])
import tests.polytestmix.test2
import tests.polytestmix.test3
import tests.polytestmix.test1
assert(tests.polytestmix.test1.result == "Success for 1: Test One")
assert(tests.polytestmix.test2.result == "Success for 2: Test Two")
assert(tests.polytestmix.test3.result == "Success for 3: Test Three")
import tests_py2.polytestmix.test2
import tests_py2.polytestmix.test3
import tests_py2.polytestmix.test1
assert(tests_py2.polytestmix.test1.result == "Success for 1: Test One")
assert(tests_py2.polytestmix.test2.result == "Success for 2: Test Two")
assert(tests_py2.polytestmix.test3.result == "Success for 3: Test Three")
class Test_Polymorph_Module(object):
def test_import3(self):
with ImportEnvironment() as sys:
polyloader.install(compiler("3"), ['3'])
polyloader.install(compiler("2"), ['2'])
from tests.polytestmix.test3 import result as result3
from tests.polytestmix.test2 import result as result2
from tests.polytestmix.test1 import result as result1
from tests_py2.polytestmix.test3 import result as result3
from tests_py2.polytestmix.test2 import result as result2
from tests_py2.polytestmix.test1 import result as result1
assert(result1 == "Success for 1: Test One")
assert(result2 == "Success for 2: Test Two")
assert(result3 == "Success for 3: Test Three")

View File

@ -1,5 +1,5 @@
[tox]
envlist = py26, py27, pypy
envlist = py26, py27, pypy, py33
; , py33, py34, py35
[testenv]
@ -8,7 +8,8 @@ setenv =
deps =
-r{toxinidir}/requirements_dev.txt
commands =
py.test --basetemp={envtmpdir} []
{py27,py26,pypy}: py.test --basetemp={envtmpdir} ./tests_py2
{py33,py34,py35}: py.test --basetemp={envtmpdir} ./tests_py3
[flake8]
max-line-length = 99