diff --git a/polyloader/_python3.py b/polyloader/_python3.py index 68c6d7d..dbbdb3d 100644 --- a/polyloader/_python3.py +++ b/polyloader/_python3.py @@ -1,12 +1,30 @@ import os import sys import marshal +import pkgutil import _imp -from importlib._bootstrap import (cache_from_source, SourceFileLoader, - FileFinder, _verbose_message, - _get_supported_file_loaders, _relax_case) +if sys.version_info[0:2] in [(3, 3), (3, 4)]: + from importlib._bootstrap import (cache_from_source, SourceFileLoader, + FileFinder, _verbose_message, + _get_supported_file_loaders, _relax_case, + _w_long, _code_type) + +if sys.version_info[0:2] in [(3, 3)]: + from importlib._bootstrap import _MAGIC_BYTES as MAGIC_NUMBER + +if sys.version_info[0:2] == (3, 4): + from importlib._bootstrap import _validate_bytecode_header, MAGIC_NUMBER + +if sys.version_info[0:2] >= (3, 5): + from importlib.machinery import SourceFileLoader, FileFinder + from importlib._bootstrap import _verbose_message + from importlib._bootstrap_external import (_w_long, _code_type, cache_from_source, + _validate_bytecode_header, + MAGIC_NUMBER, _relax_case, + _get_supported_file_loaders) + SEP = os.sep EXS = os.extsep FLS = [('%s' + SEP + '__init__' + EXS + '%s', True), @@ -22,6 +40,16 @@ def _suffixer(loaders): class _PolySourceFileLoader(SourceFileLoader): _compiler = None + def _poly_bytes_from_bytecode(self, fullname, data, path, st): + if hasattr(self, '_bytes_from_bytecode'): + return self._bytes_from_bytecode(fullname, data, + path, st) + self_module = sys.modules[__name__] + if hasattr(self_module, '_validate_bytecode_header'): + return _validate_bytecode_header(data, source_stats = st, + name = fullname, path = path) + raise ImportError("No bytecode handler found loading.") + # All this just to change one line. def get_code(self, fullname): source_path = self.get_filename(fullname) @@ -43,9 +71,9 @@ class _PolySourceFileLoader(SourceFileLoader): pass else: try: - bytes_data = self._bytes_from_bytecode(fullname, data, - bytecode_path, - st) + bytes_data = self._poly_bytes_from_bytecode(fullname, data, + bytecode_path, + st) except (ImportError, EOFError): pass else: @@ -66,7 +94,7 @@ class _PolySourceFileLoader(SourceFileLoader): _verbose_message('code object from {}', source_path) if (not sys.dont_write_bytecode and bytecode_path is not None and source_mtime is not None): - data = bytearray(_MAGIC_BYTES) + data = bytearray(MAGIC_NUMBER) data.extend(_w_long(source_mtime)) data.extend(_w_long(len(source_bytes))) data.extend(marshal.dumps(code_object)) @@ -95,7 +123,7 @@ class PolyFileFinder(FileFinder): @property def _loaders(self): - return list(self._native_loaders) + self._custom_loaders + return self._custom_loaders + list(self._native_loaders) @classmethod def _install(cls, compiler, suffixes): @@ -117,7 +145,7 @@ class PolyFileFinder(FileFinder): str(_PolySourceFileLoader).rpartition('.')[2][1:]) newloader = type(newloaderclassname, (_PolySourceFileLoader,), dict(_compiler = compiler)) - cls._custom_loaders += [(suffix, newloader) for suffix in suffixset] + cls._custom_loaders += [(EXS + suffix, newloader) for suffix in suffixset] @classmethod def getmodulename(cls, path): @@ -164,7 +192,6 @@ class PolyFileFinder(FileFinder): is_namespace = True # Check for a file w/ a proper suffix exists. for suffix, loader in self._loaders: - print("SL:", suffix, loader) full_path = os.path.join(self.path, tail_module + suffix) _verbose_message('trying {}'.format(full_path), verbosity=2) if cache_module + suffix in cache: @@ -175,59 +202,58 @@ class PolyFileFinder(FileFinder): return (None, [base_path]) return (None, []) - # In python 3, this was moved OUT of FileFinder and put into - # pkgutils, which is probably correct. I'm leaving it here, as I - # want to trigger the hit before cascading down the singledispatch - # array of iter_importer_modules. That's a hack too far. - def iter_modules(self, prefix=''): - if self.path is None or not os.path.isdir(self.path): - return - - yielded = {} - - try: - filenames = os.listdir(self.path) - except OSError: - # ignore unreadable directories like import does - filenames = [] - filenames.sort() - for fn in filenames: - modname = self.getmodulename(fn) - if modname == '__init__' or modname in yielded: - continue - - path = os.path.join(self.path, fn) - ispkg = False - - if not modname and os.path.isdir(path) and '.' not in fn: - modname = fn - try: - dircontents = os.listdir(path) - except OSError: - # ignore unreadable directories like import does - dircontents = [] - for fn in dircontents: - subname = self.getmodulename(fn) - if subname == '__init__': - ispkg = True - break - else: - continue # not a package - - if modname and '.' not in modname: - yielded[modname] = 1 - yield prefix + modname, ispkg - @classmethod def path_hook(cls, *loader_details): cls._native_loaders = loader_details def path_hook_for_PolyFileFinder(path): if not os.path.isdir(path): raise ImportError("only directories are supported", path=path) + print("Returning PolyFileFinder") return PolyFileFinder(path) return path_hook_for_PolyFileFinder +def _poly_file_finder_modules(importer, prefix=''): + print("RUNNING!") + if importer.path is None or not os.path.isdir(importer.path): + return + + yielded = {} + + try: + filenames = os.listdir(importer.path) + except OSError: + # ignore unreadable directories like import does + filenames = [] + filenames.sort() + for fn in filenames: + modname = importer.getmodulename(fn) + if modname == '__init__' or modname in yielded: + continue + + path = os.path.join(importer.path, fn) + ispkg = False + + if not modname and os.path.isdir(path) and '.' not in fn: + modname = fn + try: + dircontents = os.listdir(path) + except OSError: + # ignore unreadable directories like import does + dircontents = [] + for fn in dircontents: + subname = importer.getmodulename(fn) + if subname == '__init__': + ispkg = True + break + else: + continue # not a package + + if modname and '.' not in modname: + yielded[modname] = 1 + yield prefix + modname, ispkg + + def install(compiler, suffixes): filefinder = [(f, i) for i, f in enumerate(sys.path_hooks) if repr(f).find('.path_hook_for_FileFinder') != -1] @@ -235,6 +261,7 @@ def install(compiler, suffixes): filefinder, fpos = filefinder[0] sys.path_hooks[fpos] = PolyFileFinder.path_hook(*(_suffixer(_get_supported_file_loaders()))) sys.path_importer_cache = {} + pkgutil.iter_importer_modules.register(PolyFileFinder, _poly_file_finder_modules) PolyFileFinder._install(compiler, suffixes) diff --git a/tests_py3/test_polyloader.py b/tests_py3/test_polyloader.py index bf40fa3..d2b787e 100644 --- a/tests_py3/test_polyloader.py +++ b/tests_py3/test_polyloader.py @@ -43,8 +43,9 @@ class Compiler: self.pt = pt def __call__(self, source_text, filename, *extra): + print(type(source_text)) return compile("result='Success for %s: %s'" % - (self.pt, source_text.rstrip()), filename, "exec") + (self.pt, source_text.decode('utf-8').rstrip()), filename, "exec") def __repr__(self): return "Compiler %s" % (self.pt) @@ -80,21 +81,21 @@ class Test_Polymorph_Direct(object): with ImportEnvironment() as sys: polyloader.install(compiler("2"), ['2']) polyloader.install(compiler("3"), ['3']) - import tests_py2.polytestmix.test2 - import tests_py2.polytestmix.test3 - import tests_py2.polytestmix.test1 - assert(tests_py2.polytestmix.test1.result == "Success for 1: Test One") - assert(tests_py2.polytestmix.test2.result == "Success for 2: Test Two") - assert(tests_py2.polytestmix.test3.result == "Success for 3: Test Three") + import tests_py3.polytestmix.test2 + import tests_py3.polytestmix.test3 + import tests_py3.polytestmix.test1 + assert(tests_py3.polytestmix.test1.result == "Success for 1: Test One") + assert(tests_py3.polytestmix.test2.result == "Success for 2: Test Two") + assert(tests_py3.polytestmix.test3.result == "Success for 3: Test Three") class Test_Polymorph_Module(object): def test_import3(self): with ImportEnvironment() as sys: polyloader.install(compiler("3"), ['3']) polyloader.install(compiler("2"), ['2']) - from tests_py2.polytestmix.test3 import result as result3 - from tests_py2.polytestmix.test2 import result as result2 - from tests_py2.polytestmix.test1 import result as result1 + from tests_py3.polytestmix.test3 import result as result3 + from tests_py3.polytestmix.test2 import result as result2 + from tests_py3.polytestmix.test1 import result as result1 assert(result1 == "Success for 1: Test One") assert(result2 == "Success for 2: Test Two") assert(result3 == "Success for 3: Test Three") @@ -105,8 +106,8 @@ class Test_Polymorph_Iterator(object): with ImportEnvironment() as sys: import os import inspect - polyloader.install(compiler("2"), ['.2']) - polyloader.install(compiler("3"), ['.3']) + polyloader.install(compiler("2"), ['2']) + polyloader.install(compiler("3"), ['3']) import pkgutil target_dir = os.path.join( os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), diff --git a/tox.ini b/tox.ini index de8cf1c..53f8e9d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,5 @@ [tox] -envlist = py26, py27, pypy, py33 -; , py33, py34, py35 +envlist = py26, py27, pypy, py33, py34, py35 [testenv] setenv =