elf
/
polyloader
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Initial support for Python 3 completed.

This commit is contained in:
Elf M. Sternberg 2016-07-09 14:53:45 -07:00
parent 34342a0b3b
commit 0cfa023bc1
3 changed files with 94 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
polyloader/_python3.py
View File

@ -1,11 +1,29 @@
import os import os
import sys import sys
import marshal import marshal
import pkgutil
import _imp import _imp
if sys.version_info[0:2] in [(3, 3), (3, 4)]:
from importlib._bootstrap import (cache_from_source, SourceFileLoader, from importlib._bootstrap import (cache_from_source, SourceFileLoader,
FileFinder, _verbose_message, FileFinder, _verbose_message,
_get_supported_file_loaders, _relax_case) _get_supported_file_loaders, _relax_case,
_w_long, _code_type)
if sys.version_info[0:2] in [(3, 3)]:
from importlib._bootstrap import _MAGIC_BYTES as MAGIC_NUMBER
if sys.version_info[0:2] == (3, 4):
from importlib._bootstrap import _validate_bytecode_header, MAGIC_NUMBER
if sys.version_info[0:2] >= (3, 5):
from importlib.machinery import SourceFileLoader, FileFinder
from importlib._bootstrap import _verbose_message
from importlib._bootstrap_external import (_w_long, _code_type, cache_from_source,
_validate_bytecode_header,
MAGIC_NUMBER, _relax_case,
_get_supported_file_loaders)
SEP = os.sep SEP = os.sep
EXS = os.extsep EXS = os.extsep
@ -22,6 +40,16 @@ def _suffixer(loaders):
class _PolySourceFileLoader(SourceFileLoader): class _PolySourceFileLoader(SourceFileLoader):
_compiler = None _compiler = None
def _poly_bytes_from_bytecode(self, fullname, data, path, st):
if hasattr(self, '_bytes_from_bytecode'):
return self._bytes_from_bytecode(fullname, data,
path, st)
self_module = sys.modules[__name__]
if hasattr(self_module, '_validate_bytecode_header'):
return _validate_bytecode_header(data, source_stats = st,
name = fullname, path = path)
raise ImportError("No bytecode handler found loading.")
# All this just to change one line. # All this just to change one line.
def get_code(self, fullname): def get_code(self, fullname):
source_path = self.get_filename(fullname) source_path = self.get_filename(fullname)
@ -43,7 +71,7 @@ class _PolySourceFileLoader(SourceFileLoader):
pass pass
else: else:
try: try:
bytes_data = self._bytes_from_bytecode(fullname, data, bytes_data = self._poly_bytes_from_bytecode(fullname, data,
bytecode_path, bytecode_path,
st) st)
except (ImportError, EOFError): except (ImportError, EOFError):
@ -66,7 +94,7 @@ class _PolySourceFileLoader(SourceFileLoader):
_verbose_message('code object from {}', source_path) _verbose_message('code object from {}', source_path)
if (not sys.dont_write_bytecode and bytecode_path is not None and if (not sys.dont_write_bytecode and bytecode_path is not None and
source_mtime is not None): source_mtime is not None):
data = bytearray(_MAGIC_BYTES) data = bytearray(MAGIC_NUMBER)
data.extend(_w_long(source_mtime)) data.extend(_w_long(source_mtime))
data.extend(_w_long(len(source_bytes))) data.extend(_w_long(len(source_bytes)))
data.extend(marshal.dumps(code_object)) data.extend(marshal.dumps(code_object))
@ -95,7 +123,7 @@ class PolyFileFinder(FileFinder):
@property @property
def _loaders(self): def _loaders(self):
return list(self._native_loaders) + self._custom_loaders return self._custom_loaders + list(self._native_loaders)
@classmethod @classmethod
def _install(cls, compiler, suffixes): def _install(cls, compiler, suffixes):
@ -117,7 +145,7 @@ class PolyFileFinder(FileFinder):
str(_PolySourceFileLoader).rpartition('.')[2][1:]) str(_PolySourceFileLoader).rpartition('.')[2][1:])
newloader = type(newloaderclassname, (_PolySourceFileLoader,), newloader = type(newloaderclassname, (_PolySourceFileLoader,),
dict(_compiler = compiler)) dict(_compiler = compiler))
cls._custom_loaders += [(suffix, newloader) for suffix in suffixset] cls._custom_loaders += [(EXS + suffix, newloader) for suffix in suffixset]
@classmethod @classmethod
def getmodulename(cls, path): def getmodulename(cls, path):
@ -164,7 +192,6 @@ class PolyFileFinder(FileFinder):
is_namespace = True is_namespace = True
# Check for a file w/ a proper suffix exists. # Check for a file w/ a proper suffix exists.
for suffix, loader in self._loaders: for suffix, loader in self._loaders:
print("SL:", suffix, loader)
full_path = os.path.join(self.path, tail_module + suffix) full_path = os.path.join(self.path, tail_module + suffix)
_verbose_message('trying {}'.format(full_path), verbosity=2) _verbose_message('trying {}'.format(full_path), verbosity=2)
if cache_module + suffix in cache: if cache_module + suffix in cache:
@ -175,28 +202,36 @@ class PolyFileFinder(FileFinder):
return (None, [base_path]) return (None, [base_path])
return (None, []) return (None, [])
# In python 3, this was moved OUT of FileFinder and put into @classmethod
# pkgutils, which is probably correct. I'm leaving it here, as I def path_hook(cls, *loader_details):
# want to trigger the hit before cascading down the singledispatch cls._native_loaders = loader_details
# array of iter_importer_modules. That's a hack too far. def path_hook_for_PolyFileFinder(path):
def iter_modules(self, prefix=''): if not os.path.isdir(path):
if self.path is None or not os.path.isdir(self.path): raise ImportError("only directories are supported", path=path)
print("Returning PolyFileFinder")
return PolyFileFinder(path)
return path_hook_for_PolyFileFinder
def _poly_file_finder_modules(importer, prefix=''):
print("RUNNING!")
if importer.path is None or not os.path.isdir(importer.path):
return return
yielded = {} yielded = {}
try: try:
filenames = os.listdir(self.path) filenames = os.listdir(importer.path)
except OSError: except OSError:
# ignore unreadable directories like import does # ignore unreadable directories like import does
filenames = [] filenames = []
filenames.sort() filenames.sort()
for fn in filenames: for fn in filenames:
modname = self.getmodulename(fn) modname = importer.getmodulename(fn)
if modname == '__init__' or modname in yielded: if modname == '__init__' or modname in yielded:
continue continue
path = os.path.join(self.path, fn) path = os.path.join(importer.path, fn)
ispkg = False ispkg = False
if not modname and os.path.isdir(path) and '.' not in fn: if not modname and os.path.isdir(path) and '.' not in fn:
@ -207,7 +242,7 @@ class PolyFileFinder(FileFinder):
# ignore unreadable directories like import does # ignore unreadable directories like import does
dircontents = [] dircontents = []
for fn in dircontents: for fn in dircontents:
subname = self.getmodulename(fn) subname = importer.getmodulename(fn)
if subname == '__init__': if subname == '__init__':
ispkg = True ispkg = True
break break
@ -218,15 +253,6 @@ class PolyFileFinder(FileFinder):
yielded[modname] = 1 yielded[modname] = 1
yield prefix + modname, ispkg yield prefix + modname, ispkg
@classmethod
def path_hook(cls, *loader_details):
cls._native_loaders = loader_details
def path_hook_for_PolyFileFinder(path):
if not os.path.isdir(path):
raise ImportError("only directories are supported", path=path)
return PolyFileFinder(path)
return path_hook_for_PolyFileFinder
def install(compiler, suffixes): def install(compiler, suffixes):
filefinder = [(f, i) for i, f in enumerate(sys.path_hooks) filefinder = [(f, i) for i, f in enumerate(sys.path_hooks)
@ -235,6 +261,7 @@ def install(compiler, suffixes):
filefinder, fpos = filefinder[0] filefinder, fpos = filefinder[0]
sys.path_hooks[fpos] = PolyFileFinder.path_hook(*(_suffixer(_get_supported_file_loaders()))) sys.path_hooks[fpos] = PolyFileFinder.path_hook(*(_suffixer(_get_supported_file_loaders())))
sys.path_importer_cache = {} sys.path_importer_cache = {}
pkgutil.iter_importer_modules.register(PolyFileFinder, _poly_file_finder_modules)
PolyFileFinder._install(compiler, suffixes) PolyFileFinder._install(compiler, suffixes)

25
tests_py3/test_polyloader.py
View File

@ -43,8 +43,9 @@ class Compiler:
self.pt = pt self.pt = pt
def __call__(self, source_text, filename, *extra): def __call__(self, source_text, filename, *extra):
print(type(source_text))
return compile("result='Success for %s: %s'" % return compile("result='Success for %s: %s'" %
(self.pt, source_text.rstrip()), filename, "exec") (self.pt, source_text.decode('utf-8').rstrip()), filename, "exec")
def __repr__(self): def __repr__(self):
return "Compiler %s" % (self.pt) return "Compiler %s" % (self.pt)
@ -80,21 +81,21 @@ class Test_Polymorph_Direct(object):
with ImportEnvironment() as sys: with ImportEnvironment() as sys:
polyloader.install(compiler("2"), ['2']) polyloader.install(compiler("2"), ['2'])
polyloader.install(compiler("3"), ['3']) polyloader.install(compiler("3"), ['3'])
import tests_py2.polytestmix.test2 import tests_py3.polytestmix.test2
import tests_py2.polytestmix.test3 import tests_py3.polytestmix.test3
import tests_py2.polytestmix.test1 import tests_py3.polytestmix.test1
assert(tests_py2.polytestmix.test1.result == "Success for 1: Test One") assert(tests_py3.polytestmix.test1.result == "Success for 1: Test One")
assert(tests_py2.polytestmix.test2.result == "Success for 2: Test Two") assert(tests_py3.polytestmix.test2.result == "Success for 2: Test Two")
assert(tests_py2.polytestmix.test3.result == "Success for 3: Test Three") assert(tests_py3.polytestmix.test3.result == "Success for 3: Test Three")
class Test_Polymorph_Module(object): class Test_Polymorph_Module(object):
def test_import3(self): def test_import3(self):
with ImportEnvironment() as sys: with ImportEnvironment() as sys:
polyloader.install(compiler("3"), ['3']) polyloader.install(compiler("3"), ['3'])
polyloader.install(compiler("2"), ['2']) polyloader.install(compiler("2"), ['2'])
from tests_py2.polytestmix.test3 import result as result3 from tests_py3.polytestmix.test3 import result as result3
from tests_py2.polytestmix.test2 import result as result2 from tests_py3.polytestmix.test2 import result as result2
from tests_py2.polytestmix.test1 import result as result1 from tests_py3.polytestmix.test1 import result as result1
assert(result1 == "Success for 1: Test One") assert(result1 == "Success for 1: Test One")
assert(result2 == "Success for 2: Test Two") assert(result2 == "Success for 2: Test Two")
assert(result3 == "Success for 3: Test Three") assert(result3 == "Success for 3: Test Three")
@ -105,8 +106,8 @@ class Test_Polymorph_Iterator(object):
with ImportEnvironment() as sys: with ImportEnvironment() as sys:
import os import os
import inspect import inspect
polyloader.install(compiler("2"), ['.2']) polyloader.install(compiler("2"), ['2'])
polyloader.install(compiler("3"), ['.3']) polyloader.install(compiler("3"), ['3'])
import pkgutil import pkgutil
target_dir = os.path.join( target_dir = os.path.join(
os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),

3
tox.ini
View File

@ -1,6 +1,5 @@
[tox] [tox]
envlist = py26, py27, pypy, py33 envlist = py26, py27, pypy, py33, py34, py35
; , py33, py34, py35
[testenv] [testenv]
setenv = setenv =