benjamin.peterson
2008-11-28 23:01:28 UTC
Author: benjamin.peterson
Date: Sat Nov 29 00:01:28 2008
New Revision: 67429
Log:
Merged revisions 67428 via svnmerge from
svn+ssh://pythondev at svn.python.org/python/trunk
................
r67428 | benjamin.peterson | 2008-11-28 16:12:14 -0600 (Fri, 28 Nov 2008) | 57 lines
Merged revisions 67384,67386-67387,67389-67390,67392,67399-67400,67403-67405,67426 via svnmerge from
svn+ssh://pythondev at svn.python.org/sandbox/trunk/2to3/lib2to3
........
r67384 | benjamin.peterson | 2008-11-25 16:13:31 -0600 (Tue, 25 Nov 2008) | 4 lines
don't duplicate calls to start_tree()
RefactoringTool.pre_order values now holds a list of the fixers while pre_order_mapping holds the dict.
........
r67386 | benjamin.peterson | 2008-11-25 16:44:52 -0600 (Tue, 25 Nov 2008) | 1 line
#4423 fix_imports was still replacing usage of a module if attributes were being used
........
r67387 | benjamin.peterson | 2008-11-25 16:47:54 -0600 (Tue, 25 Nov 2008) | 1 line
fix broken test
........
r67389 | benjamin.peterson | 2008-11-25 17:13:17 -0600 (Tue, 25 Nov 2008) | 1 line
remove compatibility code; we only cater to 2.5+
........
r67390 | benjamin.peterson | 2008-11-25 22:03:36 -0600 (Tue, 25 Nov 2008) | 1 line
fix #3994; the usage of changed imports was fixed in nested cases
........
r67392 | benjamin.peterson | 2008-11-26 11:11:40 -0600 (Wed, 26 Nov 2008) | 1 line
simpilfy and comment fix_imports
........
r67399 | benjamin.peterson | 2008-11-26 11:47:03 -0600 (Wed, 26 Nov 2008) | 1 line
remove more compatibility code
........
r67400 | benjamin.peterson | 2008-11-26 12:07:41 -0600 (Wed, 26 Nov 2008) | 1 line
set svn:ignore
........
r67403 | benjamin.peterson | 2008-11-26 13:11:11 -0600 (Wed, 26 Nov 2008) | 1 line
wrap import
........
r67404 | benjamin.peterson | 2008-11-26 13:29:49 -0600 (Wed, 26 Nov 2008) | 1 line
build the fix_imports pattern in compile_pattern, so MAPPING can be changed and reflected in the pattern
........
r67405 | benjamin.peterson | 2008-11-26 14:01:24 -0600 (Wed, 26 Nov 2008) | 1 line
stop ugly messages about runtime errors being from printed
........
r67426 | benjamin.peterson | 2008-11-28 16:01:40 -0600 (Fri, 28 Nov 2008) | 5 lines
don't replace a module name if it is in the middle of a attribute lookup
This fix also stops module names from being replaced if they are not in an attribute lookup.
........
................
Modified:
python/branches/py3k/ (props changed)
python/branches/py3k/Lib/lib2to3/fixer_base.py
python/branches/py3k/Lib/lib2to3/fixer_util.py
python/branches/py3k/Lib/lib2to3/fixes/fix_dict.py
python/branches/py3k/Lib/lib2to3/fixes/fix_except.py
python/branches/py3k/Lib/lib2to3/fixes/fix_imports.py
python/branches/py3k/Lib/lib2to3/fixes/fix_imports2.py
python/branches/py3k/Lib/lib2to3/fixes/fix_next.py
python/branches/py3k/Lib/lib2to3/fixes/fix_numliterals.py
python/branches/py3k/Lib/lib2to3/fixes/fix_renames.py
python/branches/py3k/Lib/lib2to3/fixes/fix_urllib.py
python/branches/py3k/Lib/lib2to3/pgen2/parse.py
python/branches/py3k/Lib/lib2to3/pytree.py
python/branches/py3k/Lib/lib2to3/refactor.py
python/branches/py3k/Lib/lib2to3/tests/ (props changed)
python/branches/py3k/Lib/lib2to3/tests/data/ (props changed)
python/branches/py3k/Lib/lib2to3/tests/test_fixers.py
python/branches/py3k/Lib/lib2to3/tests/test_refactor.py
Modified: python/branches/py3k/Lib/lib2to3/fixer_base.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/fixer_base.py (original)
+++ python/branches/py3k/Lib/lib2to3/fixer_base.py Sat Nov 29 00:01:28 2008
@@ -7,12 +7,6 @@
import logging
import itertools
-# Get a usable 'set' constructor
-try:
- set
-except NameError:
- from sets import Set as set
-
# Local imports
from .patcomp import PatternCompiler
from . import pygram
Modified: python/branches/py3k/Lib/lib2to3/fixer_util.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/fixer_util.py (original)
+++ python/branches/py3k/Lib/lib2to3/fixer_util.py Sat Nov 29 00:01:28 2008
@@ -153,30 +153,6 @@
and node.children[0].value == "["
and node.children[-1].value == "]")
-###########################################################
-### Common portability code. This allows fixers to do, eg,
-### "from .util import set" and forget about it.
-###########################################################
-
-try:
- any = any
-except NameError:
- def any(l):
- for o in l:
- if o:
- return True
- return False
-
-try:
- set = set
-except NameError:
- from sets import Set as set
-
-try:
- reversed = reversed
-except NameError:
- def reversed(l):
- return l[::-1]
###########################################################
### Misc
Modified: python/branches/py3k/Lib/lib2to3/fixes/fix_dict.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/fixes/fix_dict.py (original)
+++ python/branches/py3k/Lib/lib2to3/fixes/fix_dict.py Sat Nov 29 00:01:28 2008
@@ -28,7 +28,7 @@
from .. import patcomp
from ..pgen2 import token
from .. import fixer_base
-from ..fixer_util import Name, Call, LParen, RParen, ArgList, Dot, set
+from ..fixer_util import Name, Call, LParen, RParen, ArgList, Dot
from .. import fixer_util
Modified: python/branches/py3k/Lib/lib2to3/fixes/fix_except.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/fixes/fix_except.py (original)
+++ python/branches/py3k/Lib/lib2to3/fixes/fix_except.py Sat Nov 29 00:01:28 2008
@@ -25,7 +25,7 @@
from .. import pytree
from ..pgen2 import token
from .. import fixer_base
-from ..fixer_util import Assign, Attr, Name, is_tuple, is_list, reversed
+from ..fixer_util import Assign, Attr, Name, is_tuple, is_list
def find_excepts(nodes):
for i, n in enumerate(nodes):
Modified: python/branches/py3k/Lib/lib2to3/fixes/fix_imports.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/fixes/fix_imports.py (original)
+++ python/branches/py3k/Lib/lib2to3/fixes/fix_imports.py Sat Nov 29 00:01:28 2008
@@ -1,9 +1,9 @@
"""Fix incompatible imports and module references."""
-# Author: Collin Winter
+# Authors: Collin Winter, Nick Edds
# Local imports
from .. import fixer_base
-from ..fixer_util import Name, attr_chain, any, set
+from ..fixer_util import Name, attr_chain
MAPPING = {'StringIO': 'io',
'cStringIO': 'io',
@@ -61,36 +61,49 @@
def build_pattern(mapping=MAPPING):
- mod_list = ' | '.join(["module='" + key + "'" for key in mapping.keys()])
- mod_name_list = ' | '.join(["module_name='" + key + "'" for key in mapping.keys()])
- yield """import_name< 'import' ((%s)
+ mod_list = ' | '.join(["module_name='%s'" % key for key in mapping])
+ bare_names = alternates(mapping.keys())
+
+ yield """name_import=import_name< 'import' ((%s)
| dotted_as_names< any* (%s) any* >) >
""" % (mod_list, mod_list)
yield """import_from< 'from' (%s) 'import' ['(']
( any | import_as_name< any 'as' any > |
import_as_names< any* >) [')'] >
- """ % mod_name_list
+ """ % mod_list
yield """import_name< 'import'
dotted_as_name< (%s) 'as' any > >
- """ % mod_name_list
- # Find usages of module members in code e.g. urllib.foo(bar)
- yield """power< (%s)
- trailer<'.' any > any* >
- """ % mod_name_list
- yield """bare_name=%s""" % alternates(mapping.keys())
+ """ % mod_list
+
+ # Find usages of module members in code e.g. thread.foo(bar)
+ yield "power< bare_with_attr=(%s) trailer<'.' any > any* >" % bare_names
+
class FixImports(fixer_base.BaseFix):
- PATTERN = "|".join(build_pattern())
+
order = "pre" # Pre-order tree traversal
+ # This is overridden in fix_imports2.
mapping = MAPPING
- # Don't match the node if it's within another match
+ def build_pattern(self):
+ return "|".join(build_pattern(self.mapping))
+
+ def compile_pattern(self):
+ # We override this, so MAPPING can be pragmatically altered and the
+ # changes will be reflected in PATTERN.
+ self.PATTERN = self.build_pattern()
+ super(FixImports, self).compile_pattern()
+
+ # Don't match the node if it's within another match.
def match(self, node):
match = super(FixImports, self).match
results = match(node)
if results:
- if any([match(obj) for obj in attr_chain(node, "parent")]):
+ # Module usage could be in the trailier of an attribute lookup, so
+ # we might have nested matches when "bare_with_attr" is present.
+ if "bare_with_attr" not in results and \
+ any([match(obj) for obj in attr_chain(node, "parent")]):
return False
return results
return False
@@ -100,20 +113,17 @@
self.replace = {}
def transform(self, node, results):
- import_mod = results.get("module")
- mod_name = results.get("module_name")
- bare_name = results.get("bare_name")
-
- if import_mod or mod_name:
- new_name = self.mapping[(import_mod or mod_name).value]
-
+ import_mod = results.get("module_name")
if import_mod:
- self.replace[import_mod.value] = new_name
+ new_name = self.mapping[(import_mod or mod_name).value]
+ if "name_import" in results:
+ # If it's not a "from x import x, y" or "import x as y" import,
+ # marked its usage to be replaced.
+ self.replace[import_mod.value] = new_name
import_mod.replace(Name(new_name, prefix=import_mod.get_prefix()))
- elif mod_name:
- mod_name.replace(Name(new_name, prefix=mod_name.get_prefix()))
- elif bare_name:
- bare_name = bare_name[0]
+ else:
+ # Replace usage of the module.
+ bare_name = results["bare_with_attr"][0]
new_name = self.replace.get(bare_name.value)
if new_name:
bare_name.replace(Name(new_name, prefix=bare_name.get_prefix()))
Modified: python/branches/py3k/Lib/lib2to3/fixes/fix_imports2.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/fixes/fix_imports2.py (original)
+++ python/branches/py3k/Lib/lib2to3/fixes/fix_imports2.py Sat Nov 29 00:01:28 2008
@@ -10,7 +10,6 @@
class FixImports2(fix_imports.FixImports):
- PATTERN = "|".join((fix_imports.build_pattern(MAPPING)))
order = "post"
Modified: python/branches/py3k/Lib/lib2to3/fixes/fix_next.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/fixes/fix_next.py (original)
+++ python/branches/py3k/Lib/lib2to3/fixes/fix_next.py Sat Nov 29 00:01:28 2008
@@ -9,7 +9,7 @@
from ..pgen2 import token
from ..pygram import python_symbols as syms
from .. import fixer_base
-from ..fixer_util import Name, Call, find_binding, any
+from ..fixer_util import Name, Call, find_binding
bind_warning = "Calls to builtin next() possibly shadowed by global binding"
Modified: python/branches/py3k/Lib/lib2to3/fixes/fix_numliterals.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/fixes/fix_numliterals.py (original)
+++ python/branches/py3k/Lib/lib2to3/fixes/fix_numliterals.py Sat Nov 29 00:01:28 2008
@@ -6,7 +6,7 @@
# Local imports
from ..pgen2 import token
from .. import fixer_base
-from ..fixer_util import Number, set
+from ..fixer_util import Number
class FixNumliterals(fixer_base.BaseFix):
Modified: python/branches/py3k/Lib/lib2to3/fixes/fix_renames.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/fixes/fix_renames.py (original)
+++ python/branches/py3k/Lib/lib2to3/fixes/fix_renames.py Sat Nov 29 00:01:28 2008
@@ -8,7 +8,7 @@
# Local imports
from .. import fixer_base
-from ..fixer_util import Name, attr_chain, any, set
+from ..fixer_util import Name, attr_chain
MAPPING = {"sys": {"maxint" : "maxsize"},
}
Modified: python/branches/py3k/Lib/lib2to3/fixes/fix_urllib.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/fixes/fix_urllib.py (original)
+++ python/branches/py3k/Lib/lib2to3/fixes/fix_urllib.py Sat Nov 29 00:01:28 2008
@@ -7,7 +7,7 @@
# Local imports
from .fix_imports import alternates, FixImports
from .. import fixer_base
-from ..fixer_util import Name, Comma, FromImport, Newline, attr_chain, any, set
+from ..fixer_util import Name, Comma, FromImport, Newline, attr_chain
MAPPING = {'urllib': [
('urllib.request',
@@ -65,7 +65,9 @@
class FixUrllib(FixImports):
- PATTERN = "|".join(build_pattern())
+
+ def build_pattern(self):
+ return "|".join(build_pattern())
def transform_import(self, node, results):
"""Transform for the basic import case. Replaces the old
Modified: python/branches/py3k/Lib/lib2to3/pgen2/parse.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/pgen2/parse.py (original)
+++ python/branches/py3k/Lib/lib2to3/pgen2/parse.py Sat Nov 29 00:01:28 2008
@@ -10,12 +10,6 @@
"""
-# Get a usable 'set' constructor
-try:
- set
-except NameError:
- from sets import Set as set
-
# Local imports
from . import token
Modified: python/branches/py3k/Lib/lib2to3/pytree.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/pytree.py (original)
+++ python/branches/py3k/Lib/lib2to3/pytree.py Sat Nov 29 00:01:28 2008
@@ -11,6 +11,9 @@
__author__ = "Guido van Rossum <guido at python.org>"
+import sys
+from io import StringIO
+
HUGE = 0x7FFFFFFF # maximum repeat count, default max
@@ -655,6 +658,11 @@
elif self.name == "bare_name":
yield self._bare_name_matches(nodes)
else:
+ # The reason for this is that hitting the recursion limit usually
+ # results in some ugly messages about how RuntimeErrors are being
+ # ignored.
+ save_stderr = sys.stderr
+ sys.stderr = StringIO()
try:
for count, r in self._recursive_matches(nodes, 0):
if self.name:
@@ -667,6 +675,8 @@
if self.name:
r[self.name] = nodes[:count]
yield count, r
+ finally:
+ sys.stderr = save_stderr
def _iterative_matches(self, nodes):
"""Helper to iteratively yield the matches."""
Modified: python/branches/py3k/Lib/lib2to3/refactor.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/refactor.py (original)
+++ python/branches/py3k/Lib/lib2to3/refactor.py Sat Nov 29 00:01:28 2008
@@ -123,8 +123,8 @@
logger=self.logger)
self.pre_order, self.post_order = self.get_fixers()
- self.pre_order = get_headnode_dict(self.pre_order)
- self.post_order = get_headnode_dict(self.post_order)
+ self.pre_order_mapping = get_headnode_dict(self.pre_order)
+ self.post_order_mapping = get_headnode_dict(self.post_order)
self.files = [] # List of files that were or should be modified
@@ -290,13 +290,12 @@
# Two calls to chain are required because pre_order.values()
# will be a list of lists of fixers:
# [[<fixer ...>, <fixer ...>], [<fixer ...>]]
- all_fixers = chain(chain(*self.pre_order.values()),\
- chain(*self.post_order.values()))
+ all_fixers = chain(self.pre_order, self.post_order)
for fixer in all_fixers:
fixer.start_tree(tree, name)
- self.traverse_by(self.pre_order, tree.pre_order())
- self.traverse_by(self.post_order, tree.post_order())
+ self.traverse_by(self.pre_order_mapping, tree.pre_order())
+ self.traverse_by(self.post_order_mapping, tree.post_order())
for fixer in all_fixers:
fixer.finish_tree(tree, name)
Modified: python/branches/py3k/Lib/lib2to3/tests/test_fixers.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/tests/test_fixers.py (original)
+++ python/branches/py3k/Lib/lib2to3/tests/test_fixers.py Sat Nov 29 00:01:28 2008
@@ -15,10 +15,7 @@
from operator import itemgetter
# Local imports
-from .. import pygram
-from .. import pytree
-from .. import refactor
-from .. import fixer_util
+from lib2to3 import pygram, pytree, refactor, fixer_util
class FixerTestCase(support.TestCase):
@@ -30,10 +27,9 @@
self.fixer_log = []
self.filename = "<string>"
- for order in (self.refactor.pre_order.values(),\
- self.refactor.post_order.values()):
- for fixer in chain(*order):
- fixer.log = self.fixer_log
+ for fixer in chain(self.refactor.pre_order,
+ self.refactor.post_order):
+ fixer.log = self.fixer_log
def _check(self, before, after):
before = support.reformat(before)
@@ -1488,6 +1484,44 @@
""" % (new, new)
self.check(b, a)
+ b = """
+ from %s import x
+ %s = 23
+ """ % (old, old)
+ a = """
+ from %s import x
+ %s = 23
+ """ % (new, old)
+ self.check(b, a)
+
+ s = """
+ def f():
+ %s.method()
+ """ % (old,)
+ self.unchanged(s)
+
+ # test nested usage
+ b = """
+ import %s
+ %s.bar(%s.foo)
+ """ % (old, old, old)
+ a = """
+ import %s
+ %s.bar(%s.foo)
+ """ % (new, new, new)
+ self.check(b, a)
+
+ b = """
+ import %s
+ x.%s
+ """ % (old, old)
+ a = """
+ import %s
+ x.%s
+ """ % (new, old)
+ self.check(b, a)
+
+
class Test_imports2(Test_imports):
fixer = "imports2"
Modified: python/branches/py3k/Lib/lib2to3/tests/test_refactor.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/tests/test_refactor.py (original)
+++ python/branches/py3k/Lib/lib2to3/tests/test_refactor.py Sat Nov 29 00:01:28 2008
@@ -161,7 +161,7 @@
self.assertEqual(len(rt.post_order), 0)
rt = self.rt(explicit=["myfixes.fix_explicit"])
- for fix in rt.post_order[None]:
+ for fix in rt.post_order:
if isinstance(fix, FixExplicit):
break
else:
Date: Sat Nov 29 00:01:28 2008
New Revision: 67429
Log:
Merged revisions 67428 via svnmerge from
svn+ssh://pythondev at svn.python.org/python/trunk
................
r67428 | benjamin.peterson | 2008-11-28 16:12:14 -0600 (Fri, 28 Nov 2008) | 57 lines
Merged revisions 67384,67386-67387,67389-67390,67392,67399-67400,67403-67405,67426 via svnmerge from
svn+ssh://pythondev at svn.python.org/sandbox/trunk/2to3/lib2to3
........
r67384 | benjamin.peterson | 2008-11-25 16:13:31 -0600 (Tue, 25 Nov 2008) | 4 lines
don't duplicate calls to start_tree()
RefactoringTool.pre_order values now holds a list of the fixers while pre_order_mapping holds the dict.
........
r67386 | benjamin.peterson | 2008-11-25 16:44:52 -0600 (Tue, 25 Nov 2008) | 1 line
#4423 fix_imports was still replacing usage of a module if attributes were being used
........
r67387 | benjamin.peterson | 2008-11-25 16:47:54 -0600 (Tue, 25 Nov 2008) | 1 line
fix broken test
........
r67389 | benjamin.peterson | 2008-11-25 17:13:17 -0600 (Tue, 25 Nov 2008) | 1 line
remove compatibility code; we only cater to 2.5+
........
r67390 | benjamin.peterson | 2008-11-25 22:03:36 -0600 (Tue, 25 Nov 2008) | 1 line
fix #3994; the usage of changed imports was fixed in nested cases
........
r67392 | benjamin.peterson | 2008-11-26 11:11:40 -0600 (Wed, 26 Nov 2008) | 1 line
simpilfy and comment fix_imports
........
r67399 | benjamin.peterson | 2008-11-26 11:47:03 -0600 (Wed, 26 Nov 2008) | 1 line
remove more compatibility code
........
r67400 | benjamin.peterson | 2008-11-26 12:07:41 -0600 (Wed, 26 Nov 2008) | 1 line
set svn:ignore
........
r67403 | benjamin.peterson | 2008-11-26 13:11:11 -0600 (Wed, 26 Nov 2008) | 1 line
wrap import
........
r67404 | benjamin.peterson | 2008-11-26 13:29:49 -0600 (Wed, 26 Nov 2008) | 1 line
build the fix_imports pattern in compile_pattern, so MAPPING can be changed and reflected in the pattern
........
r67405 | benjamin.peterson | 2008-11-26 14:01:24 -0600 (Wed, 26 Nov 2008) | 1 line
stop ugly messages about runtime errors being from printed
........
r67426 | benjamin.peterson | 2008-11-28 16:01:40 -0600 (Fri, 28 Nov 2008) | 5 lines
don't replace a module name if it is in the middle of a attribute lookup
This fix also stops module names from being replaced if they are not in an attribute lookup.
........
................
Modified:
python/branches/py3k/ (props changed)
python/branches/py3k/Lib/lib2to3/fixer_base.py
python/branches/py3k/Lib/lib2to3/fixer_util.py
python/branches/py3k/Lib/lib2to3/fixes/fix_dict.py
python/branches/py3k/Lib/lib2to3/fixes/fix_except.py
python/branches/py3k/Lib/lib2to3/fixes/fix_imports.py
python/branches/py3k/Lib/lib2to3/fixes/fix_imports2.py
python/branches/py3k/Lib/lib2to3/fixes/fix_next.py
python/branches/py3k/Lib/lib2to3/fixes/fix_numliterals.py
python/branches/py3k/Lib/lib2to3/fixes/fix_renames.py
python/branches/py3k/Lib/lib2to3/fixes/fix_urllib.py
python/branches/py3k/Lib/lib2to3/pgen2/parse.py
python/branches/py3k/Lib/lib2to3/pytree.py
python/branches/py3k/Lib/lib2to3/refactor.py
python/branches/py3k/Lib/lib2to3/tests/ (props changed)
python/branches/py3k/Lib/lib2to3/tests/data/ (props changed)
python/branches/py3k/Lib/lib2to3/tests/test_fixers.py
python/branches/py3k/Lib/lib2to3/tests/test_refactor.py
Modified: python/branches/py3k/Lib/lib2to3/fixer_base.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/fixer_base.py (original)
+++ python/branches/py3k/Lib/lib2to3/fixer_base.py Sat Nov 29 00:01:28 2008
@@ -7,12 +7,6 @@
import logging
import itertools
-# Get a usable 'set' constructor
-try:
- set
-except NameError:
- from sets import Set as set
-
# Local imports
from .patcomp import PatternCompiler
from . import pygram
Modified: python/branches/py3k/Lib/lib2to3/fixer_util.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/fixer_util.py (original)
+++ python/branches/py3k/Lib/lib2to3/fixer_util.py Sat Nov 29 00:01:28 2008
@@ -153,30 +153,6 @@
and node.children[0].value == "["
and node.children[-1].value == "]")
-###########################################################
-### Common portability code. This allows fixers to do, eg,
-### "from .util import set" and forget about it.
-###########################################################
-
-try:
- any = any
-except NameError:
- def any(l):
- for o in l:
- if o:
- return True
- return False
-
-try:
- set = set
-except NameError:
- from sets import Set as set
-
-try:
- reversed = reversed
-except NameError:
- def reversed(l):
- return l[::-1]
###########################################################
### Misc
Modified: python/branches/py3k/Lib/lib2to3/fixes/fix_dict.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/fixes/fix_dict.py (original)
+++ python/branches/py3k/Lib/lib2to3/fixes/fix_dict.py Sat Nov 29 00:01:28 2008
@@ -28,7 +28,7 @@
from .. import patcomp
from ..pgen2 import token
from .. import fixer_base
-from ..fixer_util import Name, Call, LParen, RParen, ArgList, Dot, set
+from ..fixer_util import Name, Call, LParen, RParen, ArgList, Dot
from .. import fixer_util
Modified: python/branches/py3k/Lib/lib2to3/fixes/fix_except.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/fixes/fix_except.py (original)
+++ python/branches/py3k/Lib/lib2to3/fixes/fix_except.py Sat Nov 29 00:01:28 2008
@@ -25,7 +25,7 @@
from .. import pytree
from ..pgen2 import token
from .. import fixer_base
-from ..fixer_util import Assign, Attr, Name, is_tuple, is_list, reversed
+from ..fixer_util import Assign, Attr, Name, is_tuple, is_list
def find_excepts(nodes):
for i, n in enumerate(nodes):
Modified: python/branches/py3k/Lib/lib2to3/fixes/fix_imports.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/fixes/fix_imports.py (original)
+++ python/branches/py3k/Lib/lib2to3/fixes/fix_imports.py Sat Nov 29 00:01:28 2008
@@ -1,9 +1,9 @@
"""Fix incompatible imports and module references."""
-# Author: Collin Winter
+# Authors: Collin Winter, Nick Edds
# Local imports
from .. import fixer_base
-from ..fixer_util import Name, attr_chain, any, set
+from ..fixer_util import Name, attr_chain
MAPPING = {'StringIO': 'io',
'cStringIO': 'io',
@@ -61,36 +61,49 @@
def build_pattern(mapping=MAPPING):
- mod_list = ' | '.join(["module='" + key + "'" for key in mapping.keys()])
- mod_name_list = ' | '.join(["module_name='" + key + "'" for key in mapping.keys()])
- yield """import_name< 'import' ((%s)
+ mod_list = ' | '.join(["module_name='%s'" % key for key in mapping])
+ bare_names = alternates(mapping.keys())
+
+ yield """name_import=import_name< 'import' ((%s)
| dotted_as_names< any* (%s) any* >) >
""" % (mod_list, mod_list)
yield """import_from< 'from' (%s) 'import' ['(']
( any | import_as_name< any 'as' any > |
import_as_names< any* >) [')'] >
- """ % mod_name_list
+ """ % mod_list
yield """import_name< 'import'
dotted_as_name< (%s) 'as' any > >
- """ % mod_name_list
- # Find usages of module members in code e.g. urllib.foo(bar)
- yield """power< (%s)
- trailer<'.' any > any* >
- """ % mod_name_list
- yield """bare_name=%s""" % alternates(mapping.keys())
+ """ % mod_list
+
+ # Find usages of module members in code e.g. thread.foo(bar)
+ yield "power< bare_with_attr=(%s) trailer<'.' any > any* >" % bare_names
+
class FixImports(fixer_base.BaseFix):
- PATTERN = "|".join(build_pattern())
+
order = "pre" # Pre-order tree traversal
+ # This is overridden in fix_imports2.
mapping = MAPPING
- # Don't match the node if it's within another match
+ def build_pattern(self):
+ return "|".join(build_pattern(self.mapping))
+
+ def compile_pattern(self):
+ # We override this, so MAPPING can be pragmatically altered and the
+ # changes will be reflected in PATTERN.
+ self.PATTERN = self.build_pattern()
+ super(FixImports, self).compile_pattern()
+
+ # Don't match the node if it's within another match.
def match(self, node):
match = super(FixImports, self).match
results = match(node)
if results:
- if any([match(obj) for obj in attr_chain(node, "parent")]):
+ # Module usage could be in the trailier of an attribute lookup, so
+ # we might have nested matches when "bare_with_attr" is present.
+ if "bare_with_attr" not in results and \
+ any([match(obj) for obj in attr_chain(node, "parent")]):
return False
return results
return False
@@ -100,20 +113,17 @@
self.replace = {}
def transform(self, node, results):
- import_mod = results.get("module")
- mod_name = results.get("module_name")
- bare_name = results.get("bare_name")
-
- if import_mod or mod_name:
- new_name = self.mapping[(import_mod or mod_name).value]
-
+ import_mod = results.get("module_name")
if import_mod:
- self.replace[import_mod.value] = new_name
+ new_name = self.mapping[(import_mod or mod_name).value]
+ if "name_import" in results:
+ # If it's not a "from x import x, y" or "import x as y" import,
+ # marked its usage to be replaced.
+ self.replace[import_mod.value] = new_name
import_mod.replace(Name(new_name, prefix=import_mod.get_prefix()))
- elif mod_name:
- mod_name.replace(Name(new_name, prefix=mod_name.get_prefix()))
- elif bare_name:
- bare_name = bare_name[0]
+ else:
+ # Replace usage of the module.
+ bare_name = results["bare_with_attr"][0]
new_name = self.replace.get(bare_name.value)
if new_name:
bare_name.replace(Name(new_name, prefix=bare_name.get_prefix()))
Modified: python/branches/py3k/Lib/lib2to3/fixes/fix_imports2.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/fixes/fix_imports2.py (original)
+++ python/branches/py3k/Lib/lib2to3/fixes/fix_imports2.py Sat Nov 29 00:01:28 2008
@@ -10,7 +10,6 @@
class FixImports2(fix_imports.FixImports):
- PATTERN = "|".join((fix_imports.build_pattern(MAPPING)))
order = "post"
Modified: python/branches/py3k/Lib/lib2to3/fixes/fix_next.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/fixes/fix_next.py (original)
+++ python/branches/py3k/Lib/lib2to3/fixes/fix_next.py Sat Nov 29 00:01:28 2008
@@ -9,7 +9,7 @@
from ..pgen2 import token
from ..pygram import python_symbols as syms
from .. import fixer_base
-from ..fixer_util import Name, Call, find_binding, any
+from ..fixer_util import Name, Call, find_binding
bind_warning = "Calls to builtin next() possibly shadowed by global binding"
Modified: python/branches/py3k/Lib/lib2to3/fixes/fix_numliterals.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/fixes/fix_numliterals.py (original)
+++ python/branches/py3k/Lib/lib2to3/fixes/fix_numliterals.py Sat Nov 29 00:01:28 2008
@@ -6,7 +6,7 @@
# Local imports
from ..pgen2 import token
from .. import fixer_base
-from ..fixer_util import Number, set
+from ..fixer_util import Number
class FixNumliterals(fixer_base.BaseFix):
Modified: python/branches/py3k/Lib/lib2to3/fixes/fix_renames.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/fixes/fix_renames.py (original)
+++ python/branches/py3k/Lib/lib2to3/fixes/fix_renames.py Sat Nov 29 00:01:28 2008
@@ -8,7 +8,7 @@
# Local imports
from .. import fixer_base
-from ..fixer_util import Name, attr_chain, any, set
+from ..fixer_util import Name, attr_chain
MAPPING = {"sys": {"maxint" : "maxsize"},
}
Modified: python/branches/py3k/Lib/lib2to3/fixes/fix_urllib.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/fixes/fix_urllib.py (original)
+++ python/branches/py3k/Lib/lib2to3/fixes/fix_urllib.py Sat Nov 29 00:01:28 2008
@@ -7,7 +7,7 @@
# Local imports
from .fix_imports import alternates, FixImports
from .. import fixer_base
-from ..fixer_util import Name, Comma, FromImport, Newline, attr_chain, any, set
+from ..fixer_util import Name, Comma, FromImport, Newline, attr_chain
MAPPING = {'urllib': [
('urllib.request',
@@ -65,7 +65,9 @@
class FixUrllib(FixImports):
- PATTERN = "|".join(build_pattern())
+
+ def build_pattern(self):
+ return "|".join(build_pattern())
def transform_import(self, node, results):
"""Transform for the basic import case. Replaces the old
Modified: python/branches/py3k/Lib/lib2to3/pgen2/parse.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/pgen2/parse.py (original)
+++ python/branches/py3k/Lib/lib2to3/pgen2/parse.py Sat Nov 29 00:01:28 2008
@@ -10,12 +10,6 @@
"""
-# Get a usable 'set' constructor
-try:
- set
-except NameError:
- from sets import Set as set
-
# Local imports
from . import token
Modified: python/branches/py3k/Lib/lib2to3/pytree.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/pytree.py (original)
+++ python/branches/py3k/Lib/lib2to3/pytree.py Sat Nov 29 00:01:28 2008
@@ -11,6 +11,9 @@
__author__ = "Guido van Rossum <guido at python.org>"
+import sys
+from io import StringIO
+
HUGE = 0x7FFFFFFF # maximum repeat count, default max
@@ -655,6 +658,11 @@
elif self.name == "bare_name":
yield self._bare_name_matches(nodes)
else:
+ # The reason for this is that hitting the recursion limit usually
+ # results in some ugly messages about how RuntimeErrors are being
+ # ignored.
+ save_stderr = sys.stderr
+ sys.stderr = StringIO()
try:
for count, r in self._recursive_matches(nodes, 0):
if self.name:
@@ -667,6 +675,8 @@
if self.name:
r[self.name] = nodes[:count]
yield count, r
+ finally:
+ sys.stderr = save_stderr
def _iterative_matches(self, nodes):
"""Helper to iteratively yield the matches."""
Modified: python/branches/py3k/Lib/lib2to3/refactor.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/refactor.py (original)
+++ python/branches/py3k/Lib/lib2to3/refactor.py Sat Nov 29 00:01:28 2008
@@ -123,8 +123,8 @@
logger=self.logger)
self.pre_order, self.post_order = self.get_fixers()
- self.pre_order = get_headnode_dict(self.pre_order)
- self.post_order = get_headnode_dict(self.post_order)
+ self.pre_order_mapping = get_headnode_dict(self.pre_order)
+ self.post_order_mapping = get_headnode_dict(self.post_order)
self.files = [] # List of files that were or should be modified
@@ -290,13 +290,12 @@
# Two calls to chain are required because pre_order.values()
# will be a list of lists of fixers:
# [[<fixer ...>, <fixer ...>], [<fixer ...>]]
- all_fixers = chain(chain(*self.pre_order.values()),\
- chain(*self.post_order.values()))
+ all_fixers = chain(self.pre_order, self.post_order)
for fixer in all_fixers:
fixer.start_tree(tree, name)
- self.traverse_by(self.pre_order, tree.pre_order())
- self.traverse_by(self.post_order, tree.post_order())
+ self.traverse_by(self.pre_order_mapping, tree.pre_order())
+ self.traverse_by(self.post_order_mapping, tree.post_order())
for fixer in all_fixers:
fixer.finish_tree(tree, name)
Modified: python/branches/py3k/Lib/lib2to3/tests/test_fixers.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/tests/test_fixers.py (original)
+++ python/branches/py3k/Lib/lib2to3/tests/test_fixers.py Sat Nov 29 00:01:28 2008
@@ -15,10 +15,7 @@
from operator import itemgetter
# Local imports
-from .. import pygram
-from .. import pytree
-from .. import refactor
-from .. import fixer_util
+from lib2to3 import pygram, pytree, refactor, fixer_util
class FixerTestCase(support.TestCase):
@@ -30,10 +27,9 @@
self.fixer_log = []
self.filename = "<string>"
- for order in (self.refactor.pre_order.values(),\
- self.refactor.post_order.values()):
- for fixer in chain(*order):
- fixer.log = self.fixer_log
+ for fixer in chain(self.refactor.pre_order,
+ self.refactor.post_order):
+ fixer.log = self.fixer_log
def _check(self, before, after):
before = support.reformat(before)
@@ -1488,6 +1484,44 @@
""" % (new, new)
self.check(b, a)
+ b = """
+ from %s import x
+ %s = 23
+ """ % (old, old)
+ a = """
+ from %s import x
+ %s = 23
+ """ % (new, old)
+ self.check(b, a)
+
+ s = """
+ def f():
+ %s.method()
+ """ % (old,)
+ self.unchanged(s)
+
+ # test nested usage
+ b = """
+ import %s
+ %s.bar(%s.foo)
+ """ % (old, old, old)
+ a = """
+ import %s
+ %s.bar(%s.foo)
+ """ % (new, new, new)
+ self.check(b, a)
+
+ b = """
+ import %s
+ x.%s
+ """ % (old, old)
+ a = """
+ import %s
+ x.%s
+ """ % (new, old)
+ self.check(b, a)
+
+
class Test_imports2(Test_imports):
fixer = "imports2"
Modified: python/branches/py3k/Lib/lib2to3/tests/test_refactor.py
==============================================================================
--- python/branches/py3k/Lib/lib2to3/tests/test_refactor.py (original)
+++ python/branches/py3k/Lib/lib2to3/tests/test_refactor.py Sat Nov 29 00:01:28 2008
@@ -161,7 +161,7 @@
self.assertEqual(len(rt.post_order), 0)
rt = self.rt(explicit=["myfixes.fix_explicit"])
- for fix in rt.post_order[None]:
+ for fix in rt.post_order:
if isinstance(fix, FixExplicit):
break
else: