[Python-3000-checkins] r66951 - in python/branches/py3k: Lib/test/test_pep3120.py Misc/NEWS Parser/tokenizer.c Parser/tokenizer.h Python/ast.c

[brett.cannon]

Author: brett.cannon
Date: Fri Oct 17 05:38:50 2008
New Revision: 66951

Log:
Latin-1 source code was not being properly decoded when passed through
compile(). This was due to left-over special-casing before UTF-8 became the
default source encoding.

Closes issue #3574. Thanks to Victor Stinner for help with the patch.


Modified:
python/branches/py3k/Lib/test/test_pep3120.py
python/branches/py3k/Misc/NEWS
python/branches/py3k/Parser/tokenizer.c
python/branches/py3k/Parser/tokenizer.h
python/branches/py3k/Python/ast.c

Modified: python/branches/py3k/Lib/test/test_pep3120.py
==============================================================================
Binary files. No diff available.

Modified: python/branches/py3k/Misc/NEWS
==============================================================================
--- python/branches/py3k/Misc/NEWS (original)
+++ python/branches/py3k/Misc/NEWS Fri Oct 17 05:38:50 2008
@@ -15,6 +15,8 @@
Core and Builtins
-----------------

+- Issue #3574: compile() incorrectly handled source code encoded as Latin-1.
+
- Issues #2384 and #3975: Tracebacks were not correctly printed when the
source file contains a ``coding:`` header: the wrong line was displayed, and
the encoding was not respected.

Modified: python/branches/py3k/Parser/tokenizer.c
==============================================================================
--- python/branches/py3k/Parser/tokenizer.c (original)
+++ python/branches/py3k/Parser/tokenizer.c Fri Oct 17 05:38:50 2008
@@ -135,6 +135,7 @@
tok->decoding_state = STATE_INIT;
tok->decoding_erred = 0;
tok->read_coding_spec = 0;
+ tok->enc = NULL;
tok->encoding = NULL;
tok->cont_line = 0;
#ifndef PGEN
@@ -274,8 +275,7 @@
tok->read_coding_spec = 1;
if (tok->encoding == NULL) {
assert(tok->decoding_state == STATE_RAW);
- if (strcmp(cs, "utf-8") == 0 ||
- strcmp(cs, "iso-8859-1") == 0) {
+ if (strcmp(cs, "utf-8") == 0) {
tok->encoding = cs;
} else {
r = set_readline(tok, cs);

Modified: python/branches/py3k/Parser/tokenizer.h
==============================================================================
--- python/branches/py3k/Parser/tokenizer.h (original)
+++ python/branches/py3k/Parser/tokenizer.h Fri Oct 17 05:38:50 2008
@@ -49,14 +49,14 @@
enum decoding_state decoding_state;
int decoding_erred; /* whether erred in decoding */
int read_coding_spec; /* whether 'coding:...' has been read */
- char *encoding;
+ char *encoding; /* Source encoding. */
int cont_line; /* whether we are in a continuation line. */
const char* line_start; /* pointer to start of current line */
#ifndef PGEN
PyObject *decoding_readline; /* codecs.open(...).readline */
PyObject *decoding_buffer;
#endif
- const char* enc;
+ const char* enc; /* Encoding for the current str. */
const char* str;
};


Modified: python/branches/py3k/Python/ast.c
==============================================================================
--- python/branches/py3k/Python/ast.c (original)
+++ python/branches/py3k/Python/ast.c Fri Oct 17 05:38:50 2008
@@ -3160,9 +3160,6 @@
if (encoding == NULL) {
buf = (char *)s;
u = NULL;
- } else if (strcmp(encoding, "iso-8859-1") == 0) {
- buf = (char *)s;
- u = NULL;
} else {
/* check for integer overflow */
if (len > PY_SIZE_MAX / 4)
@@ -3275,8 +3272,7 @@
}
}
need_encoding = (!*bytesmode && c->c_encoding != NULL &&
- strcmp(c->c_encoding, "utf-8") != 0 &&
- strcmp(c->c_encoding, "iso-8859-1") != 0);
+ strcmp(c->c_encoding, "utf-8") != 0);
if (rawmode || strchr(s, '\\') == NULL) {
if (need_encoding) {
PyObject *v, *u = PyUnicode_DecodeUTF8(s, len, NULL);