diff --git a/Include/errcode.h b/Include/errcode.h index dac5cf068c99d6d..10cd9b74371f2b6 100644 --- a/Include/errcode.h +++ b/Include/errcode.h @@ -38,6 +38,7 @@ extern "C" { #define E_BADSINGLE 27 /* Ill-formed single statement input */ #define E_INTERACT_STOP 28 /* Interactive mode stopped tokenization */ #define E_COLUMNOVERFLOW 29 /* Column offset overflow */ +#define E_INCOMPLETE 30 /* An unfinished token in partial input */ #ifdef __cplusplus } diff --git a/Include/internal/pycore_parser.h b/Include/internal/pycore_parser.h index b89d02035db71f9..2355016d8a8fcb5 100644 --- a/Include/internal/pycore_parser.h +++ b/Include/internal/pycore_parser.h @@ -27,13 +27,17 @@ _Py_DECLARE_STR(empty, "") }, \ } +/* The caller must initialize *incomplete to NULL if supplied. A returned AST + can also return an owned exception there for a missing line terminator. + Validate the AST before reporting that exception. */ extern struct _mod* _PyParser_ASTFromString( const char *str, PyObject* filename, int mode, PyCompilerFlags *flags, PyArena *arena, - PyObject *module); + PyObject *module, + PyObject **incomplete); extern struct _mod* _PyParser_ASTFromFile( FILE *fp, diff --git a/Lib/codeop.py b/Lib/codeop.py index 40e88423119bc4f..140e960b35f4ef7 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -62,15 +62,10 @@ def _maybe_compile(compiler, source, filename, symbol, flags): warnings.simplefilter("ignore", (SyntaxWarning, DeprecationWarning)) try: compiler(source, filename, symbol, flags=flags) + except _IncompleteInputError: + return None except SyntaxError: # Let other compile() errors propagate. - try: - compiler(source + "\n", filename, symbol, flags=flags) - return None - except _IncompleteInputError: - return None - except SyntaxError: - pass - # fallthrough + pass return compiler(source, filename, symbol, incomplete_input=False) diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py index c75c0f8627bf084..69dbc2ddfb6bbe4 100644 --- a/Lib/test/test_codeop.py +++ b/Lib/test/test_codeop.py @@ -124,6 +124,99 @@ def test_incomplete_string_diagnostics(self, symbol, prefix): self.assertEqual(cm.exception.args, ( 'incomplete input', ('', 2, 9, text, 2, -1))) + @subTests('compiler', COMPILERS) + @subTests('newline', ('\n', '\r', '\r\n')) + def test_unterminated_trailing_line(self, compiler, newline): + for head, symbol in (('1', 'eval'), ('if 1: pass', 'single'), + ('if 1:\n pass', 'single')): + for tail in (' ', '\t', '\f ', '\n '): + source = head + newline + tail + with self.subTest(source=source, symbol=symbol): + if compiler in RAW_COMPILERS: + with self.assertRaises(_IncompleteInputError): + compiler(source, '', symbol) + else: + self.assertIsNone(compiler(source, '', symbol)) + self.assertValid(source + '\n', symbol, compiler=compiler) + for tail in ('\f', ' \f', '\n\f'): + with self.subTest(head=head, tail=tail, symbol=symbol): + self.assertValid(head + newline + tail, symbol, + compiler=compiler) + + for tail in ('# comment', ' # comment'): + self.assertValid('if 1: pass' + newline + tail, compiler=compiler) + self.assertValid('1' + newline + tail, 'eval', compiler=compiler) + + for head in ('1', 'if 1: pass', 'if 1:\n pass'): + for tail in (' ', '\t', '\f', '# comment'): + self.assertValid(head + newline + tail, 'exec', + compiler=compiler) + + @subTests('symbol', ('single', 'exec', 'eval')) + def test_unfinished_token_in_invalid_position(self, symbol): + for source in ('a b', '(a b', '1 """', '(1 """', + 'f"{lambda:\\', 't"{lambda:\\'): + for compiler in COMPILERS: + with self.subTest(source=source, compiler=compiler): + with self.assertRaises(SyntaxError) as cm: + compiler(source, '', symbol) + self.assertNotIsInstance(cm.exception, _IncompleteInputError) + + @subTests('symbol', ('single', 'exec', 'eval')) + def test_invalid_partial_string_diagnostic(self, symbol): + with self.assertRaisesRegex(SyntaxError, 'invalid syntax'): + Compile()('a"""', '', symbol) + + @subTests('symbol', ('single', 'exec', 'eval')) + @subTests('prefix', ('', 'r', 'b', 'f', 't', 'rf', 'rt')) + def test_string_continuation(self, symbol, prefix): + source = prefix + '"first\\' + compile(source + '\nsecond"\n', '', symbol) + self.assertIsNone(compile_command(source, symbol=symbol)) + with self.assertRaises(_IncompleteInputError): + Compile()(source, '', symbol) + + @subTests('symbol', ('single', 'exec', 'eval')) + @subTests('newline', ('\n', '\r', '\r\n')) + def test_continued_final_line(self, symbol, newline): + source = '1\\' + newline + compile(source + '+ 2\n', '', symbol) + self.assertIsNone(compile_command(source, symbol=symbol)) + with self.assertRaises(_IncompleteInputError): + Compile()(source, '', symbol) + with self.assertRaises(SyntaxError) as cm: + Compile()('1 + \\' + newline, '', symbol) + # Appending LF to CR completes the CRLF line terminator. + if newline != '\r': + self.assertNotIsInstance(cm.exception, _IncompleteInputError) + + @subTests('symbol', ('single', 'exec')) + def test_eof_before_nested_body(self, symbol): + for source in ('if True:\n if True:', + 'if True:\n def f():', + 'match 1:\n case 1:'): + for tail in ('', '\n', '\n\n'): + with self.subTest(source=source, tail=tail): + compile(source + tail + '\n pass\n', '', symbol) + self.assertIsNone(compile_command(source + tail, symbol=symbol)) + with self.assertRaises(_IncompleteInputError): + Compile()(source + tail, '', symbol) + + def test_incomplete_suite_semantic_errors(self): + cases = ( + ('if True: break', "'break' outside loop"), + ('if True: return 1', "'return' outside function"), + ('def f(x, x):\n pass', 'duplicate parameter'), + ) + for source, message in cases: + for compiler in COMPILERS: + with self.subTest(source=source, compiler=compiler): + with self.assertRaises(SyntaxError) as cm: + compiler(source, '', 'single') + self.assertNotIsInstance(cm.exception, _IncompleteInputError) + if compiler in RAW_COMPILERS: + self.assertIn(message, str(cm.exception)) + @subTests('compiler', COMPILERS) def test_incomplete(self, compiler): ai = functools.partial(self.assertIncomplete, compiler=compiler) diff --git a/Misc/NEWS.d/next/Library/2026-10-15-13-44-03.gh-issue-159465.Ph-m8w.rst b/Misc/NEWS.d/next/Library/2026-10-15-13-44-03.gh-issue-159465.Ph-m8w.rst new file mode 100644 index 000000000000000..21199b550f9a54f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-15-13-44-03.gh-issue-159465.Ph-m8w.rst @@ -0,0 +1,2 @@ +Improve incomplete input detection in :mod:`codeop` without an extra parser +pass, and preserve syntax and compiler errors. diff --git a/Modules/_testinternalcapi/tokenizer.c b/Modules/_testinternalcapi/tokenizer.c index 0c2bdee2e803b22..f41a517deefcefd 100644 --- a/Modules/_testinternalcapi/tokenizer.c +++ b/Modules/_testinternalcapi/tokenizer.c @@ -95,6 +95,35 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), goto error; } + _PyTok_SourceClear(&source); + if (check(_PyTok_SourceAppendLineWithNewline(&source, "", 0) == 0, + "cannot append empty terminated line") < 0 || + check(_PyTok_SourceAppendLineWithNewline(&source, tail, 4) == 1, + "wrong terminated line offset") < 0 || + check(source.len == 6 && source.nlines == 2 && + memcmp(source.bytes, "\ntail\n", 7) == 0, + "wrong terminated source contents") < 0 || + check_system_error(_PyTok_SourceAppendLineWithNewline( + &source, terminated_line, 2) < 0, + "accepted newline in line payload") < 0) { + goto error; + } + source.base_offset = PY_SSIZE_T_MAX - source.len; + if (check(_PyTok_SourceAppendLineWithNewline(&source, "", 0) < 0 && + PyErr_ExceptionMatches(PyExc_MemoryError), + "accepted overflowing appended newline") < 0) { + goto error; + } + PyErr_Clear(); + source.base_offset = 0; + source.nlines = INT_MAX - 1; + if (check(_PyTok_SourceAppendLineWithNewline(&source, "", 0) < 0 && + PyErr_ExceptionMatches(PyExc_OverflowError), + "accepted overflowing newline count") < 0) { + goto error; + } + PyErr_Clear(); + _PyTok_SourceClear(&source); Py_RETURN_NONE; diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 2d51795d887993d..2f8a5d84dd9755b 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1497,11 +1497,11 @@ _PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* raw_expressions, Token*b } expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) { - Py_ssize_t bsize; - char* bstr; - if (PyBytes_AsStringAndSize(tok->bytes, &bstr, &bsize) == -1) { - return NULL; + if (tok->bytes == NULL) { + return _PyPegen_raise_incomplete_input(p); } + Py_ssize_t bsize = PyBytes_GET_SIZE(tok->bytes); + const char *bstr = PyBytes_AS_STRING(tok->bytes); int is_raw = tok->is_raw; @@ -1519,10 +1519,10 @@ expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) { } expr_ty _PyPegen_constant_from_token(Parser* p, Token* tok) { - const char* bstr = PyBytes_AsString(tok->bytes); - if (bstr == NULL) { - return NULL; + if (tok->bytes == NULL) { + return _PyPegen_raise_incomplete_input(p); } + const char *bstr = PyBytes_AS_STRING(tok->bytes); PyObject* str = PyUnicode_FromString(bstr); if (str == NULL) { return NULL; @@ -1537,10 +1537,10 @@ expr_ty _PyPegen_constant_from_token(Parser* p, Token* tok) { } expr_ty _PyPegen_constant_from_string(Parser* p, Token* tok) { - const char* the_str = PyBytes_AsString(tok->bytes); - if (the_str == NULL) { - return NULL; + if (tok->bytes == NULL) { + return _PyPegen_raise_incomplete_input(p); } + const char *the_str = PyBytes_AS_STRING(tok->bytes); PyObject *s = _PyPegen_parse_string(p, tok); if (s == NULL) { _Pypegen_raise_decode_error(p); diff --git a/Parser/lexer/layout.c b/Parser/lexer/layout.c index 6091f9adeac958c..f503f36f76ff14c 100644 --- a/Parser/lexer/layout.c +++ b/Parser/lexer/layout.c @@ -21,6 +21,9 @@ _PyLexer_ContinueLine(struct tok_state *tok) return -1; } c = tok_nextc(tok); + if (tok->implicit_newline == _PyTokenizer_APPEND_NEWLINE) { + tok->needs_newline |= _PyTokenizer_NEWLINE_CONTINUATION; + } if (c == EOF) { tok->done = E_EOF; tok->cur = tok->inp; @@ -114,6 +117,11 @@ _PyLexer_BeginLine(struct tok_state *tok) } } tok_backup(tok, c); + if (c == '\n' && tok->implicit_newline == _PyTokenizer_IMPLY_NEWLINE && + tok->level == 0 && (cont_line_col || col)) { + // Without the newline, a whitespace-only line affects indentation. + tok->needs_newline |= _PyTokenizer_NEWLINE_INDENT; + } if (c == '#' || c == '\n' || c == '\r') { int interactive = _PyTok_ReaderIsInteractive(tok); /* Lines with only whitespace and/or comments @@ -171,6 +179,10 @@ _PyLexer_IndentationToken(struct tok_state *tok, struct token *token) int _PyLexer_Newline(struct tok_state *tok, struct token *token, int blankline) { + if (!blankline && tok->implicit_newline == _PyTokenizer_IMPLY_NEWLINE) { + // The statement's terminating newline was not in the input. + tok->needs_newline |= _PyTokenizer_NEWLINE_STATEMENT; + } tok->layout.at_bol = 1; if (blankline || tok->level > 0) { if (!tok->tok_extra_tokens) { diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 83071e9e0b8ddf0..39b39b981c1fc49 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -235,6 +235,11 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token return type; } } + if (tok->implicit_newline == _PyTokenizer_IMPLY_NEWLINE && + tok->layout.depth > 0) { + // Ending this comment lets the next line dedent the suite. + tok->needs_newline |= _PyTokenizer_NEWLINE_STATEMENT; + } if (tok->tok_extra_tokens) { tok_backup(tok, c); /* don't eat the newline or EOF */ p_start = comment; @@ -516,7 +521,7 @@ _PyTokenizer_Get(struct tok_state *tok, struct token *token) Py_UNREACHABLE(); } } - if (tok_failed(tok)) { + if (tok_failed(tok) && tok->done != E_INCOMPLETE) { result = ERRORTOKEN; } token->type = result; diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 51ad278b13c40db..9c1475356b05353 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -105,6 +105,8 @@ struct tok_state { int debug; #endif int barry_as_bdfl; + unsigned char allow_incomplete_input; + unsigned char needs_newline; }; static inline ftstring_state * diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index d0f568dfa857580..c2bef9773eab937 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -21,6 +21,15 @@ string_error_token(struct tok_state *tok, struct token *token, return type; } +static int +incomplete_string_token(struct tok_state *tok, struct token *token, + int type, _PyTok_Off start, _PyTok_Loc location) +{ + string_error_token(tok, token, start, location); + tok->done = E_INCOMPLETE; + return type; +} + int _PyLexer_record_ftstring_comment(struct tok_state *tok, ftstring_state *state, _PyTok_Off start, _PyTok_Off end) @@ -337,6 +346,10 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c) if (tok_failed(tok)) { return MAKE_TOKEN(ERRORTOKEN); } + if (c == EOF && tok->allow_incomplete_input) { + return incomplete_string_token(tok, token, STRING, + tok->start, tok->start_loc); + } int end_lineno = tok->lineno; _PyTok_Loc location = tok->start_loc; const char *line = _PyTok_SourcePointer( @@ -450,6 +463,10 @@ _PyLexer_get_ftstring(struct tok_state *tok, ftstring_state *current, struct tok if (tok_failed(tok)) { return MAKE_TOKEN(ERRORTOKEN); } + if (c == EOF && tok->allow_incomplete_input) { + return incomplete_string_token(tok, token, + FTSTRING_MIDDLE(current), current->start, current->start_loc); + } if (in_format_spec && c == '\n') { return MAKE_TOKEN(_PyTokenizer_syntaxerror( diff --git a/Parser/peg_api.c b/Parser/peg_api.c index e30ca0453bd3e13..6791478ba688a68 100644 --- a/Parser/peg_api.c +++ b/Parser/peg_api.c @@ -5,14 +5,14 @@ mod_ty _PyParser_ASTFromString(const char *str, PyObject* filename, int mode, PyCompilerFlags *flags, PyArena *arena, - PyObject *module) + PyObject *module, PyObject **incomplete) { if (PySys_Audit("compile", "yO", str, filename) < 0) { return NULL; } mod_ty result = _PyPegen_run_parser_from_string(str, mode, filename, flags, - arena, module); + arena, module, incomplete); return result; } @@ -39,4 +39,4 @@ _PyParser_InteractiveASTFromFile(FILE *fp, PyObject *filename_ob, const char *en } return _PyPegen_run_parser_from_file_pointer(fp, mode, filename_ob, enc, ps1, ps2, flags, errcode, interactive_src, arena); -} \ No newline at end of file +} diff --git a/Parser/pegen.c b/Parser/pegen.c index 4ca42a407bcde80..bb14a1e78c5ad35 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -247,23 +247,26 @@ static int initialize_token(Parser *p, Token *parser_token, struct token *new_token, int token_type) { assert(parser_token != NULL); - Py_ssize_t length; - const char *text = _PyToken_TextView(p->tok, new_token, &length); - parser_token->type = token_type == NAME - ? _get_keyword_or_name_type(p, text, length) : token_type; - if (token_needs_text(parser_token->type)) { - parser_token->bytes = PyBytes_FromStringAndSize(text, length); - if (parser_token->bytes == NULL) { - return -1; + parser_token->type = token_type; + parser_token->bytes = NULL; + if (token_needs_text(token_type) && new_token->span.start >= 0) { + Py_ssize_t length; + const char *text = _PyToken_TextView(p->tok, new_token, &length); + if (token_type == NAME) { + parser_token->type = _get_keyword_or_name_type(p, text, length); } - if (_PyArena_AddPyObject(p->arena, parser_token->bytes) < 0) { - Py_DECREF(parser_token->bytes); - return -1; + // A name reclassified as a keyword does not need its text. + if (parser_token->type == token_type) { + parser_token->bytes = PyBytes_FromStringAndSize(text, length); + if (parser_token->bytes == NULL) { + return -1; + } + if (_PyArena_AddPyObject(p->arena, parser_token->bytes) < 0) { + Py_DECREF(parser_token->bytes); + return -1; + } } } - else { - parser_token->bytes = NULL; - } parser_token->metadata = NULL; if (new_token->metadata != NULL) { @@ -287,7 +290,7 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to p->fill += 1; - if (token_type == ERRORTOKEN && _PyTokenizer_GetInfo(p->tok).status == E_DECODE) { + if (token_type == ERRORTOKEN && _PyTokenizer_GetStatus(p->tok) == E_DECODE) { return _Pypegen_raise_decode_error(p); } @@ -352,6 +355,13 @@ _PyPegen_fill_token(Parser *p) } int type = new_token.type; + // EOF can finish an ordinary token. Only a boundary token means that + // the grammar requested input beyond the last physical line. + if (type == ENDMARKER || + (type == DEDENT && (p->flags & PyPARSE_ALLOW_INCOMPLETE_INPUT) && + _PyTokenizer_GetStatus(p->tok) == E_EOF)) { + p->input_exhausted = 1; + } // If we have reached the end and we are in single input mode we need to insert a newline and reset the parsing if (p->start_rule == Py_single_input && type == ENDMARKER && p->parsing_started) { @@ -953,6 +963,7 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags, } assert(tok != NULL); _PyTokenizer_SetOptions(tok, 0, (flags & PyPARSE_TYPE_COMMENTS) > 0); + _PyTokenizer_AllowIncompleteInput(tok, flags & PyPARSE_ALLOW_INCOMPLETE_INPUT); p->tok = tok; p->keywords = NULL; p->n_keyword_lists = -1; @@ -998,6 +1009,7 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags, #endif p->level = 0; p->call_invalid_rules = 0; + p->input_exhausted = 0; p->last_stmt_location.lineno = 0; p->last_stmt_location.col_offset = 0; p->last_stmt_location.end_lineno = 0; @@ -1035,16 +1047,12 @@ reset_parser_state_for_error_pass(Parser *p) p->tokens[i]->memo_mask = 0; } p->mark = 0; + p->flags &= ~PyPARSE_ALLOW_INCOMPLETE_INPUT; + _PyTokenizer_AllowIncompleteInput(p->tok, 0); p->call_invalid_rules = 1; _PyTokenizer_StopInteractive(p->tok); } -static inline int -_is_end_of_source(Parser *p) { - int err = _PyTokenizer_GetInfo(p->tok).status; - return err == E_EOF || err == E_EOFS || err == E_EOLS; -} - static void _PyPegen_set_syntax_error_metadata(Parser *p) { PyObject *exc = PyErr_GetRaisedException(); @@ -1095,13 +1103,15 @@ _PyPegen_run_parser(Parser *p) return NULL; } if (res == NULL) { - if ((p->flags & PyPARSE_ALLOW_INCOMPLETE_INPUT) && _is_end_of_source(p)) { - PyErr_Clear(); - return _PyPegen_raise_error(p, PyExc_IncompleteInputError, 0, "incomplete input"); - } - if (PyErr_Occurred() && !PyErr_ExceptionMatches(PyExc_SyntaxError)) { + if (PyErr_Occurred() && + (PyErr_ExceptionMatches(PyExc_IncompleteInputError) || + !PyErr_ExceptionMatches(PyExc_SyntaxError))) { return NULL; } + if ((p->flags & PyPARSE_ALLOW_INCOMPLETE_INPUT) && + p->input_exhausted && !PyErr_Occurred()) { + return _PyPegen_raise_incomplete_input(p); + } // Make a second parser pass. In this pass we activate heavier and slower checks // to produce better error messages and more complete diagnostics. Extra "invalid_*" // rules will be active during parsing. @@ -1194,16 +1204,21 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena mod_ty _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filename_ob, - PyCompilerFlags *flags, PyArena *arena, PyObject *module) + PyCompilerFlags *flags, PyArena *arena, PyObject *module, + PyObject **incomplete) { - int exec_input = start_rule == Py_file_input; + _PyTokenizer_Newline newline_mode = start_rule == Py_file_input + ? _PyTokenizer_IMPLY_NEWLINE : _PyTokenizer_KEEP_NEWLINE; int parser_flags = compute_parser_flags(flags); + if (parser_flags & PyPARSE_ALLOW_INCOMPLETE_INPUT) { + newline_mode = _PyTokenizer_APPEND_NEWLINE; + } struct tok_state *tok; if (flags != NULL && flags->cf_flags & PyCF_IGNORE_COOKIE) { - tok = _PyTokenizer_FromUTF8(str, exec_input, 0); + tok = _PyTokenizer_FromUTF8(str, newline_mode, 0); } else { - tok = _PyTokenizer_FromString(str, exec_input, 0); + tok = _PyTokenizer_FromString(str, newline_mode, 0); } if (tok == NULL) { if (PyErr_Occurred()) { @@ -1231,6 +1246,22 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen } result = _PyPegen_run_parser(p); + if (result != NULL && (parser_flags & PyPARSE_ALLOW_INCOMPLETE_INPUT)) { + int needs_newline = _PyTokenizer_GetNewlineRequirements(tok); + if ((needs_newline & _PyTokenizer_NEWLINE_CONTINUATION) || + (start_rule != Py_file_input && + (needs_newline & _PyTokenizer_NEWLINE_INDENT)) || + (start_rule == Py_single_input && p->input_exhausted && + (needs_newline & _PyTokenizer_NEWLINE_STATEMENT))) { + _PyPegen_raise_incomplete_input(p); + if (incomplete != NULL && PyErr_ExceptionMatches(PyExc_IncompleteInputError)) { + *incomplete = PyErr_GetRaisedException(); + } + else { + result = NULL; + } + } + } _PyPegen_Parser_Free(p); error: diff --git a/Parser/pegen.h b/Parser/pegen.h index 7454289ef6df03d..44f993101328397 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -39,6 +39,8 @@ typedef struct _memo { typedef struct { int type; // Text required by grammar actions or helpers; otherwise NULL. + // An unfinished string token also has NULL text. Its grammar action + // requests more input only if the token is valid at that position. PyObject *bytes; int level; int is_raw; @@ -86,7 +88,8 @@ typedef struct { int n_keyword_lists; int start_rule; int *errcode; - int parsing_started; + unsigned char parsing_started; + unsigned char input_exhausted; PyObject* normalize; int starting_lineno; int starting_col_offset; @@ -210,6 +213,7 @@ typedef enum { // Error handling functions and APIs int _Pypegen_raise_decode_error(Parser *p); int _Pypegen_tokenizer_error(Parser *p); +void *_PyPegen_raise_incomplete_input(Parser *p); void *_PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *errmsg, ...); void *_PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, Py_ssize_t lineno, Py_ssize_t col_offset, @@ -422,7 +426,7 @@ mod_ty _PyPegen_run_parser_from_file_pointer(FILE *, int, PyObject *, const char const char *, const char *, PyCompilerFlags *, int *, PyObject **, PyArena *); void *_PyPegen_run_parser(Parser *); -mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *, PyObject *); +mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *, PyObject *, PyObject **); asdl_stmt_seq *_PyPegen_interactive_exit(Parser *); // Generated function in parse.c - function definition in python.gram diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index 6997a1158e9310f..007397fa0358229 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -9,6 +9,16 @@ // TOKENIZER ERRORS +void * +_PyPegen_raise_incomplete_input(Parser *p) +{ + if ((p->flags & PyPARSE_ALLOW_INCOMPLETE_INPUT) && !PyErr_Occurred()) { + return _PyPegen_raise_error(p, PyExc_IncompleteInputError, 0, + "incomplete input"); + } + return NULL; +} + static inline void raise_unclosed_parentheses_error(Parser *p) { _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok); @@ -32,7 +42,17 @@ _Pypegen_tokenizer_error(Parser *p) Py_ssize_t col_offset = -1; p->error_indicator = 1; _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok); + if ((p->flags & PyPARSE_ALLOW_INCOMPLETE_INPUT) && + (info.status == E_EOF || info.status == E_INCOMPLETE)) + { + _PyPegen_raise_incomplete_input(p); + return -1; + } switch (info.status) { + case E_INCOMPLETE: + // The diagnostic pass rejected this unfinished token. Let the + // parser report the syntax error at its original failure point. + return -1; case E_TOKEN: msg = "invalid token"; break; @@ -298,6 +318,12 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, Py_ssize_t size; const char *line = _PyTokenizer_SpanView( p->tok, text_span, &size); + if (errtype == PyExc_IncompleteInputError && size > 0 && + (info.implicit_newline == _PyTokenizer_APPEND_NEWLINE || + (info.implicit_newline == _PyTokenizer_IMPLY_NEWLINE && + p->start_rule != Py_file_input))) { + size--; + } error_line = PyUnicode_DecodeUTF8(line, size, "replace"); } else if (!info.is_file) { diff --git a/Parser/tokenizer/api.c b/Parser/tokenizer/api.c index d955c6b335e1db6..99f0e4e0ddc1289 100644 --- a/Parser/tokenizer/api.c +++ b/Parser/tokenizer/api.c @@ -7,11 +7,24 @@ #include "reader_internal.h" #include "../lexer/state.h" +int +_PyTokenizer_GetStatus(const struct tok_state *tok) +{ + return tok->done; +} + +int +_PyTokenizer_GetNewlineRequirements(const struct tok_state *tok) +{ + return tok->needs_newline; +} + _PyTokenizer_Info _PyTokenizer_GetInfo(const struct tok_state *tok) { _PyTokenizer_Info info = { .status = tok->done, + .implicit_newline = tok->implicit_newline, .diagnostic = tok->diagnostic, .location = {tok->lineno, tok->line_start < 0 ? -1 : (int)(tok->cur - tok->line_start)}, @@ -103,6 +116,12 @@ _PyTokenizer_ImplyDedents(struct tok_state *tok) _PyLexer_ImplyDedents(tok); } +void +_PyTokenizer_AllowIncompleteInput(struct tok_state *tok, int allow) +{ + tok->allow_incomplete_input = allow != 0; +} + int _PyTokenizer_HasTrailingStatement(const struct tok_state *tok) { diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c index 48f906845651aab..6388b71e635cc3f 100644 --- a/Parser/tokenizer/decoder.c +++ b/Parser/tokenizer/decoder.c @@ -14,7 +14,6 @@ #define UTF16_BOM_LENGTH ((Py_ssize_t)(sizeof(UTF16_LE_BOM) - 1)) #define UTF8_BOM "\xef\xbb\xbf" #define UTF8_BOM_LENGTH ((Py_ssize_t)(sizeof(UTF8_BOM) - 1)) -#define NORMALIZED_LINE_INITIAL_CAPACITY 256 char * _PyTok_CopyBytes(const char *data, Py_ssize_t len) @@ -284,9 +283,6 @@ store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len, int preserve_crlf, int add_final_newline) { Py_ssize_t pos = 0; - // SourceAppendLine copies the bytes, so reuse this buffer for each line. - char *normalized = NULL; - Py_ssize_t capacity = 0; while (pos < len) { Py_ssize_t raw_line_len; if (preserve_crlf) { @@ -304,52 +300,45 @@ store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len, int add_newline = add_final_newline && pos + raw_line_len == len && !terminated; const char *line = data + pos; - Py_ssize_t line_len = raw_line_len; // raw_line_length stops at the first CR or LF, so any CR is in // the line terminator. Py_ssize_t terminator_len = 0; if (!preserve_crlf) { - if (line[line_len - 1] == '\r') { + if (line[raw_line_len - 1] == '\r') { terminator_len = 1; } - else if (line_len > 1 && line[line_len - 2] == '\r') { + else if (raw_line_len > 1 && line[raw_line_len - 2] == '\r') { terminator_len = 2; } } + _PyTok_Off appended; if (terminator_len || add_newline) { - Py_ssize_t payload_len = line_len - terminator_len; - if (payload_len > PY_SSIZE_T_MAX - 2) { - PyErr_NoMemory(); - tok->done = E_NOMEM; - goto error; - } - Py_ssize_t needed = payload_len + 2; - if (_PyTok_ReserveBuffer(&normalized, &capacity, needed, - NORMALIZED_LINE_INITIAL_CAPACITY) < 0) { - tok->done = E_NOMEM; - goto error; - } - memcpy(normalized, line, payload_len); - normalized[payload_len] = '\n'; - line_len = payload_len + 1; - normalized[line_len] = '\0'; - line = normalized; + appended = _PyTok_SourceAppendLineWithNewline( + &tok->source, line, raw_line_len - terminator_len); + } + else { + appended = _PyTok_SourceAppendLine(&tok->source, line, raw_line_len); } - _PyTok_Off appended = _PyTok_SourceAppendLine( - &tok->source, line, line_len); if (appended < 0) { tok->done = PyErr_ExceptionMatches(PyExc_MemoryError) ? E_NOMEM : E_ERROR; - goto error; + return -1; } - tok->reader->prepared_final_newline_is_implicit = add_newline; + tok->reader->prepared_final_newline = add_newline; pos += raw_line_len; } - PyMem_Free(normalized); + // Partial input includes the newline submitted by the next Enter key. + // A final CR pairs with that newline instead of starting a blank line. + if (add_final_newline == _PyTokenizer_APPEND_NEWLINE && + !tok->reader->prepared_final_newline && + (len == 0 || data[len - 1] != '\r')) + { + if (_PyTok_SourceAppendLine(&tok->source, "\n", 1) < 0) { + return -1; + } + tok->reader->prepared_final_newline = _PyTokenizer_APPEND_NEWLINE; + } return 0; -error: - PyMem_Free(normalized); - return -1; } int diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index ce2bb1cf5c4bc30..86cec96bdcc1249 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -180,8 +180,8 @@ next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk) chunk->data = (char *)_PyTok_SourceSpanView( &tok->source, line, &chunk->len); chunk->ownership = _PYTOK_CHUNK_BORROWED; - chunk->implicit_newline = line.end == tail.end && - tok->reader->prepared_final_newline_is_implicit; + chunk->implicit_newline = line.end == tail.end + ? tok->reader->prepared_final_newline : 0; return _PYTOK_READ_LINE; } diff --git a/Parser/tokenizer/reader_internal.h b/Parser/tokenizer/reader_internal.h index a7a68269fffaa4a..c080af3c05eef52 100644 --- a/Parser/tokenizer/reader_internal.h +++ b/Parser/tokenizer/reader_internal.h @@ -58,7 +58,7 @@ typedef struct _PyTok_Reader { Py_ssize_t decoded_len; Py_ssize_t decoded_cap; _PyTok_ReaderKind kind; - unsigned char prepared_final_newline_is_implicit; + unsigned char prepared_final_newline; // _PyTokenizer_Newline unsigned char decoded_tail_is_implicit; unsigned char file_initialized; unsigned char file_eof; diff --git a/Parser/tokenizer/source.c b/Parser/tokenizer/source.c index ebb7b05507c56ec..8688ef0f06e8302 100644 --- a/Parser/tokenizer/source.c +++ b/Parser/tokenizer/source.c @@ -71,51 +71,68 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed) static int validate_line(const _PyTok_SourceText *source, const char *bytes, - Py_ssize_t len) + Py_ssize_t len, int append_newline) { - if (len <= 0 || bytes == NULL || + if (len < 0 || (len == 0 && !append_newline) || bytes == NULL || (source->nlines > 0 && (source->len == 0 || source->bytes[source->len - 1] != '\n'))) { PyErr_SetString(PyExc_SystemError, "invalid tokenizer source line"); return -1; } const char *newline = memchr(bytes, '\n', len); - if (newline != NULL && newline != bytes + len - 1) { + if (newline != NULL && (append_newline || newline != bytes + len - 1)) { PyErr_SetString(PyExc_SystemError, "invalid tokenizer source line"); return -1; } if (source->nlines == INT_MAX || - (source->nlines == INT_MAX - 1 && newline != NULL)) { + (source->nlines == INT_MAX - 1 && (newline != NULL || append_newline))) { PyErr_SetString(PyExc_OverflowError, "too many tokenizer source lines"); return -1; } return 0; } -_PyTok_Off -_PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, - Py_ssize_t len) +static _PyTok_Off +append_line(_PyTok_SourceText *source, const char *bytes, + Py_ssize_t len, int append_newline) { - if (validate_line(source, bytes, len) < 0) { + if (validate_line(source, bytes, len, append_newline) < 0) { return -1; } - if (source->len > PY_SSIZE_T_MAX - len - 1 || - source->base_offset > PY_SSIZE_T_MAX - source->len - len) { + if (source->len > PY_SSIZE_T_MAX - len - append_newline - 1 || + source->base_offset > PY_SSIZE_T_MAX - source->len - len - append_newline) { PyErr_NoMemory(); return -1; } - if (reserve_bytes(source, source->len + len + 1) < 0) { + if (reserve_bytes(source, source->len + len + append_newline + 1) < 0) { return -1; } _PyTok_Off start = source->len; memcpy(source->bytes + start, bytes, len); source->len += len; + if (append_newline) { + source->bytes[source->len++] = '\n'; + } source->bytes[source->len] = '\0'; source->nlines++; return source->base_offset + start; } +_PyTok_Off +_PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, + Py_ssize_t len) +{ + return append_line(source, bytes, len, 0); +} + +_PyTok_Off +_PyTok_SourceAppendLineWithNewline(_PyTok_SourceText *source, const char *bytes, + Py_ssize_t len) +{ + return append_line(source, bytes, len, 1); +} + const char * _PyTok_SourceLineView(const _PyTok_SourceText *source, Py_ssize_t lineno, Py_ssize_t *len) diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h index 0fd9fe1877bed2b..d2c202830479523 100644 --- a/Parser/tokenizer/source.h +++ b/Parser/tokenizer/source.h @@ -70,6 +70,10 @@ PyAPI_FUNC(void) _PyTok_SourceDiscard(_PyTok_SourceText *); final line. The input must not point into source storage. */ PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine( _PyTok_SourceText *source, const char *bytes, Py_ssize_t len); +/* Append a line payload followed by '\n'. The payload may be empty and must + contain no newline. Other SourceAppendLine constraints still apply. */ +PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLineWithNewline( + _PyTok_SourceText *source, const char *bytes, Py_ssize_t len); /* Return borrowed bytes excluding '\n', writing the byte length to *len. Line numbers are 1-based and clamp to the first or final line; a trailing '\n' adds an empty final line. The view need not be NUL-terminated. diff --git a/Parser/tokenizer/tokenizer.h b/Parser/tokenizer/tokenizer.h index bc4009700fdc967..149bca06b8b7099 100644 --- a/Parser/tokenizer/tokenizer.h +++ b/Parser/tokenizer/tokenizer.h @@ -6,6 +6,19 @@ struct tok_state; +typedef enum { + _PyTokenizer_KEEP_NEWLINE, // Preserve the original line terminator. + _PyTokenizer_IMPLY_NEWLINE, // Add a missing line terminator. + _PyTokenizer_APPEND_NEWLINE, // Also append a blank line to terminated input. +} _PyTokenizer_Newline; + +/* Reasons that the appended newline can change the token stream. */ +enum { + _PyTokenizer_NEWLINE_STATEMENT = 1, + _PyTokenizer_NEWLINE_INDENT = 2, + _PyTokenizer_NEWLINE_CONTINUATION = 4, +}; + /* Initialize before use. metadata owns a reference released by _PyToken_Free; a consumer taking that reference must set metadata to NULL. */ struct token { @@ -37,6 +50,7 @@ typedef struct { typedef struct { int status; + unsigned char implicit_newline; // _PyTokenizer_Newline _PyTokenizer_Diagnostic diagnostic; _PyTok_Loc location; _PyTok_Off cursor; @@ -54,7 +68,9 @@ typedef struct { } _PyTokenizer_Info; /* Get replaces the initialized token, releasing its previous metadata. - Errors are returned as ERRORTOKEN, with or without a Python exception. */ + Errors are returned as ERRORTOKEN, with or without a Python exception. + In incomplete-input mode, an unfinished string retains its token type + with an absent span and E_INCOMPLETE status. No Python exception is set. */ void _PyTokenizer_Get(struct tok_state *, struct token *); void _PyTokenizer_Free(struct tok_state *); int _PyTokenizer_Traverse(struct tok_state *, visitproc, void *); @@ -69,6 +85,8 @@ _PyToken_Free(struct token *token) /* Views and borrowed snapshot references remain valid until the tokenizer is mutated or freed. Source spans may be discarded when reading more input. */ _PyTokenizer_Info _PyTokenizer_GetInfo(const struct tok_state *); +int _PyTokenizer_GetStatus(const struct tok_state *); +int _PyTokenizer_GetNewlineRequirements(const struct tok_state *); /* An absent token span has a nonnull empty text view. */ const char *_PyToken_TextView( const struct tok_state *, const struct token *, Py_ssize_t *); @@ -91,6 +109,7 @@ void _PyTokenizer_SetContext( struct tok_state *tok, PyObject *filename, PyObject *module); void _PyTokenizer_SetOptions( struct tok_state *tok, int extra_tokens, int type_comments); +void _PyTokenizer_AllowIncompleteInput(struct tok_state *, int); void _PyTokenizer_ImplyDedents(struct tok_state *); int _PyTokenizer_HasTrailingStatement(const struct tok_state *); int _PyTokenizer_IsInteractive(const struct tok_state *); diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 049eaea4994ac46..5e026a93d5c1806 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1285,7 +1285,7 @@ _PyRun_String(const char *str, PyObject* name, int start, goto done; } - mod = _PyParser_ASTFromString(str, name, start, flags, arena, module); + mod = _PyParser_ASTFromString(str, name, start, flags, arena, module, NULL); Py_XDECREF(module); if (mod != NULL) { @@ -1569,32 +1569,43 @@ _Py_CompileString(const char *str, PyObject *filename, int start, return NULL; } - PyCodeObject *co; - mod_ty mod; PyArena *arena = _PyArena_New(); if (arena == NULL) return NULL; - mod = _PyParser_ASTFromString(str, filename, start, flags, arena, module); + PyObject *result = NULL; + PyObject *incomplete = NULL; + mod_ty mod = _PyParser_ASTFromString(str, filename, start, flags, arena, + module, &incomplete); if (mod == NULL) { - _PyArena_Free(arena); - return NULL; + goto done; } if (flags && (flags->cf_flags & PyCF_ONLY_AST)) { int syntax_check_only = ((flags->cf_flags & PyCF_OPTIMIZED_AST) == PyCF_ONLY_AST); /* unoptiomized AST */ if (_PyCompile_AstPreprocess(mod, filename, flags, optimize, arena, syntax_check_only, module) < 0) { - _PyArena_Free(arena); - return NULL; + goto done; } - PyObject *result = PyAST_mod2obj(mod); + if (incomplete != NULL) { + goto done; + } + result = PyAST_mod2obj(mod); _PyArena_Free(arena); return result; } - co = _PyAST_Compile(mod, filename, flags, optimize, arena, module); + result = (PyObject *)_PyAST_Compile(mod, filename, flags, optimize, arena, module); + +done: _PyArena_Free(arena); - return (PyObject *)co; + if (incomplete != NULL && !PyErr_Occurred()) { + Py_CLEAR(result); + PyErr_SetRaisedException(incomplete); + } + else { + Py_XDECREF(incomplete); + } + return result; } PyObject * diff --git a/Python/symtable.c b/Python/symtable.c index b97778ada20c04b..7f945dad1984ed9 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -3271,7 +3271,7 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, if (arena == NULL) return NULL; - mod = _PyParser_ASTFromString(str, filename, start, flags, arena, module); + mod = _PyParser_ASTFromString(str, filename, start, flags, arena, module, NULL); if (mod == NULL) { _PyArena_Free(arena); return NULL; diff --git a/Tools/peg_generator/peg_extension/peg_extension.c b/Tools/peg_generator/peg_extension/peg_extension.c index 2fec5b0512940f3..83dd668a06987eb 100644 --- a/Tools/peg_generator/peg_extension/peg_extension.c +++ b/Tools/peg_generator/peg_extension/peg_extension.c @@ -93,7 +93,7 @@ parse_string(PyObject *self, PyObject *args, PyObject *kwds) PyCompilerFlags flags = _PyCompilerFlags_INIT; mod_ty res = _PyPegen_run_parser_from_string(the_string, Py_file_input, filename_ob, - &flags, arena, NULL); + &flags, arena, NULL, NULL); if (res == NULL) { goto error; }