gh-159465: Improve incomplete input detection

Include/errcode.h

@@ -38,6 +38,7 @@ extern "C" {
 #define E_BADSINGLE      27      /* Ill-formed single statement input */
 #define E_INTERACT_STOP  28      /* Interactive mode stopped tokenization */
 #define E_COLUMNOVERFLOW 29      /* Column offset overflow */
+#define E_INCOMPLETE     30      /* An unfinished token in partial input */
 
 #ifdef __cplusplus
 }

Include/internal/pycore_parser.h

@@ -27,13 +27,17 @@ _Py_DECLARE_STR(empty, "")
         }, \
     }
 
+/* The caller must initialize *incomplete to NULL if supplied. A returned AST
+   can also return an owned exception there for a missing line terminator.
+   Validate the AST before reporting that exception. */
 extern struct _mod* _PyParser_ASTFromString(
     const char *str,
     PyObject* filename,
     int mode,
     PyCompilerFlags *flags,
     PyArena *arena,
-    PyObject *module);
+    PyObject *module,
+    PyObject **incomplete);
 
 extern struct _mod* _PyParser_ASTFromFile(
     FILE *fp,

Lib/codeop.py

@@ -62,15 +62,10 @@ def _maybe_compile(compiler, source, filename, symbol, flags):
         warnings.simplefilter("ignore", (SyntaxWarning, DeprecationWarning))
         try:
             compiler(source, filename, symbol, flags=flags)
+        except _IncompleteInputError:
+            return None
         except SyntaxError:  # Let other compile() errors propagate.
-            try:
-                compiler(source + "\n", filename, symbol, flags=flags)
-                return None
-            except _IncompleteInputError:
-                return None
-            except SyntaxError:
-                pass
-                # fallthrough
+            pass
 
     return compiler(source, filename, symbol, incomplete_input=False)
 

Lib/test/test_codeop.py

@@ -124,6 +124,99 @@ def test_incomplete_string_diagnostics(self, symbol, prefix):
         self.assertEqual(cm.exception.args, (
             'incomplete input', ('<input>', 2, 9, text, 2, -1)))
 
+    @subTests('compiler', COMPILERS)
+    @subTests('newline', ('\n', '\r', '\r\n'))
+    def test_unterminated_trailing_line(self, compiler, newline):
+        for head, symbol in (('1', 'eval'), ('if 1: pass', 'single'),
+                             ('if 1:\n pass', 'single')):
+            for tail in (' ', '\t', '\f ', '\n '):
+                source = head + newline + tail
+                with self.subTest(source=source, symbol=symbol):
+                    if compiler in RAW_COMPILERS:
+                        with self.assertRaises(_IncompleteInputError):
+                            compiler(source, '<input>', symbol)
+                    else:
+                        self.assertIsNone(compiler(source, '<input>', symbol))
+                    self.assertValid(source + '\n', symbol, compiler=compiler)
+            for tail in ('\f', ' \f', '\n\f'):
+                with self.subTest(head=head, tail=tail, symbol=symbol):
+                    self.assertValid(head + newline + tail, symbol,
+                                     compiler=compiler)
+
+        for tail in ('# comment', ' # comment'):
+            self.assertValid('if 1: pass' + newline + tail, compiler=compiler)
+            self.assertValid('1' + newline + tail, 'eval', compiler=compiler)
+
+        for head in ('1', 'if 1: pass', 'if 1:\n pass'):
+            for tail in (' ', '\t', '\f', '# comment'):
+                self.assertValid(head + newline + tail, 'exec',
+                                 compiler=compiler)
+
+    @subTests('symbol', ('single', 'exec', 'eval'))
+    def test_unfinished_token_in_invalid_position(self, symbol):
+        for source in ('a b', '(a b', '1 """', '(1 """',
+                       'f"{lambda:\\', 't"{lambda:\\'):
+            for compiler in COMPILERS:
+                with self.subTest(source=source, compiler=compiler):
+                    with self.assertRaises(SyntaxError) as cm:
+                        compiler(source, '<input>', symbol)
+                    self.assertNotIsInstance(cm.exception, _IncompleteInputError)
+
+    @subTests('symbol', ('single', 'exec', 'eval'))
+    def test_invalid_partial_string_diagnostic(self, symbol):
+        with self.assertRaisesRegex(SyntaxError, 'invalid syntax'):
+            Compile()('a"""', '<input>', symbol)
+
+    @subTests('symbol', ('single', 'exec', 'eval'))
+    @subTests('prefix', ('', 'r', 'b', 'f', 't', 'rf', 'rt'))
+    def test_string_continuation(self, symbol, prefix):
+        source = prefix + '"first\\'
+        compile(source + '\nsecond"\n', '<input>', symbol)
+        self.assertIsNone(compile_command(source, symbol=symbol))
+        with self.assertRaises(_IncompleteInputError):
+            Compile()(source, '<input>', symbol)
+
+    @subTests('symbol', ('single', 'exec', 'eval'))
+    @subTests('newline', ('\n', '\r', '\r\n'))
+    def test_continued_final_line(self, symbol, newline):
+        source = '1\\' + newline
+        compile(source + '+ 2\n', '<input>', symbol)
+        self.assertIsNone(compile_command(source, symbol=symbol))
+        with self.assertRaises(_IncompleteInputError):
+            Compile()(source, '<input>', symbol)
+        with self.assertRaises(SyntaxError) as cm:
+            Compile()('1 + \\' + newline, '<input>', symbol)
+        # Appending LF to CR completes the CRLF line terminator.
+        if newline != '\r':
+            self.assertNotIsInstance(cm.exception, _IncompleteInputError)
+
+    @subTests('symbol', ('single', 'exec'))
+    def test_eof_before_nested_body(self, symbol):
+        for source in ('if True:\n    if True:',
+                       'if True:\n    def f():',
+                       'match 1:\n    case 1:'):
+            for tail in ('', '\n', '\n\n'):
+                with self.subTest(source=source, tail=tail):
+                    compile(source + tail + '\n        pass\n', '<input>', symbol)
+                    self.assertIsNone(compile_command(source + tail, symbol=symbol))
+                    with self.assertRaises(_IncompleteInputError):
+                        Compile()(source + tail, '<input>', symbol)
+
+    def test_incomplete_suite_semantic_errors(self):
+        cases = (
+            ('if True: break', "'break' outside loop"),
+            ('if True: return 1', "'return' outside function"),
+            ('def f(x, x):\n    pass', 'duplicate parameter'),
+        )
+        for source, message in cases:
+            for compiler in COMPILERS:
+                with self.subTest(source=source, compiler=compiler):
+                    with self.assertRaises(SyntaxError) as cm:
+                        compiler(source, '<input>', 'single')
+                    self.assertNotIsInstance(cm.exception, _IncompleteInputError)
+                    if compiler in RAW_COMPILERS:
+                        self.assertIn(message, str(cm.exception))
+
     @subTests('compiler', COMPILERS)
     def test_incomplete(self, compiler):
         ai = functools.partial(self.assertIncomplete, compiler=compiler)

Misc/NEWS.d/next/Library/2026-10-15-13-44-03.gh-issue-159465.Ph-m8w.rst

@@ -0,0 +1,2 @@
+Improve incomplete input detection in :mod:`codeop` without an extra parser
+pass, and preserve syntax and compiler errors.

Modules/_testinternalcapi/tokenizer.c

@@ -95,6 +95,35 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
         goto error;
     }
 
+    _PyTok_SourceClear(&source);
+    if (check(_PyTok_SourceAppendLineWithNewline(&source, "", 0) == 0,
+              "cannot append empty terminated line") < 0 ||
+            check(_PyTok_SourceAppendLineWithNewline(&source, tail, 4) == 1,
+                  "wrong terminated line offset") < 0 ||
+            check(source.len == 6 && source.nlines == 2 &&
+                      memcmp(source.bytes, "\ntail\n", 7) == 0,
+                  "wrong terminated source contents") < 0 ||
+            check_system_error(_PyTok_SourceAppendLineWithNewline(
+                &source, terminated_line, 2) < 0,
+                "accepted newline in line payload") < 0) {
+        goto error;
+    }
+    source.base_offset = PY_SSIZE_T_MAX - source.len;
+    if (check(_PyTok_SourceAppendLineWithNewline(&source, "", 0) < 0 &&
+                  PyErr_ExceptionMatches(PyExc_MemoryError),
+              "accepted overflowing appended newline") < 0) {
+        goto error;
+    }
+    PyErr_Clear();
+    source.base_offset = 0;
+    source.nlines = INT_MAX - 1;
+    if (check(_PyTok_SourceAppendLineWithNewline(&source, "", 0) < 0 &&
+                  PyErr_ExceptionMatches(PyExc_OverflowError),
+              "accepted overflowing newline count") < 0) {
+        goto error;
+    }
+    PyErr_Clear();
+
     _PyTok_SourceClear(&source);
     Py_RETURN_NONE;
 

Parser/action_helpers.c

@@ -1497,11 +1497,11 @@ _PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* raw_expressions, Token*b
 }
 
 expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) {
-    Py_ssize_t bsize;
-    char* bstr;
-    if (PyBytes_AsStringAndSize(tok->bytes, &bstr, &bsize) == -1) {
-        return NULL;
+    if (tok->bytes == NULL) {
+        return _PyPegen_raise_incomplete_input(p);
     }
+    Py_ssize_t bsize = PyBytes_GET_SIZE(tok->bytes);
+    const char *bstr = PyBytes_AS_STRING(tok->bytes);
 
     int is_raw = tok->is_raw;
 
@@ -1519,10 +1519,10 @@ expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) {
 }
 
 expr_ty _PyPegen_constant_from_token(Parser* p, Token* tok) {
-    const char* bstr = PyBytes_AsString(tok->bytes);
-    if (bstr == NULL) {
-        return NULL;
+    if (tok->bytes == NULL) {
+        return _PyPegen_raise_incomplete_input(p);
     }
+    const char *bstr = PyBytes_AS_STRING(tok->bytes);
     PyObject* str = PyUnicode_FromString(bstr);
     if (str == NULL) {
         return NULL;
@@ -1537,10 +1537,10 @@ expr_ty _PyPegen_constant_from_token(Parser* p, Token* tok) {
 }
 
 expr_ty _PyPegen_constant_from_string(Parser* p, Token* tok) {
-    const char* the_str = PyBytes_AsString(tok->bytes);
-    if (the_str == NULL) {
-        return NULL;
+    if (tok->bytes == NULL) {
+        return _PyPegen_raise_incomplete_input(p);
     }
+    const char *the_str = PyBytes_AS_STRING(tok->bytes);
     PyObject *s = _PyPegen_parse_string(p, tok);
     if (s == NULL) {
         _Pypegen_raise_decode_error(p);

Parser/lexer/layout.c

@@ -21,6 +21,9 @@ _PyLexer_ContinueLine(struct tok_state *tok)
         return -1;
     }
     c = tok_nextc(tok);
+    if (tok->implicit_newline == _PyTokenizer_APPEND_NEWLINE) {
+        tok->needs_newline |= _PyTokenizer_NEWLINE_CONTINUATION;
+    }
     if (c == EOF) {
         tok->done = E_EOF;
         tok->cur = tok->inp;
@@ -114,6 +117,11 @@ _PyLexer_BeginLine(struct tok_state *tok)
         }
     }
     tok_backup(tok, c);
+    if (c == '\n' && tok->implicit_newline == _PyTokenizer_IMPLY_NEWLINE &&
+        tok->level == 0 && (cont_line_col || col)) {
+        // Without the newline, a whitespace-only line affects indentation.
+        tok->needs_newline |= _PyTokenizer_NEWLINE_INDENT;
+    }
     if (c == '#' || c == '\n' || c == '\r') {
         int interactive = _PyTok_ReaderIsInteractive(tok);
         /* Lines with only whitespace and/or comments
@@ -171,6 +179,10 @@ _PyLexer_IndentationToken(struct tok_state *tok, struct token *token)
 int
 _PyLexer_Newline(struct tok_state *tok, struct token *token, int blankline)
 {
+    if (!blankline && tok->implicit_newline == _PyTokenizer_IMPLY_NEWLINE) {
+        // The statement's terminating newline was not in the input.
+        tok->needs_newline |= _PyTokenizer_NEWLINE_STATEMENT;
+    }
     tok->layout.at_bol = 1;
     if (blankline || tok->level > 0) {
         if (!tok->tok_extra_tokens) {

Parser/lexer/lexer.c

@@ -235,6 +235,11 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
                 return type;
             }
         }
+        if (tok->implicit_newline == _PyTokenizer_IMPLY_NEWLINE &&
+            tok->layout.depth > 0) {
+            // Ending this comment lets the next line dedent the suite.
+            tok->needs_newline |= _PyTokenizer_NEWLINE_STATEMENT;
+        }
         if (tok->tok_extra_tokens) {
             tok_backup(tok, c);  /* don't eat the newline or EOF */
             p_start = comment;
@@ -516,7 +521,7 @@ _PyTokenizer_Get(struct tok_state *tok, struct token *token)
             Py_UNREACHABLE();
         }
     }
-    if (tok_failed(tok)) {
+    if (tok_failed(tok) && tok->done != E_INCOMPLETE) {
         result = ERRORTOKEN;
     }
     token->type = result;

Parser/lexer/state.h

@@ -105,6 +105,8 @@ struct tok_state {
     int debug;
 #endif
     int barry_as_bdfl;
+    unsigned char allow_incomplete_input;
+    unsigned char needs_newline;
 };
 
 static inline ftstring_state *

Parser/lexer/string.c

@@ -21,6 +21,15 @@ string_error_token(struct tok_state *tok, struct token *token,
     return type;
 }
 
+static int
+incomplete_string_token(struct tok_state *tok, struct token *token,
+                        int type, _PyTok_Off start, _PyTok_Loc location)
+{
+    string_error_token(tok, token, start, location);
+    tok->done = E_INCOMPLETE;
+    return type;
+}
+
 int
 _PyLexer_record_ftstring_comment(struct tok_state *tok, ftstring_state *state,
                                  _PyTok_Off start, _PyTok_Off end)
@@ -337,6 +346,10 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
             if (tok_failed(tok)) {
                 return MAKE_TOKEN(ERRORTOKEN);
             }
+            if (c == EOF && tok->allow_incomplete_input) {
+                return incomplete_string_token(tok, token, STRING,
+                                               tok->start, tok->start_loc);
+            }
             int end_lineno = tok->lineno;
             _PyTok_Loc location = tok->start_loc;
             const char *line = _PyTok_SourcePointer(
@@ -450,6 +463,10 @@ _PyLexer_get_ftstring(struct tok_state *tok, ftstring_state *current, struct tok
             if (tok_failed(tok)) {
                 return MAKE_TOKEN(ERRORTOKEN);
             }
+            if (c == EOF && tok->allow_incomplete_input) {
+                return incomplete_string_token(tok, token,
+                    FTSTRING_MIDDLE(current), current->start, current->start_loc);
+            }
 
             if (in_format_spec && c == '\n') {
                 return MAKE_TOKEN(_PyTokenizer_syntaxerror(

Parser/peg_api.c

@@ -5,14 +5,14 @@
 mod_ty
 _PyParser_ASTFromString(const char *str, PyObject* filename, int mode,
                         PyCompilerFlags *flags, PyArena *arena,
-                        PyObject *module)
+                        PyObject *module, PyObject **incomplete)
 {
     if (PySys_Audit("compile", "yO", str, filename) < 0) {
         return NULL;
     }
 
     mod_ty result = _PyPegen_run_parser_from_string(str, mode, filename, flags,
-                                                    arena, module);
+                                                    arena, module, incomplete);
     return result;
 }
 
@@ -39,4 +39,4 @@ _PyParser_InteractiveASTFromFile(FILE *fp, PyObject *filename_ob, const char *en
     }
     return _PyPegen_run_parser_from_file_pointer(fp, mode, filename_ob, enc, ps1, ps2,
                                                  flags, errcode, interactive_src, arena);
-}
+}