gh-151464: exclude '<>' token from tokenize output (#154854)

* gh-151464: exclude '<>' token from tokenize output

Was:
```
$ echo '1 <> 2' | python -m tokenize
1,0-1,1:            NUMBER         '1'
1,2-1,4:            OP             '<>'
1,5-1,6:            NUMBER         '2'
1,6-1,7:            NEWLINE        '\n'
2,0-2,0:            ENDMARKER      ''
```

Now (regardless on ``__future__.barry_as_FLUFL`` import):
```
$ echo '1 <> 2' | ./python -m tokenize
1,0-1,1:            NUMBER         '1'
1,2-1,3:            OP             '<'
1,3-1,4:            OP             '>'
1,5-1,6:            NUMBER         '2'
1,6-1,7:            NEWLINE        '\n'
2,0-2,0:            ENDMARKER      ''
```
in accordance with the Grammar:
https://docs.python.org/3.14/reference/lexical_analysis.html#operators-and-delimiters

Also adds a custom error message for ``<>`` ("not equal" in Pascal
and Python 2).

* +1

* address review: lowercase and move invalid rule

* address review: news

* address review: move test_guido_as_bdfl_ineq_tokens()

* address review: revert _PyTokenizer_From* changes

* + revert unrelated change

* address review: remove whatsnew entry

Grammar/python.gram

@@ -789,6 +789,7 @@ compare_op_bitwise_or_pair[CmpopExprPair*]:
     | (tok='!=' { _PyPegen_check_barry_as_flufl(p, tok) ? NULL : tok }) a=bitwise_or {
         _PyPegen_cmpop_expr_pair(p, NotEq, a) }
     | '<=' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, LtE, a) }
+    | invalid_noteq
     | '<' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Lt, a) }
     | '>=' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, GtE, a) }
     | '>' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Gt, a) }
@@ -1615,3 +1616,10 @@ invalid_bitwise_or:
         ? RAISE_SYNTAX_ERROR_KNOWN_RANGE(b, c, "invalid syntax.  Maybe you meant 'or' or '|' instead of '||'?")
         : NULL
     }
+
+invalid_noteq:
+    | a='<' b='>' {
+      _PyPegen_tokens_are_adjacent(a, b)
+      ? RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax.  Maybe you meant '!=' instead of '<>'?")
+      : NULL
+    }

Include/internal/pycore_token.h

@@ -101,7 +101,7 @@ extern "C" {
 // Export these 4 symbols for 'test_peg_generator'
 PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */
 PyAPI_FUNC(int) _PyToken_OneChar(int);
-PyAPI_FUNC(int) _PyToken_TwoChars(int, int);
+PyAPI_FUNC(int) _PyToken_TwoChars(int, int, int);
 PyAPI_FUNC(int) _PyToken_ThreeChars(int, int, int);
 
 #ifdef __cplusplus

Lib/test/test_syntax.py

@@ -3644,6 +3644,31 @@ while 1:
         ]:
             self._check_error(f"x = {lhs_stmt} if 1 else {rhs_stmt}", msg)
 
+    def test_diamond_operator(self):
+        self._check_error(
+            "1<>2",
+            r"Maybe you meant '!=' instead of '<>'\?",
+            lineno=1,
+            end_lineno=1,
+            offset=2,
+            end_offset=4,
+        )
+
+    def test_diamond_operator_barry_as_flufl(self):
+        # Under barry_as_FLUFL, '<>' is the valid "not equal" operator
+        compile(
+            "from __future__ import barry_as_FLUFL\n1<>2",
+            "<test>", "exec",
+        )
+        self._check_error(
+            "from __future__ import barry_as_FLUFL\na != b",
+            "with Barry as BDFL, use '<>' instead of '!='",
+            lineno=2,
+            end_lineno=2,
+            offset=3,
+            end_offset=5,
+        )
+
     def test_double_ampersand(self):
         self._check_error(
             "a && b",

Lib/test/test_tokenize.py

@@ -1234,6 +1234,20 @@ f'''
     FSTRING_END \'"\'           (2, 2) (2, 3)
     """)
 
+    def test_ineq_tokens(self):
+        self.check_tokenize("1 != 2", """\
+    NUMBER     '1'           (1, 0) (1, 1)
+    OP         '!='          (1, 2) (1, 4)
+    NUMBER     '2'           (1, 5) (1, 6)
+""")
+        self.check_tokenize("1 <> 2", """\
+    NUMBER     '1'           (1, 0) (1, 1)
+    OP         '<'           (1, 2) (1, 3)
+    OP         '>'           (1, 3) (1, 4)
+    NUMBER     '2'           (1, 5) (1, 6)
+""")
+
+
 class GenerateTokensTest(TokenizeTest):
     def check_tokenize(self, s, expected):
         # Format the tokens in s in a table format.

Misc/NEWS.d/next/Core_and_Builtins/2026-07-29-08-43-14.gh-issue-151464.o_mtmz.rst

@@ -0,0 +1,2 @@
+Exclude invalid token ``<>`` from :mod:`tokenize` output.  :exc:`SyntaxError`
+for ``<>`` now suggests ``!=``.

Parser/action_helpers.c

@@ -6,6 +6,7 @@
 
 #include "pegen.h"
 #include "string_parser.h"          // _PyPegen_decode_string()
+#include "lexer/state.h"            // tok_state
 
 
 void *
@@ -2133,6 +2134,7 @@ _PyPegen_checked_from_import(Parser *p, asdl_seq *dots, expr_ty module_name,
             alias_ty alias = asdl_seq_GET(names, i);
             if (PyUnicode_CompareWithASCIIString(alias->name, "barry_as_FLUFL") == 0) {
                 p->flags |= PyPARSE_BARRY_AS_BDFL;
+                p->tok->barry_as_bdfl = 1;
             }
         }
     }

Parser/lexer/lexer.c

@@ -409,7 +409,7 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
     /* Check for two-character token */
     {
         int c2 = tok_nextc(tok);
-        int current_token = _PyToken_TwoChars(c, c2);
+        int current_token = _PyToken_TwoChars(c, c2, tok->barry_as_bdfl);
         if (current_token != OP) {
             int c3 = tok_nextc(tok);
             int current_token3 = _PyToken_ThreeChars(c, c2, c3);

Parser/lexer/state.h

@@ -108,6 +108,7 @@ struct tok_state {
 #ifdef Py_DEBUG
     int debug;
 #endif
+    int barry_as_bdfl;
 };
 
 static inline ftstring_state *

Parser/parser.c

@@ -350,184 +350,185 @@ static char *soft_keywords[] = {
 #define invalid_type_params_type 1261
 #define invalid_bitwise_and_type 1262  // Left-recursive
 #define invalid_bitwise_or_type 1263  // Left-recursive
-#define _loop0_1_type 1264
-#define _loop1_2_type 1265
-#define _loop0_3_type 1266
-#define _gather_4_type 1267
-#define _tmp_5_type 1268
-#define _tmp_6_type 1269
-#define _tmp_7_type 1270
-#define _tmp_8_type 1271
-#define _tmp_9_type 1272
-#define _tmp_10_type 1273
-#define _tmp_11_type 1274
-#define _loop1_12_type 1275
-#define _loop0_13_type 1276
-#define _gather_14_type 1277
-#define _tmp_15_type 1278
-#define _tmp_16_type 1279
-#define _loop0_17_type 1280
-#define _loop1_18_type 1281
-#define _loop0_19_type 1282
-#define _gather_20_type 1283
-#define _tmp_21_type 1284
-#define _loop0_22_type 1285
-#define _gather_23_type 1286
-#define _loop1_24_type 1287
-#define _tmp_25_type 1288
-#define _tmp_26_type 1289
-#define _loop0_27_type 1290
-#define _loop0_28_type 1291
-#define _loop1_29_type 1292
-#define _loop1_30_type 1293
-#define _loop0_31_type 1294
-#define _loop1_32_type 1295
-#define _loop0_33_type 1296
-#define _gather_34_type 1297
-#define _tmp_35_type 1298
-#define _loop1_36_type 1299
-#define _loop1_37_type 1300
-#define _loop1_38_type 1301
-#define _loop0_39_type 1302
-#define _gather_40_type 1303
-#define _tmp_41_type 1304
-#define _tmp_42_type 1305
-#define _tmp_43_type 1306
-#define _loop0_44_type 1307
-#define _gather_45_type 1308
-#define _loop0_46_type 1309
-#define _gather_47_type 1310
-#define _tmp_48_type 1311
-#define _loop0_49_type 1312
-#define _gather_50_type 1313
-#define _loop0_51_type 1314
-#define _gather_52_type 1315
-#define _loop0_53_type 1316
-#define _gather_54_type 1317
-#define _loop1_55_type 1318
-#define _loop1_56_type 1319
-#define _loop0_57_type 1320
-#define _gather_58_type 1321
-#define _loop1_59_type 1322
-#define _loop1_60_type 1323
-#define _loop1_61_type 1324
-#define _tmp_62_type 1325
-#define _loop0_63_type 1326
-#define _gather_64_type 1327
-#define _tmp_65_type 1328
-#define _tmp_66_type 1329
-#define _tmp_67_type 1330
-#define _tmp_68_type 1331
-#define _tmp_69_type 1332
-#define _loop0_70_type 1333
-#define _loop0_71_type 1334
-#define _loop1_72_type 1335
-#define _loop1_73_type 1336
-#define _loop0_74_type 1337
-#define _loop1_75_type 1338
-#define _loop0_76_type 1339
-#define _loop0_77_type 1340
-#define _loop0_78_type 1341
-#define _loop0_79_type 1342
-#define _loop1_80_type 1343
-#define _loop1_81_type 1344
-#define _loop0_82_type 1345
-#define _gather_83_type 1346
-#define _tmp_84_type 1347
-#define _loop0_85_type 1348
-#define _gather_86_type 1349
-#define _loop1_87_type 1350
-#define _loop0_88_type 1351
-#define _tmp_89_type 1352
-#define _loop0_90_type 1353
-#define _gather_91_type 1354
-#define _tmp_92_type 1355
-#define _loop0_93_type 1356
-#define _gather_94_type 1357
-#define _tmp_95_type 1358
-#define _loop0_96_type 1359
-#define _gather_97_type 1360
-#define _loop0_98_type 1361
-#define _tmp_99_type 1362
-#define _tmp_100_type 1363
-#define _loop0_101_type 1364
-#define _gather_102_type 1365
-#define _loop0_103_type 1366
-#define _gather_104_type 1367
-#define _tmp_105_type 1368
-#define _tmp_106_type 1369
-#define _loop0_107_type 1370
-#define _gather_108_type 1371
-#define _tmp_109_type 1372
-#define _tmp_110_type 1373
-#define _tmp_111_type 1374
-#define _tmp_112_type 1375
-#define _loop1_113_type 1376
-#define _tmp_114_type 1377
-#define _tmp_115_type 1378
-#define _tmp_116_type 1379
-#define _tmp_117_type 1380
-#define _tmp_118_type 1381
-#define _loop0_119_type 1382
-#define _loop0_120_type 1383
-#define _tmp_121_type 1384
-#define _tmp_122_type 1385
-#define _tmp_123_type 1386
-#define _tmp_124_type 1387
-#define _tmp_125_type 1388
-#define _tmp_126_type 1389
-#define _tmp_127_type 1390
-#define _tmp_128_type 1391
-#define _loop0_129_type 1392
-#define _gather_130_type 1393
-#define _tmp_131_type 1394
-#define _tmp_132_type 1395
-#define _tmp_133_type 1396
-#define _tmp_134_type 1397
-#define _loop0_135_type 1398
-#define _gather_136_type 1399
-#define _tmp_137_type 1400
-#define _loop0_138_type 1401
-#define _gather_139_type 1402
-#define _loop0_140_type 1403
-#define _gather_141_type 1404
-#define _tmp_142_type 1405
-#define _loop0_143_type 1406
-#define _tmp_144_type 1407
-#define _tmp_145_type 1408
-#define _tmp_146_type 1409
-#define _tmp_147_type 1410
-#define _tmp_148_type 1411
-#define _tmp_149_type 1412
-#define _tmp_150_type 1413
-#define _tmp_151_type 1414
-#define _tmp_152_type 1415
-#define _tmp_153_type 1416
-#define _tmp_154_type 1417
-#define _tmp_155_type 1418
-#define _tmp_156_type 1419
-#define _tmp_157_type 1420
-#define _tmp_158_type 1421
-#define _tmp_159_type 1422
-#define _tmp_160_type 1423
-#define _tmp_161_type 1424
-#define _tmp_162_type 1425
-#define _tmp_163_type 1426
-#define _tmp_164_type 1427
-#define _tmp_165_type 1428
-#define _tmp_166_type 1429
-#define _tmp_167_type 1430
-#define _tmp_168_type 1431
-#define _tmp_169_type 1432
-#define _tmp_170_type 1433
-#define _tmp_171_type 1434
-#define _loop0_172_type 1435
-#define _tmp_173_type 1436
-#define _tmp_174_type 1437
-#define _tmp_175_type 1438
-#define _tmp_176_type 1439
-#define _tmp_177_type 1440
-#define _tmp_178_type 1441
+#define invalid_noteq_type 1264
+#define _loop0_1_type 1265
+#define _loop1_2_type 1266
+#define _loop0_3_type 1267
+#define _gather_4_type 1268
+#define _tmp_5_type 1269
+#define _tmp_6_type 1270
+#define _tmp_7_type 1271
+#define _tmp_8_type 1272
+#define _tmp_9_type 1273
+#define _tmp_10_type 1274
+#define _tmp_11_type 1275
+#define _loop1_12_type 1276
+#define _loop0_13_type 1277
+#define _gather_14_type 1278
+#define _tmp_15_type 1279
+#define _tmp_16_type 1280
+#define _loop0_17_type 1281
+#define _loop1_18_type 1282
+#define _loop0_19_type 1283
+#define _gather_20_type 1284
+#define _tmp_21_type 1285
+#define _loop0_22_type 1286
+#define _gather_23_type 1287
+#define _loop1_24_type 1288
+#define _tmp_25_type 1289
+#define _tmp_26_type 1290
+#define _loop0_27_type 1291
+#define _loop0_28_type 1292
+#define _loop1_29_type 1293
+#define _loop1_30_type 1294
+#define _loop0_31_type 1295
+#define _loop1_32_type 1296
+#define _loop0_33_type 1297
+#define _gather_34_type 1298
+#define _tmp_35_type 1299
+#define _loop1_36_type 1300
+#define _loop1_37_type 1301
+#define _loop1_38_type 1302
+#define _loop0_39_type 1303
+#define _gather_40_type 1304
+#define _tmp_41_type 1305
+#define _tmp_42_type 1306
+#define _tmp_43_type 1307
+#define _loop0_44_type 1308
+#define _gather_45_type 1309
+#define _loop0_46_type 1310
+#define _gather_47_type 1311
+#define _tmp_48_type 1312
+#define _loop0_49_type 1313
+#define _gather_50_type 1314
+#define _loop0_51_type 1315
+#define _gather_52_type 1316
+#define _loop0_53_type 1317
+#define _gather_54_type 1318
+#define _loop1_55_type 1319
+#define _loop1_56_type 1320
+#define _loop0_57_type 1321
+#define _gather_58_type 1322
+#define _loop1_59_type 1323
+#define _loop1_60_type 1324
+#define _loop1_61_type 1325
+#define _tmp_62_type 1326
+#define _loop0_63_type 1327
+#define _gather_64_type 1328
+#define _tmp_65_type 1329
+#define _tmp_66_type 1330
+#define _tmp_67_type 1331
+#define _tmp_68_type 1332
+#define _tmp_69_type 1333
+#define _loop0_70_type 1334
+#define _loop0_71_type 1335
+#define _loop1_72_type 1336
+#define _loop1_73_type 1337
+#define _loop0_74_type 1338
+#define _loop1_75_type 1339
+#define _loop0_76_type 1340
+#define _loop0_77_type 1341
+#define _loop0_78_type 1342
+#define _loop0_79_type 1343
+#define _loop1_80_type 1344
+#define _loop1_81_type 1345
+#define _loop0_82_type 1346
+#define _gather_83_type 1347
+#define _tmp_84_type 1348
+#define _loop0_85_type 1349
+#define _gather_86_type 1350
+#define _loop1_87_type 1351
+#define _loop0_88_type 1352
+#define _tmp_89_type 1353
+#define _loop0_90_type 1354
+#define _gather_91_type 1355
+#define _tmp_92_type 1356
+#define _loop0_93_type 1357
+#define _gather_94_type 1358
+#define _tmp_95_type 1359
+#define _loop0_96_type 1360
+#define _gather_97_type 1361
+#define _loop0_98_type 1362
+#define _tmp_99_type 1363
+#define _tmp_100_type 1364
+#define _loop0_101_type 1365
+#define _gather_102_type 1366
+#define _loop0_103_type 1367
+#define _gather_104_type 1368
+#define _tmp_105_type 1369
+#define _tmp_106_type 1370
+#define _loop0_107_type 1371
+#define _gather_108_type 1372
+#define _tmp_109_type 1373
+#define _tmp_110_type 1374
+#define _tmp_111_type 1375
+#define _tmp_112_type 1376
+#define _loop1_113_type 1377
+#define _tmp_114_type 1378
+#define _tmp_115_type 1379
+#define _tmp_116_type 1380
+#define _tmp_117_type 1381
+#define _tmp_118_type 1382
+#define _loop0_119_type 1383
+#define _loop0_120_type 1384
+#define _tmp_121_type 1385
+#define _tmp_122_type 1386
+#define _tmp_123_type 1387
+#define _tmp_124_type 1388
+#define _tmp_125_type 1389
+#define _tmp_126_type 1390
+#define _tmp_127_type 1391
+#define _tmp_128_type 1392
+#define _loop0_129_type 1393
+#define _gather_130_type 1394
+#define _tmp_131_type 1395
+#define _tmp_132_type 1396
+#define _tmp_133_type 1397
+#define _tmp_134_type 1398
+#define _loop0_135_type 1399
+#define _gather_136_type 1400
+#define _tmp_137_type 1401
+#define _loop0_138_type 1402
+#define _gather_139_type 1403
+#define _loop0_140_type 1404
+#define _gather_141_type 1405
+#define _tmp_142_type 1406
+#define _loop0_143_type 1407
+#define _tmp_144_type 1408
+#define _tmp_145_type 1409
+#define _tmp_146_type 1410
+#define _tmp_147_type 1411
+#define _tmp_148_type 1412
+#define _tmp_149_type 1413
+#define _tmp_150_type 1414
+#define _tmp_151_type 1415
+#define _tmp_152_type 1416
+#define _tmp_153_type 1417
+#define _tmp_154_type 1418
+#define _tmp_155_type 1419
+#define _tmp_156_type 1420
+#define _tmp_157_type 1421
+#define _tmp_158_type 1422
+#define _tmp_159_type 1423
+#define _tmp_160_type 1424
+#define _tmp_161_type 1425
+#define _tmp_162_type 1426
+#define _tmp_163_type 1427
+#define _tmp_164_type 1428
+#define _tmp_165_type 1429
+#define _tmp_166_type 1430
+#define _tmp_167_type 1431
+#define _tmp_168_type 1432
+#define _tmp_169_type 1433
+#define _tmp_170_type 1434
+#define _tmp_171_type 1435
+#define _loop0_172_type 1436
+#define _tmp_173_type 1437
+#define _tmp_174_type 1438
+#define _tmp_175_type 1439
+#define _tmp_176_type 1440
+#define _tmp_177_type 1441
+#define _tmp_178_type 1442
 
 static mod_ty file_rule(Parser *p);
 static mod_ty interactive_rule(Parser *p);
@@ -793,6 +794,7 @@ static void *invalid_factor_rule(Parser *p);
 static void *invalid_type_params_rule(Parser *p);
 static void *invalid_bitwise_and_rule(Parser *p);
 static void *invalid_bitwise_or_rule(Parser *p);
+static void *invalid_noteq_rule(Parser *p);
 static asdl_seq *_loop0_1_rule(Parser *p);
 static asdl_seq *_loop1_2_rule(Parser *p);
 static asdl_seq *_loop0_3_rule(Parser *p);
@@ -12712,6 +12714,7 @@ comparison_rule(Parser *p)
 //     | '==' bitwise_or
 //     | ('!=') bitwise_or
 //     | '<=' bitwise_or
+//     | invalid_noteq
 //     | '<' bitwise_or
 //     | '>=' bitwise_or
 //     | '>' bitwise_or
@@ -12812,6 +12815,25 @@ compare_op_bitwise_or_pair_rule(Parser *p)
         D(fprintf(stderr, "%*c%s compare_op_bitwise_or_pair[%d-%d]: %s failed!\n", p->level, ' ',
                   p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'<=' bitwise_or"));
     }
+    if (p->call_invalid_rules) { // invalid_noteq
+        if (p->error_indicator) {
+            p->level--;
+            return NULL;
+        }
+        D(fprintf(stderr, "%*c> compare_op_bitwise_or_pair[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "invalid_noteq"));
+        void *invalid_noteq_var;
+        if (
+            (invalid_noteq_var = invalid_noteq_rule(p))  // invalid_noteq
+        )
+        {
+            D(fprintf(stderr, "%*c+ compare_op_bitwise_or_pair[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_noteq"));
+            _res = invalid_noteq_var;
+            goto done;
+        }
+        p->mark = _mark;
+        D(fprintf(stderr, "%*c%s compare_op_bitwise_or_pair[%d-%d]: %s failed!\n", p->level, ' ',
+                  p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_noteq"));
+    }
     { // '<' bitwise_or
         if (p->error_indicator) {
             p->level--;
@@ -27269,6 +27291,52 @@ invalid_bitwise_or_rule(Parser *p)
     return _res;
 }
 
+// invalid_noteq: '<' '>'
+static void *
+invalid_noteq_rule(Parser *p)
+{
+    if (p->level++ == MAXSTACK || _PyPegen_stack_exhausted(p)) {
+        _Pypegen_stack_overflow(p);
+    }
+    if (p->error_indicator) {
+        p->level--;
+        return NULL;
+    }
+    void * _res = NULL;
+    int _mark = p->mark;
+    { // '<' '>'
+        if (p->error_indicator) {
+            p->level--;
+            return NULL;
+        }
+        D(fprintf(stderr, "%*c> invalid_noteq[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'<' '>'"));
+        Token * a;
+        Token * b;
+        if (
+            (a = _PyPegen_expect_token(p, 20))  // token='<'
+            &&
+            (b = _PyPegen_expect_token(p, 21))  // token='>'
+        )
+        {
+            D(fprintf(stderr, "%*c+ invalid_noteq[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'<' '>'"));
+            _res = _PyPegen_tokens_are_adjacent ( a , b ) ? RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "invalid syntax.  Maybe you meant '!=' instead of '<>'?" ) : NULL;
+            if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) {
+                p->error_indicator = 1;
+                p->level--;
+                return NULL;
+            }
+            goto done;
+        }
+        p->mark = _mark;
+        D(fprintf(stderr, "%*c%s invalid_noteq[%d-%d]: %s failed!\n", p->level, ' ',
+                  p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'<' '>'"));
+    }
+    _res = NULL;
+  done:
+    p->level--;
+    return _res;
+}
+
 // _loop0_1: NEWLINE
 static asdl_seq *
 _loop0_1_rule(Parser *p)

Parser/pegen.c

@@ -7,6 +7,7 @@
 #include "pycore_unicodeobject.h" // _PyUnicode_InternImmortal
 #include <errcode.h>
 
+#include "lexer/state.h"
 #include "tokenizer/tokenizer.h"
 #include "pegen.h"
 
@@ -1124,6 +1125,7 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena
                              PyCompilerFlags *flags, int *errcode,
                              PyObject **interactive_src, PyArena *arena)
 {
+    int parser_flags = compute_parser_flags(flags);
     struct tok_state *tok = _PyTokenizer_FromFile(fp, enc, ps1, ps2);
     if (tok == NULL) {
         if (PyErr_Occurred()) {
@@ -1135,6 +1137,7 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena
         }
         return NULL;
     }
+    tok->barry_as_bdfl = parser_flags & PyPARSE_BARRY_AS_BDFL;
 
     // From here on we need to clean up even if there's an error
     mod_ty result = NULL;
@@ -1146,7 +1149,6 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena
     _PyTokenizer_SetContext(tok, filename_ob, module);
     Py_DECREF(module);
 
-    int parser_flags = compute_parser_flags(flags);
     Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, PY_MINOR_VERSION,
                                     errcode, NULL, arena);
     if (p == NULL) {
@@ -1176,6 +1178,7 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen
                        PyCompilerFlags *flags, PyArena *arena, PyObject *module)
 {
     int exec_input = start_rule == Py_file_input;
+    int parser_flags = compute_parser_flags(flags);
 
     struct tok_state *tok;
     if (flags != NULL && flags->cf_flags & PyCF_IGNORE_COOKIE) {
@@ -1193,12 +1196,13 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen
         }
         return NULL;
     }
+    tok->barry_as_bdfl = parser_flags & PyPARSE_BARRY_AS_BDFL;
+
     _PyTokenizer_SetContext(tok, filename_ob, module);
 
     // We need to clear up from here on
     mod_ty result = NULL;
 
-    int parser_flags = compute_parser_flags(flags);
     int feature_version = flags && (flags->cf_flags & PyCF_ONLY_AST) ?
         flags->cf_feature_version : PY_MINOR_VERSION;
     Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, feature_version,

Parser/token.c

@@ -113,7 +113,7 @@ _PyToken_OneChar(int c1)
 }
 
 int
-_PyToken_TwoChars(int c1, int c2)
+_PyToken_TwoChars(int c1, int c2, int BARRY_AS_BDFL)
 {
     switch (c1) {
     case '!':
@@ -163,7 +163,7 @@ _PyToken_TwoChars(int c1, int c2)
         switch (c2) {
         case '<': return LEFTSHIFT;
         case '=': return LESSEQUAL;
-        case '>': return NOTEQUAL;
+        case '>': return BARRY_AS_BDFL ? NOTEQUAL : OP;
         }
         break;
     case '=':

Tools/build/generate_token.py

@@ -94,7 +94,7 @@ extern "C" {
 // Export these 4 symbols for 'test_peg_generator'
 PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */
 PyAPI_FUNC(int) _PyToken_OneChar(int);
-PyAPI_FUNC(int) _PyToken_TwoChars(int, int);
+PyAPI_FUNC(int) _PyToken_TwoChars(int, int, int);
 PyAPI_FUNC(int) _PyToken_ThreeChars(int, int, int);
 
 #ifdef __cplusplus
@@ -142,7 +142,7 @@ _PyToken_OneChar(int c1)
 }
 
 int
-_PyToken_TwoChars(int c1, int c2)
+_PyToken_TwoChars(int c1, int c2, int BARRY_AS_BDFL)
 {
 %s\
     return OP;
@@ -171,7 +171,10 @@ def generate_chars_to_token(mapping, n=1):
             write(indent)
             write('    break;\n')
         else:
-            write("case '%s': return %s;\n" % (c, value))
+            if c == '>' and value == 'NOTEQUAL':
+                write("case '%s': return BARRY_AS_BDFL ? %s : OP;\n" % (c, value))
+            else:
+                write("case '%s': return %s;\n" % (c, value))
     write(indent)
     write('}\n')
     return ''.join(result)