unofficial mirror of bug-gnu-emacs@gnu.org 
 help / color / mirror / code / Atom feed
blob 3223875320abac19d82940340e1400fa647b7113 67083 bytes (raw)
name: doc/lispref/parsing.texi 	 # note: path name is non-authoritative(*)

   1
   2
   3
   4
   5
   6
   7
   8
   9
  10
  11
  12
  13
  14
  15
  16
  17
  18
  19
  20
  21
  22
  23
  24
  25
  26
  27
  28
  29
  30
  31
  32
  33
  34
  35
  36
  37
  38
  39
  40
  41
  42
  43
  44
  45
  46
  47
  48
  49
  50
  51
  52
  53
  54
  55
  56
  57
  58
  59
  60
  61
  62
  63
  64
  65
  66
  67
  68
  69
  70
  71
  72
  73
  74
  75
  76
  77
  78
  79
  80
  81
  82
  83
  84
  85
  86
  87
  88
  89
  90
  91
  92
  93
  94
  95
  96
  97
  98
  99
 100
 101
 102
 103
 104
 105
 106
 107
 108
 109
 110
 111
 112
 113
 114
 115
 116
 117
 118
 119
 120
 121
 122
 123
 124
 125
 126
 127
 128
 129
 130
 131
 132
 133
 134
 135
 136
 137
 138
 139
 140
 141
 142
 143
 144
 145
 146
 147
 148
 149
 150
 151
 152
 153
 154
 155
 156
 157
 158
 159
 160
 161
 162
 163
 164
 165
 166
 167
 168
 169
 170
 171
 172
 173
 174
 175
 176
 177
 178
 179
 180
 181
 182
 183
 184
 185
 186
 187
 188
 189
 190
 191
 192
 193
 194
 195
 196
 197
 198
 199
 200
 201
 202
 203
 204
 205
 206
 207
 208
 209
 210
 211
 212
 213
 214
 215
 216
 217
 218
 219
 220
 221
 222
 223
 224
 225
 226
 227
 228
 229
 230
 231
 232
 233
 234
 235
 236
 237
 238
 239
 240
 241
 242
 243
 244
 245
 246
 247
 248
 249
 250
 251
 252
 253
 254
 255
 256
 257
 258
 259
 260
 261
 262
 263
 264
 265
 266
 267
 268
 269
 270
 271
 272
 273
 274
 275
 276
 277
 278
 279
 280
 281
 282
 283
 284
 285
 286
 287
 288
 289
 290
 291
 292
 293
 294
 295
 296
 297
 298
 299
 300
 301
 302
 303
 304
 305
 306
 307
 308
 309
 310
 311
 312
 313
 314
 315
 316
 317
 318
 319
 320
 321
 322
 323
 324
 325
 326
 327
 328
 329
 330
 331
 332
 333
 334
 335
 336
 337
 338
 339
 340
 341
 342
 343
 344
 345
 346
 347
 348
 349
 350
 351
 352
 353
 354
 355
 356
 357
 358
 359
 360
 361
 362
 363
 364
 365
 366
 367
 368
 369
 370
 371
 372
 373
 374
 375
 376
 377
 378
 379
 380
 381
 382
 383
 384
 385
 386
 387
 388
 389
 390
 391
 392
 393
 394
 395
 396
 397
 398
 399
 400
 401
 402
 403
 404
 405
 406
 407
 408
 409
 410
 411
 412
 413
 414
 415
 416
 417
 418
 419
 420
 421
 422
 423
 424
 425
 426
 427
 428
 429
 430
 431
 432
 433
 434
 435
 436
 437
 438
 439
 440
 441
 442
 443
 444
 445
 446
 447
 448
 449
 450
 451
 452
 453
 454
 455
 456
 457
 458
 459
 460
 461
 462
 463
 464
 465
 466
 467
 468
 469
 470
 471
 472
 473
 474
 475
 476
 477
 478
 479
 480
 481
 482
 483
 484
 485
 486
 487
 488
 489
 490
 491
 492
 493
 494
 495
 496
 497
 498
 499
 500
 501
 502
 503
 504
 505
 506
 507
 508
 509
 510
 511
 512
 513
 514
 515
 516
 517
 518
 519
 520
 521
 522
 523
 524
 525
 526
 527
 528
 529
 530
 531
 532
 533
 534
 535
 536
 537
 538
 539
 540
 541
 542
 543
 544
 545
 546
 547
 548
 549
 550
 551
 552
 553
 554
 555
 556
 557
 558
 559
 560
 561
 562
 563
 564
 565
 566
 567
 568
 569
 570
 571
 572
 573
 574
 575
 576
 577
 578
 579
 580
 581
 582
 583
 584
 585
 586
 587
 588
 589
 590
 591
 592
 593
 594
 595
 596
 597
 598
 599
 600
 601
 602
 603
 604
 605
 606
 607
 608
 609
 610
 611
 612
 613
 614
 615
 616
 617
 618
 619
 620
 621
 622
 623
 624
 625
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
 
@c -*- mode: texinfo; coding: utf-8 -*-
@c This is part of the GNU Emacs Lisp Reference Manual.
@c Copyright (C) 2021-2022 Free Software Foundation, Inc.
@c See the file elisp.texi for copying conditions.
@node Parsing Program Source
@chapter Parsing Program Source

@cindex syntax tree, from parsing program source
Emacs provides various ways to parse program source text and produce a
@dfn{syntax tree}.  In a syntax tree, text is no longer considered a
one-dimensional stream of characters, but a structured tree of nodes,
where each node representing a piece of text.  Thus, a syntax tree can
enable interesting features like precise fontification, indentation,
navigation, structured editing, etc.

Emacs has a simple facility for parsing balanced expressions
(@pxref{Parsing Expressions}).  There is also the SMIE library for
generic navigation and indentation (@pxref{SMIE}).

In addition to those, Emacs also provides integration with
@uref{https://tree-sitter.github.io/tree-sitter, the tree-sitter
library}) if support for it was compiled in.  The tree-sitter library
implements an incremental parser and has support from a wide range of
programming languages.

@defun treesit-available-p
This function returns non-@code{nil} if tree-sitter features are
available for the current Emacs session.
@end defun

To be able to parse the program source using the tree-sitter library
and access the syntax tree of the program, a Lisp program needs to
load a language definition library, and create a parser for that
language and the current buffer.  After that, the Lisp program can
query the parser about specific nodes of the syntax tree.  Then, it
can access various kinds of information about each node, and search
for nodes using a powerful pattern-matching syntax.  This chapter
explains how to do all this, and also how a Lisp program can work with
source files that mix multiple programming languages.

@menu
* Language Definitions::     Loading tree-sitter language definitions.
* Using Parser::             Introduction to parsers.
* Retrieving Nodes::         Retrieving nodes from a syntax tree.
* Accessing Node Information:: Accessing node information.
* Pattern Matching::         Pattern matching with query patterns.
* Multiple Languages::       Parse text written in multiple languages.
* Tree-sitter major modes::  Develop major modes using tree-sitter.
* Tree-sitter C API::        Compare the C API and the ELisp API.
@end menu

@node Language Definitions
@section Tree-sitter Language Definitions
@cindex language definitions, for tree-sitter

@heading Loading a language definition
@cindex loading language definition for tree-sitter

@cindex language argument, for tree-sitter
Tree-sitter relies on language definitions to parse text in that
language.  In Emacs, a language definition is represented by a symbol.
For example, the C language definition is represented as the symbol
@code{c}, and @code{c} can be passed to tree-sitter functions as the
@var{language} argument.

@vindex treesit-extra-load-path
@vindex treesit-load-language-error
@vindex treesit-load-suffixes
Tree-sitter language definitions are distributed as dynamic libraries.
In order to use a language definition in Emacs, you need to make sure
that the dynamic library is installed on the system.  Emacs looks for
language definitions in several places, in the following order:

@itemize @bullet
@item
first, in the list of directories specified by the variable
@code{treesit-extra-load-path};
@item
then, in the @file{tree-sitter} subdirectory of the directory
specified by @code{user-emacs-directory} (@pxref{Init File});
@item
and finally, in the system's default locations for dynamic libraries.
@end itemize

In each of these directories, Emacs looks for a file with file-name
extensions specified by the variable @code{treesit-load-suffixes}.

If Emacs cannot find the library or has problems loading it, Emacs
signals the @code{treesit-load-language-error} error.  The data of
that signal could be one of the following:

@table @code
@item (not-found @var{error-msg} @dots{})
This means that Emacs could not find the language definition library.
@item (symbol-error @var{error-msg})
This means that Emacs could not find in the library the expected function
that every language definition library should export.
@item (version-mismatch @var{error-msg})
This means that the version of language definition library is incompatible
with that of the tree-sitter library.
@end table

@noindent
In all of these cases, @var{error-msg} might provide additional
details about the failure.

@defun treesit-language-available-p language &optional detail
This function returns non-@code{nil} if the language definitions for
@var{language} exist and can be loaded.

If @var{detail} is non-@code{nil}, return @code{(t . nil)} when
@var{language} is available, and @code{(nil . @var{data})} when it's
unavailable.  @var{data} is the signal data of
@code{treesit-load-language-error}.
@end defun

@vindex treesit-load-name-override-list
By convention, the file name of the dynamic library for @var{language} is
@file{libtree-sitter-@var{language}.@var{ext}}, where @var{ext} is the
system-specific extension for dynamic libraries.  Also by convention,
the function provided by that library is named
@code{tree_sitter_@var{language}}.  If a language definition library
doesn't follow this convention, you should add an entry

@example
(@var{language} @var{library-base-name} @var{function-name})
@end example

to the list in the variable @code{treesit-load-name-override-list}, where
@var{library-base-name} is the basename of the dynamic library's file name,
(usually, @file{libtree-sitter-@var{language}}), and
@var{function-name} is the function provided by the library
(usually, @code{tree_sitter_@var{language}}).  For example,

@example
(cool-lang "libtree-sitter-coool" "tree_sitter_cooool")
@end example

@noindent
for a language that considers itself too ``cool'' to abide by
conventions.

@cindex language-definition version, compatibility
@defun treesit-language-version &optional min-compatible
This function returns the version of the language-definition
Application Binary Interface (@acronym{ABI}) supported by the
tree-sitter library.  By default, it returns the latest ABI version
supported by the library, but if @var{min-compatible} is
non-@code{nil}, it returns the oldest ABI version which the library
still can support.  Language definition libraries must be built for
ABI versions between the oldest and the latest versions supported by
the tree-sitter library, otherwise the library will be unable to load
them.
@end defun

@heading Concrete syntax tree
@cindex syntax tree, concrete

A syntax tree is what a parser generates.  In a syntax tree, each node
represents a piece of text, and is connected to each other by a
parent-child relationship.  For example, if the source text is

@example
1 + 2
@end example

@noindent
its syntax tree could be

@example
@group
                  +--------------+
                  | root "1 + 2" |
                  +--------------+
                         |
        +--------------------------------+
        |       expression "1 + 2"       |
        +--------------------------------+
           |             |            |
+------------+   +--------------+   +------------+
| number "1" |   | operator "+" |   | number "2" |
+------------+   +--------------+   +------------+
@end group
@end example

We can also represent it as an s-expression:

@example
(root (expression (number) (operator) (number)))
@end example

@subheading Node types
@cindex node types, in a syntax tree

@cindex type of node, tree-sitter
@anchor{tree-sitter node type}
@cindex named node, tree-sitter
@anchor{tree-sitter named node}
@cindex anonymous node, tree-sitter
Names like @code{root}, @code{expression}, @code{number}, and
@code{operator} specify the @dfn{type} of the nodes.  However, not all
nodes in a syntax tree have a type.  Nodes that don't have a type are
known as @dfn{anonymous nodes}, and nodes with a type are @dfn{named
nodes}.  Anonymous nodes are tokens with fixed spellings, including
punctuation characters like bracket @samp{]}, and keywords like
@code{return}.

@subheading Field names

@cindex field name, tree-sitter
@cindex tree-sitter node field name
@anchor{tree-sitter node field name}
To make the syntax tree easier to analyze, many language definitions
assign @dfn{field names} to child nodes.  For example, a
@code{function_definition} node could have a @code{declarator} and a
@code{body}:

@example
@group
(function_definition
 declarator: (declaration)
 body: (compound_statement))
@end group
@end example

@heading Exploring the syntax tree
@cindex explore tree-sitter syntax tree
@cindex inspection of tree-sitter parse tree nodes

To aid in understanding the syntax of a language and in debugging of
Lisp program that use the syntax tree, Emacs provides an ``explore''
mode, which displays the syntax tree of the source in the current
buffer in real time.  Emacs also comes with an ``inspect mode'', which
displays information of the nodes at point in the mode-line.

@deffn Command treesit-explore-mode
This mode pops up a window displaying the syntax tree of the source in
the current buffer.  Selecting text in the source buffer highlights
the corresponding nodes in the syntax tree display.  Clicking
on nodes in the syntax tree highlights the corresponding text in the
source buffer.
@end deffn

@deffn Command treesit-inspect-mode
This minor mode displays on the mode-line the node that @emph{starts}
at point.  For example, the mode-line can display

@example
@var{parent} @var{field}: (@var{node} (@var{child} (@dots{})))
@end example

@noindent
where @var{node}, @var{child}, etc., are nodes which begin at point.
@var{parent} is the parent of @var{node}.  @var{node} is displayed in
a bold typeface.  @var{field-name}s are field names of @var{node} and
of @var{child}, etc.

If no node starts at point, i.e., point is in the middle of a node,
then the mode line displays the earliest node that spans point, and
its immediate parent.

This minor mode doesn't create parsers on its own.  It uses the first
parser in @code{(treesit-parser-list)} (@pxref{Using Parser}).
@end deffn

@heading Reading the grammar definition
@cindex reading grammar definition, tree-sitter

Authors of language definitions define the @dfn{grammar} of a
programming language, which determines how a parser constructs a
concrete syntax tree out of the program text.  In order to use the
syntax tree effectively, you need to consult the @dfn{grammar file}.

The grammar file is usually @file{grammar.js} in a language
definition's project repository.  The link to a language definition's
home page can be found on
@uref{https://tree-sitter.github.io/tree-sitter, tree-sitter's
homepage}.

The grammar definition is written in JavaScript.  For example, the
rule matching a @code{function_definition} node looks like

@example
@group
function_definition: $ => seq(
  $.declaration_specifiers,
  field('declarator', $.declaration),
  field('body', $.compound_statement)
)
@end group
@end example

@noindent
The rules are represented by functions that take a single argument
@var{$}, representing the whole grammar.  The function itself is
constructed by other functions: the @code{seq} function puts together
a sequence of children; the @code{field} function annotates a child
with a field name.  If we write the above definition in the so-called
@dfn{Backus-Naur Form} (@acronym{BNF}) syntax, it would look like

@example
@group
function_definition :=
  <declaration_specifiers> <declaration> <compound_statement>
@end group
@end example

@noindent
and the node returned by the parser would look like

@example
@group
(function_definition
  (declaration_specifier)
  declarator: (declaration)
  body: (compound_statement))
@end group
@end example

Below is a list of functions that one can see in a grammar definition.
Each function takes other rules as arguments and returns a new rule.

@table @code
@item seq(@var{rule1}, @var{rule2}, @dots{})
matches each rule one after another.
@item choice(@var{rule1}, @var{rule2}, @dots{})
matches one of the rules in its arguments.
@item repeat(@var{rule})
matches @var{rule} for @emph{zero or more} times.
This is like the @samp{*} operator in regular expressions.
@item repeat1(@var{rule})
matches @var{rule} for @emph{one or more} times.
This is like the @samp{+} operator in regular expressions.
@item optional(@var{rule})
matches @var{rule} for @emph{zero or one} time.
This is like the @samp{?} operator in regular expressions.
@item field(@var{name}, @var{rule})
assigns field name @var{name} to the child node matched by @var{rule}.
@item alias(@var{rule}, @var{alias})
makes nodes matched by @var{rule} appear as @var{alias} in the syntax
tree generated by the parser.  For example,

@example
alias(preprocessor_call_exp, call_expression)
@end example

@noindent
makes any node matched by @code{preprocessor_call_exp} appear as
@code{call_expression}.
@end table

Below are grammar functions of lesser importance for reading a
language definition.

@table @code
@item token(@var{rule})
marks @var{rule} to produce a single leaf node.  That is, instead of
generating a parent node with individual child nodes under it,
everything is combined into a single leaf node.  @xref{Retrieving
Nodes}.
@item token.immediate(@var{rule})
Normally, grammar rules ignore preceding whitespace; this
changes @var{rule} to match only when there is no preceding
whitespaces.
@item prec(@var{n}, @var{rule})
gives @var{rule} the level-@var{n} precedence.
@item prec.left([@var{n},] @var{rule})
marks @var{rule} as left-associative, optionally with level @var{n}.
@item prec.right([@var{n},] @var{rule})
marks @var{rule} as right-associative, optionally with level @var{n}.
@item prec.dynamic(@var{n}, @var{rule})
this is like @code{prec}, but the precedence is applied at runtime
instead.
@end table

The documentation of the tree-sitter project has
@uref{https://tree-sitter.github.io/tree-sitter/creating-parsers, more
about writing a grammar}.  Read especially ``The Grammar DSL''
section.

@node Using Parser
@section Using Tree-sitter Parser
@cindex tree-sitter parser, using

This section describes how to create and configure a tree-sitter
parser.  In Emacs, each tree-sitter parser is associated with a
buffer.  As the user edits the buffer, the associated parser and
syntax tree are automatically kept up-to-date.

@defvar treesit-max-buffer-size
This variable contains the maximum size of buffers in which
tree-sitter can be activated.  Major modes should check this value
when deciding whether to enable tree-sitter features.
@end defvar

@defun treesit-can-enable-p
This function checks whether the current buffer is suitable for
activating tree-sitter features.  It basically checks
@code{treesit-available-p} and @code{treesit-max-buffer-size}.
@end defun

@cindex creating tree-sitter parsers
@cindex tree-sitter parser, creating
@defun treesit-parser-create language &optional buffer no-reuse
Create a parser for the specified @var{buffer} and @var{language}
(@pxref{Language Definitions}).  If @var{buffer} is omitted or
@code{nil}, it stands for the current buffer.

By default, this function reuses a parser if one already exists for
@var{language} in @var{buffer}, but if @var{no-reuse} is
non-@code{nil}, this function always creates a new parser.
@end defun

Given a parser, we can query information about it.

@defun treesit-parser-buffer parser
This function returns the buffer associated with @var{parser}.
@end defun

@defun treesit-parser-language parser
This function returns the language used by @var{parser}.
@end defun

@defun treesit-parser-p object
This function checks if @var{object} is a tree-sitter parser, and
returns non-@code{nil} if it is, and @code{nil} otherwise.
@end defun

There is no need to explicitly parse a buffer, because parsing is done
automatically and lazily.  A parser only parses when a Lisp program
queries for a node in its syntax tree.  Therefore, when a parser is
first created, it doesn't parse the buffer; it waits until the Lisp
program queries for a node for the first time.  Similarly, when some
change is made in the buffer, a parser doesn't re-parse immediately.

@vindex treesit-buffer-too-large
When a parser does parse, it checks for the size of the buffer.
Tree-sitter can only handle buffer no larger than about 4GB.  If the
size exceeds that, Emacs signals the @code{treesit-buffer-too-large}
error with signal data being the buffer size.

Once a parser is created, Emacs automatically adds it to the
internal parser list.  Every time a change is made to the buffer,
Emacs updates parsers in this list so they can update their syntax
tree incrementally.

@defun treesit-parser-list &optional buffer
This function returns the parser list of @var{buffer}.  If
@var{buffer} is @code{nil} or omitted, it defaults to the current
buffer.
@end defun

@defun treesit-parser-delete parser
This function deletes @var{parser}.
@end defun

@cindex tree-sitter narrowing
@anchor{tree-sitter narrowing}
Normally, a parser ``sees'' the whole buffer, but when the buffer is
narrowed (@pxref{Narrowing}), the parser will only see the accessible
portion of the buffer.  As far as the parser can tell, the hidden
region was deleted.  When the buffer is later widened, the parser
thinks text is inserted at the beginning and at the end.  Although
parsers respect narrowing, modes should not use narrowing as a means
to handle a multi-language buffer; instead, set the ranges in which the
parser should operate.  @xref{Multiple Languages}.

Because a parser parses lazily, when the user or a Lisp program
narrows the buffer, the parser is not affected immediately; as long as
the mode doesn't query for a node while the buffer is narrowed, the
parser is oblivious of the narrowing.

@cindex tree-sitter parse string
@cindex parse string, tree-sitter
Besides creating a parser for a buffer, a Lisp program can also parse a
string.  Unlike a buffer, parsing a string is a one-off operation, and
there is no way to update the result.

@defun treesit-parse-string string language
This function parses @var{string} using @var{language}, and returns
the root node of the generated syntax tree.
@end defun

@heading Be notified by changes to the parse tree
@cindex update callback, for tree-sitter parse-tree
@cindex after-change notifier, for tree-sitter parse-tree
@cindex tree-sitter parse-tree, update and after-change callback
@cindex notifiers, tree-sitter

A Lisp program might want to be notified of text affected by
incremental parsing.  For example, inserting a comment-closing token
converts text before that token into a comment.  Even
though the text is not directly edited, it is deemed to be ``changed''
nevertheless.

Emacs lets a Lisp program to register callback functions
(a.k.a.@: @dfn{notifiers}) for this kind of changes.  A notifier
function takes two arguments: @var{ranges} and @var{parser}.
@var{ranges} is a list of cons cells of the form @w{@code{(@var{start}
. @var{end})}}, where @var{start} and @var{end} mark the start and the
end positions of a range.  @var{parser} is the parser issuing the
notification.

Every time a parser reparses a buffer, it compares the old and new
parse-tree, computes the ranges in which nodes have changed, and
passes the ranges to notifier functions.

@defun treesit-parser-add-notifier parser function
This function adds @var{function} to @var{parser}'s list of
after-change notifier functions.  @var{function} must be a function
symbol, not a lambda function (@pxref{Anonymous Functions}).
@end defun

@defun treesit-parser-remove-notifier parser function
This function removes @var{function} from the list of @var{parser}'s
after-change notifier functions.  @var{function} must be a function
symbol, rather than a lambda function.
@end defun

@defun treesit-parser-notifiers parser
This function returns the list of @var{parser}'s notifier functions.
@end defun

@node Retrieving Nodes
@section Retrieving Nodes
@cindex retrieve node, tree-sitter
@cindex tree-sitter, find node
@cindex get node, tree-sitter

@cindex terminology, for tree-sitter functions
Here's some terminology and conventions we use when documenting
tree-sitter functions.

We talk about a node being ``smaller'' or ``larger'', and ``lower'' or
``higher''.  A smaller and lower node is lower in the syntax tree and
therefore spans a smaller portion of buffer text; a larger and higher
node is higher up in the syntax tree, it contains many smaller nodes
as its children, and therefore spans a larger portion of text.

When a function cannot find a node, it returns @code{nil}.  For
convenience, all functions that take a node as argument and return
a node, also accept the node argument of @code{nil} and in that case
just return @code{nil}.

@vindex treesit-node-outdated
Nodes are not automatically updated when the associated buffer is
modified, and there is no way to update a node once it is retrieved.
Using an outdated node signals the @code{treesit-node-outdated} error.

@heading Retrieving nodes from syntax tree
@cindex retrieving tree-sitter nodes
@cindex syntax tree, retrieving nodes

@cindex leaf node, of tree-sitter parse tree
@cindex tree-sitter parse tree, leaf node
@defun treesit-node-at pos &optional parser-or-lang named
This function returns a @dfn{leaf} node at buffer position @var{pos}.
A leaf node is a node that doesn't have any child nodes.

This function tries to return a node whose span covers @var{pos}: the
node's beginning position is less or equal to @var{pos}, and the
node's end position is greater or equal to @var{pos}.

If no leaf node's span covers @var{pos} (e.g., @var{pos} is in the
whitespace between two leaf nodes), this function returns the first
leaf node after @var{pos}.

Finally, if there is no leaf node after @var{pos}, return the first
leaf node before @var{pos}.

When @var{parser-or-lang} is @code{nil} or omitted, this function uses
the first parser in @code{(treesit-parser-list)} of the current
buffer.  If @var{parser-or-lang} is a parser object, it uses that
parser; if @var{parser-or-lang} is a language, it finds the first
parser using that language in @code{(treesit-parser-list)}, and uses
that.

If this function cannot find a suitable node to return, it returns
@code{nil}.

If @var{named} is non-@code{nil}, this function looks only for named
nodes (@pxref{tree-sitter named node, named node}).

Example:

@example
@group
;; Find the node at point in a C parser's syntax tree.
(treesit-node-at (point) 'c)
  @result{} #<treesit-node (primitive_type) in 23-27>
@end group
@end example
@end defun

@defun treesit-node-on beg end &optional parser-or-lang named
This function returns the @emph{smallest} node that covers the region
of buffer text between @var{beg} and @var{end}.  In other words, the
start of the node is before or at @var{beg}, and the end of the node
is at or after @var{end}.

@emph{Beware:} calling this function on an empty line that is not
inside any top-level construct (function definition, etc.) most
probably will give you the root node, because the root node is the
smallest node that covers that empty line.  Most of the time, you want
to use @code{treesit-node-at}, described above, instead.

When @var{parser-or-lang} is @code{nil}, this function uses the first
parser in @code{(treesit-parser-list)} of the current buffer.  If
@var{parser-or-lang} is a parser object, it uses that parser; if
@var{parser-or-lang} is a language, it finds the first parser using
that language in @code{(treesit-parser-list)}, and uses that.

If @var{named} is non-@code{nil}, this function looks for a named node
only (@pxref{tree-sitter named node, named node}).
@end defun

@defun treesit-parser-root-node parser
This function returns the root node of the syntax tree generated by
@var{parser}.
@end defun

@defun treesit-buffer-root-node &optional language
This function finds the first parser that uses @var{language} in
@code{(treesit-parser-list)} of the current buffer, and returns the
root node generated by that parser.  If it cannot find an appropriate
parser, it returns @code{nil}.
@end defun

Given a node, a Lisp program can retrieve other nodes starting from
it, or query for information about this node.

@heading Retrieving nodes from other nodes
@cindex syntax tree nodes, retrieving from other nodes

@subheading By kinship
@cindex kinship, syntax tree nodes
@cindex nodes, by kinship
@cindex syntax tree nodes, by kinship

@defun treesit-node-parent node
This function returns the immediate parent of @var{node}.
@end defun

@defun treesit-node-child node n &optional named
This function returns the @var{n}'th child of @var{node}.  If
@var{named} is non-@code{nil}, it counts only named nodes
(@pxref{tree-sitter named node, named node}).

For example, in a node that represents a string @code{"text"}, there
are three children nodes: the opening quote @code{"}, the string text
@code{text}, and the closing quote @code{"}.  Among these nodes, the
first child is the opening quote @code{"}, and the first named child
is the string text.

This function returns @code{nil} if there is no @var{n}'th child.
@var{n} could be negative, e.g., @code{-1} represents the last child.
@end defun

@defun treesit-node-children node &optional named
This function returns all of @var{node}'s children as a list.  If
@var{named} is non-@code{nil}, it retrieves only named nodes.
@end defun

@defun treesit-next-sibling node &optional named
This function finds the next sibling of @var{node}.  If @var{named} is
non-@code{nil}, it finds the next named sibling.
@end defun

@defun treesit-prev-sibling node &optional named
This function finds the previous sibling of @var{node}.  If
@var{named} is non-@code{nil}, it finds the previous named sibling.
@end defun

@subheading By field name
@cindex nodes, by field name
@cindex syntax tree nodes, by field name

To make the syntax tree easier to analyze, many language definitions
assign @dfn{field names} to child nodes (@pxref{tree-sitter node field
name, field name}).  For example, a @code{function_definition} node
could have a @code{declarator} node and a @code{body} node.

@defun treesit-child-by-field-name node field-name
This function finds the child of @var{node} whose field name is
@var{field-name}, a string.

@example
@group
;; Get the child that has "body" as its field name.
(treesit-child-by-field-name node "body")
  @result{} #<treesit-node (compound_statement) in 45-89>
@end group
@end example
@end defun

@subheading By position
@cindex nodes, by position
@cindex syntax tree nodes, by position

@defun treesit-first-child-for-pos node pos &optional named
This function finds the first child of @var{node} that extends beyond
buffer position @var{pos}.  ``Extends beyond'' means the end of the
child node is greater or equal to @var{pos}.  This function only looks
for immediate children of @var{node}, and doesn't look in its
grandchildren.  If @var{named} is non-@code{nil}, it looks for the
first named child (@pxref{tree-sitter named node, named node}).
@end defun

@defun treesit-node-descendant-for-range node beg end &optional named
This function finds the @emph{smallest} descendant node of @var{node}
that spans the region of text between positions @var{beg} and
@var{end}.  It is similar to @code{treesit-node-at}.  If @var{named}
is non-@code{nil}, it looks for smallest named child.
@end defun

@heading Searching for node

@defun treesit-search-subtree node predicate &optional backward all limit
This function traverses the subtree of @var{node} (including
@var{node} itself), looking for a node for which @var{predicate}
returns non-@code{nil}.  @var{predicate} is a regexp that is matched
against each node's type, or a predicate function that takes a node
and returns non-@code{nil} if the node matches.  The function returns
the first node that matches, or @code{nil} if none does.

By default, this function only traverses named nodes, but if @var{all}
is non-@code{nil}, it traverses all the nodes.  If @var{backward} is
non-@code{nil}, it traverses backwards (i.e., it visits the last child
first when traversing down the tree).  If @var{limit} is
non-@code{nil}, it must be a number that limits the tree traversal to
that many levels down the tree.  If @var{limit} is @code{nil}, it
defaults to 1000.
@end defun

@defun treesit-search-forward start predicate &optional backward all
Like @code{treesit-search-subtree}, this function also traverses the
parse tree and matches each node with @var{predicate} (except for
@var{start}), where @var{predicate} can be a regexp or a function.
For a tree like the below where @var{start} is marked S, this function
traverses as numbered from 1 to 12:

@example
@group
              12
              |
     S--------3----------11
     |        |          |
o--o-+--o  1--+--2    6--+-----10
|  |                  |        |
o  o                +-+-+   +--+--+
                    |   |   |  |  |
                    4   5   7  8  9
@end group
@end example

Note that this function doesn't traverse the subtree of @var{start},
and it always traverse leaf nodes first, then upwards.

Like @code{treesit-search-subtree}, this function only searches for
named nodes by default, but if @var{all} is non-@code{nil}, it
searches for all nodes.  If @var{backward} is non-@code{nil}, it
searches backwards.

While @code{treesit-search-subtree} traverses the subtree of a node,
this function starts with node @var{start} and traverses every node
that comes after it in the buffer position order, i.e., nodes with
start positions greater than the end position of @var{start}.

In the tree shown above, @code{treesit-search-subtree} traverses node
S (@var{start}) and nodes marked with @code{o}, where this function
traverses the nodes marked with numbers.  This function is useful for
answering questions like ``what is the first node after @var{start} in
the buffer that satisfies some condition?''
@end defun

@defun treesit-search-forward-goto node predicate &optional start backward all
This function moves point to the start or end of the next node after
@var{node} in the buffer that matches @var{predicate}.  If @var{start}
is non-@code{nil}, stop at the beginning rather than the end of a node.

This function guarantees that the matched node it returns makes
progress in terms of buffer position: the start/end position of the
returned node is always greater than that of @var{node}.

Arguments @var{predicate}, @var{backward} and @var{all} are the same
as in @code{treesit-search-forward}.
@end defun

@defun treesit-induce-sparse-tree root predicate &optional process-fn limit
This function creates a sparse tree from @var{root}'s subtree.

It takes the subtree under @var{root}, and combs it so only the nodes
that match @var{predicate} are left.  Like previous functions, the
@var{predicate} can be a regexp string that matches against each
node's type, or a function that takes a node and return non-@code{nil}
if it matches.

For example, for a subtree on the left that consist of both numbers
and letters, if @var{predicate} is ``letter only'', the returned tree
is the one on the right.

@example
@group
    a                 a              a
    |                 |              |
+---+---+         +---+---+      +---+---+
|   |   |         |   |   |      |   |   |
b   1   2         b   |   |      b   c   d
    |   |     =>      |   |  =>      |
    c   +--+          c   +          e
    |   |  |          |   |
 +--+   d  4       +--+   d
 |  |              |
 e  5              e
@end group
@end example

If @var{process-fn} is non-@code{nil}, instead of returning the
matched nodes, this function passes each node to @var{process-fn} and
uses the returned value instead.  If non-@code{nil}, @var{limit} is
the number of levels to go down from @var{root}.  If @var{limit} is
@code{nil}, it defaults to 1000.

Each node in the returned tree looks like
@w{@code{(@var{tree-sitter-node} . (@var{child} @dots{}))}}.  The
@var{tree-sitter-node} of the root of this tree will be nil if
@var{root} doesn't match @var{predicate}.  If no node matches
@var{predicate}, the function returns @code{nil}.
@end defun

@heading More convenience functions

@defun treesit-filter-child node predicate &optional named
This function finds immediate children of @var{node} that satisfy
@var{predicate}.

The @var{predicate} function takes a node as the argument and should
return non-@code{nil} to indicate that the node should be kept.  If
@var{named} is non-@code{nil}, this function only examines the named
nodes.
@end defun

@defun treesit-parent-until node predicate
This function repeatedly finds the parents of @var{node}, and returns
the parent that satisfies @var{predicate}, a function that takes a
node as the argument.  If no parent satisfies @var{predicate}, this
function returns @code{nil}.
@end defun

@defun treesit-parent-while node predicate
This function repeatedly finds the parent of @var{node}, and keeps
doing so as long as the nodes satisfy @var{predicate}, a function that
takes a node as the argument.  That is, this function returns the
farthest parent that still satisfies @var{predicate}.
@end defun

@defun treesit-node-top-level node &optional type
This function returns the highest parent of @var{node} that has the
same type as @var{node}.  If no such parent exists, it returns
@code{nil}.  Therefore this function is also useful for testing
whether @var{node} is top-level.

If @var{type} is non-@code{nil}, this function matches each parent's
type with @var{type} as a regexp, rather than using @var{node}'s type.
@end defun

@node Accessing Node Information
@section Accessing Node Information
@cindex information of node, syntax trees
@cindex syntax trees, node information

@heading Basic information of Node

Every node is associated with a parser, and that parser is associated
with a buffer.  The following functions retrieve them.

@defun treesit-node-parser node
This function returns @var{node}'s associated parser.
@end defun

@defun treesit-node-buffer node
This function returns @var{node}'s parser's associated buffer.
@end defun

@defun treesit-node-language node
This function returns @var{node}'s parser's associated language.
@end defun

Each node represents a portion of text in the buffer.  Functions below
find relevant information about that text.

@defun treesit-node-start node
Return the start position of @var{node}.
@end defun

@defun treesit-node-end node
Return the end position of @var{node}.
@end defun

@defun treesit-node-text node &optional object
Return the buffer text that @var{node} represents, as a string.  (If
@var{node} is retrieved from parsing a string, it will be the text
from that string.)
@end defun

@cindex predicates for syntax tree nodes
Here are some predicates on tree-sitter nodes:

@defun treesit-node-p object
Checks if @var{object} is a tree-sitter syntax node.
@end defun

@defun treesit-node-eq node1 node2
Checks if @var{node1} and @var{node2} are the same node in a syntax
tree.
@end defun

@heading Property information

In general, nodes in a concrete syntax tree fall into two categories:
@dfn{named nodes} and @dfn{anonymous nodes}.  Whether a node is named
or anonymous is determined by the language definition
(@pxref{tree-sitter named node, named node}).

@cindex tree-sitter missing node
@cindex missing node, tree-sitter
Apart from being named or anonymous, a node can have other properties.
A node can be ``missing'': such nodes are inserted by the parser in
order to recover from certain kinds of syntax errors, i.e., something
should probably be there according to the grammar, but is not there.
This can happen during editing of the program source, when the source
is not yet in its final form.

@cindex tree-sitter extra node
@cindex extra node, tree-sitter
A node can be ``extra'': such nodes represent things like comments,
which can appear anywhere in the text.

@cindex tree-sitter outdated node
@cindex outdated node, tree-sitter
A node can be ``outdated'', if its parser has reparsed at least once
after the node was created.

@cindex tree-sitter node that has error
@cindex has error, tree-sitter node
A node ``has error'' if the text it spans contains a syntax error.  It
can be that the node itself has an error, or one of its descendants
has an error.

@defun treesit-node-check node property
This function checks if @var{node} has the specified @var{property}.
@var{property} can be @code{named}, @code{missing}, @code{extra},
@code{outdated}, or @code{has-error}.
@end defun

@defun treesit-node-type node
Named nodes have ``types'' (@pxref{tree-sitter node type, node type}).
For example, a named node can be a @code{string_literal} node, where
@code{string_literal} is its type.  The type of an anonymous node is
just the text that the node represents; e.g., the type of a @samp{,}
node 480is just @samp{,}.

This function returns @var{node}'s type as a string.
@end defun

@heading Information as a child or parent

@defun treesit-node-index node &optional named
This function returns the index of @var{node} as a child node of its
parent.  If @var{named} is non-@code{nil}, it only counts named nodes
(@pxref{tree-sitter named node, named node}).
@end defun

@defun treesit-node-field-name node
A child of a parent node could have a field name (@pxref{tree-sitter
node field name, field name}).  This function returns the field name
of @var{node} as a child of its parent.
@end defun

@defun treesit-node-field-name-for-child node n
This function returns the field name of the @var{n}'th child of
@var{node}.  It returns @code{nil} if there is no @var{n}'th child, or
the @var{n}'th child doesn't have a field name.

Note that @var{n} counts both named and anonymous child.  And @var{n}
could be negative, e.g., @code{-1} represents the last child.
@end defun

@defun treesit-child-count node &optional named
This function finds the number of children of @var{node}.  If
@var{named} is non-@code{nil}, it only counts named children
(@pxref{tree-sitter named node, named node}).
@end defun

@node Pattern Matching
@section Pattern Matching Tree-sitter Nodes
@cindex pattern matching with tree-sitter nodes

@cindex capturing, tree-sitter node
Tree-sitter lets Lisp programs match patterns using a small
declarative language.  This pattern matching consists of two steps:
first tree-sitter matches a @dfn{pattern} against nodes in the syntax
tree, then it @dfn{captures} specific nodes that matched the pattern
and returns the captured nodes.

We describe first how to write the most basic query pattern and how to
capture nodes in a pattern, then the pattern-matching function, and
finally the more advanced pattern syntax.

@heading Basic query syntax

@cindex tree-sitter query pattern syntax
@cindex pattern syntax, tree-sitter query
@cindex query, tree-sitter
A @dfn{query} consists of multiple @dfn{patterns}.  Each pattern is an
s-expression that matches a certain node in the syntax node.  A
pattern has the form @w{@code{(@var{type} (@var{child}@dots{}))}}

For example, a pattern that matches a @code{binary_expression} node that
contains @code{number_literal} child nodes would look like

@example
(binary_expression (number_literal))
@end example

To @dfn{capture} a node using the query pattern above, append
@code{@@@var{capture-name}} after the node pattern you want to
capture.  For example,

@example
(binary_expression (number_literal) @@number-in-exp)
@end example

@noindent
captures @code{number_literal} nodes that are inside a
@code{binary_expression} node with the capture name
@code{number-in-exp}.

We can capture the @code{binary_expression} node as well, with, for
example, the capture name @code{biexp}:

@example
(binary_expression
 (number_literal) @@number-in-exp) @@biexp
@end example

@heading Query function

@cindex query functions, tree-sitter
Now we can introduce the @dfn{query functions}.

@defun treesit-query-capture node query &optional beg end node-only
This function matches patterns in @var{query} within @var{node}.
The argument @var{query} can be either a string, a s-expression, or a
compiled query object.  For now, we focus on the string syntax;
s-expression syntax and compiled query are described at the end of the
section.

The argument @var{node} can also be a parser or a language symbol.  A
parser means using its root node, a language symbol means find or
create a parser for that language in the current buffer, and use the
root node.

The function returns all the captured nodes in a list of the form
@w{@code{(@var{capture_name} . @var{node})}}.  If @var{node-only} is
non-@code{nil}, it returns the list of nodes instead.  By default the
entire text of @var{node} is searched, but if @var{beg} and @var{end}
are both non-@code{nil}, they specify the region of buffer text where
this function should match nodes.  Any matching node whose span
overlaps with the region between @var{beg} and @var{end} are captured,
it doesn't have to be completely in the region.

@vindex treesit-query-error
@findex treesit-query-validate
This function raises the @code{treesit-query-error} error if
@var{query} is malformed.  The signal data contains a description of
the specific error.  You can use @code{treesit-query-validate} to
validate and debug the query.
@end defun

For example, suppose @var{node}'s text is @code{1 + 2}, and
@var{query} is

@example
@group
(setq query
      "(binary_expression
        (number_literal) @@number-in-exp) @@biexp")
@end group
@end example

Matching that query would return

@example
@group
(treesit-query-capture node query)
    @result{} ((biexp . @var{<node for "1 + 2">})
       (number-in-exp . @var{<node for "1">})
       (number-in-exp . @var{<node for "2">}))
@end group
@end example

As mentioned earlier, @var{query} could contain multiple patterns.
For example, it could have two top-level patterns:

@example
@group
(setq query
      "(binary_expression) @@biexp
       (number_literal)  @@number @@biexp")
@end group
@end example

@defun treesit-query-string string query language
This function parses @var{string} with @var{language}, matches its
root node with @var{query}, and returns the result.
@end defun

@heading More query syntax

Besides node type and capture, tree-sitter's pattern syntax can
express anonymous node, field name, wildcard, quantification,
grouping, alternation, anchor, and predicate.

@subheading Anonymous node

An anonymous node is written verbatim, surrounded by quotes.  A
pattern matching (and capturing) keyword @code{return} would be

@example
"return" @@keyword
@end example

@subheading Wild card

In a pattern, @samp{(_)} matches any named node, and @samp{_} matches
any named and anonymous node.  For example, to capture any named child
of a @code{binary_expression} node, the pattern would be

@example
(binary_expression (_) @@in_biexp)
@end example

@subheading Field name

It is possible to capture child nodes that have specific field names.
In the pattern below, @code{declarator} and @code{body} are field
names, indicated by the colon following them.

@example
@group
(function_definition
  declarator: (_) @@func-declarator
  body: (_) @@func-body)
@end group
@end example

It is also possible to capture a node that doesn't have a certain
field, say, a @code{function_definition} without a @code{body} field.

@example
(function_definition !body) @@func-no-body
@end example

@subheading Quantify node

@cindex quantify node, tree-sitter
Tree-sitter recognizes quantification operators @samp{*}, @samp{+} and
@samp{?}.  Their meanings are the same as in regular expressions:
@samp{*} matches the preceding pattern zero or more times, @samp{+}
matches one or more times, and @samp{?} matches zero or one time.

For example, the following pattern matches @code{type_declaration}
nodes that has @emph{zero or more} @code{long} keyword.

@example
(type_declaration "long"*) @@long-type
@end example

The following pattern matches a type declaration that has zero or one
@code{long} keyword:

@example
(type_declaration "long"?) @@long-type
@end example

@subheading Grouping

Similar to groups in regular expression, we can bundle patterns into
groups and apply quantification operators to them.  For example, to
express a comma separated list of identifiers, one could write

@example
(identifier) ("," (identifier))*
@end example

@subheading Alternation

Again, similar to regular expressions, we can express ``match anyone
from this group of patterns'' in a pattern.  The syntax is a list of
patterns enclosed in square brackets.  For example, to capture some
keywords in C, the pattern would be

@example
@group
[
  "return"
  "break"
  "if"
  "else"
] @@keyword
@end group
@end example

@subheading Anchor

The anchor operator @samp{.} can be used to enforce juxtaposition,
i.e., to enforce two things to be directly next to each other.  The
two ``things'' can be two nodes, or a child and the end of its parent.
For example, to capture the first child, the last child, or two
adjacent children:

@example
@group
;; Anchor the child with the end of its parent.
(compound_expression (_) @@last-child .)
@end group

@group
;; Anchor the child with the beginning of its parent.
(compound_expression . (_) @@first-child)
@end group

@group
;; Anchor two adjacent children.
(compound_expression
 (_) @@prev-child
 .
 (_) @@next-child)
@end group
@end example

Note that the enforcement of juxtaposition ignores any anonymous
nodes.

@subheading Predicate

It is possible to add predicate constraints to a pattern.  For
example, with the following pattern:

@example
@group
(
 (array . (_) @@first (_) @@last .)
 (#equal @@first @@last)
)
@end group
@end example

@noindent
tree-sitter only matches arrays where the first element equals to
the last element.  To attach a predicate to a pattern, we need to
group them together.  A predicate always starts with a @samp{#}.
Currently there are two predicates, @code{#equal} and @code{#match}.

@deffn Predicate equal arg1 arg2
Matches if @var{arg1} equals to @var{arg2}.  Arguments can be either
strings or capture names.  Capture names represent the text that the
captured node spans in the buffer.
@end deffn

@deffn Predicate match regexp capture-name
Matches if the text that @var{capture-name}'s node spans in the buffer
matches regular expression @var{regexp}.  Matching is case-sensitive.
@end deffn

Note that a predicate can only refer to capture names that appear in
the same pattern.  Indeed, it makes little sense to refer to capture
names in other patterns.

@heading S-expression patterns

@cindex tree-sitter patterns as sexps
@cindex patterns, tree-sitter, in sexp form
Besides strings, Emacs provides a s-expression based syntax for
tree-sitter patterns.  It largely resembles the string-based syntax.
For example, the following query

@example
@group
(treesit-query-capture
 node "(addition_expression
        left: (_) @@left
        \"+\" @@plus-sign
        right: (_) @@right) @@addition

        [\"return\" \"break\"] @@keyword")
@end group
@end example

@noindent
is equivalent to

@example
@group
(treesit-query-capture
 node '((addition_expression
         left: (_) @@left
         "+" @@plus-sign
         right: (_) @@right) @@addition

         ["return" "break"] @@keyword))
@end group
@end example

Most patterns can be written directly as strange but nevertheless
valid s-expressions.  Only a few of them needs modification:

@itemize
@item
Anchor @samp{.} is written as @code{:anchor}.
@item
@samp{?} is written as @samp{:?}.
@item
@samp{*} is written as @samp{:*}.
@item
@samp{+} is written as @samp{:+}.
@item
@code{#equal} is written as @code{:equal}.  In general, predicates
change their @samp{#} to @samp{:}.
@end itemize

For example,

@example
@group
"(
  (compound_expression . (_) @@first (_)* @@rest)
  (#match \"love\" @@first)
  )"
@end group
@end example

@noindent
is written in s-expression as

@example
@group
'((
   (compound_expression :anchor (_) @@first (_) :* @@rest)
   (:match "love" @@first)
   ))
@end group
@end example

@heading Compiling queries

@cindex compiling tree-sitter queries
@cindex queries, compiling
If a query is intended to be used repeatedly, especially in tight
loops, it is important to compile that query, because a compiled query
is much faster than an uncompiled one.  A compiled query can be used
anywhere a query is accepted.

@defun treesit-query-compile language query
This function compiles @var{query} for @var{language} into a compiled
query object and returns it.

This function raises the @code{treesit-query-error} error if
@var{query} is malformed.  The signal data contains a description of
the specific error.  You can use @code{treesit-query-validate} to
validate and debug the query.
@end defun

@defun treesit-query-language query
This function return the language of @var{query}.
@end defun

@defun treesit-query-expand query
This function converts the s-expression @var{query} into the string
format.
@end defun

@defun treesit-pattern-expand pattern
This function converts the s-expression @var{pattern} into the string
format.
@end defun

For more details, read the tree-sitter project's documentation about
pattern-matching, which can be found at
@uref{https://tree-sitter.github.io/tree-sitter/using-parsers#pattern-matching-with-queries}.

@node Multiple Languages
@section Parsing Text in Multiple Languages
@cindex multiple languages, parsing with tree-sitter
@cindex parsing multiple languages with tree-sitter
Sometimes, the source of a programming language could contain snippets
of other languages; @acronym{HTML} + @acronym{CSS} + JavaScript is one
example.  In that case, text segments written in different languages
need to be assigned different parsers.  Traditionally, this is
achieved by using narrowing.  While tree-sitter works with narrowing
(@pxref{tree-sitter narrowing, narrowing}), the recommended way is
instead to set regions of buffer text (i.e., ranges) in which a parser
will operate.  This section describes functions for setting and
getting ranges for a parser.

Lisp programs should call @code{treesit-update-ranges} to make sure
the ranges for each parser are correct before using parsers in a
buffer, and call @code{treesit-language-at} to figure out the language
responsible for the text at some position.  These two functions don't
work by themselves, they need major modes to set
@code{treesit-range-settings} and
@code{treesit-language-at-point-function}, which do the actual work.
These functions and variables are explained in more detail towards the
end of the section.

@heading Getting and setting ranges

@defun treesit-parser-set-included-ranges parser ranges
This function sets up @var{parser} to operate on @var{ranges}.  The
@var{parser} will only read the text of the specified ranges.  Each
range in @var{ranges} is a list of the form @w{@code{(@var{beg}
. @var{end})}}.

The ranges in @var{ranges} must come in order and must not overlap.
That is, in pseudo code:

@example
@group
(cl-loop for idx from 1 to (1- (length ranges))
         for prev = (nth (1- idx) ranges)
         for next = (nth idx ranges)
         should (<= (car prev) (cdr prev)
                    (car next) (cdr next)))
@end group
@end example

@vindex treesit-range-invalid
If @var{ranges} violates this constraint, or something else went
wrong, this function signals the @code{treesit-range-invalid} error.
The signal data contains a specific error message and the ranges we
are trying to set.

This function can also be used for disabling ranges.  If @var{ranges}
is @code{nil}, the parser is set to parse the whole buffer.

Example:

@example
@group
(treesit-parser-set-included-ranges
 parser '((1 . 9) (16 . 24) (24 . 25)))
@end group
@end example
@end defun

@defun treesit-parser-included-ranges parser
This function returns the ranges set for @var{parser}.  The return
value is the same as the @var{ranges} argument of
@code{treesit-parser-included-ranges}: a list of cons cells of the form
@w{@code{(@var{beg} . @var{end})}}.  If @var{parser} doesn't have any
ranges, the return value is @code{nil}.

@example
@group
(treesit-parser-included-ranges parser)
    @result{} ((1 . 9) (16 . 24) (24 . 25))
@end group
@end example
@end defun

@defun treesit-query-range source query &optional beg end
This function matches @var{source} with @var{query} and returns the
ranges of captured nodes.  The return value is a list of cons cells of
the form @w{@code{(@var{beg} . @var{end})}}, where @var{beg} and
@var{end} specify the beginning and the end of a region of text.

For convenience, @var{source} can be a language symbol, a parser, or a
node.  If it's a language symbol, this function matches in the root
node of the first parser using that language; if a parser, this
function matches in the root node of that parser; if a node, this
function matches in that node.

The argument @var{query} is the query used to capture nodes
(@pxref{Pattern Matching}).  The capture names don't matter.  The
arguments @var{beg} and @var{end}, if both non-@code{nil}, limit the
range in which this function queries.

Like other query functions, this function raises the
@code{treesit-query-error} error if @var{query} is malformed.
@end defun

@heading Supporting multiple languages in Lisp programs

It should suffice for general Lisp programs to call the following two
functions in order to support program sources that mixes multiple
languages.

@defun treesit-update-ranges &optional beg end
This function updates ranges for parsers in the buffer.  It makes sure
the parsers' ranges are set correctly between @var{beg} and @var{end},
according to @code{treesit-range-settings}.  If omitted, @var{beg}
defaults to the beginning of the buffer, and @var{end} defaults to the
end of the buffer.

For example, fontification functions use this function before querying
for nodes in a region.
@end defun

@defun treesit-language-at pos
This function returns the language of the text at buffer position
@var{pos}.  Under the hood it calls
@code{treesit-language-at-point-function} and returns its return
value.  If @code{treesit-language-at-point-function} is @code{nil},
this function returns the language of the first parser in the returned
value of @code{treesit-parser-list}.  If there is no parser in the
buffer, it returns @code{nil}.
@end defun

@heading Supporting multiple languages in major modes

@cindex host language, tree-sitter
@cindex tree-sitter host and embedded languages
@cindex embedded language, tree-sitter
Normally, in a set of languages that can be mixed together, there is a
@dfn{host language} and one or more @dfn{embedded languages}.  A Lisp
program usually first parses the whole document with the host
language's parser, retrieves some information, sets ranges for the
embedded languages with that information, and then parses the embedded
languages.

Take a buffer containing @acronym{HTML}, @acronym{CSS} and JavaScript
as an example.  A Lisp program will first parse the whole buffer with
an @acronym{HTML} parser, then query the parser for
@code{style_element} and @code{script_element} nodes, which
correspond to @acronym{CSS} and JavaScript text, respectively.  Then
it sets the range of the @acronym{CSS} and JavaScript parser to the
ranges in which their corresponding nodes span.

Given a simple @acronym{HTML} document:

@example
@group
<html>
  <script>1 + 2</script>
  <style>body @{ color: "blue"; @}</style>
</html>
@end group
@end example

@noindent
a Lisp program will first parse with a @acronym{HTML} parser, then set
ranges for @acronym{CSS} and JavaScript parsers:

@example
@group
;; Create parsers.
(setq html (treesit-get-parser-create 'html))
(setq css (treesit-get-parser-create 'css))
(setq js (treesit-get-parser-create 'javascript))
@end group

@group
;; Set CSS ranges.
(setq css-range
      (treesit-query-range
       'html
       "(style_element (raw_text) @@capture)"))
(treesit-parser-set-included-ranges css css-range)
@end group

@group
;; Set JavaScript ranges.
(setq js-range
      (treesit-query-range
       'html
       "(script_element (raw_text) @@capture)"))
(treesit-parser-set-included-ranges js js-range)
@end group
@end example

Emacs automates this process in @code{treesit-update-ranges}.  A
multi-language major mode should set @code{treesit-range-settings} so
that @code{treesit-update-ranges} knows how to perform this process
automatically.  Major modes should use the helper function
@code{treesit-range-rules} to generate a value that can be assigned to
@code{treesit-range-settings}.  The settings in the following example
directly translate into operations shown above.

@example
@group
(setq-local treesit-range-settings
            (treesit-range-rules
             :embed 'javascript
             :host 'html
             '((script_element (raw_text) @@capture))
@end group

@group
             :embed 'css
             :host 'html
             '((style_element (raw_text) @@capture))))
@end group
@end example

@defun treesit-range-rules &rest query-specs
This function is used to set @var{treesit-range-settings}.  It
takes care of compiling queries and other post-processing, and outputs
a value that @var{treesit-range-settings} can have.

It takes a series of @var{query-spec}s, where each @var{query-spec} is
a @var{query} preceded by zero or more @var{keyword}/@var{value}
pairs.  Each @var{query} is a tree-sitter query in either the
string, s-expression or compiled form, or a function.

If @var{query} is a tree-sitter query, it should be preceded by two
@var{:keyword}/@var{value} pairs, where the @code{:embed} keyword
specifies the embedded language, and the @code{:host} keyword
specified the host language.

@code{treesit-update-ranges} uses @var{query} to figure out how to set
the ranges for parsers for the embedded language.  It queries
@var{query} in a host language parser, computes the ranges in which
the captured nodes span, and applies these ranges to embedded
language parsers.

If @var{query} is a function, it doesn't need any @var{:keyword} and
@var{value} pair.  It should be a function that takes 2 arguments,
@var{start} and @var{end}, and sets the ranges for parsers in the
current buffer in the region between @var{start} and @var{end}.  It is
fine for this function to set ranges in a larger region that
encompasses the region between @var{start} and @var{end}.
@end defun

@defvar treesit-range-settings
This variable helps @code{treesit-update-ranges} in updating the
ranges for parsers in the buffer.  It is a list of @var{setting}s
where the exact format of a @var{setting} is considered internal.  You
should use @code{treesit-range-rules} to generate a value that this
variable can have.

@c Because the format is internal, we don't document them here.  Though
@c we do have it explained in the docstring.  We also expose the fact
@c that it is a list of settings, so one could combine two of them with
@c append.
@end defvar


@defvar treesit-language-at-point-function
This variable's value should be a function that takes a single
argument, @var{pos}, which is a buffer position, and returns the
language of the buffer text at @var{pos}.  This variable is used by
@code{treesit-language-at}.
@end defvar

@node Tree-sitter major modes
@section Developing major modes with tree-sitter
@cindex major mode, developing with tree-sitter

This section covers some general guidelines on developing tree-sitter
integration for a major mode.

A major mode supporting tree-sitter features should roughly follow
this pattern:

@c FIXME: Update this part once we settle on the exact format.
@example
@group
(define-derived-mode woomy-mode prog-mode "Woomy"
  "A mode for Woomy programming language."
  ;; Shared setup.
  ...
  (cond
   ;; Tree-sitter setup.
   ((treesit-ready-p 'woomy-mode 'woomy)
    (setq-local treesit-variables ...)
    (treesit-major-mode-setup))
   ;; Non-tree-sitter setup.
   (t
    ...)))
@end group
@end example

First, the major mode should use @code{treesit-ready-p} to determine
whether tree-sitter can be activated in this mode.

@defun treesit-ready-p mode language &optional quiet
This function checks for conditions for activating tree-sitter.  It
checks whether Emacs was built with tree-sitter, whether the buffer's
size is not too large for tree-sitter to handle it, and whether the
language definition for @var{language} is available on the system
(@pxref{Language Definitions}).

This function emits a warning if tree-sitter cannot be activated.  If
@var{quiet} is @code{message}, the warning is turned into a message;
if @var{quiet} is @code{nil}, no warning or message is displayed.

If all the necessary conditions are met, this function returns
non-@code{nil}; otherwise it returns @code{nil}.
@end defun

Next, the major mode should set up tree-sitter variables and call
@code{treesit-major-mode-setup}.

@defun treesit-major-mode-setup
This function activates some tree-sitter features for a major mode.

Currently, it sets up the following features:
@itemize
@item
If @code{treesit-font-lock-settings} is non-@code{nil}, it sets up
fontification.
@item
If @code{treesit-simple-indent-rules} is non-@code{nil}, it sets up
indentation.
@item
If @code{treesit-defun-type-regexp} is non-@code{nil}, it sets up
navigation functions for @code{beginning-of-defun} and
@code{end-of-defun}.
@end itemize
@end defun

For more information of these built-in tree-sitter features,
@pxref{Parser-based Font Lock}, @pxref{Parser-based Indentation}, and
@pxref{List Motion}.

For supporting mixing of multiple languages in a major mode,
@pxref{Multiple Languages}.

Setting the following local variables allows tree-sitter's indentation
engine to correctly indent multi-line comments:

@defvar treesit-comment-start
This should be a regular expression matching an opening comment token.
For example, it should match @samp{//}, @samp{////}, @samp{/*},
@samp{/****}, etc., in C.
@end defvar

@defvar treesit-comment-end
This should be a regular expression matching a closing comment token.
For example, it should match @samp{*/}, @samp{****/}, etc., in C.
@end defvar

@node Tree-sitter C API
@section Tree-sitter C API Correspondence

Emacs' tree-sitter integration doesn't expose every feature
provided by tree-sitter's C API.  Missing features include:

@itemize
@item
Creating a tree cursor and navigating the syntax tree with it.
@item
Setting timeout and cancellation flag for a parser.
@item
Setting the logger for a parser.
@item
Printing a @acronym{DOT} graph of the syntax tree to a file.
@item
Copying and modifying a syntax tree.  (Emacs doesn't expose a tree
object.)
@item
Using (row, column) coordinates as position.
@item
Updating a node with changes.  (In Emacs, retrieve a new node instead
of updating the existing one.)
@item
Querying statics of a language definition.
@end itemize

In addition, Emacs makes some changes to the C API to make the API more
convenient and idiomatic:

@itemize
@item
Instead of using byte positions, the Emacs Lisp API uses character
positions.
@item
Null nodes are converted to nil.
@end itemize

Below is the correspondence between all C API functions and their
ELisp counterparts.  Sometimes one ELisp function corresponds to
multiple C functions, and many C functions don't have an ELisp
counterpart.

@example
ts_parser_new                           treesit-parser-create
ts_parser_delete
ts_parser_set_language
ts_parser_language                      treesit-parser-language
ts_parser_set_included_ranges           treesit-parser-set-included-ranges
ts_parser_included_ranges               treesit-parser-included-ranges
ts_parser_parse
ts_parser_parse_string                  treesit-parse-string
ts_parser_parse_string_encoding
ts_parser_reset
ts_parser_set_timeout_micros
ts_parser_timeout_micros
ts_parser_set_cancellation_flag
ts_parser_cancellation_flag
ts_parser_set_logger
ts_parser_logger
ts_parser_print_dot_graphs
ts_tree_copy
ts_tree_delete
ts_tree_root_node
ts_tree_language
ts_tree_edit
ts_tree_get_changed_ranges
ts_tree_print_dot_graph
ts_node_type                            treesit-node-type
ts_node_symbol
ts_node_start_byte                      treesit-node-start
ts_node_start_point
ts_node_end_byte                        treesit-node-end
ts_node_end_point
ts_node_string                          treesit-node-string
ts_node_is_null
ts_node_is_named                        treesit-node-check
ts_node_is_missing                      treesit-node-check
ts_node_is_extra                        treesit-node-check
ts_node_has_changes
ts_node_has_error                       treesit-node-check
ts_node_parent                          treesit-node-parent
ts_node_child                           treesit-node-child
ts_node_field_name_for_child            treesit-node-field-name-for-child
ts_node_child_count                     treesit-node-child-count
ts_node_named_child                     treesit-node-child
ts_node_named_child_count               treesit-node-child-count
ts_node_child_by_field_name             treesit-node-by-field-name
ts_node_child_by_field_id
ts_node_next_sibling                    treesit-next-sibling
ts_node_prev_sibling                    treesit-prev-sibling
ts_node_next_named_sibling              treesit-next-sibling
ts_node_prev_named_sibling              treesit-prev-sibling
ts_node_first_child_for_byte            treesit-first-child-for-pos
ts_node_first_named_child_for_byte      treesit-first-child-for-pos
ts_node_descendant_for_byte_range       treesit-descendant-for-range
ts_node_descendant_for_point_range
ts_node_named_descendant_for_byte_range treesit-descendant-for-range
ts_node_named_descendant_for_point_range
ts_node_edit
ts_node_eq                              treesit-node-eq
ts_tree_cursor_new
ts_tree_cursor_delete
ts_tree_cursor_reset
ts_tree_cursor_current_node
ts_tree_cursor_current_field_name
ts_tree_cursor_current_field_id
ts_tree_cursor_goto_parent
ts_tree_cursor_goto_next_sibling
ts_tree_cursor_goto_first_child
ts_tree_cursor_goto_first_child_for_byte
ts_tree_cursor_goto_first_child_for_point
ts_tree_cursor_copy
ts_query_new
ts_query_delete
ts_query_pattern_count
ts_query_capture_count
ts_query_string_count
ts_query_start_byte_for_pattern
ts_query_predicates_for_pattern
ts_query_step_is_definite
ts_query_capture_name_for_id
ts_query_string_value_for_id
ts_query_disable_capture
ts_query_disable_pattern
ts_query_cursor_new
ts_query_cursor_delete
ts_query_cursor_exec                    treesit-query-capture
ts_query_cursor_did_exceed_match_limit
ts_query_cursor_match_limit
ts_query_cursor_set_match_limit
ts_query_cursor_set_byte_range
ts_query_cursor_set_point_range
ts_query_cursor_next_match
ts_query_cursor_remove_match
ts_query_cursor_next_capture
ts_language_symbol_count
ts_language_symbol_name
ts_language_symbol_for_name
ts_language_field_count
ts_language_field_name_for_id
ts_language_field_id_for_name
ts_language_symbol_type
ts_language_version
@end example

debug log:

solving 3223875320a ...
found 3223875320a in https://git.savannah.gnu.org/cgit/emacs.git

(*) Git path names are given by the tree(s) the blob belongs to.
    Blobs themselves have no identifier aside from the hash of its contents.^

Code repositories for project(s) associated with this public inbox

	https://git.savannah.gnu.org/cgit/emacs.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).