Make trivia first class member of token

examples/s_expressions.rs

@@ -215,9 +215,9 @@ fn test_parser() {
         "ROOT@0..15", // root node, spanning 15 bytes
     );
     assert_eq!(node.children().count(), 1);
-    let list = node.children().next().unwrap();
+    let list = node.children().next().unwrap().into_node().unwrap();
     let children = list
-        .children_with_tokens()
+        .children()
         .map(|child| format!("{:?}@{:?}", child.kind(), child.text_range()))
         .collect::<Vec<_>>();
 

src/api.rs

@@ -113,6 +113,10 @@ impl<L: Language> SyntaxNode<L> {
         self.raw.text_range()
     }
 
+    pub fn text_range_including_trivia(&self) -> TextRange {
+        self.raw.text_range_including_trivia()
+    }
+
     pub fn index(&self) -> usize {
         self.raw.index()
     }
@@ -121,10 +125,24 @@ impl<L: Language> SyntaxNode<L> {
         self.raw.text()
     }
 
+    pub fn text_including_trivia(&self) -> SyntaxText {
+        self.raw.text_including_trivia()
+    }
+
     pub fn green(&self) -> &GreenNodeData {
         self.raw.green()
     }
 
+    #[inline]
+    pub fn child_count(&self) -> usize {
+        self.raw.child_count()
+    }
+
+    /// The child at `index`, or `None` when it is out of range. O(1).
+    pub fn child(&self, index: usize) -> Option<SyntaxElement<L>> {
+        self.raw.child(index).map(NodeOrToken::from)
+    }
+
     pub fn parent(&self) -> Option<SyntaxNode<L>> {
         self.raw.parent().map(Self::from)
     }
@@ -202,7 +220,7 @@ impl<L: Language> SyntaxNode<L> {
     }
 
     /// Traverse the subtree rooted at the current node (including the current
-    /// node) in preorder, excluding tokens.
+    /// node) in preorder.
     pub fn preorder(&self) -> Preorder<L> {
         Preorder { raw: self.raw.preorder(), _p: PhantomData }
     }
@@ -213,6 +231,8 @@ impl<L: Language> SyntaxNode<L> {
         PreorderWithTokens { raw: self.raw.preorder_with_tokens(), _p: PhantomData }
     }
 
+    /// Traverse the subtree rooted at the current node (including the current
+
     /// Find a token in the subtree corresponding to this node, which covers the offset.
     /// Precondition: offset must be withing node's range.
     pub fn token_at_offset(&self, offset: TextSize) -> TokenAtOffset<SyntaxToken<L>> {
@@ -231,7 +251,7 @@ impl<L: Language> SyntaxNode<L> {
     /// there are several intersecting elements, any one can be returned.
     ///
     /// The method uses binary search internally, so it's complexity is
-    /// `O(log(N))` where `N = self.children_with_tokens().count()`.
+    /// `O(log(N))` where `N = self.children().count()`.
     pub fn child_or_token_at_range(&self, range: TextRange) -> Option<SyntaxElement<L>> {
         self.raw.child_or_token_at_range(range).map(SyntaxElement::from)
     }
@@ -253,6 +273,7 @@ impl<L: Language> SyntaxToken<L> {
         self.raw.replace_with(new_token)
     }
 
+    #[must_use]
     pub fn kind(&self) -> L::Kind {
         L::kind_from_raw(self.raw.kind())
     }
@@ -261,6 +282,10 @@ impl<L: Language> SyntaxToken<L> {
         self.raw.text_range()
     }
 
+    pub fn text_range_including_trivia(&self) -> TextRange {
+        self.raw.text_range_including_trivia()
+    }
+
     pub fn index(&self) -> usize {
         self.raw.index()
     }
@@ -269,6 +294,22 @@ impl<L: Language> SyntaxToken<L> {
         self.raw.text()
     }
 
+    pub fn text_including_trivia(&self) -> String {
+        self.raw.text_including_trivia()
+    }
+
+    pub fn leading_trivia(
+        &self,
+    ) -> impl DoubleEndedIterator<Item = SyntaxToken<L>> + ExactSizeIterator {
+        self.raw.leading_trivia().map(SyntaxToken::from)
+    }
+
+    pub fn trailing_trivia(
+        &self,
+    ) -> impl DoubleEndedIterator<Item = SyntaxToken<L>> + ExactSizeIterator {
+        self.raw.trailing_trivia().map(SyntaxToken::from)
+    }
+
     pub fn green(&self) -> &GreenTokenData {
         self.raw.green()
     }
@@ -277,6 +318,10 @@ impl<L: Language> SyntaxToken<L> {
         self.raw.parent().map(SyntaxNode::from)
     }
 
+    pub fn parent_token(&self) -> Option<SyntaxToken<L>> {
+        self.raw.parent_token().map(SyntaxToken::from)
+    }
+
     /// Iterator over all the ancestors of this token excluding itself.
     #[deprecated = "use `SyntaxToken::parent_ancestors` instead"]
     pub fn ancestors(&self) -> impl Iterator<Item = SyntaxNode<L>> {
@@ -324,6 +369,13 @@ impl<L: Language> SyntaxElement<L> {
         }
     }
 
+    pub fn text_range_including_trivia(&self) -> TextRange {
+        match self {
+            NodeOrToken::Node(it) => it.text_range_including_trivia(),
+            NodeOrToken::Token(it) => it.text_range_including_trivia(),
+        }
+    }
+
     pub fn index(&self) -> usize {
         match self {
             NodeOrToken::Node(it) => it.index(),
@@ -382,7 +434,7 @@ pub struct SyntaxNodeChildren<L: Language> {
 
 impl<L: Language> Iterator for SyntaxNodeChildren<L> {
     type Item = SyntaxNode<L>;
-    fn next(&mut self) -> Option<Self::Item> {
+    fn next(&mut self) -> Option<SyntaxNode<L>> {
         self.raw.next().map(SyntaxNode::from)
     }
 }
@@ -398,43 +450,49 @@ impl<L: Language> Iterator for SyntaxElementChildren<L> {
     fn next(&mut self) -> Option<Self::Item> {
         self.raw.next().map(NodeOrToken::from)
     }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.raw.size_hint()
+    }
 }
 
+impl<L: Language> ExactSizeIterator for SyntaxElementChildren<L> {}
+
 #[derive(Debug, Clone)]
-pub struct Preorder<L: Language> {
-    raw: cursor::Preorder,
+pub struct PreorderWithTokens<L: Language> {
+    raw: cursor::PreorderWithTokens,
     _p: PhantomData<L>,
 }
 
-impl<L: Language> Preorder<L> {
+impl<L: Language> PreorderWithTokens<L> {
     pub fn skip_subtree(&mut self) {
         self.raw.skip_subtree()
     }
 }
 
-impl<L: Language> Iterator for Preorder<L> {
-    type Item = WalkEvent<SyntaxNode<L>>;
+impl<L: Language> Iterator for PreorderWithTokens<L> {
+    type Item = WalkEvent<SyntaxElement<L>>;
     fn next(&mut self) -> Option<Self::Item> {
-        self.raw.next().map(|it| it.map(SyntaxNode::from))
+        self.raw.next().map(|it| it.map(NodeOrToken::from))
     }
 }
 
 #[derive(Debug, Clone)]
-pub struct PreorderWithTokens<L: Language> {
-    raw: cursor::PreorderWithTokens,
+pub struct Preorder<L: Language> {
+    raw: cursor::Preorder,
     _p: PhantomData<L>,
 }
 
-impl<L: Language> PreorderWithTokens<L> {
+impl<L: Language> Preorder<L> {
     pub fn skip_subtree(&mut self) {
         self.raw.skip_subtree()
     }
 }
 
-impl<L: Language> Iterator for PreorderWithTokens<L> {
-    type Item = WalkEvent<SyntaxElement<L>>;
+impl<L: Language> Iterator for Preorder<L> {
+    type Item = WalkEvent<SyntaxNode<L>>;
     fn next(&mut self) -> Option<Self::Item> {
-        self.raw.next().map(|it| it.map(SyntaxElement::from))
+        self.raw.next().map(|it| it.map(SyntaxNode::from))
     }
 }
 

src/ast.rs

@@ -188,7 +188,7 @@ impl<N: AstNode> AstChildren<N> {
 impl<N: AstNode> Iterator for AstChildren<N> {
     type Item = N;
     fn next(&mut self) -> Option<N> {
-        self.inner.find_map(N::cast)
+        self.inner.by_ref().find_map(N::cast)
     }
 }