diff --git a/src/api.rs b/src/api.rs index 9b7ca5a..47b72af 100644 --- a/src/api.rs +++ b/src/api.rs @@ -62,17 +62,28 @@ impl fmt::Display for SyntaxNode { impl fmt::Debug for SyntaxToken { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}@{:?}", self.kind(), self.text_range())?; - if self.text().len() < 25 { - return write!(f, " {:?}", self.text()); - } let text = self.text(); - for idx in 21..25 { - if text.is_char_boundary(idx) { - let text = format!("{} ...", &text[..idx]); - return write!(f, " {:?}", text); + if text.len() < 25 { + write!(f, " {text:?}")?; + } else { + let idx = (21..25).find(|&idx| text.is_char_boundary(idx)).unwrap(); + write!(f, " {:?}", format!("{} ...", &text[..idx]))?; + } + write!(f, " [")?; + for (idx, piece) in self.leading_trivia().enumerate() { + if idx > 0 { + write!(f, ", ")?; + } + write!(f, "{:?}({:?})", piece.kind(), piece.text())?; + } + write!(f, "] [")?; + for (idx, piece) in self.trailing_trivia().enumerate() { + if idx > 0 { + write!(f, ", ")?; } + write!(f, "{:?}({:?})", piece.kind(), piece.text())?; } - unreachable!() + write!(f, "]") } } @@ -113,6 +124,10 @@ impl SyntaxNode { self.raw.text_range() } + pub fn text_range_including_trivia(&self) -> TextRange { + self.raw.text_range_including_trivia() + } + pub fn index(&self) -> usize { self.raw.index() } @@ -121,6 +136,22 @@ impl SyntaxNode { self.raw.text() } + pub fn text_including_trivia(&self) -> SyntaxText { + self.raw.text_including_trivia() + } + + pub fn leading_trivia( + &self, + ) -> impl DoubleEndedIterator> + ExactSizeIterator { + self.raw.leading_trivia().map(SyntaxToken::from) + } + + pub fn trailing_trivia( + &self, + ) -> impl DoubleEndedIterator> + ExactSizeIterator { + self.raw.trailing_trivia().map(SyntaxToken::from) + } + pub fn green(&self) -> &GreenNodeData { self.raw.green() } @@ -261,6 +292,10 @@ impl SyntaxToken { self.raw.text_range() } + pub fn text_range_including_trivia(&self) -> TextRange { + self.raw.text_range_including_trivia() + } + pub fn index(&self) -> usize { self.raw.index() } @@ -269,6 +304,22 @@ impl SyntaxToken { self.raw.text() } + pub fn text_including_trivia(&self) -> String { + self.raw.text_including_trivia() + } + + pub fn leading_trivia( + &self, + ) -> impl DoubleEndedIterator> + ExactSizeIterator { + self.raw.leading_trivia().map(SyntaxToken::from) + } + + pub fn trailing_trivia( + &self, + ) -> impl DoubleEndedIterator> + ExactSizeIterator { + self.raw.trailing_trivia().map(SyntaxToken::from) + } + pub fn green(&self) -> &GreenTokenData { self.raw.green() } @@ -324,6 +375,13 @@ impl SyntaxElement { } } + pub fn text_range_including_trivia(&self) -> TextRange { + match self { + NodeOrToken::Node(it) => it.text_range_including_trivia(), + NodeOrToken::Token(it) => it.text_range_including_trivia(), + } + } + pub fn index(&self) -> usize { match self { NodeOrToken::Node(it) => it.index(), @@ -382,7 +440,7 @@ pub struct SyntaxNodeChildren { impl Iterator for SyntaxNodeChildren { type Item = SyntaxNode; - fn next(&mut self) -> Option { + fn next(&mut self) -> Option> { self.raw.next().map(SyntaxNode::from) } } diff --git a/src/cursor.rs b/src/cursor.rs index 3ebe023..b2d8ac1 100644 --- a/src/cursor.rs +++ b/src/cursor.rs @@ -35,7 +35,7 @@ use std::{ cell::Cell, fmt, hash::{Hash, Hasher}, - iter, + iter::{self, FusedIterator}, mem::ManuallyDrop, ptr, slice, }; @@ -184,7 +184,9 @@ impl NodeData { #[inline] fn parent_node(&self) -> Option { let parent = self.parent()?; - debug_assert!(matches!(parent.green, Green::Node { .. })); + if !matches!(parent.green, Green::Node { .. }) { + return None; + } parent.inc_rc(); Some(SyntaxNode { ptr: ptr::NonNull::from(parent) }) } @@ -223,9 +225,9 @@ impl NodeData { } #[inline] - fn text_range(&self) -> TextRange { + fn text_range_including_trivia(&self) -> TextRange { let offset = self.offset(); - let len = self.green().text_len(); + let len = self.green().text_len_including_trivia(); TextRange::at(offset, len) } @@ -334,7 +336,20 @@ impl SyntaxNode { #[inline] pub fn text_range(&self) -> TextRange { - self.data().text_range() + let full = self.data().text_range_including_trivia(); + if self.data().parent().is_none() { + return full; + } + let green = self.green_ref(); + TextRange::new( + full.start() + green.leading_trivia_len(), + full.end() - green.trailing_trivia_len(), + ) + } + + #[inline] + pub fn text_range_including_trivia(&self) -> TextRange { + self.data().text_range_including_trivia() } #[inline] @@ -347,6 +362,22 @@ impl SyntaxNode { SyntaxText::new(self.clone()) } + pub fn text_including_trivia(&self) -> SyntaxText { + SyntaxText::with_range(self.clone(), self.text_range_including_trivia()) + } + + pub fn leading_trivia( + &self, + ) -> impl DoubleEndedIterator + ExactSizeIterator { + SyntaxTriviaTokens::new(self.first_token(), true) + } + + pub fn trailing_trivia( + &self, + ) -> impl DoubleEndedIterator + ExactSizeIterator { + SyntaxTriviaTokens::new(self.last_token(), false) + } + #[inline] pub fn green(&self) -> &GreenNodeData { self.green_ref() @@ -493,7 +524,7 @@ impl SyntaxNode { // TODO: this could be faster if we first drill-down to node, and only // then switch to token search. We should also replace explicit // recursion with a loop. - let range = self.text_range(); + let range = self.text_range_including_trivia(); assert!( range.start() <= offset && offset <= range.end(), "Bad offset: range {:?} offset {:?}", @@ -505,7 +536,7 @@ impl SyntaxNode { } let mut children = self.children_with_tokens().filter(|child| { - let child_range = child.text_range(); + let child_range = child.text_range_including_trivia(); !child_range.is_empty() && (child_range.start() <= offset && offset <= child_range.end()) }); @@ -530,9 +561,9 @@ impl SyntaxNode { let mut res: SyntaxElement = self.clone().into(); loop { assert!( - res.text_range().contains_range(range), + res.text_range_including_trivia().contains_range(range), "Bad range: node range {:?}, range {:?}", - res.text_range(), + res.text_range_including_trivia(), range, ); res = match &res { @@ -564,6 +595,25 @@ impl SyntaxToken { SyntaxToken { ptr: NodeData::new(Some(parent), index, offset, green) } } + fn new_trivia( + green: &GreenTokenData, + parent: SyntaxToken, + index: u32, + offset: TextSize, + ) -> SyntaxToken { + let parent = ManuallyDrop::new(parent); + let green = Green::Token { ptr: green.into() }; + let data = NodeData { + _c: Count::new(), + rc: Cell::new(1), + parent: Some(parent.ptr), + index, + green, + offset, + }; + SyntaxToken { ptr: unsafe { ptr::NonNull::new_unchecked(Box::into_raw(Box::new(data))) } } + } + #[inline] fn data(&self) -> &NodeData { unsafe { self.ptr.as_ref() } @@ -585,7 +635,14 @@ impl SyntaxToken { #[inline] pub fn text_range(&self) -> TextRange { - self.data().text_range() + let green = self.green(); + let start = self.data().text_range_including_trivia().start() + green.leading_trivia_len(); + TextRange::at(start, green.text_len()) + } + + #[inline] + pub fn text_range_including_trivia(&self) -> TextRange { + self.data().text_range_including_trivia() } #[inline] @@ -608,6 +665,14 @@ impl SyntaxToken { } } + pub fn text_including_trivia(&self) -> String { + self.leading_trivia() + .chain(iter::once(self.clone())) + .chain(self.trailing_trivia()) + .map(|token| token.text().to_owned()) + .collect() + } + #[inline] pub fn green(&self) -> &GreenTokenData { self.data().green().into_token().unwrap() @@ -665,6 +730,18 @@ impl SyntaxToken { .and_then(|element| element.last_token()), } } + + pub fn leading_trivia( + &self, + ) -> impl DoubleEndedIterator + ExactSizeIterator { + SyntaxTriviaTokens::new(Some(self.clone()), true) + } + + pub fn trailing_trivia( + &self, + ) -> impl DoubleEndedIterator + ExactSizeIterator { + SyntaxTriviaTokens::new(Some(self.clone()), false) + } } impl SyntaxElement { @@ -692,6 +769,14 @@ impl SyntaxElement { } } + #[inline] + pub fn text_range_including_trivia(&self) -> TextRange { + match self { + NodeOrToken::Node(it) => it.text_range_including_trivia(), + NodeOrToken::Token(it) => it.text_range_including_trivia(), + } + } + #[inline] pub fn index(&self) -> usize { match self { @@ -760,7 +845,10 @@ impl SyntaxElement { } fn token_at_offset(&self, offset: TextSize) -> TokenAtOffset { - assert!(self.text_range().start() <= offset && offset <= self.text_range().end()); + assert!( + self.text_range_including_trivia().start() <= offset + && offset <= self.text_range_including_trivia().end() + ); match self { NodeOrToken::Token(token) => TokenAtOffset::Single(token.clone()), NodeOrToken::Node(node) => node.token_at_offset(offset), @@ -798,12 +886,7 @@ impl fmt::Debug for SyntaxNode { impl fmt::Display for SyntaxNode { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.preorder_with_tokens() - .filter_map(|event| match event { - WalkEvent::Enter(NodeOrToken::Token(token)) => Some(token), - _ => None, - }) - .try_for_each(|it| fmt::Display::fmt(&it, f)) + fmt::Display::fmt(&self.text(), f) } } @@ -1001,3 +1084,83 @@ impl Iterator for PreorderWithTokens { } } // endregion + +#[derive(Clone)] +struct SyntaxTriviaTokens { + token: Option, + is_leading: bool, + front: usize, + back: usize, + front_offset: TextSize, + back_offset: TextSize, +} + +impl SyntaxTriviaTokens { + fn new(token: Option, is_leading: bool) -> Self { + let (len, text_len) = { + let trivia = Self::green_trivia(token.as_ref(), is_leading); + (trivia.len(), trivia.iter().map(|it| it.text_len()).sum::()) + }; + let front_offset = match &token { + Some(token) if is_leading => token.text_range_including_trivia().start(), + Some(token) => token.text_range().end(), + None => TextSize::new(0), + }; + let back_offset = front_offset + text_len; + Self { token, is_leading, front: 0, back: len, front_offset, back_offset } + } + + fn green_trivia(token: Option<&SyntaxToken>, is_leading: bool) -> &[GreenToken] { + match token { + Some(token) if is_leading => token.green().leading_trivia(), + Some(token) => token.green().trailing_trivia(), + None => &[], + } + } + + fn trivia(&self) -> &[GreenToken] { + Self::green_trivia(self.token.as_ref(), self.is_leading) + } +} + +impl Iterator for SyntaxTriviaTokens { + type Item = SyntaxToken; + + fn next(&mut self) -> Option { + if self.front == self.back { + return None; + } + let token = self.token.clone()?; + let index = self.front; + self.front += 1; + let offset = self.front_offset; + self.front_offset += self.trivia()[index].text_len(); + Some(SyntaxToken::new_trivia(&self.trivia()[index], token, index as u32, offset)) + } + + fn size_hint(&self) -> (usize, Option) { + let len = self.back - self.front; + (len, Some(len)) + } +} + +impl FusedIterator for SyntaxTriviaTokens {} + +impl DoubleEndedIterator for SyntaxTriviaTokens { + fn next_back(&mut self) -> Option { + if self.front == self.back { + return None; + } + let token = self.token.clone()?; + self.back -= 1; + self.back_offset -= self.trivia()[self.back].text_len(); + Some(SyntaxToken::new_trivia( + &self.trivia()[self.back], + token, + self.back as u32, + self.back_offset, + )) + } +} + +impl ExactSizeIterator for SyntaxTriviaTokens {} diff --git a/src/green/builder.rs b/src/green/builder.rs index 1dbc8bb..2fda108 100644 --- a/src/green/builder.rs +++ b/src/green/builder.rs @@ -39,6 +39,18 @@ impl GreenNodeBuilder<'_> { self.children.push((hash, token.into())); } + #[inline] + pub fn token_with_trivia<'a, 'b>( + &mut self, + kind: SyntaxKind, + text: &str, + leading: impl IntoIterator, + trailing: impl IntoIterator, + ) { + let (hash, token) = self.cache.token_with_trivia(kind, text, leading, trailing); + self.children.push((hash, token.into())); + } + /// Start new node and make it current. #[inline] pub fn start_node(&mut self, kind: SyntaxKind) { diff --git a/src/green/element.rs b/src/green/element.rs index d2a2dcf..0b7ab5a 100644 --- a/src/green/element.rs +++ b/src/green/element.rs @@ -57,6 +57,11 @@ impl GreenElement { pub fn text_len(&self) -> TextSize { self.as_deref().text_len() } + + #[inline] + pub fn text_len_including_trivia(&self) -> TextSize { + self.as_deref().text_len_including_trivia() + } } impl GreenElementRef<'_> { @@ -77,4 +82,12 @@ impl GreenElementRef<'_> { NodeOrToken::Token(it) => it.text_len(), } } + + #[inline] + pub fn text_len_including_trivia(self) -> TextSize { + match self { + NodeOrToken::Node(it) => it.text_len_including_trivia(), + NodeOrToken::Token(it) => it.text_len_including_trivia(), + } + } } diff --git a/src/green/node.rs b/src/green/node.rs index c2eddf0..0f915fd 100644 --- a/src/green/node.rs +++ b/src/green/node.rs @@ -10,7 +10,7 @@ use countme::Count; use crate::{ arc::{Arc, HeaderSlice, ThinArc}, - green::{GreenElement, GreenElementRef, SyntaxKind}, + green::{GreenElement, GreenElementRef, GreenTokenData, SyntaxKind}, utility_types::static_assert, GreenToken, NodeOrToken, TextRange, TextSize, }; @@ -75,7 +75,7 @@ impl fmt::Debug for GreenNodeData { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("GreenNode") .field("kind", &self.kind()) - .field("text_len", &self.text_len()) + .field("text_len_including_trivia", &self.text_len_including_trivia()) .field("n_children", &self.children().len()) .finish() } @@ -124,9 +124,36 @@ impl GreenNodeData { /// Returns the length of the text covered by this node. #[inline] pub fn text_len(&self) -> TextSize { + self.text_len_including_trivia() - self.leading_trivia_len() - self.trailing_trivia_len() + } + + #[inline] + pub fn text_len_including_trivia(&self) -> TextSize { self.header().text_len } + pub(crate) fn leading_trivia_len(&self) -> TextSize { + self.first_token().map_or_else(TextSize::default, |it| it.leading_trivia_len()) + } + + pub(crate) fn trailing_trivia_len(&self) -> TextSize { + self.last_token().map_or_else(TextSize::default, |it| it.trailing_trivia_len()) + } + + fn first_token(&self) -> Option<&GreenTokenData> { + self.children().find_map(|child| match child { + NodeOrToken::Token(it) => Some(it), + NodeOrToken::Node(it) => it.first_token(), + }) + } + + fn last_token(&self) -> Option<&GreenTokenData> { + self.children().rev().find_map(|child| match child { + NodeOrToken::Token(it) => Some(it), + NodeOrToken::Node(it) => it.last_token(), + }) + } + /// Children of this node. #[inline] pub fn children(&self) -> Children<'_> { @@ -206,7 +233,7 @@ impl GreenNode { let mut text_len: TextSize = 0.into(); let children = children.into_iter().map(|el| { let rel_offset = text_len; - text_len += el.text_len(); + text_len += el.text_len_including_trivia(); match el { NodeOrToken::Node(node) => GreenChild::Node { rel_offset, node }, NodeOrToken::Token(token) => GreenChild::Token { rel_offset, token }, @@ -262,7 +289,7 @@ impl GreenChild { } #[inline] fn rel_range(&self) -> TextRange { - let len = self.as_ref().text_len(); + let len = self.as_ref().text_len_including_trivia(); TextRange::at(self.rel_offset(), len) } } diff --git a/src/green/node_cache.rs b/src/green/node_cache.rs index c73f3e6..0dd7d1f 100644 --- a/src/green/node_cache.rs +++ b/src/green/node_cache.rs @@ -43,6 +43,8 @@ fn token_hash(token: &GreenTokenData) -> u64 { let mut h = FxHasher::default(); token.kind().hash(&mut h); token.text().hash(&mut h); + token.leading_trivia().hash(&mut h); + token.trailing_trivia().hash(&mut h); h.finish() } @@ -131,22 +133,48 @@ impl NodeCache { } pub(crate) fn token(&mut self, kind: SyntaxKind, text: &str) -> (u64, GreenToken) { + self.token_with_green_trivia(kind, text, Vec::new(), Vec::new()) + } + + pub(crate) fn token_with_trivia<'a, 'b>( + &mut self, + kind: SyntaxKind, + text: &str, + leading: impl IntoIterator, + trailing: impl IntoIterator, + ) -> (u64, GreenToken) { + let leading = leading.into_iter().map(|(kind, text)| self.token(kind, text).1).collect(); + let trailing = trailing.into_iter().map(|(kind, text)| self.token(kind, text).1).collect(); + self.token_with_green_trivia(kind, text, leading, trailing) + } + + fn token_with_green_trivia( + &mut self, + kind: SyntaxKind, + text: &str, + leading: Vec, + trailing: Vec, + ) -> (u64, GreenToken) { let hash = { let mut h = FxHasher::default(); kind.hash(&mut h); text.hash(&mut h); + leading.hash(&mut h); + trailing.hash(&mut h); h.finish() }; - let entry = self - .tokens - .raw_entry_mut() - .from_hash(hash, |token| token.0.kind() == kind && token.0.text() == text); + let entry = self.tokens.raw_entry_mut().from_hash(hash, |token| { + token.0.kind() == kind + && token.0.text() == text + && token.0.leading_trivia() == leading + && token.0.trailing_trivia() == trailing + }); let token = match entry { RawEntryMut::Occupied(entry) => entry.key().0.clone(), RawEntryMut::Vacant(entry) => { - let token = GreenToken::new(kind, text); + let token = GreenToken::with_trivia(kind, text, leading, trailing); entry.insert_with_hasher(hash, NoHash(token.clone()), (), |t| token_hash(&t.0)); token } diff --git a/src/green/token.rs b/src/green/token.rs index fcc5a11..2116189 100644 --- a/src/green/token.rs +++ b/src/green/token.rs @@ -16,6 +16,8 @@ use crate::{ #[derive(PartialEq, Eq, Hash)] struct GreenTokenHead { kind: SyntaxKind, + leading: Box<[GreenToken]>, + trailing: Box<[GreenToken]>, _c: Count, } @@ -28,7 +30,10 @@ pub struct GreenTokenData { impl PartialEq for GreenTokenData { fn eq(&self, other: &Self) -> bool { - self.kind() == other.kind() && self.text() == other.text() + self.kind() == other.kind() + && self.text() == other.text() + && self.leading_trivia() == other.leading_trivia() + && self.trailing_trivia() == other.trailing_trivia() } } @@ -64,6 +69,8 @@ impl fmt::Debug for GreenTokenData { f.debug_struct("GreenToken") .field("kind", &self.kind()) .field("text", &self.text()) + .field("leading", &self.leading_trivia()) + .field("trailing", &self.trailing_trivia()) .finish() } } @@ -84,7 +91,14 @@ impl fmt::Display for GreenToken { impl fmt::Display for GreenTokenData { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.text()) + for token in self.leading_trivia() { + write!(f, "{}", token.text())?; + } + write!(f, "{}", self.text())?; + for token in self.trailing_trivia() { + write!(f, "{}", token.text())?; + } + Ok(()) } } @@ -95,29 +109,74 @@ impl GreenTokenData { self.data.header.kind } - /// Text of this Token. + /// Text of this Token, excluding its trivia. #[inline] pub fn text(&self) -> &str { unsafe { std::str::from_utf8_unchecked(self.data.slice()) } } - /// Returns the length of the text covered by this token. + /// Returns the length of the text covered by this token, excluding its trivia. #[inline] pub fn text_len(&self) -> TextSize { TextSize::of(self.text()) } + + #[inline] + pub fn leading_trivia(&self) -> &[GreenToken] { + &self.data.header.leading + } + + #[inline] + pub fn trailing_trivia(&self) -> &[GreenToken] { + &self.data.header.trailing + } + + pub(crate) fn leading_trivia_len(&self) -> TextSize { + self.leading_trivia().iter().map(|it| it.text_len()).sum() + } + + pub(crate) fn trailing_trivia_len(&self) -> TextSize { + self.trailing_trivia().iter().map(|it| it.text_len()).sum() + } + + #[inline] + pub fn text_len_including_trivia(&self) -> TextSize { + self.leading_trivia_len() + self.text_len() + self.trailing_trivia_len() + } } impl GreenToken { /// Creates new Token. #[inline] pub fn new(kind: SyntaxKind, text: &str) -> GreenToken { - let head = GreenTokenHead { kind, _c: Count::new() }; + Self::with_trivia(kind, text, Vec::new(), Vec::new()) + } + + pub fn with_trivia( + kind: SyntaxKind, + text: &str, + leading: Vec, + trailing: Vec, + ) -> Self { + assert!( + leading + .iter() + .chain(&trailing) + .all(|it| { it.leading_trivia().is_empty() && it.trailing_trivia().is_empty() }), + "trivia tokens cannot themselves carry trivia" + ); + let head = GreenTokenHead { + kind, + leading: leading.into_boxed_slice(), + trailing: trailing.into_boxed_slice(), + _c: Count::new(), + }; let ptr = ThinArc::from_header_and_iter(head, text.bytes()); GreenToken { ptr } } + #[inline] - unsafe fn from_raw(ptr: ptr::NonNull) -> GreenToken { + pub(crate) unsafe fn from_raw(ptr: ptr::NonNull) -> GreenToken { let arc = Arc::from_raw(&ptr.as_ref().data as *const ReprThin); let arc = mem::transmute::, ThinArc>(arc); GreenToken { ptr: arc } diff --git a/src/syntax_text.rs b/src/syntax_text.rs index 3ab3f6c..06fa4dc 100644 --- a/src/syntax_text.rs +++ b/src/syntax_text.rs @@ -1,4 +1,4 @@ -use std::fmt; +use std::{fmt, iter}; use crate::{ cursor::{SyntaxNode, SyntaxToken}, @@ -17,6 +17,10 @@ impl SyntaxText { SyntaxText { node, range } } + pub(crate) fn with_range(node: SyntaxNode, range: TextRange) -> SyntaxText { + SyntaxText { node, range } + } + pub fn len(&self) -> TextSize { self.range.len() } @@ -105,13 +109,20 @@ impl SyntaxText { fn tokens_with_ranges(&self) -> impl Iterator { let text_range = self.range; - self.node.descendants_with_tokens().filter_map(|element| element.into_token()).filter_map( - move |token| { + self.node + .descendants_with_tokens() + .filter_map(|element| element.into_token()) + .flat_map(|token| { + token + .leading_trivia() + .chain(iter::once(token.clone())) + .chain(token.trailing_trivia()) + }) + .filter_map(move |token| { let token_range = token.text_range(); let range = text_range.intersect(token_range)?; Some((token, range - token_range.start())) - }, - ) + }) } }