lattice_snippet/
parse.rs

1//! TextMate / LSP snippet body parser. Produces a
2//! [`SnippetBody`] from a string; consumers (renderer +
3//! active-snippet state machine) walk the resulting tokens.
4//!
5//! Grammar (matching VS Code's parser; see
6//! [LSP 3.17 §Snippet Syntax](https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#snippet_syntax)):
7//!
8//! ```text
9//! body          ::= ( token )*
10//! token         ::= literal | dollar
11//! literal       ::= ( escaped | safe_char )+
12//! escaped       ::= '\\' ( '$' | '\\' | '}' )
13//! safe_char     ::= any char except '$' | '}' | escape opener
14//!
15//! dollar        ::= '$' ( int | name | '{' block '}' )
16//! block         ::= int                            -- '${1}'
17//!                 | int ':' body                   -- '${1:foo}'
18//!                 | int '|' choices '|'            -- '${1|a,b,c|}'
19//!                 | int '/' pat '/' repl '/' flags -- '${1/.../.../i}'
20//!                 | name                           -- '${VAR}'
21//!                 | name ':' body                  -- '${VAR:fallback}'
22//!                 | name '/' pat '/' repl '/' flags
23//! choices       ::= choice ( ',' choice )*
24//! choice        ::= ( '\\,' | not(',' | '|') )*
25//! pat / repl    ::= ( '\\/' | not('/') )*
26//! flags         ::= [a-zA-Z]*
27//! int           ::= [0-9]+
28//! name          ::= [a-zA-Z_][a-zA-Z0-9_]*
29//! ```
30//!
31//! On a malformed dollar block (e.g. unclosed `${`), the parser
32//! falls back to literal output for that span -- VS Code's
33//! "be lenient" behaviour. This means a snippet body that
34//! looks-like-but-isn't a placeholder still inserts as plain
35//! text rather than failing the whole expansion. Returns a
36//! [`ParseError`] only for genuinely catastrophic input
37//! (currently never -- v1 always falls through to literal).
38
39use crate::token::{ChoiceOption, SnippetBody, SnippetToken, TransformTarget};
40
41/// Snippet parse error. Empty in v1 -- the parser is total and
42/// falls back to literal output on malformed input. Reserved
43/// for future strict-mode parsing if a use case appears.
44#[derive(Debug, thiserror::Error)]
45pub enum ParseError {
46    #[error("internal parser error: {0}")]
47    Internal(&'static str),
48}
49
50/// Parse a TextMate / LSP snippet body string.
51pub fn parse(body: &str) -> Result<SnippetBody, ParseError> {
52    let mut p = Parser::new(body);
53    let tokens = p.parse_body(None)?;
54    Ok(SnippetBody::new(merge_literals(tokens)))
55}
56
57/// Merge consecutive `Literal` tokens. The parser may emit
58/// runs of single-character literals while walking escapes;
59/// the renderer is happier with a compact stream.
60fn merge_literals(tokens: Vec<SnippetToken>) -> Vec<SnippetToken> {
61    let mut out: Vec<SnippetToken> = Vec::with_capacity(tokens.len());
62    for tok in tokens {
63        if let SnippetToken::Literal(new) = &tok
64            && let Some(SnippetToken::Literal(prev)) = out.last_mut()
65        {
66            prev.push_str(new);
67            continue;
68        }
69        out.push(tok);
70    }
71    out
72}
73
74struct Parser<'a> {
75    src: &'a [u8],
76    pos: usize,
77}
78
79impl<'a> Parser<'a> {
80    fn new(s: &'a str) -> Self {
81        Self {
82            src: s.as_bytes(),
83            pos: 0,
84        }
85    }
86
87    fn peek(&self) -> Option<u8> {
88        self.src.get(self.pos).copied()
89    }
90
91    fn bump(&mut self) -> Option<u8> {
92        let b = self.peek();
93        if b.is_some() {
94            self.pos += 1;
95        }
96        b
97    }
98
99    #[allow(dead_code)]
100    fn starts_with(&self, lit: &[u8]) -> bool {
101        self.src
102            .get(self.pos..self.pos + lit.len())
103            .map(|s| s == lit)
104            .unwrap_or(false)
105    }
106
107    /// Parse tokens until either end-of-input or one of the
108    /// stop bytes appears at the current position. The stop
109    /// list is `Some` for nested bodies (e.g. inside a
110    /// `${1:default}` block, `}` closes the block).
111    fn parse_body(&mut self, stop: Option<&[u8]>) -> Result<Vec<SnippetToken>, ParseError> {
112        let mut tokens = Vec::new();
113        while let Some(b) = self.peek() {
114            if let Some(stop) = stop
115                && stop.contains(&b)
116            {
117                break;
118            }
119            match b {
120                b'\\' => {
121                    self.bump();
122                    if let Some(next) = self.peek() {
123                        match next {
124                            b'$' | b'\\' | b'}' => {
125                                self.bump();
126                                tokens.push(SnippetToken::Literal(char::from(next).to_string()));
127                            }
128                            // Unknown escape: keep the backslash
129                            // verbatim. VS Code lenient mode.
130                            _ => {
131                                tokens.push(SnippetToken::Literal("\\".to_string()));
132                            }
133                        }
134                    } else {
135                        tokens.push(SnippetToken::Literal("\\".to_string()));
136                    }
137                }
138                b'$' => {
139                    let saved = self.pos;
140                    self.bump();
141                    match self.try_parse_dollar() {
142                        Some(tok) => tokens.push(tok),
143                        None => {
144                            // Fall back to literal '$'.
145                            self.pos = saved + 1;
146                            tokens.push(SnippetToken::Literal("$".to_string()));
147                        }
148                    }
149                }
150                _ => {
151                    // Walk one Unicode scalar at a time so
152                    // multi-byte UTF-8 chars (héllo, 你好,
153                    // emoji) round-trip cleanly. Lone-byte
154                    // bumps would yield Latin-1-shaped output
155                    // and corrupt non-ASCII content.
156                    let ch = self.peek_char_at(self.pos);
157                    self.pos += ch.len_utf8();
158                    tokens.push(SnippetToken::Literal(ch.to_string()));
159                }
160            }
161        }
162        Ok(tokens)
163    }
164
165    /// Decode the UTF-8 char starting at `pos`. The bytes are
166    /// guaranteed valid UTF-8 because the parser was constructed
167    /// from a `&str`; if `pos` somehow lands mid-codepoint we
168    /// fall back to a `?` so the parser stays total.
169    fn peek_char_at(&self, pos: usize) -> char {
170        let s = std::str::from_utf8(&self.src[pos..])
171            .ok()
172            .and_then(|s| s.chars().next());
173        s.unwrap_or('?')
174    }
175
176    /// Called after consuming a `$`. Parses the rest of the
177    /// dollar form. Returns `None` to signal "this isn't a
178    /// valid dollar form, treat the `$` as literal."
179    fn try_parse_dollar(&mut self) -> Option<SnippetToken> {
180        match self.peek()? {
181            // Bare integer tabstop: $1, $9, $123.
182            b'0'..=b'9' => {
183                let n = self.parse_uint()?;
184                Some(SnippetToken::Tabstop(n))
185            }
186            // Bare variable: $TM_FILENAME.
187            b'A'..=b'Z' | b'a'..=b'z' | b'_' => {
188                let name = self.parse_name()?;
189                Some(SnippetToken::Variable {
190                    name,
191                    default: None,
192                })
193            }
194            // Block form: ${...}.
195            b'{' => {
196                self.bump();
197                self.parse_block()
198            }
199            _ => None,
200        }
201    }
202
203    fn parse_uint(&mut self) -> Option<u32> {
204        let start = self.pos;
205        while let Some(b) = self.peek() {
206            if b.is_ascii_digit() {
207                self.bump();
208            } else {
209                break;
210            }
211        }
212        if start == self.pos {
213            return None;
214        }
215        let s = std::str::from_utf8(&self.src[start..self.pos]).ok()?;
216        s.parse().ok()
217    }
218
219    fn parse_name(&mut self) -> Option<String> {
220        let start = self.pos;
221        // First char is alpha or `_`.
222        match self.peek()? {
223            b'A'..=b'Z' | b'a'..=b'z' | b'_' => {
224                self.bump();
225            }
226            _ => return None,
227        }
228        // Subsequent chars: alpha, digit, or `_`.
229        while let Some(b) = self.peek() {
230            if matches!(b, b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'_') {
231                self.bump();
232            } else {
233                break;
234            }
235        }
236        std::str::from_utf8(&self.src[start..self.pos])
237            .ok()
238            .map(|s| s.to_string())
239    }
240
241    /// Called after consuming `${`. Parses everything up to and
242    /// including the closing `}`. Returns `None` to signal a
243    /// malformed block; the caller falls through to literal.
244    fn parse_block(&mut self) -> Option<SnippetToken> {
245        match self.peek()? {
246            b'0'..=b'9' => {
247                let idx = self.parse_uint()?;
248                match self.peek()? {
249                    b'}' => {
250                        self.bump();
251                        Some(SnippetToken::Tabstop(idx))
252                    }
253                    b':' => {
254                        self.bump();
255                        // Default body: parse until `}`.
256                        let inner = self.parse_body(Some(b"}")).ok()?;
257                        if self.peek()? != b'}' {
258                            return None;
259                        }
260                        self.bump();
261                        Some(SnippetToken::Placeholder {
262                            idx,
263                            default: SnippetBody::new(merge_literals(inner)),
264                        })
265                    }
266                    b'|' => {
267                        self.bump();
268                        let options = self.parse_choices()?;
269                        if self.peek()? != b'}' {
270                            return None;
271                        }
272                        self.bump();
273                        Some(SnippetToken::Choice { idx, options })
274                    }
275                    b'/' => {
276                        self.bump();
277                        let (pattern, replacement, flags) = self.parse_transform()?;
278                        Some(SnippetToken::Transform {
279                            target: TransformTarget::Tabstop(idx),
280                            pattern,
281                            replacement,
282                            flags,
283                        })
284                    }
285                    _ => None,
286                }
287            }
288            b'A'..=b'Z' | b'a'..=b'z' | b'_' => {
289                let name = self.parse_name()?;
290                match self.peek()? {
291                    b'}' => {
292                        self.bump();
293                        Some(SnippetToken::Variable {
294                            name,
295                            default: None,
296                        })
297                    }
298                    b':' => {
299                        self.bump();
300                        let inner = self.parse_body(Some(b"}")).ok()?;
301                        if self.peek()? != b'}' {
302                            return None;
303                        }
304                        self.bump();
305                        Some(SnippetToken::Variable {
306                            name,
307                            default: Some(SnippetBody::new(merge_literals(inner))),
308                        })
309                    }
310                    b'/' => {
311                        self.bump();
312                        let (pattern, replacement, flags) = self.parse_transform()?;
313                        Some(SnippetToken::Transform {
314                            target: TransformTarget::Variable(name),
315                            pattern,
316                            replacement,
317                            flags,
318                        })
319                    }
320                    _ => None,
321                }
322            }
323            _ => None,
324        }
325    }
326
327    fn parse_choices(&mut self) -> Option<Vec<ChoiceOption>> {
328        let mut options = Vec::new();
329        let mut current = String::new();
330        loop {
331            match self.peek()? {
332                b'|' => {
333                    self.bump();
334                    options.push(ChoiceOption {
335                        text: std::mem::take(&mut current),
336                    });
337                    return Some(options);
338                }
339                b',' => {
340                    self.bump();
341                    options.push(ChoiceOption {
342                        text: std::mem::take(&mut current),
343                    });
344                }
345                b'\\' => {
346                    self.bump();
347                    if let Some(esc) = self.peek() {
348                        match esc {
349                            b',' | b'|' | b'\\' => {
350                                self.bump();
351                                current.push(char::from(esc));
352                            }
353                            _ => current.push('\\'),
354                        }
355                    } else {
356                        current.push('\\');
357                    }
358                }
359                b => {
360                    self.bump();
361                    current.push(char::from(b));
362                }
363            }
364        }
365    }
366
367    fn parse_transform(&mut self) -> Option<(String, String, String)> {
368        // Pattern up to next unescaped `/`.
369        let pattern = self.parse_until_slash()?;
370        if self.peek()? != b'/' {
371            return None;
372        }
373        self.bump();
374        let replacement = self.parse_until_slash()?;
375        if self.peek()? != b'/' {
376            return None;
377        }
378        self.bump();
379        // Flags up to `}`.
380        let mut flags = String::new();
381        while let Some(b) = self.peek() {
382            if b == b'}' {
383                self.bump();
384                return Some((pattern, replacement, flags));
385            }
386            self.bump();
387            flags.push(char::from(b));
388        }
389        None
390    }
391
392    fn parse_until_slash(&mut self) -> Option<String> {
393        let mut out = String::new();
394        while let Some(b) = self.peek() {
395            match b {
396                b'/' => return Some(out),
397                b'\\' => {
398                    self.bump();
399                    if let Some(esc) = self.peek() {
400                        self.bump();
401                        match esc {
402                            b'/' | b'\\' => out.push(char::from(esc)),
403                            other => {
404                                // Preserve the escape so regex
405                                // engines downstream can act on
406                                // `\n` / `\t` / `\d` / etc.
407                                out.push('\\');
408                                out.push(char::from(other));
409                            }
410                        }
411                    } else {
412                        out.push('\\');
413                    }
414                }
415                b'}' => return Some(out),
416                _ => {
417                    self.bump();
418                    out.push(char::from(b));
419                }
420            }
421        }
422        Some(out)
423    }
424}
425
426#[cfg(test)]
427mod tests {
428    use super::*;
429
430    fn parse_ok(s: &str) -> SnippetBody {
431        parse(s).expect("parse succeeds (parser is total)")
432    }
433
434    #[test]
435    fn pure_literal_parses_to_one_literal_token() {
436        let b = parse_ok("hello world");
437        assert_eq!(b.tokens, vec![SnippetToken::Literal("hello world".into())]);
438    }
439
440    #[test]
441    fn bare_tabstop_parses() {
442        let b = parse_ok("$1");
443        assert_eq!(b.tokens, vec![SnippetToken::Tabstop(1)]);
444    }
445
446    #[test]
447    fn final_tabstop_parses() {
448        let b = parse_ok("$0");
449        assert_eq!(b.tokens, vec![SnippetToken::Tabstop(0)]);
450    }
451
452    #[test]
453    fn block_tabstop_parses() {
454        let b = parse_ok("${42}");
455        assert_eq!(b.tokens, vec![SnippetToken::Tabstop(42)]);
456    }
457
458    #[test]
459    fn placeholder_with_default_parses() {
460        let b = parse_ok("${1:foo}");
461        match &b.tokens[0] {
462            SnippetToken::Placeholder { idx, default } => {
463                assert_eq!(*idx, 1);
464                assert_eq!(default.tokens, vec![SnippetToken::Literal("foo".into())]);
465            }
466            other => panic!("expected Placeholder, got {other:?}"),
467        }
468    }
469
470    #[test]
471    fn nested_placeholder_inside_default_parses() {
472        let b = parse_ok("${1:outer ${2:inner} more}");
473        match &b.tokens[0] {
474            SnippetToken::Placeholder { idx, default } => {
475                assert_eq!(*idx, 1);
476                assert_eq!(
477                    default.tokens,
478                    vec![
479                        SnippetToken::Literal("outer ".into()),
480                        SnippetToken::Placeholder {
481                            idx: 2,
482                            default: SnippetBody::new(vec![SnippetToken::Literal("inner".into())]),
483                        },
484                        SnippetToken::Literal(" more".into()),
485                    ]
486                );
487            }
488            other => panic!("expected Placeholder, got {other:?}"),
489        }
490    }
491
492    #[test]
493    fn choice_placeholder_parses() {
494        let b = parse_ok("${1|alpha,beta,gamma|}");
495        match &b.tokens[0] {
496            SnippetToken::Choice { idx, options } => {
497                assert_eq!(*idx, 1);
498                let texts: Vec<&str> = options.iter().map(|c| c.text.as_str()).collect();
499                assert_eq!(texts, vec!["alpha", "beta", "gamma"]);
500            }
501            other => panic!("expected Choice, got {other:?}"),
502        }
503    }
504
505    #[test]
506    fn bare_variable_parses() {
507        let b = parse_ok("$TM_FILENAME");
508        match &b.tokens[0] {
509            SnippetToken::Variable { name, default } => {
510                assert_eq!(name, "TM_FILENAME");
511                assert!(default.is_none());
512            }
513            other => panic!("expected Variable, got {other:?}"),
514        }
515    }
516
517    #[test]
518    fn block_variable_parses() {
519        let b = parse_ok("${TM_FILENAME}");
520        match &b.tokens[0] {
521            SnippetToken::Variable { name, default } => {
522                assert_eq!(name, "TM_FILENAME");
523                assert!(default.is_none());
524            }
525            other => panic!("expected Variable, got {other:?}"),
526        }
527    }
528
529    #[test]
530    fn variable_with_fallback_parses() {
531        let b = parse_ok("${TM_FILENAME:default.txt}");
532        match &b.tokens[0] {
533            SnippetToken::Variable { name, default } => {
534                assert_eq!(name, "TM_FILENAME");
535                let d = default.as_ref().expect("has default");
536                assert_eq!(d.tokens, vec![SnippetToken::Literal("default.txt".into())]);
537            }
538            other => panic!("expected Variable, got {other:?}"),
539        }
540    }
541
542    #[test]
543    fn escapes_dollar_brace_backslash() {
544        let b = parse_ok("\\$1 \\\\ \\}");
545        // Expected: literal "$1 \ }" (the escapes consumed).
546        assert_eq!(b.tokens, vec![SnippetToken::Literal("$1 \\ }".into())]);
547    }
548
549    #[test]
550    fn malformed_block_falls_back_to_literal_dollar() {
551        // `${` without a recognised follow -- the `$` lands as
552        // a literal, the rest re-parses (the `{` becomes a
553        // literal too).
554        let b = parse_ok("$ no-block");
555        assert_eq!(b.tokens, vec![SnippetToken::Literal("$ no-block".into())]);
556    }
557
558    #[test]
559    fn transform_token_parses_pattern_replacement_flags() {
560        let b = parse_ok("${1/foo/bar/g}");
561        match &b.tokens[0] {
562            SnippetToken::Transform {
563                target,
564                pattern,
565                replacement,
566                flags,
567            } => {
568                assert!(matches!(target, TransformTarget::Tabstop(1)));
569                assert_eq!(pattern, "foo");
570                assert_eq!(replacement, "bar");
571                assert_eq!(flags, "g");
572            }
573            other => panic!("expected Transform, got {other:?}"),
574        }
575    }
576
577    #[test]
578    fn complex_body_round_trips_to_tokens() {
579        // From friendly-snippets `for-in.json`-shape.
580        let b = parse_ok("for ${1:i} in ${2:iter} {\n\t$0\n}");
581        // Just verify the structure -- 6 tokens: literal,
582        // placeholder, literal, placeholder, literal, tabstop,
583        // literal.
584        assert_eq!(b.tokens.len(), 7);
585        assert!(matches!(b.tokens[0], SnippetToken::Literal(ref s) if s == "for "));
586        assert!(matches!(
587            b.tokens[1],
588            SnippetToken::Placeholder { idx: 1, .. }
589        ));
590        assert!(matches!(b.tokens[2], SnippetToken::Literal(ref s) if s == " in "));
591        assert!(matches!(
592            b.tokens[3],
593            SnippetToken::Placeholder { idx: 2, .. }
594        ));
595        assert!(matches!(b.tokens[4], SnippetToken::Literal(ref s) if s.starts_with(" {")));
596        assert!(matches!(b.tokens[5], SnippetToken::Tabstop(0)));
597        assert!(matches!(b.tokens[6], SnippetToken::Literal(ref s) if s.contains("}")));
598    }
599
600    #[test]
601    fn unicode_literals_round_trip() {
602        let b = parse_ok("héllo wörld");
603        assert_eq!(b.tokens, vec![SnippetToken::Literal("héllo wörld".into())]);
604    }
605}
lattice_snippet/parse.rs

lattice_snippet/
parse.rs
