Merge pull request #52 from nix-community/nix24-paths

Support paths with dynamic interpolation from Nix 2.4+
nix-community · Nov 23, 2021 · 1a45713 · 1a45713
2 parents c89150a + 851e312
commit 1a45713
Show file tree

Hide file tree

Showing 12 changed files with 283 additions and 10 deletions.
diff --git a/src/kinds.rs b/src/kinds.rs
@@ -97,6 +97,7 @@ pub enum SyntaxKind {
     NODE_UNARY_OP,
     NODE_LITERAL,
     NODE_WITH,
+    NODE_PATH_WITH_INTERPOL,
 
     #[doc(hidden)]
     __LAST,

diff --git a/src/parser.rs b/src/parser.rs
@@ -212,20 +212,26 @@ where
         self.get_text_position()
     }
     fn bump(&mut self) {
-        let next = self.buffer.pop_front().or_else(|| self.iter.next());
+        let next = self.try_next();
         match next {
             Some((token, s)) => {
                 if token.is_trivia() {
                     self.trivia_buffer.push((token, s))
                 } else {
-                    self.drain_trivia_buffer();
-                    self.consumed += TextSize::of(s.as_str());
-                    self.builder.token(NixLanguage::kind_to_raw(token), s.as_str())
+                    self.manual_bump(s, token);
                 }
             }
             None => self.errors.push(ParseError::UnexpectedEOF),
         }
     }
+    fn try_next(&mut self) -> Option<(SyntaxKind, SmolStr)> {
+        self.buffer.pop_front().or_else(|| self.iter.next())
+    }
+    fn manual_bump(&mut self, s: SmolStr, token: SyntaxKind) {
+        self.drain_trivia_buffer();
+        self.consumed += TextSize::of(s.as_str());
+        self.builder.token(NixLanguage::kind_to_raw(token), s.as_str())
+    }
     fn peek_data(&mut self) -> Option<&(SyntaxKind, SmolStr)> {
         while self.peek_raw().map(|&(t, _)| t.is_trivia()).unwrap_or(false) {
             self.bump();
@@ -539,6 +545,34 @@ where
             }
             TOKEN_DYNAMIC_START => self.parse_dynamic(),
             TOKEN_STRING_START => self.parse_string(),
+            TOKEN_PATH => {
+                let next = self.try_next();
+                if let Some((token, s)) = next {
+                    let is_complex_path = self
+                        .peek()
+                        .map_or(false, |t| t == TOKEN_INTERPOL_START);
+                    self.start_node(if is_complex_path { NODE_PATH_WITH_INTERPOL } else { NODE_LITERAL });
+                    self.manual_bump(s, token);
+                    if is_complex_path {
+                        loop {
+                            match self.peek_raw().map(|(t, _)| t) {
+                                Some(TOKEN_PATH) => self.bump(),
+                                Some(TOKEN_INTERPOL_START) => {
+                                    self.start_node(NODE_STRING_INTERPOL);
+                                    self.bump();
+                                    self.parse_expr();
+                                    self.expect(TOKEN_INTERPOL_END);
+                                    self.finish_node();
+                                },
+                                _ => break
+                            }
+                        }
+                    }
+                    self.finish_node();
+                } else {
+                    self.errors.push(ParseError::UnexpectedEOF);
+                }
+            },
             t if t.is_literal() => {
                 self.start_node(NODE_LITERAL);
                 self.bump();

diff --git a/src/tokenizer.rs b/src/tokenizer.rs
@@ -31,12 +31,14 @@ struct Interpol {
     brackets: u32,
     string: bool,
     multiline: bool,
+    path_interpol: bool,
 }
 #[derive(Clone, Copy)]
 enum Todo {
     StringBody { multiline: bool },
     StringEnd,
     InterpolStart,
+    Path,
 }
 #[derive(Clone, Copy, Default)]
 struct Context {
@@ -140,7 +142,7 @@ impl<'a> Tokenizer<'a> {
                     Some('{') => {
                         self.state = start;
                         self.ctx.push(Context {
-                            interpol: Some(Interpol { brackets: 0, string: true, multiline }),
+                            interpol: Some(Interpol { brackets: 0, string: true, multiline, path_interpol: false }),
                             todo: Some(Todo::InterpolStart),
                             ..Default::default()
                         });
@@ -168,6 +170,21 @@ impl<'a> Iterator for Tokenizer<'a> {
                         return Some((TOKEN_INTERPOL_START, self.string_since(start)));
                     }
                 }
+                Some(Todo::Path) => {
+                    *todo = Some(Todo::Path);
+                    if self.starts_with_bump("${") {
+                        self.ctx.push(Context {
+                            interpol: Some(Interpol {
+                                brackets: 0,
+                                string: false,
+                                multiline: false,
+                                path_interpol: true,
+                            }),
+                            todo: None,
+                        });
+                        return Some((TOKEN_INTERPOL_START, self.string_since(start)));
+                    }
+                }
                 Some(Todo::StringBody { multiline }) => {
                     *todo = Some(Todo::StringEnd);
                     let token = self.next_string(multiline);
@@ -206,6 +223,10 @@ impl<'a> Iterator for Tokenizer<'a> {
         }
 
         if self.consume(char::is_whitespace) > 0 {
+            let ctx = self.ctx.last_mut().unwrap();
+            if matches!(ctx.todo, Some(Todo::Path)) {
+                ctx.todo = None;
+            }
             return Some((TOKEN_WHITESPACE, self.string_since(start)));
         }
 
@@ -259,7 +280,9 @@ impl<'a> Iterator for Tokenizer<'a> {
             }
             self.consume(is_valid_path_char);
             let ident = self.string_since(start);
-            if ident.ends_with('/') {
+            if self.remaining().starts_with("${") {
+                self.ctx.last_mut().unwrap().todo = Some(Todo::Path);
+            } else if ident.ends_with('/') {
                 return Some((TOKEN_ERROR, ident));
             }
             return Some((TOKEN_PATH, ident));
@@ -284,7 +307,7 @@ impl<'a> Iterator for Tokenizer<'a> {
                 Some((TOKEN_CURLY_B_OPEN, self.string_since(start)))
             }
             '}' => {
-                if let Some(Interpol { ref mut brackets, string, multiline }) =
+                if let Some(Interpol { ref mut brackets, string, multiline, path_interpol }) =
                     self.ctx.last_mut().unwrap().interpol
                 {
                     match brackets.checked_sub(1) {
@@ -296,6 +319,9 @@ impl<'a> Iterator for Tokenizer<'a> {
                                 self.ctx.last_mut().unwrap().todo =
                                     Some(Todo::StringBody { multiline });
                                 return Some((TOKEN_INTERPOL_END, self.string_since(start)));
+                            } else if path_interpol {
+                                self.ctx.last_mut().unwrap().todo = Some(Todo::Path);
+                                return Some((TOKEN_INTERPOL_END, self.string_since(start)));
                             } else {
                                 return Some((TOKEN_DYNAMIC_END, self.string_since(start)));
                             }
@@ -366,7 +392,7 @@ impl<'a> Iterator for Tokenizer<'a> {
             '$' if self.peek() == Some('{') => {
                 self.next().unwrap();
                 self.ctx.push(Context {
-                    interpol: Some(Interpol { brackets: 0, string: false, multiline: false }),
+                    interpol: Some(Interpol { brackets: 0, string: false, multiline: false, path_interpol: false, }),
                     ..Default::default()
                 });
                 Some((TOKEN_DYNAMIC_START, self.string_since(start)))
@@ -395,7 +421,13 @@ impl<'a> Iterator for Tokenizer<'a> {
                         "rec" => TOKEN_REC,
                         "then" => TOKEN_THEN,
                         "with" => TOKEN_WITH,
-                        _ => TOKEN_IDENT,
+                        _ => {
+                            if matches!(self.ctx.last_mut().unwrap().todo, Some(Todo::Path)) {
+                                TOKEN_PATH
+                            } else {
+                                TOKEN_IDENT
+                            }
+                        },
                     },
                     IdentType::Path | IdentType::Store => TOKEN_PATH,
                     IdentType::Uri => TOKEN_URI,
@@ -858,6 +890,65 @@ mod tests {
         assert_eq!(tokenize("<hello/world>"), path("<hello/world>"));
     }
     #[test]
+    fn test_path_interpol() {
+        assert_eq!(
+            tokenize("./${foo}"),
+            tokens![
+                (TOKEN_PATH, "./"),
+                (TOKEN_INTERPOL_START, "${"),
+                (TOKEN_IDENT, "foo"),
+                (TOKEN_INTERPOL_END, "}")
+            ]
+        );
+        assert_eq!(
+            tokenize("./${foo} bar"),
+            tokens![
+                (TOKEN_PATH, "./"),
+                (TOKEN_INTERPOL_START, "${"),
+                (TOKEN_IDENT, "foo"),
+                (TOKEN_INTERPOL_END, "}"),
+                (TOKEN_WHITESPACE, " "),
+                (TOKEN_IDENT, "bar"),
+            ]
+        );
+        assert_eq!(
+            tokenize("./${foo}${bar}"),
+            tokens![
+                (TOKEN_PATH, "./"),
+                (TOKEN_INTERPOL_START, "${"),
+                (TOKEN_IDENT, "foo"),
+                (TOKEN_INTERPOL_END, "}"),
+                (TOKEN_INTERPOL_START, "${"),
+                (TOKEN_IDENT, "bar"),
+                (TOKEN_INTERPOL_END, "}"),
+            ]
+        );
+        assert_eq!(
+            tokenize("./${foo}a${bar}"),
+            tokens![
+                (TOKEN_PATH, "./"),
+                (TOKEN_INTERPOL_START, "${"),
+                (TOKEN_IDENT, "foo"),
+                (TOKEN_INTERPOL_END, "}"),
+                (TOKEN_PATH, "a"),
+                (TOKEN_INTERPOL_START, "${"),
+                (TOKEN_IDENT, "bar"),
+                (TOKEN_INTERPOL_END, "}"),
+            ]
+        );
+        assert_eq!(
+            tokenize("\"./${foo}\""),
+            tokens![
+                (TOKEN_STRING_START, "\""),
+                (TOKEN_STRING_CONTENT, "./"),
+                (TOKEN_INTERPOL_START, "${"),
+                (TOKEN_IDENT, "foo"),
+                (TOKEN_INTERPOL_END, "}"),
+                (TOKEN_STRING_END, "\""),
+            ]
+        );
+    }
+    #[test]
     fn uri() {
         assert_eq!(
             tokenize("https://google.com/?q=Hello+World"),

diff --git a/src/types.rs b/src/types.rs
@@ -251,6 +251,7 @@ pub enum ParsedType {
     UnaryOp(UnaryOp),
     Value(Value),
     With(With),
+    PathWithInterpol(PathWithInterpol),
 }
 
 impl TryFrom<SyntaxNode> for ParsedType {
@@ -285,6 +286,7 @@ impl TryFrom<SyntaxNode> for ParsedType {
             NODE_UNARY_OP => Ok(ParsedType::UnaryOp(UnaryOp::cast(node).unwrap())),
             NODE_LITERAL => Ok(ParsedType::Value(Value::cast(node).unwrap())),
             NODE_WITH => Ok(ParsedType::With(With::cast(node).unwrap())),
+            NODE_PATH_WITH_INTERPOL => Ok(ParsedType::PathWithInterpol(PathWithInterpol::cast(node).unwrap())),
             other => Err(ParsedTypeError(other)),
         }
     }
@@ -320,6 +322,7 @@ impl TypedNode for ParsedType {
             ParsedType::UnaryOp(n) => n.node(),
             ParsedType::Value(n) => n.node(),
             ParsedType::With(n) => n.node(),
+            ParsedType::PathWithInterpol(n) => n.node(),
         }
     }
 
@@ -539,5 +542,6 @@ typed! [
         pub fn body(&self) -> Option<SyntaxNode> {
             nth!(self; 1)
         }
-    }
+    },
+    NODE_PATH_WITH_INTERPOL => PathWithInterpol: Wrapper
 ];
diff --git a/test_data/parser/paths/2.expect b/test_data/parser/paths/2.expect
@@ -0,0 +1,89 @@
+NODE_ROOT 0..72 {
+  NODE_LET_IN 0..72 {
+    TOKEN_LET("let") 0..3
+    TOKEN_WHITESPACE("\n  ") 3..6
+    NODE_KEY_VALUE 6..29 {
+      NODE_KEY 6..7 {
+        NODE_IDENT 6..7 {
+          TOKEN_IDENT("a") 6..7
+        }
+      }
+      TOKEN_WHITESPACE(" ") 7..8
+      TOKEN_ASSIGN("=") 8..9
+      TOKEN_WHITESPACE(" ") 9..10
+      NODE_LAMBDA 10..28 {
+        NODE_IDENT 10..11 {
+          TOKEN_IDENT("f") 10..11
+        }
+        TOKEN_COLON(":") 11..12
+        TOKEN_WHITESPACE(" ") 12..13
+        NODE_PATH_WITH_INTERPOL 13..28 {
+          TOKEN_PATH("./foo") 13..18
+          NODE_STRING_INTERPOL 18..24 {
+            TOKEN_INTERPOL_START("${") 18..20
+            NODE_IDENT 20..23 {
+              TOKEN_IDENT("bar") 20..23
+            }
+            TOKEN_INTERPOL_END("}") 23..24
+          }
+          TOKEN_PATH("/baz") 24..28
+        }
+      }
+      TOKEN_SEMICOLON(";") 28..29
+    }
+    TOKEN_WHITESPACE("\n  ") 29..32
+    NODE_KEY_VALUE 32..67 {
+      NODE_KEY 32..33 {
+        NODE_IDENT 32..33 {
+          TOKEN_IDENT("b") 32..33
+        }
+      }
+      TOKEN_WHITESPACE(" ") 33..34
+      TOKEN_ASSIGN("=") 34..35
+      TOKEN_WHITESPACE(" ") 35..36
+      NODE_APPLY 36..66 {
+        NODE_APPLY 36..53 {
+          NODE_APPLY 36..44 {
+            NODE_IDENT 36..37 {
+              TOKEN_IDENT("a") 36..37
+            }
+            TOKEN_WHITESPACE(" ") 37..38
+            TOKEN_WHITESPACE(" ") 38..39
+            NODE_LITERAL 39..44 {
+              TOKEN_PATH("./bar") 39..44
+            }
+          }
+          NODE_PATH_WITH_INTERPOL 44..53 {
+            TOKEN_PATH("./baz") 44..49
+            NODE_STRING_INTERPOL 49..53 {
+              TOKEN_INTERPOL_START("${") 49..51
+              NODE_IDENT 51..52 {
+                TOKEN_IDENT("x") 51..52
+              }
+              TOKEN_INTERPOL_END("}") 52..53
+            }
+          }
+        }
+        TOKEN_WHITESPACE(" ") 53..54
+        NODE_PATH_WITH_INTERPOL 54..66 {
+          TOKEN_PATH("./snens") 54..61
+          NODE_STRING_INTERPOL 61..65 {
+            TOKEN_INTERPOL_START("${") 61..63
+            NODE_IDENT 63..64 {
+              TOKEN_IDENT("x") 63..64
+            }
+            TOKEN_INTERPOL_END("}") 64..65
+          }
+          TOKEN_PATH("y") 65..66
+        }
+      }
+      TOKEN_SEMICOLON(";") 66..67
+    }
+    TOKEN_WHITESPACE("\n") 67..68
+    TOKEN_IN("in") 68..70
+    TOKEN_WHITESPACE(" ") 70..71
+    NODE_IDENT 71..72 {
+      TOKEN_IDENT("b") 71..72
+    }
+  }
+}
diff --git a/test_data/parser/paths/2.nix b/test_data/parser/paths/2.nix
@@ -0,0 +1,4 @@
+let
+  a = f: ./foo${bar}/baz;
+  b = a ./bar ./baz${x} ./snens${x}y;
+in b
diff --git a/test_data/parser/paths/3.expect b/test_data/parser/paths/3.expect
@@ -0,0 +1,12 @@
+NODE_ROOT 0..6 {
+  NODE_PATH_WITH_INTERPOL 0..6 {
+    TOKEN_PATH("a/") 0..2
+    NODE_STRING_INTERPOL 2..6 {
+      TOKEN_INTERPOL_START("${") 2..4
+      NODE_IDENT 4..5 {
+        TOKEN_IDENT("b") 4..5
+      }
+      TOKEN_INTERPOL_END("}") 5..6
+    }
+  }
+}
diff --git a/test_data/parser/paths/3.nix b/test_data/parser/paths/3.nix
@@ -0,0 +1 @@
+a/${b}