fix: handle unknown parser tokens as parse errors

crates/parser-generator/src/parser_generator.rs

@@ -286,26 +286,30 @@ fn generate_parser_source_code(
         .iter()
         .enumerate()
         .map(|(i, s)| {
-            format!(
-                r#"TokenKind::{} => {},"#,
-                match s {
-                    Component::Terminal(TokenKind::RAW(s)) =>
-                        format!(r#"RAW(s) if s == "{}""#, s.trim_matches('\'')),
-                    Component::Terminal(TokenKind::KEYWORD(s)) =>
-                        format!(r#"KEYWORD(s) if s == "{s}""#),
-                    Component::Terminal(s) => s.to_id(),
-                    _ => unreachable!(),
-                },
-                if matches!(
-                    s,
-                    Component::Terminal(TokenKind::C_COMMENT)
-                        | Component::Terminal(TokenKind::SQL_COMMENT)
-                ) {
-                    // Comments are not included in the parser lookahead tokens, so offset them by the number of non-terminal symbols
-                    i + non_terminal_symbols.len()
-                } else {
-                    i
+            let token_pattern = match s {
+                Component::Terminal(TokenKind::RAW(s)) => {
+                    format!(r#"RAW(s) if s == "{}""#, s.trim_matches('\''))
                 }
+                Component::Terminal(TokenKind::KEYWORD(s)) => {
+                    format!(r#"KEYWORD(s) if s == "{s}""#)
+                }
+                Component::Terminal(s) => s.to_id(),
+                _ => unreachable!(),
+            };
+            let component_id = if matches!(
+                s,
+                Component::Terminal(TokenKind::C_COMMENT)
+                    | Component::Terminal(TokenKind::SQL_COMMENT)
+            ) {
+                // Comments are not included in the parser lookahead tokens, so offset them by the number of non-terminal symbols
+                i + non_terminal_symbols.len()
+            } else {
+                i
+            };
+            format!(
+                r#"TokenKind::{} => Some({}),"#,
+                token_pattern,
+                component_id,
             )
         })
         .collect::<Vec<_>>()

crates/parser-generator/templates/parser_template.rs

@@ -33,9 +33,9 @@ pub(crate) fn end_rule_kind() -> TokenKind {{end_rule_kind}}
 
 pub(crate) fn end_rule_id() -> u32 {{end_rule_id}}
 
-pub(crate) fn token_kind_to_component_id(kind: &TokenKind) -> u32 {
+pub(crate) fn token_kind_to_component_id(kind: &TokenKind) -> Option<u32> {
     match kind {{token_to_component_id}
-    _ => unreachable!(),
+    _ => None,
     }
 }
 

crates/postgresql-cst-parser/src/cst.rs

@@ -247,7 +247,16 @@ pub fn parse_with_transformer(
             }
         };
 
-        let mut cid = token_kind_to_component_id(&token.kind);
+        let mut cid = match token_kind_to_component_id(&token.kind) {
+            Some(cid) => cid,
+            None => {
+                return Err(ParserError::ParseError {
+                    message: format!("unexpected token: {}", token.value),
+                    start_byte_pos: token.start_byte_pos,
+                    end_byte_pos: token.end_byte_pos,
+                });
+            }
+        };
 
         if matches!(token.kind, TokenKind::C_COMMENT | TokenKind::SQL_COMMENT) {
             if last_pos < token.start_byte_pos {
@@ -297,7 +306,8 @@ pub fn parse_with_transformer(
                     ParseTransform::InsertToken(token_kind) => {
                         let last_extra = extras.last().unwrap();
 
-                        cid = token_kind_to_component_id(&token_kind);
+                        cid = token_kind_to_component_id(&token_kind)
+                            .expect("inserted token should map to a syntax kind");
                         token = Token {
                             start_byte_pos: last_extra.end_byte_pos,
                             end_byte_pos: last_extra.end_byte_pos,
@@ -454,7 +464,8 @@ pub fn parse_with_transformer(
             break;
         }
 
-        let cid = token_kind_to_component_id(&token.kind);
+        let cid = token_kind_to_component_id(&token.kind)
+            .expect("remaining token should map to a syntax kind");
         let kind = SyntaxKind::from_raw(RawSyntaxKind(cid));
         extras.push(Extra {
             kind,

crates/postgresql-cst-parser/src/lib.rs

@@ -203,6 +203,20 @@ mod tests {
 
         assert_eq!(actual, expected);
     }
+
+    #[test]
+    fn test_unexpected_brace_token() {
+        let input = "{}";
+        let actual = parse(input);
+
+        let expected = Err(ParserError::ParseError {
+            message: "unexpected token: {".to_string(),
+            start_byte_pos: 0,
+            end_byte_pos: 1,
+        });
+
+        assert_eq!(actual, expected);
+    }
 }
 
 #[cfg(test)]