fix: handle comment delimiters in strings

ionide · Nov 10, 2023 · 5a41f3b · 5a41f3b
1 parent fa69ca8
commit 5a41f3b
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 35 deletions.
diff --git a/grammar.js b/grammar.js
@@ -50,6 +50,8 @@ module.exports = grammar({
   ],
 
   extras: $ => [
+    $.block_comment,
+    $.line_comment,
     /[ \s\f\uFEFF\u2060\u200B]|\\\r?n/,
   ],
 
@@ -97,8 +99,6 @@ module.exports = grammar({
 
     _module_elem: $ =>
       choice(
-        $.block_comment,
-        $.line_comment,
         $.value_declaration,
         $.module_defn,
         $.module_abbrev,
@@ -388,8 +388,6 @@ module.exports = grammar({
 
     _expression_inner: $ =>
       choice(
-        $.line_comment,
-        $.block_comment,
         $.const,
         $.paren_expression,
         $.begin_end_expression,
@@ -1580,7 +1578,7 @@ module.exports = grammar({
     ),
 
     // note: \n is allowed in strings
-    _simple_string_char: $ => /[^\t\r\u0008\a\f\v\\"]/,
+    _simple_string_char: $ => imm(prec(1, /[^\t\r\u0008\a\f\v\\"]/)),
     _string_char: $ => choice(
       $._simple_string_char,
       $._escape_char,
@@ -1605,8 +1603,7 @@ module.exports = grammar({
     bytechar: $ => seq("'", $._char_char, imm("'B")),
     bytearray: $ => seq('"', repeat($._string_char), imm('"B')),
     verbatim_bytearray: $ => seq('@"', repeat($._verbatim_string_char), imm('"B')),
-    _simple_or_escape_char: $ => choice($._escape_char, imm(/[^'\\]/)),
-    triple_quoted_string: $ => seq('"""', repeat($._simple_or_escape_char), imm('"""')),
+    triple_quoted_string: $ => seq('"""', repeat($._string_char), imm('"""')),
     _newline: $ => /\r?\n/,
 
     unit: $ => seq("(", optional(seq($._virtual_open_section, $._virtual_end_section)), ")"),
@@ -1733,7 +1730,7 @@ module.exports = grammar({
     //
 
     block_comment: $ => seq("(*", $.block_comment_content, "*)"),
-    line_comment: $ => token(seq("//", repeat(/[^\n\r]/))),
+    line_comment: $ => token(seq('//', /[^\n\r]*/)),
 
     identifier: $ => choice(
       /[_\p{XID_Start}][_'\p{XID_Continue}]*/,

diff --git a/src/grammar.json b/src/grammar.json
@@ -6061,8 +6061,15 @@
       ]
     },
     "_simple_string_char": {
-      "type": "PATTERN",
-      "value": "[^\\t\\r\\u0008\\a\\f\\v\\\\\"]"
+      "type": "IMMEDIATE_TOKEN",
+      "content": {
+        "type": "PREC",
+        "value": 1,
+        "content": {
+          "type": "PATTERN",
+          "value": "[^\\t\\r\\u0008\\a\\f\\v\\\\\"]"
+        }
+      }
     },
     "_string_char": {
       "type": "CHOICE",
@@ -6264,22 +6271,6 @@
         }
       ]
     },
-    "_simple_or_escape_char": {
-      "type": "CHOICE",
-      "members": [
-        {
-          "type": "SYMBOL",
-          "name": "_escape_char"
-        },
-        {
-          "type": "IMMEDIATE_TOKEN",
-          "content": {
-            "type": "PATTERN",
-            "value": "[^'\\\\]"
-          }
-        }
-      ]
-    },
     "triple_quoted_string": {
       "type": "SEQ",
       "members": [
@@ -6291,7 +6282,7 @@
           "type": "REPEAT",
           "content": {
             "type": "SYMBOL",
-            "name": "_simple_or_escape_char"
+            "name": "_string_char"
           }
         },
         {
@@ -7359,11 +7350,8 @@
             "value": "//"
           },
           {
-            "type": "REPEAT",
-            "content": {
-              "type": "PATTERN",
-              "value": "[^\\n\\r]"
-            }
+            "type": "PATTERN",
+            "value": "[^\\n\\r]*"
           }
         ]
       }

diff --git a/test/corpus/comments.txt b/test/corpus/comments.txt
@@ -79,7 +79,7 @@ let x = 1
       (const (int)))))
 
 ================================================================================
-coment in simple string
+comment in simple string
 ================================================================================
 
 let x = "//comment in string"
@@ -97,7 +97,7 @@ let x = "//comment in string"
         (string)))))
 
 ================================================================================
-coment in triple-quoted string
+comment in triple-quoted string
 ================================================================================
 
 let x = """
@@ -117,7 +117,7 @@ let x = """
         (triple_quoted_string)))))
 
 ================================================================================
-coment in verbatim string
+comment in verbatim string
 ================================================================================
 
 let x = @"//comment in string"