Fix full-width character alignment

lark/exceptions.py

@@ -1,6 +1,6 @@
 from .utils import logger, NO_VALUE
 from typing import Mapping, Iterable, Callable, Union, TypeVar, Tuple, Any, List, Set, Optional, Collection, TYPE_CHECKING
-
+import unicodedata
 if TYPE_CHECKING:
     from .lexer import Token
     from .parsers.lalr_interactive_parser import InteractiveParser
@@ -20,6 +20,11 @@ def assert_config(value, options: Collection, msg='Got %r, expected one of %s'):
     if value not in options:
         raise ConfigurationError(msg % (value, options))
 
+def _display_width(s):
+    width = 0
+    for ch in s:
+        width += 2 if unicodedata.east_asian_width(ch) in ('F', 'W') else 1
+    return width
 
 class GrammarError(LarkError):
     pass
@@ -66,7 +71,7 @@ def get_context(self, text: str, span: int=40) -> str:
         if not isinstance(text, bytes):
             before = text[start:pos].rsplit('\n', 1)[-1]
             after = text[pos:end].split('\n', 1)[0]
-            return before + after + '\n' + ' ' * len(before.expandtabs()) + '^\n'
+            return before + after + '\n' + ' ' * _display_width(before.expandtabs()) + '^\n'
         else:
             before = text[start:pos].rsplit(b'\n', 1)[-1]
             after = text[pos:end].split(b'\n', 1)[0]

tests/test_parser.py

@@ -204,6 +204,36 @@ def NUM(self, token):
             result = parser.parse(text)
             self.assertEqual(result, expected)
 
+    def test_fullwidth_error_context_alignment(self):
+
+        grammar = r"""
+        start: CHINESE WORD WORD
+
+        CHINESE: "你好"
+        WORD: /[a-zA-Z]+/
+
+        %ignore " "
+        """
+
+        parser = Lark(grammar)
+
+        text = "你好 hello 123"
+
+        try:
+            parser.parse(text)
+            assert False
+        except Exception as e:
+            context = e.get_context(text)
+
+            lines = context.splitlines()
+
+            assert len(lines) >= 2
+
+            caret_pos = lines[1].index("^")
+
+            # ensure full-width chars are counted properly
+            assert caret_pos >= 11
+
     def test_vargs_meta(self):
 
         @v_args(meta=True)