mewin/cxxheaderparser
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Use lex.TOKEN decorator for lexer tokens instead of docstrings

Browse Source 
- Allows usage with -OO
This commit is contained in:
Dustin Spicuzza 2022-12-10 14:08:59 -05:00
parent 19c0604603
commit b47eb7ce10
1 changed files with 6 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
cxxheaderparser/lexer.py
View File

@ -5,6 +5,7 @@ import typing
import sys import sys
from ._ply import lex from ._ply import lex
from ._ply.lex import TOKEN
if sys.version_info >= (3, 8): if sys.version_info >= (3, 8):
@ -186,14 +187,14 @@ class Lexer:
t_NUMBER = r"[0-9][0-9XxA-Fa-f]*" t_NUMBER = r"[0-9][0-9XxA-Fa-f]*"
t_FLOAT_NUMBER = r"[-+]?[0-9]*\.[0-9]+([eE][-+]?[0-9]+)?" t_FLOAT_NUMBER = r"[-+]?[0-9]*\.[0-9]+([eE][-+]?[0-9]+)?"
@TOKEN(r"[A-Za-z_~][A-Za-z0-9_]*")
def t_NAME(self, t: LexToken) -> LexToken: def t_NAME(self, t: LexToken) -> LexToken:
r"[A-Za-z_~][A-Za-z0-9_]*"
if t.value in self.keywords: if t.value in self.keywords:
t.type = t.value t.type = t.value
return t return t
@TOKEN(r"\#.*")
def t_PRECOMP_MACRO(self, t: LexToken) -> typing.Optional[LexToken]: def t_PRECOMP_MACRO(self, t: LexToken) -> typing.Optional[LexToken]:
r"\#.*"
m = _line_re.match(t.value) m = _line_re.match(t.value)
if m: if m:
filename = m.group(2) filename = m.group(2)
@ -207,8 +208,8 @@ class Lexer:
else: else:
return t return t
@TOKEN(r"\/\/.*\n?")
def t_COMMENT_SINGLELINE(self, t: LexToken) -> LexToken: def t_COMMENT_SINGLELINE(self, t: LexToken) -> LexToken:
r"\/\/.*\n?"
if t.value.startswith("///") or t.value.startswith("//!"): if t.value.startswith("///") or t.value.startswith("//!"):
self.comments.append(t.value.lstrip("\t ").rstrip("\n")) self.comments.append(t.value.lstrip("\t ").rstrip("\n"))
t.lexer.lineno += t.value.count("\n") t.lexer.lineno += t.value.count("\n")
@ -230,8 +231,8 @@ class Lexer:
t_STRING_LITERAL = r'"([^"\\]|\\.)*"' t_STRING_LITERAL = r'"([^"\\]|\\.)*"'
# Found at http://ostermiller.org/findcomment.html # Found at http://ostermiller.org/findcomment.html
@TOKEN(r"/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/\n?")
def t_COMMENT_MULTILINE(self, t: LexToken) -> LexToken: def t_COMMENT_MULTILINE(self, t: LexToken) -> LexToken:
r"/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/\n?"
if t.value.startswith("/**") or t.value.startswith("/*!"): if t.value.startswith("/**") or t.value.startswith("/*!"):
# not sure why, but get double new lines # not sure why, but get double new lines
v = t.value.replace("\n\n", "\n") v = t.value.replace("\n\n", "\n")
@ -241,8 +242,8 @@ class Lexer:
t.lexer.lineno += t.value.count("\n") t.lexer.lineno += t.value.count("\n")
return t return t
@TOKEN(r"\n+")
def t_NEWLINE(self, t: LexToken) -> LexToken: def t_NEWLINE(self, t: LexToken) -> LexToken:
r"\n+"
t.lexer.lineno += len(t.value) t.lexer.lineno += len(t.value)
del self.comments[:] del self.comments[:]
return t return t