mewin/cxxheaderparser
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Make mypy happier

Browse Source
This commit is contained in:
Dustin Spicuzza 2021-01-01 15:49:43 -05:00
parent 0329268622
commit 23b95f09f2
7 changed files with 80 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
cxxheaderparser/lexer.py
View File

@ -8,7 +8,7 @@ from ._ply import lex
if sys.version_info >= (3, 8): if sys.version_info >= (3, 8):
Protocol = typing.Protocol from typing import Protocol
else: else:
Protocol = object Protocol = object
@ -43,7 +43,7 @@ class LexToken(Protocol):
location: Location location: Location
PhonyEnding = lex.LexToken() PhonyEnding: LexToken = lex.LexToken() # type: ignore
PhonyEnding.type = "PLACEHOLDER" PhonyEnding.type = "PLACEHOLDER"
PhonyEnding.value = "" PhonyEnding.value = ""
PhonyEnding.lineno = 0 PhonyEnding.lineno = 0
@ -278,7 +278,7 @@ class Lexer:
self.lookahead = typing.Deque[LexToken]() self.lookahead = typing.Deque[LexToken]()
# For 'set_group_of_tokens' support # For 'set_group_of_tokens' support
self._get_token = self.lex.token self._get_token: typing.Callable[[], LexToken] = self.lex.token
self.lookahead_stack = typing.Deque[typing.Deque[LexToken]]() self.lookahead_stack = typing.Deque[typing.Deque[LexToken]]()
def current_location(self) -> Location: def current_location(self) -> Location:
@ -462,7 +462,7 @@ class Lexer:
def return_token(self, tok: LexToken) -> None: def return_token(self, tok: LexToken) -> None:
self.lookahead.appendleft(tok) self.lookahead.appendleft(tok)
def return_tokens(self, toks: typing.Iterable[LexToken]) -> None: def return_tokens(self, toks: typing.Sequence[LexToken]) -> None:
self.lookahead.extendleft(reversed(toks)) self.lookahead.extendleft(reversed(toks))

101
cxxheaderparser/parser.py
View File

@ -8,7 +8,6 @@ from .errors import CxxParseError
from .lexer import Lexer, LexToken, Location, PhonyEnding from .lexer import Lexer, LexToken, Location, PhonyEnding
from .options import ParserOptions from .options import ParserOptions
from .parserstate import ( from .parserstate import (
BlockState,
ClassBlockState, ClassBlockState,
EmptyBlockState, EmptyBlockState,
ExternBlockState, ExternBlockState,
@ -45,6 +44,7 @@ from .types import (
TemplateArgument, TemplateArgument,
TemplateDecl, TemplateDecl,
TemplateNonTypeParam, TemplateNonTypeParam,
TemplateParam,
TemplateSpecialization, TemplateSpecialization,
TemplateTypeParam, TemplateTypeParam,
Token, Token,
@ -60,6 +60,9 @@ from .visitor import CxxVisitor
LexTokenList = typing.List[LexToken] LexTokenList = typing.List[LexToken]
T = typing.TypeVar("T") T = typing.TypeVar("T")
ST = typing.TypeVar("ST", bound=State)
PT = typing.TypeVar("PT", Parameter, TemplateNonTypeParam)
class CxxParser: class CxxParser:
""" """
@ -82,7 +85,7 @@ class CxxParser:
global_ns = NamespaceDecl([], False) global_ns = NamespaceDecl([], False)
self.current_namespace = global_ns self.current_namespace = global_ns
self.state: BlockState = NamespaceBlockState(None, global_ns) self.state: State = NamespaceBlockState(None, global_ns)
self.anon_id = 0 self.anon_id = 0
self.options = options if options else ParserOptions() self.options = options if options else ParserOptions()
@ -103,7 +106,7 @@ class CxxParser:
# State management # State management
# #
def _push_state(self, cls: typing.Type[T], *args) -> T: def _push_state(self, cls: typing.Type[ST], *args) -> ST:
state = cls(self.state, *args) state = cls(self.state, *args)
if isinstance(state, NamespaceBlockState): if isinstance(state, NamespaceBlockState):
self.current_namespace = state.namespace self.current_namespace = state.namespace
@ -133,7 +136,7 @@ class CxxParser:
def _parse_error( def _parse_error(
self, tok: typing.Optional[LexToken], expected="" self, tok: typing.Optional[LexToken], expected=""
) -> typing.NoReturn: ) -> CxxParseError:
if not tok: if not tok:
# common case after a failed token_if # common case after a failed token_if
tok = self.lex.token() tok = self.lex.token()
@ -270,7 +273,9 @@ class CxxParser:
""" """
# non-ambiguous parsing functions for each token type # non-ambiguous parsing functions for each token type
_translation_unit_tokens = { _translation_unit_tokens: typing.Dict[
str, typing.Callable[[LexToken, typing.Optional[str]], typing.Any]
] = {
"__attribute__": self._consume_gcc_attribute, "__attribute__": self._consume_gcc_attribute,
"__declspec": self._consume_declspec, "__declspec": self._consume_declspec,
"alignas": self._consume_attribute_specifier_seq, "alignas": self._consume_attribute_specifier_seq,
@ -339,16 +344,16 @@ class CxxParser:
self, tok: LexToken, doxygen: typing.Optional[str] self, tok: LexToken, doxygen: typing.Optional[str]
) -> None: ) -> None:
value = self._preprocessor_compress_re.sub("#", tok.value) value = self._preprocessor_compress_re.sub("#", tok.value)
value = self._preprocessor_split_re.split(value, 1) svalue = self._preprocessor_split_re.split(value, 1)
if len(value) == 2: if len(svalue) == 2:
self.state.location = tok.location self.state.location = tok.location
macro = value[0].lower().replace(" ", "") macro = svalue[0].lower().replace(" ", "")
if macro.startswith("#include"): if macro.startswith("#include"):
self.visitor.on_include(self.state, value[1]) self.visitor.on_include(self.state, svalue[1])
elif macro.startswith("#define"): elif macro.startswith("#define"):
self.visitor.on_define(self.state, value[1]) self.visitor.on_define(self.state, svalue[1])
elif macro.startswith("#pragma"): elif macro.startswith("#pragma"):
self.visitor.on_pragma(self.state, value[1]) self.visitor.on_pragma(self.state, svalue[1])
# #
# Various # Various
@ -453,7 +458,7 @@ class CxxParser:
# #
def _parse_template_type_parameter( def _parse_template_type_parameter(
self, tok: LexToken, template: TemplateDecl self, tok: LexToken, template: typing.Optional[TemplateDecl]
) -> TemplateTypeParam: ) -> TemplateTypeParam:
""" """
type_parameter: "class" ["..."] [IDENTIFIER] type_parameter: "class" ["..."] [IDENTIFIER]
@ -469,12 +474,12 @@ class CxxParser:
name = None name = None
default = None default = None
tok = self.lex.token_if("NAME") otok = self.lex.token_if("NAME")
if tok: if otok:
name = tok.value name = otok.value
tok = self.lex.token_if("=") otok = self.lex.token_if("=")
if tok: if otok:
default = self._create_value(self._consume_value_until([], ",", ">")) default = self._create_value(self._consume_value_until([], ",", ">"))
return TemplateTypeParam(typekey, name, param_pack, default, template) return TemplateTypeParam(typekey, name, param_pack, default, template)
@ -492,7 +497,7 @@ class CxxParser:
| parameter_declaration | parameter_declaration
""" """
tok = self._next_token_must_be("<") tok = self._next_token_must_be("<")
params = [] params: typing.List[TemplateParam] = []
lex = self.lex lex = self.lex
@ -502,6 +507,8 @@ class CxxParser:
tok = lex.token() tok = lex.token()
tok_type = tok.type tok_type = tok.type
param: TemplateParam
if tok_type == "template": if tok_type == "template":
template = self._parse_template_decl() template = self._parse_template_decl()
tok = self._next_token_must_be("class", "typename") tok = self._next_token_must_be("class", "typename")
@ -528,9 +535,7 @@ class CxxParser:
return TemplateDecl(params) return TemplateDecl(params)
def _parse_template( def _parse_template(self, tok: LexToken, doxygen: typing.Optional[str]):
self, tok: LexToken, doxygen: typing.Optional[str]
) -> TemplateDecl:
template = self._parse_template_decl() template = self._parse_template_decl()
@ -908,7 +913,7 @@ class CxxParser:
# might start with attributes # might start with attributes
if tok.type in self._attribute_specifier_seq_start_types: if tok.type in self._attribute_specifier_seq_start_types:
self._parse_attribute_specifier_seq(tok) self._consume_attribute_specifier_seq(tok)
tok = self.lex.token() tok = self.lex.token()
tok_type = tok.type tok_type = tok.type
@ -1155,6 +1160,8 @@ class CxxParser:
doxygen = self.lex.get_doxygen() doxygen = self.lex.get_doxygen()
if is_typedef: if is_typedef:
if not name:
raise self._parse_error(None)
typedef = Typedef(dtype, name, self._current_access) typedef = Typedef(dtype, name, self._current_access)
self.visitor.on_typedef(state, typedef) self.visitor.on_typedef(state, typedef)
else: else:
@ -1423,8 +1430,8 @@ class CxxParser:
# #
def _parse_parameter( def _parse_parameter(
self, tok: typing.Optional[LexToken], cls: typing.Type[T], end: str = ")" self, tok: typing.Optional[LexToken], cls: typing.Type[PT], end: str = ")"
) -> T: ) -> PT:
""" """
Parses a single parameter (excluding vararg parameters). Also used Parses a single parameter (excluding vararg parameters). Also used
to parse template non-type parameters to parse template non-type parameters
@ -1516,13 +1523,13 @@ class CxxParser:
if self.lex.token_if("throw"): if self.lex.token_if("throw"):
tok = self._next_token_must_be("(") tok = self._next_token_must_be("(")
fn.throw = self._create_value(self._consume_balanced_items(tok)) fn.throw = self._create_value(self._consume_balanced_tokens(tok))
elif self.lex.token_if("noexcept"): elif self.lex.token_if("noexcept"):
toks = [] toks = []
tok = self.lex.token_if("(") otok = self.lex.token_if("(")
if tok: if otok:
toks = self._consume_balanced_tokens(tok)[1:-1] toks = self._consume_balanced_tokens(otok)[1:-1]
fn.noexcept = self._create_value(toks) fn.noexcept = self._create_value(toks)
if self.lex.token_if("{"): if self.lex.token_if("{"):
@ -1575,9 +1582,9 @@ class CxxParser:
method.throw = self._create_value(self._consume_balanced_tokens(tok)) method.throw = self._create_value(self._consume_balanced_tokens(tok))
elif tok_value == "noexcept": elif tok_value == "noexcept":
toks = [] toks = []
tok = self.lex.token_if("(") otok = self.lex.token_if("(")
if tok: if otok:
toks = self._consume_balanced_tokens(tok)[1:-1] toks = self._consume_balanced_tokens(otok)[1:-1]
method.noexcept = self._create_value(toks) method.noexcept = self._create_value(toks)
else: else:
self.lex.return_token(tok) self.lex.return_token(tok)
@ -1620,6 +1627,8 @@ class CxxParser:
if is_class_block: if is_class_block:
props.update(dict.fromkeys(mods.meths.keys(), True)) props.update(dict.fromkeys(mods.meths.keys(), True))
method: Method
if op: if op:
method = Operator( method = Operator(
return_type, return_type,
@ -1684,18 +1693,18 @@ class CxxParser:
assert tok.type == "[" assert tok.type == "["
toks = self._consume_balanced_tokens(tok) toks = self._consume_balanced_tokens(tok)
tok = self.lex.token_if("[") otok = self.lex.token_if("[")
if tok: if otok:
# recurses because array types are right to left # recurses because array types are right to left
dtype = self._parse_array_type(tok, dtype) dtype = self._parse_array_type(otok, dtype)
toks = toks[1:-1] toks = toks[1:-1]
if toks: size = None
value = self._create_value(toks)
else:
value = None
return Array(dtype, value) if toks:
size = self._create_value(toks)
return Array(dtype, size)
def _parse_cv_ptr( def _parse_cv_ptr(
self, self,
@ -1805,7 +1814,7 @@ class CxxParser:
if not tok: if not tok:
tok = get_token() tok = get_token()
pqname: PQName = None pqname: typing.Optional[PQName] = None
_pqname_start_tokens = self._pqname_start_tokens _pqname_start_tokens = self._pqname_start_tokens
@ -1837,12 +1846,13 @@ class CxxParser:
elif tok_type == "volatile": elif tok_type == "volatile":
volatile = True volatile = True
else: else:
if pqname is None:
raise self._parse_error(tok)
break break
tok = get_token() tok = get_token()
if pqname is None:
raise self._parse_error(tok)
self.lex.return_token(tok) self.lex.return_token(tok)
# Construct a type from the parsed name # Construct a type from the parsed name
@ -1931,6 +1941,9 @@ class CxxParser:
if is_typedef: if is_typedef:
raise self._parse_error(None) raise self._parse_error(None)
if not pqname:
raise self._parse_error(None)
return self._parse_function( return self._parse_function(
mods, mods,
dtype, dtype,
@ -2051,7 +2064,9 @@ class CxxParser:
# enum cannot be forward declared, but "enum class" can # enum cannot be forward declared, but "enum class" can
# -> but `friend enum X` is fine # -> but `friend enum X` is fine
if classkey == "enum" and not is_friend: if not classkey:
raise self._parse_error(None)
elif classkey == "enum" and not is_friend:
raise self._parse_error(None) raise self._parse_error(None)
elif template and classkey.startswith("enum"): elif template and classkey.startswith("enum"):
# enum class cannot have a template # enum class cannot have a template

17
cxxheaderparser/parserstate.py
View File

@ -33,6 +33,9 @@ class State:
#: parent state #: parent state
parent: typing.Optional["State"] parent: typing.Optional["State"]
#: Approximate location that the parsed element was found at
location: Location
def __init__(self, parent: typing.Optional["State"]) -> None: def __init__(self, parent: typing.Optional["State"]) -> None:
self.parent = parent self.parent = parent
@ -40,18 +43,12 @@ class State:
pass pass
class BlockState(State): class EmptyBlockState(State):
#: Approximate location that the parsed element was found at
location: Location
class EmptyBlockState(BlockState):
def _finish(self, visitor: "CxxVisitor") -> None: def _finish(self, visitor: "CxxVisitor") -> None:
visitor.on_empty_block_end(self) visitor.on_empty_block_end(self)
class ExternBlockState(BlockState): class ExternBlockState(State):
#: The linkage for this extern block #: The linkage for this extern block
linkage: str linkage: str
@ -64,7 +61,7 @@ class ExternBlockState(BlockState):
visitor.on_extern_block_end(self) visitor.on_extern_block_end(self)
class NamespaceBlockState(BlockState): class NamespaceBlockState(State):
#: The incremental namespace for this block #: The incremental namespace for this block
namespace: NamespaceDecl namespace: NamespaceDecl
@ -79,7 +76,7 @@ class NamespaceBlockState(BlockState):
visitor.on_namespace_end(self) visitor.on_namespace_end(self)
class ClassBlockState(BlockState): class ClassBlockState(State):
#: class decl block being processed #: class decl block being processed
class_decl: ClassDecl class_decl: ClassDecl

2
cxxheaderparser/simple.py
View File

@ -98,7 +98,7 @@ class NamespaceScope:
forward_decls: typing.List[ForwardDecl] = field(default_factory=list) forward_decls: typing.List[ForwardDecl] = field(default_factory=list)
using: typing.List[UsingDecl] = field(default_factory=list) using: typing.List[UsingDecl] = field(default_factory=list)
using_ns: typing.List[UsingDecl] = field(default_factory=list) using_ns: typing.List["UsingNamespace"] = field(default_factory=list)
using_alias: typing.List[UsingAlias] = field(default_factory=list) using_alias: typing.List[UsingAlias] = field(default_factory=list)
#: Child namespaces #: Child namespaces

6
cxxheaderparser/tokfmt.py
View File

@ -1,6 +1,6 @@
import typing import typing
from .lexer import Lexer from .lexer import LexToken, Lexer
from .types import Token from .types import Token
# key: token type, value: (left spacing, right spacing) # key: token type, value: (left spacing, right spacing)
@ -58,7 +58,7 @@ if __name__ == "__main__":
with open(lexer.filename) as fp: with open(lexer.filename) as fp:
lexer.input(fp.read()) lexer.input(fp.read())
toks = [] toks: typing.List[Token] = []
while True: while True:
tok = lexer.token_eof_ok() tok = lexer.token_eof_ok()
if not tok: if not tok:
@ -68,7 +68,7 @@ if __name__ == "__main__":
print(tokfmt(toks)) print(tokfmt(toks))
toks = [] toks = []
else: else:
toks.append(tok) toks.append(Token(tok.value, tok.type))
print(toks) print(toks)
print(tokfmt(toks)) print(tokfmt(toks))

23
cxxheaderparser/types.py
View File

@ -95,7 +95,7 @@ class NameSpecifier:
name: str name: str
specialization: typing.Optional[typing.List["TemplateSpecialization"]] = None specialization: typing.Optional["TemplateSpecialization"] = None
@dataclass @dataclass
@ -233,16 +233,15 @@ class FunctionType:
@dataclass @dataclass
class Type: class Type:
"""""" """
A type with a name associated with it
"""
typename: PQName typename: PQName
const: bool = False const: bool = False
volatile: bool = False volatile: bool = False
def get_type(self) -> "Type":
return self
@dataclass @dataclass
class Array: class Array:
@ -262,9 +261,6 @@ class Array:
#: ~~ #: ~~
size: typing.Optional[Value] size: typing.Optional[Value]
def get_type(self) -> Type:
return self.array_of.get_type()
@dataclass @dataclass
class Pointer: class Pointer:
@ -278,9 +274,6 @@ class Pointer:
const: bool = False const: bool = False
volatile: bool = False volatile: bool = False
def get_type(self) -> Type:
return self.ptr_to.get_type()
@dataclass @dataclass
class Reference: class Reference:
@ -290,9 +283,6 @@ class Reference:
ref_to: typing.Union[Array, Pointer, Type] ref_to: typing.Union[Array, Pointer, Type]
def get_type(self) -> Type:
return self.ref_to.get_type()
@dataclass @dataclass
class MoveReference: class MoveReference:
@ -302,9 +292,6 @@ class MoveReference:
moveref_to: typing.Union[Array, Pointer, Type] moveref_to: typing.Union[Array, Pointer, Type]
def get_type(self) -> Type:
return self.moveref_to.get_type()
#: A type or function type that is decorated with various things #: A type or function type that is decorated with various things
#: #:
@ -443,7 +430,7 @@ class ClassDecl:
access: typing.Optional[str] = None access: typing.Optional[str] = None
@property @property
def classkey(self) -> str: def classkey(self) -> typing.Optional[str]:
return self.typename.classkey return self.typename.classkey

3
cxxheaderparser/visitor.py
View File

@ -2,10 +2,11 @@ import sys
import typing import typing
if sys.version_info >= (3, 8): if sys.version_info >= (3, 8):
Protocol = typing.Protocol from typing import Protocol
else: else:
Protocol = object Protocol = object
from .types import ( from .types import (
EnumDecl, EnumDecl,
Field, Field,