mewin/cxxheaderparser
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #68 from robotpy/simplify-blocks

Browse Source 
breaking change: remove empty block state and update visitor types
This commit is contained in:
Dustin Spicuzza 2023-09-28 21:01:26 -04:00 committed by GitHub
commit bdcee6f9c0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 95 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
cxxheaderparser/parser.py
View File

@ -10,9 +10,9 @@ from .lexer import LexToken, Location, PhonyEnding
from .options import ParserOptions
from .parserstate import (
ClassBlockState,
EmptyBlockState,
ExternBlockState,
NamespaceBlockState,
NonClassBlockState,
ParsedTypeModifiers,
State,
)
@ -62,7 +62,6 @@ from .visitor import CxxVisitor, null_visitor
LexTokenList = typing.List[LexToken]
T = typing.TypeVar("T")
ST = typing.TypeVar("ST", bound=State)
PT = typing.TypeVar("PT", Parameter, TemplateNonTypeParam)
@ -90,7 +89,9 @@ class CxxParser:
global_ns = NamespaceDecl([], False)
self.current_namespace = global_ns
self.state: State = NamespaceBlockState(None, global_ns)
self.state: State = NamespaceBlockState(
None, self.lex.current_location(), global_ns
)
self.anon_id = 0
self.verbose = True if self.options.verbose else False
@ -111,13 +112,9 @@ class CxxParser:
# State management
#
def _push_state(self, cls: typing.Type[ST], *args) -> ST:
state = cls(self.state, *args)
def _setup_state(self, state: State):
state._prior_visitor = self.visitor
if isinstance(state, NamespaceBlockState):
self.current_namespace = state.namespace
self.state = state
return state
def _pop_state(self) -> State:
prev_state = self.state
@ -447,24 +444,37 @@ class CxxParser:
if inline and len(names) > 1:
raise CxxParseError("a nested namespace definition cannot be inline")
state = self.state
if not isinstance(state, (NamespaceBlockState, ExternBlockState)):
raise CxxParseError("namespace cannot be defined in a class")
if ns_alias:
alias = NamespaceAlias(ns_alias.value, names)
self.visitor.on_namespace_alias(self.state, alias)
self.visitor.on_namespace_alias(state, alias)
return
ns = NamespaceDecl(names, inline, doxygen)
state = self._push_state(NamespaceBlockState, ns)
state.location = location
state = NamespaceBlockState(state, location, ns)
self._setup_state(state)
self.current_namespace = state.namespace
if self.visitor.on_namespace_start(state) is False:
self.visitor = null_visitor
def _parse_extern(self, tok: LexToken, doxygen: typing.Optional[str]) -> None:
etok = self.lex.token_if("STRING_LITERAL", "template")
if etok:
# classes cannot contain extern blocks/templates
state = self.state
if isinstance(state, ClassBlockState):
raise self._parse_error(tok)
if etok.type == "STRING_LITERAL":
if self.lex.token_if("{"):
state = self._push_state(ExternBlockState, etok.value)
state.location = tok.location
state = ExternBlockState(state, tok.location, etok.value)
self._setup_state(state)
if self.visitor.on_extern_block_start(state) is False:
self.visitor = null_visitor
return
@ -510,9 +520,7 @@ class CxxParser:
def _on_empty_block_start(
self, tok: LexToken, doxygen: typing.Optional[str]
) -> None:
state = self._push_state(EmptyBlockState)
if self.visitor.on_empty_block_start(state) is False:
self.visitor = null_visitor
raise self._parse_error(tok)
def _on_block_end(self, tok: LexToken, doxygen: typing.Optional[str]) -> None:
old_state = self._pop_state()
@ -822,7 +830,7 @@ class CxxParser:
# Using directive/declaration/typealias
#
def _parse_using_directive(self) -> None:
def _parse_using_directive(self, state: NonClassBlockState) -> None:
"""
using_directive: [attribute_specifier_seq] "using" "namespace" ["::"] [nested_name_specifier] IDENTIFIER ";"
"""
@ -841,7 +849,7 @@ class CxxParser:
if not names:
raise self._parse_error(None, "NAME")
self.visitor.on_using_namespace(self.state, names)
self.visitor.on_using_namespace(state, names)
def _parse_using_declaration(self, tok: LexToken) -> None:
"""
@ -893,10 +901,11 @@ class CxxParser:
raise CxxParseError(
"unexpected using-directive when parsing alias-declaration", tok
)
if isinstance(self.state, ClassBlockState):
state = self.state
if not isinstance(state, (NamespaceBlockState, ExternBlockState)):
raise self._parse_error(tok)
self._parse_using_directive()
self._parse_using_directive(state)
elif tok.type in ("DBL_COLON", "typename") or not self.lex.token_if("="):
if template:
raise CxxParseError(
@ -1143,10 +1152,11 @@ class CxxParser:
clsdecl = ClassDecl(
typename, bases, template, explicit, final, doxygen, self._current_access
)
state = self._push_state(
ClassBlockState, clsdecl, default_access, typedef, mods
state: ClassBlockState = ClassBlockState(
self.state, location, clsdecl, default_access, typedef, mods
)
state.location = location
self._setup_state(state)
if self.visitor.on_class_start(state) is False:
self.visitor = null_visitor
@ -1853,6 +1863,7 @@ class CxxParser:
self.visitor.on_class_method(state, method)
else:
assert isinstance(state, (ExternBlockState, NamespaceBlockState))
if not method.has_body:
raise self._parse_error(None, expected="Method body")
self.visitor.on_method_impl(state, method)
@ -1912,6 +1923,9 @@ class CxxParser:
self.visitor.on_typedef(state, typedef)
return False
else:
if not isinstance(state, (ExternBlockState, NamespaceBlockState)):
raise CxxParseError("internal error")
self.visitor.on_function(state, fn)
return fn.has_body or fn.has_trailing_return

50
cxxheaderparser/parserstate.py
View File

@ -35,13 +35,13 @@ T = typing.TypeVar("T")
PT = typing.TypeVar("PT")
class State(typing.Generic[T, PT]):
class BaseState(typing.Generic[T, PT]):
#: Uninitialized user data available for use by visitor implementations. You
#: should set this in a ``*_start`` method.
user_data: T
#: parent state
parent: typing.Optional["State[PT, typing.Any]"]
parent: typing.Optional["State"]
#: Approximate location that the parsed element was found at
location: Location
@ -49,52 +49,51 @@ class State(typing.Generic[T, PT]):
#: internal detail used by parser
_prior_visitor: "CxxVisitor"
def __init__(self, parent: typing.Optional["State[PT, typing.Any]"]) -> None:
def __init__(self, parent: typing.Optional["State"], location: Location) -> None:
self.parent = parent
self.location = location
def _finish(self, visitor: "CxxVisitor") -> None:
pass
class EmptyBlockState(State[T, PT]):
parent: State[PT, typing.Any]
def _finish(self, visitor: "CxxVisitor") -> None:
visitor.on_empty_block_end(self)
class ExternBlockState(State[T, PT]):
parent: State[PT, typing.Any]
class ExternBlockState(BaseState[T, PT]):
parent: "NonClassBlockState"
#: The linkage for this extern block
linkage: str
def __init__(self, parent: typing.Optional[State], linkage: str) -> None:
super().__init__(parent)
def __init__(
self, parent: "NonClassBlockState", location: Location, linkage: str
) -> None:
super().__init__(parent, location)
self.linkage = linkage
def _finish(self, visitor: "CxxVisitor") -> None:
visitor.on_extern_block_end(self)
class NamespaceBlockState(State[T, PT]):
parent: State[PT, typing.Any]
class NamespaceBlockState(BaseState[T, PT]):
parent: "NonClassBlockState"
#: The incremental namespace for this block
namespace: NamespaceDecl
def __init__(
self, parent: typing.Optional[State], namespace: NamespaceDecl
self,
parent: typing.Optional["NonClassBlockState"],
location: Location,
namespace: NamespaceDecl,
) -> None:
super().__init__(parent)
super().__init__(parent, location)
self.namespace = namespace
def _finish(self, visitor: "CxxVisitor") -> None:
visitor.on_namespace_end(self)
class ClassBlockState(State[T, PT]):
parent: State[PT, typing.Any]
class ClassBlockState(BaseState[T, PT]):
parent: "State"
#: class decl block being processed
class_decl: ClassDecl
@ -110,13 +109,14 @@ class ClassBlockState(State[T, PT]):
def __init__(
self,
parent: typing.Optional[State],
parent: typing.Optional["State"],
location: Location,
class_decl: ClassDecl,
access: str,
typedef: bool,
mods: ParsedTypeModifiers,
) -> None:
super().__init__(parent)
super().__init__(parent, location)
self.class_decl = class_decl
self.access = access
self.typedef = typedef
@ -127,3 +127,9 @@ class ClassBlockState(State[T, PT]):
def _finish(self, visitor: "CxxVisitor") -> None:
visitor.on_class_end(self)
State = typing.Union[
NamespaceBlockState[T, PT], ExternBlockState[T, PT], ClassBlockState[T, PT]
]
NonClassBlockState = typing.Union[ExternBlockState[T, PT], NamespaceBlockState[T, PT]]

36
cxxheaderparser/simple.py
View File

@ -51,7 +51,6 @@ from .types import (
from .parserstate import (
State,
EmptyBlockState,
ClassBlockState,
ExternBlockState,
NamespaceBlockState,
@ -180,11 +179,12 @@ class ParsedData:
#
# define what user data we store in each state type
SState = State[Block, Block]
SEmptyBlockState = EmptyBlockState[Block, Block]
SExternBlockState = ExternBlockState[Block, Block]
SNamespaceBlockState = NamespaceBlockState[NamespaceScope, NamespaceScope]
SClassBlockState = ClassBlockState[ClassScope, Block]
SExternBlockState = ExternBlockState[NamespaceScope, NamespaceScope]
SNamespaceBlockState = NamespaceBlockState[NamespaceScope, NamespaceScope]
SState = typing.Union[SClassBlockState, SExternBlockState, SNamespaceBlockState]
SNonClassBlockState = typing.Union[SExternBlockState, SNamespaceBlockState]
class SimpleCxxVisitor:
@ -209,16 +209,6 @@ class SimpleCxxVisitor:
def on_include(self, state: SState, filename: str) -> None:
self.data.includes.append(Include(filename))
def on_empty_block_start(self, state: SEmptyBlockState) -> typing.Optional[bool]:
# this matters for some scope/resolving purposes, but you're
# probably going to want to use clang if you care about that
# level of detail
state.user_data = state.parent.user_data
return None
def on_empty_block_end(self, state: SEmptyBlockState) -> None:
pass
def on_extern_block_start(self, state: SExternBlockState) -> typing.Optional[bool]:
state.user_data = state.parent.user_data
return None
@ -254,8 +244,9 @@ class SimpleCxxVisitor:
def on_namespace_end(self, state: SNamespaceBlockState) -> None:
pass
def on_namespace_alias(self, state: SState, alias: NamespaceAlias) -> None:
assert isinstance(state.user_data, NamespaceScope)
def on_namespace_alias(
self, state: SNonClassBlockState, alias: NamespaceAlias
) -> None:
state.user_data.ns_alias.append(alias)
def on_forward_decl(self, state: SState, fdecl: ForwardDecl) -> None:
@ -269,19 +260,18 @@ class SimpleCxxVisitor:
assert isinstance(state.user_data, NamespaceScope)
state.user_data.variables.append(v)
def on_function(self, state: SState, fn: Function) -> None:
assert isinstance(state.user_data, NamespaceScope)
def on_function(self, state: SNonClassBlockState, fn: Function) -> None:
state.user_data.functions.append(fn)
def on_method_impl(self, state: SState, method: Method) -> None:
assert isinstance(state.user_data, NamespaceScope)
def on_method_impl(self, state: SNonClassBlockState, method: Method) -> None:
state.user_data.method_impls.append(method)
def on_typedef(self, state: SState, typedef: Typedef) -> None:
state.user_data.typedefs.append(typedef)
def on_using_namespace(self, state: SState, namespace: typing.List[str]) -> None:
assert isinstance(state.user_data, NamespaceScope)
def on_using_namespace(
self, state: SNonClassBlockState, namespace: typing.List[str]
) -> None:
ns = UsingNamespace("::".join(namespace))
state.user_data.using_ns.append(ns)

52
cxxheaderparser/visitor.py
View File

@ -25,10 +25,10 @@ from .types import (
from .parserstate import (
State,
EmptyBlockState,
ClassBlockState,
ExternBlockState,
NamespaceBlockState,
NonClassBlockState,
)
@ -52,26 +52,6 @@ class CxxVisitor(Protocol):
Called once for each ``#include`` directive encountered
"""
def on_empty_block_start(self, state: EmptyBlockState) -> typing.Optional[bool]:
"""
Called when a ``{`` is encountered that isn't associated with or
consumed by other declarations.
.. code-block:: c++
{
// stuff
}
If this function returns False, the visitor will not be called for any
items inside this block (including on_empty_block_end)
"""
def on_empty_block_end(self, state: EmptyBlockState) -> None:
"""
Called when an empty block ends
"""
def on_extern_block_start(self, state: ExternBlockState) -> typing.Optional[bool]:
"""
.. code-block:: c++
@ -102,7 +82,9 @@ class CxxVisitor(Protocol):
Called at the end of a ``namespace`` block
"""
def on_namespace_alias(self, state: State, alias: NamespaceAlias) -> None:
def on_namespace_alias(
self, state: NonClassBlockState, alias: NamespaceAlias
) -> None:
"""
Called when a ``namespace`` alias is encountered
"""
@ -122,12 +104,12 @@ class CxxVisitor(Protocol):
Called when a global variable is encountered
"""
def on_function(self, state: State, fn: Function) -> None:
def on_function(self, state: NonClassBlockState, fn: Function) -> None:
"""
Called when a function is encountered that isn't part of a class
"""
def on_method_impl(self, state: State, method: Method) -> None:
def on_method_impl(self, state: NonClassBlockState, method: Method) -> None:
"""
Called when a method implementation is encountered outside of a class
declaration. For example:
@ -155,7 +137,9 @@ class CxxVisitor(Protocol):
once for ``*PT``
"""
def on_using_namespace(self, state: State, namespace: typing.List[str]) -> None:
def on_using_namespace(
self, state: NonClassBlockState, namespace: typing.List[str]
) -> None:
"""
.. code-block:: c++
@ -258,12 +242,6 @@ class NullVisitor:
def on_include(self, state: State, filename: str) -> None:
return None
def on_empty_block_start(self, state: EmptyBlockState) -> typing.Optional[bool]:
return None
def on_empty_block_end(self, state: EmptyBlockState) -> None:
return None
def on_extern_block_start(self, state: ExternBlockState) -> typing.Optional[bool]:
return None
@ -276,7 +254,9 @@ class NullVisitor:
def on_namespace_end(self, state: NamespaceBlockState) -> None:
return None
def on_namespace_alias(self, state: State, alias: NamespaceAlias) -> None:
def on_namespace_alias(
self, state: NonClassBlockState, alias: NamespaceAlias
) -> None:
return None
def on_forward_decl(self, state: State, fdecl: ForwardDecl) -> None:
@ -288,16 +268,18 @@ class NullVisitor:
def on_variable(self, state: State, v: Variable) -> None:
return None
def on_function(self, state: State, fn: Function) -> None:
def on_function(self, state: NonClassBlockState, fn: Function) -> None:
return None
def on_method_impl(self, state: State, method: Method) -> None:
def on_method_impl(self, state: NonClassBlockState, method: Method) -> None:
return None
def on_typedef(self, state: State, typedef: Typedef) -> None:
return None
def on_using_namespace(self, state: State, namespace: typing.List[str]) -> None:
def on_using_namespace(
self, state: NonClassBlockState, namespace: typing.List[str]
) -> None:
return None
def on_using_alias(self, state: State, using: UsingAlias) -> None:

26
tests/test_misc.py
View File

@ -351,29 +351,3 @@ def test_warning_directive() -> None:
data = parse_string(content, cleandoc=True)
assert data == ParsedData()
def test_empty_block() -> None:
"""
Ensure the simple visitor doesn't break with an empty block
"""
content = """
{
class X {};
}
"""
data = parse_string(content, cleandoc=True)
assert data == ParsedData(
namespace=NamespaceScope(
classes=[
ClassScope(
class_decl=ClassDecl(
typename=PQName(
segments=[NameSpecifier(name="X")], classkey="class"
)
)
)
]
)
)

48
tests/test_skip.py
View File

@ -10,7 +10,6 @@ from cxxheaderparser.simple import (
NamespaceScope,
ParsedData,
SClassBlockState,
SEmptyBlockState,
SExternBlockState,
SNamespaceBlockState,
SimpleCxxVisitor,
@ -148,53 +147,6 @@ def test_skip_class() -> None:
)
#
# ensure empty block is skipped
#
class SkipEmptyBlock(SimpleCxxVisitor):
def on_empty_block_start(self, state: SEmptyBlockState) -> typing.Optional[bool]:
return False
def test_skip_empty_block() -> None:
content = """
void fn1();
{
void fn2();
}
void fn3();
"""
v = SkipEmptyBlock()
parser = CxxParser("<str>", inspect.cleandoc(content), v)
parser.parse()
data = v.data
assert data == ParsedData(
namespace=NamespaceScope(
functions=[
Function(
return_type=Type(
typename=PQName(segments=[FundamentalSpecifier(name="void")])
),
name=PQName(segments=[NameSpecifier(name="fn1")]),
parameters=[],
),
Function(
return_type=Type(
typename=PQName(segments=[FundamentalSpecifier(name="void")])
),
name=PQName(segments=[NameSpecifier(name="fn3")]),
parameters=[],
),
]
)
)
#
# ensure namespace 'skip' is skipped
#