commit ef5c22972bc8bb16b6295a34eaa865555a26b6e7 Author: Dustin Spicuzza Date: Mon Dec 28 03:35:30 2020 -0500 Initial commit diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml new file mode 100644 index 0000000..e1d2b82 --- /dev/null +++ b/.github/workflows/dist.yml @@ -0,0 +1,94 @@ +--- +name: dist + +on: [push, pull_request] + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: psf/black@stable + + # check-doc: + # runs-on: ubuntu-18.04 + + # steps: + # - uses: actions/checkout@v2 + # with: + # submodules: recursive + # fetch-depth: 0 + + # - uses: actions/setup-python@v2 + # with: + # python-version: 3.8 + # - name: Sphinx + # run: | + # pip --disable-pip-version-check install -e . + # pip --disable-pip-version-check install -r docs/requirements.txt + # cd docs && make clean html SPHINXOPTS="-W --keep-going" + + test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [windows-latest, macos-latest, ubuntu-18.04] + python_version: [3.6, 3.7, 3.8, 3.9] + architecture: [x86, x64] + exclude: + - os: macos-latest + architecture: x86 + - os: ubuntu-18.04 + architecture: x86 + + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + fetch-depth: 0 + + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python_version }} + architecture: ${{ matrix.architecture }} + + - name: Install build dependencies + run: python -m pip --disable-pip-version-check install wheel + + - name: Build wheel + run: python setup.py bdist_wheel + + - name: Install test dependencies + run: python -m pip --disable-pip-version-check install pytest + + - name: Test wheel + shell: bash + run: | + cd dist + python -m pip --disable-pip-version-check install *.whl + cd ../tests + python -m pytest + + publish: + runs-on: ubuntu-latest + needs: [check, check-doc, test] + if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') + + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + fetch-depth: 0 + + - uses: actions/setup-python@v2 + with: + python-version: 3.8 + - run: pip --disable-pip-version-check install wheel + + - name: Build packages + run: python setup.py sdist bdist_wheel + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@master + with: + user: __token__ + password: ${{ secrets.PYPI_PASSWORD }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c2ae541 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +*.py[cod] +*.egg-info +/build +/dist +/cxxheaderparser/version.py +/.vscode + +.coverage +.pytest_cache \ No newline at end of file diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..1d8f05b --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,104 @@ +cxxheaderparser license: + +Copyright (c) 2020 Dustin Spicuzza +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +----------------------------------------------------------------------------- + +CppHeaderParser license: + +Copyright (C) 2011, Jashua R. Cloutier +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +* Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + +* Neither the name of Jashua R. Cloutier nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. Stories, + blog entries etc making reference to this project may mention the + name Jashua R. Cloutier in terms of project originator/creator etc. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +----------------------------------------------------------------------------- + +PLY license: + +Copyright (C) 2001-2020 +David M. Beazley (Dabeaz LLC) +All rights reserved. + +Latest version: https://github.com/dabeaz/ply + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +* Neither the name of David Beazley or Dabeaz LLC may be used to + endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +----------------------------------------------------------------------------- diff --git a/README.md b/README.md new file mode 100644 index 0000000..64bb1bb --- /dev/null +++ b/README.md @@ -0,0 +1,108 @@ +cxxheaderparser +=============== + +**Note**: This is still a work in progress, but should be stable in a few weeks +once I port robotpy-build over to use it + +A pure python C++ header parser that parses C++ headers in a mildly naive +manner that allows it to handle many C++ constructs, including many modern +(C++11 and beyond) features. + +This is a complete rewrite of the `CppHeaderParser` library. `CppHeaderParser` +is really useful for some tasks, but it's implementation is a truly terrible +ugly hack built on top of other terrible hacks. This rewrite tries to learn +from `CppHeaderParser` and leave its ugly baggage behind. + +Goals: + +* Parse syntatically valid C++ and provide a useful (and documented!) pure + python API to work with the parsed data +* Process incomplete headers (doesn't need to process includes) +* Provide enough information for binding generators to wrap C++ code +* Handle common C++ features, but it may struggle with obscure or overly + complex things (feel free to make a PR to fix it!) + +Non-goals: + +* **Does not produce a full AST**, use Clang if you need that +* **Not intended to validate C++**, which means this will not reject all + invalid C++ headers! Use a compiler if you need that +* **No C preprocessor substitution support implemented**. If you are parsing + headers that contain macros, you should preprocess your code using the + excellent pure python preprocessor [pcpp](https://github.com/ned14/pcpp) + or your favorite compiler +* Probably won't be able to parse most IOCCC entries + +There are two APIs available: + +* A visitor-style interface to build up your own custom data structures +* A simple visitor that stores everything in a giant data structure + +Documentation +------------- + +TODO: documentation site + +Install +------- + +Requires Python 3.6+, no non-stdlib dependencies if using Python 3.7+. + +TODO: distribute on pip + +Usage +----- + +To see a dump of the data parsed from a header: + +``` +# pprint format +python -m cxxheaderparser myheader.h + +# JSON format +python -m cxxheaderparser --mode=json myheader.h + +# dataclasses repr format +python -m cxxheaderparser --mode=repr myheader.h + +# dataclasses repr format (formatted with black) +python -m cxxheaderparser --mode=brepr myheader.h +``` + +See the documentation for anything more complex. + +Bugs +---- + +This should handle even complex C++ code with few problems, but there are +almost certainly weird edge cases that it doesn't handle. Additionally, +not all C++17/20 constructs are supported yet (but contributions welcome!). + +If you find an bug, we encourage you to submit a pull request! New +changes will only be accepted if there are tests to cover the change you +made (and if they don’t break existing tests). + +Author +------ + +cxxheaderparser was created by Dustin Spicuzza + +Credit +------ + +* Partially derived from and inspired by the `CppHeaderParser` project + originally developed by Jashua Cloutier +* An embedded version of PLY is used for lexing tokens +* Portions of the lexer grammar and other ideas were derived from pycparser +* The source code is liberally sprinkled with comments containing C++ parsing + grammar mostly derived from the [Hyperlinked C++ BNF Grammar](https://www.nongnu.org/hcb/) +* cppreference.com has been invaluable for understanding many of the weird + quirks of C++, and some of the unit tests use examples from there +* [Compiler Explorer](godbolt.org) has been invaluable for validating my + understanding of C++ by allowing me to quickly type in quirky C++ + constructs to see if they actually compile + +License +------- + +BSD License \ No newline at end of file diff --git a/cxxheaderparser/__init__.py b/cxxheaderparser/__init__.py new file mode 100644 index 0000000..7deded2 --- /dev/null +++ b/cxxheaderparser/__init__.py @@ -0,0 +1,4 @@ +try: + from .version import __version__ +except ImportError: + __version__ = "master" \ No newline at end of file diff --git a/cxxheaderparser/__main__.py b/cxxheaderparser/__main__.py new file mode 100644 index 0000000..cc68d38 --- /dev/null +++ b/cxxheaderparser/__main__.py @@ -0,0 +1,4 @@ +from cxxheaderparser.dump import dumpmain + +if __name__ == "__main__": + dumpmain() diff --git a/cxxheaderparser/_ply/__init__.py b/cxxheaderparser/_ply/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cxxheaderparser/_ply/lex.py b/cxxheaderparser/_ply/lex.py new file mode 100644 index 0000000..37a29b6 --- /dev/null +++ b/cxxheaderparser/_ply/lex.py @@ -0,0 +1,902 @@ +# fmt: off +# ----------------------------------------------------------------------------- +# ply: lex.py +# +# Copyright (C) 2001-2020 +# David M. Beazley (Dabeaz LLC) +# All rights reserved. +# +# Latest version: https://github.com/dabeaz/ply +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of David Beazley or Dabeaz LLC may be used to +# endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# ----------------------------------------------------------------------------- + +import re +import sys +import types +import copy +import os +import inspect + +# This tuple contains acceptable string types +StringTypes = (str, bytes) + +# This regular expression is used to match valid token names +_is_identifier = re.compile(r'^[a-zA-Z0-9_]+$') + +# Exception thrown when invalid token encountered and no default error +# handler is defined. +class LexError(Exception): + def __init__(self, message, s): + self.args = (message,) + self.text = s + +# Token class. This class is used to represent the tokens produced. +class LexToken(object): + def __repr__(self): + return f'LexToken({self.type},{self.value!r},{self.lineno},{self.lexpos})' + +# This object is a stand-in for a logging object created by the +# logging module. + +class PlyLogger(object): + def __init__(self, f): + self.f = f + + def critical(self, msg, *args, **kwargs): + self.f.write((msg % args) + '\n') + + def warning(self, msg, *args, **kwargs): + self.f.write('WARNING: ' + (msg % args) + '\n') + + def error(self, msg, *args, **kwargs): + self.f.write('ERROR: ' + (msg % args) + '\n') + + info = critical + debug = critical + +# ----------------------------------------------------------------------------- +# === Lexing Engine === +# +# The following Lexer class implements the lexer runtime. There are only +# a few public methods and attributes: +# +# input() - Store a new string in the lexer +# token() - Get the next token +# clone() - Clone the lexer +# +# lineno - Current line number +# lexpos - Current position in the input string +# ----------------------------------------------------------------------------- + +class Lexer: + def __init__(self): + self.lexre = None # Master regular expression. This is a list of + # tuples (re, findex) where re is a compiled + # regular expression and findex is a list + # mapping regex group numbers to rules + self.lexretext = None # Current regular expression strings + self.lexstatere = {} # Dictionary mapping lexer states to master regexs + self.lexstateretext = {} # Dictionary mapping lexer states to regex strings + self.lexstaterenames = {} # Dictionary mapping lexer states to symbol names + self.lexstate = 'INITIAL' # Current lexer state + self.lexstatestack = [] # Stack of lexer states + self.lexstateinfo = None # State information + self.lexstateignore = {} # Dictionary of ignored characters for each state + self.lexstateerrorf = {} # Dictionary of error functions for each state + self.lexstateeoff = {} # Dictionary of eof functions for each state + self.lexreflags = 0 # Optional re compile flags + self.lexdata = None # Actual input data (as a string) + self.lexpos = 0 # Current position in input text + self.lexlen = 0 # Length of the input text + self.lexerrorf = None # Error rule (if any) + self.lexeoff = None # EOF rule (if any) + self.lextokens = None # List of valid tokens + self.lexignore = '' # Ignored characters + self.lexliterals = '' # Literal characters that can be passed through + self.lexmodule = None # Module + self.lineno = 1 # Current line number + + def clone(self, object=None): + c = copy.copy(self) + + # If the object parameter has been supplied, it means we are attaching the + # lexer to a new object. In this case, we have to rebind all methods in + # the lexstatere and lexstateerrorf tables. + + if object: + newtab = {} + for key, ritem in self.lexstatere.items(): + newre = [] + for cre, findex in ritem: + newfindex = [] + for f in findex: + if not f or not f[0]: + newfindex.append(f) + continue + newfindex.append((getattr(object, f[0].__name__), f[1])) + newre.append((cre, newfindex)) + newtab[key] = newre + c.lexstatere = newtab + c.lexstateerrorf = {} + for key, ef in self.lexstateerrorf.items(): + c.lexstateerrorf[key] = getattr(object, ef.__name__) + c.lexmodule = object + return c + + # ------------------------------------------------------------ + # input() - Push a new string into the lexer + # ------------------------------------------------------------ + def input(self, s): + self.lexdata = s + self.lexpos = 0 + self.lexlen = len(s) + + # ------------------------------------------------------------ + # begin() - Changes the lexing state + # ------------------------------------------------------------ + def begin(self, state): + if state not in self.lexstatere: + raise ValueError(f'Undefined state {state!r}') + self.lexre = self.lexstatere[state] + self.lexretext = self.lexstateretext[state] + self.lexignore = self.lexstateignore.get(state, '') + self.lexerrorf = self.lexstateerrorf.get(state, None) + self.lexeoff = self.lexstateeoff.get(state, None) + self.lexstate = state + + # ------------------------------------------------------------ + # push_state() - Changes the lexing state and saves old on stack + # ------------------------------------------------------------ + def push_state(self, state): + self.lexstatestack.append(self.lexstate) + self.begin(state) + + # ------------------------------------------------------------ + # pop_state() - Restores the previous state + # ------------------------------------------------------------ + def pop_state(self): + self.begin(self.lexstatestack.pop()) + + # ------------------------------------------------------------ + # current_state() - Returns the current lexing state + # ------------------------------------------------------------ + def current_state(self): + return self.lexstate + + # ------------------------------------------------------------ + # skip() - Skip ahead n characters + # ------------------------------------------------------------ + def skip(self, n): + self.lexpos += n + + # ------------------------------------------------------------ + # token() - Return the next token from the Lexer + # + # Note: This function has been carefully implemented to be as fast + # as possible. Don't make changes unless you really know what + # you are doing + # ------------------------------------------------------------ + def token(self): + # Make local copies of frequently referenced attributes + lexpos = self.lexpos + lexlen = self.lexlen + lexignore = self.lexignore + lexdata = self.lexdata + + while lexpos < lexlen: + # This code provides some short-circuit code for whitespace, tabs, and other ignored characters + if lexdata[lexpos] in lexignore: + lexpos += 1 + continue + + # Look for a regular expression match + for lexre, lexindexfunc in self.lexre: + m = lexre.match(lexdata, lexpos) + if not m: + continue + + # Create a token for return + tok = LexToken() + tok.value = m.group() + tok.lineno = self.lineno + tok.lexpos = lexpos + + i = m.lastindex + func, tok.type = lexindexfunc[i] + + if not func: + # If no token type was set, it's an ignored token + if tok.type: + self.lexpos = m.end() + return tok + else: + lexpos = m.end() + break + + lexpos = m.end() + + # If token is processed by a function, call it + + tok.lexer = self # Set additional attributes useful in token rules + self.lexmatch = m + self.lexpos = lexpos + newtok = func(tok) + del tok.lexer + del self.lexmatch + + # Every function must return a token, if nothing, we just move to next token + if not newtok: + lexpos = self.lexpos # This is here in case user has updated lexpos. + lexignore = self.lexignore # This is here in case there was a state change + break + return newtok + else: + # No match, see if in literals + if lexdata[lexpos] in self.lexliterals: + tok = LexToken() + tok.value = lexdata[lexpos] + tok.lineno = self.lineno + tok.type = tok.value + tok.lexpos = lexpos + self.lexpos = lexpos + 1 + return tok + + # No match. Call t_error() if defined. + if self.lexerrorf: + tok = LexToken() + tok.value = self.lexdata[lexpos:] + tok.lineno = self.lineno + tok.type = 'error' + tok.lexer = self + tok.lexpos = lexpos + self.lexpos = lexpos + newtok = self.lexerrorf(tok) + if lexpos == self.lexpos: + # Error method didn't change text position at all. This is an error. + raise LexError(f"Scanning error. Illegal character {lexdata[lexpos]!r}", + lexdata[lexpos:]) + lexpos = self.lexpos + if not newtok: + continue + return newtok + + self.lexpos = lexpos + raise LexError(f"Illegal character {lexdata[lexpos]!r} at index {lexpos}", + lexdata[lexpos:]) + + if self.lexeoff: + tok = LexToken() + tok.type = 'eof' + tok.value = '' + tok.lineno = self.lineno + tok.lexpos = lexpos + tok.lexer = self + self.lexpos = lexpos + newtok = self.lexeoff(tok) + return newtok + + self.lexpos = lexpos + 1 + if self.lexdata is None: + raise RuntimeError('No input string given with input()') + return None + + # Iterator interface + def __iter__(self): + return self + + def __next__(self): + t = self.token() + if t is None: + raise StopIteration + return t + +# ----------------------------------------------------------------------------- +# ==== Lex Builder === +# +# The functions and classes below are used to collect lexing information +# and build a Lexer object from it. +# ----------------------------------------------------------------------------- + +# ----------------------------------------------------------------------------- +# _get_regex(func) +# +# Returns the regular expression assigned to a function either as a doc string +# or as a .regex attribute attached by the @TOKEN decorator. +# ----------------------------------------------------------------------------- +def _get_regex(func): + return getattr(func, 'regex', func.__doc__) + +# ----------------------------------------------------------------------------- +# get_caller_module_dict() +# +# This function returns a dictionary containing all of the symbols defined within +# a caller further down the call stack. This is used to get the environment +# associated with the yacc() call if none was provided. +# ----------------------------------------------------------------------------- +def get_caller_module_dict(levels): + f = sys._getframe(levels) + return { **f.f_globals, **f.f_locals } + +# ----------------------------------------------------------------------------- +# _form_master_re() +# +# This function takes a list of all of the regex components and attempts to +# form the master regular expression. Given limitations in the Python re +# module, it may be necessary to break the master regex into separate expressions. +# ----------------------------------------------------------------------------- +def _form_master_re(relist, reflags, ldict, toknames): + if not relist: + return [], [], [] + regex = '|'.join(relist) + try: + lexre = re.compile(regex, reflags) + + # Build the index to function map for the matching engine + lexindexfunc = [None] * (max(lexre.groupindex.values()) + 1) + lexindexnames = lexindexfunc[:] + + for f, i in lexre.groupindex.items(): + handle = ldict.get(f, None) + if type(handle) in (types.FunctionType, types.MethodType): + lexindexfunc[i] = (handle, toknames[f]) + lexindexnames[i] = f + elif handle is not None: + lexindexnames[i] = f + if f.find('ignore_') > 0: + lexindexfunc[i] = (None, None) + else: + lexindexfunc[i] = (None, toknames[f]) + + return [(lexre, lexindexfunc)], [regex], [lexindexnames] + except Exception: + m = (len(relist) // 2) + 1 + llist, lre, lnames = _form_master_re(relist[:m], reflags, ldict, toknames) + rlist, rre, rnames = _form_master_re(relist[m:], reflags, ldict, toknames) + return (llist+rlist), (lre+rre), (lnames+rnames) + +# ----------------------------------------------------------------------------- +# def _statetoken(s,names) +# +# Given a declaration name s of the form "t_" and a dictionary whose keys are +# state names, this function returns a tuple (states,tokenname) where states +# is a tuple of state names and tokenname is the name of the token. For example, +# calling this with s = "t_foo_bar_SPAM" might return (('foo','bar'),'SPAM') +# ----------------------------------------------------------------------------- +def _statetoken(s, names): + parts = s.split('_') + for i, part in enumerate(parts[1:], 1): + if part not in names and part != 'ANY': + break + + if i > 1: + states = tuple(parts[1:i]) + else: + states = ('INITIAL',) + + if 'ANY' in states: + states = tuple(names) + + tokenname = '_'.join(parts[i:]) + return (states, tokenname) + + +# ----------------------------------------------------------------------------- +# LexerReflect() +# +# This class represents information needed to build a lexer as extracted from a +# user's input file. +# ----------------------------------------------------------------------------- +class LexerReflect(object): + def __init__(self, ldict, log=None, reflags=0): + self.ldict = ldict + self.error_func = None + self.tokens = [] + self.reflags = reflags + self.stateinfo = {'INITIAL': 'inclusive'} + self.modules = set() + self.error = False + self.log = PlyLogger(sys.stderr) if log is None else log + + # Get all of the basic information + def get_all(self): + self.get_tokens() + self.get_literals() + self.get_states() + self.get_rules() + + # Validate all of the information + def validate_all(self): + self.validate_tokens() + self.validate_literals() + self.validate_rules() + return self.error + + # Get the tokens map + def get_tokens(self): + tokens = self.ldict.get('tokens', None) + if not tokens: + self.log.error('No token list is defined') + self.error = True + return + + if not isinstance(tokens, (list, tuple)): + self.log.error('tokens must be a list or tuple') + self.error = True + return + + if not tokens: + self.log.error('tokens is empty') + self.error = True + return + + self.tokens = tokens + + # Validate the tokens + def validate_tokens(self): + terminals = {} + for n in self.tokens: + if not _is_identifier.match(n): + self.log.error(f"Bad token name {n!r}") + self.error = True + if n in terminals: + self.log.warning(f"Token {n!r} multiply defined") + terminals[n] = 1 + + # Get the literals specifier + def get_literals(self): + self.literals = self.ldict.get('literals', '') + if not self.literals: + self.literals = '' + + # Validate literals + def validate_literals(self): + try: + for c in self.literals: + if not isinstance(c, StringTypes) or len(c) > 1: + self.log.error(f'Invalid literal {c!r}. Must be a single character') + self.error = True + + except TypeError: + self.log.error('Invalid literals specification. literals must be a sequence of characters') + self.error = True + + def get_states(self): + self.states = self.ldict.get('states', None) + # Build statemap + if self.states: + if not isinstance(self.states, (tuple, list)): + self.log.error('states must be defined as a tuple or list') + self.error = True + else: + for s in self.states: + if not isinstance(s, tuple) or len(s) != 2: + self.log.error("Invalid state specifier %r. Must be a tuple (statename,'exclusive|inclusive')", s) + self.error = True + continue + name, statetype = s + if not isinstance(name, StringTypes): + self.log.error('State name %r must be a string', name) + self.error = True + continue + if not (statetype == 'inclusive' or statetype == 'exclusive'): + self.log.error("State type for state %r must be 'inclusive' or 'exclusive'", name) + self.error = True + continue + if name in self.stateinfo: + self.log.error("State %r already defined", name) + self.error = True + continue + self.stateinfo[name] = statetype + + # Get all of the symbols with a t_ prefix and sort them into various + # categories (functions, strings, error functions, and ignore characters) + + def get_rules(self): + tsymbols = [f for f in self.ldict if f[:2] == 't_'] + + # Now build up a list of functions and a list of strings + self.toknames = {} # Mapping of symbols to token names + self.funcsym = {} # Symbols defined as functions + self.strsym = {} # Symbols defined as strings + self.ignore = {} # Ignore strings by state + self.errorf = {} # Error functions by state + self.eoff = {} # EOF functions by state + + for s in self.stateinfo: + self.funcsym[s] = [] + self.strsym[s] = [] + + if len(tsymbols) == 0: + self.log.error('No rules of the form t_rulename are defined') + self.error = True + return + + for f in tsymbols: + t = self.ldict[f] + states, tokname = _statetoken(f, self.stateinfo) + self.toknames[f] = tokname + + if hasattr(t, '__call__'): + if tokname == 'error': + for s in states: + self.errorf[s] = t + elif tokname == 'eof': + for s in states: + self.eoff[s] = t + elif tokname == 'ignore': + line = t.__code__.co_firstlineno + file = t.__code__.co_filename + self.log.error("%s:%d: Rule %r must be defined as a string", file, line, t.__name__) + self.error = True + else: + for s in states: + self.funcsym[s].append((f, t)) + elif isinstance(t, StringTypes): + if tokname == 'ignore': + for s in states: + self.ignore[s] = t + if '\\' in t: + self.log.warning("%s contains a literal backslash '\\'", f) + + elif tokname == 'error': + self.log.error("Rule %r must be defined as a function", f) + self.error = True + else: + for s in states: + self.strsym[s].append((f, t)) + else: + self.log.error('%s not defined as a function or string', f) + self.error = True + + # Sort the functions by line number + for f in self.funcsym.values(): + f.sort(key=lambda x: x[1].__code__.co_firstlineno) + + # Sort the strings by regular expression length + for s in self.strsym.values(): + s.sort(key=lambda x: len(x[1]), reverse=True) + + # Validate all of the t_rules collected + def validate_rules(self): + for state in self.stateinfo: + # Validate all rules defined by functions + + for fname, f in self.funcsym[state]: + line = f.__code__.co_firstlineno + file = f.__code__.co_filename + module = inspect.getmodule(f) + self.modules.add(module) + + tokname = self.toknames[fname] + if isinstance(f, types.MethodType): + reqargs = 2 + else: + reqargs = 1 + nargs = f.__code__.co_argcount + if nargs > reqargs: + self.log.error("%s:%d: Rule %r has too many arguments", file, line, f.__name__) + self.error = True + continue + + if nargs < reqargs: + self.log.error("%s:%d: Rule %r requires an argument", file, line, f.__name__) + self.error = True + continue + + if not _get_regex(f): + self.log.error("%s:%d: No regular expression defined for rule %r", file, line, f.__name__) + self.error = True + continue + + try: + c = re.compile('(?P<%s>%s)' % (fname, _get_regex(f)), self.reflags) + if c.match(''): + self.log.error("%s:%d: Regular expression for rule %r matches empty string", file, line, f.__name__) + self.error = True + except re.error as e: + self.log.error("%s:%d: Invalid regular expression for rule '%s'. %s", file, line, f.__name__, e) + if '#' in _get_regex(f): + self.log.error("%s:%d. Make sure '#' in rule %r is escaped with '\\#'", file, line, f.__name__) + self.error = True + + # Validate all rules defined by strings + for name, r in self.strsym[state]: + tokname = self.toknames[name] + if tokname == 'error': + self.log.error("Rule %r must be defined as a function", name) + self.error = True + continue + + if tokname not in self.tokens and tokname.find('ignore_') < 0: + self.log.error("Rule %r defined for an unspecified token %s", name, tokname) + self.error = True + continue + + try: + c = re.compile('(?P<%s>%s)' % (name, r), self.reflags) + if (c.match('')): + self.log.error("Regular expression for rule %r matches empty string", name) + self.error = True + except re.error as e: + self.log.error("Invalid regular expression for rule %r. %s", name, e) + if '#' in r: + self.log.error("Make sure '#' in rule %r is escaped with '\\#'", name) + self.error = True + + if not self.funcsym[state] and not self.strsym[state]: + self.log.error("No rules defined for state %r", state) + self.error = True + + # Validate the error function + efunc = self.errorf.get(state, None) + if efunc: + f = efunc + line = f.__code__.co_firstlineno + file = f.__code__.co_filename + module = inspect.getmodule(f) + self.modules.add(module) + + if isinstance(f, types.MethodType): + reqargs = 2 + else: + reqargs = 1 + nargs = f.__code__.co_argcount + if nargs > reqargs: + self.log.error("%s:%d: Rule %r has too many arguments", file, line, f.__name__) + self.error = True + + if nargs < reqargs: + self.log.error("%s:%d: Rule %r requires an argument", file, line, f.__name__) + self.error = True + + for module in self.modules: + self.validate_module(module) + + # ----------------------------------------------------------------------------- + # validate_module() + # + # This checks to see if there are duplicated t_rulename() functions or strings + # in the parser input file. This is done using a simple regular expression + # match on each line in the source code of the given module. + # ----------------------------------------------------------------------------- + + def validate_module(self, module): + try: + lines, linen = inspect.getsourcelines(module) + except IOError: + return + + fre = re.compile(r'\s*def\s+(t_[a-zA-Z_0-9]*)\(') + sre = re.compile(r'\s*(t_[a-zA-Z_0-9]*)\s*=') + + counthash = {} + linen += 1 + for line in lines: + m = fre.match(line) + if not m: + m = sre.match(line) + if m: + name = m.group(1) + prev = counthash.get(name) + if not prev: + counthash[name] = linen + else: + filename = inspect.getsourcefile(module) + self.log.error('%s:%d: Rule %s redefined. Previously defined on line %d', filename, linen, name, prev) + self.error = True + linen += 1 + +# ----------------------------------------------------------------------------- +# lex(module) +# +# Build all of the regular expression rules from definitions in the supplied module +# ----------------------------------------------------------------------------- +def lex(*, module=None, object=None, debug=False, + reflags=int(re.VERBOSE), debuglog=None, errorlog=None): + + global lexer + + ldict = None + stateinfo = {'INITIAL': 'inclusive'} + lexobj = Lexer() + global token, input + + if errorlog is None: + errorlog = PlyLogger(sys.stderr) + + if debug: + if debuglog is None: + debuglog = PlyLogger(sys.stderr) + + # Get the module dictionary used for the lexer + if object: + module = object + + # Get the module dictionary used for the parser + if module: + _items = [(k, getattr(module, k)) for k in dir(module)] + ldict = dict(_items) + # If no __file__ attribute is available, try to obtain it from the __module__ instead + if '__file__' not in ldict: + ldict['__file__'] = sys.modules[ldict['__module__']].__file__ + else: + ldict = get_caller_module_dict(2) + + # Collect parser information from the dictionary + linfo = LexerReflect(ldict, log=errorlog, reflags=reflags) + linfo.get_all() + if linfo.validate_all(): + raise SyntaxError("Can't build lexer") + + # Dump some basic debugging information + if debug: + debuglog.info('lex: tokens = %r', linfo.tokens) + debuglog.info('lex: literals = %r', linfo.literals) + debuglog.info('lex: states = %r', linfo.stateinfo) + + # Build a dictionary of valid token names + lexobj.lextokens = set() + for n in linfo.tokens: + lexobj.lextokens.add(n) + + # Get literals specification + if isinstance(linfo.literals, (list, tuple)): + lexobj.lexliterals = type(linfo.literals[0])().join(linfo.literals) + else: + lexobj.lexliterals = linfo.literals + + lexobj.lextokens_all = lexobj.lextokens | set(lexobj.lexliterals) + + # Get the stateinfo dictionary + stateinfo = linfo.stateinfo + + regexs = {} + # Build the master regular expressions + for state in stateinfo: + regex_list = [] + + # Add rules defined by functions first + for fname, f in linfo.funcsym[state]: + regex_list.append('(?P<%s>%s)' % (fname, _get_regex(f))) + if debug: + debuglog.info("lex: Adding rule %s -> '%s' (state '%s')", fname, _get_regex(f), state) + + # Now add all of the simple rules + for name, r in linfo.strsym[state]: + regex_list.append('(?P<%s>%s)' % (name, r)) + if debug: + debuglog.info("lex: Adding rule %s -> '%s' (state '%s')", name, r, state) + + regexs[state] = regex_list + + # Build the master regular expressions + + if debug: + debuglog.info('lex: ==== MASTER REGEXS FOLLOW ====') + + for state in regexs: + lexre, re_text, re_names = _form_master_re(regexs[state], reflags, ldict, linfo.toknames) + lexobj.lexstatere[state] = lexre + lexobj.lexstateretext[state] = re_text + lexobj.lexstaterenames[state] = re_names + if debug: + for i, text in enumerate(re_text): + debuglog.info("lex: state '%s' : regex[%d] = '%s'", state, i, text) + + # For inclusive states, we need to add the regular expressions from the INITIAL state + for state, stype in stateinfo.items(): + if state != 'INITIAL' and stype == 'inclusive': + lexobj.lexstatere[state].extend(lexobj.lexstatere['INITIAL']) + lexobj.lexstateretext[state].extend(lexobj.lexstateretext['INITIAL']) + lexobj.lexstaterenames[state].extend(lexobj.lexstaterenames['INITIAL']) + + lexobj.lexstateinfo = stateinfo + lexobj.lexre = lexobj.lexstatere['INITIAL'] + lexobj.lexretext = lexobj.lexstateretext['INITIAL'] + lexobj.lexreflags = reflags + + # Set up ignore variables + lexobj.lexstateignore = linfo.ignore + lexobj.lexignore = lexobj.lexstateignore.get('INITIAL', '') + + # Set up error functions + lexobj.lexstateerrorf = linfo.errorf + lexobj.lexerrorf = linfo.errorf.get('INITIAL', None) + if not lexobj.lexerrorf: + errorlog.warning('No t_error rule is defined') + + # Set up eof functions + lexobj.lexstateeoff = linfo.eoff + lexobj.lexeoff = linfo.eoff.get('INITIAL', None) + + # Check state information for ignore and error rules + for s, stype in stateinfo.items(): + if stype == 'exclusive': + if s not in linfo.errorf: + errorlog.warning("No error rule is defined for exclusive state %r", s) + if s not in linfo.ignore and lexobj.lexignore: + errorlog.warning("No ignore rule is defined for exclusive state %r", s) + elif stype == 'inclusive': + if s not in linfo.errorf: + linfo.errorf[s] = linfo.errorf.get('INITIAL', None) + if s not in linfo.ignore: + linfo.ignore[s] = linfo.ignore.get('INITIAL', '') + + # Create global versions of the token() and input() functions + token = lexobj.token + input = lexobj.input + lexer = lexobj + + return lexobj + +# ----------------------------------------------------------------------------- +# runmain() +# +# This runs the lexer as a main program +# ----------------------------------------------------------------------------- + +def runmain(lexer=None, data=None): + if not data: + try: + filename = sys.argv[1] + with open(filename) as f: + data = f.read() + except IndexError: + sys.stdout.write('Reading from standard input (type EOF to end):\n') + data = sys.stdin.read() + + if lexer: + _input = lexer.input + else: + _input = input + _input(data) + if lexer: + _token = lexer.token + else: + _token = token + + while True: + tok = _token() + if not tok: + break + sys.stdout.write(f'({tok.type},{tok.value!r},{tok.lineno},{tok.lexpos})\n') + +# ----------------------------------------------------------------------------- +# @TOKEN(regex) +# +# This decorator function can be used to set the regex expression on a function +# when its docstring might need to be set in an alternative way +# ----------------------------------------------------------------------------- + +def TOKEN(r): + def set_regex(f): + if hasattr(r, '__call__'): + f.regex = _get_regex(r) + else: + f.regex = r + return f + return set_regex diff --git a/cxxheaderparser/dump.py b/cxxheaderparser/dump.py new file mode 100644 index 0000000..f9e194b --- /dev/null +++ b/cxxheaderparser/dump.py @@ -0,0 +1,53 @@ +import argparse +import dataclasses +import json +import pprint +import subprocess +import sys + +from .options import ParserOptions +from .simple import parse_file + + +def dumpmain(): + + parser = argparse.ArgumentParser() + parser.add_argument("header") + parser.add_argument( + "-w", + "--width", + default=80, + type=int, + help="Width of output when in pprint mode", + ) + parser.add_argument("-v", "--verbose", default=False, action="store_true") + parser.add_argument( + "--mode", choices=["json", "pprint", "repr", "brepr"], default="pprint" + ) + + args = parser.parse_args() + + options = ParserOptions(verbose=args.verbose) + data = parse_file(args.header, options=options) + + if args.mode == "pprint": + ddata = dataclasses.asdict(data) + pprint.pprint(ddata, width=args.width, compact=True) + + elif args.mode == "json": + ddata = dataclasses.asdict(data) + json.dump(ddata, sys.stdout, indent=2) + + elif args.mode == "brepr": + stmt = repr(data) + stmt = subprocess.check_output( + ["black", "-", "-q"], input=stmt.encode("utf-8") + ).decode("utf-8") + + print(stmt) + + elif args.mode == "repr": + print(data) + + else: + parser.error("Invalid mode") diff --git a/cxxheaderparser/errors.py b/cxxheaderparser/errors.py new file mode 100644 index 0000000..c92e177 --- /dev/null +++ b/cxxheaderparser/errors.py @@ -0,0 +1,12 @@ +import typing +from .lexer import LexToken + + +class CxxParseError(Exception): + """ + Exception raised when a parsing error occurs + """ + + def __init__(self, msg: str, tok: typing.Optional[LexToken] = None) -> None: + Exception.__init__(self, msg) + self.tok = tok diff --git a/cxxheaderparser/gentest.py b/cxxheaderparser/gentest.py new file mode 100644 index 0000000..19bca76 --- /dev/null +++ b/cxxheaderparser/gentest.py @@ -0,0 +1,98 @@ +import argparse +import dataclasses +import inspect +import subprocess + +from .options import ParserOptions +from .simple import parse_string + + +def nondefault_repr(data): + """ + Similar to the default dataclass repr, but exclude any + default parameters or parameters with compare=False + """ + + is_dataclass = dataclasses.is_dataclass + get_fields = dataclasses.fields + MISSING = dataclasses.MISSING + + def _inner_repr(o) -> str: + if is_dataclass(o): + vals = [] + for f in get_fields(o): + if f.repr and f.compare: + v = getattr(o, f.name) + if f.default_factory is not MISSING: + default = f.default_factory() + else: + default = f.default + + if v != default: + vals.append(f"{f.name}={_inner_repr(v)}") + + return f"{o.__class__.__qualname__ }({', '.join(vals)})" + + elif isinstance(o, list): + return f"[{','.join(_inner_repr(l) for l in o)}]" + elif isinstance(o, dict): + vals = [] + for k, v in o.items(): + vals.append(f'"{k}": {_inner_repr(v)}') + return "{" + ",".join(vals) + "}" + else: + return repr(o) + + return _inner_repr(data) + + +def gentest(infile: str, name: str, outfile: str, verbose: bool): + # Goal is to allow making a unit test as easy as running this dumper + # on a file and copy/pasting this into a test + + with open(infile, "r") as fp: + content = fp.read() + + options = ParserOptions(verbose=verbose) + + data = parse_string(content, options=options) + + stmt = nondefault_repr(data) + + content = content.replace("\n", "\n ") + + stmt = inspect.cleandoc( + f''' + + def test_{name}(): + content = """ + {content} + """ + data = parse_string(content, cleandoc=True) + + assert data == {stmt} + + ''' + ) + + # format it with black + stmt = subprocess.check_output( + ["black", "-", "-q"], input=stmt.encode("utf-8") + ).decode("utf-8") + + if outfile == "-": + print(stmt) + else: + with open(outfile, "w") as fp: + fp.write(stmt) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("header") + parser.add_argument("name", nargs="?", default="TODO") + parser.add_argument("-v", "--verbose", default=False, action="store_true") + parser.add_argument("-o", "--output", default="-") + args = parser.parse_args() + + gentest(args.header, args.name, args.output, args.verbose) diff --git a/cxxheaderparser/lexer.py b/cxxheaderparser/lexer.py new file mode 100644 index 0000000..93bb3e7 --- /dev/null +++ b/cxxheaderparser/lexer.py @@ -0,0 +1,425 @@ +from collections import deque +import re +import typing +import sys + + +from ._ply import lex + + +if sys.version_info >= (3, 8): + Protocol = typing.Protocol +else: + Protocol = object + +_line_re = re.compile(r'^#line (\d+) "(.*)"') +_multicomment_re = re.compile("\n[\\s]+\\*") + + +class Location(typing.NamedTuple): + """ + Location that something was found at, takes #line directives into account + """ + + filename: str + lineno: int + + +class LexToken(Protocol): + """ + Token as emitted by PLY and modified by our lexer + """ + + #: Lexer type for this token + type: str + + #: Raw value for this token + value: str + + lineno: int + lexpos: int + + #: Location token was found at + location: Location + + +class Lexer: + + keywords = { + "__attribute__", + "alignas", + "alignof", + "asm", + "auto", + "bool", + "break", + "case", + "catch", + "char", + "char8_t", + "char16_t", + "char32_t", + "class", + "const", + "constexpr", + "const_cast", + "continue", + "decltype", + "__declspec", + "default", + "delete", + "do", + "double", + "dynamic_cast", + "else", + "enum", + "explicit", + "export", + "extern", + "false", + "final", + "float", + "for", + "friend", + "goto", + "if", + "inline", + "int", + "long", + "mutable", + "namespace", + "new", + "noexcept", + "nullptr", + "nullptr_t", # not a keyword, but makes things easier + "operator", + "private", + "protected", + "public", + "register", + "reinterpret_cast", + "return", + "short", + "signed", + "sizeof", + "static", + "static_assert", + "static_cast", + "struct", + "switch", + "template", + "this", + "thread_local", + "throw", + "true", + "try", + "typedef", + "typeid", + "typename", + "union", + "unsigned", + "using", + "virtual", + "void", + "volatile", + "wchar_t", + "while", + } + + tokens = [ + "NUMBER", + "FLOAT_NUMBER", + "NAME", + "COMMENT_SINGLELINE", + "COMMENT_MULTILINE", + "PRECOMP_MACRO", + "DIVIDE", + "CHAR_LITERAL", + "STRING_LITERAL", + "NEWLINE", + "ELLIPSIS", + "DBL_LBRACKET", + "DBL_RBRACKET", + "DBL_COLON", + "DBL_AMP", + "SHIFT_LEFT", + ] + list(keywords) + + literals = [ + "<", + ">", + "(", + ")", + "{", + "}", + "[", + "]", + ";", + ":", + ",", + "\\", + "|", + "%", + "^", + "!", + "*", + "-", + "+", + "&", + "=", + "'", + ".", + ] + + t_ignore = " \t\r?@\f" + t_NUMBER = r"[0-9][0-9XxA-Fa-f]*" + t_FLOAT_NUMBER = r"[-+]?[0-9]*\.[0-9]+([eE][-+]?[0-9]+)?" + + def t_NAME(self, t): + r"[A-Za-z_~][A-Za-z0-9_]*" + if t.value in self.keywords: + t.type = t.value + return t + + def t_PRECOMP_MACRO(self, t): + r"\#.*" + m = _line_re.match(t.value) + if m: + filename = m.group(2) + if filename not in self._filenames_set: + self.filenames.append(filename) + self._filenames_set.add(filename) + self.filename = filename + + self.line_offset = 1 + self.lex.lineno - int(m.group(1)) + + else: + return t + + def t_COMMENT_SINGLELINE(self, t): + r"\/\/.*\n?" + if t.value.startswith("///") or t.value.startswith("//!"): + self.comments.append(t.value.lstrip("\t ").rstrip("\n")) + t.lexer.lineno += t.value.count("\n") + return t + + t_DIVIDE = r"/(?!/)" + t_CHAR_LITERAL = "'.'" + t_ELLIPSIS = r"\.\.\." + t_DBL_LBRACKET = r"\[\[" + t_DBL_RBRACKET = r"\]\]" + t_DBL_COLON = r"::" + t_DBL_AMP = r"&&" + t_SHIFT_LEFT = r"<<" + # SHIFT_RIGHT introduces ambiguity + + # found at http://wordaligned.org/articles/string-literals-and-regular-expressions + # TODO: This does not work with the string "bla \" bla" + t_STRING_LITERAL = r'"([^"\\]|\\.)*"' + + # Found at http://ostermiller.org/findcomment.html + def t_COMMENT_MULTILINE(self, t): + r"/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/\n?" + if t.value.startswith("/**") or t.value.startswith("/*!"): + # not sure why, but get double new lines + v = t.value.replace("\n\n", "\n") + # strip prefixing whitespace + v = _multicomment_re.sub("\n*", v) + self.comments = v.splitlines() + t.lexer.lineno += t.value.count("\n") + return t + + def t_NEWLINE(self, t): + r"\n+" + t.lexer.lineno += len(t.value) + del self.comments[:] + return t + + def t_error(self, v): + print("Lex error: ", v) + + _lexer = None + + def __new__(cls, *args, **kwargs): + # only build the lexer once + inst = super().__new__(cls) + if cls._lexer is None: + cls._lexer = lex.lex(module=inst) + + inst.lex = cls._lexer.clone(inst) + return inst + + def __init__(self, filename: typing.Optional[str] = None): + self.input = self.lex.input + + # For tracking current file/line position + self.filename = filename + self.line_offset = 0 + + self.filenames = [] + self._filenames_set = set() + + if self.filename: + self.filenames.append(filename) + self._filenames_set.add(filename) + + # Doxygen comments + self.comments = [] + + self.lookahead = typing.Deque[LexToken]() + + def current_location(self) -> Location: + if self.lookahead: + return self.lookahead[0].location + return Location(self.filename, self.lex.lineno - self.line_offset) + + def get_doxygen(self) -> typing.Optional[str]: + """ + This should be called after the first element of something has + been consumed. + + It will lookahead for comments that come after the item, if prior + comments don't exist. + """ + + # Assumption: This function is either called at the beginning of a + # statement or at the end of a statement + + if self.comments: + comments = self.comments + else: + comments = [] + # only look for comments until a newline (including lookahead) + for tok in self.lookahead: + if tok.type == "NEWLINE": + return None + + while True: + tok = self.lex.token() + comments.extend(self.comments) + + if tok is None: + break + + tok.location = Location(self.filename, tok.lineno - self.line_offset) + ttype = tok.type + if ttype == "NEWLINE": + self.lookahead.append(tok) + break + + if ttype not in self._discard_types: + self.lookahead.append(tok) + + if ttype == "NAME": + break + + del self.comments[:] + + comments = "\n".join(comments) + del self.comments[:] + if comments: + return comments + + return None + + _discard_types = {"NEWLINE", "COMMENT_SINGLELINE", "COMMENT_MULTILINE"} + + def token(self) -> LexToken: + tok = None + while self.lookahead: + tok = self.lookahead.popleft() + if tok.type not in self._discard_types: + return tok + + while True: + tok = self.lex.token() + if tok is None: + raise EOFError("unexpected end of file") + + if tok.type not in self._discard_types: + tok.location = Location(self.filename, tok.lineno - self.line_offset) + break + + return tok + + def token_eof_ok(self) -> typing.Optional[LexToken]: + tok = None + while self.lookahead: + tok = self.lookahead.popleft() + if tok.type not in self._discard_types: + return tok + + while True: + tok = self.lex.token() + if tok is None: + break + + if tok.type not in self._discard_types: + tok.location = Location(self.filename, tok.lineno - self.line_offset) + break + + return tok + + def token_if(self, *types: str) -> typing.Optional[LexToken]: + tok = self.token_eof_ok() + if tok is None: + return None + if tok.type not in types: + # put it back on the left in case it was retrieved + # from the lookahead buffer + self.lookahead.appendleft(tok) + return None + return tok + + def token_if_in_set(self, types: typing.Set[str]) -> typing.Optional[LexToken]: + tok = self.token_eof_ok() + if tok is None: + return None + if tok.type not in types: + # put it back on the left in case it was retrieved + # from the lookahead buffer + self.lookahead.appendleft(tok) + return None + return tok + + def token_if_val(self, *vals: str) -> typing.Optional[LexToken]: + tok = self.token_eof_ok() + if tok is None: + return None + if tok.value not in vals: + # put it back on the left in case it was retrieved + # from the lookahead buffer + self.lookahead.appendleft(tok) + return None + return tok + + def token_if_not(self, *types: str) -> typing.Optional[LexToken]: + tok = self.token_eof_ok() + if tok is None: + return None + if tok.type in types: + # put it back on the left in case it was retrieved + # from the lookahead buffer + self.lookahead.appendleft(tok) + return None + return tok + + def token_peek_if(self, *types: str) -> bool: + tok = self.token_eof_ok() + if not tok: + return False + self.lookahead.appendleft(tok) + return tok.type in types + + def return_token(self, tok: LexToken) -> None: + self.lookahead.appendleft(tok) + + def return_tokens(self, toks: typing.Iterable[LexToken]) -> None: + self.lookahead.extendleft(reversed(toks)) + + +if __name__ == "__main__": + try: + lex.runmain(lexer=Lexer(None)) + except EOFError: + pass diff --git a/cxxheaderparser/options.py b/cxxheaderparser/options.py new file mode 100644 index 0000000..b9ecb82 --- /dev/null +++ b/cxxheaderparser/options.py @@ -0,0 +1,14 @@ +from dataclasses import dataclass + + +@dataclass +class ParserOptions: + """ + Options that control parsing behaviors + """ + + #: If true, prints out + verbose: bool = False + + #: If true, converts a single void parameter to zero parameters + convert_void_to_zero_params: bool = True diff --git a/cxxheaderparser/parser.py b/cxxheaderparser/parser.py new file mode 100644 index 0000000..ae9060b --- /dev/null +++ b/cxxheaderparser/parser.py @@ -0,0 +1,2082 @@ +from collections import deque + +import inspect +import re +import typing + +from .errors import CxxParseError +from .lexer import Lexer, LexToken, Location +from .options import ParserOptions +from .parserstate import ( + BlockState, + ClassBlockState, + EmptyBlockState, + ExternBlockState, + NamespaceBlockState, + ParsedTypeModifiers, + State, +) +from .types import ( + AnonymousName, + Array, + AutoSpecifier, + BaseClass, + ClassDecl, + DecltypeSpecifier, + DecoratedType, + EnumDecl, + Enumerator, + Field, + ForwardDecl, + FriendDecl, + Function, + FunctionType, + FundamentalSpecifier, + Method, + MoveReference, + NameSpecifier, + NamespaceDecl, + Operator, + PQNameSegment, + Parameter, + PQName, + Pointer, + Reference, + TemplateArgument, + TemplateDecl, + TemplateNonTypeParam, + TemplateSpecialization, + TemplateTypeParam, + Token, + Type, + Typedef, + UsingAlias, + UsingDecl, + Value, + Variable, +) +from .visitor import CxxVisitor + +LexTokenList = typing.List[LexToken] +T = typing.TypeVar("T") + + +class CxxParser: + """ + Single-use parser object + """ + + def __init__( + self, + filename: str, + content: str, + visitor: CxxVisitor, + options: typing.Optional[ParserOptions] = None, + ) -> None: + self.visitor = visitor + self.filename = filename + + self.lex = Lexer(filename) + self.lex.input(content) + + global_ns = NamespaceDecl([], False) + self.current_namespace = global_ns + + self.state: BlockState = NamespaceBlockState(None, global_ns) + self.anon_id = 0 + + self.options = options if options else ParserOptions() + + self.verbose = True if self.options.verbose else False + if self.verbose: + + def debug_print(fmt: str, *args: typing.Any): + fmt = f"[%4d] {fmt}" + args = (inspect.currentframe().f_back.f_lineno,) + args + print(fmt % args) + + self.debug_print = debug_print + else: + self.debug_print = lambda fmt, *args: None + + # + # State management + # + + def _push_state(self, cls: typing.Type[T], *args) -> T: + state = cls(self.state, *args) + if isinstance(state, NamespaceBlockState): + self.current_namespace = state.namespace + self.state = state + return state + + def _pop_state(self) -> State: + prev_state = self.state + prev_state._finish(self.visitor) + state = prev_state.parent + if state is None: + raise CxxParseError("INTERNAL ERROR: unbalanced state") + + if isinstance(state, NamespaceBlockState): + self.current_namespace = state.namespace + + self.state = state + return prev_state + + @property + def _current_access(self) -> typing.Optional[str]: + return getattr(self.state, "access", None) + + # + # Utility parsing functions used by the rest of the code + # + + def _parse_error( + self, tok: typing.Optional[LexToken], expected="" + ) -> typing.NoReturn: + if not tok: + # common case after a failed token_if + tok = self.lex.token() + + if expected: + expected = f", expected '{expected}'" + + msg = f"unexpected '{tok.value}'{expected}" + + # TODO: better error message + return CxxParseError(msg, tok) + + def _next_token_must_be(self, *tokenTypes: str) -> LexToken: + tok = self.lex.token() + if tok.type not in tokenTypes: + raise self._parse_error(tok, "' or '".join(tokenTypes)) + return tok + + def _next_token_in_set(self, tokenTypes: typing.Set[str]) -> LexToken: + tok = self.lex.token() + if tok.type not in tokenTypes: + raise self._parse_error(tok, "' or '".join(sorted(tokenTypes))) + return tok + + def _consume_up_to(self, rtoks: LexTokenList, *token_types: str) -> LexTokenList: + # includes the last token + get_token = self.lex.token + while True: + tok = get_token() + rtoks.append(tok) + if tok.type in token_types: + break + return rtoks + + def _consume_until(self, rtoks: LexTokenList, *token_types: str) -> LexTokenList: + # does not include the found token + token_if_not = self.lex.token_if_not + while True: + tok = token_if_not(*token_types) + if tok is None: + break + rtoks.append(tok) + return rtoks + + def _consume_value_until( + self, rtoks: LexTokenList, *token_types: str + ) -> LexTokenList: + # does not include the found token + token_if_not = self.lex.token_if_not + while True: + tok = token_if_not(*token_types) + if tok is None: + break + + if tok.type in self._balanced_token_map: + rtoks.extend(self._consume_balanced_tokens(tok)) + else: + rtoks.append(tok) + + return rtoks + + _end_balanced_tokens = {">", "}", "]", ")", "DBL_RBRACKET"} + _balanced_token_map = { + "<": ">", + "{": "}", + "(": ")", + "[": "]", + "DBL_LBRACKET": "DBL_RBRACKET", + } + + def _consume_balanced_tokens( + self, + *init_tokens: LexToken, + token_map: typing.Optional[typing.Dict[str, str]] = None, + ) -> LexTokenList: + + if token_map is None: + token_map = self._balanced_token_map + + consumed = list(init_tokens) + match_stack = deque((token_map[tok.type] for tok in consumed)) + get_token = self.lex.token + + while True: + tok = get_token() + consumed.append(tok) + + if tok.type in self._end_balanced_tokens: + expected = match_stack.pop() + if tok.type != expected: + # hack: ambiguous right-shift issues here, really + # should be looking at the context + if tok.type == ">": + tok = self.lex.token_if(">") + if tok: + consumed.append(tok) + match_stack.append(expected) + continue + + raise self._parse_error(tok, expected) + if len(match_stack) == 0: + return consumed + + continue + + next_end = token_map.get(tok.type) + if next_end: + match_stack.append(next_end) + + def _discard_contents(self, start_type: str, end_type: str) -> None: + # use this instead of consume_balanced_tokens because + # we don't care at all about the internals + level = 1 + get_token = self.lex.token + while True: + tok = get_token() + if tok.type == start_type: + level += 1 + elif tok.type == end_type: + level -= 1 + if level == 0: + break + + def _create_value(self, toks: LexTokenList) -> Value: + return Value([Token(tok.value, tok.type) for tok in toks]) + + # + # Parsing begins here + # + + def parse(self) -> None: + """ + Parse the header contents + """ + + # non-ambiguous parsing functions for each token type + _translation_unit_tokens = { + "__attribute__": self._consume_gcc_attribute, + "__declspec": self._consume_declspec, + "alignas": self._consume_attribute_specifier_seq, + "extern": self._parse_extern, + "friend": self._parse_friend_decl, + "namespace": self._parse_namespace, + "private": self._process_access_specifier, + "protected": self._process_access_specifier, + "public": self._process_access_specifier, + "static_assert": self._consume_static_assert, + "template": self._parse_template, + "typedef": self._parse_typedef, + "using": self._parse_using, + "{": self._on_empty_block_start, + "}": self._on_block_end, + "DBL_LBRACKET": self._consume_attribute_specifier_seq, + "PRECOMP_MACRO": self._process_preprocessor_token, + ";": lambda _1, _2: None, + } + + tok = None + + get_token_eof_ok = self.lex.token_eof_ok + get_doxygen = self.lex.get_doxygen + + try: + while True: + tok = get_token_eof_ok() + if not tok: + break + + doxygen = get_doxygen() + + fn = _translation_unit_tokens.get(tok.type) + if fn: + fn(tok, doxygen) + else: + # this processes ambiguous declarations + self._parse_declarations(tok, doxygen) + + except Exception as e: + if self.verbose: + raise + context = "" + if isinstance(e, CxxParseError): + context = ": " + str(e) + if e.tok: + tok = e.tok + + if tok: + filename, lineno = tok.location + msg = f"Not able to parse {filename} on line {lineno} evaluating '{tok.value}'{context}" + else: + msg = f"Error parsing {self.filename}{context}" + + raise CxxParseError(msg) from e + + # + # Preprocessor directives + # + + _preprocessor_compress_re = re.compile(r"^#[\t ]+") + _preprocessor_split_re = re.compile(r"[\t ]+") + + def _process_preprocessor_token( + self, tok: LexToken, doxygen: typing.Optional[str] + ) -> None: + value = self._preprocessor_compress_re.sub("#", tok.value) + value = self._preprocessor_split_re.split(value, 1) + if len(value) == 2: + self.state.location = tok.location + macro = value[0].lower().replace(" ", "") + if macro.startswith("#include"): + self.visitor.on_include(self.state, value[1]) + elif macro.startswith("#define"): + self.visitor.on_define(self.state, value[1]) + elif macro.startswith("#pragma"): + self.visitor.on_pragma(self.state, value[1]) + + # + # Various + # + + def _parse_namespace( + self, tok: LexToken, doxygen: typing.Optional[str], inline: bool = False + ) -> None: + """ + namespace_definition: named_namespace_definition + | unnamed_namespace_definition + + named_namespace_definition: ["inline"] "namespace" IDENTIFIER "{" namespace_body "}" + + unnamed_namespace_definition: ["inline"] "namespace" "{" namespace_body "}" + + namespace_alias_definition: "namespace" IDENTIFIER "=" qualified_namespace_specifier ";" + """ + + names = [] + location = tok.location + + tok = self._next_token_must_be("NAME", "{") + if tok.type != "{": + while True: + names.append(tok.value) + tok = self._next_token_must_be("DBL_COLON", "{") + if tok.type == "{": + break + + tok = self._next_token_must_be("NAME") + + # TODO: namespace_alias_definition + + ns = NamespaceDecl(names, inline) + state = self._push_state(NamespaceBlockState, ns) + state.location = location + self.visitor.on_namespace_start(state) + + def _parse_extern(self, tok: LexToken, doxygen: typing.Optional[str]) -> None: + + etok = self.lex.token_if("STRING_LITERAL") + if etok: + if self.lex.token_if("{"): + state = self._push_state(ExternBlockState, etok.value) + state.location = tok.location + self.visitor.on_extern_block_start(state) + return + + # an extern variable/function with specific linkage + self.lex.return_token(etok) + + self._parse_declarations(tok, doxygen) + + def _parse_friend_decl( + self, + tok: LexToken, + doxygen: typing.Optional[str], + template: typing.Optional[TemplateDecl] = None, + ) -> None: + if not isinstance(self.state, ClassBlockState): + raise self._parse_error(tok) + + tok = self.lex.token() + self._parse_declarations(tok, doxygen, template, is_friend=True) + + def _parse_inline(self, tok: LexToken, doxygen: typing.Optional[str]) -> None: + itok = self.lex.token_if("namespace") + if itok: + self._parse_namespace(itok, doxygen, inline=True) + else: + self._parse_declarations(tok, doxygen) + + def _parse_mutable(self, tok: LexToken, doxygen: typing.Optional[str]) -> None: + if not isinstance(self.state, ClassBlockState): + raise self._parse_error(tok) + + self._parse_declarations(tok, doxygen) + + def _parse_typedef(self, tok: LexToken, doxygen: typing.Optional[str]) -> None: + tok = self.lex.token() + self._parse_declarations(tok, doxygen, is_typedef=True) + + def _consume_static_assert( + self, tok: LexToken, doxygen: typing.Optional[str] + ) -> None: + self._next_token_must_be("(") + self._discard_contents("(", ")") + + def _on_empty_block_start( + self, tok: LexToken, doxygen: typing.Optional[str] + ) -> None: + self._push_state(EmptyBlockState) + + def _on_block_end(self, tok: LexToken, doxygen: typing.Optional[str]) -> None: + old_state = self._pop_state() + if isinstance(old_state, ClassBlockState): + self._finish_class_decl(old_state) + + # + # Template parsing + # + + def _parse_template_type_parameter( + self, tok: LexToken, template: TemplateDecl + ) -> TemplateTypeParam: + """ + type_parameter: "class" ["..."] [IDENTIFIER] + | "class" [IDENTIFIER] "=" type_id + | "typename" ["..."] [IDENTIFIER] + | "typename" [IDENTIFIER] "=" type_id + | "template" "<" template_parameter_list ">" "class" ["..."] [IDENTIFIER] + | "template" "<" template_parameter_list ">" "class" [IDENTIFIER] "=" id_expression + """ + # entry: token is either class or typename + typekey = tok.type + param_pack = True if self.lex.token_if("ELLIPSIS") else False + name = None + default = None + + tok = self.lex.token_if("NAME") + if tok: + name = tok.value + + tok = self.lex.token_if("=") + if tok: + default = self._create_value(self._consume_value_until([], ",", ">")) + + return TemplateTypeParam(typekey, name, param_pack, default, template) + + def _parse_template_decl(self) -> TemplateDecl: + """ + template_declaration: "template" "<" template_parameter_list ">" declaration + + explicit_specialization: "template" "<" ">" declaration + + template_parameter_list: template_parameter + | template_parameter_list "," template_parameter + + template_parameter: type_parameter + | parameter_declaration + """ + tok = self._next_token_must_be("<") + params = [] + + lex = self.lex + + if not lex.token_if(">"): + while True: + + tok = lex.token() + tok_type = tok.type + + if tok_type == "template": + template = self._parse_template_decl() + tok = self._next_token_must_be("class", "typename") + param = self._parse_template_type_parameter(tok, template) + elif tok_type == "class": + param = self._parse_template_type_parameter(tok, None) + elif tok_type == "typename": + ptok = lex.token() + if ptok.type in ("ELLIPSIS", "=", ",", ">") or ( + ptok.type == "NAME" and lex.token_peek_if("=", ",", ">") + ): + lex.return_token(ptok) + param = self._parse_template_type_parameter(tok, None) + else: + param = self._parse_parameter(ptok, TemplateNonTypeParam, ">") + else: + param = self._parse_parameter(tok, TemplateNonTypeParam, ">") + + params.append(param) + + tok = self._next_token_must_be(",", ">") + if tok.type == ">": + break + + return TemplateDecl(params) + + def _parse_template( + self, tok: LexToken, doxygen: typing.Optional[str] + ) -> TemplateDecl: + + template = self._parse_template_decl() + + tok = self.lex.token() + if tok.type == "using": + self._parse_using(tok, doxygen, template) + elif tok.type == "friend": + self._parse_friend_decl(tok, doxygen, template) + else: + self._parse_declarations(tok, doxygen, template) + + def _parse_template_specialization(self) -> TemplateSpecialization: + """ + template_argument_list: template_argument ["..."] + | template_argument_list "," template_argument ["..."] + + template_argument: constant_expression + | type_id + | id_expression + + template_id: simple_template_id + | operator_function_id "<" [template_argument_list] ">" + | literal_operator_id "<" [template_argument_list] ">" + + simple_template_id: IDENTIFIER "<" [template_argument_list] ">" + """ + + args: typing.List[TemplateArgument] = [] + param_pack = False + + # On entry, < has just been consumed + + while True: + raw_name = self._consume_value_until([], ",", ">", "...") + raw_name = [Token(tok.value, tok.type) for tok in raw_name] + args.append(TemplateArgument(raw_name)) + + tok = self._next_token_must_be(",", ">", "ELLIPSIS") + if tok.type == ">": + break + elif tok.type == "ELLIPSIS": + param_pack = True + self._next_token_must_be(">") + break + + return TemplateSpecialization(args, param_pack) + + # + # Attributes + # + + def _consume_gcc_attribute( + self, tok: LexToken, doxygen: typing.Optional[str] = None + ) -> None: + tok1 = self._next_token_must_be("(") + tok2 = self._next_token_must_be("(") + self._consume_balanced_tokens(tok1, tok2) + + def _consume_declspec( + self, tok: LexToken, doxygen: typing.Optional[str] = None + ) -> None: + tok = self._next_token_must_be("(") + self._consume_balanced_tokens(tok) + + _attribute_specifier_seq_start_types = ("DBL_LBRACKET", "alignas") + + def _consume_attribute_specifier_seq( + self, tok: LexToken, doxygen: typing.Optional[str] = None + ) -> None: + """ + attribute_specifier_seq: attribute_specifier + | attribute_specifier_seq attribute_specifier + + attribute_specifier: "[[" attribute_list "]]" + | alignment_specifier + + alignment_specifier: "alignas" "(" type_id ["..."] ")" + | "alignas" "(" alignment_expression ["..."] ")" + + attribute_list: [attribute] + | attribute_list "," [attribute] + | attribute "..." + | attribute_list "," attribute "..." + + attribute: attribute_token [attribute_argument_clause] + + attribute_token: IDENTIFIER + | attribute_scoped_token + + attribute_scoped_token: attribute_namespace "::" IDENTIFIER + + attribute_namespace: IDENTIFIER + + attribute_argument_clause: "(" balanced_token_seq ")" + + balanced_token_seq: balanced_token + | balanced_token_seq balanced_token + + balanced_token: "(" balanced_token_seq ")" + | "[" balanced_token_seq "]" + | "{" balanced_token_seq "}" + | token + """ + + # TODO: retain the attributes and do something with them + # attrs = [] + + while True: + if tok.type == "DBL_LBRACKET": + tokens = self._consume_balanced_tokens(tok) + # attrs.append(Attribute(tokens)) + elif tok.type == "alignas": + next_tok = self._next_token_must_be("(") + tokens = self._consume_balanced_tokens(next_tok) + # attrs.append(AlignasAttribute(tokens)) + else: + self.lex.return_token(tok) + break + + # multiple attributes can be specified + tok = self.lex.token_if(*self._attribute_specifier_seq_start_types) + if tok is None: + break + + # TODO return attrs + + # + # Using directive/declaration/typealias + # + + def _parse_using_directive(self) -> None: + """ + using_directive: [attribute_specifier_seq] "using" "namespace" ["::"] [nested_name_specifier] IDENTIFIER ";" + """ + + names = [] + if self.lex.token_if("DBL_COLON"): + names.append("") + + while True: + tok = self._next_token_must_be("NAME") + names.append(tok.value) + + if not self.lex.token_if("DBL_COLON"): + break + + if not names: + raise self._parse_error(None, "NAME") + + self.visitor.on_using_namespace(self.state, names) + + def _parse_using_declaration(self, tok: LexToken) -> None: + """ + using_declaration: "using" ["typename"] ["::"] nested_name_specifier unqualified_id ";" + | "using" "::" unqualified_id ";" + + """ + if tok.type == "typename": + tok = self.lex.token() + + typename, _ = self._parse_pqname( + tok, fn_ok=True, compound_ok=True, fund_ok=True + ) + decl = UsingDecl(typename, self._current_access) + + self.visitor.on_using_declaration(self.state, decl) + + def _parse_using_typealias( + self, id_tok: LexToken, template: typing.Optional[TemplateDecl] + ) -> None: + """ + alias_declaration: "using" IDENTIFIER "=" type_id ";" + """ + + parsed_type, mods = self._parse_type(None) + mods.validate(var_ok=False, meth_ok=False, msg="parsing typealias") + + dtype = self._parse_cv_ptr(parsed_type) + + alias = UsingAlias(id_tok.value, dtype, template, self._current_access) + + self.visitor.on_using_alias(self.state, alias) + + def _parse_using( + self, + tok: LexToken, + doxygen: typing.Optional[str], + template: typing.Optional[TemplateDecl] = None, + ) -> None: + self.state.location = tok.location + + tok = self._next_token_must_be("NAME", "DBL_COLON", "namespace", "typename") + + if tok.type == "namespace": + if template: + raise CxxParseError( + "unexpected using-directive when parsing alias-declaration", tok + ) + if isinstance(self.state, ClassBlockState): + raise self._parse_error(tok) + + self._parse_using_directive() + elif tok.type in ("DBL_COLON", "typename") or not self.lex.token_if("="): + if template: + raise CxxParseError( + "unexpected using-declaration when parsing alias-declaration", tok + ) + self._parse_using_declaration(tok) + else: + self._parse_using_typealias(tok, template) + + # All using things end with a semicolon + self._next_token_must_be(";") + + # + # Enum parsing + # + + def _parse_enum_decl( + self, + typename: PQName, + tok: LexToken, + doxygen: typing.Optional[str], + is_typedef: bool, + location: Location, + mods: ParsedTypeModifiers, + ) -> None: + """ + opaque_enum_declaration: enum_key [attribute_specifier_seq] IDENTIFIER [enum_base] ";" + + enum_specifier: enum_head "{" [enumerator_list] "}" + | enum_head "{" enumerator_list "," "}" + + enum_head: enum_key [attribute_specifier_seq] [IDENTIFIER] [enum_base] + | enum_key [attribute_specifier_seq] nested_name_specifier IDENTIFIER [enum_base] + + enum_key: "enum" + | "enum" "class" + | "enum" "struct" + + enum_base: ":" type_specifier_seq + """ + + self.state.location = location + + tok_type = tok.type + + # entry: tok is one of _class_enum_stage2 + if tok_type not in (":", "{"): + raise self._parse_error(tok) + + base = None + values: typing.List[Enumerator] = [] + + if tok_type == ":": + base, _ = self._parse_pqname(None, fund_ok=True) + tok = self._next_token_must_be("{", ";") + tok_type = tok.type + + if tok_type == ";": + if is_typedef: + raise self._parse_error(tok) + + # enum forward declaration with base + fdecl = ForwardDecl( + typename, None, doxygen, base, access=self._current_access + ) + self.visitor.on_forward_decl(self.state, fdecl) + return + + values = self._parse_enumerator_list() + + enum = EnumDecl(typename, values, base, doxygen, self._current_access) + self.visitor.on_enum(self.state, enum) + + # Finish it up + self._finish_class_or_enum(enum.typename, is_typedef, mods) + + def _parse_enumerator_list(self) -> typing.List[Enumerator]: + """ + enumerator_list: enumerator_definition + | enumerator_list "," enumerator_definition + + enumerator_definition: enumerator + | enumerator "=" constant_expression + + enumerator: IDENTIFIER [attribute-specifier-seq] + """ + + values: typing.List[Enumerator] = [] + + while True: + name_tok = self._next_token_must_be("}", "NAME") + if name_tok.value == "}": + break + + name = name_tok.value + doxygen = self.lex.get_doxygen() + value = None + + tok = self._next_token_must_be("}", ",", "=", "DBL_LBRACKET") + if tok.type == "DBL_LBRACKET": + self._consume_attribute_specifier_seq(tok) + tok = self._next_token_must_be("}", ",", "=") + + if tok.type == "=": + value = self._create_value(self._consume_value_until([], ",", "}")) + tok = self._next_token_must_be("}", ",") + + values.append(Enumerator(name, value, doxygen)) + + if tok.type == "}": + break + + return values + + # + # Type parsing + # + + _base_access_virtual = {"public", "private", "protected", "virtual"} + + def _parse_class_decl_base_clause( + self, default_access: str + ) -> typing.List[BaseClass]: + """ + base_clause: ":" base_specifier_list + + base_specifier_list: base_specifier ["..."] + | base_specifier_list "," base_specifier ["..."] + + base_specifier: [attribute_specifier_seq] base_type_specifier + | [attribute_specifier_seq] "virtual" [access_specifier] base_type_specifier + | [attribute_specifier_seq] access_specifier ["virtual"] base_type_specifier + + base_type_specifier: class_or_decltype + + class_or_decltype: ["::"] [nested_name_specifier] class_name + | decltype_specifier + """ + + bases = [] + + while True: + tok = self.lex.token() + tok_type = tok.type + + # might start with attributes + if tok.type in self._attribute_specifier_seq_start_types: + self._parse_attribute_specifier_seq(tok) + tok = self.lex.token() + tok_type = tok.type + + access = default_access + virtual = False + parameter_pack = False + + # virtual/access specifier comes next + while tok_type in self._base_access_virtual: + if tok_type == "virtual": + virtual = True + else: + access = tok_type + + tok = self.lex.token() + tok_type = tok.type + + # Followed by the name + typename, _ = self._parse_pqname(tok) + + # And potentially a parameter pack + if self.lex.token_if("ELLIPSIS"): + parameter_pack = True + + bases.append(BaseClass(access, typename, virtual, parameter_pack)) + + if not self.lex.token_if(","): + break + + return bases + + def _parse_class_decl( + self, + typename: PQName, + tok: LexToken, + doxygen: typing.Optional[str], + template: typing.Optional[TemplateDecl], + typedef: bool, + location: Location, + props: typing.Dict[str, LexToken], + ) -> None: + """ + class_specifier: class_head "{" [member_specification] "}" + + class_head: class_key [attribute_specifier_seq] class_head_name [class_virt_specifier_seq] [base_clause] + | class_key [attribute_specifier_seq] [base_clause] + + class_key: "class" + | "struct" + | "union" + + class_head_name: [nested_name_specifier] class_name + + class_name: IDENTIFIER + | simple_template_id + + class_virt_specifier_seq: class_virt_specifier + | class_virt_specifier_seq class_virt_specifier + + class_virt_specifier: "final" + | "explicit" + """ + + bases = [] + explicit = False + final = False + + default_access = "private" if typename.classkey == "class" else "public" + + #: entry: token is one of _class_enum_stage2 + tok_type = tok.type + + while True: + if tok_type == "final": + final = True + elif tok_type == "explicit": + explicit = True + else: + break + + tok = self.lex.token() + tok_type = tok.type + + if tok_type == ":": + bases = self._parse_class_decl_base_clause(default_access) + tok = self.lex.token() + tok_type = tok.type + + if tok_type != "{": + raise self._parse_error(tok, "{") + + clsdecl = ClassDecl( + typename, bases, template, explicit, final, doxygen, self._current_access + ) + state = self._push_state( + ClassBlockState, clsdecl, default_access, typedef, props + ) + state.location = location + self.visitor.on_class_start(state) + + def _finish_class_decl(self, state: ClassBlockState) -> None: + self._finish_class_or_enum(state.class_decl.typename, state.typedef, state.mods) + + def _process_access_specifier( + self, tok: LexToken, doxygen: typing.Optional[str] + ) -> None: + + state = self.state + if not isinstance(state, ClassBlockState): + raise self._parse_error(tok) + + state._set_access(tok.value) + self._next_token_must_be(":") + + def _discard_ctor_initializer(self) -> None: + """ + ctor_initializer: ":" mem_initializer_list + + mem_initializer_list: mem_initializer ["..."] + | mem_initializer "," mem_initializer_list ["..."] + + mem_initializer: mem_initializer_id "(" [expression_list] ")" + | mem_initializer_id braced_init_list + + mem_initializer_id: class_or_decltype + | IDENTIFIER + """ + self.debug_print("discarding ctor intializer") + # all of this is discarded.. the challenge is to determine + # when the initializer ends and the function starts + get_token = self.lex.token + while True: + tok = get_token() + if tok.type == "DBL_COLON": + tok = get_token() + + if tok.type == "decltype": + tok = self._next_token_must_be("(") + self._consume_balanced_tokens(tok) + tok = get_token() + + # each initializer is either foo() or foo{}, so look for that + while True: + if tok.type not in ("{", "("): + tok = get_token() + continue + + if tok.type == "{": + self._discard_contents("{", "}") + elif tok.type == "(": + self._discard_contents("(", ")") + + tok = get_token() + break + + # at the end + if tok.type == "ELLIPSIS": + tok = get_token() + + if tok.type == ",": + continue + elif tok.type == "{": + # reached the function + self._discard_contents("{", "}") + return + else: + raise self._parse_error(tok, ",' or '{") + + # + # Variable parsing + # + + def _parse_bitfield(self) -> int: + # is a integral constant expression... for now, just do integers + tok = self._next_token_must_be("NUMBER") + return int(tok.value) + + def _parse_field( + self, + mods: ParsedTypeModifiers, + dtype: DecoratedType, + pqname: typing.Optional[PQName], + template: typing.Optional[TemplateDecl], + doxygen: typing.Optional[str], + location: Location, + is_typedef: bool, + ) -> None: + + state = self.state + state.location = location + is_class_block = isinstance(state, ClassBlockState) + name = None + bits = None + default = None + + if not pqname: + if is_typedef: + raise CxxParseError("empty name not allowed in typedef") + if not is_class_block: + raise CxxParseError("variables must have names") + + else: + last_segment = pqname.segments[-1] + if not isinstance(last_segment, NameSpecifier): + raise CxxParseError(f"invalid name for variable: {pqname}") + + if is_typedef or is_class_block: + name = last_segment.name + if len(pqname.segments) > 1: + raise CxxParseError(f"name cannot have multiple segments: {pqname}") + + # check for array + tok = self.lex.token_if("[") + if tok: + dtype = self._parse_array_type(tok, dtype) + + # bitfield + tok = self.lex.token_if(":") + if tok: + if is_typedef or not is_class_block: + raise self._parse_error(tok) + + bits = self._parse_bitfield() + + # default values + tok = self.lex.token_if("=") + if tok: + if is_typedef: + raise self._parse_error(tok) + + default = self._create_value(self._consume_value_until([], ",", ";")) + + else: + # default value initializer + tok = self.lex.token_if("{") + if tok: + if is_typedef: + raise self._parse_error(tok) + + default = self._create_value(self._consume_balanced_tokens(tok)) + + if is_typedef: + typedef = Typedef(dtype, name, self._current_access) + self.visitor.on_typedef(state, typedef) + else: + props = dict.fromkeys(mods.both.keys(), True) + props.update(dict.fromkeys(mods.vars.keys(), True)) + + if is_class_block: + f = Field( + name=name, + type=dtype, + access=self._current_access, + value=default, + bits=bits, + doxygen=doxygen, + **props, + ) + self.visitor.on_class_field(state, f) + else: + v = Variable( + pqname, dtype, default, doxygen=doxygen, template=template, **props + ) + self.visitor.on_variable(state, v) + + # + # PQName parsing + # + + def _parse_pqname_decltype_specifier(self) -> DecltypeSpecifier: + """ + decltype_specifier: "decltype" "(" expression ")" + """ + # entry: decltype just consumed + tok = self._next_token_must_be("(") + toks = self._consume_balanced_tokens(tok)[1:-1] + return DecltypeSpecifier([Token(tok.value, tok.type) for tok in toks]) + + _name_compound_start = {"struct", "enum", "class", "union"} + _compound_fundamentals = { + "unsigned", + "signed", + "short", + "int", + "long", + "float", + "double", + "char", + } + + _fundamentals = _compound_fundamentals | { + "bool", + "char16_t", + "char32_t", + "nullptr_t", + "wchar_t", + "void", + } + + def _parse_pqname_fundamental(self, tok_value: str) -> FundamentalSpecifier: + fnames = [tok_value] + _compound_fundamentals = self._compound_fundamentals + + # compound fundamentals can show up in groups in any order + # -> TODO: const can show up in here + if tok_value in _compound_fundamentals: + while True: + tok = self.lex.token_if_in_set(_compound_fundamentals) + if not tok: + break + fnames.append(tok.value) + + return FundamentalSpecifier(" ".join(fnames)) + + def _parse_pqname_name_operator(self) -> LexTokenList: + # last tok was 'operator' -- collect until ( is reached + # - no validation done here, we assume the code is valid + + tok = self.lex.token() + parts = [tok] + + # special case: operator() + if tok.value == "(": + tok = self._next_token_must_be(")") + parts.append(tok) + return parts + + self._consume_until(parts, "(") + return parts + + _pqname_start_tokens = ( + {"auto", "decltype", "NAME", "operator", "template", "DBL_COLON"} + | _name_compound_start + | _fundamentals + ) + + def _parse_pqname_name( + self, tok_value: str + ) -> typing.Tuple[NameSpecifier, typing.Optional[str]]: + name = "" + specialization = None + op = None + + # parse out operators as that's generally useful + if tok_value == "operator": + + op_parts = self._parse_pqname_name_operator() + op = "".join(o.value for o in op_parts) + name = f"operator{op}" + + else: + name = tok_value + + if self.lex.token_if("<"): + # template specialization + specialization = self._parse_template_specialization() + + return NameSpecifier(name, specialization), op + + def _parse_pqname( + self, + tok: typing.Optional[LexToken], + *, + fn_ok: bool = False, + compound_ok: bool = False, + fund_ok: bool = False, + ) -> typing.Tuple[PQName, typing.Optional[str]]: + """ + Parses a possibly qualified function name or a type name, returns when + unexpected item encountered (but does not consume it) + + :param fn_ok: Operator functions ok + :param compound_ok: Compound types ok + :param fund_ok: Fundamental types ok + + qualified_id: ["::"] nested_name_specifier ["template"] unqualified_id + | "::" IDENTIFIER + | "::" operator_function_id + | "::" literal_operator_id + | "::" template_id + + unqualified_id: IDENTIFIER + | operator_function_id + | conversion_function_id + | literal_operator_id + | "~" class_name + | "~" decltype_specifier + | template_id + + conversion_function_id: "operator" conversion_type_id + + conversion_type_id: type_specifier_seq [conversion_declarator] + + conversion_declarator: ptr_operator [conversion_declarator] + + nested_name_specifier: type_name "::" + | IDENTIFIER "::" + | decltype_specifier "::" + | nested_name_specifier IDENTIFIER "::" + | nested_name_specifier ["template"] simple_template_id "::" + + type_name: IDENTIFIER + | simple_template_id + + """ + + classkey = None + segments: typing.List[PQNameSegment] = [] + op = None + + if tok is None: + tok = self.lex.token() + + if tok.type not in self._pqname_start_tokens: + raise self._parse_error(tok) + + if tok.type == "auto": + return PQName([AutoSpecifier()]), None + + _fundamentals = self._fundamentals + + # The pqname could start with class/enum/struct/union + if tok.value in self._name_compound_start: + if not compound_ok: + raise self._parse_error(tok) + classkey = tok.value + if classkey == "enum": + # check for enum class/struct + tok = self.lex.token_if("class", "struct") + if tok: + classkey = f"{classkey} {tok.value}" + + tok = self.lex.token_if( + "alignas", "__attribute__", "__declspec", "DBL_LBRACKET" + ) + if tok: + if tok.type == "__attribute__": + self._consume_gcc_attribute(tok) + elif tok.type == "__declspec": + self._consume_declspec(tok) + else: + self._consume_attribute_specifier_seq(tok) + + tok = self.lex.token_if("NAME", "DBL_COLON") + if not tok: + # Handle unnamed class/enum/struct + self.anon_id += 1 + segments.append(AnonymousName(self.anon_id)) + return PQName(segments, classkey), None + + # First section of the name: Add empty segment if starting out with a + # namespace specifier + if tok.type == "DBL_COLON": + segments.append(NameSpecifier("")) + tok = self._next_token_must_be("NAME", "template", "operator") + + while True: + tok_value = tok.value + + if tok_value == "decltype": + segments.append(self._parse_pqname_decltype_specifier()) + + # check for fundamentals, consume them + elif tok_value in _fundamentals: + if not fund_ok: + raise self._parse_error(tok) + + segments.append(self._parse_pqname_fundamental(tok_value)) + # no additional parts after fundamentals + break + + else: + if tok_value == "[[": + self._consume_attribute_specifier_seq(tok) + + if tok_value == "template": + tok = self._next_token_must_be("NAME") + tok_value = tok.value + + name, op = self._parse_pqname_name(tok_value) + segments.append(name) + if op: + if not fn_ok: + # encountered unexpected operator + raise self._parse_error(tok, "NAME") + + # nothing after operator + break + + # If no more segments, we're done + if not self.lex.token_if("DBL_COLON"): + break + + tok = self._next_token_must_be("NAME", "operator", "template", "decltype") + + pqname = PQName(segments, classkey) + + self.debug_print( + "parse_pqname: %s op=%s", + pqname, + op, + ) + + return pqname, op + + # + # Function parsing + # + + def _parse_parameter( + self, tok: typing.Optional[LexToken], cls: typing.Type[T], end: str = ")" + ) -> T: + """ + Parses a single parameter (excluding vararg parameters). Also used + to parse template non-type parameters + """ + + param_name = None + default = None + param_pack = False + + # required typename + decorators + parsed_type, mods = self._parse_type(tok) + mods.validate(var_ok=False, meth_ok=False, msg="parsing parameter") + + dtype = self._parse_cv_ptr(parsed_type) + + # optional parameter pack + if self.lex.token_if("ELLIPSIS"): + param_pack = True + + # name can be surrounded by parens + tok = self.lex.token_if("(") + if tok: + toks = self._consume_balanced_tokens(tok) + self.lex.return_tokens(toks[1:-1]) + + # optional name + tok = self.lex.token_if("NAME") + if tok: + param_name = tok.value + + # optional array parameter + tok = self.lex.token_if("[") + if tok: + dtype = self._parse_array_type(tok, dtype) + + # optional default value + if self.lex.token_if("="): + default = self._create_value(self._consume_value_until([], ",", end)) + + param = cls(type=dtype, name=param_name, default=default, param_pack=param_pack) + self.debug_print("parameter: %s", param) + return param + + def _parse_parameters(self) -> typing.Tuple[typing.List[Parameter], bool]: + """ + Consumes function parameters and returns them, and vararg if found + """ + + # starting at a ( + + # special case: zero parameters + if self.lex.token_if(")"): + return [], False + + params: typing.List[Parameter] = [] + vararg = False + + while True: + + if self.lex.token_if("ELLIPSIS"): + vararg = True + self._next_token_must_be(")") + break + + param = self._parse_parameter(None, Parameter) + params.append(param) + + tok = self._next_token_must_be(",", ")") + if tok.value == ")": + break + + # convert fn(void) to fn() + if self.options.convert_void_to_zero_params and len(params) == 1: + p0_type = params[0].type + if ( + isinstance(p0_type, Type) + and len(p0_type.typename.segments) == 1 + and getattr(p0_type.typename.segments[0], "name", None) == "void" + ): + params = [] + + return params, vararg + + def _parse_fn_end(self, fn: Function) -> None: + """ + Consumes the various keywords after the parameters in a function + declaration, and definition if present. + """ + + if self.lex.token_if("throw"): + tok = self._next_token_must_be("(") + fn.throw = self._create_value(self._consume_balanced_items(tok)) + + elif self.lex.token_if("noexcept"): + toks = [] + tok = self.lex.token_if("(") + if tok: + toks = self._consume_balanced_tokens(tok)[1:-1] + fn.noexcept = self._create_value(toks) + + if self.lex.token_if("{"): + self._discard_contents("{", "}") + fn.has_body = True + + def _parse_method_end(self, method: Method) -> None: + """ + Consumes the various keywords after the parameters in a method + declaration, and definition if present. + """ + + # various keywords at the end of a method + get_token = self.lex.token + + while True: + tok = get_token() + tok_value = tok.value + + if tok_value in (":", "{"): + method.has_body = True + + if tok_value == ":": + self._discard_ctor_initializer() + elif tok_value == "{": + self._discard_contents("{", "}") + + break + + elif tok_value == "=": + tok = get_token() + tok_value = tok.value + + if tok_value == "0": + method.pure_virtual = True + elif tok_value == "delete": + method.deleted = True + elif tok_value == "default": + method.default = True + else: + raise self._parse_error(tok, "0/delete/default") + + break + elif tok_value in ("const", "volatile", "override", "final"): + setattr(method, tok_value, True) + elif tok_value in ("&", "&&"): + method.ref_qualifier = tok_value + elif tok_value == "throw": + tok = self._next_token_must_be("(") + method.throw = self._create_value(self._consume_balanced_tokens(tok)) + elif tok_value == "noexcept": + toks = [] + tok = self.lex.token_if("(") + if tok: + toks = self._consume_balanced_tokens(tok)[1:-1] + method.noexcept = self._create_value(toks) + else: + self.lex.return_token(tok) + break + + def _parse_function( + self, + mods: ParsedTypeModifiers, + return_type: typing.Optional[DecoratedType], + pqname: PQName, + op: typing.Optional[str], + template: typing.Optional[TemplateDecl], + doxygen: typing.Optional[str], + location: Location, + constructor: bool, + destructor: bool, + is_friend: bool, + ) -> bool: + """ + Assumes the caller has already consumed the return type and name, this consumes the + rest of the function including the definition if present + + Returns True if function has a body and it was consumed + """ + + # TODO: explicit (operator int)() in C++20 + + # last segment must be NameSpecifier + if not isinstance(pqname.segments[-1], NameSpecifier): + raise self._parse_error(None) + + props = dict.fromkeys(mods.both.keys(), True) + + state = self.state + state.location = location + is_class_block = isinstance(state, ClassBlockState) + + params, vararg = self._parse_parameters() + + if is_class_block: + props.update(dict.fromkeys(mods.meths.keys(), True)) + + if op: + method = Operator( + return_type, + pqname, + params, + vararg, + doxygen=doxygen, + operator=op, + template=template, + access=self._current_access, + **props, + ) + else: + method = Method( + return_type, + pqname, + params, + vararg, + doxygen=doxygen, + constructor=constructor, + destructor=destructor, + template=template, + access=self._current_access, + **props, + ) + + self._parse_method_end(method) + + if is_friend: + friend = FriendDecl(fn=method) + self.visitor.on_class_friend(state, friend) + else: + # method must not have multiple segments + if len(pqname.segments) != 1: + raise self._parse_error(None) + + self.visitor.on_class_method(state, method) + + return method.has_body + + else: + fn = Function( + return_type, + pqname, + params, + vararg, + doxygen=doxygen, + template=template, + **props, + ) + self._parse_fn_end(fn) + self.visitor.on_function(state, fn) + + return fn.has_body + + # + # Decorated type parsing + # + + def _parse_array_type(self, tok: LexToken, dtype: DecoratedType) -> Array: + + assert tok.type == "[" + + toks = self._consume_balanced_tokens(tok) + tok = self.lex.token_if("[") + if tok: + # recurses because array types are right to left + dtype = self._parse_array_type(tok, dtype) + + toks = toks[1:-1] + if toks: + value = self._create_value(toks) + else: + value = None + + return Array(dtype, value) + + def _parse_cv_ptr( + self, dtype: typing.Union[Array, FunctionType, Pointer, Type] + ) -> DecoratedType: + + while True: + tok = self.lex.token_if("*", "const", "volatile", "(") + if not tok: + break + + if tok.type == "*": + dtype = Pointer(dtype) + elif tok.type == "const": + dtype.const = True + elif tok.type == "volatile": + dtype.volatile = True + else: + # Check to see if this is a grouping paren or something else + if not self.lex.token_peek_if("*", "&"): + self.lex.return_token(tok) + break + + # this is a grouping paren, so consume it + toks = self._consume_balanced_tokens(tok) + + # Now check to see if we have either an array or a function pointer + aptok = self.lex.token_if("[", "(") + if aptok: + if aptok.type == "[": + dtype = self._parse_array_type(aptok, dtype) + elif aptok.type == "(": + fn_params, vararg = self._parse_parameters() + # the type we already have is the return type of the function pointer + + dtype = FunctionType(dtype, fn_params, vararg) + + # the inner tokens must either be a * or a pqname that ends + # with ::* (member function pointer) + # ... TODO member function pointer :( + + # return the inner toks and recurse + # -> this could return some weird results for invalid code, but + # we don't support that anyways so it's fine? + self.lex.return_tokens(toks[1:-1]) + dtype = self._parse_cv_ptr(dtype) + break + + tok = self.lex.token_if("&", "DBL_AMP") + if tok: + if tok.type == "&": + dtype = Reference(dtype) + else: + dtype = MoveReference(dtype) + + return dtype + + # Applies to variables and return values + _type_kwd_both = {"const", "constexpr", "extern", "inline", "static", "volatile"} + + # Only found on methods + _type_kwd_meth = {"explicit", "virtual"} + + _parse_type_ptr_ref_paren = {"*", "&", "DBL_AMP", "("} + + def _parse_type( + self, + tok: typing.Optional[LexToken], + ) -> typing.Tuple[Type, ParsedTypeModifiers]: + """ + This parses a typename and stops parsing when it hits something + that it doesn't understand. The caller uses the results to figure + out what got parsed + + This only parses the base type, does not parse pointers, references, + or additional const/volatile qualifiers + """ + + const = False + volatile = False + + # Modifiers that apply to the variable/function + # -> key is name of modifier, value is a token so that we can emit an + # appropriate error + + vars: typing.Dict[str, LexToken] = {} # only found on variables + both: typing.Dict[str, LexToken] = {} # found on either + meths: typing.Dict[str, LexToken] = {} # only found on methods + + get_token = self.lex.token + + if not tok: + tok = get_token() + + pqname: PQName = None + + _pqname_start_tokens = self._pqname_start_tokens + + # This loop parses until it finds two pqname or ptr/ref + while True: + tok_type = tok.type + if tok_type in _pqname_start_tokens: + if pqname is not None: + # found second set of names, done here + break + pqname, _ = self._parse_pqname( + tok, compound_ok=True, fn_ok=False, fund_ok=True + ) + elif tok_type in self._parse_type_ptr_ref_paren: + if pqname is None: + raise self._parse_error(tok) + break + elif tok_type == "const": + const = True + elif tok_type in self._type_kwd_both: + if tok_type == "extern": + # TODO: store linkage + self.lex.token_if("STRING_LITERAL") + both[tok_type] = tok + elif tok_type in self._type_kwd_meth: + meths[tok_type] = tok + elif tok_type == "mutable": + vars["mutable"] = tok + elif tok_type == "volatile": + volatile = True + else: + if pqname is None: + raise self._parse_error(tok) + break + + tok = get_token() + + self.lex.return_token(tok) + + # Construct a type from the parsed name + parsed_type = Type(pqname, const, volatile) + mods = ParsedTypeModifiers(vars, both, meths) + + return parsed_type, mods + + def _parse_decl( + self, + parsed_type: Type, + mods: ParsedTypeModifiers, + location: Location, + doxygen: typing.Optional[str], + template: typing.Optional[TemplateDecl], + is_typedef: bool, + is_friend: bool, + ) -> bool: + toks = [] + + # On entry we only have the base type, decorate it + dtype = self._parse_cv_ptr(parsed_type) + + state = self.state + + pqname = None + constructor = False + destructor = False + op = None + + # If we have a leading (, that's either an obnoxious grouping + # paren or it's a constructor + tok = self.lex.token_if("(") + if tok: + + # Check to see if this is a constructor/destructor + if isinstance(state, ClassBlockState) and isinstance(dtype, Type): + + dsegments = dtype.typename.segments + + if not is_friend: + # class name to match against is this class + cls_name = getattr( + state.class_decl.typename.segments[-1], "name", None + ) + else: + # class name to match against is the other class + if len(dsegments) >= 2: + cls_name = getattr(dsegments[-2], "name", None) + else: + cls_name = None + + ret_name = getattr(dsegments[-1], "name", None) + + if cls_name: + if cls_name == ret_name: + pqname = dtype.typename + dtype = None + constructor = True + self.lex.return_token(tok) + elif f"~{cls_name}" == ret_name: + pqname = dtype.typename + dtype = None + destructor = True + self.lex.return_token(tok) + + if dtype: + # if it's not a constructor/destructor, it could be a + # grouping paren like "void (name(int x));" + toks = self._consume_balanced_tokens(tok) + + # .. not sure what it's grouping, so put it back? + self.lex.return_tokens(toks[1:-1]) + + if dtype: + tok = self.lex.token_if_in_set(self._pqname_start_tokens) + if tok: + pqname, op = self._parse_pqname(tok, fn_ok=True) + + # TODO: "type fn(x);" is ambiguous here. Because this is a header + # parser, we assume it's a function, not a variable declaration + # calling a constructor + + # if ( then it's a function/method + if self.lex.token_if("("): + if is_typedef: + raise self._parse_error(None) + + return self._parse_function( + mods, + dtype, + pqname, + op, + template, + doxygen, + location, + constructor, + destructor, + is_friend, + ) + + # anything else is a field/variable + if is_friend: + # "friend Foo;" + if not self.lex.token_if(";"): + raise self._parse_error(None) + + fwd = ForwardDecl( + parsed_type.typename, + template, + doxygen, + access=self._current_access, + ) + friend = FriendDecl(cls=fwd) + state.location = location + self.visitor.on_class_friend(state, friend) + return True + if op: + raise self._parse_error(None) + + self._parse_field(mods, dtype, pqname, template, doxygen, location, is_typedef) + return False + + _class_enum_stage2 = {":", "final", "explicit", "{"} + + def _parse_declarations( + self, + tok: LexToken, + doxygen: typing.Optional[str], + template: typing.Optional[TemplateDecl] = None, + is_typedef: bool = False, + is_friend: bool = False, + ) -> None: + """ + Parses a sequence of declarations + """ + + # On entry to this function, we don't have enough context to know + # what we're going to find, so keep trying to deduce it + + location = tok.location + + # Always starts out with some kind of type name + parsed_type, mods = self._parse_type(tok) + + # Check to see if this might be a class/enum declaration + if parsed_type.typename.classkey and self._maybe_parse_class_enum_decl( + parsed_type, mods, doxygen, template, is_typedef, is_friend, location + ): + return + + var_ok = True + + if is_typedef: + var_ok = False + meth_ok = False + msg = "parsing typedef" + elif isinstance(self.state, ClassBlockState): + meth_ok = True + msg = "parsing declaration in class" + else: + meth_ok = False + msg = "parsing declaration" + + mods.validate(var_ok=var_ok, meth_ok=meth_ok, msg=msg) + + # Ok, dealing with a variable or function/method + while True: + if self._parse_decl( + parsed_type, mods, location, doxygen, template, is_typedef, is_friend + ): + # if it returns True then it handled the end of the statement + break + + # Unset the doxygen, location + doxygen = None + + # Check for multiple declarations + tok = self._next_token_must_be(",", ";") + location = tok.location + if tok.type == ";": + break + + def _maybe_parse_class_enum_decl( + self, + parsed_type: Type, + mods: ParsedTypeModifiers, + doxygen: typing.Optional[str], + template: typing.Optional[TemplateDecl], + is_typedef: bool, + is_friend: bool, + location: Location, + ) -> bool: + + # check for forward declaration or friend declaration + if self.lex.token_if(";"): + if is_typedef: + raise self._parse_error(None) + + classkey = parsed_type.typename.classkey + mods.validate( + var_ok=False, + meth_ok=False, + msg=f"parsing {classkey} forward declaration", + ) + + # enum cannot be forward declared, but "enum class" can + # -> but `friend enum X` is fine + if classkey == "enum" and not is_friend: + raise self._parse_error(None) + elif template and classkey.startswith("enum"): + # enum class cannot have a template + raise self._parse_error(None) + + fdecl = ForwardDecl( + parsed_type.typename, template, doxygen, access=self._current_access + ) + self.state.location = location + if is_friend: + friend = FriendDecl(cls=fdecl) + self.visitor.on_class_friend(self.state, friend) + else: + self.visitor.on_forward_decl(self.state, fdecl) + return True + + tok = self.lex.token_if_in_set(self._class_enum_stage2) + if tok: + + classkey = parsed_type.typename.classkey + # var is ok because it could be carried on to any variables + mods.validate( + var_ok=not is_typedef, + meth_ok=False, + msg=f"parsing {classkey} declaration", + ) + typename = parsed_type.typename + + if is_friend: + # friend declaration doesn't have extra context + raise self._parse_error(tok) + + if typename.classkey in ("class", "struct", "union"): + self._parse_class_decl( + typename, tok, doxygen, template, is_typedef, location, mods + ) + else: + if template: + # enum cannot have a template + raise self._parse_error(tok) + self._parse_enum_decl( + typename, tok, doxygen, is_typedef, location, mods + ) + + return True + + # Otherwise this must be a variable or function + return False + + def _finish_class_or_enum( + self, + name: PQName, + is_typedef: bool, + mods: ParsedTypeModifiers, + ) -> None: + parsed_type = Type(name) + + tok = self.lex.token_if("__attribute__") + if tok: + self._consume_gcc_attribute(tok) + + if not is_typedef and self.lex.token_if(";"): + return + + while True: + location = self.lex.current_location() + if self._parse_decl( + parsed_type, mods, location, None, None, is_typedef, False + ): + # if it returns True then it handled the end of the statement + break + + # Check for multiple declarations + tok = self._next_token_must_be(",", ";") + if tok.type == ";": + break diff --git a/cxxheaderparser/parserstate.py b/cxxheaderparser/parserstate.py new file mode 100644 index 0000000..46abe33 --- /dev/null +++ b/cxxheaderparser/parserstate.py @@ -0,0 +1,114 @@ +import typing + +if typing.TYPE_CHECKING: + from .visitor import CxxVisitor + +from .errors import CxxParseError +from .lexer import LexToken, Location +from .types import ClassDecl, NamespaceDecl + + +class ParsedTypeModifiers(typing.NamedTuple): + vars: typing.Dict[str, LexToken] # only found on variables + both: typing.Dict[str, LexToken] # found on either variables or functions + meths: typing.Dict[str, LexToken] # only found on methods + + def validate(self, *, var_ok: bool, meth_ok: bool, msg: str) -> None: + # Almost there! Do any checks the caller asked for + if not var_ok and self.vars: + for tok in self.vars.values(): + raise CxxParseError(f"{msg}: unexpected '{tok.value}'") + + if not meth_ok and self.meths: + for tok in self.meths.values(): + raise CxxParseError(f"{msg}: unexpected '{tok.value}'") + + if not meth_ok and not var_ok and self.both: + for tok in self.both.values(): + raise CxxParseError(f"{msg}: unexpected '{tok.value}'") + + +class State: + + #: parent state + parent: typing.Optional["State"] + + def __init__(self, parent: typing.Optional["State"]) -> None: + self.parent = parent + + def _finish(self, visitor: "CxxVisitor") -> None: + pass + + +class BlockState(State): + + #: Approximate location that the parsed element was found at + location: Location + + +class EmptyBlockState(BlockState): + def _finish(self, visitor: "CxxVisitor") -> None: + visitor.on_empty_block_end(self) + + +class ExternBlockState(BlockState): + + #: The linkage for this extern block + linkage: str + + def __init__(self, parent: typing.Optional[State], linkage: str) -> None: + super().__init__(parent) + self.linkage = linkage + + def _finish(self, visitor: "CxxVisitor"): + visitor.on_extern_block_end(self) + + +class NamespaceBlockState(BlockState): + + #: The incremental namespace for this block + namespace: NamespaceDecl + + def __init__( + self, parent: typing.Optional[State], namespace: NamespaceDecl + ) -> None: + super().__init__(parent) + self.namespace = namespace + + def _finish(self, visitor: "CxxVisitor") -> None: + visitor.on_namespace_end(self) + + +class ClassBlockState(BlockState): + + #: class decl block being processed + class_decl: ClassDecl + + #: Current access level for items encountered + access: str + + #: Currently parsing as a typedef + typedef: bool + + #: modifiers to apply to following variables + mods: ParsedTypeModifiers + + def __init__( + self, + parent: typing.Optional[State], + class_decl: ClassDecl, + access: str, + typedef: bool, + mods: ParsedTypeModifiers, + ) -> None: + super().__init__(parent) + self.class_decl = class_decl + self.access = access + self.typedef = typedef + self.mods = mods + + def _set_access(self, access: str) -> None: + self.access = access + + def _finish(self, visitor: "CxxVisitor") -> None: + visitor.on_class_end(self) diff --git a/cxxheaderparser/simple.py b/cxxheaderparser/simple.py new file mode 100644 index 0000000..7230ddb --- /dev/null +++ b/cxxheaderparser/simple.py @@ -0,0 +1,294 @@ +""" + +The simple parser/collector iterates over the C++ file and returns a data +structure with all elements in it. Not quite as flexible as implementing +your own parser listener, but you can accomplish most things with it. + +cxxheaderparser's unit tests predominantly use the simple API for parsing, +so you can expect it to be pretty stable. + +""" + +import inspect +import typing + + +from dataclasses import dataclass, field + +from .types import ( + ClassDecl, + EnumDecl, + Field, + ForwardDecl, + FriendDecl, + Function, + Method, + Typedef, + UsingAlias, + UsingDecl, + Variable, +) + +from .parserstate import ( + State, + EmptyBlockState, + ClassBlockState, + ExternBlockState, + NamespaceBlockState, +) +from .parser import CxxParser +from .options import ParserOptions + +# +# Data structure +# + + +@dataclass +class ClassScope: + + class_decl: ClassDecl + + #: Nested classes + classes: typing.List["ClassScope"] = field(default_factory=list) + enums: typing.List[EnumDecl] = field(default_factory=list) + fields: typing.List[Field] = field(default_factory=list) + friends: typing.List[FriendDecl] = field(default_factory=list) + methods: typing.List[Method] = field(default_factory=list) + typedefs: typing.List[Typedef] = field(default_factory=list) + + forward_decls: typing.List[ForwardDecl] = field(default_factory=list) + using: typing.List[UsingDecl] = field(default_factory=list) + using_alias: typing.List[UsingAlias] = field(default_factory=list) + + +@dataclass +class NamespaceScope: + + name: str = "" + + classes: typing.List["ClassScope"] = field(default_factory=list) + enums: typing.List[EnumDecl] = field(default_factory=list) + functions: typing.List[Method] = field(default_factory=list) + typedefs: typing.List[Typedef] = field(default_factory=list) + variables: typing.List[Variable] = field(default_factory=list) + + forward_decls: typing.List[ForwardDecl] = field(default_factory=list) + using: typing.List[UsingDecl] = field(default_factory=list) + using_ns: typing.List[UsingDecl] = field(default_factory=list) + using_alias: typing.List[UsingAlias] = field(default_factory=list) + + #: Child namespaces + namespaces: typing.Dict[str, "NamespaceScope"] = field(default_factory=dict) + + +Block = typing.Union[ClassScope, NamespaceScope] + + +@dataclass +class Define: + content: str + + +@dataclass +class Pragma: + content: str + + +@dataclass +class Include: + #: The filename includes the surrounding ``<>`` or ``"`` + filename: str + + +@dataclass +class UsingNamespace: + ns: str + + +@dataclass +class ParsedData: + + namespace: NamespaceScope = field(default_factory=lambda: NamespaceScope()) + + defines: typing.List[Define] = field(default_factory=list) + pragmas: typing.List[Pragma] = field(default_factory=list) + includes: typing.List[Include] = field(default_factory=list) + + +# +# Visitor implementation +# + + +class SimpleCxxVisitor: + """ + A simple visitor that stores all of the C++ elements passed to it + in an "easy" to use data structure + + .. warning:: Names are not resolved, so items are stored in the scope that + they are found. For example: + + .. code-block:: c++ + + namespace N { + class C; + } + + class N::C { + void fn(); + }; + + The 'C' class would be a forward declaration in the 'N' namespace, + but the ClassDecl for 'C' would be stored in the global + namespace instead of the 'N' namespace. + """ + + data: ParsedData + namespace: NamespaceScope + block: Block + + def __init__(self): + self.namespace = NamespaceScope("") + self.block = self.namespace + + self.ns_stack = typing.Deque[NamespaceScope]() + self.block_stack = typing.Deque[Block]() + + self.data = ParsedData(self.namespace) + + def on_define(self, state: State, content: str) -> None: + self.data.defines.append(Define(content)) + + def on_pragma(self, state: State, content: str) -> None: + self.data.pragmas.append(Pragma(content)) + + def on_include(self, state: State, filename: str) -> None: + self.data.includes.append(Include(filename)) + + def on_empty_block_start(self, state: EmptyBlockState) -> None: + # this matters for some scope/resolving purposes, but you're + # probably going to want to use clang if you care about that + # level of detail + pass + + def on_empty_block_end(self, state: EmptyBlockState) -> None: + pass + + def on_extern_block_start(self, state: ExternBlockState) -> None: + pass # TODO + + def on_extern_block_end(self, state: ExternBlockState) -> None: + pass + + def on_namespace_start(self, state: NamespaceBlockState) -> None: + parent_ns = self.namespace + self.block_stack.append(parent_ns) + self.ns_stack.append(parent_ns) + + ns = None + names = state.namespace.names + if not names: + # all anonymous namespaces in a translation unit are the same + names = [""] + + for name in names: + ns = parent_ns.namespaces.get(name) + if ns is None: + ns = NamespaceScope(name) + parent_ns.namespaces[name] = ns + parent_ns = ns + + self.block = ns + self.namespace = ns + + def on_namespace_end(self, state: NamespaceBlockState) -> None: + self.block = self.block_stack.pop() + self.namespace = self.ns_stack.pop() + + def on_forward_decl(self, state: State, fdecl: ForwardDecl) -> None: + self.block.forward_decls.append(fdecl) + + def on_variable(self, state: State, v: Variable) -> None: + self.block.variables.append(v) + + def on_function(self, state: State, fn: Function) -> None: + self.block.functions.append(fn) + + def on_typedef(self, state: State, typedef: Typedef) -> None: + self.block.typedefs.append(typedef) + + def on_using_namespace(self, state: State, namespace: typing.List[str]) -> None: + ns = UsingNamespace("::".join(namespace)) + self.block.using_ns.append(ns) + + def on_using_alias(self, state: State, using: UsingAlias): + self.block.using_alias.append(using) + + def on_using_declaration(self, state: State, using: UsingDecl) -> None: + self.block.using.append(using) + + # + # Enums + # + + def on_enum(self, state: State, enum: EnumDecl) -> None: + self.block.enums.append(enum) + + # + # Class/union/struct + # + + def on_class_start(self, state: ClassBlockState) -> None: + block = ClassScope(state.class_decl) + self.block.classes.append(block) + self.block_stack.append(self.block) + self.block = block + + def on_class_field(self, state: State, f: Field) -> None: + self.block.fields.append(f) + + def on_class_method(self, state: ClassBlockState, method: Method) -> None: + self.block.methods.append(method) + + def on_class_friend(self, state: ClassBlockState, friend: FriendDecl): + self.block.friends.append(friend) + + def on_class_end(self, state: ClassBlockState) -> None: + self.block = self.block_stack.pop() + + +def parse_string( + content: str, + *, + filename="", + options: typing.Optional[ParserOptions] = None, + cleandoc: bool = False, +) -> ParsedData: + """ + Simple function to parse a header and return a data structure + """ + if cleandoc: + content = inspect.cleandoc(content) + + visitor = SimpleCxxVisitor() + parser = CxxParser(filename, content, visitor, options) + parser.parse() + + return visitor.data + + +def parse_file( + filename: str, + encoding: typing.Optional[str] = None, + *, + options: typing.Optional[ParserOptions] = None, +) -> ParsedData: + """ + Simple function to parse a header from a file and return a data structure + """ + + with open(filename, encoding=encoding) as fp: + content = fp.read() + + return parse_string(content, filename=filename, options=options) diff --git a/cxxheaderparser/tokfmt.py b/cxxheaderparser/tokfmt.py new file mode 100644 index 0000000..2072f48 --- /dev/null +++ b/cxxheaderparser/tokfmt.py @@ -0,0 +1,74 @@ +import typing + +from .lexer import Lexer +from .types import Token + +# key: token type, value: (left spacing, right spacing) +_want_spacing = { + "NUMBER": (2, 2), + "FLOAT_NUMBER": (2, 2), + "NAME": (2, 2), + "CHAR_LITERAL": (2, 2), + "STRING_LITERAL": (2, 2), + "ELLIPSIS": (2, 2), + ">": (0, 2), + ")": (0, 1), + "(": (1, 0), + ",": (0, 3), + "*": (1, 2), + "&": (0, 2), +} + +_want_spacing.update(dict.fromkeys(Lexer.keywords, (2, 2))) + + +def tokfmt(toks: typing.List[Token]) -> str: + """ + Helper function that takes a list of tokens and converts them to a string + """ + last = 0 + vals = [] + default = (0, 0) + ws = _want_spacing + + for tok in toks: + value = tok.value + # special case + if value == "operator": + l, r = 2, 0 + else: + l, r = ws.get(tok.type, default) + if l + last >= 3: + vals.append(" ") + + last = r + vals.append(value) + + return "".join(vals) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("header") + args = parser.parse_args() + + lexer = Lexer(args.header) + with open(lexer.filename) as fp: + lexer.input(fp.read()) + + toks = [] + while True: + tok = lexer.token_eof_ok() + if not tok: + break + if tok.type == ";": + print(toks) + print(tokfmt(toks)) + toks = [] + else: + toks.append(tok) + + print(toks) + print(tokfmt(toks)) diff --git a/cxxheaderparser/types.py b/cxxheaderparser/types.py new file mode 100644 index 0000000..287fb22 --- /dev/null +++ b/cxxheaderparser/types.py @@ -0,0 +1,650 @@ +import typing +from dataclasses import dataclass, field + + +@dataclass +class Token: + """ + In an ideal world, this Token class would not be exposed via the user + visible API. Unfortunately, getting to that point would take a significant + amount of effort. + + It is not expected that these will change, but they might. + + At the moment, the only supported use of Token objects are in conjunction + with the ``tokfmt`` function. As this library matures, we'll try to clarify + the expectations around these. File an issue on github if you have ideas! + """ + + #: Raw value of the token + value: str + + #: Lex type of the token + type: str = field(repr=False, compare=False, default="") + + +@dataclass +class Value: + """ + A unparsed list of tokens + + .. code-block:: c++ + + int x = 0x1337; + ~~~~~~ + """ + + #: Tokens corresponding to the value + tokens: typing.List[Token] + + +@dataclass +class NamespaceDecl: + """ + Namespace declarations + + .. code-block:: c++ + + namespace foo::bar {} + ~~~~~~~~ + """ + + #: These are the names (split by ::) for this namespace declaration, + #: but does not include any parent namespace names + #: + #: An anonymous namespace is an empty list + names: typing.List[str] + inline: bool = False + + +@dataclass +class DecltypeSpecifier: + """ + Contents of a decltype (inside the parentheses) + + .. code-block:: c++ + + decltype(Foo::Bar) + ~~~~~~~~ + """ + + #: Unparsed tokens within the decltype + tokens: typing.List[Token] + + +@dataclass +class FundamentalSpecifier: + """ + A specifier that only contains fundamental types + """ + + name: str + + +@dataclass +class NameSpecifier: + """ + An individual segment of a type name + + .. code-block:: c++ + + Foo::Bar + ~~~ + + """ + + name: str + + specialization: typing.Optional[typing.List["TemplateSpecialization"]] = None + + +@dataclass +class AutoSpecifier: + """ + Used for an auto return type + """ + + name: str = "auto" + + +@dataclass +class AnonymousName: + """ + A name for an anonymous class, such as in a typedef. There is no string + associated with this name, only an integer id. Things that share the same + anonymous name have anonymous name instances with the same id + """ + + #: Unique id associated with this name (only unique per parser instance!) + id: int + + +PQNameSegment = typing.Union[ + AnonymousName, FundamentalSpecifier, NameSpecifier, DecltypeSpecifier, AutoSpecifier +] + + +@dataclass +class PQName: + """ + Possibly qualified name of a C++ type. + """ + + #: All of the segments of the name. This is always guaranteed to have at + #: least one element in it. Name is segmented by '::' + #: + #: If a name refers to the global namespace, the first segment will be an + #: empty NameSpecifier + segments: typing.List[PQNameSegment] + + #: Set if the name starts with class/enum/struct + classkey: typing.Optional[str] = None + + +@dataclass +class Enumerator: + """ + An individual value of an enumeration + """ + + #: The enumerator key name + name: str + + #: None if not explicitly specified + value: typing.Optional[Value] = None + + #: Documentation if present + doxygen: typing.Optional[str] = None + + +@dataclass +class EnumDecl: + """ + An enumeration type + """ + + typename: PQName + + values: typing.List[Enumerator] + + base: typing.Optional[PQName] = None + + #: Documentation if present + doxygen: typing.Optional[str] = None + + #: If within a class, the access level for this decl + access: typing.Optional[str] = None + + +@dataclass +class TemplateArgument: + """ + A single argument for a template specialization + + .. code-block:: c++ + + Foo + ~~~ + + """ + + #: This contains unparsed arbitrary expressions, including additional + #: specializations or decltypes or whatever + tokens: typing.List[Token] + + +@dataclass +class TemplateSpecialization: + """ + Contains the arguments of a template specialization + + .. code-block:: c++s + + Foo + ~~~~~~~~~~~ + + """ + + args: typing.List[TemplateArgument] + + #: If True, indicates a parameter pack (...) on the last parameter + param_pack: bool = False + + +@dataclass +class FunctionType: + """ + A function type, currently only used in a function pointer + + .. note:: There can only be one of FunctionType or Type in a DecoratedType + chain + """ + + return_type: "DecoratedType" + parameters: typing.List["Parameter"] + + #: If a member function pointer + # TODO classname: typing.Optional[PQName] + + #: Set to True if ends with ``...`` + vararg: bool = False + + +@dataclass +class Type: + """""" + + typename: PQName + + const: bool = False + volatile: bool = False + + def get_type(self) -> "Type": + return self + + +@dataclass +class Array: + """ + Information about an array. Multidimensional arrays are represented as + an array of array. + """ + + #: The type that this is an array of + array_of: typing.Union["Array", "Pointer", Type] + + #: Size of the array + #: + #: .. code-block:: c++ + #: + #: int x[10]; + #: ~~ + size: typing.Optional[Value] + + def get_type(self) -> Type: + return self.array_of.get_type() + + +@dataclass +class Pointer: + """ + A pointer + """ + + #: Thing that this points to + ptr_to: typing.Union[Array, FunctionType, "Pointer", Type] + + const: bool = False + volatile: bool = False + + def get_type(self) -> Type: + return self.ptr_to.get_type() + + +@dataclass +class Reference: + """ + A lvalue (``&``) reference + """ + + ref_to: typing.Union[Array, Pointer, Type] + + def get_type(self) -> Type: + return self.ref_to.get_type() + + +@dataclass +class MoveReference: + """ + An rvalue (``&&``) reference + """ + + moveref_to: typing.Union[Array, Pointer, Type] + + def get_type(self) -> Type: + return self.moveref_to.get_type() + + +#: A type or function type that is decorated with various things +#: +#: .. note:: There can only be one of FunctionType or Type in a DecoratedType +#: chain +DecoratedType = typing.Union[Array, Pointer, MoveReference, Reference, Type] + + +@dataclass +class TemplateNonTypeParam: + """ + + .. code-block:: c++ + + template + ~~~~~ + + template + ~~~~~~~~~~~~~~~~~~~ + + template + ~~~~~~ + """ + + type: DecoratedType + name: typing.Optional[str] = None + default: typing.Optional[Value] = None + + #: Contains a ``...`` + param_pack: bool = False + + +@dataclass +class TemplateTypeParam: + """ + + .. code-block:: c++ + + template + ~~~~~~~~~~ + """ + + #: 'typename' or 'class' + typekey: str + + name: typing.Optional[str] = None + + param_pack: bool = False + + default: typing.Optional[Value] = None + + #: A template-template param + template: typing.Optional["TemplateDecl"] = None + + +#: A parameter for a template declaration +#: +#: .. code-block:: c++ +#: +#: template +#: ~~~~~~~~~~ +TemplateParam = typing.Union[TemplateNonTypeParam, TemplateTypeParam] + + +@dataclass +class TemplateDecl: + """ + Template declaration for a function or class + + .. code-block:: c++ + + template + class Foo {}; + + template + T fn(); + + """ + + params: typing.List[TemplateParam] = field(default_factory=list) + + +@dataclass +class ForwardDecl: + """ + Represents a forward declaration of a user defined type + """ + + typename: PQName + template: typing.Optional[TemplateDecl] = None + doxygen: typing.Optional[str] = None + + #: Set if this is a forward declaration of an enum and it has a base + enum_base: typing.Optional[PQName] = None + + #: If within a class, the access level for this decl + access: typing.Optional[str] = None + + +@dataclass +class BaseClass: + """ + Base class declarations for a class + """ + + #: access specifier for this base + access: str + + #: possibly qualified type name for the base + typename: PQName + + #: Virtual inheritance + virtual: bool = False + + #: Contains a ``...`` + param_pack: bool = False + + +@dataclass +class ClassDecl: + """ + A class is a user defined type (class/struct/union) + """ + + typename: PQName + + bases: typing.List[BaseClass] = field(default_factory=list) + template: typing.Optional[TemplateDecl] = None + + explicit: bool = False + final: bool = False + + doxygen: typing.Optional[str] = None + + #: If within a class, the access level for this decl + access: typing.Optional[str] = None + + @property + def classkey(self) -> str: + return self.typename.classkey + + +@dataclass +class Parameter: + """ + A parameter of a function/method + """ + + type: DecoratedType + name: typing.Optional[str] = None + default: typing.Optional[Value] = None + param_pack: bool = False + + +@dataclass +class Function: + """ + A function declaration, potentially with the function body + """ + + return_type: DecoratedType + name: PQName + parameters: typing.List[Parameter] + + #: Set to True if ends with ``...`` + vararg: bool = False + + doxygen: typing.Optional[str] = None + + constexpr: bool = False + extern: typing.Union[bool, str] = False + static: bool = False + inline: bool = False + + #: If true, the body of the function is present + has_body: bool = False + + template: typing.Optional[TemplateDecl] = None + + throw: typing.Optional[Value] = None + noexcept: typing.Optional[Value] = None + + +@dataclass +class Method(Function): + """ + A method declaration, potentially with the method body + """ + + #: constructors and destructors don't have a return type + return_type: typing.Optional[DecoratedType] + + access: str = "" + + const: bool = False + volatile: bool = False + + #: ref-qualifier for this method, either lvalue (&) or rvalue (&&) + #: + #: .. code-block:: c++ + #: + #: void foo() &&; + #: ~~ + #: + ref_qualifier: typing.Optional[str] = None + + constructor: bool = False + explicit: bool = False + default: bool = False + deleted: bool = False + + destructor: bool = False + + pure_virtual: bool = False + virtual: bool = False + final: bool = False + override: bool = False + + +@dataclass +class Operator(Method): + operator: str = "" + + +@dataclass +class FriendDecl: + """ + Represents a friend declaration -- friends can only be classes or functions + """ + + cls: typing.Optional[ForwardDecl] = None + + fn: typing.Optional[Function] = None + + +@dataclass +class Typedef: + """ + A typedef specifier. A unique typedef specifier is created for each alias + created by the typedef. + + .. code-block:: c++ + + typedef type name, *pname; + + """ + + #: The aliased type + #: + #: .. code-block:: c++ + #: + #: typedef type *pname; + #: ~~~~~~ + type: DecoratedType + + #: The alias introduced for the specified type + #: + #: .. code-block:: c++ + #: + #: typedef type *pname; + #: ~~~~~ + name: str + + #: If within a class, the access level for this decl + access: typing.Optional[str] = None + + +@dataclass +class Variable: + """ + A variable declaration + """ + + name: PQName + type: DecoratedType + + value: typing.Optional[Value] = None + + constexpr: bool = False + extern: typing.Union[bool, str] = False + static: bool = False + inline: bool = False + + #: Can occur for a static variable for a templated class + template: typing.Optional[TemplateDecl] = None + + doxygen: typing.Optional[str] = None + + +@dataclass +class Field: + """ + A field of a class + """ + + #: public/private/protected + access: str + + type: DecoratedType + name: typing.Optional[str] = None + + value: typing.Optional[Value] = None + bits: typing.Optional[int] = None + + constexpr: bool = False + mutable: bool = False + static: bool = False + + doxygen: typing.Optional[str] = None + + +@dataclass +class UsingDecl: + """ + .. code-block:: c++ + + using NS::ClassName; + """ + + typename: PQName + + #: If within a class, the access level for this decl + access: typing.Optional[str] = None + + +@dataclass +class UsingAlias: + """ + .. code-block:: c++ + + using foo = int; + + template + using VectorT = std::vector; + + """ + + alias: str + type: DecoratedType + + template: typing.Optional[TemplateDecl] = None + + #: If within a class, the access level for this decl + access: typing.Optional[str] = None diff --git a/cxxheaderparser/visitor.py b/cxxheaderparser/visitor.py new file mode 100644 index 0000000..f9243ab --- /dev/null +++ b/cxxheaderparser/visitor.py @@ -0,0 +1,197 @@ +import sys +import typing + +if sys.version_info >= (3, 8): + Protocol = typing.Protocol +else: + Protocol = object + +from .types import ( + EnumDecl, + Field, + ForwardDecl, + FriendDecl, + Function, + Method, + Typedef, + UsingAlias, + UsingDecl, + Variable, +) + +from .parserstate import ( + State, + EmptyBlockState, + ClassBlockState, + ExternBlockState, + NamespaceBlockState, +) + + +class CxxVisitor(Protocol): + """ + Defines the interface used by the parser to emit events + """ + + def on_define(self, state: State, content: str) -> None: + """ + .. warning:: cxxheaderparser intentionally does not have a C preprocessor + implementation. If you are parsing code with macros in it, + use a conforming preprocessor like ``pcpp`` + """ + + def on_pragma(self, state: State, content: str) -> None: + """ + Called once for each ``#pragma`` directive encountered + """ + + def on_include(self, state: State, filename: str) -> None: + """ + Called once for each ``#include`` directive encountered + """ + + def on_empty_block_start(self, state: EmptyBlockState) -> None: + """ + Called when a ``{`` is encountered that isn't associated with or + consumed by other declarations. + + .. code-block:: c++ + + { + // stuff + } + """ + + def on_empty_block_end(self, state: EmptyBlockState) -> None: + ... + + def on_extern_block_start(self, state: ExternBlockState) -> None: + """ + .. code-block:: c++ + + extern "C" { + + } + + """ + + def on_extern_block_end(self, state: ExternBlockState) -> None: + ... + + def on_namespace_start(self, state: NamespaceBlockState) -> None: + """ + Called when a ``namespace`` directive is encountered + """ + + def on_namespace_end(self, state: NamespaceBlockState) -> None: + """ + Called at the end of a ``namespace`` block + """ + + def on_forward_decl(self, state: State, fdecl: ForwardDecl) -> None: + """ + Called when a forward declaration is encountered + """ + + def on_variable(self, state: State, v: Variable) -> None: + ... + + def on_function(self, state: State, fn: Function) -> None: + ... + + def on_typedef(self, state: State, typedef: Typedef) -> None: + """ + Called for each typedef instance encountered. For example: + + .. code-block:: c++ + + typedef int T, *PT; + + Will result in ``on_typedef`` being called twice, once for ``T`` and + once for ``*PT`` + """ + + def on_using_namespace(self, state: State, namespace: typing.List[str]) -> None: + """ + .. code-block:: c++ + + using namespace std; + """ + + def on_using_alias(self, state: State, using: UsingAlias): + """ + .. code-block:: c++ + + using foo = int; + + template + using VectorT = std::vector; + + """ + + def on_using_declaration(self, state: State, using: UsingDecl) -> None: + """ + .. code-block:: c++ + + using NS::ClassName; + + """ + + # + # Enums + # + + def on_enum(self, state: State, enum: EnumDecl) -> None: + """ + Called after an enum is encountered + """ + + # + # Class/union/struct + # + + def on_class_start(self, state: ClassBlockState) -> None: + """ + Called when a class/struct/union is encountered + + When part of a typedef: + + .. code-block:: c++ + + typedef struct { } X; + + This is called first, followed by on_typedef for each typedef instance + encountered. The compound type object is passed as the type to the + typedef. + """ + + def on_class_field(self, state: ClassBlockState, f: Field) -> None: + """ + Called when a field of a class is encountered + """ + + def on_class_friend(self, state: ClassBlockState, friend: FriendDecl): + """ + Called when a friend declaration is encountered + """ + + def on_class_method(self, state: ClassBlockState, method: Method) -> None: + """ + Called when a method of a class is encountered + """ + + def on_class_end(self, state: ClassBlockState) -> None: + """ + Called when the end of a class/struct/union is encountered. + + When a variable like this is declared: + + .. code-block:: c++ + + struct X { + + } x; + + Then ``on_class_start``, .. ``on_class_end`` are emitted, along with + ``on_variable`` for each instance declared. + """ diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..241ca7d --- /dev/null +++ b/setup.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from __future__ import print_function + +from os.path import dirname, exists, join +import sys, subprocess + +from setuptools import find_packages, setup + +setup_dir = dirname(__file__) +git_dir = join(setup_dir, ".git") +version_file = join(setup_dir, "cxxheaderparser", "version.py") + +# Automatically generate a version.py based on the git version +if exists(git_dir): + p = subprocess.Popen( + ["git", "describe", "--tags", "--long", "--dirty=-dirty"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + out, err = p.communicate() + # Make sure the git version has at least one tag + if err: + print("Error: You need to create a tag for this repo to use the builder") + sys.exit(1) + + # Convert git version to PEP440 compliant version + # - Older versions of pip choke on local identifiers, so we can't include the git commit + v, commits, local = out.decode("utf-8").rstrip().split("-", 2) + if commits != "0" or "-dirty" in local: + v = "%s.post0.dev%s" % (v, commits) + + # Create the version.py file + with open(version_file, "w") as fp: + fp.write("# Autogenerated by setup.py\n__version__ = '{0}'".format(v)) + +with open(version_file, "r") as fp: + exec(fp.read(), globals()) + +DESCRIPTION = ( + "Parse C++ header files and generate a data structure representing the class" +) + + +CLASSIFIERS = [ + "Development Status :: 4 - Beta", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: C++", + "License :: OSI Approved :: BSD License", + "Intended Audience :: Developers", + "Topic :: Software Development", + "Topic :: Software Development :: Code Generators", + "Topic :: Software Development :: Compilers", +] + +setup( + name="cxxheaderparser", + version=__version__, + author="Dustin Spicuzza", + author_email="dustin@virtualroadside.com", + maintainer="RobotPy Development Team", + maintainer_email="robotpy@googlegroups.com", + url="https://github.com/robotpy/cxxheaderparser", + description=DESCRIPTION, + long_description=open("README.md").read(), + long_description_content_type="text/markdown", + install_requires=["dataclasses; python_version < '3.7'"], + license="BSD", + platforms="Platform Independent", + packages=find_packages(), + keywords="c++ header parser ply", + python_requires=">= 3.6", + classifiers=CLASSIFIERS, +) diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..0ba917a --- /dev/null +++ b/tests/README.md @@ -0,0 +1,25 @@ +Tests +===== + +To run the tests, install `cxxheaderparser` and `pytest`, then just run: + + pytest + +Adding new tests +---------------- + +There's a helper script in cxxheaderparser explicitly for generating many of the +unit tests in this directory. To run it: + +* Create a file with your C++ content in it +* Run `python -m cxxheaderparser.gentest FILENAME.h some_name` +* Copy the stdout to one of these `test_*.py` files + +Content origin +-------------- + +* Some are scraps of real code derived from various sources +* Some were derived from the original `CppHeaderParser` tests +* Some have been derived from examples found on https://en.cppreference.com, + which are available under Creative Commons Attribution-Sharealike 3.0 + Unported License (CC-BY-SA) diff --git a/tests/test_attributes.py b/tests/test_attributes.py new file mode 100644 index 0000000..a84fd47 --- /dev/null +++ b/tests/test_attributes.py @@ -0,0 +1,148 @@ +# Note: testcases generated via `python -m cxxheaderparser.gentest` + +from cxxheaderparser.types import ( + ClassDecl, + EnumDecl, + Enumerator, + Field, + Function, + FundamentalSpecifier, + NameSpecifier, + PQName, + Pointer, + Token, + Type, + Typedef, + Value, + Variable, +) +from cxxheaderparser.simple import ( + ClassScope, + NamespaceScope, + parse_string, + ParsedData, +) + + +def test_attributes_everywhere(): + # TODO: someday we'll actually support storing attributes, + # but for now just make sure they don't get in the way + + content = """ + struct [[deprecated]] S {}; + [[deprecated]] typedef S *PS; + + [[deprecated]] int x; + union U { + [[deprecated]] int n; + }; + [[deprecated]] void f(); + + enum [[deprecated]] E{A [[deprecated]], B [[deprecated]] = 42}; + + struct alignas(8) AS {}; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="S")], classkey="struct" + ) + ) + ), + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="U")], classkey="union" + ) + ), + fields=[ + Field( + access="public", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="n", + ) + ], + ), + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="AS")], classkey="struct" + ) + ) + ), + ], + enums=[ + EnumDecl( + typename=PQName( + segments=[NameSpecifier(name="E")], classkey="enum" + ), + values=[ + Enumerator(name="A"), + Enumerator(name="B", value=Value(tokens=[Token(value="42")])), + ], + ) + ], + functions=[ + Function( + return_type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="void")]) + ), + name=PQName(segments=[NameSpecifier(name="f")]), + parameters=[], + ) + ], + typedefs=[ + Typedef( + type=Pointer( + ptr_to=Type(typename=PQName(segments=[NameSpecifier(name="S")])) + ), + name="PS", + ) + ], + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="x")]), + type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + ) + ], + ) + ) + + +def test_attributes_gcc_enum_packed(): + content = """ + enum Wheat { + w1, + w2, + w3, + } __attribute__((packed)); + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName( + segments=[NameSpecifier(name="Wheat")], classkey="enum" + ), + values=[ + Enumerator(name="w1"), + Enumerator(name="w2"), + Enumerator(name="w3"), + ], + ) + ] + ) + ) diff --git a/tests/test_class.py b/tests/test_class.py new file mode 100644 index 0000000..d0727df --- /dev/null +++ b/tests/test_class.py @@ -0,0 +1,2843 @@ +# Note: testcases generated via `python -m cxxheaderparser.gentest` + +from cxxheaderparser.types import ( + AnonymousName, + Array, + BaseClass, + ClassDecl, + EnumDecl, + Enumerator, + Field, + ForwardDecl, + Function, + FundamentalSpecifier, + Method, + MoveReference, + NameSpecifier, + Operator, + PQName, + Parameter, + Pointer, + Reference, + TemplateArgument, + TemplateDecl, + TemplateSpecialization, + TemplateTypeParam, + Token, + Type, + Typedef, + UsingDecl, + Value, + Variable, +) +from cxxheaderparser.simple import ( + ClassScope, + NamespaceScope, + parse_string, + ParsedData, +) + + +def test_class_member_spec_1(): + content = """ + class S { + int d1; // non-static data member + int a[10] = {1, 2}; // non-static data member with initializer (C++11) + static const int d2 = 1; // static data member with initializer + virtual void f1(int) = 0; // pure virtual member function + std::string d3, *d4, f2(int); // two data members and a member function + enum { NORTH, SOUTH, EAST, WEST }; + struct NestedS { + std::string s; + } d5, *d6; + typedef NestedS value_type, *pointer_type; + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="S")], classkey="class" + ) + ), + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="NestedS")], + classkey="struct", + ), + access="private", + ), + fields=[ + Field( + name="s", + type=Type( + typename=PQName( + segments=[ + NameSpecifier(name="std"), + NameSpecifier(name="string"), + ] + ) + ), + access="public", + ) + ], + ) + ], + enums=[ + EnumDecl( + typename=PQName( + segments=[AnonymousName(id=1)], classkey="enum" + ), + values=[ + Enumerator(name="NORTH"), + Enumerator(name="SOUTH"), + Enumerator(name="EAST"), + Enumerator(name="WEST"), + ], + access="private", + ) + ], + fields=[ + Field( + name="d1", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + access="private", + ), + Field( + name="a", + type=Array( + array_of=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + size=Value(tokens=[Token(value="10")]), + ), + access="private", + value=Value( + tokens=[ + Token(value="{"), + Token(value="1"), + Token(value=","), + Token(value="2"), + Token(value="}"), + ] + ), + ), + Field( + name="d2", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ), + const=True, + ), + access="private", + value=Value(tokens=[Token(value="1")]), + static=True, + ), + Field( + name="d3", + type=Type( + typename=PQName( + segments=[ + NameSpecifier(name="std"), + NameSpecifier(name="string"), + ] + ) + ), + access="private", + ), + Field( + name="d4", + type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[ + NameSpecifier(name="std"), + NameSpecifier(name="string"), + ] + ) + ) + ), + access="private", + ), + Field( + name="d5", + type=Type( + typename=PQName( + segments=[NameSpecifier(name="NestedS")], + classkey="struct", + ) + ), + access="private", + ), + Field( + name="d6", + type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[NameSpecifier(name="NestedS")], + classkey="struct", + ) + ) + ), + access="private", + ), + ], + methods=[ + Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="f1")]), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ) + ) + ], + access="private", + pure_virtual=True, + virtual=True, + ), + Method( + return_type=Type( + typename=PQName( + segments=[ + NameSpecifier(name="std"), + NameSpecifier(name="string"), + ] + ) + ), + name=PQName(segments=[NameSpecifier(name="f2")]), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ) + ) + ], + access="private", + ), + ], + typedefs=[ + Typedef( + type=Type( + typename=PQName( + segments=[NameSpecifier(name="NestedS")] + ) + ), + name="value_type", + access="private", + ), + Typedef( + type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[NameSpecifier(name="NestedS")] + ) + ) + ), + name="pointer_type", + access="private", + ), + ], + ) + ] + ) + ) + + +def test_class_member_spec_2(): + content = """ + class M { + std::size_t C; + std::vector data; + + public: + M(std::size_t R, std::size_t C) + : C(C), data(R * C) {} // constructor definition + int operator()(size_t r, size_t c) const { // member function definition + return data[r * C + c]; + } + int &operator()(size_t r, size_t c) { // another member function definition + return data[r * C + c]; + } + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="M")], classkey="class" + ) + ), + fields=[ + Field( + name="C", + type=Type( + typename=PQName( + segments=[ + NameSpecifier(name="std"), + NameSpecifier(name="size_t"), + ] + ) + ), + access="private", + ), + Field( + name="data", + type=Type( + typename=PQName( + segments=[ + NameSpecifier(name="std"), + NameSpecifier( + name="vector", + specialization=TemplateSpecialization( + args=[ + TemplateArgument( + tokens=[Token(value="int")] + ) + ] + ), + ), + ] + ) + ), + access="private", + ), + ], + methods=[ + Method( + return_type=None, + name=PQName(segments=[NameSpecifier(name="M")]), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[ + NameSpecifier(name="std"), + NameSpecifier(name="size_t"), + ] + ) + ), + name="R", + ), + Parameter( + type=Type( + typename=PQName( + segments=[ + NameSpecifier(name="std"), + NameSpecifier(name="size_t"), + ] + ) + ), + name="C", + ), + ], + has_body=True, + access="public", + constructor=True, + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator()")]), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[NameSpecifier(name="size_t")] + ) + ), + name="r", + ), + Parameter( + type=Type( + typename=PQName( + segments=[NameSpecifier(name="size_t")] + ) + ), + name="c", + ), + ], + has_body=True, + access="public", + const=True, + operator="()", + ), + Operator( + return_type=Reference( + ref_to=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ) + ), + name=PQName(segments=[NameSpecifier(name="operator()")]), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[NameSpecifier(name="size_t")] + ) + ), + name="r", + ), + Parameter( + type=Type( + typename=PQName( + segments=[NameSpecifier(name="size_t")] + ) + ), + name="c", + ), + ], + has_body=True, + access="public", + operator="()", + ), + ], + ) + ] + ) + ) + + +def test_class_member_spec_3(): + content = """ + class S { + public: + S(); // public constructor + S(const S &); // public copy constructor + virtual ~S(); // public virtual destructor + private: + int *ptr; // private data member + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="S")], classkey="class" + ) + ), + fields=[ + Field( + name="ptr", + type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ) + ), + access="private", + ) + ], + methods=[ + Method( + return_type=None, + name=PQName(segments=[NameSpecifier(name="S")]), + parameters=[], + access="public", + constructor=True, + ), + Method( + return_type=None, + name=PQName(segments=[NameSpecifier(name="S")]), + parameters=[ + Parameter( + type=Reference( + ref_to=Type( + typename=PQName( + segments=[NameSpecifier(name="S")] + ), + const=True, + ) + ) + ) + ], + access="public", + constructor=True, + ), + Method( + return_type=None, + name=PQName(segments=[NameSpecifier(name="~S")]), + parameters=[], + access="public", + destructor=True, + virtual=True, + ), + ], + ) + ] + ) + ) + + +def test_class_using(): + content = """ + class Base { + protected: + int d; + }; + class Derived : public Base { + public: + using Base::Base; // inherit all parent's constructors (C++11) + using Base::d; // make Base's protected member d a public member of Derived + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="Base")], classkey="class" + ) + ), + fields=[ + Field( + name="d", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + access="protected", + ) + ], + ), + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="Derived")], classkey="class" + ), + bases=[ + BaseClass( + access="public", + typename=PQName(segments=[NameSpecifier(name="Base")]), + ) + ], + ), + using=[ + UsingDecl( + typename=PQName( + segments=[ + NameSpecifier(name="Base"), + NameSpecifier(name="Base"), + ] + ), + access="public", + ), + UsingDecl( + typename=PQName( + segments=[ + NameSpecifier(name="Base"), + NameSpecifier(name="d"), + ] + ), + access="public", + ), + ], + ), + ] + ) + ) + + +def test_class_member_spec_6(): + content = """ + struct S { + template + void f(T&& n); + + template + struct NestedS { + std::basic_string s; + }; + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="S")], classkey="struct" + ) + ), + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="NestedS")], + classkey="struct", + ), + template=TemplateDecl( + params=[ + TemplateTypeParam(typekey="class", name="CharT") + ] + ), + access="public", + ), + fields=[ + Field( + name="s", + type=Type( + typename=PQName( + segments=[ + NameSpecifier(name="std"), + NameSpecifier( + name="basic_string", + specialization=TemplateSpecialization( + args=[ + TemplateArgument( + tokens=[ + Token(value="CharT") + ] + ) + ] + ), + ), + ] + ) + ), + access="public", + ) + ], + ) + ], + methods=[ + Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="f")]), + parameters=[ + Parameter( + type=MoveReference( + moveref_to=Type( + typename=PQName( + segments=[NameSpecifier(name="T")] + ) + ) + ), + name="n", + ) + ], + template=TemplateDecl( + params=[TemplateTypeParam(typekey="typename", name="T")] + ), + access="public", + ) + ], + ) + ] + ) + ) + + +def test_class_fn_default_params(): + content = """ + // clang-format off + class Hen + { + public: + void add(int a=100, b=0xfd, float c=1.7e-3, float d=3.14); + void join(string s1="", string s2="nothing"); + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="Hen")], classkey="class" + ) + ), + methods=[ + Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="add")]), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="a", + default=Value(tokens=[Token(value="100")]), + ), + Parameter( + type=Type( + typename=PQName( + segments=[NameSpecifier(name="b")] + ) + ), + default=Value(tokens=[Token(value="0xfd")]), + ), + Parameter( + type=Type( + typename=PQName( + segments=[ + FundamentalSpecifier(name="float") + ] + ) + ), + name="c", + default=Value(tokens=[Token(value="1.7e-3")]), + ), + Parameter( + type=Type( + typename=PQName( + segments=[ + FundamentalSpecifier(name="float") + ] + ) + ), + name="d", + default=Value(tokens=[Token(value="3.14")]), + ), + ], + access="public", + ), + Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="join")]), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[NameSpecifier(name="string")] + ) + ), + name="s1", + default=Value(tokens=[Token(value='""')]), + ), + Parameter( + type=Type( + typename=PQName( + segments=[NameSpecifier(name="string")] + ) + ), + name="s2", + default=Value(tokens=[Token(value='"nothing"')]), + ), + ], + access="public", + ), + ], + ) + ] + ) + ) + + +def test_class_fn_inline_virtual(): + content = """ + class B { + public: + virtual inline int aMethod(); + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="B")], classkey="class" + ) + ), + methods=[ + Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name=PQName(segments=[NameSpecifier(name="aMethod")]), + parameters=[], + inline=True, + access="public", + virtual=True, + ) + ], + ) + ] + ) + ) + + +def test_class_fn_pure_virtual_const(): + content = """ + class StoneClass { + virtual int getNum2() const = 0; + int getNum3(); + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="StoneClass")], + classkey="class", + ) + ), + methods=[ + Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name=PQName(segments=[NameSpecifier(name="getNum2")]), + parameters=[], + access="private", + const=True, + pure_virtual=True, + virtual=True, + ), + Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name=PQName(segments=[NameSpecifier(name="getNum3")]), + parameters=[], + access="private", + ), + ], + ) + ] + ) + ) + + +def test_class_fn_return_global_ns(): + content = """ + struct Avacado { + uint8_t foo() { return 4; } + ::uint8_t bar() { return 0; } + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="Avacado")], classkey="struct" + ) + ), + methods=[ + Method( + return_type=Type( + typename=PQName( + segments=[NameSpecifier(name="uint8_t")] + ) + ), + name=PQName(segments=[NameSpecifier(name="foo")]), + parameters=[], + has_body=True, + access="public", + ), + Method( + return_type=Type( + typename=PQName( + segments=[ + NameSpecifier(name=""), + NameSpecifier(name="uint8_t"), + ] + ) + ), + name=PQName(segments=[NameSpecifier(name="bar")]), + parameters=[], + has_body=True, + access="public", + ), + ], + ) + ] + ) + ) + + +def test_class_ns_class(): + content = """ + namespace ns { + class N; + }; + + class ns::N {}; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[ + NameSpecifier(name="ns"), + NameSpecifier(name="N"), + ], + classkey="class", + ) + ) + ) + ], + namespaces={ + "ns": NamespaceScope( + name="ns", + forward_decls=[ + ForwardDecl( + typename=PQName( + segments=[NameSpecifier(name="N")], classkey="class" + ) + ) + ], + ) + }, + ) + ) + + +def test_class_ns_w_base(): + content = """ + class Herb::Cilantro : public Plant {}; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[ + NameSpecifier(name="Herb"), + NameSpecifier(name="Cilantro"), + ], + classkey="class", + ), + bases=[ + BaseClass( + access="public", + typename=PQName(segments=[NameSpecifier(name="Plant")]), + ) + ], + ) + ) + ] + ) + ) + + +def test_class_inner_class(): + content = """ + class C { + class Inner {}; + }; + + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="C")], classkey="class" + ) + ), + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="Inner")], + classkey="class", + ), + access="private", + ) + ) + ], + ) + ] + ) + ) + + +def test_class_inner_fwd_class(): + content = """ + class C { + class N; + }; + + class C::N {}; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="C")], classkey="class" + ) + ), + forward_decls=[ + ForwardDecl( + typename=PQName( + segments=[NameSpecifier(name="N")], classkey="class" + ), + access="private", + ) + ], + ), + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="C"), NameSpecifier(name="N")], + classkey="class", + ) + ) + ), + ] + ) + ) + + +def test_class_inner_var_access(): + content = """ + class Bug_3488053 { + public: + class Bug_3488053_Nested { + public: + int x; + }; + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="Bug_3488053")], + classkey="class", + ) + ), + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="Bug_3488053_Nested")], + classkey="class", + ), + access="public", + ), + fields=[ + Field( + access="public", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="x", + ) + ], + ) + ], + ) + ] + ) + ) + + +def test_class_ns_and_inner(): + content = """ + namespace RoosterNamespace { + class RoosterOuterClass { + public: + int member1; + + class RoosterSubClass1 { + public: + int publicMember1; + + private: + int privateMember1; + }; + + private: + int member2; + class RoosterSubClass2 { + public: + int publicMember2; + + private: + int privateMember2; + }; + }; + } // namespace RoosterNamespace + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + namespaces={ + "RoosterNamespace": NamespaceScope( + name="RoosterNamespace", + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="RoosterOuterClass")], + classkey="class", + ) + ), + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[ + NameSpecifier(name="RoosterSubClass1") + ], + classkey="class", + ), + access="public", + ), + fields=[ + Field( + access="public", + type=Type( + typename=PQName( + segments=[ + FundamentalSpecifier(name="int") + ] + ) + ), + name="publicMember1", + ), + Field( + access="private", + type=Type( + typename=PQName( + segments=[ + FundamentalSpecifier(name="int") + ] + ) + ), + name="privateMember1", + ), + ], + ), + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[ + NameSpecifier(name="RoosterSubClass2") + ], + classkey="class", + ), + access="private", + ), + fields=[ + Field( + access="public", + type=Type( + typename=PQName( + segments=[ + FundamentalSpecifier(name="int") + ] + ) + ), + name="publicMember2", + ), + Field( + access="private", + type=Type( + typename=PQName( + segments=[ + FundamentalSpecifier(name="int") + ] + ) + ), + name="privateMember2", + ), + ], + ), + ], + fields=[ + Field( + access="public", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="member1", + ), + Field( + access="private", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="member2", + ), + ], + ) + ], + ) + } + ) + ) + + +def test_class_struct_access(): + content = """ + struct SampleStruct { + unsigned int meth(); + + private: + int prop; + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="SampleStruct")], + classkey="struct", + ) + ), + fields=[ + Field( + access="private", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="prop", + ) + ], + methods=[ + Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="unsigned int")] + ) + ), + name=PQName(segments=[NameSpecifier(name="meth")]), + parameters=[], + access="public", + ) + ], + ) + ] + ) + ) + + +def test_class_volatile_move_deleted_fn(): + content = """ + struct C { + void foo() volatile && = delete; + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="C")], classkey="struct" + ) + ), + methods=[ + Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="foo")]), + parameters=[], + access="public", + volatile=True, + ref_qualifier="&&", + deleted=True, + ) + ], + ) + ] + ) + ) + + +def test_class_bitfield_1(): + content = """ + struct S { + // will usually occupy 2 bytes: + // 3 bits: value of b1 + // 2 bits: unused + // 6 bits: value of b2 + // 2 bits: value of b3 + // 3 bits: unused + unsigned char b1 : 3, : 2, b2 : 6, b3 : 2; + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="S")], classkey="struct" + ) + ), + fields=[ + Field( + name="b1", + type=Type( + typename=PQName( + segments=[ + FundamentalSpecifier(name="unsigned char") + ] + ) + ), + access="public", + bits=3, + ), + Field( + type=Type( + typename=PQName( + segments=[ + FundamentalSpecifier(name="unsigned char") + ] + ) + ), + access="public", + bits=2, + ), + Field( + name="b2", + type=Type( + typename=PQName( + segments=[ + FundamentalSpecifier(name="unsigned char") + ] + ) + ), + access="public", + bits=6, + ), + Field( + name="b3", + type=Type( + typename=PQName( + segments=[ + FundamentalSpecifier(name="unsigned char") + ] + ) + ), + access="public", + bits=2, + ), + ], + ) + ] + ) + ) + + +def test_class_bitfield_2(): + content = """ + struct HAL_ControlWord { + int x : 1; + int y : 1; + }; + typedef struct HAL_ControlWord HAL_ControlWord; + int HAL_GetControlWord(HAL_ControlWord *controlWord); + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="HAL_ControlWord")], + classkey="struct", + ) + ), + fields=[ + Field( + name="x", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + access="public", + bits=1, + ), + Field( + name="y", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + access="public", + bits=1, + ), + ], + ) + ], + functions=[ + Function( + return_type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + name=PQName(segments=[NameSpecifier(name="HAL_GetControlWord")]), + parameters=[ + Parameter( + type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[NameSpecifier(name="HAL_ControlWord")] + ) + ) + ), + name="controlWord", + ) + ], + ) + ], + typedefs=[ + Typedef( + type=Type( + typename=PQName( + segments=[NameSpecifier(name="HAL_ControlWord")], + classkey="struct", + ) + ), + name="HAL_ControlWord", + ) + ], + ) + ) + + +def test_class_anon_struct_as_globalvar(): + content = """ + struct { + int m; + } unnamed, *p_unnamed; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + classkey="struct", segments=[AnonymousName(id=1)] + ) + ), + fields=[ + Field( + name="m", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")], + ) + ), + access="public", + ) + ], + ) + ], + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="unnamed")]), + type=Type( + typename=PQName( + classkey="struct", segments=[AnonymousName(id=1)] + ) + ), + ), + Variable( + name=PQName(segments=[NameSpecifier(name="p_unnamed")]), + type=Pointer( + ptr_to=Type( + typename=PQName( + classkey="struct", segments=[AnonymousName(id=1)] + ) + ) + ), + ), + ], + ) + ) + + +def test_class_anon_struct_as_classvar(): + content = """ + struct AnonHolderClass { + struct { + int x; + } a; + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="AnonHolderClass")], + classkey="struct", + ) + ), + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[AnonymousName(id=1)], classkey="struct" + ), + access="public", + ), + fields=[ + Field( + access="public", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="x", + ) + ], + ) + ], + fields=[ + Field( + access="public", + type=Type( + typename=PQName( + segments=[AnonymousName(id=1)], classkey="struct" + ) + ), + name="a", + ) + ], + ) + ] + ) + ) + + +def test_initializer_with_initializer_list_1(): + content = """ + struct ComplexInit : SomeBase { + ComplexInit(int i) : m_stuff{i, 2} { auto i = something(); } + + void fn(); + + std::vector m_stuff; + }; + + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="ComplexInit")], + classkey="struct", + ), + bases=[ + BaseClass( + access="public", + typename=PQName( + segments=[NameSpecifier(name="SomeBase")] + ), + ) + ], + ), + fields=[ + Field( + access="public", + type=Type( + typename=PQName( + segments=[ + NameSpecifier(name="std"), + NameSpecifier( + name="vector", + specialization=TemplateSpecialization( + args=[ + TemplateArgument( + tokens=[Token(value="int")] + ) + ] + ), + ), + ] + ) + ), + name="m_stuff", + ) + ], + methods=[ + Method( + return_type=None, + name=PQName(segments=[NameSpecifier(name="ComplexInit")]), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="i", + ) + ], + has_body=True, + access="public", + constructor=True, + ), + Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="fn")]), + parameters=[], + access="public", + ), + ], + ) + ] + ) + ) + + +def test_initializer_with_initializer_list_2(): + content = """ + template class future final { + public: + template + future(future &&oth) noexcept + : future(oth.then([](R &&val) -> T { return val; })) {} + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="future")], classkey="class" + ), + template=TemplateDecl( + params=[TemplateTypeParam(typekey="typename", name="T")] + ), + final=True, + ), + methods=[ + Method( + return_type=None, + name=PQName(segments=[NameSpecifier(name="future")]), + parameters=[ + Parameter( + type=MoveReference( + moveref_to=Type( + typename=PQName( + segments=[ + NameSpecifier( + name="future", + specialization=TemplateSpecialization( + args=[ + TemplateArgument( + tokens=[ + Token(value="R") + ] + ) + ] + ), + ) + ] + ) + ) + ), + name="oth", + ) + ], + has_body=True, + template=TemplateDecl( + params=[TemplateTypeParam(typekey="typename", name="R")] + ), + noexcept=Value(tokens=[]), + access="public", + constructor=True, + ) + ], + ) + ] + ) + ) + + +def test_class_with_arrays(): + content = """ + const int MAX_ITEM = 7; + class Bird { + int items[MAX_ITEM]; + int otherItems[7]; + int oneItem; + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="Bird")], classkey="class" + ) + ), + fields=[ + Field( + access="private", + type=Array( + array_of=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + size=Value(tokens=[Token(value="MAX_ITEM")]), + ), + name="items", + ), + Field( + access="private", + type=Array( + array_of=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + size=Value(tokens=[Token(value="7")]), + ), + name="otherItems", + ), + Field( + access="private", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="oneItem", + ), + ], + ) + ], + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="MAX_ITEM")]), + type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]), + const=True, + ), + value=Value(tokens=[Token(value="7")]), + ) + ], + ) + ) + + +def test_class_fn_inline_impl(): + content = """ + class Monkey { + private: + static void Create(); + }; + inline void Monkey::Create() {} + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="Monkey")], classkey="class" + ) + ), + methods=[ + Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="Create")]), + parameters=[], + static=True, + access="private", + ) + ], + ) + ], + functions=[ + Function( + return_type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="void")]) + ), + name=PQName( + segments=[ + NameSpecifier(name="Monkey"), + NameSpecifier(name="Create"), + ] + ), + parameters=[], + inline=True, + has_body=True, + ) + ], + ) + ) + + +def test_class_fn_virtual_final_override(): + content = """ + struct Lemon { + virtual void foo() final; + virtual void foo2(); + }; + + struct Lime final : Lemon { + void abc(); + void foo2() override; + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="Lemon")], classkey="struct" + ) + ), + methods=[ + Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="foo")]), + parameters=[], + access="public", + virtual=True, + final=True, + ), + Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="foo2")]), + parameters=[], + access="public", + virtual=True, + ), + ], + ), + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="Lime")], classkey="struct" + ), + bases=[ + BaseClass( + access="public", + typename=PQName(segments=[NameSpecifier(name="Lemon")]), + ) + ], + final=True, + ), + methods=[ + Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="abc")]), + parameters=[], + access="public", + ), + Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="foo2")]), + parameters=[], + access="public", + override=True, + ), + ], + ), + ] + ) + ) + + +def test_class_fn_return_class(): + content = """ + class Peach { + int abc; + }; + + class Plumb { + class Peach *doSomethingGreat(class Peach *pInCurPtr); + class Peach *var; + }; + + class Peach *Plumb::myMethod(class Peach *pInPtr) { + return pInPtr; + } + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="Peach")], classkey="class" + ) + ), + fields=[ + Field( + access="private", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="abc", + ) + ], + ), + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="Plumb")], classkey="class" + ) + ), + fields=[ + Field( + access="private", + type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[NameSpecifier(name="Peach")], + classkey="class", + ) + ) + ), + name="var", + ) + ], + methods=[ + Method( + return_type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[NameSpecifier(name="Peach")], + classkey="class", + ) + ) + ), + name=PQName( + segments=[NameSpecifier(name="doSomethingGreat")] + ), + parameters=[ + Parameter( + type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[NameSpecifier(name="Peach")], + classkey="class", + ) + ) + ), + name="pInCurPtr", + ) + ], + access="private", + ) + ], + ), + ], + functions=[ + Function( + return_type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[NameSpecifier(name="Peach")], classkey="class" + ) + ) + ), + name=PQName( + segments=[ + NameSpecifier(name="Plumb"), + NameSpecifier(name="myMethod"), + ] + ), + parameters=[ + Parameter( + type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[NameSpecifier(name="Peach")], + classkey="class", + ) + ) + ), + name="pInPtr", + ) + ], + has_body=True, + ) + ], + ) + ) + + +def test_class_fn_template_impl(): + content = """ + class Owl { + private: + template int *tFunc(int count); + }; + + template int *Owl::tFunc(int count) { + if (count == 0) { + return NULL; + } + return NULL; + } + + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="Owl")], classkey="class" + ) + ), + methods=[ + Method( + return_type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ) + ), + name=PQName(segments=[NameSpecifier(name="tFunc")]), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="count", + ) + ], + template=TemplateDecl( + params=[TemplateTypeParam(typekey="typename", name="T")] + ), + access="private", + ) + ], + ) + ], + functions=[ + Function( + return_type=Pointer( + ptr_to=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ) + ), + name=PQName( + segments=[ + NameSpecifier(name="Owl"), + NameSpecifier(name="tFunc"), + ] + ), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="count", + ) + ], + has_body=True, + template=TemplateDecl( + params=[TemplateTypeParam(typekey="typename", name="T")] + ), + ) + ], + ) + ) + + +def test_class_fn_inline_template_impl(): + content = """ + class Chicken { + template static T Get(); + }; + template T Chicken::Get() { return T(); } + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="Chicken")], classkey="class" + ) + ), + methods=[ + Method( + return_type=Type( + typename=PQName(segments=[NameSpecifier(name="T")]) + ), + name=PQName(segments=[NameSpecifier(name="Get")]), + parameters=[], + static=True, + template=TemplateDecl( + params=[TemplateTypeParam(typekey="typename", name="T")] + ), + access="private", + ) + ], + ) + ], + functions=[ + Function( + return_type=Type( + typename=PQName(segments=[NameSpecifier(name="T")]) + ), + name=PQName( + segments=[ + NameSpecifier(name="Chicken"), + NameSpecifier(name="Get"), + ] + ), + parameters=[], + has_body=True, + template=TemplateDecl( + params=[TemplateTypeParam(typekey="typename", name="T")] + ), + ) + ], + ) + ) + + +def test_class_fn_explicit_constructors(): + content = """ + class Lizzard { + Lizzard(); + explicit Lizzard(int a); + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="Lizzard")], classkey="class" + ) + ), + methods=[ + Method( + return_type=None, + name=PQName(segments=[NameSpecifier(name="Lizzard")]), + parameters=[], + access="private", + constructor=True, + ), + Method( + return_type=None, + name=PQName(segments=[NameSpecifier(name="Lizzard")]), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="a", + ) + ], + access="private", + constructor=True, + explicit=True, + ), + ], + ) + ] + ) + ) + + +def test_class_fn_default_constructor(): + content = """ + class DefaultConstDest { + public: + DefaultConstDest() = default; + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="DefaultConstDest")], + classkey="class", + ) + ), + methods=[ + Method( + return_type=None, + name=PQName( + segments=[NameSpecifier(name="DefaultConstDest")] + ), + parameters=[], + access="public", + constructor=True, + default=True, + ) + ], + ) + ] + ) + ) + + +def test_class_fn_delete_constructor(): + content = """ + class A { + public: + A() = delete; + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="A")], classkey="class" + ) + ), + methods=[ + Method( + return_type=None, + name=PQName(segments=[NameSpecifier(name="A")]), + parameters=[], + access="public", + constructor=True, + deleted=True, + ) + ], + ) + ] + ) + ) + + +def test_class_multi_vars(): + content = """ + class Grape { + public: + int a, b, c; + map d; + map e, f; + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="Grape")], classkey="class" + ) + ), + fields=[ + Field( + access="public", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="a", + ), + Field( + access="public", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="b", + ), + Field( + access="public", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="c", + ), + Field( + access="public", + type=Type( + typename=PQName( + segments=[ + NameSpecifier( + name="map", + specialization=TemplateSpecialization( + args=[ + TemplateArgument( + tokens=[Token(value="string")] + ), + TemplateArgument( + tokens=[Token(value="int")] + ), + ] + ), + ) + ] + ) + ), + name="d", + ), + Field( + access="public", + type=Type( + typename=PQName( + segments=[ + NameSpecifier( + name="map", + specialization=TemplateSpecialization( + args=[ + TemplateArgument( + tokens=[Token(value="string")] + ), + TemplateArgument( + tokens=[Token(value="int")] + ), + ] + ), + ) + ] + ) + ), + name="e", + ), + Field( + access="public", + type=Type( + typename=PQName( + segments=[ + NameSpecifier( + name="map", + specialization=TemplateSpecialization( + args=[ + TemplateArgument( + tokens=[Token(value="string")] + ), + TemplateArgument( + tokens=[Token(value="int")] + ), + ] + ), + ) + ] + ) + ), + name="f", + ), + ], + ) + ] + ) + ) + + +def test_class_static_const_var_expr(): + content = """ + class PandaClass { + static const int CONST_A = (1 << 7) - 1; + static const int CONST_B = sizeof(int); + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="PandaClass")], + classkey="class", + ) + ), + fields=[ + Field( + access="private", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ), + const=True, + ), + name="CONST_A", + value=Value( + tokens=[ + Token(value="("), + Token(value="1"), + Token(value="<<"), + Token(value="7"), + Token(value=")"), + Token(value="-"), + Token(value="1"), + ] + ), + static=True, + ), + Field( + access="private", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ), + const=True, + ), + name="CONST_B", + value=Value( + tokens=[ + Token(value="sizeof"), + Token(value="("), + Token(value="int"), + Token(value=")"), + ] + ), + static=True, + ), + ], + ) + ] + ) + ) + + +def test_class_fwd_struct(): + content = """ + class PotatoClass { + struct FwdStruct; + FwdStruct *ptr; + struct FwdStruct { + int a; + }; + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="PotatoClass")], + classkey="class", + ) + ), + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="FwdStruct")], + classkey="struct", + ), + access="private", + ), + fields=[ + Field( + access="public", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="a", + ) + ], + ) + ], + fields=[ + Field( + access="private", + type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[NameSpecifier(name="FwdStruct")] + ) + ) + ), + name="ptr", + ) + ], + forward_decls=[ + ForwardDecl( + typename=PQName( + segments=[NameSpecifier(name="FwdStruct")], + classkey="struct", + ), + access="private", + ) + ], + ) + ] + ) + ) + + +def test_class_multi_array(): + content = """ + struct Picture { + char name[25]; + unsigned int pdata[128][256]; + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="Picture")], classkey="struct" + ) + ), + fields=[ + Field( + access="public", + type=Array( + array_of=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="char")] + ) + ), + size=Value(tokens=[Token(value="25")]), + ), + name="name", + ), + Field( + access="public", + type=Array( + array_of=Array( + array_of=Type( + typename=PQName( + segments=[ + FundamentalSpecifier( + name="unsigned int" + ) + ] + ) + ), + size=Value(tokens=[Token(value="256")]), + ), + size=Value(tokens=[Token(value="128")]), + ), + name="pdata", + ), + ], + ) + ] + ) + ) + + +def test_class_noexcept(): + content = """ + struct Grackle { + void no_noexcept(); + void just_noexcept() noexcept; + void const_noexcept() const noexcept; + void noexcept_bool() noexcept(true); + void const_noexcept_bool() const noexcept(true); + void noexcept_noexceptOperator() noexcept(noexcept(Grackle())); + void const_noexcept_noexceptOperator() const noexcept(noexcept(Grackle())); + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="Grackle")], classkey="struct" + ) + ), + methods=[ + Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="no_noexcept")]), + parameters=[], + access="public", + ), + Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="just_noexcept")]), + parameters=[], + noexcept=Value(tokens=[]), + access="public", + ), + Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName( + segments=[NameSpecifier(name="const_noexcept")] + ), + parameters=[], + noexcept=Value(tokens=[]), + access="public", + const=True, + ), + Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="noexcept_bool")]), + parameters=[], + noexcept=Value(tokens=[Token(value="true")]), + access="public", + ), + Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName( + segments=[NameSpecifier(name="const_noexcept_bool")] + ), + parameters=[], + noexcept=Value(tokens=[Token(value="true")]), + access="public", + const=True, + ), + Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName( + segments=[ + NameSpecifier(name="noexcept_noexceptOperator") + ] + ), + parameters=[], + noexcept=Value( + tokens=[ + Token(value="noexcept"), + Token(value="("), + Token(value="Grackle"), + Token(value="("), + Token(value=")"), + Token(value=")"), + ] + ), + access="public", + ), + Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName( + segments=[ + NameSpecifier( + name="const_noexcept_noexceptOperator" + ) + ] + ), + parameters=[], + noexcept=Value( + tokens=[ + Token(value="noexcept"), + Token(value="("), + Token(value="Grackle"), + Token(value="("), + Token(value=")"), + Token(value=")"), + ] + ), + access="public", + const=True, + ), + ], + ) + ] + ) + ) diff --git a/tests/test_class_base.py b/tests/test_class_base.py new file mode 100644 index 0000000..70e57f5 --- /dev/null +++ b/tests/test_class_base.py @@ -0,0 +1,247 @@ +# Note: testcases generated via `python -m cxxheaderparser.gentest` + +from cxxheaderparser.types import ( + BaseClass, + ClassDecl, + Field, + FundamentalSpecifier, + Method, + NameSpecifier, + PQName, + TemplateArgument, + TemplateSpecialization, + Token, + Type, +) +from cxxheaderparser.simple import ( + ClassScope, + NamespaceScope, + parse_string, + ParsedData, +) + + +def test_class_private_base(): + content = """ + namespace Citrus + { + class BloodOrange { }; + } + + class Bananna: public Citrus::BloodOrange + { + }; + + class ExcellentCake: private Citrus::BloodOrange, Convoluted::Nested::Mixin + { + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="Bananna")], classkey="class" + ), + bases=[ + BaseClass( + access="public", + typename=PQName( + segments=[ + NameSpecifier(name="Citrus"), + NameSpecifier(name="BloodOrange"), + ] + ), + ) + ], + ) + ), + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="ExcellentCake")], + classkey="class", + ), + bases=[ + BaseClass( + access="private", + typename=PQName( + segments=[ + NameSpecifier(name="Citrus"), + NameSpecifier(name="BloodOrange"), + ] + ), + ), + BaseClass( + access="private", + typename=PQName( + segments=[ + NameSpecifier(name="Convoluted"), + NameSpecifier(name="Nested"), + NameSpecifier(name="Mixin"), + ] + ), + ), + ], + ) + ), + ], + namespaces={ + "Citrus": NamespaceScope( + name="Citrus", + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="BloodOrange")], + classkey="class", + ) + ) + ) + ], + ) + }, + ) + ) + + +def test_class_virtual_base(): + content = """ + class BaseMangoClass {}; + class MangoClass : virtual public BaseMangoClass {}; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="BaseMangoClass")], + classkey="class", + ) + ) + ), + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="MangoClass")], + classkey="class", + ), + bases=[ + BaseClass( + access="public", + typename=PQName( + segments=[NameSpecifier(name="BaseMangoClass")] + ), + virtual=True, + ) + ], + ) + ), + ] + ) + ) + + +def test_class_multiple_base_with_virtual(): + content = """ + class BlueJay : public Bird, public virtual Food { + public: + BlueJay() {} + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="BlueJay")], classkey="class" + ), + bases=[ + BaseClass( + access="public", + typename=PQName(segments=[NameSpecifier(name="Bird")]), + ), + BaseClass( + access="public", + typename=PQName(segments=[NameSpecifier(name="Food")]), + virtual=True, + ), + ], + ), + methods=[ + Method( + return_type=None, + name=PQName(segments=[NameSpecifier(name="BlueJay")]), + parameters=[], + has_body=True, + access="public", + constructor=True, + ) + ], + ) + ] + ) + ) + + +def test_class_base_specialized(): + content = """ + class Pea : public Vegetable { + int i; + }; + + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="Pea")], classkey="class" + ), + bases=[ + BaseClass( + access="public", + typename=PQName( + segments=[ + NameSpecifier( + name="Vegetable", + specialization=TemplateSpecialization( + args=[ + TemplateArgument( + tokens=[Token(value="Green")] + ) + ] + ), + ) + ] + ), + ) + ], + ), + fields=[ + Field( + access="private", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="i", + ) + ], + ) + ] + ) + ) diff --git a/tests/test_doxygen.py b/tests/test_doxygen.py new file mode 100644 index 0000000..c303f17 --- /dev/null +++ b/tests/test_doxygen.py @@ -0,0 +1,221 @@ +# Note: testcases generated via `python -m cxxheaderparser.gentest` + +from cxxheaderparser.types import ( + AnonymousName, + Array, + BaseClass, + ClassDecl, + EnumDecl, + Enumerator, + Field, + ForwardDecl, + Function, + FundamentalSpecifier, + Method, + MoveReference, + NameSpecifier, + Operator, + PQName, + Parameter, + Pointer, + Reference, + TemplateArgument, + TemplateDecl, + TemplateSpecialization, + TemplateTypeParam, + Token, + Type, + Typedef, + UsingDecl, + Value, + Variable, +) +from cxxheaderparser.simple import ( + ClassScope, + NamespaceScope, + parse_string, + ParsedData, +) + +r""" +class SampleClass: public BaseSampleClass +{ +public: + enum Elephant + { + EL_ONE = 1, + EL_TWO = 2, + EL_NINE = 9, + EL_TEN, + }; + + SampleClass(); + /*! + * Method 1 + */ + string meth1(); + + /// + /// Method 2 description + /// + /// @param v1 Variable 1 + /// + int meth2(int v1); + + /** + * Method 3 description + * + * \param v1 Variable 1 with a really long + * wrapping description + * \param v2 Variable 2 + */ + void meth3(const string & v1, vector & v2); + + /********************************** + * Method 4 description + * + * @return Return value + *********************************/ + unsigned int meth4(); +private: + void * meth5(){return NULL;} + + /// prop1 description + string prop1; + //! prop5 description + int prop5; + + bool prop6; /*!< prop6 description */ + + double prop7; //!< prop7 description + //!< with two lines + + /// prop8 description + int prop8; +}; +namespace Alpha +{ + class AlphaClass + { + public: + AlphaClass(); + + void alphaMethod(); + + string alphaString; + protected: + typedef enum + { + Z_A, + Z_B = 0x2B, + Z_C = 'j', + Z_D, + } Zebra; + }; + + namespace Omega + { + class OmegaClass + { + public: + OmegaClass(); + + string omegaString; + protected: + /// + /// @brief Rino Numbers, not that that means anything + /// + typedef enum + { + RI_ZERO, /// item zero + RI_ONE, /** item one */ + RI_TWO, //!< item two + RI_THREE, + /// item four + RI_FOUR, + } Rino; + }; + }; +} + +""" + + +# def test_doxygen_messy(): +# content = """ +# // clang-format off + +# /// fn comment +# void +# fn(); + +# /// var comment +# int +# v1 = 0; + +# int +# v2 = 0; /// var2 comment + +# /// cls comment +# class +# C {}; + +# /// template comment +# template +# class +# C2 {}; +# """ +# data = parse_string(content, cleandoc=True) + +# assert data == ParsedData( +# namespace=NamespaceScope( +# classes=[ +# ClassScope( +# class_decl=ClassDecl( +# typename=PQName( +# segments=[NameSpecifier(name="C")], classkey="class" +# ), +# doxygen="/// cls comment", +# ) +# ), +# ClassScope( +# class_decl=ClassDecl( +# typename=PQName( +# segments=[NameSpecifier(name="C2")], classkey="class" +# ), +# template=TemplateDecl( +# params=[TemplateTypeParam(typekey="typename", name="T")] +# ), +# doxygen="/// template comment", +# ) +# ), +# ], +# functions=[ +# Function( +# return_type=Type( +# typename=PQName(segments=[FundamentalSpecifier(name="void")]) +# ), +# name=PQName(segments=[NameSpecifier(name="fn")]), +# parameters=[], +# doxygen="/// fn comment", +# ) +# ], +# variables=[ +# Variable( +# name=PQName(segments=[NameSpecifier(name="v1")]), +# type=Type( +# typename=PQName(segments=[FundamentalSpecifier(name="int")]) +# ), +# value=Value(tokens=[Token(value="0")]), +# doxygen="/// var comment", +# ), +# Variable( +# name=PQName(segments=[NameSpecifier(name="v2")]), +# type=Type( +# typename=PQName(segments=[FundamentalSpecifier(name="int")]) +# ), +# value=Value(tokens=[Token(value="0")]), +# ), +# ], +# ) +# ) diff --git a/tests/test_enum.py b/tests/test_enum.py new file mode 100644 index 0000000..8b1953f --- /dev/null +++ b/tests/test_enum.py @@ -0,0 +1,649 @@ +from cxxheaderparser.types import ( + AnonymousName, + ClassDecl, + EnumDecl, + Enumerator, + Field, + ForwardDecl, + Function, + FundamentalSpecifier, + NameSpecifier, + PQName, + Parameter, + Pointer, + Token, + Type, + Typedef, + Value, + Variable, +) +from cxxheaderparser.simple import ( + ClassScope, + NamespaceScope, + parse_string, + ParsedData, +) + + +def test_basic_enum(): + content = """ + enum Foo { + A, + B, + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName( + segments=[NameSpecifier(name="Foo")], classkey="enum" + ), + values=[Enumerator(name="A"), Enumerator(name="B")], + ) + ] + ) + ) + + +def test_enum_w_expr(): + content = """ + enum Foo { + A = (1 / 2), + B = 3, + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName( + segments=[NameSpecifier(name="Foo")], classkey="enum" + ), + values=[ + Enumerator( + name="A", + value=Value( + tokens=[ + Token(value="("), + Token(value="1"), + Token(value="/"), + Token(value="2"), + Token(value=")"), + ] + ), + ), + Enumerator(name="B", value=Value(tokens=[Token(value="3")])), + ], + ) + ] + ) + ) + + +def test_enum_w_multiline_expr(): + content = r""" + // clang-format off + enum Author + { + NAME = ('J' << 24 | \ + 'A' << 16 | \ + 'S' << 8 | \ + 'H'), + }; + + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName( + segments=[NameSpecifier(name="Author")], classkey="enum" + ), + values=[ + Enumerator( + name="NAME", + value=Value( + tokens=[ + Token(value="("), + Token(value="'J'"), + Token(value="<<"), + Token(value="24"), + Token(value="|"), + Token(value="\\"), + Token(value="'A'"), + Token(value="<<"), + Token(value="16"), + Token(value="|"), + Token(value="\\"), + Token(value="'S'"), + Token(value="<<"), + Token(value="8"), + Token(value="|"), + Token(value="\\"), + Token(value="'H'"), + Token(value=")"), + ] + ), + ) + ], + ) + ] + ) + ) + + +def test_basic_enum_class(): + content = """ + enum class BE { BEX }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName( + segments=[NameSpecifier(name="BE")], classkey="enum class" + ), + values=[Enumerator(name="BEX")], + ) + ] + ) + ) + + +def test_basic_enum_struct(): + content = """ + enum struct BE { BEX }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName( + segments=[NameSpecifier(name="BE")], classkey="enum struct" + ), + values=[Enumerator(name="BEX")], + ) + ] + ) + ) + + +def test_enum_base(): + content = """ + enum class E : int {}; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName( + segments=[NameSpecifier(name="E")], classkey="enum class" + ), + values=[], + base=PQName(segments=[FundamentalSpecifier(name="int")]), + ) + ] + ) + ) + + +# instances + + +def test_enum_instance_1(): + content = """ + enum class BE { BEX } be1; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName( + segments=[NameSpecifier(name="BE")], classkey="enum class" + ), + values=[Enumerator(name="BEX")], + ) + ], + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="be1")]), + type=Type( + typename=PQName( + segments=[NameSpecifier(name="BE")], classkey="enum class" + ) + ), + ) + ], + ) + ) + + +def test_enum_instance_2(): + content = """ + enum class BE { BEX } be1, *be2; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName( + segments=[NameSpecifier(name="BE")], classkey="enum class" + ), + values=[Enumerator(name="BEX")], + ) + ], + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="be1")]), + type=Type( + typename=PQName( + segments=[NameSpecifier(name="BE")], classkey="enum class" + ) + ), + ), + Variable( + name=PQName(segments=[NameSpecifier(name="be2")]), + type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[NameSpecifier(name="BE")], + classkey="enum class", + ) + ) + ), + ), + ], + ) + ) + + +# bases in namespaces + + +def test_enum_base_in_ns(): + content = """ + namespace EN { + typedef int EINT; + }; + + enum class BE : EN::EINT { BEX }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName( + segments=[NameSpecifier(name="BE")], classkey="enum class" + ), + values=[Enumerator(name="BEX")], + base=PQName( + segments=[NameSpecifier(name="EN"), NameSpecifier(name="EINT")] + ), + ) + ], + namespaces={ + "EN": NamespaceScope( + name="EN", + typedefs=[ + Typedef( + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="EINT", + ) + ], + ) + }, + ) + ) + + +# forward declarations + + +def test_enum_fwd(): + content = """ + enum class BE1; + enum class BE2 : EN::EINT; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + forward_decls=[ + ForwardDecl( + typename=PQName( + segments=[NameSpecifier(name="BE1")], classkey="enum class" + ) + ), + ForwardDecl( + typename=PQName( + segments=[NameSpecifier(name="BE2")], classkey="enum class" + ), + enum_base=PQName( + segments=[NameSpecifier(name="EN"), NameSpecifier(name="EINT")] + ), + ), + ] + ) + ) + + +def test_enum_private_in_class(): + content = """ + + class C { + enum E { E1 }; + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="C")], classkey="class" + ) + ), + enums=[ + EnumDecl( + typename=PQName( + segments=[NameSpecifier(name="E")], classkey="enum" + ), + values=[Enumerator(name="E1")], + access="private", + ) + ], + ) + ] + ) + ) + + +def test_enum_public_in_class(): + content = """ + + class C { + public: + enum E { E1 }; + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="C")], classkey="class" + ) + ), + enums=[ + EnumDecl( + typename=PQName( + segments=[NameSpecifier(name="E")], classkey="enum" + ), + values=[Enumerator(name="E1")], + access="public", + ) + ], + ) + ] + ) + ) + + +def test_default_enum(): + content = """ + class A { + enum { + v1, + v2, + } m_v1 = v1; + + enum { vv1, vv2, vv3 } m_v2 = vv2, m_v3 = vv3; + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="A")], classkey="class" + ) + ), + enums=[ + EnumDecl( + typename=PQName( + segments=[AnonymousName(id=1)], classkey="enum" + ), + values=[Enumerator(name="v1"), Enumerator(name="v2")], + access="private", + ), + EnumDecl( + typename=PQName( + segments=[AnonymousName(id=2)], classkey="enum" + ), + values=[ + Enumerator(name="vv1"), + Enumerator(name="vv2"), + Enumerator(name="vv3"), + ], + access="private", + ), + ], + fields=[ + Field( + access="private", + type=Type( + typename=PQName( + segments=[AnonymousName(id=1)], classkey="enum" + ) + ), + name="m_v1", + value=Value(tokens=[Token(value="v1")]), + ), + Field( + access="private", + type=Type( + typename=PQName( + segments=[AnonymousName(id=2)], classkey="enum" + ) + ), + name="m_v2", + value=Value(tokens=[Token(value="vv2")]), + ), + Field( + access="private", + type=Type( + typename=PQName( + segments=[AnonymousName(id=2)], classkey="enum" + ) + ), + name="m_v3", + value=Value(tokens=[Token(value="vv3")]), + ), + ], + ) + ] + ) + ) + + +def test_enum_template_vals(): + content = """ + enum { + IsRandomAccess = std::is_base_of::value, + IsBidirectional = std::is_base_of::value, + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName(segments=[AnonymousName(id=1)], classkey="enum"), + values=[ + Enumerator( + name="IsRandomAccess", + value=Value( + tokens=[ + Token(value="std"), + Token(value="::"), + Token(value="is_base_of"), + Token(value="<"), + Token(value="std"), + Token(value="::"), + Token(value="random_access_iterator_tag"), + Token(value=","), + Token(value="IteratorCategoryT"), + Token(value=">"), + Token(value="::"), + Token(value="value"), + ] + ), + ), + Enumerator( + name="IsBidirectional", + value=Value( + tokens=[ + Token(value="std"), + Token(value="::"), + Token(value="is_base_of"), + Token(value="<"), + Token(value="std"), + Token(value="::"), + Token(value="bidirectional_iterator_tag"), + Token(value=","), + Token(value="IteratorCategoryT"), + Token(value=">"), + Token(value="::"), + Token(value="value"), + ] + ), + ), + ], + ) + ] + ) + ) + + +def test_enum_fn(): + content = """ + enum E { + VALUE, + }; + + void fn_with_enum_param1(const enum E e); + + void fn_with_enum_param2(const enum E e) { + // code here + } + + enum E fn_with_enum_retval1(void); + + enum E fn_with_enum_retval2(void) { + // code here + } + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName( + segments=[NameSpecifier(name="E")], classkey="enum" + ), + values=[Enumerator(name="VALUE")], + ) + ], + functions=[ + Function( + return_type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="void")]) + ), + name=PQName(segments=[NameSpecifier(name="fn_with_enum_param1")]), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[NameSpecifier(name="E")], classkey="enum" + ), + const=True, + ), + name="e", + ) + ], + ), + Function( + return_type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="void")]) + ), + name=PQName(segments=[NameSpecifier(name="fn_with_enum_param2")]), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[NameSpecifier(name="E")], classkey="enum" + ), + const=True, + ), + name="e", + ) + ], + has_body=True, + ), + Function( + return_type=Type( + typename=PQName( + segments=[NameSpecifier(name="E")], classkey="enum" + ) + ), + name=PQName(segments=[NameSpecifier(name="fn_with_enum_retval1")]), + parameters=[], + ), + Function( + return_type=Type( + typename=PQName( + segments=[NameSpecifier(name="E")], classkey="enum" + ) + ), + name=PQName(segments=[NameSpecifier(name="fn_with_enum_retval2")]), + parameters=[], + has_body=True, + ), + ], + ) + ) diff --git a/tests/test_fn.py b/tests/test_fn.py new file mode 100644 index 0000000..e5f112a --- /dev/null +++ b/tests/test_fn.py @@ -0,0 +1,506 @@ +# Note: testcases generated via `python -m cxxheaderparser.gentest` + +from cxxheaderparser.types import ( + Array, + AutoSpecifier, + Function, + FundamentalSpecifier, + NameSpecifier, + PQName, + Parameter, + Pointer, + Reference, + TemplateArgument, + TemplateDecl, + TemplateSpecialization, + TemplateTypeParam, + Token, + Type, +) +from cxxheaderparser.simple import ( + NamespaceScope, + parse_string, + ParsedData, +) + + +def test_fn_returns_class(): + content = """ + class X *fn1(); + struct Y fn2(); + enum E fn3(); + + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + functions=[ + Function( + return_type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[NameSpecifier(name="X")], classkey="class" + ) + ) + ), + name=PQName(segments=[NameSpecifier(name="fn1")]), + parameters=[], + ), + Function( + return_type=Type( + typename=PQName( + segments=[NameSpecifier(name="Y")], classkey="struct" + ) + ), + name=PQName(segments=[NameSpecifier(name="fn2")]), + parameters=[], + ), + Function( + return_type=Type( + typename=PQName( + segments=[NameSpecifier(name="E")], classkey="enum" + ) + ), + name=PQName(segments=[NameSpecifier(name="fn3")]), + parameters=[], + ), + ] + ) + ) + + +def test_fn_pointer_params(): + content = """ + int fn1(int *); + int fn2(int *p); + int fn3(int(*p)); + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + functions=[ + Function( + return_type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + name=PQName(segments=[NameSpecifier(name="fn1")]), + parameters=[ + Parameter( + type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ) + ), + ) + ], + ), + Function( + return_type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + name=PQName(segments=[NameSpecifier(name="fn2")]), + parameters=[ + Parameter( + name="p", + type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ) + ), + ) + ], + ), + Function( + return_type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + name=PQName(segments=[NameSpecifier(name="fn3")]), + parameters=[ + Parameter( + name="p", + type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ) + ), + ) + ], + ), + ] + ) + ) + + +def test_fn_void_is_no_params(): + content = """ + int fn(void); + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + functions=[ + Function( + return_type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + name=PQName(segments=[NameSpecifier(name="fn")]), + parameters=[], + ) + ] + ) + ) + + +def test_fn_array_param(): + content = """ + void fn(int array[]); + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + functions=[ + Function( + return_type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="void")]) + ), + name=PQName(segments=[NameSpecifier(name="fn")]), + parameters=[ + Parameter( + name="array", + type=Array( + array_of=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + size=None, + ), + ) + ], + ) + ] + ) + ) + + +def test_fn_weird_refs(): + content = """ + int aref(int(&x)); + void ptr_ref(int(*&name)); + void ref_to_array(int (&array)[]); + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + functions=[ + Function( + return_type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + name=PQName(segments=[NameSpecifier(name="aref")]), + parameters=[ + Parameter( + name="x", + type=Reference( + ref_to=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ) + ), + ) + ], + ), + Function( + return_type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="void")]) + ), + name=PQName(segments=[NameSpecifier(name="ptr_ref")]), + parameters=[ + Parameter( + name="name", + type=Reference( + ref_to=Pointer( + ptr_to=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ) + ) + ), + ) + ], + ), + Function( + return_type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="void")]) + ), + name=PQName(segments=[NameSpecifier(name="ref_to_array")]), + parameters=[ + Parameter( + name="array", + type=Reference( + ref_to=Array( + array_of=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + size=None, + ) + ), + ) + ], + ), + ] + ) + ) + + +def test_fn_too_many_parens(): + content = """ + int fn1(int (x)); + void (fn2 (int (*const (name)))); + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + functions=[ + Function( + return_type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + name=PQName(segments=[NameSpecifier(name="fn1")]), + parameters=[ + Parameter( + name="x", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + ) + ], + ), + Function( + return_type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="void")]) + ), + name=PQName(segments=[NameSpecifier(name="fn2")]), + parameters=[ + Parameter( + name="name", + type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + const=True, + ), + ) + ], + ), + ] + ) + ) + + +# TODO calling conventions +""" +void __stdcall fn(); +void (__stdcall * fn) +""" + + +def test_fn_same_line(): + # multiple functions on the same line + content = """ + void fn1(), fn2(); + void *fn3(), fn4(); + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + functions=[ + Function( + return_type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="void")]) + ), + name=PQName(segments=[NameSpecifier(name="fn1")]), + parameters=[], + ), + Function( + return_type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="void")]) + ), + name=PQName(segments=[NameSpecifier(name="fn2")]), + parameters=[], + ), + Function( + return_type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ) + ), + name=PQName(segments=[NameSpecifier(name="fn3")]), + parameters=[], + ), + Function( + return_type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="void")]) + ), + name=PQName(segments=[NameSpecifier(name="fn4")]), + parameters=[], + ), + ] + ) + ) + + +def test_fn_auto_template(): + content = """ + template + auto add(T t, U u) { return t + u; } + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + functions=[ + Function( + return_type=Type(typename=PQName(segments=[AutoSpecifier()])), + name=PQName(segments=[NameSpecifier(name="add")]), + parameters=[ + Parameter( + type=Type( + typename=PQName(segments=[NameSpecifier(name="T")]) + ), + name="t", + ), + Parameter( + type=Type( + typename=PQName(segments=[NameSpecifier(name="U")]) + ), + name="u", + ), + ], + has_body=True, + template=TemplateDecl( + params=[ + TemplateTypeParam(typekey="class", name="T"), + TemplateTypeParam(typekey="class", name="U"), + ] + ), + ) + ] + ) + ) + + +def test_fn_template_ptr(): + content = """ + std::vector *fn(std::vector *ps); + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + functions=[ + Function( + return_type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[ + NameSpecifier(name="std"), + NameSpecifier( + name="vector", + specialization=TemplateSpecialization( + args=[ + TemplateArgument( + tokens=[ + Token(value="Pointer"), + Token(value="*"), + ] + ) + ] + ), + ), + ] + ) + ) + ), + name=PQName(segments=[NameSpecifier(name="fn")]), + parameters=[ + Parameter( + type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[ + NameSpecifier(name="std"), + NameSpecifier( + name="vector", + specialization=TemplateSpecialization( + args=[ + TemplateArgument( + tokens=[ + Token(value="Pointer"), + Token(value="*"), + ] + ) + ] + ), + ), + ] + ) + ) + ), + name="ps", + ) + ], + ) + ] + ) + ) + + +def test_fn_with_impl(): + content = """ + // clang-format off + void termite(void) + { + return ((structA*) (Func())->element); + } + + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + functions=[ + Function( + return_type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="void")]) + ), + name=PQName(segments=[NameSpecifier(name="termite")]), + parameters=[], + has_body=True, + ) + ] + ) + ) diff --git a/tests/test_friends.py b/tests/test_friends.py new file mode 100644 index 0000000..c600e1b --- /dev/null +++ b/tests/test_friends.py @@ -0,0 +1,382 @@ +# Note: testcases generated via `python -m cxxheaderparser.gentest` + +from cxxheaderparser.types import ( + ClassDecl, + Field, + ForwardDecl, + FriendDecl, + FundamentalSpecifier, + Method, + NameSpecifier, + PQName, + Parameter, + Reference, + TemplateDecl, + TemplateTypeParam, + Type, +) +from cxxheaderparser.simple import ( + ClassScope, + NamespaceScope, + parse_string, + ParsedData, +) + +# friends +def test_various_friends(): + content = """ + class FX { + public: + FX(char); + ~FX(); + void fn() const; + }; + + class FF { + friend class FX; + friend FX::FX(char), FX::~FX(); + friend void FX::fn() const; + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="FX")], classkey="class" + ) + ), + methods=[ + Method( + return_type=None, + name=PQName(segments=[NameSpecifier(name="FX")]), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="char")] + ) + ) + ) + ], + access="public", + constructor=True, + ), + Method( + return_type=None, + name=PQName(segments=[NameSpecifier(name="~FX")]), + parameters=[], + access="public", + destructor=True, + ), + Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="fn")]), + parameters=[], + access="public", + const=True, + ), + ], + ), + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="FF")], classkey="class" + ) + ), + friends=[ + FriendDecl( + cls=ForwardDecl( + typename=PQName( + segments=[NameSpecifier(name="FX")], + classkey="class", + ), + access="private", + ) + ), + FriendDecl( + fn=Method( + return_type=None, + name=PQName( + segments=[ + NameSpecifier(name="FX"), + NameSpecifier(name="FX"), + ] + ), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[ + FundamentalSpecifier(name="char") + ] + ) + ) + ) + ], + access="private", + constructor=True, + ) + ), + FriendDecl( + fn=Method( + return_type=Type( + typename=PQName( + segments=[ + NameSpecifier(name="FX"), + NameSpecifier(name="FX"), + ] + ) + ), + name=PQName( + segments=[ + NameSpecifier(name="FX"), + NameSpecifier(name="~FX"), + ] + ), + parameters=[], + access="private", + ) + ), + FriendDecl( + fn=Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName( + segments=[ + NameSpecifier(name="FX"), + NameSpecifier(name="fn"), + ] + ), + parameters=[], + access="private", + const=True, + ) + ), + ], + ), + ] + ) + ) + + +def test_more_friends(): + content = """ + template struct X { static int x; }; + + struct BFF { + void fn() const; + }; + + struct F { + friend enum B; + friend void BFF::fn() const; + + template friend class X; + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="X")], classkey="struct" + ), + template=TemplateDecl( + params=[TemplateTypeParam(typekey="typename", name="T")] + ), + ), + fields=[ + Field( + name="x", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + access="public", + static=True, + ) + ], + ), + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="BFF")], classkey="struct" + ) + ), + methods=[ + Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="fn")]), + parameters=[], + access="public", + const=True, + ) + ], + ), + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="F")], classkey="struct" + ) + ), + friends=[ + FriendDecl( + cls=ForwardDecl( + typename=PQName( + segments=[NameSpecifier(name="B")], classkey="enum" + ), + access="public", + ) + ), + FriendDecl( + fn=Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName( + segments=[ + NameSpecifier(name="BFF"), + NameSpecifier(name="fn"), + ] + ), + parameters=[], + access="public", + const=True, + ) + ), + FriendDecl( + cls=ForwardDecl( + typename=PQName( + segments=[NameSpecifier(name="X")], classkey="class" + ), + template=TemplateDecl( + params=[ + TemplateTypeParam(typekey="typename", name="T") + ] + ), + access="public", + ) + ), + ], + ), + ] + ) + ) + + +def test_friend_type_no_class(): + content = """ + class DogClass; + class CatClass { + friend DogClass; + }; + + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="CatClass")], classkey="class" + ) + ), + friends=[ + FriendDecl( + cls=ForwardDecl( + typename=PQName( + segments=[NameSpecifier(name="DogClass")] + ), + access="private", + ) + ) + ], + ) + ], + forward_decls=[ + ForwardDecl( + typename=PQName( + segments=[NameSpecifier(name="DogClass")], classkey="class" + ) + ) + ], + ) + ) + + +def test_friend_with_impl(): + content = """ + // clang-format off + class Garlic { + public: + friend int genNum(C& a) + { + return obj.meth().num(); + } + }; + + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="Garlic")], classkey="class" + ) + ), + friends=[ + FriendDecl( + fn=Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name=PQName(segments=[NameSpecifier(name="genNum")]), + parameters=[ + Parameter( + type=Reference( + ref_to=Type( + typename=PQName( + segments=[NameSpecifier(name="C")] + ) + ) + ), + name="a", + ) + ], + has_body=True, + access="public", + ) + ) + ], + ) + ] + ) + ) diff --git a/tests/test_misc.py b/tests/test_misc.py new file mode 100644 index 0000000..dfbc143 --- /dev/null +++ b/tests/test_misc.py @@ -0,0 +1,170 @@ +# Note: testcases generated via `python -m cxxheaderparser.gentest` + +from cxxheaderparser.types import ( + Function, + FundamentalSpecifier, + NameSpecifier, + PQName, + Parameter, + Type, + Variable, +) +from cxxheaderparser.simple import ( + Include, + NamespaceScope, + Pragma, + parse_string, + ParsedData, + Define, +) + +# +# minimal preprocessor support +# + + +def test_define(): + content = """ + #define simple + #define complex(thing) stuff(thing) + # define spaced + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + defines=[ + Define(content="simple"), + Define(content="complex(thing) stuff(thing)"), + Define(content="spaced"), + ], + ) + + +def test_includes(): + content = """ + #include + #include "local.h" + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData(includes=[Include(""), Include('"local.h"')]) + + +def test_pragma(): + content = """ + + #pragma once + + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData(pragmas=[Pragma(content="once")]) + + +# +# extern "C" +# + + +def test_extern_c(): + content = """ + extern "C" { + int x; + }; + + int y; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="x")]), + type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + ), + Variable( + name=PQName(segments=[NameSpecifier(name="y")]), + type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + ), + ] + ) + ) + + +def test_misc_extern_inline(): + content = """ + extern "C++" { + inline HAL_Value HAL_GetSimValue(HAL_SimValueHandle handle) { + HAL_Value v; + return v; + } + } // extern "C++" + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + functions=[ + Function( + return_type=Type( + typename=PQName(segments=[NameSpecifier(name="HAL_Value")]) + ), + name=PQName(segments=[NameSpecifier(name="HAL_GetSimValue")]), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[NameSpecifier(name="HAL_SimValueHandle")] + ) + ), + name="handle", + ) + ], + inline=True, + has_body=True, + ) + ] + ) + ) + + +# +# Misc +# + + +def test_static_assert_1(): + # static_assert should be ignored + content = """ + static_assert(x == 1); + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData() + + +def test_static_assert_2(): + # static_assert should be ignored + content = """ + static_assert(sizeof(int) == 4, + "integer size is wrong" + "for some reason"); + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData() + + +def test_comment_eof(): + content = """ + namespace a {} // namespace a""" + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope(namespaces={"a": NamespaceScope(name="a")}) + ) diff --git a/tests/test_namespaces.py b/tests/test_namespaces.py new file mode 100644 index 0000000..2b99f33 --- /dev/null +++ b/tests/test_namespaces.py @@ -0,0 +1,121 @@ +# Note: testcases generated via `python -m cxxheaderparser.gentest` + +from cxxheaderparser.types import ( + FundamentalSpecifier, + NameSpecifier, + PQName, + Token, + Type, + Value, + Variable, +) +from cxxheaderparser.simple import ( + NamespaceScope, + parse_string, + ParsedData, +) + + +def test_dups_in_different_ns(): + content = """ + + namespace { + int x = 4; + } + + int x = 5; + + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="x")]), + type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + value=Value(tokens=[Token(value="5")]), + ) + ], + namespaces={ + "": NamespaceScope( + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="x")]), + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + value=Value(tokens=[Token(value="4")]), + ) + ] + ) + }, + ) + ) + + +def test_correct_ns(): + content = """ + namespace a::b::c { + int i1; + } + + namespace a { + namespace b { + namespace c { + int i2; + } + } + } + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + namespaces={ + "a": NamespaceScope( + name="a", + namespaces={ + "b": NamespaceScope( + name="b", + namespaces={ + "c": NamespaceScope( + name="c", + variables=[ + Variable( + name=PQName( + segments=[NameSpecifier(name="i1")] + ), + type=Type( + typename=PQName( + segments=[ + FundamentalSpecifier(name="int") + ] + ) + ), + ), + Variable( + name=PQName( + segments=[NameSpecifier(name="i2")] + ), + type=Type( + typename=PQName( + segments=[ + FundamentalSpecifier(name="int") + ] + ) + ), + ), + ], + ) + }, + ) + }, + ) + } + ) + ) diff --git a/tests/test_operators.py b/tests/test_operators.py new file mode 100644 index 0000000..7104caf --- /dev/null +++ b/tests/test_operators.py @@ -0,0 +1,554 @@ +# Note: testcases generated via `python -m cxxheaderparser.gentest` + +from cxxheaderparser.types import ( + ClassDecl, + FundamentalSpecifier, + NameSpecifier, + Operator, + PQName, + Parameter, + Reference, + Type, +) +from cxxheaderparser.simple import ( + ClassScope, + NamespaceScope, + parse_string, + ParsedData, +) + + +def test_class_operators(): + content = r""" + class OperatorClass { + public: + void operator=(const Sample25Class &); + void operator-=(const Sample25Class &); + void operator+=(); + void operator[](); + bool operator==(const int &b); + OperatorClass &operator+(); + void operator-(); + void operator*(); + void operator\(); + void operator%(); + void operator^(); + void operator|(); + void operator&(); + void operator~(); + void operator<<(); + void operator>>(); + void operator!=(); + void operator<(); + void operator>(); + void operator>=(); + void operator<=(); + void operator!(); + void operator&&(); + void operator||(); + void operator+=(); + void operator-=(); + void operator*=(); + void operator\=(); + void operator%=(); + void operator&=(); + void operator|=(); + void operator^=(); + void operator<<=(); + void operator>>=(); + void operator++(); + void operator--(); + void operator()(); + void operator->(); + void operator,(); + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="OperatorClass")], + classkey="class", + ) + ), + methods=[ + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator=")]), + parameters=[ + Parameter( + type=Reference( + ref_to=Type( + typename=PQName( + segments=[ + NameSpecifier(name="Sample25Class") + ] + ), + const=True, + ) + ) + ) + ], + access="public", + operator="=", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator-=")]), + parameters=[ + Parameter( + type=Reference( + ref_to=Type( + typename=PQName( + segments=[ + NameSpecifier(name="Sample25Class") + ] + ), + const=True, + ) + ) + ) + ], + access="public", + operator="-=", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator+=")]), + parameters=[], + access="public", + operator="+=", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator[]")]), + parameters=[], + access="public", + operator="[]", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="bool")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator==")]), + parameters=[ + Parameter( + type=Reference( + ref_to=Type( + typename=PQName( + segments=[ + FundamentalSpecifier(name="int") + ] + ), + const=True, + ) + ), + name="b", + ) + ], + access="public", + operator="==", + ), + Operator( + return_type=Reference( + ref_to=Type( + typename=PQName( + segments=[NameSpecifier(name="OperatorClass")] + ) + ) + ), + name=PQName(segments=[NameSpecifier(name="operator+")]), + parameters=[], + access="public", + operator="+", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator-")]), + parameters=[], + access="public", + operator="-", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator*")]), + parameters=[], + access="public", + operator="*", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator\\")]), + parameters=[], + access="public", + operator="\\", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator%")]), + parameters=[], + access="public", + operator="%", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator^")]), + parameters=[], + access="public", + operator="^", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator|")]), + parameters=[], + access="public", + operator="|", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator&")]), + parameters=[], + access="public", + operator="&", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator~")]), + parameters=[], + access="public", + operator="~", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator<<")]), + parameters=[], + access="public", + operator="<<", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator>>")]), + parameters=[], + access="public", + operator=">>", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator!=")]), + parameters=[], + access="public", + operator="!=", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator<")]), + parameters=[], + access="public", + operator="<", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator>")]), + parameters=[], + access="public", + operator=">", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator>=")]), + parameters=[], + access="public", + operator=">=", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator<=")]), + parameters=[], + access="public", + operator="<=", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator!")]), + parameters=[], + access="public", + operator="!", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator&&")]), + parameters=[], + access="public", + operator="&&", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator||")]), + parameters=[], + access="public", + operator="||", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator+=")]), + parameters=[], + access="public", + operator="+=", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator-=")]), + parameters=[], + access="public", + operator="-=", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator*=")]), + parameters=[], + access="public", + operator="*=", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator\\=")]), + parameters=[], + access="public", + operator="\\=", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator%=")]), + parameters=[], + access="public", + operator="%=", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator&=")]), + parameters=[], + access="public", + operator="&=", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator|=")]), + parameters=[], + access="public", + operator="|=", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator^=")]), + parameters=[], + access="public", + operator="^=", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator<<=")]), + parameters=[], + access="public", + operator="<<=", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator>>=")]), + parameters=[], + access="public", + operator=">>=", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator++")]), + parameters=[], + access="public", + operator="++", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator--")]), + parameters=[], + access="public", + operator="--", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator()")]), + parameters=[], + access="public", + operator="()", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator->")]), + parameters=[], + access="public", + operator="->", + ), + Operator( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="operator,")]), + parameters=[], + access="public", + operator=",", + ), + ], + ) + ] + ) + ) diff --git a/tests/test_template.py b/tests/test_template.py new file mode 100644 index 0000000..1c45e86 --- /dev/null +++ b/tests/test_template.py @@ -0,0 +1,1290 @@ +# Note: testcases generated via `python -m cxxheaderparser.gentest` + +from cxxheaderparser.types import ( + Array, + BaseClass, + ClassDecl, + Field, + ForwardDecl, + Function, + FunctionType, + FundamentalSpecifier, + Method, + NameSpecifier, + PQName, + Parameter, + Pointer, + Reference, + TemplateArgument, + TemplateDecl, + TemplateNonTypeParam, + TemplateSpecialization, + TemplateTypeParam, + Token, + Type, + Value, + Variable, +) +from cxxheaderparser.simple import ClassScope, NamespaceScope, ParsedData, parse_string + + +def test_template_base_template_ns(): + content = """ + class A : public B::C {}; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="A")], classkey="class" + ), + bases=[ + BaseClass( + access="public", + typename=PQName( + segments=[ + NameSpecifier( + name="B", + specialization=TemplateSpecialization( + args=[ + TemplateArgument( + tokens=[Token(value="int")] + ), + TemplateArgument( + tokens=[Token(value="int")] + ), + ] + ), + ), + NameSpecifier(name="C"), + ] + ), + ) + ], + ) + ) + ] + ) + ) + + +def test_template_non_type_various(): + content = """ + // simple non-type template parameter + template struct S { int a[N]; }; + + template struct S2 {}; + + // complicated non-type example + template + struct Complicated { + // calls the function selected at compile time + // and stores the result in the array selected at compile time + void foo(char base) { ra[4] = pf(c - base); } + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="S")], classkey="struct" + ), + template=TemplateDecl( + params=[ + TemplateNonTypeParam( + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="N", + ) + ] + ), + ), + fields=[ + Field( + name="a", + type=Array( + array_of=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + size=Value(tokens=[Token(value="N")]), + ), + access="public", + ) + ], + ), + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="S2")], classkey="struct" + ), + template=TemplateDecl( + params=[ + TemplateNonTypeParam( + type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[ + FundamentalSpecifier(name="char") + ] + ), + const=True, + ) + ) + ) + ] + ), + ) + ), + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="Complicated")], + classkey="struct", + ), + template=TemplateDecl( + params=[ + TemplateNonTypeParam( + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="char")] + ) + ), + name="c", + ), + TemplateNonTypeParam( + type=Reference( + ref_to=Array( + array_of=Type( + typename=PQName( + segments=[ + FundamentalSpecifier(name="int") + ] + ) + ), + size=Value(tokens=[Token(value="5")]), + ) + ), + name="ra", + ), + TemplateNonTypeParam( + type=Pointer( + ptr_to=FunctionType( + return_type=Type( + typename=PQName( + segments=[ + FundamentalSpecifier(name="int") + ] + ) + ), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[ + FundamentalSpecifier( + name="int" + ) + ] + ) + ) + ) + ], + ) + ), + name="pf", + ), + ] + ), + ), + methods=[ + Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="foo")]), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="char")] + ) + ), + name="base", + ) + ], + has_body=True, + access="public", + ) + ], + ), + ] + ) + ) + + +def test_template_dependent_nontype_default(): + content = """ + template class X; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + forward_decls=[ + ForwardDecl( + typename=PQName( + segments=[NameSpecifier(name="X")], classkey="class" + ), + template=TemplateDecl( + params=[ + TemplateTypeParam(typekey="class", name="T"), + TemplateNonTypeParam( + type=Type( + typename=PQName( + segments=[ + NameSpecifier(name="T"), + NameSpecifier(name="type"), + ] + ) + ), + name="n", + default=Value(tokens=[Token(value="0")]), + ), + ] + ), + ) + ] + ) + ) + + +def test_template_optional_names(): + content = """ + template class My_vector; + template struct My_op_functor; + template class My_tuple; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + forward_decls=[ + ForwardDecl( + typename=PQName( + segments=[NameSpecifier(name="My_vector")], classkey="class" + ), + template=TemplateDecl(params=[TemplateTypeParam(typekey="class")]), + ), + ForwardDecl( + typename=PQName( + segments=[NameSpecifier(name="My_op_functor")], + classkey="struct", + ), + template=TemplateDecl( + params=[ + TemplateTypeParam( + typekey="class", + default=Value(tokens=[Token(value="void")]), + ) + ] + ), + ), + ForwardDecl( + typename=PQName( + segments=[NameSpecifier(name="My_tuple")], classkey="class" + ), + template=TemplateDecl( + params=[TemplateTypeParam(typekey="typename", param_pack=True)] + ), + ), + ] + ) + ) + + +def test_template_template_template(): + content = """ + template struct eval; // primary template + + template class TT, typename T1, typename... Rest> + struct eval> {}; // partial specialization of eval + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[ + NameSpecifier( + name="eval", + specialization=TemplateSpecialization( + args=[ + TemplateArgument( + tokens=[ + Token(value="TT"), + Token(value="<"), + Token(value="T1"), + Token(value=","), + Token(value="Rest"), + Token(value="..."), + Token(value=">"), + ] + ) + ] + ), + ) + ], + classkey="struct", + ), + template=TemplateDecl( + params=[ + TemplateTypeParam( + typekey="class", + name="TT", + template=TemplateDecl( + params=[ + TemplateTypeParam(typekey="typename"), + TemplateTypeParam( + typekey="typename", param_pack=True + ), + ] + ), + ), + TemplateTypeParam(typekey="typename", name="T1"), + TemplateTypeParam( + typekey="typename", name="Rest", param_pack=True + ), + ] + ), + ) + ) + ], + forward_decls=[ + ForwardDecl( + typename=PQName( + segments=[NameSpecifier(name="eval")], classkey="struct" + ), + template=TemplateDecl( + params=[TemplateTypeParam(typekey="typename", name="T")] + ), + ) + ], + ) + ) + + +def test_template_static_var(): + content = """ + template + struct X { + static int x; + }; + + template + int X::x = 5; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="X")], classkey="struct" + ), + template=TemplateDecl( + params=[TemplateTypeParam(typekey="typename", name="T")] + ), + ), + fields=[ + Field( + name="x", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + access="public", + static=True, + ) + ], + ) + ], + variables=[ + Variable( + name=PQName( + segments=[ + NameSpecifier( + name="X", + specialization=TemplateSpecialization( + args=[TemplateArgument(tokens=[Token(value="T")])] + ), + ), + NameSpecifier(name="x"), + ] + ), + type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + value=Value(tokens=[Token(value="5")]), + template=TemplateDecl( + params=[TemplateTypeParam(typekey="typename", name="T")] + ), + ) + ], + ) + ) + + +def test_template_fn_template(): + content = """ + class S { + template StringRef copy(Allocator &A) const { + // Don't request a length 0 copy from the allocator. + if (empty()) + return StringRef(); + char *S = A.template Allocate(Length); + std::copy(begin(), end(), S); + return StringRef(S, Length); + } + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="S")], classkey="class" + ) + ), + methods=[ + Method( + return_type=Type( + typename=PQName( + segments=[NameSpecifier(name="StringRef")] + ) + ), + name=PQName(segments=[NameSpecifier(name="copy")]), + parameters=[ + Parameter( + type=Reference( + ref_to=Type( + typename=PQName( + segments=[ + NameSpecifier(name="Allocator") + ] + ) + ) + ), + name="A", + ) + ], + has_body=True, + template=TemplateDecl( + params=[ + TemplateTypeParam( + typekey="typename", name="Allocator" + ) + ] + ), + access="private", + const=True, + ) + ], + ) + ] + ) + ) + + +def test_template_fn_param_initializer(): + content = """ + template + void fn(something s = something{1, 2, 3}); + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + functions=[ + Function( + return_type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="void")]) + ), + name=PQName(segments=[NameSpecifier(name="fn")]), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[ + NameSpecifier( + name="something", + specialization=TemplateSpecialization( + args=[ + TemplateArgument( + tokens=[Token(value="T")] + ), + TemplateArgument( + tokens=[Token(value="U")] + ), + ] + ), + ) + ] + ) + ), + name="s", + default=Value( + tokens=[ + Token(value="something"), + Token(value="<"), + Token(value="T"), + Token(value=","), + Token(value="U"), + Token(value=">"), + Token(value="{"), + Token(value="1"), + Token(value=","), + Token(value="2"), + Token(value=","), + Token(value="3"), + Token(value="}"), + ] + ), + ) + ], + template=TemplateDecl( + params=[ + TemplateTypeParam(typekey="typename", name="T"), + TemplateTypeParam(typekey="typename", name="U"), + ] + ), + ) + ] + ) + ) + + +def test_template_huge(): + content = """ + // clang-format off + class AlmondClass + { + public: + std::map > > meth(bool flag, + std::map > > bigArg); + }; + + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="AlmondClass")], + classkey="class", + ) + ), + methods=[ + Method( + return_type=Type( + typename=PQName( + segments=[ + NameSpecifier(name="std"), + NameSpecifier( + name="map", + specialization=TemplateSpecialization( + args=[ + TemplateArgument( + tokens=[Token(value="unsigned")] + ), + TemplateArgument( + tokens=[ + Token(value="std"), + Token(value="::"), + Token(value="pair"), + Token(value="<"), + Token(value="unsigned"), + Token(value=","), + Token( + value="SnailTemplateClass" + ), + Token(value="<"), + Token( + value="SnailNamespace" + ), + Token(value="::"), + Token(value="SnailClass"), + Token(value=">"), + Token(value=">"), + ] + ), + ] + ), + ), + ] + ) + ), + name=PQName(segments=[NameSpecifier(name="meth")]), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="bool")] + ) + ), + name="flag", + ), + Parameter( + type=Type( + typename=PQName( + segments=[ + NameSpecifier(name="std"), + NameSpecifier( + name="map", + specialization=TemplateSpecialization( + args=[ + TemplateArgument( + tokens=[ + Token( + value="unsigned" + ) + ] + ), + TemplateArgument( + tokens=[ + Token(value="std"), + Token(value="::"), + Token(value="pair"), + Token(value="<"), + Token( + value="unsigned" + ), + Token(value=","), + Token( + value="SnailTemplateClass" + ), + Token(value="<"), + Token( + value="SnailNamespace" + ), + Token(value="::"), + Token( + value="SnailClass" + ), + Token(value=">"), + Token(value=">"), + ] + ), + ] + ), + ), + ] + ) + ), + name="bigArg", + ), + ], + access="public", + ) + ], + ) + ] + ) + ) + + +def test_template_specialized(): + content = """ + template <> class FruitFly : public Fly {}; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[ + NameSpecifier( + name="FruitFly", + specialization=TemplateSpecialization( + args=[ + TemplateArgument( + tokens=[Token(value="int")] + ) + ] + ), + ) + ], + classkey="class", + ), + bases=[ + BaseClass( + access="public", + typename=PQName(segments=[NameSpecifier(name="Fly")]), + ) + ], + template=TemplateDecl(), + ) + ) + ] + ) + ) + + +def test_template_class_defaults(): + content = """ + template > + class Raddish_SetIterator : public Raddish_Iterator { + protected: + VALUE_SET_ITERATOR _beg, _end; + + public: + Raddish_SetIterator(const VALUE_SET_ITERATOR &begin, + const VALUE_SET_ITERATOR &end) { + init(begin, end); + } + }; + + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="Raddish_SetIterator")], + classkey="class", + ), + bases=[ + BaseClass( + access="public", + typename=PQName( + segments=[ + NameSpecifier( + name="Raddish_Iterator", + specialization=TemplateSpecialization( + args=[ + TemplateArgument( + tokens=[Token(value="VALUE")] + ) + ] + ), + ) + ] + ), + ) + ], + template=TemplateDecl( + params=[ + TemplateTypeParam(typekey="typename", name="VALUE"), + TemplateTypeParam( + typekey="typename", name="VALUE_SET_ITERATOR" + ), + TemplateTypeParam( + typekey="typename", + name="ACCESSOR", + default=Value( + tokens=[ + Token(value="Raddish"), + Token(value="::"), + Token(value="SimpleAccessor"), + Token(value="<"), + Token(value="VALUE"), + Token(value=","), + Token(value="VALUE_SET_ITERATOR"), + Token(value=">"), + ] + ), + ), + ] + ), + ), + fields=[ + Field( + access="protected", + type=Type( + typename=PQName( + segments=[NameSpecifier(name="VALUE_SET_ITERATOR")] + ) + ), + name="_beg", + ), + Field( + access="protected", + type=Type( + typename=PQName( + segments=[NameSpecifier(name="VALUE_SET_ITERATOR")] + ) + ), + name="_end", + ), + ], + methods=[ + Method( + return_type=None, + name=PQName( + segments=[NameSpecifier(name="Raddish_SetIterator")] + ), + parameters=[ + Parameter( + type=Reference( + ref_to=Type( + typename=PQName( + segments=[ + NameSpecifier( + name="VALUE_SET_ITERATOR" + ) + ] + ), + const=True, + ) + ), + name="begin", + ), + Parameter( + type=Reference( + ref_to=Type( + typename=PQName( + segments=[ + NameSpecifier( + name="VALUE_SET_ITERATOR" + ) + ] + ), + const=True, + ) + ), + name="end", + ), + ], + has_body=True, + access="public", + constructor=True, + ) + ], + ) + ] + ) + ) + + +def test_template_many_packs(): + content = """ + // clang-format off + + template + class XYZ : public MyBaseClass + { + public: + XYZ(); + }; + + template + class concat_iterator + : public iterator_facade_base, + std::forward_iterator_tag, ValueT> { + }; + + template + struct build_index_impl : build_index_impl {}; + + template + struct build_index_impl<0, I...> : index_sequence {}; + + template + struct is_callable, + void_t()).*std::declval())(std::declval()...))>> + : std::true_type {}; + + template + struct S : public T... {}; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="XYZ")], classkey="class" + ), + bases=[ + BaseClass( + access="public", + typename=PQName( + segments=[ + NameSpecifier( + name="MyBaseClass", + specialization=TemplateSpecialization( + args=[ + TemplateArgument( + tokens=[Token(value="Type")] + ), + TemplateArgument( + tokens=[Token(value="int")] + ), + ] + ), + ) + ] + ), + ) + ], + template=TemplateDecl( + params=[TemplateTypeParam(typekey="typename", name="Type")] + ), + ), + methods=[ + Method( + return_type=None, + name=PQName(segments=[NameSpecifier(name="XYZ")]), + parameters=[], + access="public", + constructor=True, + ) + ], + ), + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="concat_iterator")], + classkey="class", + ), + bases=[ + BaseClass( + access="public", + typename=PQName( + segments=[ + NameSpecifier( + name="iterator_facade_base", + specialization=TemplateSpecialization( + args=[ + TemplateArgument( + tokens=[ + Token( + value="concat_iterator" + ), + Token(value="<"), + Token(value="ValueT"), + Token(value=","), + Token(value="IterTs"), + Token(value="..."), + Token(value=">"), + ] + ), + TemplateArgument( + tokens=[ + Token(value="std"), + Token(value="::"), + Token( + value="forward_iterator_tag" + ), + ] + ), + TemplateArgument( + tokens=[Token(value="ValueT")] + ), + ] + ), + ) + ] + ), + ) + ], + template=TemplateDecl( + params=[ + TemplateTypeParam(typekey="typename", name="ValueT"), + TemplateTypeParam( + typekey="typename", name="IterTs", param_pack=True + ), + ] + ), + ) + ), + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="build_index_impl")], + classkey="struct", + ), + bases=[ + BaseClass( + access="public", + typename=PQName( + segments=[ + NameSpecifier( + name="build_index_impl", + specialization=TemplateSpecialization( + args=[ + TemplateArgument( + tokens=[ + Token(value="N"), + Token(value="-"), + Token(value="1"), + ] + ), + TemplateArgument( + tokens=[ + Token(value="N"), + Token(value="-"), + Token(value="1"), + ] + ), + TemplateArgument( + tokens=[ + Token(value="I"), + Token(value="..."), + ] + ), + ] + ), + ) + ] + ), + ) + ], + template=TemplateDecl( + params=[ + TemplateNonTypeParam( + type=Type( + typename=PQName( + segments=[ + NameSpecifier(name="std"), + NameSpecifier(name="size_t"), + ] + ) + ), + name="N", + ), + TemplateNonTypeParam( + type=Type( + typename=PQName( + segments=[ + NameSpecifier(name="std"), + NameSpecifier(name="size_t"), + ] + ) + ), + name="I", + param_pack=True, + ), + ] + ), + ) + ), + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[ + NameSpecifier( + name="build_index_impl", + specialization=TemplateSpecialization( + args=[ + TemplateArgument(tokens=[Token(value="0")]), + TemplateArgument( + tokens=[ + Token(value="I"), + Token(value="..."), + ] + ), + ] + ), + ) + ], + classkey="struct", + ), + bases=[ + BaseClass( + access="public", + typename=PQName( + segments=[ + NameSpecifier( + name="index_sequence", + specialization=TemplateSpecialization( + args=[ + TemplateArgument( + tokens=[ + Token(value="I"), + Token(value="..."), + ] + ) + ] + ), + ) + ] + ), + ) + ], + template=TemplateDecl( + params=[ + TemplateNonTypeParam( + type=Type( + typename=PQName( + segments=[ + NameSpecifier(name="std"), + NameSpecifier(name="size_t"), + ] + ) + ), + name="I", + param_pack=True, + ) + ] + ), + ) + ), + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[ + NameSpecifier( + name="is_callable", + specialization=TemplateSpecialization( + args=[ + TemplateArgument(tokens=[Token(value="F")]), + TemplateArgument(tokens=[Token(value="P")]), + TemplateArgument( + tokens=[ + Token(value="typelist"), + Token(value="<"), + Token(value="T"), + Token(value="..."), + Token(value=">"), + ] + ), + TemplateArgument( + tokens=[ + Token(value="void_t"), + Token(value="<"), + Token(value="decltype"), + Token(value="("), + Token(value="("), + Token(value="("), + Token(value="*"), + Token(value="std"), + Token(value="::"), + Token(value="declval"), + Token(value="<"), + Token(value="P"), + Token(value=">"), + Token(value="("), + Token(value=")"), + Token(value=")"), + Token(value="."), + Token(value="*"), + Token(value="std"), + Token(value="::"), + Token(value="declval"), + Token(value="<"), + Token(value="F"), + Token(value=">"), + Token(value="("), + Token(value=")"), + Token(value=")"), + Token(value="("), + Token(value="std"), + Token(value="::"), + Token(value="declval"), + Token(value="<"), + Token(value="T"), + Token(value=">"), + Token(value="("), + Token(value=")"), + Token(value="..."), + Token(value=")"), + Token(value=")"), + Token(value=">"), + ] + ), + ] + ), + ) + ], + classkey="struct", + ), + bases=[ + BaseClass( + access="public", + typename=PQName( + segments=[ + NameSpecifier(name="std"), + NameSpecifier(name="true_type"), + ] + ), + ) + ], + template=TemplateDecl( + params=[ + TemplateTypeParam(typekey="typename", name="F"), + TemplateTypeParam(typekey="typename", name="P"), + TemplateTypeParam( + typekey="typename", name="T", param_pack=True + ), + ] + ), + ) + ), + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="S")], classkey="struct" + ), + bases=[ + BaseClass( + access="public", + typename=PQName(segments=[NameSpecifier(name="T")]), + param_pack=True, + ) + ], + template=TemplateDecl( + params=[ + TemplateNonTypeParam( + type=Type( + typename=PQName( + segments=[NameSpecifier(name="T")] + ) + ), + param_pack=True, + ) + ] + ), + ) + ), + ] + ) + ) diff --git a/tests/test_tokfmt.py b/tests/test_tokfmt.py new file mode 100644 index 0000000..cff0f1a --- /dev/null +++ b/tests/test_tokfmt.py @@ -0,0 +1,52 @@ +import pytest + +from cxxheaderparser.lexer import Lexer +from cxxheaderparser.tokfmt import tokfmt + + +@pytest.mark.parametrize( + "instr", + [ + "int", + "unsigned int", + "::uint8_t", + "void *", + "const char *", + "const char[]", + "void * (*)()", + "void (*)(void * buf, int buflen)", + "void (* fnType)(void * buf, int buflen)", + "TypeName& x", + "vector&", + "std::vector *", + "Alpha::Omega", + "Convoluted::Nested::Mixin", + "std::function", + "std::shared_ptr>", + "tr1::shared_ptr>", + "std::map>>", + "std::is_base_of::value", + "const char&&", + "something{1, 2, 3}", + "operator-=", + "operator[]", + "operator*", + "operator>=", + ], +) +def test_tokfmt(instr): + """ + Each input string is exactly what the output of tokfmt should be + """ + toks = [] + lexer = Lexer("") + lexer.input(instr) + + while True: + tok = lexer.token_eof_ok() + if not tok: + break + + toks.append(tok) + + assert tokfmt(toks) == instr diff --git a/tests/test_typedef.py b/tests/test_typedef.py new file mode 100644 index 0000000..df96813 --- /dev/null +++ b/tests/test_typedef.py @@ -0,0 +1,870 @@ +# Note: testcases generated via `python -m cxxheaderparser.gentest` + + +from cxxheaderparser.types import ( + AnonymousName, + Array, + BaseClass, + ClassDecl, + EnumDecl, + Enumerator, + Field, + FunctionType, + FundamentalSpecifier, + NameSpecifier, + PQName, + Parameter, + Pointer, + Reference, + TemplateArgument, + TemplateSpecialization, + Token, + Type, + Typedef, + Value, +) +from cxxheaderparser.simple import ClassScope, NamespaceScope, ParsedData, parse_string + + +def test_simple_typedef(): + content = """ + typedef std::vector IntVector; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + typedefs=[ + Typedef( + type=Type( + typename=PQName( + segments=[ + NameSpecifier(name="std"), + NameSpecifier( + name="vector", + specialization=TemplateSpecialization( + args=[ + TemplateArgument( + tokens=[Token(value="int")] + ) + ] + ), + ), + ] + ) + ), + name="IntVector", + ) + ] + ) + ) + + +def test_struct_typedef_1(): + content = """ + typedef struct { + int m; + } unnamed_struct, *punnamed_struct; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[AnonymousName(id=1)], classkey="struct" + ) + ), + fields=[ + Field( + name="m", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + access="public", + ) + ], + ) + ], + typedefs=[ + Typedef( + type=Type( + typename=PQName( + segments=[AnonymousName(id=1)], classkey="struct" + ) + ), + name="unnamed_struct", + ), + Typedef( + type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[AnonymousName(id=1)], classkey="struct" + ) + ) + ), + name="punnamed_struct", + ), + ], + ) + ) + + +def test_struct_typedef_2(): + content = """ + typedef struct { + int m; + } * punnamed_struct, unnamed_struct; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[AnonymousName(id=1)], classkey="struct" + ) + ), + fields=[ + Field( + name="m", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + access="public", + ) + ], + ) + ], + typedefs=[ + Typedef( + type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[AnonymousName(id=1)], classkey="struct" + ) + ) + ), + name="punnamed_struct", + ), + Typedef( + type=Type( + typename=PQName( + segments=[AnonymousName(id=1)], classkey="struct" + ) + ), + name="unnamed_struct", + ), + ], + ) + ) + + +def test_typedef_array(): + content = """ + typedef char TenCharArray[10]; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + typedefs=[ + Typedef( + type=Array( + array_of=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="char")] + ) + ), + size=Value(tokens=[Token(value="10")]), + ), + name="TenCharArray", + ) + ] + ) + ) + + +def test_typedef_array_of_struct(): + content = """ + typedef struct{} tx[3], ty; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[AnonymousName(id=1)], classkey="struct" + ) + ) + ) + ], + typedefs=[ + Typedef( + type=Array( + array_of=Type( + typename=PQName( + segments=[AnonymousName(id=1)], classkey="struct" + ) + ), + size=Value(tokens=[Token(value="3")]), + ), + name="tx", + ), + Typedef( + type=Type( + typename=PQName( + segments=[AnonymousName(id=1)], classkey="struct" + ) + ), + name="ty", + ), + ], + ) + ) + + +def test_typedef_class_w_base(): + content = """ + typedef class XX : public F {} G; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="XX")], classkey="class" + ), + bases=[ + BaseClass( + access="public", + typename=PQName(segments=[NameSpecifier(name="F")]), + ) + ], + ) + ) + ], + typedefs=[ + Typedef( + type=Type( + typename=PQName( + segments=[NameSpecifier(name="XX")], classkey="class" + ) + ), + name="G", + ) + ], + ) + ) + + +def test_complicated_typedef(): + content = """ + typedef int int_t, *intp_t, (&fp)(int, ulong), arr_t[10]; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + typedefs=[ + Typedef( + type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + name="int_t", + ), + Typedef( + type=Pointer( + ptr_to=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ) + ), + name="intp_t", + ), + Typedef( + type=Reference( + ref_to=FunctionType( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ) + ), + Parameter( + type=Type( + typename=PQName( + segments=[NameSpecifier(name="ulong")] + ) + ) + ), + ], + ) + ), + name="fp", + ), + Typedef( + type=Array( + array_of=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + size=Value(tokens=[Token(value="10")]), + ), + name="arr_t", + ), + ] + ) + ) + + +def test_typedef_c_struct_idiom(): + content = """ + // common C idiom to avoid having to write "struct S" + typedef struct {int a; int b;} S, *pS; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[AnonymousName(id=1)], classkey="struct" + ) + ), + fields=[ + Field( + name="a", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + access="public", + ), + Field( + name="b", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + access="public", + ), + ], + ) + ], + typedefs=[ + Typedef( + type=Type( + typename=PQName( + segments=[AnonymousName(id=1)], classkey="struct" + ) + ), + name="S", + ), + Typedef( + type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[AnonymousName(id=1)], classkey="struct" + ) + ) + ), + name="pS", + ), + ], + ) + ) + + +def test_typedef_struct_same_name(): + content = """ + typedef struct Fig { + int a; + } Fig; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="Fig")], classkey="struct" + ) + ), + fields=[ + Field( + access="public", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="a", + ) + ], + ) + ], + typedefs=[ + Typedef( + type=Type( + typename=PQName( + segments=[NameSpecifier(name="Fig")], classkey="struct" + ) + ), + name="Fig", + ) + ], + ) + ) + + +def test_typedef_struct_w_enum(): + content = """ + typedef struct { + enum BeetEnum : int { FAIL = 0, PASS = 1 }; + } BeetStruct; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[AnonymousName(id=1)], classkey="struct" + ) + ), + enums=[ + EnumDecl( + typename=PQName( + segments=[NameSpecifier(name="BeetEnum")], + classkey="enum", + ), + values=[ + Enumerator( + name="FAIL", value=Value(tokens=[Token(value="0")]) + ), + Enumerator( + name="PASS", value=Value(tokens=[Token(value="1")]) + ), + ], + base=PQName(segments=[FundamentalSpecifier(name="int")]), + access="public", + ) + ], + ) + ], + typedefs=[ + Typedef( + type=Type( + typename=PQName( + segments=[AnonymousName(id=1)], classkey="struct" + ) + ), + name="BeetStruct", + ) + ], + ) + ) + + +def test_typedef_union(): + content = """ + typedef union apricot_t { + int i; + float f; + char s[20]; + } Apricot; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="apricot_t")], classkey="union" + ) + ), + fields=[ + Field( + access="public", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="i", + ), + Field( + access="public", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="float")] + ) + ), + name="f", + ), + Field( + access="public", + type=Array( + array_of=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="char")] + ) + ), + size=Value(tokens=[Token(value="20")]), + ), + name="s", + ), + ], + ) + ], + typedefs=[ + Typedef( + type=Type( + typename=PQName( + segments=[NameSpecifier(name="apricot_t")], classkey="union" + ) + ), + name="Apricot", + ) + ], + ) + ) + + +def test_typedef_fnptr(): + content = """ + typedef void *(*fndef)(int); + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + typedefs=[ + Typedef( + type=Pointer( + ptr_to=FunctionType( + return_type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ) + ), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ) + ) + ], + ) + ), + name="fndef", + ) + ] + ) + ) + + +def test_typedef_const(): + content = """ + typedef int theint, *const ptheint; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + typedefs=[ + Typedef( + type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + name="theint", + ), + Typedef( + type=Pointer( + ptr_to=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + const=True, + ), + name="ptheint", + ), + ] + ) + ) + + +def test_enum_typedef_1(): + content = """ + typedef enum {} E; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName(segments=[AnonymousName(id=1)], classkey="enum"), + values=[], + ) + ], + typedefs=[ + Typedef( + type=Type( + typename=PQName(segments=[AnonymousName(id=1)], classkey="enum") + ), + name="E", + ) + ], + ) + ) + + +def test_enum_typedef_2(): + content = """ + typedef enum { E1 } BE; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName(segments=[AnonymousName(id=1)], classkey="enum"), + values=[Enumerator(name="E1")], + ) + ], + typedefs=[ + Typedef( + type=Type( + typename=PQName(segments=[AnonymousName(id=1)], classkey="enum") + ), + name="BE", + ) + ], + ) + ) + + +def test_enum_typedef_3(): + content = """ + typedef enum { E1, E2, } E; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName(segments=[AnonymousName(id=1)], classkey="enum"), + values=[Enumerator(name="E1"), Enumerator(name="E2")], + ) + ], + typedefs=[ + Typedef( + type=Type( + typename=PQName(segments=[AnonymousName(id=1)], classkey="enum") + ), + name="E", + ) + ], + ) + ) + + +def test_enum_typedef_3(): + content = """ + typedef enum { E1 } * PBE; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName(segments=[AnonymousName(id=1)], classkey="enum"), + values=[Enumerator(name="E1")], + ) + ], + typedefs=[ + Typedef( + type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[AnonymousName(id=1)], classkey="enum" + ) + ) + ), + name="PBE", + ) + ], + ) + ) + + +def test_enum_typedef_4(): + content = """ + typedef enum { E1 } * PBE, BE; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName(segments=[AnonymousName(id=1)], classkey="enum"), + values=[Enumerator(name="E1")], + ) + ], + typedefs=[ + Typedef( + type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[AnonymousName(id=1)], classkey="enum" + ) + ) + ), + name="PBE", + ), + Typedef( + type=Type( + typename=PQName(segments=[AnonymousName(id=1)], classkey="enum") + ), + name="BE", + ), + ], + ) + ) + + +def test_enum_typedef_5(): + content = """ + typedef enum { E1 } BE, *PBE; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName(segments=[AnonymousName(id=1)], classkey="enum"), + values=[Enumerator(name="E1")], + ) + ], + typedefs=[ + Typedef( + type=Type( + typename=PQName(segments=[AnonymousName(id=1)], classkey="enum") + ), + name="BE", + ), + Typedef( + type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[AnonymousName(id=1)], classkey="enum" + ) + ) + ), + name="PBE", + ), + ], + ) + ) + + +def test_enum_typedef_fwd(): + content = """ + typedef enum BE BET; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + typedefs=[ + Typedef( + type=Type( + typename=PQName( + segments=[NameSpecifier(name="BE")], classkey="enum" + ) + ), + name="BET", + ) + ] + ) + ) + + +def test_typedef_enum_expr(): + content = """ + typedef enum { StarFruit = (2 + 2) / 2 } Carambola; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName(segments=[AnonymousName(id=1)], classkey="enum"), + values=[ + Enumerator( + name="StarFruit", + value=Value( + tokens=[ + Token(value="("), + Token(value="2"), + Token(value="+"), + Token(value="2"), + Token(value=")"), + Token(value="/"), + Token(value="2"), + ] + ), + ) + ], + ) + ], + typedefs=[ + Typedef( + type=Type( + typename=PQName(segments=[AnonymousName(id=1)], classkey="enum") + ), + name="Carambola", + ) + ], + ) + ) diff --git a/tests/test_union.py b/tests/test_union.py new file mode 100644 index 0000000..d9a660f --- /dev/null +++ b/tests/test_union.py @@ -0,0 +1,154 @@ +# Note: testcases generated via `python -m cxxheaderparser.gentest` + +from cxxheaderparser.types import ( + AnonymousName, + ClassDecl, + Field, + FundamentalSpecifier, + NameSpecifier, + PQName, + Type, +) +from cxxheaderparser.simple import ( + ClassScope, + NamespaceScope, + parse_string, + ParsedData, +) + + +def test_union_basic(): + content = """ + + struct HAL_Value { + union { + int v_int; + HAL_Bool v_boolean; + } data; + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="HAL_Value")], + classkey="struct", + ) + ), + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[AnonymousName(id=1)], classkey="union" + ), + access="public", + ), + fields=[ + Field( + access="public", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="v_int", + ), + Field( + access="public", + type=Type( + typename=PQName( + segments=[NameSpecifier(name="HAL_Bool")] + ) + ), + name="v_boolean", + ), + ], + ) + ], + fields=[ + Field( + access="public", + type=Type( + typename=PQName( + segments=[AnonymousName(id=1)], classkey="union" + ) + ), + name="data", + ) + ], + ) + ] + ) + ) + + +def test_union_anon_in_struct(): + content = """ + struct Outer { + union { + int x; + int y; + }; + int z; + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="Outer")], classkey="struct" + ) + ), + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[AnonymousName(id=1)], classkey="union" + ), + access="public", + ), + fields=[ + Field( + access="public", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="x", + ), + Field( + access="public", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="y", + ), + ], + ) + ], + fields=[ + Field( + access="public", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="z", + ) + ], + ) + ] + ) + ) diff --git a/tests/test_using.py b/tests/test_using.py new file mode 100644 index 0000000..31d45ef --- /dev/null +++ b/tests/test_using.py @@ -0,0 +1,511 @@ +# Note: testcases generated via `python -m cxxheaderparser.gentest` + +from cxxheaderparser.types import ( + BaseClass, + ClassDecl, + Function, + FundamentalSpecifier, + Method, + NameSpecifier, + PQName, + Parameter, + Pointer, + Reference, + TemplateArgument, + TemplateDecl, + TemplateSpecialization, + TemplateTypeParam, + Token, + Type, + UsingAlias, + UsingDecl, +) +from cxxheaderparser.simple import ( + ClassScope, + NamespaceScope, + UsingNamespace, + parse_string, + ParsedData, +) + + +def test_using_namespace(): + content = """ + using namespace foo; + using namespace foo::bar; + using namespace ::foo; + using namespace ::foo::bar; + + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + using_ns=[ + UsingNamespace(ns="foo"), + UsingNamespace(ns="foo::bar"), + UsingNamespace(ns="::foo"), + UsingNamespace(ns="::foo::bar"), + ] + ) + ) + + +def test_using_declaration(): + content = """ + using ::foo; + using foo::bar; + using ::foo::bar; + using typename ::foo::bar; + using typename foo::bar; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + using=[ + UsingDecl( + typename=PQName( + segments=[NameSpecifier(name=""), NameSpecifier(name="foo")] + ) + ), + UsingDecl( + typename=PQName( + segments=[NameSpecifier(name="foo"), NameSpecifier(name="bar")] + ) + ), + UsingDecl( + typename=PQName( + segments=[ + NameSpecifier(name=""), + NameSpecifier(name="foo"), + NameSpecifier(name="bar"), + ] + ) + ), + UsingDecl( + typename=PQName( + segments=[ + NameSpecifier(name=""), + NameSpecifier(name="foo"), + NameSpecifier(name="bar"), + ] + ) + ), + UsingDecl( + typename=PQName( + segments=[NameSpecifier(name="foo"), NameSpecifier(name="bar")] + ) + ), + ] + ) + ) + + +# alias-declaration +def test_alias_declaration_1(): + content = """ + using alias = foo; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + using_alias=[ + UsingAlias( + alias="alias", + type=Type(typename=PQName(segments=[NameSpecifier(name="foo")])), + ) + ] + ) + ) + + +def test_alias_declaration_2(): + content = """ + template using alias = foo; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + using_alias=[ + UsingAlias( + alias="alias", + type=Type( + typename=PQName( + segments=[ + NameSpecifier( + name="foo", + specialization=TemplateSpecialization( + args=[ + TemplateArgument(tokens=[Token(value="T")]) + ] + ), + ) + ] + ) + ), + template=TemplateDecl( + params=[TemplateTypeParam(typekey="typename", name="T")] + ), + ) + ] + ) + ) + + +def test_alias_declaration_3(): + content = """ + using alias = ::foo::bar; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + using_alias=[ + UsingAlias( + alias="alias", + type=Type( + typename=PQName( + segments=[ + NameSpecifier(name=""), + NameSpecifier(name="foo"), + NameSpecifier(name="bar"), + ] + ) + ), + ) + ] + ) + ) + + +def test_alias_declaration_4(): + content = """ + template using alias = ::foo::bar; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + using_alias=[ + UsingAlias( + alias="alias", + type=Type( + typename=PQName( + segments=[ + NameSpecifier(name=""), + NameSpecifier(name="foo"), + NameSpecifier( + name="bar", + specialization=TemplateSpecialization( + args=[ + TemplateArgument(tokens=[Token(value="T")]) + ] + ), + ), + ] + ) + ), + template=TemplateDecl( + params=[TemplateTypeParam(typekey="typename", name="T")] + ), + ) + ] + ) + ) + + +def test_alias_declaration_5(): + content = """ + using alias = foo::bar; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + using_alias=[ + UsingAlias( + alias="alias", + type=Type( + typename=PQName( + segments=[ + NameSpecifier(name="foo"), + NameSpecifier(name="bar"), + ] + ) + ), + ) + ] + ) + ) + + +def test_alias_declaration_6(): + content = """ + template using alias = foo::bar; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + using_alias=[ + UsingAlias( + alias="alias", + type=Type( + typename=PQName( + segments=[ + NameSpecifier( + name="foo", + specialization=TemplateSpecialization( + args=[ + TemplateArgument(tokens=[Token(value="T")]) + ] + ), + ), + NameSpecifier(name="bar"), + ] + ) + ), + template=TemplateDecl( + params=[TemplateTypeParam(typekey="typename", name="T")] + ), + ) + ] + ) + ) + + +def test_using_many_things(): + content = """ + // clang-format off + + using std::thing; + using MyThing = SomeThing; + namespace a { + using std::string; + using VoidFunction = std::function; + + void fn(string &s, VoidFunction fn, thing * t); + + class A : public B { + public: + using B::B; + using IntFunction = std::function; + + void a(string &s, IntFunction fn, thing * t); + }; + } + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + using=[ + UsingDecl( + typename=PQName( + segments=[ + NameSpecifier(name="std"), + NameSpecifier(name="thing"), + ] + ) + ) + ], + using_alias=[ + UsingAlias( + alias="MyThing", + type=Type( + typename=PQName(segments=[NameSpecifier(name="SomeThing")]) + ), + ) + ], + namespaces={ + "a": NamespaceScope( + name="a", + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="A")], classkey="class" + ), + bases=[ + BaseClass( + access="public", + typename=PQName( + segments=[NameSpecifier(name="B")] + ), + ) + ], + ), + methods=[ + Method( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="a")]), + parameters=[ + Parameter( + type=Reference( + ref_to=Type( + typename=PQName( + segments=[ + NameSpecifier(name="string") + ] + ) + ) + ), + name="s", + ), + Parameter( + type=Type( + typename=PQName( + segments=[ + NameSpecifier( + name="IntFunction" + ) + ] + ) + ), + name="fn", + ), + Parameter( + type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[ + NameSpecifier(name="thing") + ] + ) + ) + ), + name="t", + ), + ], + access="public", + ) + ], + using=[ + UsingDecl( + typename=PQName( + segments=[ + NameSpecifier(name="B"), + NameSpecifier(name="B"), + ] + ), + access="public", + ) + ], + using_alias=[ + UsingAlias( + alias="IntFunction", + type=Type( + typename=PQName( + segments=[ + NameSpecifier(name="std"), + NameSpecifier( + name="function", + specialization=TemplateSpecialization( + args=[ + TemplateArgument( + tokens=[ + Token(value="int"), + Token(value="("), + Token(value=")"), + ] + ) + ] + ), + ), + ] + ) + ), + access="public", + ) + ], + ) + ], + functions=[ + Function( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + name=PQName(segments=[NameSpecifier(name="fn")]), + parameters=[ + Parameter( + type=Reference( + ref_to=Type( + typename=PQName( + segments=[NameSpecifier(name="string")] + ) + ) + ), + name="s", + ), + Parameter( + type=Type( + typename=PQName( + segments=[ + NameSpecifier(name="VoidFunction") + ] + ) + ), + name="fn", + ), + Parameter( + type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[NameSpecifier(name="thing")] + ) + ) + ), + name="t", + ), + ], + ) + ], + using=[ + UsingDecl( + typename=PQName( + segments=[ + NameSpecifier(name="std"), + NameSpecifier(name="string"), + ] + ) + ) + ], + using_alias=[ + UsingAlias( + alias="VoidFunction", + type=Type( + typename=PQName( + segments=[ + NameSpecifier(name="std"), + NameSpecifier( + name="function", + specialization=TemplateSpecialization( + args=[ + TemplateArgument( + tokens=[ + Token(value="void"), + Token(value="("), + Token(value=")"), + ] + ) + ] + ), + ), + ] + ) + ), + ) + ], + ) + }, + ) + ) diff --git a/tests/test_var.py b/tests/test_var.py new file mode 100644 index 0000000..3621959 --- /dev/null +++ b/tests/test_var.py @@ -0,0 +1,768 @@ +# Note: testcases generated via `python -m cxxheaderparser.gentest` + + +from cxxheaderparser.types import ( + Array, + ClassDecl, + EnumDecl, + Enumerator, + Field, + FunctionType, + FundamentalSpecifier, + NameSpecifier, + PQName, + Parameter, + Pointer, + Reference, + Token, + Type, + Value, + Variable, +) +from cxxheaderparser.simple import ClassScope, NamespaceScope, ParsedData, parse_string + + +def test_var_unixwiz_ridiculous(): + # http://unixwiz.net/techtips/reading-cdecl.html + # + # .. "we have no idea how this variable is useful, but at least we can + # describe the type correctly" + + content = """ + char *(*(**foo[][8])())[]; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="foo")]), + type=Array( + array_of=Array( + array_of=Pointer( + ptr_to=Pointer( + ptr_to=FunctionType( + return_type=Pointer( + ptr_to=Array( + array_of=Pointer( + ptr_to=Type( + typename=PQName( + segments=[ + FundamentalSpecifier( + name="char" + ) + ] + ) + ) + ), + size=None, + ) + ), + parameters=[], + ) + ) + ), + size=Value(tokens=[Token(value="8")]), + ), + size=None, + ), + ) + ] + ) + ) + + +def test_var_ptr_to_array15_of_ptr_to_int(): + content = """ + int *(*crocodile)[15]; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="crocodile")]), + type=Pointer( + ptr_to=Array( + array_of=Pointer( + ptr_to=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ) + ), + size=Value(tokens=[Token(value="15")]), + ) + ), + ) + ] + ) + ) + + +def test_var_ref_to_array(): + content = """ + int abase[3]; + int (&aname)[3] = abase; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="abase")]), + type=Array( + array_of=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + size=Value(tokens=[Token(value="3")]), + ), + ), + Variable( + name=PQName(segments=[NameSpecifier(name="aname")]), + type=Reference( + ref_to=Array( + array_of=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + size=Value(tokens=[Token(value="3")]), + ) + ), + value=Value(tokens=[Token(value="abase")]), + ), + ] + ) + ) + + +def test_var_ptr_to_array(): + content = """ + int zz, (*aname)[3] = &abase; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="zz")]), + type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + ), + Variable( + name=PQName(segments=[NameSpecifier(name="aname")]), + type=Pointer( + ptr_to=Array( + array_of=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + size=Value(tokens=[Token(value="3")]), + ) + ), + value=Value(tokens=[Token(value="&"), Token(value="abase")]), + ), + ] + ) + ) + + +def test_var_multi_1(): + content = """ + int zz, (&aname)[3] = abase; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="zz")]), + type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + ), + Variable( + name=PQName(segments=[NameSpecifier(name="aname")]), + type=Reference( + ref_to=Array( + array_of=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + size=Value(tokens=[Token(value="3")]), + ) + ), + value=Value(tokens=[Token(value="abase")]), + ), + ] + ) + ) + + +def test_var_array_of_fnptr_varargs(): + content = """ + void (*a3[3])(int, ...); + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="a3")]), + type=Array( + array_of=Pointer( + ptr_to=FunctionType( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[ + FundamentalSpecifier(name="int") + ] + ) + ) + ) + ], + vararg=True, + ) + ), + size=Value(tokens=[Token(value="3")]), + ), + ) + ] + ) + ) + + +def test_var_double_fnptr_varargs(): + content = """ + void (*(*a4))(int, ...); + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="a4")]), + type=Pointer( + ptr_to=Pointer( + ptr_to=FunctionType( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[ + FundamentalSpecifier(name="int") + ] + ) + ) + ) + ], + vararg=True, + ) + ) + ), + ) + ] + ) + ) + + +def test_var_fnptr_voidstar(): + content = """ + void(*(*a5)(int)); + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="a5")]), + type=Pointer( + ptr_to=FunctionType( + return_type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ) + ), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ) + ) + ], + ) + ), + ) + ] + ) + ) + + +def test_var_fnptr_moreparens(): + content = """ + void (*x)(int(p1), int); + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="x")]), + type=Pointer( + ptr_to=FunctionType( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + parameters=[ + Parameter( + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="p1", + ), + Parameter( + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ) + ), + ], + ) + ), + ) + ] + ) + ) + + +# From pycparser: +# Pointer decls nest from inside out. This is important when different +# levels have different qualifiers. For example: +# +# char * const * p; +# +# Means "pointer to const pointer to char" +# +# While: +# +# char ** const p; +# +# Means "const pointer to pointer to char" + + +def test_var_ptr_to_const_ptr_to_char(): + content = """ + char *const *p; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="p")]), + type=Pointer( + ptr_to=Pointer( + ptr_to=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="char")] + ) + ), + const=True, + ) + ), + ) + ] + ) + ) + + +def test_var_const_ptr_to_ptr_to_char(): + content = """ + char **const p; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="p")]), + type=Pointer( + ptr_to=Pointer( + ptr_to=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="char")] + ) + ) + ), + const=True, + ), + ) + ] + ) + ) + + +def test_var_array_initializer1(): + content = """ + int x[3]{1, 2, 3}; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="x")]), + type=Array( + array_of=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + size=Value(tokens=[Token(value="3")]), + ), + value=Value( + tokens=[ + Token(value="{"), + Token(value="1"), + Token(value=","), + Token(value="2"), + Token(value=","), + Token(value="3"), + Token(value="}"), + ] + ), + ) + ] + ) + ) + + +def test_var_array_initializer2(): + content = """ + int x[3] = {1, 2, 3}; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="x")]), + type=Array( + array_of=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + size=Value(tokens=[Token(value="3")]), + ), + value=Value( + tokens=[ + Token(value="{"), + Token(value="1"), + Token(value=","), + Token(value="2"), + Token(value=","), + Token(value="3"), + Token(value="}"), + ] + ), + ) + ] + ) + ) + + +def test_var_extern_c(): + content = """ + extern "C" int x; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="x")]), + type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + # TODO: store linkage + extern=True, + ) + ] + ) + ) + + +def test_var_ns_1(): + content = """ + int N::x; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + variables=[ + Variable( + name=PQName( + segments=[NameSpecifier(name="N"), NameSpecifier(name="x")] + ), + type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + ) + ] + ) + ) + + +def test_var_ns_2(): + content = """ + int N::x = 4; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + variables=[ + Variable( + name=PQName( + segments=[NameSpecifier(name="N"), NameSpecifier(name="x")] + ), + type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + value=Value(tokens=[Token(value="4")]), + ) + ] + ) + ) + + +def test_var_ns_3(): + content = """ + int N::x{4}; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + variables=[ + Variable( + name=PQName( + segments=[NameSpecifier(name="N"), NameSpecifier(name="x")] + ), + type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + value=Value( + tokens=[Token(value="{"), Token(value="4"), Token(value="}")] + ), + ) + ] + ) + ) + + +def test_var_static_struct(): + content = """ + constexpr static struct SS {} s; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="SS")], classkey="struct" + ) + ) + ) + ], + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="s")]), + type=Type( + typename=PQName( + segments=[NameSpecifier(name="SS")], classkey="struct" + ) + ), + constexpr=True, + static=True, + ) + ], + ) + ) + + +def test_var_constexpr_enum(): + content = """ + constexpr enum E { EE } e = EE; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName( + segments=[NameSpecifier(name="E")], classkey="enum" + ), + values=[Enumerator(name="EE")], + ) + ], + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="e")]), + type=Type( + typename=PQName( + segments=[NameSpecifier(name="E")], classkey="enum" + ) + ), + value=Value(tokens=[Token(value="EE")]), + constexpr=True, + ) + ], + ) + ) + + +def test_var_fnptr_in_class(): + content = """ + struct DriverFuncs { + void *(*init)(); + void (*write)(void *buf, int buflen); + }; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="DriverFuncs")], + classkey="struct", + ) + ), + fields=[ + Field( + access="public", + type=Pointer( + ptr_to=FunctionType( + return_type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[ + FundamentalSpecifier(name="void") + ] + ) + ) + ), + parameters=[], + ) + ), + name="init", + ), + Field( + access="public", + type=Pointer( + ptr_to=FunctionType( + return_type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="void")] + ) + ), + parameters=[ + Parameter( + type=Pointer( + ptr_to=Type( + typename=PQName( + segments=[ + FundamentalSpecifier( + name="void" + ) + ] + ) + ) + ), + name="buf", + ), + Parameter( + type=Type( + typename=PQName( + segments=[ + FundamentalSpecifier(name="int") + ] + ) + ), + name="buflen", + ), + ], + ) + ), + name="write", + ), + ], + ) + ] + ) + ) + + +def test_var_extern(): + content = """ + extern int externVar; + """ + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="externVar")]), + type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + extern=True, + ) + ] + ) + )