Merge pull request #61 from robotpy/preprocessor

Add easy to use preprocessor support

.github/workflows/dist.yml

@@ -27,7 +27,7 @@ jobs:
         python-version: 3.8
     - name: Install requirements
       run: |
-        pip --disable-pip-version-check install mypy pytest
+        pip --disable-pip-version-check install mypy pytest pcpp
     - name: Run mypy
       run: |
         mypy .

README.md

@@ -31,6 +31,7 @@ Non-goals:
   headers that contain macros, you should preprocess your code using the
   excellent pure python preprocessor [pcpp](https://github.com/ned14/pcpp)
   or your favorite compiler
+  * See `cxxheaderparser.preprocessor` for how to use
 * Probably won't be able to parse most IOCCC entries
 
 There are two APIs available:

cxxheaderparser/dump.py

@@ -23,10 +23,19 @@ def dumpmain() -> None:
     parser.add_argument(
         "--mode", choices=["json", "pprint", "repr", "brepr"], default="pprint"
     )
+    parser.add_argument(
+        "--pcpp", default=False, action="store_true", help="Use pcpp preprocessor"
+    )
 
     args = parser.parse_args()
 
-    options = ParserOptions(verbose=args.verbose)
+    preprocessor = None
+    if args.pcpp:
+        from .preprocessor import make_pcpp_preprocessor
+
+        preprocessor = make_pcpp_preprocessor()
+
+    options = ParserOptions(verbose=args.verbose, preprocessor=preprocessor)
     data = parse_file(args.header, options=options)
 
     if args.mode == "pprint":

cxxheaderparser/options.py

@@ -1,4 +1,8 @@
 from dataclasses import dataclass
+from typing import Callable, Optional
+
+#: arguments are (filename, content)
+PreprocessorFunction = Callable[[str, str], str]
 
 
 @dataclass
@@ -12,3 +16,7 @@ class ParserOptions:
 
     #: If true, converts a single void parameter to zero parameters
     convert_void_to_zero_params: bool = True
+
+    #: A function that will preprocess the header before parsing. See
+    #: :py:mod:`cxxheaderparser.preprocessor` for available preprocessors
+    preprocessor: Optional[PreprocessorFunction] = None

cxxheaderparser/parser.py

@@ -81,6 +81,10 @@ class CxxParser:
     ) -> None:
         self.visitor = visitor
         self.filename = filename
+        self.options = options if options else ParserOptions()
+
+        if options and options.preprocessor is not None:
+            content = options.preprocessor(filename, content)
 
         self.lex: lexer.TokenStream = lexer.LexerTokenStream(filename, content)
 
@@ -90,8 +94,6 @@ class CxxParser:
         self.state: State = NamespaceBlockState(None, global_ns)
         self.anon_id = 0
 
-        self.options = options if options else ParserOptions()
-
         self.verbose = True if self.options.verbose else False
         if self.verbose:
 

cxxheaderparser/preprocessor.py

@@ -0,0 +1,106 @@
+"""
+Contains optional preprocessor support via pcpp
+"""
+
+import io
+from os.path import relpath
+import typing
+from .options import PreprocessorFunction
+
+from pcpp import Preprocessor, OutputDirective, Action
+
+
+class PreprocessorError(Exception):
+    pass
+
+
+class _CustomPreprocessor(Preprocessor):
+    def __init__(self):
+        Preprocessor.__init__(self)
+        self.errors = []
+
+    def on_error(self, file, line, msg):
+        self.errors.append(f"{file}:{line} error: {msg}")
+
+    def on_include_not_found(self, *ignored):
+        raise OutputDirective(Action.IgnoreAndPassThrough)
+
+    def on_comment(self, *ignored):
+        return True
+
+
+def _filter_self(fname: str, fp: typing.TextIO) -> str:
+    # the output of pcpp includes the contents of all the included files, which
+    # isn't what a typical user of cxxheaderparser would want, so we strip out
+    # the line directives and any content that isn't in our original file
+
+    # Compute the filename to match based on how pcpp does it
+    try:
+        relfname = relpath(fname)
+    except Exception:
+        relfname = fname
+    relfname = relfname.replace("\\", "/")
+
+    relfname += '"\n'
+
+    new_output = io.StringIO()
+    keep = True
+
+    for line in fp:
+        if line.startswith("#line"):
+            keep = line.endswith(relfname)
+
+        if keep:
+            new_output.write(line)
+
+    new_output.seek(0)
+    return new_output.read()
+
+
+def make_pcpp_preprocessor(
+    *,
+    defines: typing.List[str] = [],
+    include_paths: typing.List[str] = [],
+    retain_all_content: bool = False,
+) -> PreprocessorFunction:
+    """
+    Creates a preprocessor function that uses pcpp (which must be installed
+    separately) to preprocess the input text.
+
+    .. code-block:: python
+
+        pp = make_pcpp_preprocessor()
+        options = ParserOptions(preprocessor=pp)
+
+        parse_file(content, options=options)
+
+    """
+
+    def _preprocess_file(filename: str, content: str) -> str:
+        pp = _CustomPreprocessor()
+        if include_paths:
+            for p in include_paths:
+                pp.add_path(p)
+
+        for define in defines:
+            pp.define(define)
+
+        if not retain_all_content:
+            pp.line_directive = "#line"
+
+        pp.parse(content, filename)
+
+        if pp.errors:
+            raise PreprocessorError("\n".join(pp.errors))
+        elif pp.return_code:
+            raise PreprocessorError("failed with exit code %d" % pp.return_code)
+
+        fp = io.StringIO()
+        pp.write(fp)
+        fp.seek(0)
+        if retain_all_content:
+            return fp.read()
+        else:
+            return _filter_self(filename, fp)
+
+    return _preprocess_file

docs/custom.rst

@@ -35,3 +35,10 @@ Parser state
 .. automodule:: cxxheaderparser.parserstate
    :members:
    :undoc-members:
+
+Preprocessor
+------------
+
+.. automodule:: cxxheaderparser.preprocessor
+   :members:
+   :undoc-members:

docs/index.rst

@@ -10,10 +10,11 @@ A pure python C++ header parser that parses C++ headers in a mildly naive
 manner that allows it to handle many C++ constructs, including many modern
 (C++11 and beyond) features.
 
-.. warning:: cxxheaderparser intentionally does not have a C preprocessor
+.. warning:: cxxheaderparser intentionally does not use a C preprocessor by
-             implementation! If you are parsing code with macros in it, use
+             default. If you are parsing code with macros in it, you need to
-             a conforming preprocessor like the pure python preprocessor
+             provide a preprocessor function in :py:class:`.ParserOptions`.
-             `pcpp`_ or your favorite C++ compiler.
+
+             .. seealso:: :py:attr:`cxxheaderparser.options.ParserOptions.preprocessor`
 
 .. _pcpp: https://github.com/ned14/pcpp
 

docs/requirements.txt

@@ -1,3 +1,4 @@
 sphinx >= 3.0
 sphinx-rtd-theme
 sphinx-autodoc-typehints
+pcpp

mypy.ini

@@ -1,5 +1,8 @@
 [mypy]
 exclude = setup\.py|docs
 
+[mypy-pcpp.*]
+ignore_missing_imports = True
+
 [mypy-cxxheaderparser._ply.*]
 ignore_errors = True

setup.py

@@ -68,6 +68,7 @@ setup(
     long_description=open("README.md").read(),
     long_description_content_type="text/markdown",
     install_requires=["dataclasses; python_version < '3.7'"],
+    extras_require={"pcpp": ["pcpp~=1.30"]},
     license="BSD",
     platforms="Platform Independent",
     packages=find_packages(),