Merge pull request #78 from robotpy/msvc-preprocessor

Add MSVC compatible preprocessing function

.github/workflows/dist.yml

@@ -112,6 +112,10 @@ jobs:
     - name: Install test dependencies
       run: python -m pip --disable-pip-version-check install -r tests/requirements.txt
     
+    - name: Setup MSVC compiler
+      uses: ilammy/msvc-dev-cmd@v1
+      if: matrix.os == 'windows-latest'
+
     - name: Test wheel
       shell: bash
       run: |

.readthedocs.yml

@@ -0,0 +1,15 @@
+version: 2
+
+sphinx:
+  configuration: docs/conf.py
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+python:
+  install:
+  - requirements: docs/requirements.txt
+  - method: pip
+    path: .

cxxheaderparser/lexer.py

@@ -17,7 +17,7 @@ if sys.version_info >= (3, 8):
 else:
     Protocol = object
 
-_line_re = re.compile(r'^\#[\t ]*line (\d+) "(.*)"')
+_line_re = re.compile(r'^\#[\t ]*(line)? (\d+) "(.*)"')
 _multicomment_re = re.compile("\n[\\s]+\\*")
 
 
@@ -448,8 +448,8 @@ class PlyLexer:
         # handle line macros
         m = _line_re.match(t.value)
         if m:
-            self.filename = m.group(2)
-            self.line_offset = 1 + self.lex.lineno - int(m.group(1))
+            self.filename = m.group(3)
+            self.line_offset = 1 + self.lex.lineno - int(m.group(2))
             return None
         # ignore C++23 warning directive
         if t.value.startswith("#warning"):

cxxheaderparser/options.py

@@ -2,7 +2,7 @@ from dataclasses import dataclass
 from typing import Callable, Optional
 
 #: arguments are (filename, content)
-PreprocessorFunction = Callable[[str, str], str]
+PreprocessorFunction = Callable[[str, Optional[str]], str]
 
 
 @dataclass

cxxheaderparser/parser.py

@@ -74,9 +74,10 @@ class CxxParser:
     def __init__(
         self,
         filename: str,
-        content: str,
+        content: typing.Optional[str],
         visitor: CxxVisitor,
         options: typing.Optional[ParserOptions] = None,
+        encoding: typing.Optional[str] = None,
     ) -> None:
         self.visitor = visitor
         self.filename = filename
@@ -85,6 +86,13 @@ class CxxParser:
         if options and options.preprocessor is not None:
             content = options.preprocessor(filename, content)
 
+        if content is None:
+            if encoding is None:
+                encoding = "utf-8-sig"
+
+            with open(filename, "r", encoding=encoding) as fp:
+                content = fp.read()
+
         self.lex: lexer.TokenStream = lexer.LexerTokenStream(filename, content)
 
         global_ns = NamespaceDecl([], False)

cxxheaderparser/preprocessor.py

@@ -1,37 +1,248 @@
 """
-Contains optional preprocessor support via pcpp
+Contains optional preprocessor support functions
 """
 
 import io
 import re
 import os
+import subprocess
+import sys
+import tempfile
 import typing
-from .options import PreprocessorFunction
 
-from pcpp import Preprocessor, OutputDirective, Action
+from .options import PreprocessorFunction
 
 
 class PreprocessorError(Exception):
     pass
 
 
-class _CustomPreprocessor(Preprocessor):
-    def __init__(self, encoding: typing.Optional[str]):
-        Preprocessor.__init__(self)
-        self.errors: typing.List[str] = []
-        self.assume_encoding = encoding
-
-    def on_error(self, file, line, msg):
-        self.errors.append(f"{file}:{line} error: {msg}")
-
-    def on_include_not_found(self, *ignored):
-        raise OutputDirective(Action.IgnoreAndPassThrough)
-
-    def on_comment(self, *ignored):
-        return True
+#
+# GCC preprocessor support
+#
 
 
-def _filter_self(fname: str, fp: typing.TextIO) -> str:
+def _gcc_filter(fname: str, fp: typing.TextIO) -> str:
+    new_output = io.StringIO()
+    keep = True
+    fname = fname.replace("\\", "\\\\")
+
+    for line in fp:
+        if line.startswith("# "):
+            last_quote = line.rfind('"')
+            if last_quote != -1:
+                keep = line[:last_quote].endswith(fname)
+
+        if keep:
+            new_output.write(line)
+
+    new_output.seek(0)
+    return new_output.read()
+
+
+def make_gcc_preprocessor(
+    *,
+    defines: typing.List[str] = [],
+    include_paths: typing.List[str] = [],
+    retain_all_content: bool = False,
+    encoding: typing.Optional[str] = None,
+    gcc_args: typing.List[str] = ["g++"],
+    print_cmd: bool = True,
+) -> PreprocessorFunction:
+    """
+    Creates a preprocessor function that uses g++ to preprocess the input text.
+
+    gcc is a high performance and accurate precompiler, but if an #include
+    directive can't be resolved or other oddity exists in your input it will
+    throw an error.
+
+    :param defines: list of #define macros specified as "key value"
+    :param include_paths: list of directories to search for included files
+    :param retain_all_content: If False, only the parsed file content will be retained
+    :param encoding: If specified any include files are opened with this encoding
+    :param gcc_args: This is the path to G++ and any extra args you might want
+    :param print_cmd: Prints the gcc command as its executed
+
+    .. code-block:: python
+
+        pp = make_gcc_preprocessor()
+        options = ParserOptions(preprocessor=pp)
+
+        parse_file(content, options=options)
+
+    """
+
+    if not encoding:
+        encoding = "utf-8"
+
+    def _preprocess_file(filename: str, content: typing.Optional[str]) -> str:
+        cmd = gcc_args + ["-w", "-E", "-C"]
+
+        for p in include_paths:
+            cmd.append(f"-I{p}")
+        for d in defines:
+            cmd.append(f"-D{d.replace(' ', '=')}")
+
+        kwargs = {"encoding": encoding}
+        if filename == "<str>":
+            cmd.append("-")
+            filename = "<stdin>"
+            if content is None:
+                raise PreprocessorError("no content specified for stdin")
+            kwargs["input"] = content
+        else:
+            cmd.append(filename)
+
+        if print_cmd:
+            print("+", " ".join(cmd), file=sys.stderr)
+
+        result: str = subprocess.check_output(cmd, **kwargs)  # type: ignore
+        if not retain_all_content:
+            result = _gcc_filter(filename, io.StringIO(result))
+
+        return result
+
+    return _preprocess_file
+
+
+#
+# Microsoft Visual Studio preprocessor support
+#
+
+
+def _msvc_filter(fp: typing.TextIO) -> str:
+    # MSVC outputs the original file as the very first #line directive
+    # so we just use that
+    new_output = io.StringIO()
+    keep = True
+
+    first = fp.readline()
+    assert first.startswith("#line")
+    fname = first[first.find('"') :]
+
+    for line in fp:
+        if line.startswith("#line"):
+            keep = line.endswith(fname)
+
+        if keep:
+            new_output.write(line)
+
+    new_output.seek(0)
+    return new_output.read()
+
+
+def make_msvc_preprocessor(
+    *,
+    defines: typing.List[str] = [],
+    include_paths: typing.List[str] = [],
+    retain_all_content: bool = False,
+    encoding: typing.Optional[str] = None,
+    msvc_args: typing.List[str] = ["cl.exe"],
+    print_cmd: bool = True,
+) -> PreprocessorFunction:
+    """
+    Creates a preprocessor function that uses cl.exe from Microsoft Visual Studio
+    to preprocess the input text. cl.exe is not typically on the path, so you
+    may need to open the correct developer tools shell or pass in the correct path
+    to cl.exe in the `msvc_args` parameter.
+
+    cl.exe will throw an error if a file referenced by an #include directive is not found.
+
+    :param defines: list of #define macros specified as "key value"
+    :param include_paths: list of directories to search for included files
+    :param retain_all_content: If False, only the parsed file content will be retained
+    :param encoding: If specified any include files are opened with this encoding
+    :param msvc_args: This is the path to cl.exe and any extra args you might want
+    :param print_cmd: Prints the command as its executed
+
+    .. code-block:: python
+
+        pp = make_msvc_preprocessor()
+        options = ParserOptions(preprocessor=pp)
+
+        parse_file(content, options=options)
+
+    """
+
+    if not encoding:
+        encoding = "utf-8"
+
+    def _preprocess_file(filename: str, content: typing.Optional[str]) -> str:
+        cmd = msvc_args + ["/nologo", "/E", "/C"]
+
+        for p in include_paths:
+            cmd.append(f"/I{p}")
+        for d in defines:
+            cmd.append(f"/D{d.replace(' ', '=')}")
+
+        tfpname = None
+
+        try:
+            kwargs = {"encoding": encoding}
+            if filename == "<str>":
+                if content is None:
+                    raise PreprocessorError("no content specified for stdin")
+
+                tfp = tempfile.NamedTemporaryFile(
+                    mode="w", encoding=encoding, suffix=".h", delete=False
+                )
+                tfpname = tfp.name
+                tfp.write(content)
+                tfp.close()
+
+                cmd.append(tfpname)
+            else:
+                cmd.append(filename)
+
+            if print_cmd:
+                print("+", " ".join(cmd), file=sys.stderr)
+
+            result: str = subprocess.check_output(cmd, **kwargs)  # type: ignore
+            if not retain_all_content:
+                result = _msvc_filter(io.StringIO(result))
+        finally:
+            if tfpname:
+                os.unlink(tfpname)
+
+        return result
+
+    return _preprocess_file
+
+
+#
+# PCPP preprocessor support (not installed by default)
+#
+
+
+try:
+    import pcpp
+    from pcpp import Preprocessor, OutputDirective, Action
+
+    class _CustomPreprocessor(Preprocessor):
+        def __init__(
+            self,
+            encoding: typing.Optional[str],
+            passthru_includes: typing.Optional["re.Pattern"],
+        ):
+            Preprocessor.__init__(self)
+            self.errors: typing.List[str] = []
+            self.assume_encoding = encoding
+            self.passthru_includes = passthru_includes
+
+        def on_error(self, file, line, msg):
+            self.errors.append(f"{file}:{line} error: {msg}")
+
+        def on_include_not_found(self, *ignored):
+            raise OutputDirective(Action.IgnoreAndPassThrough)
+
+        def on_comment(self, *ignored):
+            return True
+
+except ImportError:
+    pcpp = None
+
+
+def _pcpp_filter(fname: str, fp: typing.TextIO) -> str:
     # the output of pcpp includes the contents of all the included files, which
     # isn't what a typical user of cxxheaderparser would want, so we strip out
     # the line directives and any content that isn't in our original file
@@ -58,12 +269,22 @@ def make_pcpp_preprocessor(
     include_paths: typing.List[str] = [],
     retain_all_content: bool = False,
     encoding: typing.Optional[str] = None,
+    passthru_includes: typing.Optional["re.Pattern"] = None,
 ) -> PreprocessorFunction:
     """
     Creates a preprocessor function that uses pcpp (which must be installed
     separately) to preprocess the input text.
 
+    If missing #include files are encountered, this preprocessor will ignore the
+    error. This preprocessor is pure python so it's very portable, and is a good
+    choice if performance isn't critical.
+
+    :param defines: list of #define macros specified as "key value"
+    :param include_paths: list of directories to search for included files
+    :param retain_all_content: If False, only the parsed file content will be retained
     :param encoding: If specified any include files are opened with this encoding
+    :param passthru_includes: If specified any #include directives that match the
+                              compiled regex pattern will be part of the output.
 
     .. code-block:: python
 
@@ -74,8 +295,11 @@ def make_pcpp_preprocessor(
 
     """
 
-    def _preprocess_file(filename: str, content: str) -> str:
-        pp = _CustomPreprocessor(encoding)
+    if pcpp is None:
+        raise PreprocessorError("pcpp is not installed")
+
+    def _preprocess_file(filename: str, content: typing.Optional[str]) -> str:
+        pp = _CustomPreprocessor(encoding, passthru_includes)
         if include_paths:
             for p in include_paths:
                 pp.add_path(p)
@@ -86,6 +310,10 @@ def make_pcpp_preprocessor(
         if not retain_all_content:
             pp.line_directive = "#line"
 
+        if content is None:
+            with open(filename, "r", encoding=encoding) as fp:
+                content = fp.read()
+
         pp.parse(content, filename)
 
         if pp.errors:
@@ -111,6 +339,6 @@ def make_pcpp_preprocessor(
                         filename = filename.replace(os.sep, "/")
                     break
 
-            return _filter_self(filename, fp)
+            return _pcpp_filter(filename, fp)
 
     return _preprocess_file

cxxheaderparser/simple.py

@@ -348,7 +348,10 @@ def parse_file(
     if filename == "-":
         content = sys.stdin.read()
     else:
-        with open(filename, encoding=encoding) as fp:
-            content = fp.read()
+        content = None
 
-    return parse_string(content, filename=filename, options=options)
+    visitor = SimpleCxxVisitor()
+    parser = CxxParser(filename, content, visitor, options)
+    parser.parse()
+
+    return visitor.data

docs/conf.py

@@ -12,27 +12,19 @@ import pkg_resources
 # -- Project information -----------------------------------------------------
 
 project = "cxxheaderparser"
-copyright = "2020-2021, Dustin Spicuzza"
+copyright = "2020-2023, Dustin Spicuzza"
 author = "Dustin Spicuzza"
 
 # The full version, including alpha/beta/rc tags
 release = pkg_resources.get_distribution("cxxheaderparser").version
 
-# -- RTD configuration ------------------------------------------------
-
-# on_rtd is whether we are on readthedocs.org, this line of code grabbed from docs.readthedocs.org
-on_rtd = os.environ.get("READTHEDOCS", None) == "True"
-
 
 # -- General configuration ---------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = [
-    "sphinx.ext.autodoc",
-    "sphinx_autodoc_typehints",
-]
+extensions = ["sphinx.ext.autodoc", "sphinx_autodoc_typehints", "sphinx_rtd_theme"]
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
@@ -47,13 +39,7 @@ exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-#
-if not on_rtd:  # only import and set the theme if we're building docs locally
-    import sphinx_rtd_theme
 
-    html_theme = "sphinx_rtd_theme"
-    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
-else:
-    html_theme = "default"
+html_theme = "sphinx_rtd_theme"
 
 always_document_param_types = True

tests/test_preprocessor.py

@@ -1,9 +1,20 @@
 import os
 import pathlib
+import pytest
+import re
+import shutil
+import subprocess
+import typing
 
-from cxxheaderparser.options import ParserOptions
-from cxxheaderparser.preprocessor import make_pcpp_preprocessor
-from cxxheaderparser.simple import NamespaceScope, ParsedData, parse_file, parse_string
+from cxxheaderparser.options import ParserOptions, PreprocessorFunction
+from cxxheaderparser import preprocessor
+from cxxheaderparser.simple import (
+    NamespaceScope,
+    ParsedData,
+    parse_file,
+    parse_string,
+    Include,
+)
 from cxxheaderparser.types import (
     FundamentalSpecifier,
     NameSpecifier,
@@ -15,12 +26,39 @@ from cxxheaderparser.types import (
 )
 
 
-def test_basic_preprocessor() -> None:
+@pytest.fixture(params=["gcc", "msvc", "pcpp"])
+def make_pp(request) -> typing.Callable[..., PreprocessorFunction]:
+    param = request.param
+    if param == "gcc":
+        gcc_path = shutil.which("g++")
+        if not gcc_path:
+            pytest.skip("g++ not found")
+
+        subprocess.run([gcc_path, "--version"])
+        return preprocessor.make_gcc_preprocessor
+    elif param == "msvc":
+        gcc_path = shutil.which("cl.exe")
+        if not gcc_path:
+            pytest.skip("cl.exe not found")
+
+        return preprocessor.make_msvc_preprocessor
+    elif param == "pcpp":
+        if preprocessor.pcpp is None:
+            pytest.skip("pcpp not installed")
+        return preprocessor.make_pcpp_preprocessor
+    else:
+        assert False
+
+
+def test_basic_preprocessor(
+    make_pp: typing.Callable[..., PreprocessorFunction]
+) -> None:
     content = """
       #define X 1
       int x = X;
     """
-    options = ParserOptions(preprocessor=make_pcpp_preprocessor())
+
+    options = ParserOptions(preprocessor=make_pp())
     data = parse_string(content, cleandoc=True, options=options)
 
     assert data == ParsedData(
@@ -38,7 +76,10 @@ def test_basic_preprocessor() -> None:
     )
 
 
-def test_preprocessor_omit_content(tmp_path: pathlib.Path) -> None:
+def test_preprocessor_omit_content(
+    make_pp: typing.Callable[..., PreprocessorFunction],
+    tmp_path: pathlib.Path,
+) -> None:
     """Ensure that content in other headers is omitted"""
     h_content = '#include "t2.h"' "\n" "int x = X;\n"
     h2_content = "#define X 2\n" "int omitted = 1;\n"
@@ -49,7 +90,7 @@ def test_preprocessor_omit_content(tmp_path: pathlib.Path) -> None:
     with open(tmp_path / "t2.h", "w") as fp:
         fp.write(h2_content)
 
-    options = ParserOptions(preprocessor=make_pcpp_preprocessor())
+    options = ParserOptions(preprocessor=make_pp())
     data = parse_file(tmp_path / "t1.h", options=options)
 
     assert data == ParsedData(
@@ -67,7 +108,10 @@ def test_preprocessor_omit_content(tmp_path: pathlib.Path) -> None:
     )
 
 
-def test_preprocessor_omit_content2(tmp_path: pathlib.Path) -> None:
+def test_preprocessor_omit_content2(
+    make_pp: typing.Callable[..., PreprocessorFunction],
+    tmp_path: pathlib.Path,
+) -> None:
     """
     Ensure that content in other headers is omitted while handling pcpp
     relative path quirk
@@ -84,9 +128,7 @@ def test_preprocessor_omit_content2(tmp_path: pathlib.Path) -> None:
     with open(tmp_path2 / "t2.h", "w") as fp:
         fp.write(h2_content)
 
-    options = ParserOptions(
-        preprocessor=make_pcpp_preprocessor(include_paths=[str(tmp_path)])
-    )
+    options = ParserOptions(preprocessor=make_pp(include_paths=[str(tmp_path)]))
 
     # Weirdness happens here
     os.chdir(tmp_path)
@@ -107,7 +149,9 @@ def test_preprocessor_omit_content2(tmp_path: pathlib.Path) -> None:
     )
 
 
-def test_preprocessor_encoding(tmp_path: pathlib.Path) -> None:
+def test_preprocessor_encoding(
+    make_pp: typing.Callable[..., PreprocessorFunction], tmp_path: pathlib.Path
+) -> None:
     """Ensure we can handle alternate encodings"""
     h_content = b"// \xa9 2023 someone\n" b'#include "t2.h"' b"\n" b"int x = X;\n"
 
@@ -119,7 +163,7 @@ def test_preprocessor_encoding(tmp_path: pathlib.Path) -> None:
     with open(tmp_path / "t2.h", "wb") as fp:
         fp.write(h2_content)
 
-    options = ParserOptions(preprocessor=make_pcpp_preprocessor(encoding="cp1252"))
+    options = ParserOptions(preprocessor=make_pp(encoding="cp1252"))
     data = parse_file(tmp_path / "t1.h", options=options, encoding="cp1252")
 
     assert data == ParsedData(
@@ -135,3 +179,26 @@ def test_preprocessor_encoding(tmp_path: pathlib.Path) -> None:
             ]
         )
     )
+
+
+@pytest.mark.skipif(preprocessor.pcpp is None, reason="pcpp not installed")
+def test_preprocessor_passthru_includes(tmp_path: pathlib.Path) -> None:
+    """Ensure that all #include pass through"""
+    h_content = '#include "t2.h"\n'
+
+    with open(tmp_path / "t1.h", "w") as fp:
+        fp.write(h_content)
+
+    with open(tmp_path / "t2.h", "w") as fp:
+        fp.write("")
+
+    options = ParserOptions(
+        preprocessor=preprocessor.make_pcpp_preprocessor(
+            passthru_includes=re.compile(".+")
+        )
+    )
+    data = parse_file(tmp_path / "t1.h", options=options)
+
+    assert data == ParsedData(
+        namespace=NamespaceScope(), includes=[Include(filename='"t2.h"')]
+    )