From 7be4b8282dcb19f34e0b08c34d9883e56e8b1c8d Mon Sep 17 00:00:00 2001 From: Dejan Mircevski Date: Wed, 17 Jun 2015 11:40:33 -0400 Subject: [PATCH] Add #include processing to glslang (though turned off by default). When an include directive is recognized by the preprocessor, it executes a callback on the filepath argument to obtain the file contents. That way the compilation client can deal with the file system, include paths, etc. Currently only accepts quoted filepaths -- no angle brackets yet. --- StandAlone/StandAlone.cpp | 4 +-- .../baseResults/preprocessor.include.vert.err | 8 +++++ .../baseResults/preprocessor.include.vert.out | 0 Test/preprocessor.include.vert | 6 ++++ Test/test-preprocessor-list | 1 + glslang/MachineIndependent/ParseHelper.h | 1 + glslang/MachineIndependent/Scan.h | 3 ++ glslang/MachineIndependent/ShaderLang.cpp | 33 ++++++++++------- .../MachineIndependent/preprocessor/Pp.cpp | 35 +++++++++++++++++++ .../preprocessor/PpAtom.cpp | 3 ++ .../preprocessor/PpContext.cpp | 4 +-- .../preprocessor/PpContext.h | 35 ++++++++++++++++++- .../preprocessor/PpTokens.h | 3 ++ glslang/Public/ShaderLang.h | 27 ++++++++++++-- 14 files changed, 142 insertions(+), 21 deletions(-) create mode 100644 Test/baseResults/preprocessor.include.vert.err create mode 100644 Test/baseResults/preprocessor.include.vert.out create mode 100644 Test/preprocessor.include.vert diff --git a/StandAlone/StandAlone.cpp b/StandAlone/StandAlone.cpp index b88d8ad5..cc6e279b 100644 --- a/StandAlone/StandAlone.cpp +++ b/StandAlone/StandAlone.cpp @@ -700,8 +700,8 @@ void CompileAndLinkShaders() shader->setStrings(shaderStrings, 1); if (Options & EOptionOutputPreprocessed) { std::string str; - if (shader->preprocess(&Resources, defaultVersion, ENoProfile, - false, false, messages, &str)) { + if (shader->preprocess(&Resources, defaultVersion, ENoProfile, false, false, + messages, &str, glslang::TShader::ForbidInclude())) { PutsIfNonEmpty(str.c_str()); } else { CompileFailed = true; diff --git a/Test/baseResults/preprocessor.include.vert.err b/Test/baseResults/preprocessor.include.vert.err new file mode 100644 index 00000000..edc9ba9d --- /dev/null +++ b/Test/baseResults/preprocessor.include.vert.err @@ -0,0 +1,8 @@ +ERROR: 0:8000: '#include' : must be followed by a file designation +ERROR: 0:8001: '#include' : must be followed by a file designation +ERROR: 0:8002: '#error' : unexpected include directive +ERROR: 0:8003: '#include' : extra content after file designation +ERROR: 0:8004: '#error' : unexpected include directive +ERROR: 5 compilation errors. No code generated. + + diff --git a/Test/baseResults/preprocessor.include.vert.out b/Test/baseResults/preprocessor.include.vert.out new file mode 100644 index 00000000..e69de29b diff --git a/Test/preprocessor.include.vert b/Test/preprocessor.include.vert new file mode 100644 index 00000000..512327ba --- /dev/null +++ b/Test/preprocessor.include.vert @@ -0,0 +1,6 @@ +#line 8000 +#include +#include 123 +#include "foo" +#include "foo" garbage +#include "no-eol" \ No newline at end of file diff --git a/Test/test-preprocessor-list b/Test/test-preprocessor-list index e75ac1b4..195b4a79 100644 --- a/Test/test-preprocessor-list +++ b/Test/test-preprocessor-list @@ -4,6 +4,7 @@ preprocessor.edge_cases.vert preprocessor.errors.vert preprocessor.extensions.vert preprocessor.function_macro.vert +preprocessor.include.vert preprocessor.line.vert preprocessor.line.frag preprocessor.pragma.vert diff --git a/glslang/MachineIndependent/ParseHelper.h b/glslang/MachineIndependent/ParseHelper.h index 6f5645ef..f0c63bb3 100644 --- a/glslang/MachineIndependent/ParseHelper.h +++ b/glslang/MachineIndependent/ParseHelper.h @@ -214,6 +214,7 @@ public: int getNumErrors() const { return numErrors; } const TSourceLoc& getCurrentLoc() const { return currentScanner->getSourceLoc(); } void setCurrentLine(int line) { currentScanner->setLine(line); } + void setCurrentColumn(int col) { currentScanner->setColumn(col); } void setCurrentSourceName(const char* name) { currentScanner->setFile(name); } void setCurrentString(int string) { currentScanner->setString(string); } void setScanner(TInputScanner* scanner) { currentScanner = scanner; } diff --git a/glslang/MachineIndependent/Scan.h b/glslang/MachineIndependent/Scan.h index ff7bfa2f..eade05fd 100644 --- a/glslang/MachineIndependent/Scan.h +++ b/glslang/MachineIndependent/Scan.h @@ -154,6 +154,9 @@ public: loc[getLastValidSourceIndex()].name = nullptr; } + // for #include content indentation + void setColumn(int col) { loc[getLastValidSourceIndex()].column = col; } + const TSourceLoc& getSourceLoc() const { return loc[std::max(0, std::min(currentSource, numSources - finale - 1))]; } // Returns the index (starting from 0) of the most recent valid source string we are reading from. int getLastValidSourceIndex() const { return std::min(currentSource, numSources - 1); } diff --git a/glslang/MachineIndependent/ShaderLang.cpp b/glslang/MachineIndependent/ShaderLang.cpp index e98df614..617f9e21 100644 --- a/glslang/MachineIndependent/ShaderLang.cpp +++ b/glslang/MachineIndependent/ShaderLang.cpp @@ -130,7 +130,7 @@ bool InitializeSymbolTable(const TString& builtIns, int version, EProfile profil TIntermediate intermediate(language, version, profile); TParseContext parseContext(symbolTable, intermediate, true, version, profile, language, infoSink); - TPpContext ppContext(parseContext); + TPpContext ppContext(parseContext, TShader::ForbidInclude()); TScanContext scanContext(parseContext); parseContext.setScanContext(&scanContext); parseContext.setPpContext(&ppContext); @@ -463,7 +463,8 @@ bool ProcessDeferred( EShMessages messages, // warnings/errors/AST; things to print out TIntermediate& intermediate, // returned tree, etc. ProcessingContext& processingContext, - bool requireNonempty + bool requireNonempty, + const TShader::Includer& includer ) { if (! InitThread()) @@ -565,7 +566,7 @@ bool ProcessDeferred( TParseContext parseContext(symbolTable, intermediate, false, version, profile, compiler->getLanguage(), compiler->infoSink, forwardCompatible, messages); glslang::TScanContext scanContext(parseContext); - TPpContext ppContext(parseContext); + TPpContext ppContext(parseContext, includer); parseContext.setScanContext(&scanContext); parseContext.setPpContext(&ppContext); parseContext.setLimits(*resources); @@ -838,6 +839,7 @@ bool PreprocessDeferred( bool forceDefaultVersionAndProfile, bool forwardCompatible, // give errors for use of deprecated features EShMessages messages, // warnings/errors/AST; things to print out + const TShader::Includer& includer, TIntermediate& intermediate, // returned tree, etc. std::string* outputString) { @@ -845,7 +847,8 @@ bool PreprocessDeferred( return ProcessDeferred(compiler, shaderStrings, numStrings, inputLengths, stringNames, preamble, optLevel, resources, defaultVersion, defaultProfile, forceDefaultVersionAndProfile, - forwardCompatible, messages, intermediate, parser, false); + forwardCompatible, messages, intermediate, parser, + false, includer); } @@ -874,13 +877,15 @@ bool CompileDeferred( bool forceDefaultVersionAndProfile, bool forwardCompatible, // give errors for use of deprecated features EShMessages messages, // warnings/errors/AST; things to print out - TIntermediate& intermediate) // returned tree, etc. + TIntermediate& intermediate,// returned tree, etc. + const TShader::Includer& includer) { DoFullParse parser; return ProcessDeferred(compiler, shaderStrings, numStrings, inputLengths, stringNames, preamble, optLevel, resources, defaultVersion, defaultProfile, forceDefaultVersionAndProfile, - forwardCompatible, messages, intermediate, parser, true); + forwardCompatible, messages, intermediate, parser, + true, includer); } } // end anonymous namespace for local functions @@ -1024,7 +1029,7 @@ int ShCompile( TIntermediate intermediate(compiler->getLanguage()); bool success = CompileDeferred(compiler, shaderStrings, numStrings, inputLengths, nullptr, "", optLevel, resources, defaultVersion, ENoProfile, false, - forwardCompatible, messages, intermediate); + forwardCompatible, messages, intermediate, TShader::ForbidInclude()); // // Call the machine dependent compiler @@ -1327,7 +1332,7 @@ void TShader::setStringsWithLengthsAndNames( // Returns true for success. // bool TShader::parse(const TBuiltInResource* builtInResources, int defaultVersion, EProfile defaultProfile, bool forceDefaultVersionAndProfile, - bool forwardCompatible, EShMessages messages) + bool forwardCompatible, EShMessages messages, const Includer& includer) { if (! InitThread()) return false; @@ -1340,7 +1345,7 @@ bool TShader::parse(const TBuiltInResource* builtInResources, int defaultVersion return CompileDeferred(compiler, strings, numStrings, lengths, stringNames, preamble, EShOptNone, builtInResources, defaultVersion, defaultProfile, forceDefaultVersionAndProfile, - forwardCompatible, messages, *intermediate); + forwardCompatible, messages, *intermediate, includer); } bool TShader::parse(const TBuiltInResource* builtInResources, int defaultVersion, bool forwardCompatible, EShMessages messages) @@ -1351,9 +1356,11 @@ bool TShader::parse(const TBuiltInResource* builtInResources, int defaultVersion // Fill in a string with the result of preprocessing ShaderStrings // Returns true if all extensions, pragmas and version strings were valid. bool TShader::preprocess(const TBuiltInResource* builtInResources, - int defaultVersion, EProfile defaultProfile, bool forceDefaultVersionAndProfile, - bool forwardCompatible, - EShMessages message, std::string* output_string) + int defaultVersion, EProfile defaultProfile, + bool forceDefaultVersionAndProfile, + bool forwardCompatible, EShMessages message, + std::string* output_string, + const TShader::Includer& includer) { if (! InitThread()) return false; @@ -1366,7 +1373,7 @@ bool TShader::preprocess(const TBuiltInResource* builtInResources, return PreprocessDeferred(compiler, strings, numStrings, lengths, stringNames, preamble, EShOptNone, builtInResources, defaultVersion, defaultProfile, forceDefaultVersionAndProfile, - forwardCompatible, message, *intermediate, output_string); + forwardCompatible, message, includer, *intermediate, output_string); } const char* TShader::getInfoLog() diff --git a/glslang/MachineIndependent/preprocessor/Pp.cpp b/glslang/MachineIndependent/preprocessor/Pp.cpp index 279706ec..9e126824 100644 --- a/glslang/MachineIndependent/preprocessor/Pp.cpp +++ b/glslang/MachineIndependent/preprocessor/Pp.cpp @@ -596,6 +596,38 @@ int TPpContext::CPPifdef(int defined, TPpToken* ppToken) return token; } +// Handle #include +int TPpContext::CPPinclude(TPpToken* ppToken) +{ + const TSourceLoc directiveLoc = ppToken->loc; + int token = scanToken(ppToken); + if (token != PpAtomConstString) { + // TODO: handle angle brackets. + parseContext.ppError(directiveLoc, "must be followed by a file designation", "#include", ""); + } else { + const char* name = GetAtomString(ppToken->atom); + token = scanToken(ppToken); + if (token != '\n' && token != EndOfInput) { + parseContext.ppError(ppToken->loc, "extra content after file designation", "#include", ""); + } else { + if (!inputStack.empty()) ungetChar(); + std::string replacement; + bool success; + std::tie(success, replacement) = includer.include(name); + if (success) { + pushInput(new TokenizableString(replacement, this)); + // At EOF, there's no "current" location anymore. + if (token != EndOfInput) parseContext.setCurrentColumn(0); + // Don't accidentally return EndOfInput, which will end all preprocessing. + return '\n'; + } else { + parseContext.ppError(ppToken->loc, "not found", name, ""); + } + } + } + return token; +} + // Handle #line int TPpContext::CPPline(TPpToken* ppToken) { @@ -845,6 +877,9 @@ int TPpContext::readCPPline(TPpToken* ppToken) case PpAtomIfndef: token = CPPifdef(0, ppToken); break; + case PpAtomInclude: + token = CPPinclude(ppToken); + break; case PpAtomLine: token = CPPline(ppToken); break; diff --git a/glslang/MachineIndependent/preprocessor/PpAtom.cpp b/glslang/MachineIndependent/preprocessor/PpAtom.cpp index 04e0d630..b3cb2b07 100644 --- a/glslang/MachineIndependent/preprocessor/PpAtom.cpp +++ b/glslang/MachineIndependent/preprocessor/PpAtom.cpp @@ -120,6 +120,9 @@ const struct { { PpAtomLineMacro, "__LINE__" }, { PpAtomFileMacro, "__FILE__" }, { PpAtomVersionMacro, "__VERSION__" }, + + { PpAtomInclude, "include" }, + }; } // end anonymous namespace diff --git a/glslang/MachineIndependent/preprocessor/PpContext.cpp b/glslang/MachineIndependent/preprocessor/PpContext.cpp index 828764b0..b8d2c737 100644 --- a/glslang/MachineIndependent/preprocessor/PpContext.cpp +++ b/glslang/MachineIndependent/preprocessor/PpContext.cpp @@ -83,8 +83,8 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. namespace glslang { -TPpContext::TPpContext(TParseContext& pc) : - preamble(0), strings(0), parseContext(pc), inComment(false) +TPpContext::TPpContext(TParseContext& pc, const TShader::Includer& inclr) : + preamble(0), strings(0), parseContext(pc), includer(inclr), inComment(false) { InitAtomTable(); InitScanner(); diff --git a/glslang/MachineIndependent/preprocessor/PpContext.h b/glslang/MachineIndependent/preprocessor/PpContext.h index 74a9f5dd..82564ebc 100644 --- a/glslang/MachineIndependent/preprocessor/PpContext.h +++ b/glslang/MachineIndependent/preprocessor/PpContext.h @@ -118,7 +118,7 @@ class TInputScanner; // Don't expect too much in terms of OO design. class TPpContext { public: - TPpContext(TParseContext&); + TPpContext(TParseContext&, const TShader::Includer&); virtual ~TPpContext(); void setPreamble(const char* preamble, size_t length); @@ -281,6 +281,8 @@ protected: // from Pp.cpp // TSourceLoc ifloc; /* outermost #if */ + // Used to obtain #include content. + const TShader::Includer& includer; int InitCPP(); int CPPdefine(TPpToken * ppToken); @@ -291,6 +293,7 @@ protected: int evalToToken(int token, bool shortCircuit, int& res, bool& err, TPpToken * ppToken); int CPPif (TPpToken * ppToken); int CPPifdef(int defined, TPpToken * ppToken); + int CPPinclude(TPpToken * ppToken); int CPPline(TPpToken * ppToken); int CPPerror(TPpToken * ppToken); int CPPpragma(TPpToken * ppToken); @@ -419,6 +422,36 @@ protected: TInputScanner* input; }; + // Holds a string that can be tokenized via the tInput interface. + class TokenizableString : public tInput { + public: + // Copies str, which must be non-empty. + TokenizableString(const std::string& str, TPpContext* pp) + : tInput(pp), + str_(str), + strings(str_.data()), + length(str_.size()), + scanner(1, &strings, &length), + stringInput(pp, scanner) {} + + // tInput methods: + int scan(TPpToken* t) override { return stringInput.scan(t); } + int getch() override { return stringInput.getch(); } + void ungetch() override { stringInput.ungetch(); } + + private: + // Stores the titular string. + const std::string str_; + // Will point to str_[0] and be passed to scanner constructor. + const char* const strings; + // Length of str_, passed to scanner constructor. + size_t length; + // Scans over str_. + TInputScanner scanner; + // Delegate object implementing the tInput interface. + tStringInput stringInput; + }; + int InitScanner(); int ScanFromString(char* s); void missingEndifCheck(); diff --git a/glslang/MachineIndependent/preprocessor/PpTokens.h b/glslang/MachineIndependent/preprocessor/PpTokens.h index 0d116f9b..391b04ad 100644 --- a/glslang/MachineIndependent/preprocessor/PpTokens.h +++ b/glslang/MachineIndependent/preprocessor/PpTokens.h @@ -158,6 +158,9 @@ enum EFixedAtoms { PpAtomFileMacro, PpAtomVersionMacro, + // #include + PpAtomInclude, + PpAtomLast, }; diff --git a/glslang/Public/ShaderLang.h b/glslang/Public/ShaderLang.h index edca638c..7b12ad07 100644 --- a/glslang/Public/ShaderLang.h +++ b/glslang/Public/ShaderLang.h @@ -249,6 +249,7 @@ SH_IMPORT_EXPORT int ShGetUniformLocation(const ShHandle uniformMap, const char* #include #include +#include class TCompiler; class TInfoSink; @@ -288,14 +289,34 @@ public: void setStringsWithLengthsAndNames( const char* const* s, const int* l, const char* const* names, int n); void setPreamble(const char* s) { preamble = s; } - bool parse(const TBuiltInResource*, int defaultVersion, EProfile defaultProfile, bool forceDefaultVersionAndProfile, bool forwardCompatible, EShMessages); + + // Interface to #include handlers. + class Includer { + public: + // On success, returns true and the content that replaces "#include + // filename". On failure, returns false and an arbitrary string. + virtual std::pair include(const char* filename) const = 0; + }; + + // Generates #error as #include content. + class ForbidInclude : public Includer { + public: + std::pair include(const char* filename) const override + { + return std::make_pair(true, "#error unexpected include directive\n"); + } + }; + + bool parse(const TBuiltInResource*, int defaultVersion, EProfile defaultProfile, bool forceDefaultVersionAndProfile, + bool forwardCompatible, EShMessages, const Includer& = ForbidInclude()); + // Equivalent to parse() without a default profile and without forcing defaults. // Provided for backwards compatibility. bool parse(const TBuiltInResource*, int defaultVersion, bool forwardCompatible, EShMessages); bool preprocess(const TBuiltInResource* builtInResources, int defaultVersion, EProfile defaultProfile, bool forceDefaultVersionAndProfile, - bool forwardCompatible, - EShMessages message, std::string* outputString); + bool forwardCompatible, EShMessages message, std::string* outputString, + const TShader::Includer& includer); const char* getInfoLog(); const char* getInfoDebugLog();