Add #include processing to glslang (though turned off by default).

When an include directive is recognized by the preprocessor, it
executes a callback on the filepath argument to obtain the file
contents.  That way the compilation client can deal with the file
system, include paths, etc.

Currently only accepts quoted filepaths -- no angle brackets yet.

StandAlone/StandAlone.cpp

@@ -700,8 +700,8 @@ void CompileAndLinkShaders()
         shader->setStrings(shaderStrings, 1);
         if (Options & EOptionOutputPreprocessed) {
             std::string str;
-            if (shader->preprocess(&Resources, defaultVersion, ENoProfile,
+            if (shader->preprocess(&Resources, defaultVersion, ENoProfile, false, false,
-                                   false, false, messages, &str)) {
+                                   messages, &str, glslang::TShader::ForbidInclude())) {
                 PutsIfNonEmpty(str.c_str());
             } else {
                 CompileFailed = true;

Test/baseResults/preprocessor.include.vert.err

@@ -0,0 +1,8 @@
+ERROR: 0:8000: '#include' : must be followed by a file designation 
+ERROR: 0:8001: '#include' : must be followed by a file designation 
+ERROR: 0:8002: '#error' : unexpected include directive  
+ERROR: 0:8003: '#include' : extra content after file designation 
+ERROR: 0:8004: '#error' : unexpected include directive  
+ERROR: 5 compilation errors.  No code generated.
+
+

Test/preprocessor.include.vert

@@ -0,0 +1,6 @@
+#line 8000
+#include
+#include 123
+#include "foo"
+#include "foo" garbage
+#include "no-eol"

Test/test-preprocessor-list

@@ -4,6 +4,7 @@ preprocessor.edge_cases.vert
 preprocessor.errors.vert
 preprocessor.extensions.vert
 preprocessor.function_macro.vert
+preprocessor.include.vert
 preprocessor.line.vert
 preprocessor.line.frag
 preprocessor.pragma.vert

glslang/MachineIndependent/ParseHelper.h

@@ -214,6 +214,7 @@ public:
     int getNumErrors() const { return numErrors; }
     const TSourceLoc& getCurrentLoc() const { return currentScanner->getSourceLoc(); }
     void setCurrentLine(int line) { currentScanner->setLine(line); }
+    void setCurrentColumn(int col) { currentScanner->setColumn(col); }
     void setCurrentSourceName(const char* name) { currentScanner->setFile(name); }
     void setCurrentString(int string) { currentScanner->setString(string); }
     void setScanner(TInputScanner* scanner) { currentScanner  = scanner; }

glslang/MachineIndependent/Scan.h

@@ -154,6 +154,9 @@ public:
         loc[getLastValidSourceIndex()].name = nullptr;
     }
 
+    // for #include content indentation
+    void setColumn(int col) { loc[getLastValidSourceIndex()].column = col; }
+
     const TSourceLoc& getSourceLoc() const { return loc[std::max(0, std::min(currentSource, numSources - finale - 1))]; }
     // Returns the index (starting from 0) of the most recent valid source string we are reading from.
     int getLastValidSourceIndex() const { return std::min(currentSource, numSources - 1); }

glslang/MachineIndependent/ShaderLang.cpp

@@ -130,7 +130,7 @@ bool InitializeSymbolTable(const TString& builtIns, int version, EProfile profil
     TIntermediate intermediate(language, version, profile);
     
     TParseContext parseContext(symbolTable, intermediate, true, version, profile, language, infoSink);
-    TPpContext ppContext(parseContext);
+    TPpContext ppContext(parseContext, TShader::ForbidInclude());
     TScanContext scanContext(parseContext);
     parseContext.setScanContext(&scanContext);
     parseContext.setPpContext(&ppContext);
@@ -463,7 +463,8 @@ bool ProcessDeferred(
     EShMessages messages,       // warnings/errors/AST; things to print out
     TIntermediate& intermediate, // returned tree, etc.
     ProcessingContext& processingContext,
-    bool requireNonempty
+    bool requireNonempty,
+    const TShader::Includer& includer
     )
 {
     if (! InitThread())
@@ -565,7 +566,7 @@ bool ProcessDeferred(
 
     TParseContext parseContext(symbolTable, intermediate, false, version, profile, compiler->getLanguage(), compiler->infoSink, forwardCompatible, messages);
     glslang::TScanContext scanContext(parseContext);
-    TPpContext ppContext(parseContext);
+    TPpContext ppContext(parseContext, includer);
     parseContext.setScanContext(&scanContext);
     parseContext.setPpContext(&ppContext);
     parseContext.setLimits(*resources);
@@ -838,6 +839,7 @@ bool PreprocessDeferred(
     bool forceDefaultVersionAndProfile,
     bool forwardCompatible,     // give errors for use of deprecated features
     EShMessages messages,       // warnings/errors/AST; things to print out
+    const TShader::Includer& includer,
     TIntermediate& intermediate, // returned tree, etc.
     std::string* outputString)
 {
@@ -845,7 +847,8 @@ bool PreprocessDeferred(
     return ProcessDeferred(compiler, shaderStrings, numStrings, inputLengths, stringNames,
                            preamble, optLevel, resources, defaultVersion,
                            defaultProfile, forceDefaultVersionAndProfile,
-                           forwardCompatible, messages, intermediate, parser, false);
+                           forwardCompatible, messages, intermediate, parser,
+                           false, includer);
 }
 
 
@@ -874,13 +877,15 @@ bool CompileDeferred(
     bool forceDefaultVersionAndProfile,
     bool forwardCompatible,     // give errors for use of deprecated features
     EShMessages messages,       // warnings/errors/AST; things to print out
-    TIntermediate& intermediate) // returned tree, etc.
+    TIntermediate& intermediate,// returned tree, etc.
+    const TShader::Includer& includer)
 {
     DoFullParse parser;
     return ProcessDeferred(compiler, shaderStrings, numStrings, inputLengths, stringNames,
                            preamble, optLevel, resources, defaultVersion,
                            defaultProfile, forceDefaultVersionAndProfile,
-                           forwardCompatible, messages, intermediate, parser, true);
+                           forwardCompatible, messages, intermediate, parser,
+                           true, includer);
 }
 
 } // end anonymous namespace for local functions
@@ -1024,7 +1029,7 @@ int ShCompile(
     TIntermediate intermediate(compiler->getLanguage());
     bool success = CompileDeferred(compiler, shaderStrings, numStrings, inputLengths, nullptr,
                                    "", optLevel, resources, defaultVersion, ENoProfile, false,
-                                   forwardCompatible, messages, intermediate);
+                                   forwardCompatible, messages, intermediate, TShader::ForbidInclude());
 
     //
     // Call the machine dependent compiler
@@ -1327,7 +1332,7 @@ void TShader::setStringsWithLengthsAndNames(
 // Returns true for success.
 //
 bool TShader::parse(const TBuiltInResource* builtInResources, int defaultVersion, EProfile defaultProfile, bool forceDefaultVersionAndProfile,
-                    bool forwardCompatible, EShMessages messages)
+                    bool forwardCompatible, EShMessages messages, const Includer& includer)
 {
     if (! InitThread())
         return false;
@@ -1340,7 +1345,7 @@ bool TShader::parse(const TBuiltInResource* builtInResources, int defaultVersion
     return CompileDeferred(compiler, strings, numStrings, lengths, stringNames,
                            preamble, EShOptNone, builtInResources, defaultVersion,
                            defaultProfile, forceDefaultVersionAndProfile,
-                           forwardCompatible, messages, *intermediate);
+                           forwardCompatible, messages, *intermediate, includer);
 }
 
 bool TShader::parse(const TBuiltInResource* builtInResources, int defaultVersion, bool forwardCompatible, EShMessages messages)
@@ -1351,9 +1356,11 @@ bool TShader::parse(const TBuiltInResource* builtInResources, int defaultVersion
 // Fill in a string with the result of preprocessing ShaderStrings
 // Returns true if all extensions, pragmas and version strings were valid.
 bool TShader::preprocess(const TBuiltInResource* builtInResources,
-                         int defaultVersion, EProfile defaultProfile, bool forceDefaultVersionAndProfile,
+                         int defaultVersion, EProfile defaultProfile,
-                         bool forwardCompatible,
+                         bool forceDefaultVersionAndProfile,
-                         EShMessages message, std::string* output_string)
+                         bool forwardCompatible, EShMessages message,
+                         std::string* output_string,
+                         const TShader::Includer& includer)
 {
     if (! InitThread())
         return false;
@@ -1366,7 +1373,7 @@ bool TShader::preprocess(const TBuiltInResource* builtInResources,
     return PreprocessDeferred(compiler, strings, numStrings, lengths, stringNames, preamble,
                               EShOptNone, builtInResources, defaultVersion,
                               defaultProfile, forceDefaultVersionAndProfile,
-                              forwardCompatible, message, *intermediate, output_string);
+                              forwardCompatible, message, includer, *intermediate, output_string);
 }
 
 const char* TShader::getInfoLog()

glslang/MachineIndependent/preprocessor/Pp.cpp

@@ -596,6 +596,38 @@ int TPpContext::CPPifdef(int defined, TPpToken* ppToken)
     return token;
 }
 
+// Handle #include
+int TPpContext::CPPinclude(TPpToken* ppToken)
+{
+    const TSourceLoc directiveLoc = ppToken->loc;
+    int token = scanToken(ppToken);
+    if (token != PpAtomConstString) {
+        // TODO: handle angle brackets.
+        parseContext.ppError(directiveLoc, "must be followed by a file designation", "#include", "");
+    } else {
+        const char* name = GetAtomString(ppToken->atom);
+        token = scanToken(ppToken);
+        if (token != '\n' && token != EndOfInput) {
+            parseContext.ppError(ppToken->loc, "extra content after file designation", "#include", "");
+        } else {
+            if (!inputStack.empty()) ungetChar();
+            std::string replacement;
+            bool success;
+            std::tie(success, replacement) = includer.include(name);
+            if (success) {
+                pushInput(new TokenizableString(replacement, this));
+                // At EOF, there's no "current" location anymore.
+                if (token != EndOfInput) parseContext.setCurrentColumn(0);
+                // Don't accidentally return EndOfInput, which will end all preprocessing.
+                return '\n';
+            } else {
+                parseContext.ppError(ppToken->loc, "not found", name, "");
+            }
+        }
+    }
+    return token;
+}
+
 // Handle #line
 int TPpContext::CPPline(TPpToken* ppToken) 
 {
@@ -845,6 +877,9 @@ int TPpContext::readCPPline(TPpToken* ppToken)
         case PpAtomIfndef:
             token = CPPifdef(0, ppToken);
             break;
+        case PpAtomInclude:
+            token = CPPinclude(ppToken);
+            break;
         case PpAtomLine:
             token = CPPline(ppToken);
             break;

glslang/MachineIndependent/preprocessor/PpAtom.cpp

@@ -120,6 +120,9 @@ const struct {
     { PpAtomLineMacro,       "__LINE__" },
     { PpAtomFileMacro,       "__FILE__" },
     { PpAtomVersionMacro,    "__VERSION__" },
+
+    { PpAtomInclude,        "include" },
+
 };
 
 } // end anonymous namespace

glslang/MachineIndependent/preprocessor/PpContext.cpp

@@ -83,8 +83,8 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 namespace glslang {
 
-TPpContext::TPpContext(TParseContext& pc) : 
+TPpContext::TPpContext(TParseContext& pc, const TShader::Includer& inclr) : 
-    preamble(0), strings(0), parseContext(pc), inComment(false)
+    preamble(0), strings(0), parseContext(pc), includer(inclr), inComment(false)
 {
     InitAtomTable();
     InitScanner();

glslang/MachineIndependent/preprocessor/PpContext.h

@@ -118,7 +118,7 @@ class TInputScanner;
 // Don't expect too much in terms of OO design.
 class TPpContext {
 public:
-    TPpContext(TParseContext&);
+    TPpContext(TParseContext&, const TShader::Includer&);
     virtual ~TPpContext();
 
     void setPreamble(const char* preamble, size_t length);
@@ -281,6 +281,8 @@ protected:
     // from Pp.cpp
     //
     TSourceLoc ifloc; /* outermost #if */
+    // Used to obtain #include content.
+    const TShader::Includer& includer;
 
     int InitCPP();
     int CPPdefine(TPpToken * ppToken);
@@ -291,6 +293,7 @@ protected:
     int evalToToken(int token, bool shortCircuit, int& res, bool& err, TPpToken * ppToken);
     int CPPif (TPpToken * ppToken); 
     int CPPifdef(int defined, TPpToken * ppToken);
+    int CPPinclude(TPpToken * ppToken);
     int CPPline(TPpToken * ppToken); 
     int CPPerror(TPpToken * ppToken); 
     int CPPpragma(TPpToken * ppToken);
@@ -419,6 +422,36 @@ protected:
         TInputScanner* input;
     };
 
+    // Holds a string that can be tokenized via the tInput interface.
+    class TokenizableString : public tInput {
+    public:
+        // Copies str, which must be non-empty.
+        TokenizableString(const std::string& str, TPpContext* pp)
+            : tInput(pp),
+              str_(str),
+              strings(str_.data()),
+              length(str_.size()),
+              scanner(1, &strings, &length),
+              stringInput(pp, scanner) {}
+
+        // tInput methods:
+        int scan(TPpToken* t) override { return stringInput.scan(t); }
+        int getch() override { return stringInput.getch(); }
+        void ungetch() override { stringInput.ungetch(); }
+
+    private:
+        // Stores the titular string.
+        const std::string str_;
+        // Will point to str_[0] and be passed to scanner constructor.
+        const char* const strings;
+        // Length of str_, passed to scanner constructor.
+        size_t length;
+        // Scans over str_.
+        TInputScanner scanner;
+        // Delegate object implementing the tInput interface.
+        tStringInput stringInput;
+    };
+
     int InitScanner();
     int ScanFromString(char* s);
     void missingEndifCheck();

glslang/MachineIndependent/preprocessor/PpTokens.h

@@ -158,6 +158,9 @@ enum EFixedAtoms {
     PpAtomFileMacro,
     PpAtomVersionMacro,
 
+    // #include
+    PpAtomInclude,
+
     PpAtomLast,
 };
 

glslang/Public/ShaderLang.h

@@ -249,6 +249,7 @@ SH_IMPORT_EXPORT int ShGetUniformLocation(const ShHandle uniformMap, const char*
 
 #include <list>
 #include <string>
+#include <utility>
 
 class TCompiler;
 class TInfoSink;
@@ -288,14 +289,34 @@ public:
     void setStringsWithLengthsAndNames(
         const char* const* s, const int* l, const char* const* names, int n);
     void setPreamble(const char* s) { preamble = s; }
-    bool parse(const TBuiltInResource*, int defaultVersion, EProfile defaultProfile, bool forceDefaultVersionAndProfile, bool forwardCompatible, EShMessages);
+
+    // Interface to #include handlers.
+    class Includer {
+    public:
+        // On success, returns true and the content that replaces "#include
+        // filename".  On failure, returns false and an arbitrary string.
+        virtual std::pair<bool, std::string> include(const char* filename) const = 0;
+    };
+
+    // Generates #error as #include content.
+    class ForbidInclude : public Includer {
+    public:
+        std::pair<bool, std::string> include(const char* filename) const override
+        {
+            return std::make_pair<bool, std::string>(true, "#error unexpected include directive\n");
+        }
+    };
+
+    bool parse(const TBuiltInResource*, int defaultVersion, EProfile defaultProfile, bool forceDefaultVersionAndProfile,
+               bool forwardCompatible, EShMessages, const Includer& = ForbidInclude());
+
     // Equivalent to parse() without a default profile and without forcing defaults.
     // Provided for backwards compatibility.
     bool parse(const TBuiltInResource*, int defaultVersion, bool forwardCompatible, EShMessages);
     bool preprocess(const TBuiltInResource* builtInResources,
                     int defaultVersion, EProfile defaultProfile, bool forceDefaultVersionAndProfile,
-                    bool forwardCompatible,
+                    bool forwardCompatible, EShMessages message, std::string* outputString,
-                    EShMessages message, std::string* outputString);
+                    const TShader::Includer& includer);
 
     const char* getInfoLog();
     const char* getInfoDebugLog();