Merge pull request #198 from AWoloszyn/update-includer

Updated the includer interface to allow relative includes.

StandAlone/StandAlone.cpp

@@ -699,8 +699,9 @@ void CompileAndLinkShaderUnits(std::vector<ShaderCompUnit> compUnits)
 
         if (Options & EOptionOutputPreprocessed) {
             std::string str;
+            glslang::TShader::ForbidInclude includer;
             if (shader->preprocess(&Resources, defaultVersion, ENoProfile, false, false,
-                                   messages, &str, glslang::TShader::ForbidInclude())) {
+                                   messages, &str, includer)) {
                 PutsIfNonEmpty(str.c_str());
             } else {
                 CompileFailed = true;

glslang/MachineIndependent/Scan.h

@@ -51,10 +51,10 @@ const int EndOfInput = -1;
 //
 class TInputScanner {
 public:
-    TInputScanner(int n, const char* const s[], size_t L[], const char* const* names = nullptr, int b = 0, int f = 0) :
+    TInputScanner(int n, const char* const s[], size_t L[], const char* const* names = nullptr, int b = 0, int f = 0, bool single = false) :
         numSources(n),
         sources(reinterpret_cast<const unsigned char* const *>(s)), // up to this point, common usage is "char*", but now we need positive 8-bit characters
-        lengths(L), currentSource(0), currentChar(0), stringBias(b), finale(f)
+        lengths(L), currentSource(0), currentChar(0), stringBias(b), finale(f), singleLogical(single)
     {
         loc = new TSourceLoc[numSources];
         for (int i = 0; i < numSources; ++i) {
@@ -67,6 +67,10 @@ public:
         loc[currentSource].string = -stringBias;
         loc[currentSource].line = 1;
         loc[currentSource].column = 0;
+        logicalSourceLoc.string = 0;
+        logicalSourceLoc.line = 1;
+        logicalSourceLoc.column = 0;
+        logicalSourceLoc.name = loc[0].name;
     }
 
     virtual ~TInputScanner()
@@ -82,8 +86,11 @@ public:
 
         int ret = peek();
         ++loc[currentSource].column;
+        ++logicalSourceLoc.column;
         if (ret == '\n') {
             ++loc[currentSource].line;
+            ++logicalSourceLoc.line;
+            logicalSourceLoc.column = 0;
             loc[currentSource].column = 0;
         }
         advance();
@@ -118,6 +125,7 @@ public:
         if (currentChar > 0) {
             --currentChar;
             --loc[currentSource].column;
+            --logicalSourceLoc.column;
             if (loc[currentSource].column < 0) {
                 // We've moved back past a new line. Find the
                 // previous newline (or start of the file) to compute
@@ -129,6 +137,7 @@ public:
                     }
                     --chIndex;
                 }
+                logicalSourceLoc.column = (int)(currentChar - chIndex);
                 loc[currentSource].column = (int)(currentChar - chIndex);
             }
         } else {
@@ -141,23 +150,49 @@ public:
             } else
                 currentChar = lengths[currentSource] - 1;
         }
-        if (peek() == '\n')
+        if (peek() == '\n') {
             --loc[currentSource].line;
+            --logicalSourceLoc.line;
+        }
     }
 
     // for #line override
-    void setLine(int newLine) { loc[getLastValidSourceIndex()].line = newLine; }
+    void setLine(int newLine)
-    void setFile(const char* filename) { loc[getLastValidSourceIndex()].name = filename; }
+    {
+        logicalSourceLoc.line = newLine;
+        loc[getLastValidSourceIndex()].line = newLine;
+    }
+
+    // for #line override in filename based parsing
+    void setFile(const char* filename)
+    {
+        logicalSourceLoc.name = filename;
+        loc[getLastValidSourceIndex()].name = filename;
+    }
+
     void setString(int newString)
     {
+        logicalSourceLoc.string = newString;
         loc[getLastValidSourceIndex()].string = newString;
+        logicalSourceLoc.name = nullptr;
         loc[getLastValidSourceIndex()].name = nullptr;
     }
 
     // for #include content indentation
-    void setColumn(int col) { loc[getLastValidSourceIndex()].column = col; }
+    void setColumn(int col)
+    {
+        logicalSourceLoc.column = col;
+        loc[getLastValidSourceIndex()].column = col;
+    }
 
-    const TSourceLoc& getSourceLoc() const { return loc[std::max(0, std::min(currentSource, numSources - finale - 1))]; }
+    const TSourceLoc& getSourceLoc() const
+    {
+        if (singleLogical) {
+            return logicalSourceLoc;
+        } else {
+            return loc[std::max(0, std::min(currentSource, numSources - finale - 1))];
+        }
+    }
     // Returns the index (starting from 0) of the most recent valid source string we are reading from.
     int getLastValidSourceIndex() const { return std::min(currentSource, numSources - 1); }
 
@@ -204,6 +239,10 @@ protected:
 
     int stringBias;   // the first string that is the user's string number 0
     int finale;       // number of internal strings after user's last string
+
+    TSourceLoc logicalSourceLoc;
+    bool singleLogical; // treats the strings as a single logical string.
+                        // locations will be reported from the first string.
 };
 
 } // end namespace glslang

glslang/MachineIndependent/ShaderLang.cpp

@@ -131,7 +131,8 @@ bool InitializeSymbolTable(const TString& builtIns, int version, EProfile profil
     TIntermediate intermediate(language, version, profile);
     
     TParseContext parseContext(symbolTable, intermediate, true, version, profile, spv, vulkan, language, infoSink);
-    TPpContext ppContext(parseContext, TShader::ForbidInclude());
+    TShader::ForbidInclude includer;
+    TPpContext ppContext(parseContext, "", includer);
     TScanContext scanContext(parseContext);
     parseContext.setScanContext(&scanContext);
     parseContext.setPpContext(&ppContext);
@@ -482,7 +483,7 @@ bool ProcessDeferred(
     TIntermediate& intermediate, // returned tree, etc.
     ProcessingContext& processingContext,
     bool requireNonempty,
-    const TShader::Includer& includer
+    TShader::Includer& includer
     )
 {
     if (! InitThread())
@@ -550,7 +551,6 @@ bool ProcessDeferred(
         version = defaultVersion;
         profile = defaultProfile;
     }
-
     int spv = (messages & EShMsgSpvRules) ? 100 : 0;         // TODO find path to get real version number here, for now non-0 is what matters
     bool goodVersion = DeduceVersionProfile(compiler->infoSink, compiler->getLanguage(), versionNotFirst, defaultVersion, version, profile, spv);
     bool versionWillBeError = (versionNotFound || (profile == EEsProfile && version >= 300 && versionNotFirst));
@@ -590,7 +590,7 @@ bool ProcessDeferred(
 
     TParseContext parseContext(symbolTable, intermediate, false, version, profile, spv, vulkan, compiler->getLanguage(), compiler->infoSink, forwardCompatible, messages);
     glslang::TScanContext scanContext(parseContext);
-    TPpContext ppContext(parseContext, includer);
+    TPpContext ppContext(parseContext, names[numPre]? names[numPre]: "", includer);
     parseContext.setScanContext(&scanContext);
     parseContext.setPpContext(&ppContext);
     parseContext.setLimits(*resources);
@@ -863,7 +863,7 @@ bool PreprocessDeferred(
     bool forceDefaultVersionAndProfile,
     bool forwardCompatible,     // give errors for use of deprecated features
     EShMessages messages,       // warnings/errors/AST; things to print out
-    const TShader::Includer& includer,
+    TShader::Includer& includer,
     TIntermediate& intermediate, // returned tree, etc.
     std::string* outputString)
 {
@@ -902,7 +902,7 @@ bool CompileDeferred(
     bool forwardCompatible,     // give errors for use of deprecated features
     EShMessages messages,       // warnings/errors/AST; things to print out
     TIntermediate& intermediate,// returned tree, etc.
-    const TShader::Includer& includer)
+    TShader::Includer& includer)
 {
     DoFullParse parser;
     return ProcessDeferred(compiler, shaderStrings, numStrings, inputLengths, stringNames,
@@ -1051,9 +1051,10 @@ int ShCompile(
     compiler->infoSink.debug.erase();
 
     TIntermediate intermediate(compiler->getLanguage());
+    TShader::ForbidInclude includer;
     bool success = CompileDeferred(compiler, shaderStrings, numStrings, inputLengths, nullptr,
                                    "", optLevel, resources, defaultVersion, ENoProfile, false,
-                                   forwardCompatible, messages, intermediate, TShader::ForbidInclude());
+                                   forwardCompatible, messages, intermediate, includer);
 
     //
     // Call the machine dependent compiler
@@ -1361,7 +1362,7 @@ void TShader::setStringsWithLengthsAndNames(
 // Returns true for success.
 //
 bool TShader::parse(const TBuiltInResource* builtInResources, int defaultVersion, EProfile defaultProfile, bool forceDefaultVersionAndProfile,
-                    bool forwardCompatible, EShMessages messages, const Includer& includer)
+                    bool forwardCompatible, EShMessages messages, Includer& includer)
 {
     if (! InitThread())
         return false;
@@ -1389,7 +1390,7 @@ bool TShader::preprocess(const TBuiltInResource* builtInResources,
                          bool forceDefaultVersionAndProfile,
                          bool forwardCompatible, EShMessages message,
                          std::string* output_string,
-                         const TShader::Includer& includer)
+                         Includer& includer)
 {
     if (! InitThread())
         return false;

glslang/MachineIndependent/preprocessor/Pp.cpp

@@ -611,24 +611,28 @@ int TPpContext::CPPinclude(TPpToken* ppToken)
         if (token != '\n' && token != EndOfInput) {
             parseContext.ppError(ppToken->loc, "extra content after file designation", "#include", "");
         } else {
-            auto include = includer.include(filename.c_str());
+            TShader::Includer::IncludeResult* res = includer.include(filename.c_str(), TShader::Includer::EIncludeRelative, currentSourceFile.c_str(), includeStack.size() + 1);
-            std::string sourceName = include.first;
+            if (res && !res->file_name.empty()) {
-            std::string replacement = include.second;
+                if (res->file_data && res->file_length) {
-            if (!sourceName.empty()) {
-                if (!replacement.empty()) {
                     const bool forNextLine = parseContext.lineDirectiveShouldSetNextLine();
-                    std::ostringstream content;
+                    std::ostringstream prologue;
-                    content << "#line " << forNextLine << " " << "\"" << sourceName << "\"\n";
+                    std::ostringstream epilogue;
-                    content << replacement << (replacement.back() == '\n' ? "" : "\n");
+                    prologue << "#line " << forNextLine << " " << "\"" << res->file_name << "\"\n";
-                    content << "#line " << directiveLoc.line + forNextLine << " " << directiveLoc.getStringNameOrNum() << "\n";
+                    epilogue << (res->file_data[res->file_length - 1] == '\n'? "" : "\n") << "#line " << directiveLoc.line + forNextLine << " " << directiveLoc.getStringNameOrNum() << "\n";
-                    pushInput(new TokenizableString(directiveLoc, content.str(), this));
+                    pushInput(new TokenizableIncludeFile(directiveLoc, prologue.str(), res, epilogue.str(), this));
                 }
                 // At EOF, there's no "current" location anymore.
                 if (token != EndOfInput) parseContext.setCurrentColumn(0);
                 // Don't accidentally return EndOfInput, which will end all preprocessing.
                 return '\n';
             } else {
-                parseContext.ppError(directiveLoc, replacement.c_str(), "#include", "");
+                std::string message =
+                    res ? std::string(res->file_data, res->file_length)
+                        : std::string("Could not process include directive");
+                parseContext.ppError(directiveLoc, message.c_str(), "#include", "");
+                if (res) {
+                    includer.releaseInclude(res);
+                }
             }
         }
     }

glslang/MachineIndependent/preprocessor/PpContext.cpp

@@ -83,8 +83,10 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 namespace glslang {
 
-TPpContext::TPpContext(TParseContext& pc, const TShader::Includer& inclr) : 
+TPpContext::TPpContext(TParseContext& pc, const std::string& rootFileName, TShader::Includer& inclr) : 
-    preamble(0), strings(0), parseContext(pc), includer(inclr), inComment(false)
+    preamble(0), strings(0), parseContext(pc), includer(inclr), inComment(false),
+    rootFileName(rootFileName),
+    currentSourceFile(rootFileName)
 {
     InitAtomTable();
     InitScanner();

glslang/MachineIndependent/preprocessor/PpContext.h

@@ -78,6 +78,7 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifndef PPCONTEXT_H
 #define PPCONTEXT_H
 
+#include <stack>
 #include <unordered_map>
 
 #include "../ParseHelper.h"
@@ -121,7 +122,7 @@ class TInputScanner;
 // Don't expect too much in terms of OO design.
 class TPpContext {
 public:
-    TPpContext(TParseContext&, const TShader::Includer&);
+    TPpContext(TParseContext&, const std::string& rootFileName, TShader::Includer&);
     virtual ~TPpContext();
 
     void setPreamble(const char* preamble, size_t length);
@@ -290,7 +291,7 @@ protected:
     //
 
     // Used to obtain #include content.
-    const TShader::Includer& includer;
+    TShader::Includer& includer;
 
     int InitCPP();
     int CPPdefine(TPpToken * ppToken);
@@ -430,16 +431,30 @@ protected:
         TInputScanner* input;
     };
 
-    // Holds a string that can be tokenized via the tInput interface.
+    // Holds a reference to included file data, as well as a 
-    class TokenizableString : public tInput {
+    // prologue and an epilogue string. This can be scanned using the tInput
+    // interface and acts as a single source string.
+    class TokenizableIncludeFile : public tInput {
     public:
-        // Copies str, which must be non-empty.
+        // Copies prologue and epilogue. The includedFile must remain valid
-        TokenizableString(const TSourceLoc& startLoc, const std::string& str, TPpContext* pp)
+        // until this TokenizableIncludeFile is no longer used.
+        TokenizableIncludeFile(const TSourceLoc& startLoc,
+                          const std::string& prologue,
+                          TShader::Includer::IncludeResult* includedFile,
+                          const std::string& epilogue,
+                          TPpContext* pp)
             : tInput(pp),
-              str_(str),
+              prologue_(prologue),
-              strings(str_.data()),
+              includedFile_(includedFile),
-              length(str_.size()),
+              epilogue_(epilogue),
-              scanner(1, &strings, &length),
+              strings({prologue_.data(), includedFile_->file_data, epilogue_.data()}),
+              lengths({prologue_.size(), includedFile_->file_length, epilogue_.size()}),
+              names({
+                      includedFile_->file_name.c_str(),
+                      includedFile_->file_name.c_str(),
+                      includedFile_->file_name.c_str()
+              }),
+              scanner(3, strings, lengths, names, 0, 0, true),
               prevScanner(nullptr),
               stringInput(pp, scanner) {
                   scanner.setLine(startLoc.line);
@@ -456,16 +471,34 @@ protected:
         {
             prevScanner = pp->parseContext.getScanner();
             pp->parseContext.setScanner(&scanner);
+            pp->push_include(includedFile_);
+        }
+
+        void notifyDeleted() override
+        {
+            pp->parseContext.setScanner(prevScanner);
+            pp->pop_include();
         }
-        void notifyDeleted() override { pp->parseContext.setScanner(prevScanner); }
 
     private:
-        // Stores the titular string.
+        // Stores the prologue for this string.
-        const std::string str_;
+        const std::string prologue_;
-        // Will point to str_[0] and be passed to scanner constructor.
+
-        const char* const strings;
+        // Stores the epilogue for this string.
+        const std::string epilogue_;
+
+        // Points to the IncludeResult that this TokenizableIncludeFile represents.
+        TShader::Includer::IncludeResult* includedFile_;
+
+        // Will point to prologue_, includedFile_->file_data and epilogue_
+        // This is passed to scanner constructor.
+        // These do not own the storage and it must remain valid until this
+        // object has been destroyed.
+        const char* strings[3];
         // Length of str_, passed to scanner constructor.
-        size_t length;
+        size_t lengths[3];
+        // String names
+        const char* names[3];
         // Scans over str_.
         TInputScanner scanner;
         // The previous effective scanner before the scanner in this instance
@@ -480,6 +513,24 @@ protected:
     void missingEndifCheck();
     int lFloatConst(int len, int ch, TPpToken* ppToken);
 
+    void push_include(TShader::Includer::IncludeResult* result)
+    {
+        currentSourceFile = result->file_name;
+        includeStack.push(result);
+    }
+
+    void pop_include()
+    {
+        TShader::Includer::IncludeResult* include = includeStack.top();
+        includeStack.pop();
+        includer.releaseInclude(include);
+        if (includeStack.empty()) {
+            currentSourceFile = rootFileName;
+        } else {
+            currentSourceFile = includeStack.top()->file_name;
+        }
+    }
+
     bool inComment;
 
     //
@@ -487,8 +538,12 @@ protected:
     //
     typedef TUnorderedMap<TString, int> TAtomMap;
     typedef TVector<const TString*> TStringMap;
+
     TAtomMap atomMap;
     TStringMap stringMap;
+    std::stack<TShader::Includer::IncludeResult*> includeStack;
+    std::string currentSourceFile;
+    std::string rootFileName;
     int nextAtom;
     void InitAtomTable();
     void AddAtomFixed(const char* s, int atom);

glslang/Public/ShaderLang.h

@@ -292,25 +292,89 @@ public:
     void setPreamble(const char* s) { preamble = s; }
 
     // Interface to #include handlers.
+    //
+    // To support #include, a client of Glslang does the following:
+    // 1. Call setStringsWithNames to set the source strings and associated
+    //    names.  For example, the names could be the names of the files
+    //    containing the shader sources.
+    // 2. Call parse with an Includer.
+    //
+    // When the Glslang parser encounters an #include directive, it calls
+    // the Includer's include method with the the requested include name
+    // together with the current string name.  The returned IncludeResult
+    // contains the fully resolved name of the included source, together
+    // with the source text that should replace the #include directive
+    // in the source stream.  After parsing that source, Glslang will
+    // release the IncludeResult object.
     class Includer {
     public:
-        // On success, returns the full path and content of the file with the given
+        typedef enum {
-        // filename that replaces "#include filename". On failure, returns an empty
+          EIncludeRelative, // For #include "something"
-        // string and an error message.
+          EIncludeStandard  // Reserved. For #include <something>
-        virtual std::pair<std::string, std::string> include(const char* filename) const = 0;
+        } IncludeType;
+
+        // An IncludeResult contains the resolved name and content of a source
+        // inclusion.
+        struct IncludeResult {
+            // For a successful inclusion, the fully resolved name of the requested
+            // include.  For example, in a filesystem-based includer, full resolution
+            // should convert a relative path name into an absolute path name.
+            // For a failed inclusion, this is an empty string.
+            std::string file_name;
+            // The content and byte length of the requested inclusion.  The
+            // Includer producing this IncludeResult retains ownership of the
+            // storage.
+            // For a failed inclusion, the file_data
+            // field points to a string containing error details.
+            const char* file_data;
+            const size_t file_length;
+            // Include resolver's context.
+            void* user_data;
+        };
+
+        // Resolves an inclusion request by name, type, current source name,
+        // and include depth.
+        // On success, returns an IncludeResult containing the resolved name
+        // and content of the include.  On failure, returns an IncludeResult
+        // with an empty string for the file_name and error details in the
+        // file_data field.  The Includer retains ownership of the contents
+        // of the returned IncludeResult value, and those contents must
+        // remain valid until the releaseInclude method is called on that
+        // IncludeResult object.
+        virtual IncludeResult* include(const char* requested_source,
+                                      IncludeType type,
+                                      const char* requesting_source,
+                                      size_t inclusion_depth) = 0;
+        // Signals that the parser will no longer use the contents of the
+        // specified IncludeResult.
+        virtual void releaseInclude(IncludeResult* result) = 0;
     };
 
     // Returns an error message for any #include directive.
     class ForbidInclude : public Includer {
     public:
-        std::pair<std::string, std::string> include(const char* /*filename*/) const override
+        IncludeResult* include(const char*, IncludeType, const char*, size_t) override
         {
-            return std::make_pair<std::string, std::string>("", "unexpected include directive");
+            static const char unexpected_include[] =
+                "unexpected include directive";
+            return new IncludeResult(
+                {"", unexpected_include, sizeof(unexpected_include) - 1, nullptr});
+        }
+        virtual void releaseInclude(IncludeResult* result) override
+        {
+            delete result;
         }
     };
 
+    bool parse(const TBuiltInResource* res, int defaultVersion, EProfile defaultProfile, bool forceDefaultVersionAndProfile,
+               bool forwardCompatible, EShMessages messages)
+    {
+        TShader::ForbidInclude includer;
+        return parse(res, defaultVersion, defaultProfile, forceDefaultVersionAndProfile, forwardCompatible, messages, includer);
+    }
+
     bool parse(const TBuiltInResource*, int defaultVersion, EProfile defaultProfile, bool forceDefaultVersionAndProfile,
-               bool forwardCompatible, EShMessages, const Includer& = ForbidInclude());
+               bool forwardCompatible, EShMessages, Includer&);
 
     // Equivalent to parse() without a default profile and without forcing defaults.
     // Provided for backwards compatibility.
@@ -318,7 +382,7 @@ public:
     bool preprocess(const TBuiltInResource* builtInResources,
                     int defaultVersion, EProfile defaultProfile, bool forceDefaultVersionAndProfile,
                     bool forwardCompatible, EShMessages message, std::string* outputString,
-                    const TShader::Includer& includer);
+                    Includer& includer);
 
     const char* getInfoLog();
     const char* getInfoDebugLog();