Merge pull request #907 from KhronosGroup/include

HLSL: Add an Includer to handle #include for local HLSL paths.

StandAlone/CMakeLists.txt

@@ -6,7 +6,7 @@ target_include_directories(glslang-default-resource-limits
                            PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}
                            PUBLIC ${PROJECT_SOURCE_DIR})
 
-set(SOURCES StandAlone.cpp)
+set(SOURCES StandAlone.cpp DirStackFileIncluder.h)
 set(REMAPPER_SOURCES spirv-remap.cpp)
 
 add_executable(glslangValidator ${SOURCES})
@@ -22,7 +22,6 @@ set(LIBRARIES
     SPVRemapper
     glslang-default-resource-limits)
 
-
 if(WIN32)
     set(LIBRARIES ${LIBRARIES} psapi)
 elseif(UNIX)

StandAlone/DirStackFileIncluder.h

@@ -0,0 +1,125 @@
+//
+// Copyright (C) 2002-2005  3Dlabs Inc. Ltd.
+// Copyright (C) 2017 Google, Inc.
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//
+//    Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+//
+//    Redistributions in binary form must reproduce the above
+//    copyright notice, this list of conditions and the following
+//    disclaimer in the documentation and/or other materials provided
+//    with the distribution.
+//
+//    Neither the name of 3Dlabs Inc. Ltd. nor the names of its
+//    contributors may be used to endorse or promote products derived
+//    from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+// COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+
+#pragma once
+
+#include <vector>
+#include <string>
+#include <fstream>
+#include <algorithm>
+
+#include "./../glslang/Public/ShaderLang.h"
+
+// Default include class for normal include convention of search backward
+// through the stack of active include paths (for nested includes).
+// Can be overridden to customize.
+class DirStackFileIncluder : public glslang::TShader::Includer {
+public:
+    virtual IncludeResult* includeLocal(const char* headerName,
+                                        const char* includerName,
+                                        size_t inclusionDepth) override
+    {
+        return readLocalPath(headerName, includerName, inclusionDepth);
+    }
+
+    virtual IncludeResult* includeSystem(const char* headerName,
+                                         const char* /*includerName*/,
+                                         size_t /*inclusionDepth*/) override
+    {
+        return readSystemPath(headerName);
+    }
+
+    virtual void releaseInclude(IncludeResult* result) override
+    {
+        if (result != nullptr) {
+            delete [] static_cast<tUserDataElement*>(result->userData);
+            delete result;
+        }
+    }
+
+    virtual ~DirStackFileIncluder() override { }
+
+protected:
+    typedef char tUserDataElement;
+    std::vector<std::string> directoryStack;
+
+    // Search for a valid "local" path based on combining the stack of include
+    // directories and the nominal name of the header.
+    virtual IncludeResult* readLocalPath(const char* headerName, const char* includerName, int depth)
+    {
+        // Discard popped include directories, and if first level, initialize.
+        directoryStack.resize(depth);
+        if (depth == 1)
+            directoryStack.front() = getDirectory(includerName);
+
+        // find a directory that works, reverse search of include stack
+        for (auto it = directoryStack.rbegin(); it != directoryStack.rend(); ++it) {
+            std::string path = *it + '/' + headerName;
+            std::replace(path.begin(), path.end(), '\\', '/');
+            std::ifstream file(path, std::ios_base::binary | std::ios_base::ate);
+            if (file) {
+                directoryStack.push_back(getDirectory(path));
+                return newIncludeResult(path, file, (int)file.tellg());
+            }
+        }
+
+        return nullptr;
+    }
+
+    // Search for a valid <system> path.
+    // Not implemented yet; returning nullptr signals failure to find.
+    virtual IncludeResult* readSystemPath(const char* /*headerName*/) const
+    {
+        return nullptr;
+    }
+
+    // Do actual reading of the file, filling in a new include result.
+    virtual IncludeResult* newIncludeResult(const std::string& path, std::ifstream& file, int length) const
+    {
+        char* content = new tUserDataElement [length];
+        file.seekg(0, file.beg);
+        file.read(content, length);
+        return new IncludeResult(path, content, length, content);
+    }
+
+    // If no path markers, return current working directory.
+    // Otherwise, strip file name and return path leading up to it.
+    virtual std::string getDirectory(const std::string path) const
+    {
+        size_t last = path.find_last_of("/\\");
+        return last == std::string::npos ? "." : path.substr(0, last);
+    }
+};

StandAlone/StandAlone.cpp

@@ -41,6 +41,7 @@
 
 #include "ResourceLimits.h"
 #include "Worklist.h"
+#include "DirStackFileIncluder.h"
 #include "./../glslang/Include/ShHandle.h"
 #include "./../glslang/Include/revision.h"
 #include "./../glslang/Public/ShaderLang.h"
@@ -48,6 +49,7 @@
 #include "../SPIRV/GLSL.std.450.h"
 #include "../SPIRV/doc.h"
 #include "../SPIRV/disassemble.h"
+
 #include <cstring>
 #include <cstdlib>
 #include <cctype>
@@ -665,9 +667,9 @@ void CompileAndLinkShaderUnits(std::vector<ShaderCompUnit> compUnits)
 
         const int defaultVersion = Options & EOptionDefaultDesktop? 110: 100;
 
+        DirStackFileIncluder includer;
         if (Options & EOptionOutputPreprocessed) {
             std::string str;
-            glslang::TShader::ForbidIncluder includer;
             if (shader->preprocess(&Resources, defaultVersion, ENoProfile, false, false,
                                    messages, &str, includer)) {
                 PutsIfNonEmpty(str.c_str());
@@ -678,7 +680,7 @@ void CompileAndLinkShaderUnits(std::vector<ShaderCompUnit> compUnits)
             StderrIfNonEmpty(shader->getInfoDebugLog());
             continue;
         }
-        if (! shader->parse(&Resources, defaultVersion, false, messages))
+        if (! shader->parse(&Resources, defaultVersion, false, messages, includer))
             CompileFailed = true;
 
         program.addShader(shader);

Test/bar.h

@@ -0,0 +1 @@
+float4 i1;

Test/baseResults/hlsl.include.vert.out

@@ -0,0 +1,75 @@
+../Test/hlsl.include.vert
+// Module Version 10000
+// Generated by (magic number): 80001
+// Id's are bound by 44
+
+                              Capability Shader
+               1:             ExtInstImport  "GLSL.std.450"
+                              MemoryModel Logical GLSL450
+                              EntryPoint Vertex 4  "main" 42
+                              Source HLSL 500
+                              Name 4  "main"
+                              Name 9  "@main("
+                              Name 11  "$Global"
+                              MemberName 11($Global) 0  "i1"
+                              MemberName 11($Global) 1  "i2"
+                              MemberName 11($Global) 2  "i4"
+                              MemberName 11($Global) 3  "i3"
+                              MemberName 11($Global) 4  "i6"
+                              MemberName 11($Global) 5  "i5"
+                              Name 13  ""
+                              Name 42  "@entryPointOutput"
+                              MemberDecorate 11($Global) 0 Offset 0
+                              MemberDecorate 11($Global) 1 Offset 16
+                              MemberDecorate 11($Global) 2 Offset 32
+                              MemberDecorate 11($Global) 3 Offset 48
+                              MemberDecorate 11($Global) 4 Offset 64
+                              MemberDecorate 11($Global) 5 Offset 80
+                              Decorate 11($Global) Block
+                              Decorate 13 DescriptorSet 0
+                              Decorate 42(@entryPointOutput) BuiltIn Position
+               2:             TypeVoid
+               3:             TypeFunction 2
+               6:             TypeFloat 32
+               7:             TypeVector 6(float) 4
+               8:             TypeFunction 7(fvec4)
+     11($Global):             TypeStruct 7(fvec4) 7(fvec4) 7(fvec4) 7(fvec4) 7(fvec4) 7(fvec4)
+              12:             TypePointer Uniform 11($Global)
+              13:     12(ptr) Variable Uniform
+              14:             TypeInt 32 1
+              15:     14(int) Constant 0
+              16:             TypePointer Uniform 7(fvec4)
+              19:     14(int) Constant 1
+              23:     14(int) Constant 3
+              27:     14(int) Constant 2
+              31:     14(int) Constant 5
+              35:     14(int) Constant 4
+              41:             TypePointer Output 7(fvec4)
+42(@entryPointOutput):     41(ptr) Variable Output
+         4(main):           2 Function None 3
+               5:             Label
+              43:    7(fvec4) FunctionCall 9(@main()
+                              Store 42(@entryPointOutput) 43
+                              Return
+                              FunctionEnd
+       9(@main():    7(fvec4) Function None 8
+              10:             Label
+              17:     16(ptr) AccessChain 13 15
+              18:    7(fvec4) Load 17
+              20:     16(ptr) AccessChain 13 19
+              21:    7(fvec4) Load 20
+              22:    7(fvec4) FAdd 18 21
+              24:     16(ptr) AccessChain 13 23
+              25:    7(fvec4) Load 24
+              26:    7(fvec4) FAdd 22 25
+              28:     16(ptr) AccessChain 13 27
+              29:    7(fvec4) Load 28
+              30:    7(fvec4) FAdd 26 29
+              32:     16(ptr) AccessChain 13 31
+              33:    7(fvec4) Load 32
+              34:    7(fvec4) FAdd 30 33
+              36:     16(ptr) AccessChain 13 35
+              37:    7(fvec4) Load 36
+              38:    7(fvec4) FAdd 34 37
+                              ReturnValue 38
+                              FunctionEnd

Test/baseResults/hlsl.includeNegative.vert.out

@@ -0,0 +1,10 @@
+hlsl.includeNegative.vert
+ERROR: ./foo.h:1: '#error' : should not be included  
+ERROR: ./inc2/../foo.h:1: '#error' : should not be included  
+ERROR: ./parentBad:3: '#error' : bad parent  
+ERROR: hlsl.includeNegative.vert:7: '#error' : in main  
+hlsl.includeNegative.vert(8): error at column 0, HLSL parsing failed.
+ERROR: 5 compilation errors.  No code generated.
+
+
+SPIR-V is not generated for failed compile or link

Test/baseResults/include.vert.out

@@ -0,0 +1,69 @@
+include.vert
+Warning, version 450 is not yet complete; most version-specific features are present, but some are missing.
+
+Shader version: 450
+Requested GL_GOOGLE_cpp_style_line_directive
+Requested GL_GOOGLE_include_directive
+0:? Sequence
+0:13  Function Definition: main( ( global void)
+0:13    Function Parameters: 
+0:15    Sequence
+0:15      move second child to first child ( temp 4-component vector of float)
+0:15        'color' ( smooth out 4-component vector of float)
+0:15        add ( temp 4-component vector of float)
+0:15          add ( temp 4-component vector of float)
+0:15            add ( temp 4-component vector of float)
+0:15              add ( temp 4-component vector of float)
+0:15                add ( temp 4-component vector of float)
+0:15                  'i1' ( global 4-component vector of float)
+0:15                  'i2' ( global 4-component vector of float)
+0:15                'i3' ( global 4-component vector of float)
+0:15              'i4' ( global 4-component vector of float)
+0:15            'i5' ( global 4-component vector of float)
+0:15          'i6' ( global 4-component vector of float)
+0:?   Linker Objects
+0:?     'i1' ( global 4-component vector of float)
+0:?     'i2' ( global 4-component vector of float)
+0:?     'i4' ( global 4-component vector of float)
+0:?     'i3' ( global 4-component vector of float)
+0:?     'i6' ( global 4-component vector of float)
+0:?     'i5' ( global 4-component vector of float)
+0:?     'color' ( smooth out 4-component vector of float)
+0:?     'gl_VertexID' ( gl_VertexId int VertexId)
+0:?     'gl_InstanceID' ( gl_InstanceId int InstanceId)
+
+
+Linked vertex stage:
+
+
+Shader version: 450
+Requested GL_GOOGLE_cpp_style_line_directive
+Requested GL_GOOGLE_include_directive
+0:? Sequence
+0:13  Function Definition: main( ( global void)
+0:13    Function Parameters: 
+0:15    Sequence
+0:15      move second child to first child ( temp 4-component vector of float)
+0:15        'color' ( smooth out 4-component vector of float)
+0:15        add ( temp 4-component vector of float)
+0:15          add ( temp 4-component vector of float)
+0:15            add ( temp 4-component vector of float)
+0:15              add ( temp 4-component vector of float)
+0:15                add ( temp 4-component vector of float)
+0:15                  'i1' ( global 4-component vector of float)
+0:15                  'i2' ( global 4-component vector of float)
+0:15                'i3' ( global 4-component vector of float)
+0:15              'i4' ( global 4-component vector of float)
+0:15            'i5' ( global 4-component vector of float)
+0:15          'i6' ( global 4-component vector of float)
+0:?   Linker Objects
+0:?     'i1' ( global 4-component vector of float)
+0:?     'i2' ( global 4-component vector of float)
+0:?     'i4' ( global 4-component vector of float)
+0:?     'i3' ( global 4-component vector of float)
+0:?     'i6' ( global 4-component vector of float)
+0:?     'i5' ( global 4-component vector of float)
+0:?     'color' ( smooth out 4-component vector of float)
+0:?     'gl_VertexID' ( gl_VertexId int VertexId)
+0:?     'gl_InstanceID' ( gl_InstanceId int InstanceId)
+

Test/foo.h

@@ -0,0 +1 @@
+#error should not be included

Test/hlsl.include.vert

@@ -0,0 +1,8 @@
+#include "bar.h"
+#include "./inc1/bar.h"
+#include "inc2\bar.h"
+
+float4 main() : SV_Position
+{
+    return i1 + i2 + i3 + i4 + i5 + i6;
+}

Test/hlsl.includeNegative.vert

@@ -0,0 +1,8 @@
+#include "foo.h"
+#include "inc2/../foo.h"
+#include "inc1/badInc.h"
+
+float4 main() : SV_Position
+{
+#error in main
+}

Test/inc1/badInc.h

@@ -0,0 +1 @@
+#include "parentBad"

Test/inc1/bar.h

@@ -0,0 +1,3 @@
+float4 i2;
+
+#include "foo.h"

Test/inc1/foo.h

@@ -0,0 +1,3 @@
+#include "parent.h"
+
+float4 i3;

Test/inc2/bar.h

@@ -0,0 +1,2 @@
+#include "foo.h"
+float4 i5;

Test/inc2/foo.h

@@ -0,0 +1 @@
+float4 i6;

Test/include.vert

@@ -0,0 +1,16 @@
+#version 450
+
+#extension GL_GOOGLE_include_directive : enable
+
+#define float4 vec4
+
+#include "bar.h"
+#include "./inc1/bar.h"
+#include "inc2\bar.h"
+
+out vec4 color;
+
+void main()
+{
+    color = i1 + i2 + i3 + i4 + i5 + i6;
+}

Test/parent.h

@@ -0,0 +1 @@
+float4 i4;

Test/parentBad

@@ -0,0 +1,3 @@
+int a;
+
+#error bad parent

Test/runtests

@@ -106,6 +106,17 @@ echo Testing SPV Debug Information
 $EXE -g -H spv.debugInfo.frag > $TARGETDIR/spv.debugInfo.frag.out
 diff -b $BASEDIR/spv.debugInfo.frag.out $TARGETDIR/spv.debugInfo.frag.out || HASERROR=1
 
+#
+# Testing Includer
+#
+echo Testing Includer
+$EXE -D -e main -H ../Test/hlsl.include.vert > $TARGETDIR/hlsl.include.vert.out
+diff -b $BASEDIR/hlsl.include.vert.out $TARGETDIR/hlsl.include.vert.out || HASERROR=1
+$EXE -D -e main -H hlsl.includeNegative.vert > $TARGETDIR/hlsl.includeNegative.vert.out
+diff -b $BASEDIR/hlsl.includeNegative.vert.out $TARGETDIR/hlsl.includeNegative.vert.out || HASERROR=1
+$EXE -l -i include.vert > $TARGETDIR/include.vert.out
+diff -b $BASEDIR/include.vert.out $TARGETDIR/include.vert.out || HASERROR=1
+
 #
 # Final checking
 #

glslang/MachineIndependent/ShaderLang.cpp

@@ -1607,11 +1607,6 @@ bool TShader::parse(const TBuiltInResource* builtInResources, int defaultVersion
                            forwardCompatible, messages, *intermediate, includer, sourceEntryPointName);
 }
 
-bool TShader::parse(const TBuiltInResource* builtInResources, int defaultVersion, bool forwardCompatible, EShMessages messages)
-{
-    return parse(builtInResources, defaultVersion, ENoProfile, false, forwardCompatible, messages);
-}
-
 // Fill in a string with the result of preprocessing ShaderStrings
 // Returns true if all extensions, pragmas and version strings were valid.
 bool TShader::preprocess(const TBuiltInResource* builtInResources,

glslang/MachineIndependent/preprocessor/Pp.cpp

@@ -613,14 +613,14 @@ int TPpContext::CPPinclude(TPpToken* ppToken)
     TShader::Includer::IncludeResult* res = nullptr;
     if (startWithLocalSearch)
         res = includer.includeLocal(filename.c_str(), currentSourceFile.c_str(), includeStack.size() + 1);
-    if (! res || res->headerName.empty()) {
+    if (res == nullptr || res->headerName.empty()) {
         includer.releaseInclude(res);
         res = includer.includeSystem(filename.c_str(), currentSourceFile.c_str(), includeStack.size() + 1);
     }
 
     // Process the results
-    if (res && !res->headerName.empty()) {
+    if (res != nullptr && !res->headerName.empty()) {
-        if (res->headerData && res->headerLength) {
+        if (res->headerData != nullptr && res->headerLength > 0) {
             // path for processing one or more tokens from an included header, hand off 'res'
             const bool forNextLine = parseContext.lineDirectiveShouldSetNextLine();
             std::ostringstream prologue;
@@ -638,7 +638,7 @@ int TPpContext::CPPinclude(TPpToken* ppToken)
     } else {
         // error path, clean up
         std::string message =
-            res ? std::string(res->headerData, res->headerLength)
+            res != nullptr ? std::string(res->headerData, res->headerLength)
                            : std::string("Could not process include directive");
         parseContext.ppError(directiveLoc, message.c_str(), "#include", "for header name: %s", filename.c_str());
         includer.releaseInclude(res);

glslang/Public/ShaderLang.h

@@ -407,6 +407,9 @@ public:
         virtual void releaseInclude(IncludeResult*) override { }
     };
 
+    bool parse(const TBuiltInResource*, int defaultVersion, EProfile defaultProfile, bool forceDefaultVersionAndProfile,
+               bool forwardCompatible, EShMessages, Includer&);
+
     bool parse(const TBuiltInResource* res, int defaultVersion, EProfile defaultProfile, bool forceDefaultVersionAndProfile,
                bool forwardCompatible, EShMessages messages)
     {
@@ -414,12 +417,18 @@ public:
         return parse(res, defaultVersion, defaultProfile, forceDefaultVersionAndProfile, forwardCompatible, messages, includer);
     }
 
-    bool parse(const TBuiltInResource*, int defaultVersion, EProfile defaultProfile, bool forceDefaultVersionAndProfile,
-               bool forwardCompatible, EShMessages, Includer&);
-
     // Equivalent to parse() without a default profile and without forcing defaults.
-    // Provided for backwards compatibility.
+    bool parse(const TBuiltInResource* builtInResources, int defaultVersion, bool forwardCompatible, EShMessages messages)
-    bool parse(const TBuiltInResource*, int defaultVersion, bool forwardCompatible, EShMessages);
+    {
+        return parse(builtInResources, defaultVersion, ENoProfile, false, forwardCompatible, messages);
+    }
+
+    bool parse(const TBuiltInResource* builtInResources, int defaultVersion, bool forwardCompatible, EShMessages messages,
+               Includer& includer)
+    {
+        return parse(builtInResources, defaultVersion, ENoProfile, false, forwardCompatible, messages, includer);
+    }
+
     bool preprocess(const TBuiltInResource* builtInResources,
                     int defaultVersion, EProfile defaultProfile, bool forceDefaultVersionAndProfile,
                     bool forwardCompatible, EShMessages message, std::string* outputString,