Improve preprocessor by using GLSL scanner, allowing read-only strings to be compiled, unifying of line # tracking, and correct detection that ES #version appeared after a comment.

git-svn-id: https://cvs.khronos.org/svn/repos/ogl/trunk/ecosystem/public/sdk/tools/glslang@23721 e7fa87d3-cd2b-0410-9028-fcbf551c1848

StandAlone/StandAlone.cpp

@@ -92,12 +92,12 @@ ShBindingTable FixedAttributeTable = { 3, FixedAttributeBindings };
 EShLanguage FindLanguage(const std::string& name);
 bool CompileFile(const char *fileName, ShHandle, int options);
 void usage();
-void FreeFileData(char **data);
+void FreeFileData(char** data);
-char** ReadFileData(const char *fileName);
+char** ReadFileData(const char* fileName);
 void InfoLogMsg(const char* msg, const char* name, const int num);
 
 // Use to test breaking up a single shader file into multiple strings.
-int NumShaderStrings = 1;
+int NumShaderStrings;
 
 TBuiltInResource Resources;
 std::string ConfigFile;
@@ -205,7 +205,7 @@ void ProcessConfigFile()
     char** configStrings = 0;
     char *config = 0;
     if (ConfigFile.size() > 0) {
-        char** configStrings = ReadFileData(ConfigFile.c_str());
+        configStrings = ReadFileData(ConfigFile.c_str());
         if (configStrings)
             config = *configStrings;
         else {
@@ -731,9 +731,11 @@ bool CompileFile(const char *fileName, ShHandle compiler, int Options)
         for (int j = 0; j < ((Options & EOptionMemoryLeakMode) ? 100 : 1); ++j) {
             //ret = ShCompile(compiler, shaderStrings, NumShaderStrings, lengths, EShOptNone, &Resources, Options, 100, false, messages);
             ret = ShCompile(compiler, shaderStrings, NumShaderStrings, 0, EShOptNone, &Resources, Options, 100, false, messages);
-            //const char* multi[4] = { "# ve", "rsion", " 300 e", "s" };
+            //const char* multi[12] = { "# ve", "rsion", " 300 e", "s", "\n#err", 
+            //                         "or should be l", "ine 1", "string 5\n", "float glo", "bal", 
+            //                         ";\n#error should be line 2\n void main() {", "global = 2.3;}" };
             //const char* multi[7] = { "/", "/", "\\", "\n", "\n", "#", "version 300 es" };
-            //ret = ShCompile(compiler, multi, 4, 0, EShOptNone, &Resources, Options, 100, false, messages);
+            //ret = ShCompile(compiler, multi, 7, 0, EShOptNone, &Resources, Options, 100, false, messages);
         }
 
         if (Options & EOptionMemoryLeakMode)
@@ -836,12 +838,13 @@ char** ReadFileData(const char *fileName)
     }
     fdata[count] = '\0';
     fclose(in);
-    if(count==0){
+    if (count == 0) {
         return_data[0]=(char*)malloc(count+2);
         return_data[0][0]='\0';
-        NumShaderStrings=0;
+        NumShaderStrings = 0;
         return return_data;       
-    }
+    } else
+        NumShaderStrings = 1;
 
 	int len = (int)(ceil)((float)count/(float)NumShaderStrings);
     int ptr_len=0,i=0;

Test/baseResults/empty.frag.out

@@ -0,0 +1,30 @@
+empty.frag
+WARNING: #version: statement missing; use #version on first line of shader
+
+0:? Sequence
+0:?   Linker Objects
+
+empty2.frag
+WARNING: #version: statement missing; use #version on first line of shader
+
+0:? Sequence
+0:?   Linker Objects
+
+empty3.frag
+Warning, version 110 is not yet complete; most features are present, but a few are missing.
+
+0:? Sequence
+0:?   Linker Objects
+
+
+Linked fragment stage:
+
+ERROR: Linking fragment stage: Cannot mix ES profile with non-ES profile shaders
+
+ERROR: Linking fragment stage: Cannot mix ES profile with non-ES profile shaders
+
+ERROR: Linking fragment stage: Missing entry point: Each stage requires one "void main()" entry point
+
+0:? Sequence
+0:?   Linker Objects
+

Test/baseResults/versionsClean.frag.out

@@ -1,5 +1,6 @@
 ERROR: #version: statement must appear first in es-profile shader; before comments or newlines
-ERROR: 1 compilation errors.  No code generated.
+ERROR: 0:34: '#version' : must occur first in shader 
+ERROR: 2 compilation errors.  No code generated.
 
 ERROR: node is still EOpNull!
 0:41  Function Definition: main( (void)

Test/empty2.frag

@@ -0,0 +1 @@
+

Test/empty3.frag

@@ -0,0 +1 @@
+#version 110

Test/runtests

@@ -41,6 +41,7 @@ runLinkTest recurse1.vert recurse1.frag recurse2.frag
 runLinkTest 300link.frag
 runLinkTest 300link2.frag
 runLinkTest 300link3.frag
+runLinkTest empty.frag empty2.frag empty3.frag
 
 #
 # multi-threaded test

glslang/MachineIndependent/ParseHelper.cpp

@@ -35,6 +35,7 @@
 //
 
 #include "ParseHelper.h"
+#include "Scan.h"
 
 #include "osinclude.h"
 #include <stdarg.h>
@@ -51,12 +52,10 @@ TParseContext::TParseContext(TSymbolTable& symt, TIntermediate& interm, bool pb,
             intermediate(interm), symbolTable(symt), infoSink(is), language(L),
             version(v), profile(p), forwardCompatible(fc), messages(m),
             contextPragma(true, false), loopNestingLevel(0), structNestingLevel(0),
-            tokensBeforeEOF(false),
+            tokensBeforeEOF(false), currentScanner(0),
-            numErrors(0), parsingBuiltins(pb), afterEOF(false)
+            numErrors(0), parsingBuiltins(pb), afterEOF(false),
+            anyIndexLimits(false)
 {
-    currentLoc.line = 1;
-    currentLoc.string = 0;
-
     // ensure we always have a linkage node, even if empty, to simplify tree topology algorithms
     linkage = new TIntermAggregate;
 
@@ -106,43 +105,9 @@ TParseContext::TParseContext(TSymbolTable& symt, TIntermediate& interm, bool pb,
         globalOutputDefaults.layoutStream = 0;
 }
 
-//
+void TParseContext::setLimits(const TLimits& L)
-// Parse an array of strings using yyparse, going through the
-// preprocessor to tokenize the shader strings, then through
-// the GLSL scanner.
-//
-// Returns true for successful acceptance of the shader, false if any errors.
-//
-bool TParseContext::parseShaderStrings(TPpContext& ppContext, char* strings[], size_t lengths[], int numStrings)
 {
-    // empty shaders are okay
+    limits = L;
-    if (! strings || numStrings == 0 || lengths[0] == 0)
-        return true;
-
-    for (int i = 0; i < numStrings; ++i) {
-        if (! strings[i]) {
-            TSourceLoc loc;
-            loc.string = i;
-            loc.line = 1;
-            error(loc, "Null shader source string", "", "");
-
-            return false;
-        }
-    }
-
-    if (getPreamble())
-        ppContext.setPreamble(getPreamble(), strlen(getPreamble()));
-    ppContext.setShaderStrings(strings, lengths, numStrings);
-
-    // TODO: desktop PP: a shader containing nothing but white space and comments is valid, even though it has no parse tokens
-    size_t len = 0;
-    while (strings[0][len] == ' '  ||
-           strings[0][len] == '\t' ||
-           strings[0][len] == '\n' ||
-           strings[0][len] == '\r') {
-        if (++len >= lengths[0])
-            return true;
-    }
 
     anyIndexLimits = ! limits.generalAttributeMatrixVectorIndexing ||
                      ! limits.generalConstantMatrixVectorIndexing ||
@@ -150,10 +115,21 @@ bool TParseContext::parseShaderStrings(TPpContext& ppContext, char* strings[], s
                      ! limits.generalUniformIndexing ||
                      ! limits.generalVariableIndexing ||
                      ! limits.generalVaryingIndexing;
+}
 
+//
+// Parse an array of strings using yyparse, going through the
+// preprocessor to tokenize the shader strings, then through
+// the GLSL scanner.
+//
+// Returns true for successful acceptance of the shader, false if any errors.
+//
+bool TParseContext::parseShaderStrings(TPpContext& ppContext, TInputScanner& input, bool versionWillBeError)
+{
+    currentScanner = &input;
+    ppContext.setInput(input, versionWillBeError);
     yyparse((void*)this);
-
+    finalErrorCheck();
-    finalize();
 
     return numErrors == 0;
 }
@@ -163,21 +139,21 @@ void TParseContext::parserError(const char *s)
 {
     if (afterEOF) {
         if (tokensBeforeEOF == 1)
-            error(currentLoc, "", "pre-mature EOF", s, "");
+            error(getCurrentLoc(), "", "pre-mature EOF", s, "");
     } else
-        error(currentLoc, "", "", s, "");
+        error(getCurrentLoc(), "", "", s, "");
 }
 
 void TParseContext::handlePragma(const char **tokens, int numTokens)
 {
     if (!strcmp(tokens[0], "optimize")) {
         if (numTokens != 4) {
-            error(currentLoc, "optimize pragma syntax is incorrect", "#pragma", "");
+            error(getCurrentLoc(), "optimize pragma syntax is incorrect", "#pragma", "");
             return;
         }
 
         if (strcmp(tokens[1], "(")) {
-            error(currentLoc, "\"(\" expected after 'optimize' keyword", "#pragma", "");
+            error(getCurrentLoc(), "\"(\" expected after 'optimize' keyword", "#pragma", "");
             return;
         }
 
@@ -186,22 +162,22 @@ void TParseContext::handlePragma(const char **tokens, int numTokens)
         else if (!strcmp(tokens[2], "off"))
             contextPragma.optimize = false;
         else {
-            error(currentLoc, "\"on\" or \"off\" expected after '(' for 'optimize' pragma", "#pragma", "");
+            error(getCurrentLoc(), "\"on\" or \"off\" expected after '(' for 'optimize' pragma", "#pragma", "");
             return;
         }
 
         if (strcmp(tokens[3], ")")) {
-            error(currentLoc, "\")\" expected to end 'optimize' pragma", "#pragma", "");
+            error(getCurrentLoc(), "\")\" expected to end 'optimize' pragma", "#pragma", "");
             return;
         }
     } else if (!strcmp(tokens[0], "debug")) {
         if (numTokens != 4) {
-            error(currentLoc, "debug pragma syntax is incorrect", "#pragma", "");
+            error(getCurrentLoc(), "debug pragma syntax is incorrect", "#pragma", "");
             return;
         }
 
         if (strcmp(tokens[1], "(")) {
-            error(currentLoc, "\"(\" expected after 'debug' keyword", "#pragma", "");
+            error(getCurrentLoc(), "\"(\" expected after 'debug' keyword", "#pragma", "");
             return;
         }
 
@@ -210,12 +186,12 @@ void TParseContext::handlePragma(const char **tokens, int numTokens)
         else if (!strcmp(tokens[2], "off"))
             contextPragma.debug = false;
         else {
-            error(currentLoc, "\"on\" or \"off\" expected after '(' for 'debug' pragma", "#pragma", "");
+            error(getCurrentLoc(), "\"on\" or \"off\" expected after '(' for 'debug' pragma", "#pragma", "");
             return;
         }
 
         if (strcmp(tokens[3], ")")) {
-            error(currentLoc, "\")\" expected to end 'debug' pragma", "#pragma", "");
+            error(getCurrentLoc(), "\")\" expected to end 'debug' pragma", "#pragma", "");
             return;
         }
     } else {
@@ -2282,7 +2258,7 @@ void TParseContext::inductiveLoopCheck(TSourceLoc loc, TIntermNode* init, TInter
 //
 // Do any additional error checking, etc., once we know the parsing is done.
 //
-void TParseContext::finalize()
+void TParseContext::finalErrorCheck()
 {
     // Check on array indexes for ES 2.0 (version 100) limitations.
     for (size_t i = 0; i < needsIndexLimitationChecking.size(); ++i)

glslang/MachineIndependent/ParseHelper.h

@@ -40,6 +40,7 @@
 #include "../Include/ShHandle.h"
 #include "SymbolTable.h"
 #include "localintermediate.h"
+#include "Scan.h"
 
 namespace glslang {
 
@@ -64,9 +65,10 @@ public:
     TParseContext(TSymbolTable&, TIntermediate&, bool parsingBuiltins, int version, EProfile, EShLanguage, TInfoSink&,
                   bool forwardCompatible = false, EShMessages messages = EShMsgDefault);
 
-public:
+    void setLimits(const TLimits&);
-    bool parseShaderStrings(TPpContext&, char* strings[], size_t strLen[], int numStrings);
+    bool parseShaderStrings(TPpContext&, TInputScanner& input, bool versionWillBeError = false);
     void parserError(const char *s);     // for bison's yyerror
+    const char* getPreamble();
 
     void C_DECL error(TSourceLoc, const char *szReason, const char *szToken,
                       const char *szExtraInfoFormat, ...);
@@ -163,6 +165,9 @@ public:
     TPpContext* getPpContext() const { return ppContext; }
     void addError() { ++numErrors; }
     int getNumErrors() const { return numErrors; }
+    const TSourceLoc& getCurrentLoc() const { return currentScanner->getSourceLoc(); }
+    void setCurrentLine(int line) { currentScanner->setLine(line); }
+    void setCurrentString(int string) { currentScanner->setString(string); }
 
     // The following are implemented in Versions.cpp to localize version/profile/stage/extensions control
     void initializeExtensionBehavior();
@@ -177,14 +182,13 @@ public:
     void doubleCheck(TSourceLoc, const char* op);
 
 protected:
-    const char* getPreamble();
     void nonInitConstCheck(TSourceLoc, TString& identifier, TType& type);
     TVariable* declareNonArray(TSourceLoc, TString& identifier, TType&, bool& newDeclaration);
     void declareArray(TSourceLoc, TString& identifier, const TType&, TSymbol*&, bool& newDeclaration);
     TIntermNode* executeInitializer(TSourceLoc, TString& identifier, TIntermTyped* initializer, TVariable* variable);
     TIntermTyped* convertInitializerList(TSourceLoc, const TType&, TIntermTyped* initializer);
     TOperator mapTypeToConstructorOp(const TType&);
-    void finalize();
+    void finalErrorCheck();
 
 public:
     //
@@ -213,13 +217,13 @@ public:
     TQualifier currentBlockQualifier;
     TIntermAggregate *linkage;   // aggregate node of objects the linker may need, if not referenced by the rest of the AST
     TPrecisionQualifier defaultPrecision[EbtNumTypes];
-    TSourceLoc currentLoc;
     bool tokensBeforeEOF;
     TLimits limits;
 
 protected:
     TScanContext* scanContext;
     TPpContext* ppContext;
+    TInputScanner* currentScanner;
     int numErrors;               // number of compile-time errors encountered
     bool parsingBuiltins;        // true if parsing built-in symbols/functions
     TMap<TString, TExtensionBehavior> extensionBehavior;    // for each extension string, what its current behavior is set to

glslang/MachineIndependent/Scan.cpp

@@ -40,12 +40,12 @@
 
 #include <string.h>
 
-#include "Scan.h"
 #include "../Include/Types.h"
 #include "SymbolTable.h"
 #include "glslang_tab.cpp.h"
 #include "ParseHelper.h"
 #include "ScanContext.h"
+#include "Scan.h"
 
 // preprocessor includes
 #include "preprocessor/PpContext.h"
@@ -54,37 +54,37 @@
 namespace glslang {
     
 // read past any white space
-void ConsumeWhiteSpace(TInputScanner& input, bool& foundNonSpaceTab)
+void TInputScanner::consumeWhiteSpace(bool& foundNonSpaceTab)
 {
-    char c = input.peek();  // don't accidentally consume anything other than whitespace
+    char c = peek();  // don't accidentally consume anything other than whitespace
     while (c == ' ' || c == '\t' || c == '\r' || c == '\n') {
         if (c == '\r' || c == '\n')
             foundNonSpaceTab = true;
-        input.get();
+        get();
-        c = input.peek();
+        c = peek();
     }
 }
 
 // return true if a comment was actually consumed
-bool ConsumeComment(TInputScanner& input)
+bool TInputScanner::consumeComment()
 {
-    if (input.peek() != '/')
+    if (peek() != '/')
         return false;
 
-    input.get();  // consume the '/'
+    get();  // consume the '/'
-    char c = input.peek();
+    char c = peek();
     if (c == '/') {
 
         // a '//' style comment
-        input.get();  // consume the second '/'
+        get();  // consume the second '/'
-        c = input.get();
+        c = get();
         do {
             while (c > 0 && c != '\\' && c != '\r' && c != '\n')
-                c = input.get();
+                c = get();
 
             if (c <= 0 || c == '\r' || c == '\n') {
                 while (c == '\r' || c == '\n')
-                    c = input.get();
+                    c = get();
 
                 // we reached the end of the comment
                 break;
@@ -92,30 +92,30 @@ bool ConsumeComment(TInputScanner& input)
                 // it's a '\', so we need to keep going, after skipping what's escaped
                     
                 // read the skipped character
-                c = input.get();
+                c = get();
 
                 // if it's a two-character newline, skip both characters
-                if (c == '\r' && input.peek() == '\n')
+                if (c == '\r' && peek() == '\n')
-                    input.get();
+                    get();
-                c = input.get();
+                c = get();
             }
         } while (true);
 
         // put back the last non-comment character
         if (c > 0)
-            input.unget();
+            unget();
 
         return true;
     } else if (c == '*') {
 
         // a '/*' style comment
-        input.get();  // consume the '*'
+        get();  // consume the '*'
-        c = input.get();
+        c = get();
         do {
             while (c > 0 && c != '*')
-                c = input.get();
+                c = get();
             if (c == '*') {
-                c = input.get();
+                c = get();
                 if (c == '/')
                     break;  // end of comment
                 // not end of comment
@@ -126,26 +126,26 @@ bool ConsumeComment(TInputScanner& input)
         return true;
     } else {
         // it's not a comment, put the '/' back
-        input.unget();
+        unget();
 
         return false;
     }
 }
 
 // skip whitespace, then skip a comment, rinse, repeat
-void ConsumeWhitespaceComment(TInputScanner& input, bool& foundNonSpaceTab)
+void TInputScanner::consumeWhitespaceComment(bool& foundNonSpaceTab)
 {
     do {
-        ConsumeWhiteSpace(input, foundNonSpaceTab);
+        consumeWhiteSpace(foundNonSpaceTab);
  
         // if not starting a comment now, then done
-        char c = input.peek();
+        char c = peek();
         if (c != '/' || c < 0)
             return;
 
         // skip potential comment 
         foundNonSpaceTab = true;
-        if (! ConsumeComment(input))
+        if (! consumeComment())
             return;
 
     } while (true);
@@ -159,9 +159,9 @@ void ConsumeWhitespaceComment(TInputScanner& input, bool& foundNonSpaceTab)
 // is that scanning will start anew, following the rules for the chosen version/profile,
 // and with a corresponding parsing context.
 //
-bool ScanVersion(TInputScanner& input, int& version, EProfile& profile)
+bool TInputScanner::scanVersion(int& version, EProfile& profile)
 {
-    // This function doesn't have to get all the semantics correct, 
+    // This function doesn't have to get all the semantics correct,
     // just find the #version if there is a correct one present.
     // The preprocessor will have the responsibility of getting all the semantics right.
 
@@ -169,43 +169,43 @@ bool ScanVersion(TInputScanner& input, int& version, EProfile& profile)
     profile = ENoProfile;
 
     bool foundNonSpaceTab = false;
-    ConsumeWhitespaceComment(input, foundNonSpaceTab);
+    consumeWhitespaceComment(foundNonSpaceTab);
 
     // #
-    if (input.get() != '#')
+    if (get() != '#')
         return true;
 
     // whitespace
     char c;
     do {
-        c = input.get();
+        c = get();
     } while (c == ' ' || c == '\t');
 
-    if (          c != 'v' ||
+    if (    c != 'v' ||
-        input.get() != 'e' ||
+        get() != 'e' ||
-        input.get() != 'r' ||
+        get() != 'r' ||
-        input.get() != 's' ||
+        get() != 's' ||
-        input.get() != 'i' ||
+        get() != 'i' ||
-        input.get() != 'o' ||
+        get() != 'o' ||
-        input.get() != 'n')
+        get() != 'n')
         return true;
 
     // whitespace
     do {
-        c = input.get();
+        c = get();
     } while (c == ' ' || c == '\t');
 
     // version number
     while (c >= '0' && c <= '9') {
         version = 10 * version + (c - '0');
-        c = input.get();
+        c = get();
     }
     if (version == 0)
         return true;
     
     // whitespace
     while (c == ' ' || c == '\t')
-        c = input.get();
+        c = get();
 
     // profile
     const int maxProfileLength = 13;  // not including any 0
@@ -215,7 +215,7 @@ bool ScanVersion(TInputScanner& input, int& version, EProfile& profile)
         if (c < 0 || c == ' ' || c == '\t' || c == '\n' || c == '\r')
             break;
         profileString[profileLength] = c;
-        c = input.get();
+        c = get();
     }
     if (c > 0 && c != ' ' && c != '\t' && c != '\n' && c != '\r')
         return true;

glslang/MachineIndependent/Scan.h

@@ -33,6 +33,8 @@
 //ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 //POSSIBILITY OF SUCH DAMAGE.
 //
+#ifndef _GLSLANG_SCAN_INCLUDED_
+#define _GLSLANG_SCAN_INCLUDED_
 
 #include "Versions.h"
 
@@ -44,7 +46,17 @@ namespace glslang {
 //
 class TInputScanner {
 public:
-    TInputScanner(int n, const char* const i[], size_t L[]) : numSources(n), sources(i), lengths(L), currentSource(0), currentChar(0) { }
+    TInputScanner(int n, const char* const s[], size_t L[], int b = 0) : 
+        numSources(n), sources(s), lengths(L), currentSource(0), currentChar(0), stringBias(b)
+    {
+        loc = new TSourceLoc[numSources];
+        loc[currentSource].string = -stringBias;
+        loc[currentSource].line = 1;
+    }
+    virtual ~TInputScanner()
+    {
+        delete [] loc;
+    }
 
     // return of -1 means end of strings,
     // anything else is the next character
@@ -56,23 +68,13 @@ public:
             return -1;
 
         char ret = sources[currentSource][currentChar];
+        if (ret == '\n')
+            ++loc[currentSource].line;
         advance();
 
         return ret;
     }
 
-    // advance one character
-    void advance()
-    {
-        ++currentChar;
-        if (currentChar >= static_cast<int>(lengths[currentSource])) {
-            ++currentSource;
-            currentChar = 0;
-            while (currentSource < numSources && lengths[currentSource] == 0)
-                ++currentSource;
-        }
-    }
-
     // retrieve the next character, no advance
     char peek()
     {
@@ -95,21 +97,58 @@ public:
             if (currentChar < 0)
                 currentChar = 0;
         }
+        if (peek() == '\n')
+            --loc[currentSource].line;
     }
 
+    // for #line override
+    void setLine(int newLine) { loc[currentSource].line = newLine; }
+    void setString(int newString) { loc[currentSource].string = newString; }
+
+    const TSourceLoc& getSourceLoc() const { return loc[currentSource]; }
+
+    void consumeWhiteSpace(bool& foundNonSpaceTab);
+    bool consumeComment();
+    void consumeWhitespaceComment(bool& foundNonSpaceTab);
+    bool scanVersion(int& version, EProfile& profile);
+
 protected:
+
+    // advance one character
+    void advance()
+    {
+        ++currentChar;
+        if (currentChar >= static_cast<int>(lengths[currentSource])) {
+            ++currentSource;
+            if (currentSource < numSources) {
+                loc[currentSource].string = loc[currentSource - 1].string + 1;
+                loc[currentSource].line = 1;
+            }
+            while (currentSource < numSources && lengths[currentSource] == 0) {
+                ++currentSource;
+                if (currentSource < numSources) {
+                    loc[currentSource].string = loc[currentSource - 1].string + 1;
+                    loc[currentSource].line = 1;
+                }
+            }
+            currentChar = 0;
+        }
+    }
+
     int numSources;             // number of strings in source
     const char* const *sources; // array of strings
     const size_t *lengths;      // length of each string
     int currentSource;
     int currentChar;
+
+    // This is for reporting what string/line an error occurred on, and can be overridden by #line.
+    // It remembers the last state of each source string as it is left for the next one, so unget() 
+    // can restore that state.
+    TSourceLoc* loc;  // an array
+
+    int stringBias;   // the first string that is the user's string number 0
 };
 
-// TODO: The location of these is still pending a grand design for going to a singular
-// scanner for version finding, preprocessing, and tokenizing:
-void ConsumeWhiteSpace(TInputScanner& input, bool& foundNonSpaceTab);
-bool ConsumeComment(TInputScanner& input);
-void ConsumeWhitespaceComment(TInputScanner& input, bool& foundNonSpaceTab);
-bool ScanVersion(TInputScanner& input, int& version, EProfile& profile);
-
 } // end namespace glslang
+
+#endif // _GLSLANG_SCAN_INCLUDED_

glslang/MachineIndependent/ShaderLang.cpp

@@ -143,7 +143,8 @@ bool InitializeSymbolTable(const TString& builtIns, int version, EProfile profil
     builtInShaders[0] = builtIns.c_str();
     builtInLengths[0] = builtIns.size();
 
-    if (! parseContext.parseShaderStrings(ppContext, const_cast<char**>(builtInShaders), builtInLengths, 1) != 0) {
+    TInputScanner input(1, builtInShaders, builtInLengths);
+    if (! parseContext.parseShaderStrings(ppContext, input) != 0) {
         infoSink.info.message(EPrefixInternalError, "Unable to parse built-ins");
         printf("Unable to parse built-ins\n%s\n", infoSink.info.c_str());
 
@@ -420,27 +421,42 @@ bool CompileDeferred(
     if (! InitThread())
         return false;
 
-    if (numStrings == 0)
-        return true;
-
     // This must be undone (.pop()) by the caller, after it finishes consuming the created tree.
     GetThreadPoolAllocator().push();
+
+    if (numStrings == 0)
+        return true;
     
-    // move to length-based strings, rather than null-terminated strings
+    // Move to length-based strings, rather than null-terminated strings.
-    size_t* lengths = new size_t[numStrings];
+    // Also, add strings to include the preamble and to ensure the shader is not null,
+    // which lets the grammar accept what was a null (post preprocessing) shader.
+    //
+    // Shader will look like
+    //   string 0:                preamble
+    //   string 1...numStrings:   user's shader
+    //   string numStrings+1:     "int;"
+    //
+    size_t* lengths = new size_t[numStrings + 2];
+    const char** strings = new const char*[numStrings + 2];
     for (int s = 0; s < numStrings; ++s) {
+        strings[s + 1] = shaderStrings[s];
         if (inputLengths == 0 || inputLengths[s] < 0)
-            lengths[s] = strlen(shaderStrings[s]);
+            lengths[s + 1] = strlen(shaderStrings[s]);
         else
-            lengths[s] = inputLengths[s];
+            lengths[s + 1] = inputLengths[s];
     }
 
+    // First, without using the preprocessor or parser, find the #version, so we know what
+    // symbol tables, processing rules, etc. to set up.  This does not need the extra strings
+    // outlined above, just the user shader.
     int version;
     EProfile profile;
-    glslang::TInputScanner input(numStrings, shaderStrings, lengths);
+    glslang::TInputScanner userInput(numStrings, &strings[1], &lengths[1]);  // no preamble
-    bool versionNotFirst = ScanVersion(input, version, profile);
+    bool versionNotFirst = userInput.scanVersion(version, profile);
+    bool versionNotFound = version == 0;
     bool goodVersion = DeduceVersionProfile(compiler->infoSink, compiler->getLanguage(), versionNotFirst, defaultVersion, version, profile);
-
+    bool versionWillBeError = (versionNotFound || (profile == EEsProfile && versionNotFirst));
+    
     intermediate.setVersion(version);
     intermediate.setProfile(profile);
     SetupBuiltinSymbolTable(version, profile);
@@ -458,31 +474,36 @@ bool CompileDeferred(
     // Add built-in symbols that are potentially context dependent;
     // they get popped again further down.
     AddContextSpecificSymbols(resources, compiler->infoSink, symbolTable, version, profile, compiler->getLanguage());
+    
+    //
+    // Now we can process the full shader under proper symbols and rules.
+    //
 
     TParseContext parseContext(symbolTable, intermediate, false, version, profile, compiler->getLanguage(), compiler->infoSink, forwardCompatible, messages);
     glslang::TScanContext scanContext(parseContext);
     TPpContext ppContext(parseContext);
     parseContext.setScanContext(&scanContext);
     parseContext.setPpContext(&ppContext);
-    parseContext.limits = resources->limits;
+    parseContext.setLimits(resources->limits);
     if (! goodVersion)
         parseContext.addError();
 
     parseContext.initializeExtensionBehavior();
 
-    //
-    // Parse the application's shaders.  All the following symbol table
-    // work will be throw-away, so push a new allocation scope that can
-    // be thrown away, then push a scope for the current shader's globals.
-    //
     bool success = true;
     
-    symbolTable.push();
+    // Fill in the strings as outlined above.
-    if (! symbolTable.atGlobalLevel())
+    strings[0] = parseContext.getPreamble();
-        parseContext.infoSink.info.message(EPrefixInternalError, "Wrong symbol table level");
+    lengths[0] = strlen(strings[0]);
+    strings[numStrings + 1] = "\n int;";
+    lengths[numStrings + 1] = strlen(strings[numStrings + 1]);
+    TInputScanner fullInput(numStrings + 2, strings, lengths, 1);
 
-    bool ret = parseContext.parseShaderStrings(ppContext, const_cast<char**>(shaderStrings), lengths, numStrings);
+    // Push a new symbol allocation scope that can for the shader's globals.
-    if (! ret)
+    symbolTable.push();
+
+    // Parse the full shader.
+    if (! parseContext.parseShaderStrings(ppContext, fullInput, versionWillBeError))
         success = false;
     intermediate.addSymbolLinkageNodes(parseContext.linkage, parseContext.language, symbolTable);
 
@@ -503,6 +524,7 @@ bool CompileDeferred(
         intermediate.outputTree(parseContext.infoSink);
 
     delete [] lengths;
+    delete [] strings;
 
     return success;
 }

glslang/MachineIndependent/Versions.cpp

@@ -330,14 +330,14 @@ void TParseContext::updateExtensionBehavior(const char* extName, const char* beh
     else if (! strcmp("warn", behaviorString))
         behavior = EBhWarn;
     else
-        error(currentLoc, "behavior not supported", "#extension", behaviorString);
+        error(getCurrentLoc(), "behavior not supported", "#extension", behaviorString);
 
     // Update the current behavior
     TMap<TString, TExtensionBehavior>::iterator iter;
     if (! strcmp(extName, "all")) {
         // special case for the 'all' extension; apply it to every extension present
         if (behavior == EBhRequire || behavior == EBhEnable) {
-            error(currentLoc, "extension 'all' cannot have 'require' or 'enable' behavior", "#extension", "");
+            error(getCurrentLoc(), "extension 'all' cannot have 'require' or 'enable' behavior", "#extension", "");
             return;
         } else {
             for (iter = extensionBehavior.begin(); iter != extensionBehavior.end(); ++iter)
@@ -349,12 +349,12 @@ void TParseContext::updateExtensionBehavior(const char* extName, const char* beh
         if (iter == extensionBehavior.end()) {
             switch (behavior) {
             case EBhRequire:
-                error(currentLoc, "extension not supported", "#extension", extName);
+                error(getCurrentLoc(), "extension not supported", "#extension", extName);
                 break;
             case EBhEnable:
             case EBhWarn:
             case EBhDisable:
-                warn(currentLoc, "extension not supported", "#extension", extName);
+                warn(getCurrentLoc(), "extension not supported", "#extension", extName);
                 break;
             default:
                 assert(0 && "unexpected behavior");

glslang/MachineIndependent/preprocessor/Pp.cpp

@@ -134,7 +134,7 @@ int TPpContext::FinalCPP()
     mem_FreePool(pool);
 
     if (ifdepth)
-        parseContext.error(parseContext.currentLoc, "missing #endif", "#if", "");
+        parseContext.error(parseContext.getCurrentLoc(), "missing #endif", "#if", "");
 
     return 1;
 }
@@ -552,21 +552,21 @@ int TPpContext::CPPline(TPpToken * ppToken)
         return token;
     }
     else if (token == CPP_INTCONSTANT) {
-        parseContext.currentLoc.line = atoi(ppToken->name);
+        parseContext.setCurrentLine(atoi(ppToken->name));
         token = currentInput->scan(this, currentInput, ppToken);
 
         if (token == CPP_INTCONSTANT) {
-            parseContext.currentLoc.string = atoi(ppToken->name);
+            parseContext.setCurrentString(atoi(ppToken->name));
             token = currentInput->scan(this, currentInput, ppToken);
             if (token != '\n')
-                parseContext.error(parseContext.currentLoc, "cannot be followed by more than two integral literals", "#line", "");
+                parseContext.error(parseContext.getCurrentLoc(), "cannot be followed by more than two integral literals", "#line", "");
         } else if (token == '\n')
 
             return token;
         else
-            parseContext.error(parseContext.currentLoc, "second argument can only be an integral literal", "#line", "");
+            parseContext.error(parseContext.getCurrentLoc(), "second argument can only be an integral literal", "#line", "");
     } else
-        parseContext.error(parseContext.currentLoc, "first argument can only be an integral literal", "#line", "");
+        parseContext.error(parseContext.getCurrentLoc(), "first argument can only be an integral literal", "#line", "");
 
     return token;
 }
@@ -662,8 +662,8 @@ int TPpContext::CPPversion(TPpToken * ppToken)
 {
     int token = currentInput->scan(this, currentInput, ppToken);
 
-    if (notAVersionToken)
+    if (errorOnVersion)
-        parseContext.error(ppToken->loc, "must occur before any other statement in the program", "#version", "");
+        parseContext.error(ppToken->loc, "must occur first in shader", "#version", "");
 
     if (token == '\n') {
         parseContext.error(ppToken->loc, "must be followed by version number", "#version", "");
@@ -759,7 +759,6 @@ int TPpContext::readCPPline(TPpToken * ppToken)
             } else {
                 parseContext.error(ppToken->loc, "#else after a #else", "#else", "");
                 ifdepth = 0;
-                notAVersionToken = true;
                 return 0;
             }
         } else if (ppToken->atom == elifAtom) {
@@ -805,8 +804,6 @@ int TPpContext::readCPPline(TPpToken * ppToken)
         token = currentInput->scan(this, currentInput, ppToken);
     }
 
-    notAVersionToken = ! isVersion;
-
     return token;
 } // readCPPline
 
@@ -931,7 +928,7 @@ int TPpContext::MacroExpand(int atom, TPpToken* ppToken, int expandUndef)
     int depth = 0;
 
     if (atom == __LINE__Atom) {
-        ppToken->ival = parseContext.currentLoc.line;
+        ppToken->ival = parseContext.getCurrentLoc().line;
         sprintf(ppToken->name, "%d", ppToken->ival);
         UngetToken(CPP_INTCONSTANT, ppToken);
 
@@ -939,7 +936,7 @@ int TPpContext::MacroExpand(int atom, TPpToken* ppToken, int expandUndef)
     }
 
     if (atom == __FILE__Atom) {
-        ppToken->ival = parseContext.currentLoc.string;
+        ppToken->ival = parseContext.getCurrentLoc().string;
         sprintf(ppToken->name, "%d", ppToken->ival);
         UngetToken(CPP_INTCONSTANT, ppToken);
 
@@ -964,8 +961,6 @@ int TPpContext::MacroExpand(int atom, TPpToken* ppToken, int expandUndef)
 
     in = (MacroInputSrc*)malloc(sizeof(*in));
     memset(in, 0, sizeof(*in));
-    in->base.line = currentInput->line;
-    in->base.name = currentInput->name;
 
     if ((! sym || sym->mac.undef) && expandUndef) {
         // push input

glslang/MachineIndependent/preprocessor/PpContext.cpp

@@ -84,7 +84,7 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace glslang {
 
 TPpContext::TPpContext(TParseContext& pc) : 
-    preamble(0), strings(0), notAVersionToken(false), parseContext(pc)
+    preamble(0), strings(0), parseContext(pc)
 {
     InitAtomTable();
     InitScanner(this);
@@ -101,28 +101,17 @@ TPpContext::~TPpContext()
     delete [] preamble;
 }
 
-void TPpContext::setPreamble(const char* p, size_t l)
+void TPpContext::setInput(TInputScanner& input, bool versionWillBeError)
 {
-    if (p && l > 0) {
+    StringInputSrc *in = (StringInputSrc *)malloc(sizeof(StringInputSrc));
-        // preAmble could be a hard-coded string; make writable copy
+    memset(in, 0, sizeof(StringInputSrc));
-        // TODO: efficiency PP: make it not need writable strings
+    in->input = &input;
-        preambleLength = l;
+    in->base.scan = byte_scan;
-        preamble = new char[preambleLength + 1];
+    in->base.getch = (int (*)(TPpContext*, InputSrc *, TPpToken *))str_getch;
-        memcpy(preamble, p, preambleLength + 1);  // TODO: PP: assuming nul-terminated strings
+    in->base.ungetch = (void (*)(TPpContext*, InputSrc *, int, TPpToken *))str_ungetch;
-        ScanFromString(preamble);
+    in->base.prev = currentInput;
-        currentString = -1;
+    currentInput = &in->base;
-    }
+    errorOnVersion = versionWillBeError;
-}
-
-void TPpContext::setShaderStrings(char* s[], size_t l[], int n)
-{
-    strings = s;
-    lengths = l;
-    numStrings = n;
-    if (! preamble) {
-        ScanFromString(strings[0]);
-        currentString = 0;
-    }
 }
 
 } // end namespace glslang

glslang/MachineIndependent/preprocessor/PpContext.h

@@ -95,6 +95,8 @@ public:
     char   name[maxTokenLength+1];
 };
 
+class TInputScanner;
+
 // This class is the result of turning a huge pile of C code communicating through globals
 // into a class.  This was done to allowing instancing to attain thread safety.
 // Don't expect too much in terms of OO design.
@@ -104,7 +106,7 @@ public:
     virtual ~TPpContext();
 
     void setPreamble(const char* preamble, size_t length);
-    void setShaderStrings(char* strings[], size_t lengths[], int numStrings);
+    void setInput(TInputScanner& input, bool versionWillBeError);
 
     const char* tokenize(TPpToken* ppToken);
 
@@ -113,8 +115,6 @@ public:
         int			(*scan)(TPpContext*, struct InputSrc *, TPpToken *);
         int			(*getch)(TPpContext*, struct InputSrc *, TPpToken *);
         void		(*ungetch)(TPpContext*, struct InputSrc *, int, TPpToken *);
-        int			name;  /* atom */
-        int			line;
     };
 
     struct TokenBlock {
@@ -177,7 +177,6 @@ protected:
     // Scanner data:
     int mostRecentToken;        // Most recent token seen by the scanner
     int previous_token;
-    bool notAVersionToken;      // used to make sure that #version is the first token seen in the file, if present
     TParseContext& parseContext;
 
     static const int maxMacroArgs = 64;
@@ -195,6 +194,7 @@ protected:
     };
 
     InputSrc *currentInput;
+    bool errorOnVersion;
 
     //
     // from Pp.cpp
@@ -289,7 +289,7 @@ protected:
     //
     struct StringInputSrc {
         InputSrc base;
-        char *p;
+        TInputScanner* input;
     };
     int InitScanner(TPpContext *cpp);
     static int str_getch(TPpContext*, StringInputSrc *in);

glslang/MachineIndependent/preprocessor/PpScanner.cpp

@@ -88,23 +88,7 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "PpContext.h"
 #include "PpTokens.h"
-
+#include "Scan.h"
-namespace {
-
-using namespace glslang;
-
-int eof_scan(TPpContext*, TPpContext::InputSrc*, TPpToken*)
-{
-    return EOF;
-}
-
-void noop(TPpContext*, TPpContext::InputSrc *in, int ch, TPpToken * ppToken)
-{
-}
-
-TPpContext::InputSrc eof_inputsrc = { 0, &eof_scan, &eof_scan, &noop };
-
-} // end anonymous namespace
 
 namespace glslang {
 
@@ -115,79 +99,26 @@ int TPpContext::InitScanner(TPpContext *cpp)
         return 0;
 
     mostRecentToken = 0;
-    currentInput = &eof_inputsrc;
+    currentInput = 0;
     previous_token = '\n';
-    notAVersionToken = false;
-
-    return 1;
-} // InitScanner
-
-/*
-* str_getch()
-* takes care of reading from multiple strings.
-* returns the next-char from the input stream.
-* returns EOF when the complete shader is parsed.
-*/
-int TPpContext::str_getch(TPpContext* pp, StringInputSrc *in)
-{
-    for(;;) {
-        if (*in->p) {
-            if (*in->p == '\n') {
-                in->base.line++;
-                ++pp->parseContext.currentLoc.line;
-            }
-            return *in->p++;
-        }
-        if (pp->currentString < 0) {
-            // we only parsed the built-in pre-amble; start with clean slate for user code
-            pp->notAVersionToken = false;
-        }
-        if (++(pp->currentString) < pp->numStrings) {
-            free(in);
-            pp->parseContext.currentLoc.string = pp->currentString;
-            pp->parseContext.currentLoc.line = 1;
-            pp->ScanFromString(pp->strings[pp->currentString]);
-            in=(StringInputSrc*)pp->currentInput;
-            continue;             
-        } else {
-            pp->currentInput = in->base.prev;
-            pp->currentString = 0;
-            free(in);
-            return EOF;
-        }  
-    }
-} // str_getch
-
-void TPpContext::str_ungetch(TPpContext* pp, StringInputSrc *in, int ch, TPpToken *type)
-{
-    if (in->p[-1] == ch)in->p--;
-    else {
-        *(in->p)='\0'; //this would take care of shifting to the previous string.
-        pp->currentString--;
-        pp->parseContext.currentLoc.string = pp->currentString;
-    }  
-    if (ch == '\n') {
-        in->base.line--;
-        --pp->parseContext.currentLoc.line;
-    }
-} // str_ungetch
-
-int TPpContext::ScanFromString(char *s)
-{
-
-    StringInputSrc *in = (StringInputSrc *)malloc(sizeof(StringInputSrc));
-    memset(in, 0, sizeof(StringInputSrc));
-    in->p = s;
-    in->base.line = 1;
-    in->base.scan = byte_scan;
-    in->base.getch = (int (*)(TPpContext*, InputSrc *, TPpToken *))str_getch;
-    in->base.ungetch = (void (*)(TPpContext*, InputSrc *, int, TPpToken *))str_ungetch;
-    in->base.prev = currentInput;
-    currentInput = &in->base;
 
     return 1;
 }
 
+int TPpContext::str_getch(TPpContext* pp, StringInputSrc *in)
+{
+    int ch = in->input->get();
+
+    if (ch == EOF)
+        free(in);
+
+    return ch;
+}
+
+void TPpContext::str_ungetch(TPpContext* pp, StringInputSrc *in, int ch, TPpToken *type)
+{
+    in->input->unget();
+}
 
 ///////////////////////////////////////////////////////////////////////////////////////////////
 /////////////////////////////////// Floating point constants: /////////////////////////////////
@@ -332,7 +263,7 @@ int TPpContext::byte_scan(TPpContext* pp, InputSrc *in, TPpToken * ppToken)
             ch = pp->currentInput->getch(pp, pp->currentInput, ppToken);
         }
 
-        ppToken->loc = pp->parseContext.currentLoc;
+        ppToken->loc = pp->parseContext.getCurrentLoc();
         len = 0;
         switch (ch) {
         default:
@@ -696,6 +627,7 @@ int TPpContext::byte_scan(TPpContext* pp, InputSrc *in, TPpToken * ppToken)
                 return '.';
             }
         case '/':
+            // TODO: preprocessor: use the Scan.cpp comment scanner
             ch = pp->currentInput->getch(pp, pp->currentInput, ppToken);
             if (ch == '/') {
                 do {
@@ -801,8 +733,6 @@ const char* TPpContext::tokenize(TPpToken* ppToken)
         if (token == '\n')
             continue;
 
-        notAVersionToken = true;
-
         // expand macros
         if (token == CPP_IDENTIFIER && MacroExpand(ppToken->atom, ppToken, 0) == 1)
             continue;
@@ -831,7 +761,7 @@ int TPpContext::check_EOF(int token)
 {
     if (token == EOF) {
         if (ifdepth > 0)
-            parseContext.error(parseContext.currentLoc, "missing #endif", "#if", "");
+            parseContext.error(parseContext.getCurrentLoc(), "missing #endif", "#if", "");
         return 1;
     }
     return 0;

glslang/MachineIndependent/preprocessor/PpTokens.cpp

@@ -306,7 +306,7 @@ int TPpContext::ReadToken(TokenStream *pTok, TPpToken *ppToken)
     char ch;
 
     ltoken = lReadByte(pTok);
-    ppToken->loc = parseContext.currentLoc;
+    ppToken->loc = parseContext.getCurrentLoc();
     if (ltoken >= 0) {
         if (ltoken > 127)
             ltoken += 128;
@@ -399,12 +399,9 @@ int TPpContext::scan_token(TPpContext* pp, TokenInputSrc *in, TPpToken * ppToken
 {
     int token = pp->ReadToken(in->tokens, ppToken);
     int (*final)(TPpContext *);
-    if (token == '\n') {
-        in->base.line++;
-        return token;
-    }
     if (token > 0)
         return token;
+    
     pp->currentInput = in->base.prev;
     final = in->final;
     free(in);
@@ -418,10 +415,8 @@ int TPpContext::ReadFromTokenStream(TokenStream *ts, int name, int (*final)(TPpC
 {
     TokenInputSrc *in = (TokenInputSrc *) malloc(sizeof(TokenInputSrc));
     memset(in, 0, sizeof(TokenInputSrc));
-    in->base.name = name;
     in->base.prev = currentInput;
     in->base.scan = (int (*)(TPpContext*, InputSrc*, TPpToken*))scan_token;
-    in->base.line = 1;
     in->tokens = ts;
     in->final = final;
     RewindTokenStream(ts);
@@ -449,8 +444,6 @@ void TPpContext::UngetToken(int token, TPpToken * ppToken)
     t->lval = *ppToken;
     t->base.scan = (int(*)(TPpContext*, struct InputSrc *, TPpToken *))reget_token;
     t->base.prev = currentInput;
-    t->base.name = currentInput->name;
-    t->base.line = currentInput->line;
     currentInput = &t->base;
 }
 

glslang/Public/ShaderLang.h

@@ -270,7 +270,7 @@ class TShader {
 public:
     explicit TShader(EShLanguage);
     virtual ~TShader();
-    void setStrings(char** s, int n) { strings = s; numStrings = n; }
+    void setStrings(const char* const* s, int n) { strings = s; numStrings = n; }
     bool parse(const TBuiltInResource*, int defaultVersion, bool forwardCompatible, EShMessages);
     const char* getInfoLog();
     const char* getInfoDebugLog();
@@ -280,7 +280,7 @@ protected:
     TCompiler* compiler;
     TIntermediate* intermediate;
     TInfoSink* infoSink;
-    char** strings;
+    const char* const* strings;
     int numStrings;
 
     friend class TProgram;