From d67e15e9b4680ff2e788facee5e219531045a216 Mon Sep 17 00:00:00 2001 From: Lei Zhang Date: Wed, 15 Jul 2015 11:41:20 -0400 Subject: [PATCH 1/2] Create a new class to keep track of line number in preprocessor. SourceLineSynchronizer is added for keeping track of the last line number and output newlines appropriately if we switch to a new line in the preprocessor. --- glslang/MachineIndependent/ShaderLang.cpp | 108 ++++++++++++---------- 1 file changed, 57 insertions(+), 51 deletions(-) diff --git a/glslang/MachineIndependent/ShaderLang.cpp b/glslang/MachineIndependent/ShaderLang.cpp index 02fdc094..9611ac74 100644 --- a/glslang/MachineIndependent/ShaderLang.cpp +++ b/glslang/MachineIndependent/ShaderLang.cpp @@ -599,6 +599,36 @@ bool ProcessDeferred( return success; } +// Responsible for keeping track of the most recent line in the preprocessor +// and outputting newlines appropriately if the line changes. +class SourceLineSynchronizer { +public: + SourceLineSynchronizer(std::stringstream* output) : output(output) {} + SourceLineSynchronizer(const SourceLineSynchronizer&) = delete; + SourceLineSynchronizer& operator=(const SourceLineSynchronizer&) = delete; + + // If we switched to a new line, returns true and inserts newlines + // appropriately. Otherwise, returns false and outputs nothing. + bool syncToLine(int tokenLine) { + const bool newLineStarted = lastLine < tokenLine; + for (; lastLine < tokenLine; ++lastLine) { + if (lastLine > 0) *output << std::endl; + } + return newLineStarted; + } + + // Sets the line number of the most recent line to newLineNum. + void setLineNum(int newLineNum) { lastLine = newLineNum; } + +private: + // output stream for newlines. + std::stringstream* output; + // lastLine is the line number (starting from 1) of the last token processed. + // It is tracked in order for newlines to be inserted when a token appears + // on a new line. 0 means we haven't started processing any line in the + // current source string. + int lastLine = 0; +}; // DoPreprocessing is a valid ProcessingContext template argument, // which only performs the preprocessing step of compilation. @@ -615,99 +645,75 @@ struct DoPreprocessing { static const std::string noSpaceBeforeTokens = ","; glslang::TPpToken token; - std::stringstream outputStream; - int lastLine = -1; // lastLine is the line number of the last token - // processed. It is tracked in order for new-lines to be inserted when - // a token appears on a new line. - int lastToken = -1; parseContext.setScanner(&input); ppContext.setInput(input, versionWillBeError); - // Inserts newlines and incremnets lastLine until - // lastLine >= line. - auto adjustLine = [&lastLine, &outputStream](int line) { - int tokenLine = line - 1; - while(lastLine < tokenLine) { - if (lastLine >= 0) { - outputStream << std::endl; - } - ++lastLine; - } - }; + std::stringstream outputStream; + SourceLineSynchronizer lineSync(&outputStream); - parseContext.setExtensionCallback([&adjustLine, &outputStream]( + parseContext.setExtensionCallback([&lineSync, &outputStream]( int line, const char* extension, const char* behavior) { - adjustLine(line); + lineSync.syncToLine(line); outputStream << "#extension " << extension << " : " << behavior; }); - parseContext.setLineCallback([&adjustLine, &lastLine, &outputStream, &parseContext]( - int curLineNo, int newLineNo, bool hasSource, int sourceNum) { + parseContext.setLineCallback([&lineSync, &outputStream, &parseContext]( + int curLineNum, int newLineNum, bool hasSource, int sourceNum) { // SourceNum is the number of the source-string that is being parsed. - adjustLine(curLineNo); - outputStream << "#line " << newLineNo; + lineSync.syncToLine(curLineNum); + outputStream << "#line " << newLineNum; if (hasSource) { outputStream << " " << sourceNum; } if (parseContext.lineDirectiveShouldSetNextLine()) { - // newLineNo is the new line number for the line following the #line + // newLineNum is the new line number for the line following the #line // directive. So the new line number for the current line is - newLineNo -= 1; + newLineNum -= 1; } outputStream << std::endl; - // Line number starts from 1. And we are at the next line of the #line - // directive now. So lastLine (from 0) should be (newLineNo - 1) + 1. - lastLine = newLineNo; + // And we are at the next line of the #line directive now. + lineSync.setLineNum(newLineNum + 1); }); parseContext.setVersionCallback( - [&adjustLine, &outputStream](int line, int version, const char* str) { - adjustLine(line); + [&lineSync, &outputStream](int line, int version, const char* str) { + lineSync.syncToLine(line); outputStream << "#version " << version; if (str) { outputStream << " " << str; } }); - parseContext.setPragmaCallback([&adjustLine, &outputStream]( + parseContext.setPragmaCallback([&lineSync, &outputStream]( int line, const glslang::TVector& ops) { - adjustLine(line); + lineSync.syncToLine(line); outputStream << "#pragma "; for(size_t i = 0; i < ops.size(); ++i) { outputStream << ops[i]; } }); - parseContext.setErrorCallback([&adjustLine, &outputStream]( + parseContext.setErrorCallback([&lineSync, &outputStream]( int line, const char* errorMessage) { - adjustLine(line); + lineSync.syncToLine(line); outputStream << "#error " << errorMessage; }); + int lastToken = EOF; // lastToken records the last token processed. while (const char* tok = ppContext.tokenize(&token)) { - int tokenLine = token.loc.line - 1; // start at 0; - bool newLine = false; - while (lastLine < tokenLine) { - if (lastLine > -1) { - outputStream << std::endl; - newLine = true; - } - ++lastLine; - if (lastLine == tokenLine) { - // Don't emit whitespace onto empty lines. - // Copy any whitespace characters at the start of a line - // from the input to the output. - for(int i = 0; i < token.loc.column - 1; ++i) { - outputStream << " "; - } - } + bool isNewLine = lineSync.syncToLine(token.loc.line); + + if (isNewLine) { + // Don't emit whitespace onto empty lines. + // Copy any whitespace characters at the start of a line + // from the input to the output. + outputStream << std::string(token.loc.column - 1, ' '); } // Output a space in between tokens, but not at the start of a line, // and also not around special tokens. This helps with readability // and consistency. - if (!newLine && - lastToken != -1 && + if (!isNewLine && lastToken != -1 && (unNeededSpaceTokens.find((char)token.token) == std::string::npos) && (unNeededSpaceTokens.find((char)lastToken) == std::string::npos) && (noSpaceBeforeTokens.find((char)token.token) == std::string::npos)) { From 79f6451b5398e72b136bef4898510968dc74110c Mon Sep 17 00:00:00 2001 From: Lei Zhang Date: Thu, 9 Jul 2015 16:58:42 -0400 Subject: [PATCH 2/2] DoPreprocessing() should consider source string change. Everytime we switch to a new source string, the line are reset. We need to reset lastLine and also output newlines appropriately. --- glslang/MachineIndependent/ShaderLang.cpp | 49 +++++++++++++++++++---- 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/glslang/MachineIndependent/ShaderLang.cpp b/glslang/MachineIndependent/ShaderLang.cpp index 9611ac74..3cc26a4c 100644 --- a/glslang/MachineIndependent/ShaderLang.cpp +++ b/glslang/MachineIndependent/ShaderLang.cpp @@ -599,17 +599,40 @@ bool ProcessDeferred( return success; } -// Responsible for keeping track of the most recent line in the preprocessor -// and outputting newlines appropriately if the line changes. +// Responsible for keeping track of the most recent source string and line in +// the preprocessor and outputting newlines appropriately if the source string +// or line changes. class SourceLineSynchronizer { public: - SourceLineSynchronizer(std::stringstream* output) : output(output) {} + SourceLineSynchronizer(const std::function& lastSourceIndex, + std::stringstream* output) + : getLastSourceIndex(lastSourceIndex), output(output) {} SourceLineSynchronizer(const SourceLineSynchronizer&) = delete; SourceLineSynchronizer& operator=(const SourceLineSynchronizer&) = delete; - // If we switched to a new line, returns true and inserts newlines - // appropriately. Otherwise, returns false and outputs nothing. + // Sets the internally tracked source string index to that of the most + // recently read token. If we switched to a new source string, returns + // true and inserts a newline. Otherwise, returns false and outputs nothing. + bool syncToMostRecentString() { + if (getLastSourceIndex() != lastSource) { + // After switching to a new source string, we need to reset lastLine + // because line number resets every time a new source string is + // used. We also need to output a newline to separate the output + // from the previous source string (if there is one). + if (lastSource != -1 || lastLine != 0) + *output << std::endl; + lastSource = getLastSourceIndex(); + lastLine = -1; + return true; + } + return false; + } + + // Calls syncToMostRecentString() and then sets the internally tracked line + // number to tokenLine. If we switched to a new line, returns true and inserts + // newlines appropriately. Otherwise, returns false and outputs nothing. bool syncToLine(int tokenLine) { + syncToMostRecentString(); const bool newLineStarted = lastLine < tokenLine; for (; lastLine < tokenLine; ++lastLine) { if (lastLine > 0) *output << std::endl; @@ -617,12 +640,20 @@ public: return newLineStarted; } - // Sets the line number of the most recent line to newLineNum. + // Sets the internally tracked line number to newLineNum. void setLineNum(int newLineNum) { lastLine = newLineNum; } private: + // A function for getting the index of the last valid source string we've + // read tokens from. + const std::function getLastSourceIndex; // output stream for newlines. std::stringstream* output; + // lastSource is the source string index (starting from 0) of the last token + // processed. It is tracked in order for newlines to be inserted when a new + // source string starts. -1 means we haven't started processing any source + // string. + int lastSource = -1; // lastLine is the line number (starting from 1) of the last token processed. // It is tracked in order for newlines to be inserted when a token appears // on a new line. 0 means we haven't started processing any line in the @@ -649,7 +680,8 @@ struct DoPreprocessing { ppContext.setInput(input, versionWillBeError); std::stringstream outputStream; - SourceLineSynchronizer lineSync(&outputStream); + SourceLineSynchronizer lineSync( + std::bind(&TInputScanner::getLastValidSourceIndex, &input), &outputStream); parseContext.setExtensionCallback([&lineSync, &outputStream]( int line, const char* extension, const char* behavior) { @@ -701,6 +733,7 @@ struct DoPreprocessing { int lastToken = EOF; // lastToken records the last token processed. while (const char* tok = ppContext.tokenize(&token)) { + bool isNewString = lineSync.syncToMostRecentString(); bool isNewLine = lineSync.syncToLine(token.loc.line); if (isNewLine) { @@ -713,7 +746,7 @@ struct DoPreprocessing { // Output a space in between tokens, but not at the start of a line, // and also not around special tokens. This helps with readability // and consistency. - if (!isNewLine && lastToken != -1 && + if (!isNewString && !isNewLine && lastToken != -1 && (unNeededSpaceTokens.find((char)token.token) == std::string::npos) && (unNeededSpaceTokens.find((char)lastToken) == std::string::npos) && (noSpaceBeforeTokens.find((char)token.token) == std::string::npos)) {