diff --git a/Test/baseResults/120.vert.out b/Test/baseResults/120.vert.out index 94a97b53..c63d95a2 100644 --- a/Test/baseResults/120.vert.out +++ b/Test/baseResults/120.vert.out @@ -77,9 +77,8 @@ ERROR: 0:192: 'gl_ClipDistance' : left of '[' is not of type array, matrix, or ERROR: 0:192: 'assign' : l-value required (can't modify a const) ERROR: 0:195: 'gl_ModelViewMatrix' : identifiers starting with "gl_" are reserved ERROR: 0:200: 'token pasting (##)' : not supported for this version or the enabled extensions -ERROR: 0:200: '##' : token pasting not implemented (internal error) -ERROR: 0:200: '' : syntax error -ERROR: 80 compilation errors. No code generated. +ERROR: 0:203: 'token pasting (##)' : not supported for this version or the enabled extensions +ERROR: 79 compilation errors. No code generated. Shader version: 120 @@ -427,7 +426,8 @@ ERROR: node is still EOpNull! 0:? 'c2D' (in 2-component vector of float) 0:? 'c3D' (in 3-component vector of float) 0:? 'v4' (uniform 4-component vector of float) -0:? 'abc' (global int) +0:? 'abcdef' (global int) +0:? 'qrstuv' (global int) Linked vertex stage: @@ -499,5 +499,6 @@ ERROR: node is still EOpNull! 0:? 'c2D' (in 2-component vector of float) 0:? 'c3D' (in 3-component vector of float) 0:? 'v4' (uniform 4-component vector of float) -0:? 'abc' (global int) +0:? 'abcdef' (global int) +0:? 'qrstuv' (global int) diff --git a/Test/baseResults/130.vert.out b/Test/baseResults/130.vert.out index 1b3a0e19..7ca32a13 100644 --- a/Test/baseResults/130.vert.out +++ b/Test/baseResults/130.vert.out @@ -3,9 +3,7 @@ ERROR: 0:59: 'gl_InstanceID' : undeclared identifier ERROR: 0:59: '=' : cannot convert from 'temp float' to 'temp int' ERROR: 0:61: 'texelFetch' : no matching overloaded function found ERROR: 0:61: 'assign' : cannot convert from 'const float' to 'temp int' -ERROR: 0:75: '##' : token pasting not implemented (internal error) -ERROR: 0:75: '' : syntax error -ERROR: 6 compilation errors. No code generated. +ERROR: 4 compilation errors. No code generated. Shader version: 130 @@ -149,7 +147,8 @@ ERROR: node is still EOpNull! 0:? 'v4' (uniform 4-component vector of float) 0:? 'gl_ClipDistance' (smooth out implicitly-sized array of float ClipDistance) 0:? 'gl_TexCoord' (smooth out implicitly-sized array of 4-component vector of float TexCoord) -0:? 'abc' (global int) +0:? 'abcdef' (global int) +0:? 'qrstuv' (global int) 0:? 'gl_VertexID' (gl_VertexId int VertexId) @@ -281,6 +280,7 @@ ERROR: node is still EOpNull! 0:? 'v4' (uniform 4-component vector of float) 0:? 'gl_ClipDistance' (smooth out 2-element array of float ClipDistance) 0:? 'gl_TexCoord' (smooth out 1-element array of 4-component vector of float TexCoord) -0:? 'abc' (global int) +0:? 'abcdef' (global int) +0:? 'qrstuv' (global int) 0:? 'gl_VertexID' (gl_VertexId int VertexId) diff --git a/Test/baseResults/tokenPaste.vert.out b/Test/baseResults/tokenPaste.vert.out new file mode 100755 index 00000000..59971515 --- /dev/null +++ b/Test/baseResults/tokenPaste.vert.out @@ -0,0 +1,60 @@ +tokenPaste.vert +Warning, version 450 is not yet complete; most version-specific features are present, but some are missing. +ERROR: 0:38: '##' : unexpected location +ERROR: 0:40: '##' : unexpected location; end of replacement list +ERROR: 0:49: '##' : combined tokens are too long +ERROR: 0:52: '##' : only supported for preprocessing identifiers +ERROR: 4 compilation errors. No code generated. + + +Shader version: 450 +ERROR: node is still EOpNull! +0:52 Sequence +0:52 move second child to first child (temp int) +0:52 'a' (global int) +0:52 Constant: +0:52 11 (const int) +0:? Linker Objects +0:? 'SecondExpansion' (global int) +0:? 'PostPasteExpansion' (global int) +0:? 'foo27' (global float) +0:? 'foo155' (uniform float) +0:? 'foo719' (global float) +0:? 'barfoo' (uniform float) +0:? 'argless' (global float) +0:? 'dc1' (global float) +0:? 'dc2' (global float) +0:? 'foo875' (uniform float) +0:? 'ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123451234' (global float) +0:? 'a' (global int) +0:? 'gl_VertexID' (gl_VertexId int VertexId) +0:? 'gl_InstanceID' (gl_InstanceId int InstanceId) + + +Linked vertex stage: + +ERROR: Linking vertex stage: Missing entry point: Each stage requires one entry point + +Shader version: 450 +ERROR: node is still EOpNull! +0:52 Sequence +0:52 move second child to first child (temp int) +0:52 'a' (global int) +0:52 Constant: +0:52 11 (const int) +0:? Linker Objects +0:? 'SecondExpansion' (global int) +0:? 'PostPasteExpansion' (global int) +0:? 'foo27' (global float) +0:? 'foo155' (uniform float) +0:? 'foo719' (global float) +0:? 'barfoo' (uniform float) +0:? 'argless' (global float) +0:? 'dc1' (global float) +0:? 'dc2' (global float) +0:? 'foo875' (uniform float) +0:? 'ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123451234' (global float) +0:? 'a' (global int) +0:? 'gl_VertexID' (gl_VertexId int VertexId) +0:? 'gl_InstanceID' (gl_InstanceId int InstanceId) + diff --git a/Test/tokenPaste.vert b/Test/tokenPaste.vert new file mode 100644 index 00000000..6d6212cc --- /dev/null +++ b/Test/tokenPaste.vert @@ -0,0 +1,52 @@ +#version 450 + +// side test verifies multiple rounds of argument expansion +#define bear SecondExpansion +#define mmmB bear +#define mmmA(a) a +int mmmA(mmmB); // mmmB -> bear, and then in mmmA(), bear -> SecondExpansion + +// pasting skips the first round of expansion +#define mmcatmmdog PostPasteExpansion +#define mmcat cat +#define mmdog dog +#define mmp(a,b) a## b +int mmp(mmcat, mmdog); // mmcat/mmdog not expanded, mmcatmmdog -> PostPasteExpansion + +// multi-token pre +#define mmtokpastepre(a) a##27 +mmtokpastepre(float foo); // should declare "float foo27;" + +// multi-token post +#define mmtokpastepost(a) uni ##a +mmtokpastepost(form float foo155); // should declare "uniform float foo155;" + +// non-first argument +#define foo ShouldntExpandToThis +#define semi ; +#define bothpaste(a,b) a##b +float bothpaste(foo, 719); // should declare "float foo719;" +#define secpaste(a,b) a bar ## b +secpaste(uniform float, foo semi) // should declare "uniform float barfoo;" + +// no args +#define noArg fl##oat +noArg argless; + +// bad location +#define bad1 ## float +bad1 dc1; +#define bad2 float ## +bad2 dc2; + +// multiple ## +#define multiPaste(a, b, c) a##or##b flo##at foo##c +multiPaste(unif, m, 875); + +// too long +#define simplePaste(a,b) a##b +// 1020 + 5 characters +float simplePaste(ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF012345, 12345); + +// non-identifiers +int a = simplePaste(11,12); diff --git a/glslang/Include/revision.h b/glslang/Include/revision.h index ea129596..d90b8c78 100644 --- a/glslang/Include/revision.h +++ b/glslang/Include/revision.h @@ -2,5 +2,5 @@ // For the version, it uses the latest git tag followed by the number of commits. // For the date, it uses the current date (when then script is run). -#define GLSLANG_REVISION "Overload400-PrecQual.1695" -#define GLSLANG_DATE "16-Dec-2016" +#define GLSLANG_REVISION "Overload400-PrecQual.1696" +#define GLSLANG_DATE "19-Dec-2016" diff --git a/glslang/MachineIndependent/preprocessor/Pp.cpp b/glslang/MachineIndependent/preprocessor/Pp.cpp index 824a69a9..1985b8bc 100644 --- a/glslang/MachineIndependent/preprocessor/Pp.cpp +++ b/glslang/MachineIndependent/preprocessor/Pp.cpp @@ -933,36 +933,38 @@ int TPpContext::readCPPline(TPpToken* ppToken) return token; } -TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream* a, TPpToken* ppToken, bool newLineOkay) +// Macro-expand a macro argument 'arg' to create 'expandedArg'. +// Does not replace 'arg'. +// Returns nullptr if no expanded argument is created. +TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream* arg, TPpToken* ppToken, bool newLineOkay) { int token; - TokenStream *n; - RewindTokenStream(a); + RewindTokenStream(arg); do { - token = ReadToken(a, ppToken); + token = ReadToken(arg, ppToken); if (token == PpAtomIdentifier && LookUpSymbol(ppToken->atom)) break; } while (token != EndOfInput); if (token == EndOfInput) - return a; + return nullptr; - n = new TokenStream; + TokenStream* expandedArg = new TokenStream; pushInput(new tMarkerInput(this)); - pushTokenStreamInput(a); + pushTokenStreamInput(arg); while ((token = scanToken(ppToken)) != tMarkerInput::marker) { if (token == PpAtomIdentifier && MacroExpand(ppToken->atom, ppToken, false, newLineOkay) != 0) continue; - RecordToken(n, token, ppToken); + RecordToken(expandedArg, token, ppToken); } popInput(); - delete a; - return n; + return expandedArg; } // -// Return the next token for a macro expansion, handling macro args. +// Return the next token for a macro expansion, handling macro arguments, +// whose semantics are dependent on being adjacent to ##. // int TPpContext::tMacroInput::scan(TPpToken* ppToken) { @@ -971,6 +973,39 @@ int TPpContext::tMacroInput::scan(TPpToken* ppToken) token = pp->ReadToken(mac->body, ppToken); } while (token == ' '); // handle white space in macro + // Hash operators basically turn off a round of macro substitution + // (the round done on the argument before the round done on the RHS of the + // macro definition): + // + // "A parameter in the replacement list, unless preceded by a # or ## + // preprocessing token or followed by a ## preprocessing token (see below), + // is replaced by the corresponding argument after all macros contained + // therein have been expanded." + // + // "If, in the replacement list, a parameter is immediately preceded or + // followed by a ## preprocessing token, the parameter is replaced by the + // corresponding argument's preprocessing token sequence." + + bool pasting = false; + if (postpaste) { + // don't expand next token + pasting = true; + postpaste = false; + } + + if (prepaste) { + // already know we should be on a ##, verify + assert(token == PpAtomPaste); + prepaste = false; + postpaste = true; + } + + // see if are preceding a ## + if (peekMacPasting()) { + prepaste = true; + pasting = true; + } + // TODO: preprocessor: properly handle whitespace (or lack of it) between tokens when expanding if (token == PpAtomIdentifier) { int i; @@ -978,7 +1013,10 @@ int TPpContext::tMacroInput::scan(TPpToken* ppToken) if (mac->args[i] == ppToken->atom) break; if (i >= 0) { - pp->pushTokenStreamInput(args[i]); + TokenStream* arg = expandedArgs[i]; + if (arg == nullptr || pasting) + arg = args[i]; + pp->pushTokenStreamInput(arg, prepaste); return pp->scanToken(ppToken); } @@ -990,6 +1028,31 @@ int TPpContext::tMacroInput::scan(TPpToken* ppToken) return token; } +// See if the next non-white-space token in the macro is ## +bool TPpContext::tMacroInput::peekMacPasting() +{ + // don't return early, have to restore this + size_t savePos = mac->body->current; + + // skip white-space + int ltoken; + do { + ltoken = pp->lReadByte(mac->body); + } while (ltoken == ' '); + + // check for ## + bool pasting = false; + if (ltoken == '#') { + ltoken = pp->lReadByte(mac->body); + if (ltoken == '#') + pasting = true; + } + + mac->body->current = savePos; + + return pasting; +} + // return a textual zero, for scanning a macro that was never defined int TPpContext::tZeroInput::scan(TPpToken* ppToken) { @@ -1080,6 +1143,9 @@ int TPpContext::MacroExpand(int atom, TPpToken* ppToken, bool expandUndef, bool in->args.resize(in->mac->argc); for (int i = 0; i < in->mac->argc; i++) in->args[i] = new TokenStream; + in->expandedArgs.resize(in->mac->argc); + for (int i = 0; i < in->mac->argc; i++) + in->expandedArgs[i] = nullptr; int arg = 0; bool tokenRecorded = false; do { @@ -1143,8 +1209,11 @@ int TPpContext::MacroExpand(int atom, TPpToken* ppToken, bool expandUndef, bool } parseContext.ppError(loc, "Too many args in macro", "macro expansion", GetAtomString(atom)); } + + // We need both expanded and non-expanded forms of the argument, for whether or + // not token pasting is in play. for (int i = 0; i < in->mac->argc; i++) - in->args[i] = PrescanMacroArg(in->args[i], ppToken, newLineOkay); + in->expandedArgs[i] = PrescanMacroArg(in->args[i], ppToken, newLineOkay); } pushInput(in); diff --git a/glslang/MachineIndependent/preprocessor/PpContext.h b/glslang/MachineIndependent/preprocessor/PpContext.h index 013c90e5..707280f0 100644 --- a/glslang/MachineIndependent/preprocessor/PpContext.h +++ b/glslang/MachineIndependent/preprocessor/PpContext.h @@ -129,6 +129,7 @@ public: void setPreamble(const char* preamble, size_t length); const char* tokenize(TPpToken* ppToken); + int tokenPaste(TPpToken&); class tInput { public: @@ -138,6 +139,8 @@ public: virtual int scan(TPpToken*) = 0; virtual int getch() = 0; virtual void ungetch() = 0; + virtual bool peekPasting() { return false; } // true when about to see ## + virtual bool endOfReplacementList() { return false; } // true when at the end of a macro replacement list (RHS of #define) // Will be called when we start reading tokens from this instance virtual void notifyActivated() {} @@ -235,6 +238,8 @@ protected: } int getChar() { return inputStack.back()->getch(); } void ungetChar() { inputStack.back()->ungetch(); } + bool peekPasting() { return !inputStack.empty() && inputStack.back()->peekPasting(); } + bool endOfReplacementList() { return inputStack.empty() || inputStack.back()->endOfReplacementList(); } static const int maxMacroArgs = 64; static const int maxIfNesting = 64; @@ -245,18 +250,29 @@ protected: class tMacroInput : public tInput { public: - tMacroInput(TPpContext* pp) : tInput(pp) { } + tMacroInput(TPpContext* pp) : tInput(pp), prepaste(false), postpaste(false) { } virtual ~tMacroInput() { for (size_t i = 0; i < args.size(); ++i) delete args[i]; + for (size_t i = 0; i < expandedArgs.size(); ++i) + delete expandedArgs[i]; } virtual int scan(TPpToken*); virtual int getch() { assert(0); return EndOfInput; } virtual void ungetch() { assert(0); } + bool peekPasting() override { return prepaste; } + bool endOfReplacementList() override { return mac->body->current >= mac->body->data.size(); } + MacroSymbol *mac; TVector args; + TVector expandedArgs; + + protected: + bool peekMacPasting(); + bool prepaste; // true if we are just before ## + bool postpaste; // true if we are right after ## }; class tMarkerInput : public tInput { @@ -329,17 +345,19 @@ protected: void RecordToken(TokenStream* pTok, int token, TPpToken* ppToken); void RewindTokenStream(TokenStream *pTok); int ReadToken(TokenStream* pTok, TPpToken* ppToken); - void pushTokenStreamInput(TokenStream *ts); + void pushTokenStreamInput(TokenStream *ts, bool pasting = false); void UngetToken(int token, TPpToken* ppToken); class tTokenInput : public tInput { public: - tTokenInput(TPpContext* pp, TokenStream* t) : tInput(pp), tokens(t) { } + tTokenInput(TPpContext* pp, TokenStream* t, bool prepasting) : tInput(pp), tokens(t), lastTokenPastes(prepasting) { } virtual int scan(TPpToken *); virtual int getch() { assert(0); return EndOfInput; } virtual void ungetch() { assert(0); } + virtual bool peekPasting() override; protected: TokenStream *tokens; + bool lastTokenPastes; // true if the last token in the input is to be pasted, rather than consumed as a token }; class tUngotTokenInput : public tInput { diff --git a/glslang/MachineIndependent/preprocessor/PpScanner.cpp b/glslang/MachineIndependent/preprocessor/PpScanner.cpp index 518dbdee..bc5c0866 100644 --- a/glslang/MachineIndependent/preprocessor/PpScanner.cpp +++ b/glslang/MachineIndependent/preprocessor/PpScanner.cpp @@ -739,6 +739,11 @@ const char* TPpContext::tokenize(TPpToken* ppToken) for(;;) { token = scanToken(ppToken); ppToken->token = token; + + // Handle token-pasting logic + token = tokenPaste(*ppToken); + ppToken->token = token; + if (token == EndOfInput) { missingEndifCheck(); return nullptr; @@ -800,6 +805,47 @@ const char* TPpContext::tokenize(TPpToken* ppToken) } } +// +// Do all token-pasting related combining of two pasted tokens when getting a +// stream of tokens from a replacement list. Degenerates to no processing if a +// replacement list is not the source of the token stream. +// +int TPpContext::tokenPaste(TPpToken& ppToken) +{ + // starting with ## is illegal, skip to next token + if (ppToken.token == PpAtomPaste) { + parseContext.ppError(ppToken.loc, "unexpected location", "##", ""); + ppToken.token = scanToken(&ppToken); + } + + // ## can be chained, process all in the chain at once + while (peekPasting()) { + TPpToken pastedPpToken; + + // next token has to be ## + pastedPpToken.token = scanToken(&pastedPpToken); + assert(pastedPpToken.token == PpAtomPaste); + + if (endOfReplacementList()) { + parseContext.ppError(ppToken.loc, "unexpected location; end of replacement list", "##", ""); + break; + } + + // get the token after the ## + scanToken(&pastedPpToken); + + // combine the tokens + if (strlen(ppToken.name) + strlen(pastedPpToken.name) > MaxTokenLength) + parseContext.ppError(ppToken.loc, "combined tokens are too long", "##", ""); + strncat(ppToken.name, pastedPpToken.name, MaxTokenLength - strlen(ppToken.name)); + ppToken.atom = LookUpAddString(ppToken.name); + if (ppToken.token != PpAtomIdentifier) + parseContext.ppError(ppToken.loc, "only supported for preprocessing identifiers", "##", ""); + } + + return ppToken.token; +} + // Checks if we've seen balanced #if...#endif void TPpContext::missingEndifCheck() { diff --git a/glslang/MachineIndependent/preprocessor/PpTokens.cpp b/glslang/MachineIndependent/preprocessor/PpTokens.cpp index 23b617d3..724cedd0 100644 --- a/glslang/MachineIndependent/preprocessor/PpTokens.cpp +++ b/glslang/MachineIndependent/preprocessor/PpTokens.cpp @@ -160,7 +160,7 @@ void TPpContext::RecordToken(TokenStream *pTok, int token, TPpToken* ppToken) } /* -* Reset a token stream in preperation for reading. +* Reset a token stream in preparation for reading. */ void TPpContext::RewindTokenStream(TokenStream *pTok) { @@ -187,9 +187,7 @@ int TPpContext::ReadToken(TokenStream *pTok, TPpToken *ppToken) if (lReadByte(pTok) == '#') { parseContext.requireProfile(ppToken->loc, ~EEsProfile, "token pasting (##)"); parseContext.profileRequires(ppToken->loc, ~EEsProfile, 130, 0, "token pasting (##)"); - parseContext.error(ppToken->loc, "token pasting not implemented (internal error)", "##", ""); - //return PpAtomPaste; - return ReadToken(pTok, ppToken); + ltoken = PpAtomPaste; } else lUnreadByte(pTok); } @@ -279,9 +277,34 @@ int TPpContext::tTokenInput::scan(TPpToken* ppToken) return pp->ReadToken(tokens, ppToken); } -void TPpContext::pushTokenStreamInput(TokenStream* ts) +// We are pasting if the entire macro is preceding a pasting operator +// (lastTokenPastes) and we are also on the last token. +bool TPpContext::tTokenInput::peekPasting() { - pushInput(new tTokenInput(this, ts)); + if (! lastTokenPastes) + return false; + // Getting here means the last token will be pasted. + + // Are we at the last non-whitespace token? + size_t savePos = tokens->current; + bool moreTokens = false; + do { + int byte = pp->lReadByte(tokens); + if (byte == EndOfInput) + break; + if (byte != ' ') { + moreTokens = true; + break; + } + } while (true); + tokens->current = savePos; + + return !moreTokens; +} + +void TPpContext::pushTokenStreamInput(TokenStream* ts, bool prepasting) +{ + pushInput(new tTokenInput(this, ts, prepasting)); RewindTokenStream(ts); } diff --git a/gtests/AST.FromFile.cpp b/gtests/AST.FromFile.cpp index a2e961e8..70705337 100644 --- a/gtests/AST.FromFile.cpp +++ b/gtests/AST.FromFile.cpp @@ -178,6 +178,7 @@ INSTANTIATE_TEST_CASE_P( "syntaxError.frag", "test.frag", "texture.frag", + "tokenPaste.vert", "types.frag", "uniformArray.frag", "variableArrayIndex.frag",