Merge pull request #637 from KhronosGroup/token-paste

PP: Implement token pasting for PP identifiers.

Test/baseResults/120.vert.out

@@ -77,9 +77,8 @@ ERROR: 0:192: 'gl_ClipDistance' :  left of '[' is not of type array, matrix, or
 ERROR: 0:192: 'assign' :  l-value required (can't modify a const)
 ERROR: 0:195: 'gl_ModelViewMatrix' : identifiers starting with "gl_" are reserved 
 ERROR: 0:200: 'token pasting (##)' : not supported for this version or the enabled extensions 
-ERROR: 0:200: '##' : token pasting not implemented (internal error) 
+ERROR: 0:203: 'token pasting (##)' : not supported for this version or the enabled extensions 
-ERROR: 0:200: '' :  syntax error
+ERROR: 79 compilation errors.  No code generated.
-ERROR: 80 compilation errors.  No code generated.
 
 
 Shader version: 120
@@ -427,7 +426,8 @@ ERROR: node is still EOpNull!
 0:?     'c2D' (in 2-component vector of float)
 0:?     'c3D' (in 3-component vector of float)
 0:?     'v4' (uniform 4-component vector of float)
-0:?     'abc' (global int)
+0:?     'abcdef' (global int)
+0:?     'qrstuv' (global int)
 
 
 Linked vertex stage:
@@ -499,5 +499,6 @@ ERROR: node is still EOpNull!
 0:?     'c2D' (in 2-component vector of float)
 0:?     'c3D' (in 3-component vector of float)
 0:?     'v4' (uniform 4-component vector of float)
-0:?     'abc' (global int)
+0:?     'abcdef' (global int)
+0:?     'qrstuv' (global int)
 

Test/baseResults/130.vert.out

@@ -3,9 +3,7 @@ ERROR: 0:59: 'gl_InstanceID' : undeclared identifier
 ERROR: 0:59: '=' :  cannot convert from 'temp float' to 'temp int'
 ERROR: 0:61: 'texelFetch' : no matching overloaded function found 
 ERROR: 0:61: 'assign' :  cannot convert from 'const float' to 'temp int'
-ERROR: 0:75: '##' : token pasting not implemented (internal error) 
+ERROR: 4 compilation errors.  No code generated.
-ERROR: 0:75: '' :  syntax error
-ERROR: 6 compilation errors.  No code generated.
 
 
 Shader version: 130
@@ -149,7 +147,8 @@ ERROR: node is still EOpNull!
 0:?     'v4' (uniform 4-component vector of float)
 0:?     'gl_ClipDistance' (smooth out implicitly-sized array of float ClipDistance)
 0:?     'gl_TexCoord' (smooth out implicitly-sized array of 4-component vector of float TexCoord)
-0:?     'abc' (global int)
+0:?     'abcdef' (global int)
+0:?     'qrstuv' (global int)
 0:?     'gl_VertexID' (gl_VertexId int VertexId)
 
 
@@ -281,6 +280,7 @@ ERROR: node is still EOpNull!
 0:?     'v4' (uniform 4-component vector of float)
 0:?     'gl_ClipDistance' (smooth out 2-element array of float ClipDistance)
 0:?     'gl_TexCoord' (smooth out 1-element array of 4-component vector of float TexCoord)
-0:?     'abc' (global int)
+0:?     'abcdef' (global int)
+0:?     'qrstuv' (global int)
 0:?     'gl_VertexID' (gl_VertexId int VertexId)
 

Test/baseResults/tokenPaste.vert.out

@@ -0,0 +1,60 @@
+tokenPaste.vert
+Warning, version 450 is not yet complete; most version-specific features are present, but some are missing.
+ERROR: 0:38: '##' : unexpected location 
+ERROR: 0:40: '##' : unexpected location; end of replacement list 
+ERROR: 0:49: '##' : combined tokens are too long 
+ERROR: 0:52: '##' : only supported for preprocessing identifiers 
+ERROR: 4 compilation errors.  No code generated.
+
+
+Shader version: 450
+ERROR: node is still EOpNull!
+0:52  Sequence
+0:52    move second child to first child (temp int)
+0:52      'a' (global int)
+0:52      Constant:
+0:52        11 (const int)
+0:?   Linker Objects
+0:?     'SecondExpansion' (global int)
+0:?     'PostPasteExpansion' (global int)
+0:?     'foo27' (global float)
+0:?     'foo155' (uniform float)
+0:?     'foo719' (global float)
+0:?     'barfoo' (uniform float)
+0:?     'argless' (global float)
+0:?     'dc1' (global float)
+0:?     'dc2' (global float)
+0:?     'foo875' (uniform float)
+0:?     'ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123451234' (global float)
+0:?     'a' (global int)
+0:?     'gl_VertexID' (gl_VertexId int VertexId)
+0:?     'gl_InstanceID' (gl_InstanceId int InstanceId)
+
+
+Linked vertex stage:
+
+ERROR: Linking vertex stage: Missing entry point: Each stage requires one entry point
+
+Shader version: 450
+ERROR: node is still EOpNull!
+0:52  Sequence
+0:52    move second child to first child (temp int)
+0:52      'a' (global int)
+0:52      Constant:
+0:52        11 (const int)
+0:?   Linker Objects
+0:?     'SecondExpansion' (global int)
+0:?     'PostPasteExpansion' (global int)
+0:?     'foo27' (global float)
+0:?     'foo155' (uniform float)
+0:?     'foo719' (global float)
+0:?     'barfoo' (uniform float)
+0:?     'argless' (global float)
+0:?     'dc1' (global float)
+0:?     'dc2' (global float)
+0:?     'foo875' (uniform float)
+0:?     'ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123451234' (global float)
+0:?     'a' (global int)
+0:?     'gl_VertexID' (gl_VertexId int VertexId)
+0:?     'gl_InstanceID' (gl_InstanceId int InstanceId)
+

Test/tokenPaste.vert

@@ -0,0 +1,52 @@
+#version 450
+
+// side test verifies multiple rounds of argument expansion
+#define bear SecondExpansion
+#define mmmB bear
+#define mmmA(a) a
+int mmmA(mmmB);                    // mmmB -> bear, and then in mmmA(), bear -> SecondExpansion
+
+// pasting skips the first round of expansion
+#define mmcatmmdog PostPasteExpansion
+#define mmcat cat
+#define mmdog dog
+#define mmp(a,b) a## b
+int mmp(mmcat, mmdog);             // mmcat/mmdog not expanded, mmcatmmdog -> PostPasteExpansion
+
+// multi-token pre
+#define mmtokpastepre(a) a##27
+mmtokpastepre(float foo);          // should declare "float foo27;"
+
+// multi-token post
+#define mmtokpastepost(a) uni ##a
+mmtokpastepost(form float foo155); // should declare "uniform float foo155;"
+
+// non-first argument
+#define foo ShouldntExpandToThis
+#define semi ;
+#define bothpaste(a,b) a##b
+float bothpaste(foo, 719);          // should declare "float foo719;"
+#define secpaste(a,b) a bar ## b
+secpaste(uniform float, foo semi)   // should declare "uniform float barfoo;"
+
+// no args
+#define noArg fl##oat
+noArg argless;
+
+// bad location
+#define bad1 ## float
+bad1 dc1;
+#define bad2 float ##
+bad2 dc2;
+
+// multiple ##
+#define multiPaste(a, b, c) a##or##b flo##at foo##c
+multiPaste(unif, m, 875);
+
+// too long
+#define simplePaste(a,b) a##b
+// 1020 + 5 characters
+float simplePaste(ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF012345, 12345);
+
+// non-identifiers
+int a = simplePaste(11,12);

glslang/Include/revision.h

@@ -2,5 +2,5 @@
 // For the version, it uses the latest git tag followed by the number of commits.
 // For the date, it uses the current date (when then script is run).
 
-#define GLSLANG_REVISION "Overload400-PrecQual.1695"
+#define GLSLANG_REVISION "Overload400-PrecQual.1696"
-#define GLSLANG_DATE "16-Dec-2016"
+#define GLSLANG_DATE "19-Dec-2016"

glslang/MachineIndependent/preprocessor/Pp.cpp

@@ -933,36 +933,38 @@ int TPpContext::readCPPline(TPpToken* ppToken)
     return token;
 }
 
-TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream* a, TPpToken* ppToken, bool newLineOkay)
+// Macro-expand a macro argument 'arg' to create 'expandedArg'.
+// Does not replace 'arg'.
+// Returns nullptr if no expanded argument is created.
+TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream* arg, TPpToken* ppToken, bool newLineOkay)
 {
     int token;
-    TokenStream *n;
+    RewindTokenStream(arg);
-    RewindTokenStream(a);
     do {
-        token = ReadToken(a, ppToken);
+        token = ReadToken(arg, ppToken);
         if (token == PpAtomIdentifier && LookUpSymbol(ppToken->atom))
             break;
     } while (token != EndOfInput);
 
     if (token == EndOfInput)
-        return a;
+        return nullptr;
 
-    n = new TokenStream;
+    TokenStream* expandedArg = new TokenStream;
     pushInput(new tMarkerInput(this));
-    pushTokenStreamInput(a);
+    pushTokenStreamInput(arg);
     while ((token = scanToken(ppToken)) != tMarkerInput::marker) {
         if (token == PpAtomIdentifier && MacroExpand(ppToken->atom, ppToken, false, newLineOkay) != 0)
             continue;
-        RecordToken(n, token, ppToken);
+        RecordToken(expandedArg, token, ppToken);
     }
     popInput();
-    delete a;
 
-    return n;
+    return expandedArg;
 }
 
 // 
-// Return the next token for a macro expansion, handling macro args.
+// Return the next token for a macro expansion, handling macro arguments,
+// whose semantics are dependent on being adjacent to ##.
 //
 int TPpContext::tMacroInput::scan(TPpToken* ppToken)
 {
@@ -971,6 +973,39 @@ int TPpContext::tMacroInput::scan(TPpToken* ppToken)
         token = pp->ReadToken(mac->body, ppToken);
     } while (token == ' ');  // handle white space in macro
 
+    // Hash operators basically turn off a round of macro substitution
+    // (the round done on the argument before the round done on the RHS of the
+    // macro definition):
+    //
+    // "A parameter in the replacement list, unless preceded by a # or ##
+    // preprocessing token or followed by a ## preprocessing token (see below),
+    // is replaced by the corresponding argument after all macros contained
+    // therein have been expanded."
+    //
+    // "If, in the replacement list, a parameter is immediately preceded or
+    // followed by a ## preprocessing token, the parameter is replaced by the
+    // corresponding argument's preprocessing token sequence."
+
+    bool pasting = false;
+    if (postpaste) {
+        // don't expand next token
+        pasting = true;
+        postpaste = false;
+    }
+
+    if (prepaste) {
+        // already know we should be on a ##, verify
+        assert(token == PpAtomPaste);
+        prepaste = false;
+        postpaste = true;
+    }
+
+    // see if are preceding a ##
+    if (peekMacPasting()) {
+        prepaste = true;
+        pasting = true;
+    }
+
     // TODO: preprocessor:  properly handle whitespace (or lack of it) between tokens when expanding
     if (token == PpAtomIdentifier) {
         int i;
@@ -978,7 +1013,10 @@ int TPpContext::tMacroInput::scan(TPpToken* ppToken)
             if (mac->args[i] == ppToken->atom) 
                 break;
         if (i >= 0) {
-            pp->pushTokenStreamInput(args[i]);
+            TokenStream* arg = expandedArgs[i];
+            if (arg == nullptr || pasting)
+                arg = args[i];
+            pp->pushTokenStreamInput(arg, prepaste);
 
             return pp->scanToken(ppToken);
         }
@@ -990,6 +1028,31 @@ int TPpContext::tMacroInput::scan(TPpToken* ppToken)
     return token;
 }
 
+// See if the next non-white-space token in the macro is ##
+bool TPpContext::tMacroInput::peekMacPasting()
+{
+    // don't return early, have to restore this
+    size_t savePos = mac->body->current;
+
+    // skip white-space
+    int ltoken;
+    do {
+        ltoken = pp->lReadByte(mac->body);
+    } while (ltoken == ' ');
+
+    // check for ##
+    bool pasting = false;
+    if (ltoken == '#') {
+        ltoken = pp->lReadByte(mac->body);
+        if (ltoken == '#')
+            pasting = true;
+    }
+
+    mac->body->current = savePos;
+
+    return pasting;
+}
+
 // return a textual zero, for scanning a macro that was never defined
 int TPpContext::tZeroInput::scan(TPpToken* ppToken)
 {
@@ -1080,6 +1143,9 @@ int TPpContext::MacroExpand(int atom, TPpToken* ppToken, bool expandUndef, bool
         in->args.resize(in->mac->argc);
         for (int i = 0; i < in->mac->argc; i++)
             in->args[i] = new TokenStream;
+        in->expandedArgs.resize(in->mac->argc);
+        for (int i = 0; i < in->mac->argc; i++)
+            in->expandedArgs[i] = nullptr;
         int arg = 0;
         bool tokenRecorded = false;
         do {
@@ -1143,8 +1209,11 @@ int TPpContext::MacroExpand(int atom, TPpToken* ppToken, bool expandUndef, bool
             }
             parseContext.ppError(loc, "Too many args in macro", "macro expansion", GetAtomString(atom));
         }
+
+        // We need both expanded and non-expanded forms of the argument, for whether or
+        // not token pasting is in play.
         for (int i = 0; i < in->mac->argc; i++)
-            in->args[i] = PrescanMacroArg(in->args[i], ppToken, newLineOkay);
+            in->expandedArgs[i] = PrescanMacroArg(in->args[i], ppToken, newLineOkay);
     }
 
     pushInput(in);

glslang/MachineIndependent/preprocessor/PpContext.h

@@ -129,6 +129,7 @@ public:
     void setPreamble(const char* preamble, size_t length);
 
     const char* tokenize(TPpToken* ppToken);
+    int tokenPaste(TPpToken&);
 
     class tInput {
     public:
@@ -138,6 +139,8 @@ public:
         virtual int scan(TPpToken*) = 0;
         virtual int getch() = 0;
         virtual void ungetch() = 0;
+        virtual bool peekPasting() { return false; }          // true when about to see ##
+        virtual bool endOfReplacementList() { return false; } // true when at the end of a macro replacement list (RHS of #define)
 
         // Will be called when we start reading tokens from this instance
         virtual void notifyActivated() {}
@@ -235,6 +238,8 @@ protected:
     }
     int  getChar() { return inputStack.back()->getch(); }
     void ungetChar() { inputStack.back()->ungetch(); }
+    bool peekPasting() { return !inputStack.empty() && inputStack.back()->peekPasting(); }
+    bool endOfReplacementList() { return inputStack.empty() || inputStack.back()->endOfReplacementList(); }
 
     static const int maxMacroArgs = 64;
     static const int maxIfNesting = 64;
@@ -245,18 +250,29 @@ protected:
 
     class tMacroInput : public tInput {
     public:
-        tMacroInput(TPpContext* pp) : tInput(pp) { }
+        tMacroInput(TPpContext* pp) : tInput(pp), prepaste(false), postpaste(false) { }
         virtual ~tMacroInput()
         {
             for (size_t i = 0; i < args.size(); ++i)
                 delete args[i];
+            for (size_t i = 0; i < expandedArgs.size(); ++i)
+                delete expandedArgs[i];
         }
 
         virtual int scan(TPpToken*);
         virtual int getch() { assert(0); return EndOfInput; }
         virtual void ungetch() { assert(0); }
+        bool peekPasting() override { return prepaste; }
+        bool endOfReplacementList() override { return mac->body->current >= mac->body->data.size(); }
+
         MacroSymbol *mac;
         TVector<TokenStream*> args;
+        TVector<TokenStream*> expandedArgs;
+
+    protected:
+        bool peekMacPasting();
+        bool prepaste;         // true if we are just before ##
+        bool postpaste;        // true if we are right after ##
     };
 
     class tMarkerInput : public tInput {
@@ -329,17 +345,19 @@ protected:
     void RecordToken(TokenStream* pTok, int token, TPpToken* ppToken);
     void RewindTokenStream(TokenStream *pTok);
     int ReadToken(TokenStream* pTok, TPpToken* ppToken);
-    void pushTokenStreamInput(TokenStream *ts);
+    void pushTokenStreamInput(TokenStream *ts, bool pasting = false);
     void UngetToken(int token, TPpToken* ppToken);
     
     class tTokenInput : public tInput {
     public:
-        tTokenInput(TPpContext* pp, TokenStream* t) : tInput(pp), tokens(t) { }
+        tTokenInput(TPpContext* pp, TokenStream* t, bool prepasting) : tInput(pp), tokens(t), lastTokenPastes(prepasting) { }
         virtual int scan(TPpToken *);
         virtual int getch() { assert(0); return EndOfInput; }
         virtual void ungetch() { assert(0); }
+        virtual bool peekPasting() override;
     protected:
         TokenStream *tokens;
+        bool lastTokenPastes;     // true if the last token in the input is to be pasted, rather than consumed as a token
     };
 
     class tUngotTokenInput : public tInput {

glslang/MachineIndependent/preprocessor/PpScanner.cpp

@@ -739,6 +739,11 @@ const char* TPpContext::tokenize(TPpToken* ppToken)
     for(;;) {
         token = scanToken(ppToken);
         ppToken->token = token;
+
+        // Handle token-pasting logic
+        token = tokenPaste(*ppToken);
+        ppToken->token = token;
+
         if (token == EndOfInput) {
             missingEndifCheck();
             return nullptr;
@@ -800,6 +805,47 @@ const char* TPpContext::tokenize(TPpToken* ppToken)
     }
 }
 
+//
+// Do all token-pasting related combining of two pasted tokens when getting a
+// stream of tokens from a replacement list. Degenerates to no processing if a
+// replacement list is not the source of the token stream.
+//
+int TPpContext::tokenPaste(TPpToken& ppToken)
+{
+    // starting with ## is illegal, skip to next token
+    if (ppToken.token == PpAtomPaste) {
+        parseContext.ppError(ppToken.loc, "unexpected location", "##", "");
+        ppToken.token = scanToken(&ppToken);
+    }
+
+    // ## can be chained, process all in the chain at once
+    while (peekPasting()) {
+        TPpToken pastedPpToken;
+
+        // next token has to be ##
+        pastedPpToken.token = scanToken(&pastedPpToken);
+        assert(pastedPpToken.token == PpAtomPaste);
+
+        if (endOfReplacementList()) {
+            parseContext.ppError(ppToken.loc, "unexpected location; end of replacement list", "##", "");
+            break;
+        }
+
+        // get the token after the ##
+        scanToken(&pastedPpToken);
+
+        // combine the tokens
+        if (strlen(ppToken.name) + strlen(pastedPpToken.name) > MaxTokenLength)
+            parseContext.ppError(ppToken.loc, "combined tokens are too long", "##", "");
+        strncat(ppToken.name, pastedPpToken.name, MaxTokenLength - strlen(ppToken.name));
+        ppToken.atom = LookUpAddString(ppToken.name);
+        if (ppToken.token != PpAtomIdentifier)
+            parseContext.ppError(ppToken.loc, "only supported for preprocessing identifiers", "##", "");
+    }
+
+    return ppToken.token;
+}
+
 // Checks if we've seen balanced #if...#endif
 void TPpContext::missingEndifCheck()
 {

glslang/MachineIndependent/preprocessor/PpTokens.cpp

@@ -160,7 +160,7 @@ void TPpContext::RecordToken(TokenStream *pTok, int token, TPpToken* ppToken)
 }
 
 /*
-* Reset a token stream in preperation for reading.
+* Reset a token stream in preparation for reading.
 */
 void TPpContext::RewindTokenStream(TokenStream *pTok)
 {
@@ -187,9 +187,7 @@ int TPpContext::ReadToken(TokenStream *pTok, TPpToken *ppToken)
             if (lReadByte(pTok) == '#') {
                 parseContext.requireProfile(ppToken->loc, ~EEsProfile, "token pasting (##)");
                 parseContext.profileRequires(ppToken->loc, ~EEsProfile, 130, 0, "token pasting (##)");
-                parseContext.error(ppToken->loc, "token pasting not implemented (internal error)", "##", "");
+                ltoken = PpAtomPaste;
-                //return PpAtomPaste;
-                return ReadToken(pTok, ppToken);
             } else
                 lUnreadByte(pTok);
         }
@@ -279,9 +277,34 @@ int TPpContext::tTokenInput::scan(TPpToken* ppToken)
     return pp->ReadToken(tokens, ppToken);
 }
 
-void TPpContext::pushTokenStreamInput(TokenStream* ts)
+// We are pasting if the entire macro is preceding a pasting operator
+// (lastTokenPastes) and we are also on the last token.
+bool TPpContext::tTokenInput::peekPasting()
 {
-    pushInput(new tTokenInput(this, ts));
+    if (! lastTokenPastes)
+        return false;
+    // Getting here means the last token will be pasted.
+
+    // Are we at the last non-whitespace token?
+    size_t savePos = tokens->current;
+    bool moreTokens = false;
+    do {
+        int byte = pp->lReadByte(tokens);
+        if (byte == EndOfInput)
+            break;
+        if (byte != ' ') {
+            moreTokens = true;
+            break;
+        }
+    } while (true);
+    tokens->current = savePos;
+
+    return !moreTokens;
+}
+
+void TPpContext::pushTokenStreamInput(TokenStream* ts, bool prepasting)
+{
+    pushInput(new tTokenInput(this, ts, prepasting));
     RewindTokenStream(ts);
 }
 

gtests/AST.FromFile.cpp

@@ -178,6 +178,7 @@ INSTANTIATE_TEST_CASE_P(
         "syntaxError.frag",
         "test.frag",
         "texture.frag",
+        "tokenPaste.vert",
         "types.frag",
         "uniformArray.frag",
         "variableArrayIndex.frag",