Fix double expansion of macro arguments.

This adds a new fullyExpanded flag that makes sure that macro arguments
only get expanded once. This can happen either in PrescanMacroArg, or, if
there is token pasting or a function-like macro name has been passed as
an argument and may need to be expanded when used as a function.

Test/baseResults/preprocessor.function_macro.vert.out

@@ -13,9 +13,12 @@
 
 
 
+
+
 int main(){
   gl_Position = vec4(3 + 1, 3 + 4, 3 + 1);
   gl_Position = vec4(1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12);
   gl_Position = vec4(4 + 3 + 3);
+  gl_Position = 4 + 3 + F . a;
 }
 

Test/preprocessor.function_macro.vert

@@ -13,8 +13,11 @@
   )\
 4 + 3 + Y
 
+#define F F.a
+
 int main() {
   gl_Position = vec4(X(3), Y(3, 4), Z(3));
   gl_Position = vec4(REALLY_LONG_MACRO_NAME_WITH_MANY_PARAMETERS(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12));
   gl_Position = vec4(A(3));
+  gl_Position = A(F);
 }

glslang/MachineIndependent/preprocessor/Pp.cpp

@@ -1126,9 +1126,6 @@ int TPpContext::tMacroInput::scan(TPpToken* ppToken)
         pasting = true;
     }
 
-    // HLSL does expand macros before concatenation
-    if (pasting && pp->parseContext.isReadingHLSL())
-        pasting = false;
 
     // TODO: preprocessor:  properly handle whitespace (or lack of it) between tokens when expanding
     if (token == PpAtomIdentifier) {
@@ -1138,9 +1135,12 @@ int TPpContext::tMacroInput::scan(TPpToken* ppToken)
                 break;
         if (i >= 0) {
             TokenStream* arg = expandedArgs[i];
-            if (arg == nullptr || pasting)
+            bool expanded = !!arg && !pasting;
+            // HLSL does expand macros before concatenation
+            if (arg == nullptr || (pasting && !pp->parseContext.isReadingHLSL()) ) {
                 arg = args[i];
-            pp->pushTokenStreamInput(*arg, prepaste);
+            }
+            pp->pushTokenStreamInput(*arg, prepaste, expanded);
 
             return pp->scanToken(ppToken);
         }
@@ -1183,6 +1183,9 @@ MacroExpandResult TPpContext::MacroExpand(TPpToken* ppToken, bool expandUndef, b
 {
     ppToken->space = false;
     int macroAtom = atomStrings.getAtom(ppToken->name);
+    if (ppToken->fullyExpanded)
+        return MacroExpandNotStarted;
+
     switch (macroAtom) {
     case PpAtomLineMacro:
         // Arguments which are macro have been replaced in the first stage.
@@ -1214,8 +1217,10 @@ MacroExpandResult TPpContext::MacroExpand(TPpToken* ppToken, bool expandUndef, b
     MacroSymbol* macro = macroAtom == 0 ? nullptr : lookupMacroDef(macroAtom);
 
     // no recursive expansions
-    if (macro != nullptr && macro->busy)
+    if (macro != nullptr && macro->busy) {
+        ppToken->fullyExpanded = true;
         return MacroExpandNotStarted;
+    }
 
     // not expanding undefined macros
     if ((macro == nullptr || macro->undef) && ! expandUndef)

glslang/MachineIndependent/preprocessor/PpContext.h

@@ -102,6 +102,7 @@ public:
         i64val = 0;
         loc.init();
         name[0] = 0;
+        fullyExpanded = false;
     }
 
     // Used for comparing macro definitions, so checks what is relevant for that.
@@ -117,6 +118,8 @@ public:
     // True if a space (for white space or a removed comment) should also be
     // recognized, in front of the token returned:
     bool space;
+
+    bool fullyExpanded;
     // Numeric value of the token:
     union {
         int ival;
@@ -475,16 +478,27 @@ protected:
     //
     // From PpTokens.cpp
     //
-    void pushTokenStreamInput(TokenStream&, bool pasting = false);
+    void pushTokenStreamInput(TokenStream&, bool pasting = false, bool expanded = false);
     void UngetToken(int token, TPpToken*);
 
     class tTokenInput : public tInput {
     public:
-        tTokenInput(TPpContext* pp, TokenStream* t, bool prepasting) :
+        tTokenInput(TPpContext* pp, TokenStream* t, bool prepasting, bool expanded) :
             tInput(pp),
             tokens(t),
-            lastTokenPastes(prepasting) { }
+            lastTokenPastes(prepasting),
-        virtual int scan(TPpToken *ppToken) override { return tokens->getToken(pp->parseContext, ppToken); }
+            preExpanded(expanded) { }
+        virtual int scan(TPpToken *ppToken) override {
+            int token = tokens->getToken(pp->parseContext, ppToken);
+            ppToken->fullyExpanded = preExpanded;
+            if (tokens->atEnd() && token == PpAtomIdentifier) {
+                int macroAtom = pp->atomStrings.getAtom(ppToken->name);
+                MacroSymbol* macro = macroAtom == 0 ? nullptr : pp->lookupMacroDef(macroAtom);
+                if (macro && macro->functionLike)
+                    ppToken->fullyExpanded = false;
+            }
+            return token;
+        }
         virtual int getch() override { assert(0); return EndOfInput; }
         virtual void ungetch() override { assert(0); }
         virtual bool peekPasting() override { return tokens->peekTokenizedPasting(lastTokenPastes); }
@@ -492,6 +506,7 @@ protected:
     protected:
         TokenStream* tokens;
         bool lastTokenPastes; // true if the last token in the input is to be pasted, rather than consumed as a token
+        bool preExpanded;
     };
 
     class tUngotTokenInput : public tInput {

glslang/MachineIndependent/preprocessor/PpScanner.cpp

@@ -480,9 +480,7 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
         E_GL_EXT_shader_explicit_arithmetic_types_int16 };
     static const int Num_Int16_Extensions = sizeof(Int16_Extensions) / sizeof(Int16_Extensions[0]);
 
-    ppToken->ival = 0;
+    ppToken->clear();
-    ppToken->i64val = 0;
-    ppToken->space = false;
     ch = getch();
     for (;;) {
         while (ch == ' ' || ch == '\t') {

glslang/MachineIndependent/preprocessor/PpTokens.cpp

@@ -195,9 +195,9 @@ bool TPpContext::TokenStream::peekUntokenizedPasting()
     return pasting;
 }
 
-void TPpContext::pushTokenStreamInput(TokenStream& ts, bool prepasting)
+void TPpContext::pushTokenStreamInput(TokenStream& ts, bool prepasting, bool expanded)
 {
-    pushInput(new tTokenInput(this, &ts, prepasting));
+    pushInput(new tTokenInput(this, &ts, prepasting, expanded));
     ts.reset();
 }