PP: Remove second parsing of numbers recorded in macros; save/use original.

This partly addresses #1228 and #234 by reducing usage of strtod (or atof).
There is now only place to parse a floating-point number.

glslang/MachineIndependent/preprocessor/PpContext.h

@@ -92,13 +92,16 @@ namespace glslang {
 
 class TPpToken {
 public:
-    TPpToken() : space(false), i64val(0)
+    TPpToken() { clear(); }
+    void clear()
     {
+        space = false;
+        i64val = 0;
         loc.init();
         name[0] = 0;
     }
 
-    // This is used for comparing macro definitions, so checks what is relevant for that.
+    // Used for comparing macro definitions, so checks what is relevant for that.
     bool operator==(const TPpToken& right)
     {
         return space == right.space &&
@@ -108,15 +111,17 @@ public:
     bool operator!=(const TPpToken& right) { return ! operator==(right); }
 
     TSourceLoc loc;
-    bool space;  // true if a space (for white space or a removed comment) should also be recognized, in front of the token returned
+    // True if a space (for white space or a removed comment) should also be
-
+    // recognized, in front of the token returned:
+    bool space;
+    // Numeric value of the token:
     union {
         int ival;
         double dval;
         long long i64val;
     };
-
+    // Text string of the token:
-    char   name[MaxTokenLength + 1];
+    char name[MaxTokenLength + 1];
 };
 
 class TStringAtomMap {

glslang/MachineIndependent/preprocessor/PpTokens.cpp

@@ -97,6 +97,56 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 namespace glslang {
 
+
+namespace {
+
+    // When recording (and playing back) should the backing name string
+    // be saved (restored)?
+    bool SaveName(int atom)
+    {
+        switch (atom) {
+        case PpAtomIdentifier:
+        case PpAtomConstString:
+        case PpAtomConstInt:
+        case PpAtomConstUint:
+        case PpAtomConstInt64:
+        case PpAtomConstUint64:
+    #ifdef AMD_EXTENSIONS
+        case PpAtomConstInt16:
+        case PpAtomConstUint16:
+    #endif
+        case PpAtomConstFloat:
+        case PpAtomConstDouble:
+        case PpAtomConstFloat16:
+            return true;
+        default:
+            return false;
+        }
+    }
+
+    // When recording (and playing back) should the numeric value
+    // be saved (restored)?
+    bool SaveValue(int atom)
+    {
+        switch (atom) {
+        case PpAtomConstInt:
+        case PpAtomConstUint:
+        case PpAtomConstInt64:
+        case PpAtomConstUint64:
+    #ifdef AMD_EXTENSIONS
+        case PpAtomConstInt16:
+        case PpAtomConstUint16:
+    #endif
+        case PpAtomConstFloat:
+        case PpAtomConstDouble:
+        case PpAtomConstFloat16:
+            return true;
+        default:
+            return false;
+        }
+    }
+}
+
 // push onto back of stream
 void TPpContext::TokenStream::putSubtoken(char subtoken)
 {
@@ -121,42 +171,25 @@ void TPpContext::TokenStream::ungetSubtoken()
 
 // Add a complete token (including backing string) to the end of a list
 // for later playback.
-void TPpContext::TokenStream::putToken(int token, TPpToken* ppToken)
+void TPpContext::TokenStream::putToken(int atom, TPpToken* ppToken)
 {
-    const char* s;
+    // save the atom
-    char* str = NULL;
+    assert((atom & ~0xff) == 0);
+    putSubtoken(static_cast<char>(atom));
 
-    assert((token & ~0xff) == 0);
+    // save the backing name string
-    putSubtoken(static_cast<char>(token));
+    if (SaveName(atom)) {
-
+        const char* s = ppToken->name;
-    switch (token) {
-    case PpAtomIdentifier:
-    case PpAtomConstString:
-        s = ppToken->name;
         while (*s)
             putSubtoken(*s++);
         putSubtoken(0);
-        break;
+    }
-    case PpAtomConstInt:
+
-    case PpAtomConstUint:
+    // save the numeric value
-    case PpAtomConstInt64:
+    if (SaveValue(atom)) {
-    case PpAtomConstUint64:
+        const char* n = reinterpret_cast<const char*>(&ppToken->i64val);
-#ifdef AMD_EXTENSIONS
+        for (int i = 0; i < sizeof(ppToken->i64val); ++i)
-    case PpAtomConstInt16:
+            putSubtoken(*n++);
-    case PpAtomConstUint16:
-#endif
-    case PpAtomConstFloat:
-    case PpAtomConstDouble:
-    case PpAtomConstFloat16:
-        str = ppToken->name;
-        while (*str) {
-            putSubtoken(*str);
-            str++;
-        }
-        putSubtoken(0);
-        break;
-    default:
-        break;
     }
 }
 
@@ -164,38 +197,19 @@ void TPpContext::TokenStream::putToken(int token, TPpToken* ppToken)
 // (Not the source stream, but a stream used to hold a tokenized macro).
 int TPpContext::TokenStream::getToken(TParseContextBase& parseContext, TPpToken *ppToken)
 {
-    int len;
+    // get the atom
-    int ch;
+    int atom = getSubtoken();
+    if (atom == EndOfInput)
+        return atom;
 
-    int subtoken = getSubtoken();
+    // init the token
+    ppToken->clear();
     ppToken->loc = parseContext.getCurrentLoc();
-    switch (subtoken) {
+
-    case '#':
+    // get the backing name string
-        // Check for ##, unless the current # is the last character
+    if (SaveName(atom)) {
-        if (current < data.size()) {
+        int ch = getSubtoken();
-            if (getSubtoken() == '#') {
+        int len = 0;
-                parseContext.requireProfile(ppToken->loc, ~EEsProfile, "token pasting (##)");
-                parseContext.profileRequires(ppToken->loc, ~EEsProfile, 130, 0, "token pasting (##)");
-                subtoken = PpAtomPaste;
-            } else
-                ungetSubtoken();
-        }
-        break;
-    case PpAtomConstString:
-    case PpAtomIdentifier:
-    case PpAtomConstFloat:
-    case PpAtomConstDouble:
-    case PpAtomConstFloat16:
-    case PpAtomConstInt:
-    case PpAtomConstUint:
-    case PpAtomConstInt64:
-    case PpAtomConstUint64:
-#ifdef AMD_EXTENSIONS
-    case PpAtomConstInt16:
-    case PpAtomConstUint16:
-#endif
-        len = 0;
-        ch = getSubtoken();
         while (ch != 0 && ch != EndOfInput) {
             if (len < MaxTokenLength) {
                 ppToken->name[len] = (char)ch;
@@ -207,63 +221,28 @@ int TPpContext::TokenStream::getToken(TParseContextBase& parseContext, TPpToken
             }
         }
         ppToken->name[len] = 0;
+    }
 
-        switch (subtoken) {
+    // Check for ##, unless the current # is the last character
-        case PpAtomIdentifier:
+    if (atom == '#') {
-            break;
+        if (current < data.size()) {
-        case PpAtomConstString:
+            if (getSubtoken() == '#') {
-            break;
+                parseContext.requireProfile(ppToken->loc, ~EEsProfile, "token pasting (##)");
-        case PpAtomConstFloat:
+                parseContext.profileRequires(ppToken->loc, ~EEsProfile, 130, 0, "token pasting (##)");
-        case PpAtomConstDouble:
+                atom = PpAtomPaste;
-        case PpAtomConstFloat16:
-            ppToken->dval = atof(ppToken->name);
-            break;
-        case PpAtomConstInt:
-#ifdef AMD_EXTENSIONS
-        case PpAtomConstInt16:
-#endif
-            if (len > 0 && ppToken->name[0] == '0') {
-                if (len > 1 && (ppToken->name[1] == 'x' || ppToken->name[1] == 'X'))
-                    ppToken->ival = (int)strtol(ppToken->name, 0, 16);
-                else
-                    ppToken->ival = (int)strtol(ppToken->name, 0, 8);
             } else
-                ppToken->ival = atoi(ppToken->name);
+                ungetSubtoken();
-            break;
-        case PpAtomConstUint:
-#ifdef AMD_EXTENSIONS
-        case PpAtomConstUint16:
-#endif
-            if (len > 0 && ppToken->name[0] == '0') {
-                if (len > 1 && (ppToken->name[1] == 'x' || ppToken->name[1] == 'X'))
-                    ppToken->ival = (int)strtoul(ppToken->name, 0, 16);
-                else
-                    ppToken->ival = (int)strtoul(ppToken->name, 0, 8);
-            } else
-                ppToken->ival = (int)strtoul(ppToken->name, 0, 10);
-            break;
-        case PpAtomConstInt64:
-            if (len > 0 && ppToken->name[0] == '0') {
-                if (len > 1 && (ppToken->name[1] == 'x' || ppToken->name[1] == 'X'))
-                    ppToken->i64val = strtoll(ppToken->name, nullptr, 16);
-                else
-                    ppToken->i64val = strtoll(ppToken->name, nullptr, 8);
-            } else
-                ppToken->i64val = atoll(ppToken->name);
-            break;
-        case PpAtomConstUint64:
-            if (len > 0 && ppToken->name[0] == '0') {
-                if (len > 1 && (ppToken->name[1] == 'x' || ppToken->name[1] == 'X'))
-                    ppToken->i64val = (long long)strtoull(ppToken->name, nullptr, 16);
-                else
-                    ppToken->i64val = (long long)strtoull(ppToken->name, nullptr, 8);
-            } else
-                ppToken->i64val = (long long)strtoull(ppToken->name, 0, 10);
-            break;
         }
     }
 
-    return subtoken;
+    // get the numeric value
+    if (SaveValue(atom)) {
+        char* n = reinterpret_cast<char*>(&ppToken->i64val);
+        for (int i = 0; i < sizeof(ppToken->i64val); ++i)
+            *n++ = getSubtoken();
+    }
+
+    return atom;
 }
 
 // We are pasting if