PP: Implement locale-independent strtod, using istringstream and a fast path.

Fixes #1228. Fixes #234.

This uses imbue() to be locale independent.  Notes:

- 'sstream >> double' is much slower than strtod()
  * this was measurable in the test suite as a whole, despite being
    a tiny fraction of what the test suite does
- so, this embeds a fast path that bypasses sstream most of the time
  => the test suite is faster than before
- sstream is probably slower, because it does more accurate rounding than strtod()
- sstream does not create INFINITY by itself, this was done based on failure inferencing

glslang/MachineIndependent/preprocessor/PpContext.cpp

@@ -77,6 +77,7 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 \****************************************************************************/
 
 #include <cstdlib>
+#include <locale>
 
 #include "PpContext.h"
 
@@ -91,6 +92,8 @@ TPpContext::TPpContext(TParseContextBase& pc, const std::string& rootFileName, T
     for (elsetracker = 0; elsetracker < maxIfNesting; elsetracker++)
         elseSeen[elsetracker] = false;
     elsetracker = 0;
+
+    strtodStream.imbue(std::locale::classic());
 }
 
 TPpContext::~TPpContext()

glslang/MachineIndependent/preprocessor/PpContext.h

@@ -80,6 +80,7 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <stack>
 #include <unordered_map>
+#include <sstream>
 
 #include "../ParseHelper.h"
 
@@ -620,6 +621,8 @@ protected:
     std::string rootFileName;
     std::stack<TShader::Includer::IncludeResult*> includeStack;
     std::string currentSourceFile;
+
+    std::istringstream strtodStream;
 };
 
 } // end namespace glslang

glslang/MachineIndependent/preprocessor/PpScanner.cpp

@@ -102,20 +102,36 @@ namespace glslang {
 
 int TPpContext::lFloatConst(int len, int ch, TPpToken* ppToken)
 {
-    bool HasDecimalOrExponent = false;
-    int isDouble = 0;
-
     const auto saveName = [&](int ch) {
         if (len <= MaxTokenLength)
             ppToken->name[len++] = static_cast<char>(ch);
     };
 
-    // Decimal:
+    // find the range of non-zero digits before the decimal point
+    int startNonZero = 0;
+    while (startNonZero < len && ppToken->name[startNonZero] == '0')
+        ++startNonZero;
+    int endNonZero = len;
+    while (endNonZero > startNonZero && ppToken->name[endNonZero-1] == '0')
+        --endNonZero;
+    int numWholeNumberDigits = endNonZero - startNonZero;
 
+    // accumulate the range's value
+    bool fastPath = numWholeNumberDigits <= 15;  // when the number gets too complex, set to false
+    unsigned long long wholeNumber = 0;
+    if (fastPath) {
+        for (int i = startNonZero; i < endNonZero; ++i)
+            wholeNumber = wholeNumber * 10 + (ppToken->name[i] - '0');
+    }
+    int decimalShift = len - endNonZero;
+
+    // Decimal point:
+    bool hasDecimalOrExponent = false;
     if (ch == '.') {
-        HasDecimalOrExponent = true;
+        hasDecimalOrExponent = true;
         saveName(ch);
         ch = getChar();
+        int firstDecimal = len;
 
         // 1.#INF or -1.#INF
         if (ch == '#' && (ifdepth > 0 || parseContext.intermediate.getSource() == EShSourceHlsl)) {
@@ -145,38 +161,97 @@ int TPpContext::lFloatConst(int len, int ch, TPpToken* ppToken)
             }
         }
 
-        while (ch >= '0' && ch <= '9') {
+        // Consume leading-zero digits after the decimal point
+        while (ch == '0') {
             saveName(ch);
             ch = getChar();
         }
+        int startNonZeroDecimal = len;
+        int endNonZeroDecimal = len;
+
+        // Consume remaining digits, up to the exponent
+        while (ch >= '0' && ch <= '9') {
+            saveName(ch);
+            if (ch != '0')
+                endNonZeroDecimal = len;
+            ch = getChar();
+        }
+
+        // Compute accumulation up to the last non-zero digit
+        if (endNonZeroDecimal > startNonZeroDecimal) {
+            numWholeNumberDigits += endNonZeroDecimal - endNonZero - 1; // don't include the "."
+            if (numWholeNumberDigits > 15)
+                fastPath = false;
+            if (fastPath) {
+                for (int i = endNonZero; i < endNonZeroDecimal; ++i) {
+                    if (ppToken->name[i] != '.')
+                        wholeNumber = wholeNumber * 10 + (ppToken->name[i] - '0');
+                }
+            }
+            decimalShift = firstDecimal - endNonZeroDecimal;
+        }
     }
 
     // Exponent:
-
+    bool negativeExponent = false;
-    if (ch == 'e' || ch == 'E') {
+    double exponentValue = 0.0;
-        HasDecimalOrExponent = true;
+    int exponent = 0;
-        saveName(ch);
+    {
-        ch = getChar();
+        if (ch == 'e' || ch == 'E') {
-        if (ch == '+' || ch == '-') {
+            hasDecimalOrExponent = true;
             saveName(ch);
             ch = getChar();
-        }
+            if (ch == '+' || ch == '-') {
-        if (ch >= '0' && ch <= '9') {
+                negativeExponent = ch == '-';
-            while (ch >= '0' && ch <= '9') {
                 saveName(ch);
                 ch = getChar();
             }
-        } else {
+            if (ch >= '0' && ch <= '9') {
-            parseContext.ppError(ppToken->loc, "bad character in float exponent", "", "");
+                while (ch >= '0' && ch <= '9') {
+                    exponent = exponent * 10 + (ch - '0');
+                    saveName(ch);
+                    ch = getChar();
+                }
+            } else {
+                parseContext.ppError(ppToken->loc, "bad character in float exponent", "", "");
+            }
+        }
+
+        // Compensate for location of decimal
+        if (negativeExponent)
+            exponent -= decimalShift;
+        else {
+            exponent += decimalShift;
+            if (exponent < 0) {
+                negativeExponent = true;
+                exponent = -exponent;
+            }
+        }
+        if (exponent > 22)
+            fastPath = false;
+
+        if (fastPath) {
+            // Compute the floating-point value of the exponent
+            exponentValue = 1.0;
+            if (exponent > 0) {
+                double expFactor = 10;
+                while (exponent > 0) {
+                    if (exponent & 0x1)
+                        exponentValue *= expFactor;
+                    expFactor *= expFactor;
+                    exponent >>= 1;
+                }
+            }
         }
     }
 
     // Suffix:
+    bool isDouble = false;
     bool isFloat16 = false;
     if (ch == 'l' || ch == 'L') {
         if (ifdepth == 0 && parseContext.intermediate.getSource() == EShSourceGlsl)
             parseContext.doubleCheck(ppToken->loc, "double floating-point suffix");
-        if (ifdepth == 0 && !HasDecimalOrExponent)
+        if (ifdepth == 0 && !hasDecimalOrExponent)
             parseContext.ppError(ppToken->loc, "float literal needs a decimal point or exponent", "", "");
         if (parseContext.intermediate.getSource() == EShSourceGlsl) {
             int ch2 = getChar();
@@ -186,16 +261,16 @@ int TPpContext::lFloatConst(int len, int ch, TPpToken* ppToken)
             } else {
                 saveName(ch);
                 saveName(ch2);
-                isDouble = 1;
+                isDouble = true;
             }
         } else if (parseContext.intermediate.getSource() == EShSourceHlsl) {
             saveName(ch);
-            isDouble = 1;
+            isDouble = true;
         }
     } else if (ch == 'h' || ch == 'H') {
         if (ifdepth == 0 && parseContext.intermediate.getSource() == EShSourceGlsl)
             parseContext.float16Check(ppToken->loc, "half floating-point suffix");
-        if (ifdepth == 0 && !HasDecimalOrExponent)
+        if (ifdepth == 0 && !hasDecimalOrExponent)
             parseContext.ppError(ppToken->loc, "float literal needs a decimal point or exponent", "", "");
         if (parseContext.intermediate.getSource() == EShSourceGlsl) {
             int ch2 = getChar();
@@ -216,13 +291,13 @@ int TPpContext::lFloatConst(int len, int ch, TPpToken* ppToken)
             parseContext.profileRequires(ppToken->loc,  EEsProfile, 300, nullptr, "floating-point suffix");
         if (ifdepth == 0 && !parseContext.relaxedErrors())
             parseContext.profileRequires(ppToken->loc, ~EEsProfile, 120, nullptr, "floating-point suffix");
-        if (ifdepth == 0 && !HasDecimalOrExponent)
+        if (ifdepth == 0 && !hasDecimalOrExponent)
             parseContext.ppError(ppToken->loc, "float literal needs a decimal point or exponent", "", "");
         saveName(ch);
     } else
         ungetChar();
 
-    // Patch up the name, length, etc.
+    // Patch up the name and length for overflow
 
     if (len > MaxTokenLength) {
         len = MaxTokenLength;
@@ -230,8 +305,29 @@ int TPpContext::lFloatConst(int len, int ch, TPpToken* ppToken)
     }
     ppToken->name[len] = '\0';
 
-    // Get the numerical value
+    // Compute the numerical value
-    ppToken->dval = strtod(ppToken->name, nullptr);
+    if (fastPath) {
+        // compute the floating-point value of the exponent
+        if (exponentValue == 0.0)
+            ppToken->dval = (double)wholeNumber;
+        else if (negativeExponent)
+            ppToken->dval = (double)wholeNumber / exponentValue;
+        else
+            ppToken->dval = (double)wholeNumber * exponentValue;
+    } else {
+        // slow path
+        strtodStream.clear();
+        strtodStream.str(ppToken->name);
+        strtodStream >> ppToken->dval;
+        // Assume failure combined with a large exponent was overflow, in
+        // an attempt to set INF.  Otherwise, assume underflow, and set 0.0.
+        if (strtodStream.fail()) {
+            if (!negativeExponent && exponent + numWholeNumberDigits > 300)
+                ppToken->i64val = 0x7ff0000000000000; // +Infinity
+            else
+                ppToken->dval = 0.0;
+        }
+    }
 
     // Return the right token type
     if (isDouble)