PP: Clean up and rationalize floating-point-number scanner.

This encapsulates where the string could overflow, removing 40 lines of fragile code. It also improves handling of numbers that are too long. There are a couple of open issues that could related to this function being more rational (locale dependence, 1.#INF).
2017-01-12 12:00:59 -07:00
parent e86fc0946f
commit 913e3b686a
3 changed files with 73 additions and 112 deletions
--- a/Test/baseResults/tokenLength.vert.out
+++ b/Test/baseResults/tokenLength.vert.out
@@ -11,8 +11,6 @@ ERROR: 0:34: '' : octal literal too big
 ERROR: 0:35: '' : numeric literal too long 
 ERROR: 0:35: '' : numeric literal too big 
 ERROR: 0:36: '' : float literal too long 
-ERROR: 0:36: '' : float literal too long 
-ERROR: 0:36: '' : float literal too long 
 WARNING: 0:39: '#extension' : extension not supported: a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhooooooooooooooooooooooooooooooohhhhhhhhhhhhhhhhh01234
 ERROR: 0:40: '' : name too long 
 WARNING: 0:40: '#extension' : extension not supported: a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhooooooooooooooooooooooooooooooohhhhhhhhhhhhhhhhh01234
@@ -29,7 +27,7 @@ ERROR: 0:62: 'preprocessor evaluation' : undefined macro in expression not allow
 ERROR: 0:67: '' : numeric literal too long 
 ERROR: 0:70: '' : name too long 
 ERROR: 0:70: 'preprocessor evaluation' : undefined macro in expression not allowed in es profile A000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
-ERROR: 28 compilation errors.  No code generated.
+ERROR: 26 compilation errors.  No code generated.


 Shader version: 300
@@ -73,7 +71,7 @@ ERROR: node is still EOpNull!
 0:23    move second child to first child (temp highp float)
 0:23      'E3' (global highp float)
 0:23      Constant:
-0:23        12.000000
+0:23        1.012346
 0:25  Function Definition: main( (global void)
 0:25    Function Parameters: 
 0:27    Sequence
@@ -104,7 +102,7 @@ ERROR: node is still EOpNull!
 0:36    move second child to first child (temp highp float)
 0:36      'superF' (global highp float)
 0:36      Constant:
-0:36        inf
+0:36        1.012346
 0:?   Linker Objects
 0:?     'BCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789' (in highp float)
 0:?     'ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789' (in highp float)
@@ -169,7 +167,7 @@ ERROR: node is still EOpNull!
 0:23    move second child to first child (temp highp float)
 0:23      'E3' (global highp float)
 0:23      Constant:
-0:23        12.000000
+0:23        1.012346
 0:25  Function Definition: main( (global void)
 0:25    Function Parameters: 
 0:27    Sequence
@@ -200,7 +198,7 @@ ERROR: node is still EOpNull!
 0:36    move second child to first child (temp highp float)
 0:36      'superF' (global highp float)
 0:36      Constant:
-0:36        inf
+0:36        1.012346
 0:?   Linker Objects
 0:?     'BCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789' (in highp float)
 0:?     'ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789' (in highp float)
--- a/glslang/Include/revision.h
+++ b/glslang/Include/revision.h
@@ -2,5 +2,5 @@
 // For the version, it uses the latest git tag followed by the number of commits.
 // For the date, it uses the current date (when then script is run).

-#define GLSLANG_REVISION "Overload400-PrecQual.1760"
-#define GLSLANG_DATE "11-Jan-2017"
+#define GLSLANG_REVISION "Overload400-PrecQual.1764"
+#define GLSLANG_DATE "12-Jan-2017"
--- a/glslang/MachineIndependent/preprocessor/PpScanner.cpp
+++ b/glslang/MachineIndependent/preprocessor/PpScanner.cpp
@@ -100,36 +100,26 @@ namespace glslang {
 int TPpContext::lFloatConst(int len, int ch, TPpToken* ppToken)
 {
    bool HasDecimalOrExponent = false;
-    int declen;
-    int str_len;
    int isDouble = 0;
 #ifdef AMD_EXTENSIONS
    int isFloat16 = 0;
    bool enableFloat16 = parseContext.version >= 450 && parseContext.extensionTurnedOn(E_GL_AMD_gpu_shader_half_float);
 #endif

-    declen = 0;
+    const auto saveName = [&](int ch) {
+        if (len <= MaxTokenLength)
+            ppToken->name[len++] = static_cast<char>(ch);
+    };
+
+    // Decimal:

-    str_len=len;
-    char* str = ppToken->name;
    if (ch == '.') {
        HasDecimalOrExponent = true;
-        str[len++] = (char)ch;
+        saveName(ch);
        ch = getChar();
        while (ch >= '0' && ch <= '9') {
-            if (len < MaxTokenLength) {
-                declen++;
-                if (len > 0 || ch != '0') {
-                    str[len] = (char)ch;
-                    len++;
-                    str_len++;
-                }
-                ch = getChar();
-            } else {
-                parseContext.ppError(ppToken->loc, "float literal too long", "", "");
-                len = 1;
-                str_len = 1;
-            }
+            saveName(ch);
+            ch = getChar();
        }
    }

@@ -137,101 +127,74 @@ int TPpContext::lFloatConst(int len, int ch, TPpToken* ppToken)

    if (ch == 'e' || ch == 'E') {
        HasDecimalOrExponent = true;
-        if (len >= MaxTokenLength) {
-            parseContext.ppError(ppToken->loc, "float literal too long", "", "");
-            len = 1;
-            str_len = 1;
-        } else {
-            str[len++] = (char)ch;
+        saveName(ch);
+        ch = getChar();
+        if (ch == '+' || ch == '-') {
+            saveName(ch);
            ch = getChar();
-            if (ch == '+') {
-                str[len++] = (char)ch;
-                ch = getChar();
-            } else if (ch == '-') {
-                str[len++] = (char)ch;
+        }
+        if (ch >= '0' && ch <= '9') {
+            while (ch >= '0' && ch <= '9') {
+                saveName(ch);
                ch = getChar();
            }
-            if (ch >= '0' && ch <= '9') {
-                while (ch >= '0' && ch <= '9') {
-                    if (len < MaxTokenLength) {
-                        str[len++] = (char)ch;
-                        ch = getChar();
-                    } else {
-                        parseContext.ppError(ppToken->loc, "float literal too long", "", "");
-                        len = 1;
-                        str_len = 1;
-                    }
-                }
-            } else {
-                parseContext.ppError(ppToken->loc, "bad character in float exponent", "", "");
-            }
+        } else {
+            parseContext.ppError(ppToken->loc, "bad character in float exponent", "", "");
        }
    }

-    if (len == 0) {
-        ppToken->dval = 0.0;
-        strcpy(str, "0.0");
-    } else {
-        if (ch == 'l' || ch == 'L') {
-            parseContext.doubleCheck(ppToken->loc, "double floating-point suffix");
-            if (! HasDecimalOrExponent)
-                parseContext.ppError(ppToken->loc, "float literal needs a decimal point or exponent", "", "");
-            int ch2 = getChar();
-            if (ch2 != 'f' && ch2 != 'F') {
-                ungetChar();
-                ungetChar();
-            } else {
-                if (len < MaxTokenLength) {
-                    str[len++] = (char)ch;
-                    str[len++] = (char)ch2;
-                    isDouble = 1;
-                } else {
-                    parseContext.ppError(ppToken->loc, "float literal too long", "", "");
-                    len = 1,str_len=1;
-                }
-            }
-#ifdef AMD_EXTENSIONS
-        } else if (enableFloat16 && (ch == 'h' || ch == 'H')) {
-            parseContext.float16Check(ppToken->loc, "half floating-point suffix");
-            if (!HasDecimalOrExponent)
-                parseContext.ppError(ppToken->loc, "float literal needs a decimal point or exponent", "", "");
-            int ch2 = getChar();
-            if (ch2 != 'f' && ch2 != 'F') {
-                ungetChar();
-                ungetChar();
-            }
-            else {
-                if (len < MaxTokenLength) {
-                    str[len++] = (char)ch;
-                    str[len++] = (char)ch2;
-                    isFloat16 = 1;
-                }
-                else {
-                    parseContext.ppError(ppToken->loc, "float literal too long", "", "");
-                    len = 1, str_len = 1;
-                }
-            }
-#endif
-        } else if (ch == 'f' || ch == 'F') {
-            parseContext.profileRequires(ppToken->loc,  EEsProfile, 300, nullptr, "floating-point suffix");
-            if (! parseContext.relaxedErrors())
-                parseContext.profileRequires(ppToken->loc, ~EEsProfile, 120, nullptr, "floating-point suffix");
-            if (! HasDecimalOrExponent)
-                parseContext.ppError(ppToken->loc, "float literal needs a decimal point or exponent", "", "");
-            if (len < MaxTokenLength)
-                str[len++] = (char)ch;
-            else {
-                parseContext.ppError(ppToken->loc, "float literal too long", "", "");
-                len = 1,str_len=1;
-            }
-        } else
+    // Suffix:
+
+    if (ch == 'l' || ch == 'L') {
+        parseContext.doubleCheck(ppToken->loc, "double floating-point suffix");
+        if (! HasDecimalOrExponent)
+            parseContext.ppError(ppToken->loc, "float literal needs a decimal point or exponent", "", "");
+        int ch2 = getChar();
+        if (ch2 != 'f' && ch2 != 'F') {
            ungetChar();
+            ungetChar();
+        } else {
+            saveName(ch);
+            saveName(ch2);
+            isDouble = 1;
+        }
+#ifdef AMD_EXTENSIONS
+    } else if (enableFloat16 && (ch == 'h' || ch == 'H')) {
+        parseContext.float16Check(ppToken->loc, "half floating-point suffix");
+        if (!HasDecimalOrExponent)
+            parseContext.ppError(ppToken->loc, "float literal needs a decimal point or exponent", "", "");
+        int ch2 = getChar();
+        if (ch2 != 'f' && ch2 != 'F') {
+            ungetChar();
+            ungetChar();
+        } else {
+            saveName(ch);
+            saveName(ch2);
+            isFloat16 = 1;
+        }
+#endif
+    } else if (ch == 'f' || ch == 'F') {
+        parseContext.profileRequires(ppToken->loc,  EEsProfile, 300, nullptr, "floating-point suffix");
+        if (! parseContext.relaxedErrors())
+            parseContext.profileRequires(ppToken->loc, ~EEsProfile, 120, nullptr, "floating-point suffix");
+        if (! HasDecimalOrExponent)
+            parseContext.ppError(ppToken->loc, "float literal needs a decimal point or exponent", "", "");
+        saveName(ch);
+    } else
+        ungetChar();

-        str[len]='\0';
+    // Patch up the name, length, etc.

-        ppToken->dval = strtod(str, nullptr);
+    if (len > MaxTokenLength) {
+        len = MaxTokenLength;
+        parseContext.ppError(ppToken->loc, "float literal too long", "", "");
    }
+    ppToken->name[len] = '\0';

+    // Get the numerical value
+    ppToken->dval = strtod(ppToken->name, nullptr);
+
+    // Return the right token type
    if (isDouble)
        return PpAtomConstDouble;
 #ifdef AMD_EXTENSIONS