Merge pull request #1129 from xorgy/const-fold-int-min-modulo-negative-one

GLSL: Fold constant SHRT_MIN/INT_MIN/LLONG_MIN % -1 to 0.

Test/baseResults/constFoldIntMin.frag.out

@@ -0,0 +1,51 @@
+constFoldIntMin.frag
+Shader version: 460
+Requested GL_AMD_gpu_shader_int16
+Requested GL_ARB_gpu_shader_int64
+0:? Sequence
+0:5  Function Definition: a( ( global void)
+0:5    Function Parameters: 
+0:6    Sequence
+0:6      Sequence
+0:6        move second child to first child ( temp int16_t)
+0:6          'u' ( temp int16_t)
+0:6          Constant:
+0:6            32768 (const int)
+0:7      Sequence
+0:7        move second child to first child ( temp int)
+0:7          'v' ( temp int)
+0:7          Constant:
+0:7            -2147483648 (const int)
+0:8      Sequence
+0:8        move second child to first child ( temp int64_t)
+0:8          'w' ( temp int64_t)
+0:8          Constant:
+0:8            -9223372036854775808 (const int64_t)
+0:9      Sequence
+0:9        move second child to first child ( temp int16_t)
+0:9          'x' ( temp int16_t)
+0:9          Constant:
+0:9            0 (const int)
+0:10      Sequence
+0:10        move second child to first child ( temp int)
+0:10          'y' ( temp int)
+0:10          Constant:
+0:10            0 (const int)
+0:11      Sequence
+0:11        move second child to first child ( temp int64_t)
+0:11          'z' ( temp int64_t)
+0:11          Constant:
+0:11            0 (const int64_t)
+0:?   Linker Objects
+
+
+Linked fragment stage:
+
+ERROR: Linking fragment stage: Missing entry point: Each stage requires one entry point
+
+Shader version: 460
+Requested GL_AMD_gpu_shader_int16
+Requested GL_ARB_gpu_shader_int64
+0:? Sequence
+0:?   Linker Objects
+

Test/constFoldIntMin.frag

@@ -0,0 +1,12 @@
+#version 460 core
+#extension GL_AMD_gpu_shader_int16 : enable
+#extension GL_ARB_gpu_shader_int64 : enable
+
+void a(){
+    int16_t u = -32768S / -1S; // SHRT_MIN
+    int v = -2147483648 / -1; // INT_MIN
+    int64_t w = -9223372036854775808L / -1L; // LLONG_MIN
+    int16_t x = -32768S % -1S; // SHRT_MIN
+    int y = -2147483648 % -1; // INT_MIN
+    int64_t z = -9223372036854775808L % -1L; // LLONG_MIN
+}

glslang/MachineIndependent/Constant.cpp

@@ -38,6 +38,7 @@
 #include <cmath>
 #include <cfloat>
 #include <cstdlib>
+#include <climits>
 
 namespace {
 
@@ -264,7 +265,29 @@ TIntermTyped* TIntermConstantUnion::fold(TOperator op, const TIntermTyped* right
             if (rightUnionArray[i] == 0)
                 newConstArray[i] = leftUnionArray[i];
             else
-                newConstArray[i] = leftUnionArray[i] % rightUnionArray[i];
+                switch (getType().getBasicType()) {
+                case EbtInt:
+                    if (rightUnionArray[i].getIConst() == -1 && leftUnionArray[i].getIConst() == INT_MIN) {
+                        newConstArray[i].setIConst(0);
+                        break;
+                    } else goto modulo_default;
+
+                case EbtInt64:
+                    if (rightUnionArray[i].getI64Const() == -1 && leftUnionArray[i].getI64Const() == LLONG_MIN) {
+                        newConstArray[i].setI64Const(0);
+                        break;
+                    } else goto modulo_default;
+#ifdef AMD_EXTENSIONS
+                case EbtInt16:
+                    if (rightUnionArray[i].getIConst() == -1 && leftUnionArray[i].getIConst() == SHRT_MIN) {
+                        newConstArray[i].setIConst(0);
+                        break;
+                    } else goto modulo_default;
+#endif
+                default:
+                modulo_default:
+                    newConstArray[i] = leftUnionArray[i] % rightUnionArray[i];
+                }
         }
         break;
 

gtests/AST.FromFile.cpp

@@ -115,6 +115,7 @@ INSTANTIATE_TEST_CASE_P(
         "330comp.frag",
         "constErrors.frag",
         "constFold.frag",
+        "constFoldIntMin.frag",
         "errors.frag",
         "forwardRef.frag",
         "uint.frag",