Bug 1215127

Created attachment 1018413 [details]
type.o.i (with some __attribute__ tries which I've added)

Created attachment 1018414 [details]
0001-spec-file-patch-for-GCC-reproducer.patch

Created attachment 1018415 [details]
relink.sh

With all the optimize attributes you have in there this seems to have started with http://gcc.gnu.org/r219823 , but wonder if that isn't exactly because of the optimize attributes with that revision.

So, after removing all the __attribute__((optimize ("O0"))) stuff, I've bisected this to http://gcc.gnu.org/r215697 , which is indeed a VRP change and looking e.g. at vrp1 diffs, I see changes e.g. in TypeInt::xdual or TypeInt::make (i.e. functions that inline normalize_int_widen or normalize_long_widen).

And, by adding noinline/noclone on both normalize_{int,long}_widen, and then optimize (0) on them selectively to find which one is the problem, I've came up with short reproducer:

__attribute__((noinline, noclone)) int
normalize_long_widen (long long lo, long long hi, int w)
{
  if (lo <= hi)
    {
      if ((unsigned long long) (hi - lo) <= 3ULL)
	w = 0;
      if ((unsigned long long) (hi - lo) >= -1ULL)
	w = 3;
    }
  else
    {
      if ((unsigned long long) (lo - hi) <= 3ULL)
	w = 0;
      if ((unsigned long long) (lo - hi) >= -1ULL)
	w = 0;
    }
  return w;
}

int
main ()
{
  __builtin_printf ("%d\n", normalize_long_widen (-__LONG_LONG_MAX__ - 1,
                                                  __LONG_LONG_MAX__, 16));
  return 0;
}

and that just seems that hotspot relies on undefined behavior, as LONG_LONG_MIN is smaller than LONG_LONG_MAX (thus lo <= hi is true), but hi - lo is undefined behavior.

Guess you want something like (so that the subtraction is performed in unsigned type with well defined overflow):
--- type.o.i~	2015-04-24 14:17:08.000000000 +0200
+++ type.o.i	2015-04-24 17:07:08.258819716 +0200
@@ -129471,11 +129471,11 @@ static int normalize_int_widen( jint lo,
 
 
   if (lo <= hi) {
-    if ((juint)(hi - lo) <= ((juint)3)) w = Type::WidenMin;
-    if ((juint)(hi - lo) >= max_juint) w = Type::WidenMax;
+    if ((juint)hi - lo <= ((juint)3)) w = Type::WidenMin;
+    if ((juint)hi - lo >= max_juint) w = Type::WidenMax;
   } else {
-    if ((juint)(lo - hi) <= ((juint)3)) w = Type::WidenMin;
-    if ((juint)(lo - hi) >= max_juint) w = Type::WidenMin;
+    if ((juint)lo - hi <= ((juint)3)) w = Type::WidenMin;
+    if ((juint)lo - hi >= max_juint) w = Type::WidenMin;
   }
   return w;
 }
@@ -129753,11 +129753,11 @@ static int normalize_long_widen( jlong l
 
 
   if (lo <= hi) {
-    if ((julong)(hi - lo) <= ((juint)3)) w = Type::WidenMin;
-    if ((julong)(hi - lo) >= max_julong) w = Type::WidenMax;
+    if ((julong)hi - lo <= ((juint)3)) w = Type::WidenMin;
+    if ((julong)hi - lo >= max_julong) w = Type::WidenMax;
   } else {
-    if ((julong)(lo - hi) <= ((juint)3)) w = Type::WidenMin;
-    if ((julong)(lo - hi) >= max_julong) w = Type::WidenMin;
+    if ((julong)lo - hi <= ((juint)3)) w = Type::WidenMin;
+    if ((julong)lo - hi >= max_julong) w = Type::WidenMin;
   }
   return w;
 }

(of course applied to the source, not preprocessed source) as a fix.
Or ((juint)hi - (juint)lo) instead of (juint)hi - lo, depends on the projects coding conventions...
You could have tried -fsanitize=undefined first before filing bugs... (next time).

(In reply to Jakub Jelinek from comment #5)
> You could have tried -fsanitize=undefined first before filing bugs... (next
> time).

Thank you very much for the analysis, Jakub. I'll certainly use -fsanitize=undefined next time before filing bugs. Thanks again!