Towards Beautiful Test Cases for Compiler Bugs John Regehr University of Utah

template __attribute__((always_inline)) typename boost::dispatch::meta::call ::type shift_right ( A0 const& a0, A1 const& a1 ) { typename boost::dispatch::make_functor ::type callee; return callee( a0, a1);; } template __attribute__((always_inline)) typename boost::dispatch::meta::call ::type shr ( A0 const& a0, A1 const& a1 ) { typename boost::dispatch::make_functor ::type callee; return callee( a0, a1);; } } # 5 "/home/gaunard/build/may_alias/include/boost/simd/toolbox/operator/include/functions/shift_right.hpp" 2 # 1 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/scalar/shift_right.hpp" 1 # 14 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/scalar/shift_right.hpp" namespace boost { namespace simd { namespace ext { } } } namespace boost { namespace dispatch { namespace meta { template __attribute__((always_inline)) boost :: simd :: ext :: implement >, scalar_ >), tag::cpu_ > dispatching( boost::simd::tag::shift_right_, tag::cpu_, scalar_ > const, scalar_ > const, adl_helper = adl_helper() ) { boost :: simd :: ext :: implement >, scalar_ > ), tag::cpu_ > that; return that; } } } } namespace boost { namespace simd { namespace ext { template struct implement >, scalar_ > ), tag::cpu_, Dummy > { typedef A0 result_type; inline result_type operator()( A0 const& a0, A1 const& a1 ) const { typedef typename dispatch::meta::as_integer ::type itype; return bitwise_cast (shift_right(bitwise_cast (a0),a1)); } }; } } } namespace boost { namespace simd { namespace ext{ } } } namespace boost { namespace dispatch { namespace meta { template __attribute__((always_inline)) boost :: simd :: ext :: implement >, scalar_ >), tag::cpu_ > dispatching( boost::simd::tag::shift_right_, tag::cpu_, scalar_ > const, scalar_ > const, adl_helper = adl_helper() ) { boost :: simd :: ext :: implement >, scalar_ > ), tag::cpu_ > that; return that; } } } } namespace boost { namespace simd { namespace ext { template struct implement >, scalar_ > ), tag::cpu_, Dummy > { typedef A0 result_type; inline result_type operator()( A0 const& a0, A1 const& a1 ) const { return a0 >> a1; } }; } } } # 6 "/home/gaunard/build/may_alias/include/boost/simd/toolbox/operator/include/functions/shift_right.hpp" 2 # 1 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/simd/common/shift_ri ght.hpp" 1 # 20 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/simd/common/shift_ri ght.hpp" namespace boost { namespace simd { namespace ext { 2

template __attribute__((always_inline)) typename boost::dispatch::meta::call ::type shift_right ( A0 const& a0, A1 const& a1 ) { typename boost::dispatch::make_functor ::type callee; return callee( a0, a1);; } template __attribute__((always_inline)) typename boost::dispatch::meta::call ::type shr ( A0 const& a0, A1 const& a1 ) { typename boost::dispatch::make_functor ::type callee; return callee( a0, a1);; } } # 5 "/home/gaunard/build/may_alias/include/boost/simd/toolbox/operator/include/functions/shift_right.hpp" 2 # 1 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/scalar/shift_right.hpp" 1 # 14 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/scalar/shift_right.hpp" namespace boost { namespace simd { namespace ext { } } } namespace boost { namespace dispatch { namespace meta { template __attribute__((always_inline)) boost :: simd :: ext :: implement >, scalar_ >), tag::cpu_ > dispatching( boost::simd::tag::shift_right_, tag::cpu_, scalar_ > const, scalar_ > const, adl_helper = adl_helper() ) { boost :: simd :: ext :: implement >, scalar_ > ), tag::cpu_ > that; return that; } } } } namespace boost { namespace simd { namespace ext { template struct implement >, scalar_ > ), tag::cpu_, Dummy > { typedef A0 result_type; inline result_type operator()( A0 const& a0, A1 const& a1 ) const { typedef typename dispatch::meta::as_integer ::type itype; return bitwise_cast (shift_right(bitwise_cast (a0),a1)); } }; } } } namespace boost { namespace simd { namespace ext{ } } } namespace boost { namespace dispatch { namespace meta { template __attribute__((always_inline)) boost :: simd :: ext :: implement >, scalar_ >), tag::cpu_ > dispatching( boost::simd::tag::shift_right_, tag::cpu_, scalar_ > const, scalar_ > const, adl_helper = adl_helper() ) { boost :: simd :: ext :: implement >, scalar_ > ), tag::cpu_ > that; return that; } } } } namespace boost { namespace simd { namespace ext { template struct implement >, scalar_ > ), tag::cpu_, Dummy > { typedef A0 result_type; inline result_type operator()( A0 const& a0, A1 const& a1 ) const { return a0 >> a1; } }; } } } # 6 "/home/gaunard/build/may_alias/include/boost/simd/toolbox/operator/include/functions/shift_right.hpp" 2 # 1 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/simd/common/shift_ri ght.hpp" 1 # 20 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/simd/common/shift_ri ght.hpp" namespace boost { namespace simd { namespace ext { 3 From GCC PR 50800: “Testcase is [here] (couldn't attach it due to bugzilla size restrictions)”

template __attribute__((always_inline)) typename boost::dispatch::meta::call ::type shift_right ( A0 const& a0, A1 const& a1 ) { typename boost::dispatch::make_functor ::type callee; return callee( a0, a1);; } template __attribute__((always_inline)) typename boost::dispatch::meta::call ::type shr ( A0 const& a0, A1 const& a1 ) { typename boost::dispatch::make_functor ::type callee; return callee( a0, a1);; } } # 5 "/home/gaunard/build/may_alias/include/boost/simd/toolbox/operator/include/functions/shift_right.hpp" 2 # 1 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/scalar/shift_right.hpp" 1 # 14 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/scalar/shift_right.hpp" namespace boost { namespace simd { namespace ext { } } } namespace boost { namespace dispatch { namespace meta { template __attribute__((always_inline)) boost :: simd :: ext :: implement >, scalar_ >), tag::cpu_ > dispatching( boost::simd::tag::shift_right_, tag::cpu_, scalar_ > const, scalar_ > const, adl_helper = adl_helper() ) { boost :: simd :: ext :: implement >, scalar_ > ), tag::cpu_ > that; return that; } } } } namespace boost { namespace simd { namespace ext { template struct implement >, scalar_ > ), tag::cpu_, Dummy > { typedef A0 result_type; inline result_type operator()( A0 const& a0, A1 const& a1 ) const { typedef typename dispatch::meta::as_integer ::type itype; return bitwise_cast (shift_right(bitwise_cast (a0),a1)); } }; } } } namespace boost { namespace simd { namespace ext{ } } } namespace boost { namespace dispatch { namespace meta { template __attribute__((always_inline)) boost :: simd :: ext :: implement >, scalar_ >), tag::cpu_ > dispatching( boost::simd::tag::shift_right_, tag::cpu_, scalar_ > const, scalar_ > const, adl_helper = adl_helper() ) { boost :: simd :: ext :: implement >, scalar_ > ), tag::cpu_ > that; return that; } } } } namespace boost { namespace simd { namespace ext { template struct implement >, scalar_ > ), tag::cpu_, Dummy > { typedef A0 result_type; inline result_type operator()( A0 const& a0, A1 const& a1 ) const { return a0 >> a1; } }; } } } # 6 "/home/gaunard/build/may_alias/include/boost/simd/toolbox/operator/include/functions/shift_right.hpp" 2 # 1 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/simd/common/shift_ri ght.hpp" 1 # 20 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/simd/common/shift_ri ght.hpp" namespace boost { namespace simd { namespace ext { 4 From GCC PR 50800: “Testcase is [here] (couldn't attach it due to bugzilla size restrictions)” Next comment: “That you couldn't attach it should tell you something…”

template __attribute__((always_inline)) typename boost::dispatch::meta::call ::type shift_right ( A0 const& a0, A1 const& a1 ) { typename boost::dispatch::make_functor ::type callee; return callee( a0, a1);; } template __attribute__((always_inline)) typename boost::dispatch::meta::call ::type shr ( A0 const& a0, A1 const& a1 ) { typename boost::dispatch::make_functor ::type callee; return callee( a0, a1);; } } # 5 "/home/gaunard/build/may_alias/include/boost/simd/toolbox/operator/include/functions/shift_right.hpp" 2 # 1 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/scalar/shift_right.hpp" 1 # 14 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/scalar/shift_right.hpp" namespace boost { namespace simd { namespace ext { } } } namespace boost { namespace dispatch { namespace meta { template __attribute__((always_inline)) boost :: simd :: ext :: implement >, scalar_ >), tag::cpu_ > dispatching( boost::simd::tag::shift_right_, tag::cpu_, scalar_ > const, scalar_ > const, adl_helper = adl_helper() ) { boost :: simd :: ext :: implement >, scalar_ > ), tag::cpu_ > that; return that; } } } } namespace boost { namespace simd { namespace ext { template struct implement >, scalar_ > ), tag::cpu_, Dummy > { typedef A0 result_type; inline result_type operator()( A0 const& a0, A1 const& a1 ) const { typedef typename dispatch::meta::as_integer ::type itype; return bitwise_cast (shift_right(bitwise_cast (a0),a1)); } }; } } } namespace boost { namespace simd { namespace ext{ } } } namespace boost { namespace dispatch { namespace meta { template __attribute__((always_inline)) boost :: simd :: ext :: implement >, scalar_ >), tag::cpu_ > dispatching( boost::simd::tag::shift_right_, tag::cpu_, scalar_ > const, scalar_ > const, adl_helper = adl_helper() ) { boost :: simd :: ext :: implement >, scalar_ > ), tag::cpu_ > that; return that; } } } } namespace boost { namespace simd { namespace ext { template struct implement >, scalar_ > ), tag::cpu_, Dummy > { typedef A0 result_type; inline result_type operator()( A0 const& a0, A1 const& a1 ) const { return a0 >> a1; } }; } } } # 6 "/home/gaunard/build/may_alias/include/boost/simd/toolbox/operator/include/functions/shift_right.hpp" 2 # 1 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/simd/common/shift_ri ght.hpp" 1 # 20 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/simd/common/shift_ri ght.hpp" namespace boost { namespace simd { namespace ext { 5 From GCC PR 50800: “Testcase is [here] (couldn't attach it due to bugzilla size restrictions)” Next comment: “That you couldn't attach it should tell you something…” Next comment: 203 KB reduced test case attached

(safe_47 (((g_1161 ^ 0x2893L), (((safe_9 ((!l_1747), ((l_1749 = func_37 (func_58 ((l_1748, UL), g_110, l_1168, p_7, l_1747), g_381, p_9)), g_894))) != l_1747) <= p_10)), p_8))), g_11) 4UL), 0x93L)) ^ g_1161))) <= l_1750), p_8), g_458); g_1122 = (l_1749 && func_37 (l_1750, (g_1161 = ((safe_17 (((safe_60 (l_1755, 4)) | (g_703, (safe_55 ((p_10, 3UL), p_7)))), ((safe_51 ((safe_42 (0UL, ((l_1747 != 0x3AL) < g_28))))) | 0xF862D3BEL))) != 0xC7L)), g_1487)); g_146 = 0xBC806FC8L; l_1788 = func_24 (((safe_13 ((~ (((g_458 != (safe_54 (((~ (l_1749 = (safe_43 ((safe_14 ((((safe_62 ((safe_4 ((safe_63 (((l_1750 = 1L), (safe_59 (l_1168, 1))), (((p_10 = func_46 (func_46 (l_1749, g_894, (safe_23 (l_1749, (l_1167 = g_1161))), ((l_1469 = 6

(safe_47 (((g_1161 ^ 0x2893L), (((safe_9 ((!l_1747), ((l_1749 = func_37 (func_58 ((l_1748, UL), g_110, l_1168, p_7, l_1747), g_381, p_9)), g_894))) != l_1747) <= p_10)), p_8))), g_11) 4UL), 0x93L)) ^ g_1161))) <= l_1750), p_8), g_458); g_1122 = (l_1749 && func_37 (l_1750, (g_1161 = ((safe_17 (((safe_60 (l_1755, 4)) | (g_703, (safe_55 ((p_10, 3UL), p_7)))), ((safe_51 ((safe_42 (0UL, ((l_1747 != 0x3AL) < g_28))))) | 0xF862D3BEL))) != 0xC7L)), g_1487)); g_146 = 0xBC806FC8L; l_1788 = func_24 (((safe_13 ((~ (((g_458 != (safe_54 (((~ (l_1749 = (safe_43 ((safe_14 ((((safe_62 ((safe_4 ((safe_63 (((l_1750 = 1L), (safe_59 (l_1168, 1))), (((p_10 = func_46 (func_46 (l_1749, g_894, (safe_23 (l_1749, (l_1167 = g_1161))), ((l_1469 = 7 Exposes an LLVM miscompilation bug Part of an 89 KB program randomly generated by Csmith

(safe_47 (((g_1161 ^ 0x2893L), (((safe_9 ((!l_1747), ((l_1749 = func_37 (func_58 ((l_1748, UL), g_110, l_1168, p_7, l_1747), g_381, p_9)), g_894))) != l_1747) <= p_10)), p_8))), g_11) 4UL), 0x93L)) ^ g_1161))) <= l_1750), p_8), g_458); g_1122 = (l_1749 && func_37 (l_1750, (g_1161 = ((safe_17 (((safe_60 (l_1755, 4)) | (g_703, (safe_55 ((p_10, 3UL), p_7)))), ((safe_51 ((safe_42 (0UL, ((l_1747 != 0x3AL) < g_28))))) | 0xF862D3BEL))) != 0xC7L)), g_1487)); g_146 = 0xBC806FC8L; l_1788 = func_24 (((safe_13 ((~ (((g_458 != (safe_54 (((~ (l_1749 = (safe_43 ((safe_14 ((((safe_62 ((safe_4 ((safe_63 (((l_1750 = 1L), (safe_59 (l_1168, 1))), (((p_10 = func_46 (func_46 (l_1749, g_894, (safe_23 (l_1749, (l_1167 = g_1161))), ((l_1469 = 8 Part of an 89 KB program randomly generated by Csmith Exposes an LLVM miscompilation bug Csmith is our C compiler fuzzer ~425 compiler bugs discovered and reported so far

short a; int b, d, k, l, m; int fn1 (short p1, int p2) { return p1 - p2; } int fn2 (short p1) { return p1 == 0 || a == 0 ? a : a % p1; } int fn3 (int p1, unsigned short p2) { return p2 == 0 ? 0 : p1 % p2; } void fn4 (unsigned char p1) { if (p1) b = 1; } void fn5 (p1) { short n = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; } 9 $ clang -O1 foo.c ;./a.out 0 $ clang -O2 foo.c ;./a.out 1

short a; int b, d, k, l, m; int fn1 (short p1, int p2) { return p1 - p2; } int fn2 (short p1) { return p1 == 0 || a == 0 ? a : a % p1; } int fn3 (int p1, unsigned short p2) { return p2 == 0 ? 0 : p1 % p2; } void fn4 (unsigned char p1) { if (p1) b = 1; } void fn5 (p1) { short n = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; } 10 $ clang -O1 foo.c ;./a.out 0 $ clang -O2 foo.c ;./a.out 1 Reported as LLVM PR on 3/5/2012, fixed same day

int printf (const char *, …); char f[] = { -9L }; int main (void) { printf ("%d\n", 255 | f[0]); } 11 Intel CC for x86-64 is wrong at “-Ofast -ipo”

int printf (const char *,...); const union { short f1; int f2 : 13; } a = { }; int main (void) { printf ("%d\n", a.f1); printf ("%d\n", a.f2); return 0; } 12 GCC from Ubuntu LTS for x86-64 is wrong at -O1

int printf (const char *,...); int b; static struct S0 { int f2 : 1; } c; int main (void) { b = -1 ^ c.f2; printf ("%d\n", b); } 13 GCC from mid-Feb 2012 for x86-64 is wrong at -O0 (PR 52209)

Beautiful test case == 1.Small (but not obfuscated) 2.Well-defined 3.Unambiguously elicits bad behavior 4.Makes the compiler bug obvious Not all bugs have beautiful test cases 14

short a; int b, d, k, l, m; int fn1 (short p1, int p2) { return p1 - p2; } int fn2 (short p1) { return p1 == 0 || a == 0 ? a : a % p1; } int fn3 (int p1, unsigned short p2) { return p2 == 0 ? 0 : p1 % p2; } void fn4 (unsigned char p1) { if (p1) b = 1; } void fn5 (p1) { short n = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; } 15 Delta Debugging algorithm “ddmin” == Greedy search Delete chunks Test results for “interestingness” Keep reducing granularity

short a; int b, d, k, l, m; int fn1 (short p1, int p2) { return p1 - p2; } int fn2 (short p1) { return p1 == 0 || a == 0 ? a : a % p1; } int fn3 (int p1, unsigned short p2) { return p2 == 0 ? 0 : p1 % p2; } void fn4 (unsigned char p1) { if (p1) b = 1; } void fn5 (p1) { short n = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; } 16 ddmin == Greedy search Delete chunks Test for “interestingness” Keep reducing granularity Is this still interesting?

#!/bin/bash clang -O1 test.c&&\./a.out > out1.txt&&\ clang -O2 test.c&&\./a.out > out2.txt&&\ ! diff out1.txt out2.txt 17

short a; int b, d, k, l, m; int fn1 (short p1, int p2) { return p1 - p2; } int fn2 (short p1) { return p1 == 0 || a == 0 ? a : a % p1; } int fn3 (int p1, unsigned short p2) { return p2 == 0 ? 0 : p1 % p2; } void fn4 (unsigned char p1) { if (p1) b = 1; } void fn5 (p1) { short n = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; } 18 ddmin == Greedy search Delete chunks Test for “interestingness” Keep reducing granularity Assume not interesting, so we backtrack

short a; int b, d, k, l, m; int fn1 (short p1, int p2) { return p1 - p2; } int fn2 (short p1) { return p1 == 0 || a == 0 ? a : a % p1; } int fn3 (int p1, unsigned short p2) { return p2 == 0 ? 0 : p1 % p2; } void fn4 (unsigned char p1) { if (p1) b = 1; } void fn5 (p1) { short n = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; } 19 ddmin == Greedy search Delete chunks Test for “interestingness” Keep reducing granularity

short a; int b, d, k, l, m; int fn1 (short p1, int p2) { return p1 - p2; } int fn2 (short p1) { return p1 == 0 || a == 0 ? a : a % p1; } int fn3 (int p1, unsigned short p2) { return p2 == 0 ? 0 : p1 % p2; } void fn4 (unsigned char p1) { if (p1) b = 1; } void fn5 (p1) { short n = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; } 20 ddmin == Greedy search Delete chunks Test for “interestingness” Keep reducing granularity

short a; int b, d, k, l, m; int fn1 (short p1, int p2) { return p1 - p2; } int fn2 (short p1) { return p1 == 0 || a == 0 ? a : a % p1; } int fn3 (int p1, unsigned short p2) { return p2 == 0 ? 0 : p1 % p2; } void fn4 (unsigned char p1) { if (p1) b = 1; } void fn5 (p1) { short n = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; } 21 ddmin == Greedy search Delete chunks Test for “interestingness” Keep reducing granularity

short a; int b, d, k, l, m; int fn1 (short p1, int p2) { return p1 - p2; } int fn2 (short p1) { return p1 == 0 || a == 0 ? a : a % p1; } int fn3 (int p1, unsigned short p2) { return p2 == 0 ? 0 : p1 % p2; } void fn4 (unsigned char p1) { if (p1) b = 1; } void fn5 (p1) { short n = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; } 22 ddmin == Greedy search Delete chunks Test for “interestingness” Keep reducing granularity

short a; int b, d, k, l, m; int fn1 (short p1, int p2) { return p1 - p2; } int fn2 (short p1) { return p1 == 0 || a == 0 ? a : a % p1; } int fn3 (int p1, unsigned short p2) { return p2 == 0 ? 0 : p1 % p2; } void fn4 (unsigned char p1) { if (p1) b = 1; } void fn5 (p1) { short n = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; } 23 ddmin == Greedy search Delete chunks Test for “interestingness” Keep reducing granularity

short a; int b, d, k, l, m; int fn1 (short p1, int p2) { return p1 - p2; } int fn2 (short p1) { return p1 == 0 || a == 0 ? a : a % p1; } int fn3 (int p1, unsigned short p2) { return p2 == 0 ? 0 : p1 % p2; } void fn4 (unsigned char p1) { if (p1) b = 1; } void fn5 (p1) { short n = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; } 24 ddmin == Greedy search Delete chunks Test for “interestingness” Keep reducing granularity

Problem: ddmin gets stuck – Output is often not beautiful Our premise: – The Delta Debugging concept is sound – But ddmin is too hard-coded, inflexible 25

Generalized Delta Debugging 1.One or more transformations – E.g. “remove chunks of text” 2.Interestingness test 3.Fitness function – E.g. “prefer smaller test cases” 4.Search algorithm – E.g. “greedy” 26

Generalized Delta Debugging 1.One or more transformations – E.g. “remove chunks of text” 2.Interestingness test 3.Fitness function – E.g. “prefer smaller test cases” 4.Search algorithm – E.g. “greedy” 27

C-Reduce: A Generalized Delta Debugger Fixpoint computation Parameterized by – Interestingness test – Plugins that perform transformations Plugins are iterators implementing: reset() advance() transform(file) 28

void fn5 (p1) { short n = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; } 29

void fn5 (p1) { short 0 = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; } 30

void fn5 (p1) { short n = 0; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; } 31

void fn5 (p1) { short n = 60018; 0 = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; } 32

void fn5 (p1) { short n = 60018; k = 0 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; } 33

void fn5 (p1) { short n = 60018; k = fn2 (0); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; } 34

Example transformations: Remove chunks of text (like ddmin) C-specific peephole transformations – 0xfeedbeefULL  1 – x ^= y  x = y – (x + 1)  x + 1 – while (…)  if (…) – x ? y : z  y C-Reduce has 49 plugins so far 35

Some transformations make coordinated changes across different parts of the program – Inline a function call – Scalar replacement of aggregates – Un-nest nested function calls – Remove dead arguments – Make function return void – Reduce array dimension or pointer level – Shorten identifier name These use Clang as a rewriter 36

37 transformation current test case interesting? yes no

Beautiful test case == 1.Small (but not obfuscated) 2.Well-defined 3.Unambiguously elicits bad behavior 4.Makes the compiler bug obvious 38

#include using namespace std; int r[3], x[3], y[3]; int main() { int xa=2,ya=5,xb=4,yb=2,n=3; x[0] = 3; x[1] = 5; x[2] = 1; y[0] = 1; y[1] = 3; y[2] = 3; r[0] = 2; r[1] = 1; r[2] = 2; int tcount = 0; for (int k=min(xa,xb); k<=max(xa,xb); k++) { bool found1,found2 = false; for (int j=0; j<n; j++) { if (((k-x[j])*(k-x[j])+(y[j]-ya)*(y[j]-ya))<=r[j]*r[j]) { found1 = true; } if (((k-x[j])*(k-x[j])+(y[j]-yb)*(y[j]-yb))<=r[j]*r[j]) { found2 = true; } if (found1 && found2) break; } if (!found1) tcount++; if (!found2) tcount++; } cout << tcount << endl; return 0; } 39 GCC PR 51962

#include using namespace std; int r[3], x[3], y[3]; int main() { int xa=2,ya=5,xb=4,yb=2,n=3; x[0] = 3; x[1] = 5; x[2] = 1; y[0] = 1; y[1] = 3; y[2] = 3; r[0] = 2; r[1] = 1; r[2] = 2; int tcount = 0; for (int k=min(xa,xb); k<=max(xa,xb); k++) { bool found1,found2 = false; for (int j=0; j<n; j++) { if (((k-x[j])*(k-x[j])+(y[j]-ya)*(y[j]-ya))<=r[j]*r[j]) { found1 = true; } if (((k-x[j])*(k-x[j])+(y[j]-yb)*(y[j]-yb))<=r[j]*r[j]) { found2 = true; } if (found1 && found2) break; } if (!found1) tcount++; if (!found2) tcount++; } cout << tcount << endl; return 0; } 40 GCC PR Bug report says: “Compile the following simple code without -O3, and run. Now compile it with -O3 option (for optimization), run again. Surprisingly 2 different outputs appear.”

#include using namespace std; int r[3], x[3], y[3]; int main() { int xa=2,ya=5,xb=4,yb=2,n=3; x[0] = 3; x[1] = 5; x[2] = 1; y[0] = 1; y[1] = 3; y[2] = 3; r[0] = 2; r[1] = 1; r[2] = 2; int tcount = 0; for (int k=min(xa,xb); k<=max(xa,xb); k++) { bool found1,found2 = false; for (int j=0; j<n; j++) { if (((k-x[j])*(k-x[j])+(y[j]-ya)*(y[j]-ya))<=r[j]*r[j]) { found1 = true; } if (((k-x[j])*(k-x[j])+(y[j]-yb)*(y[j]-yb))<=r[j]*r[j]) { found2 = true; } if (found1 && found2) break; } if (!found1) tcount++; if (!found2) tcount++; } cout << tcount << endl; return 0; } 41 GCC PR Bug report says: “Compile the following simple code without -O3, and run. Now compile it with -O3 option (for optimization), run again. Surprisingly 2 different outputs appear.” GCC developer responds: “You do not initialise found1.” PR is RESOLVED INVALID And this person may have a hard time getting someone to read his next bug report

#include using namespace std; int r[3], x[3], y[3]; int main() { int xa=2,ya=5,xb=4,yb=2,n=3; x[0] = 3; x[1] = 5; x[2] = 1; y[0] = 1; y[1] = 3; y[2] = 3; r[0] = 2; r[1] = 1; r[2] = 2; int tcount = 0; for (int k=min(xa,xb); k<=max(xa,xb); k++) { bool found1,found2 = false; for (int j=0; j<n; j++) { if (((k-x[j])*(k-x[j])+(y[j]-ya)*(y[j]-ya))<=r[j]*r[j]) { found1 = true; } if (((k-x[j])*(k-x[j])+(y[j]-yb)*(y[j]-yb))<=r[j]*r[j]) { found2 = true; } if (found1 && found2) break; } if (!found1) tcount++; if (!found2) tcount++; } cout << tcount << endl; return 0; } 42 GCC PR 51962

C99 has – 191 kinds of undefined behavior – 52 kinds of unspecified behavior Code in a bug report must not execute these behaviors ddmin and C-Reduce tend to introduce these behaviors 43

#!/bin/bash is_valid_c99 test.c &&\ clang -O1 test.c&&\./a.out > out1.txt&&\ clang -O2 test.c&&\./a.out > out2.txt&&\ ! diff out1.txt out2.txt 44

Validity checkers: KCC: executable semantics for C99 Frama-C: static analyzer that supports an interpreter mode 45

Median size output from reducers For 57 compiler-crash bugs – ddmin*:8.6 KB – C-Reduce: 0.33 KB For 43 wrong-code bugs – ddmin*: 6.5 KB – C-Reduce: 0.51 KB * 46

Reduction fails when compiler is non-deterministic – Usually due to memory unsafety + ASLR Reduction works poorly when compiler bug stems from a resource overflow – Infinite loop, memory leak, register spill 47

Can C-Reduce create reportable test cases from proprietary codes? We hope so Clearly “declassification” has to be done on a case-by-case basis We would love to get feedback about successes and failures 48

Does C-Reduce work for C++ code? – Yes, but not (yet) very well C++-specific transformations are needed – Template instantiation – Namespace and class hierarchy flattening Problem: No validity checker for C++ 49

Factoring the transformations out of a delta debugger is a good idea C-Reduce can (sometimes) produce beautiful test cases Ensuring validity of test cases is hard 50

C-Reduce is here: 51