def build_libc(lib_filename, files):
     o_s = []
     prev_cxx = os.environ.get("EMMAKEN_CXX")
     if prev_cxx:
         os.environ["EMMAKEN_CXX"] = ""
     musl_internal_includes = shared.path_from_root("system", "lib", "libc", "musl", "src", "internal")
     for src in files:
         o = in_temp(os.path.basename(src) + ".o")
         execute(
             [
                 shared.PYTHON,
                 shared.EMCC,
                 shared.path_from_root("system", "lib", src),
                 "-o",
                 o,
                 "-I",
                 musl_internal_includes,
             ]
             + lib_opts,
             stdout=stdout,
             stderr=stderr,
         )
         o_s.append(o)
     if prev_cxx:
         os.environ["EMMAKEN_CXX"] = prev_cxx
     shared.Building.link(o_s, in_temp(lib_filename))
     return in_temp(lib_filename)

 def create_wasm_compiler_rt(libname):
   srcdir = shared.path_from_root('system', 'lib', 'compiler-rt', 'lib', 'builtins')
   filenames = ['addtf3.c', 'ashlti3.c', 'ashrti3.c', 'comparetf2.c', 'divtf3.c', 'divti3.c', 'udivmodti4.c',
                'extenddftf2.c', 'extendsftf2.c',
                'fixdfti.c', 'fixsfti.c', 'fixtfdi.c', 'fixtfsi.c', 'fixtfti.c',
                'fixunsdfti.c', 'fixunssfti.c', 'fixunstfdi.c', 'fixunstfsi.c', 'fixunstfti.c',
                'floatditf.c', 'floatsitf.c', 'floattidf.c', 'floattisf.c',
                'floatunditf.c', 'floatunsitf.c', 'floatuntidf.c', 'floatuntisf.c', 'lshrti3.c',
                'modti3.c', 'multf3.c', 'multi3.c', 'subtf3.c', 'udivti3.c', 'umodti3.c', 'ashrdi3.c',
                'ashldi3.c', 'fixdfdi.c', 'floatdidf.c', 'lshrdi3.c', 'moddi3.c',
                'trunctfdf2.c', 'trunctfsf2.c', 'umoddi3.c', 'fixunsdfdi.c', 'muldi3.c',
                'divdi3.c', 'divmoddi4.c', 'udivdi3.c', 'udivmoddi4.c']
   files = (os.path.join(srcdir, f) for f in filenames)
   o_s = []
   commands = []
   for src in files:
     o = in_temp(os.path.basename(src) + '.o')
     # Use clang directly instead of emcc. Since emcc's intermediate format (produced by -S) is LLVM IR, there's no way to
     # get emcc to output wasm .s files, which is what we archive in compiler_rt.
     commands.append([shared.CLANG_CC, '--target=wasm32', '-S', shared.path_from_root('system', 'lib', src), '-O2', '-o', o] + shared.EMSDK_OPTS)
     o_s.append(o)
   run_commands(commands)
   lib = in_temp(libname)
   run_commands([[shared.LLVM_AR, 'cr', '-format=gnu', lib] + o_s])
   return lib

  def create_wasm_libc(libname):
    # in asm.js we just use Math.sin etc., which is good for code size. But wasm doesn't have such builtins, so
    # we need to bundle in more code
    # we also build in musl versions of things that we have hand-optimized asm.js for
    files = ([shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'math', x) for x in ('cos.c', 'cosf.c', 'cosl.c', 'sin.c', 'sinf.c', 'sinl.c', 'tan.c', 'tanf.c', 'tanl.c', 'acos.c', 'acosf.c', 'acosl.c', 'asin.c', 'asinf.c', 'asinl.c', 'atan.c', 'atanf.c', 'atanl.c', 'atan2.c', 'atan2f.c', 'atan2l.c', 'exp.c', 'expf.c', 'expl.c', 'log.c', 'logf.c', 'logl.c', 'pow.c', 'powf.c', 'powl.c')] +
             [shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'string', x) for x in ('memcpy.c', 'memset.c', 'memmove.c')])

    return build_libc(libname, files, ['-O2'])

 def create_dlmalloc_split(libname):
   dlmalloc_o = in_temp('dl' + libname)
   check_call([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'dlmalloc.c'), '-o', dlmalloc_o, '-O2', '-DMSPACES', '-DONLY_MSPACES'])
   split_malloc_o = in_temp('sm' + libname)
   check_call([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'split_malloc.cpp'), '-o', split_malloc_o, '-O2'])
   lib = in_temp(libname)
   shared.Building.link([dlmalloc_o, split_malloc_o], lib)
   return lib

def create_load_wasm_worker():
  emscripten.logging.debug('building load-wasm-worker')
  output = emscripten.Cache.get_path('load-wasm-worker.js')
  emscripten.try_delete(output)
  check_call([PYTHON, emscripten.EMCC, emscripten.path_from_root('third_party', 'wasm-polyfill', 'src', 'unpack.cpp'),
                      emscripten.path_from_root('tools', 'optimizer', 'parser.cpp'),
                      '-o', output] + \
                      '-O3 -std=c++11 --memory-init-file 0 --llvm-lto 1 -s TOTAL_MEMORY=67108864 -s WASM=0'.split(' '))
  assert os.path.exists(output)
  open(output, 'a').write(open(emscripten.path_from_root('third_party', 'wasm-polyfill', 'src', 'load-wasm-worker.js')).read())
  return output

 def build_libc(lib_filename, files):
   o_s = []
   prev_cxx = os.environ.get('EMMAKEN_CXX')
   if prev_cxx: os.environ['EMMAKEN_CXX'] = ''
   musl_internal_includes = ['-I', shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'internal'), '-I', shared.path_from_root('system', 'lib', 'libc', 'musl', 'arch', 'js')]
   for src in files:
     o = in_temp(os.path.basename(src) + '.o')
     execute([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', src), '-o', o] + musl_internal_includes + lib_opts, stdout=stdout, stderr=stderr)
     o_s.append(o)
   if prev_cxx: os.environ['EMMAKEN_CXX'] = prev_cxx
   shared.Building.link(o_s, in_temp(lib_filename))
   return in_temp(lib_filename)

def create_pack_asmjs():
  emscripten.logging.debug('building pack-asmjs')
  output = emscripten.Cache.get_path('pack-asmjs.js')
  emscripten.try_delete(output)
  check_call([PYTHON, emscripten.EMCC, emscripten.path_from_root('third_party', 'wasm-polyfill', 'src', 'pack-asmjs.cpp'),
                      emscripten.path_from_root('third_party', 'wasm-polyfill', 'src', 'unpack.cpp'),
                      emscripten.path_from_root('tools', 'optimizer', 'parser.cpp'),
                      '-o', output] + \
                      '-O3 -std=c++11 -DCHECKED_OUTPUT_SIZE --memory-init-file 0 --llvm-lto 1 -s TOTAL_MEMORY=67108864 -s WASM=0 -s INVOKE_RUN=0'.split(' ') + \
                     ['-I' + emscripten.path_from_root('tools', 'optimizer')])
  assert os.path.exists(output)
  open(output, 'a').write(open(emscripten.path_from_root('third_party', 'wasm-polyfill', 'src', 'pack-asmjs.js')).read())
  return output

 def build_libc(lib_filename, files, lib_opts):
   o_s = []
   musl_internal_includes = ['-I', shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'internal'), '-I', shared.path_from_root('system', 'lib', 'libc', 'musl', 'arch', 'js')]
   commands = []
   # Hide several musl warnings that produce a lot of spam to unit test build server logs.
   # TODO: When updating musl the next time, feel free to recheck which of their warnings might have been fixed, and which ones of these could be cleaned up.
   c_opts = ['-Wno-dangling-else', '-Wno-unknown-pragmas', '-Wno-shift-op-parentheses', '-Wno-string-plus-int', '-Wno-logical-op-parentheses', '-Wno-bitwise-op-parentheses', '-Wno-visibility']
   for src in files:
     o = in_temp(os.path.basename(src) + '.o')
     commands.append([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', src), '-o', o] + musl_internal_includes + default_opts + c_opts + lib_opts)
     o_s.append(o)
   run_commands(commands)
   shared.Building.link(o_s, in_temp(lib_filename))
   return in_temp(lib_filename)

  def create_compiler_rt(libname):
    srcdir = shared.path_from_root('system', 'lib', 'compiler-rt')
    filenames = ['divdc3.c', 'divsc3.c', 'muldc3.c', 'mulsc3.c']
    files = (os.path.join(srcdir, f) for f in filenames)

    o_s = []
    commands = []
    for src in files:
      o = in_temp(os.path.basename(src) + '.o')
      commands.append([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', src), '-O2', '-o', o])
      o_s.append(o)
    run_commands(commands)
    shared.Building.link(o_s, in_temp(libname))
    return in_temp(libname)

 def create_libc(libname):
   logging.debug(' building libc for cache')
   libc_files = [
   ]
   musl_srcdir = shared.path_from_root('system', 'lib', 'libc', 'musl', 'src')
   blacklist = set(
     ['ipc', 'passwd', 'thread', 'signal', 'sched', 'ipc', 'time', 'linux', 'aio', 'exit', 'legacy', 'mq', 'process', 'search', 'setjmp', 'env', 'ldso', 'conf'] + # musl modules
     ['memcpy.c', 'memset.c', 'memmove.c', 'getaddrinfo.c', 'getnameinfo.c', 'inet_addr.c', 'res_query.c', 'gai_strerror.c', 'proto.c', 'gethostbyaddr.c', 'gethostbyaddr_r.c', 'gethostbyname.c', 'gethostbyname2_r.c', 'gethostbyname_r.c', 'gethostbyname2.c', 'usleep.c', 'alarm.c', 'syscall.c'] + # individual files
     ['abs.c', 'cos.c', 'cosf.c', 'cosl.c', 'sin.c', 'sinf.c', 'sinl.c', 'tan.c', 'tanf.c', 'tanl.c', 'acos.c', 'acosf.c', 'acosl.c', 'asin.c', 'asinf.c', 'asinl.c', 'atan.c', 'atanf.c', 'atanl.c', 'atan2.c', 'atan2f.c', 'atan2l.c', 'exp.c', 'expf.c', 'expl.c', 'log.c', 'logf.c', 'logl.c', 'sqrt.c', 'sqrtf.c', 'sqrtl.c', 'fabs.c', 'fabsf.c', 'fabsl.c', 'ceil.c', 'ceilf.c', 'ceill.c', 'floor.c', 'floorf.c', 'floorl.c', 'pow.c', 'powf.c', 'powl.c', 'round.c', 'roundf.c'] # individual math files
   )
   # TODO: consider using more math code from musl, doing so makes box2d faster
   for dirpath, dirnames, filenames in os.walk(musl_srcdir):
     for f in filenames:
       if f.endswith('.c'):
         if f in blacklist: continue
         dir_parts = os.path.split(dirpath)
         cancel = False
         for part in dir_parts:
           if part in blacklist:
             cancel = True
             break
         if not cancel:
           libc_files.append(os.path.join(musl_srcdir, dirpath, f))
   args = ['-Os']
   if shared.Settings.USE_PTHREADS:
     args += ['-s', 'USE_PTHREADS=1']
     assert '-mt' in libname
   else:
     assert '-mt' not in libname
   return build_libc(libname, libc_files, args)

 def create_gl():
   prev_cxx = os.environ.get('EMMAKEN_CXX')
   if prev_cxx: os.environ['EMMAKEN_CXX'] = ''
   o = in_temp('gl.o')
   check_call([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'gl.c'), '-o', o])
   if prev_cxx: os.environ['EMMAKEN_CXX'] = prev_cxx
   return o

 def create_optimizer():
   shared.logging.debug('building native optimizer: ' + name)
   output = shared.Cache.get_path(name)
   shared.try_delete(output)
   for compiler in [shared.CLANG, 'g++', 'clang++']: # try our clang first, otherwise hope for a system compiler in the path
     shared.logging.debug('  using ' + compiler)
     try:
       subprocess.Popen([compiler,
                         shared.path_from_root('tools', 'optimizer', 'parser.cpp'),
                         shared.path_from_root('tools', 'optimizer', 'simple_ast.cpp'),
                         shared.path_from_root('tools', 'optimizer', 'optimizer.cpp'),
                         '-O3', '-std=c++11', '-fno-exceptions', '-fno-rtti', '-o', output] + args).communicate()
     except OSError:
       if compiler == shared.CLANG: raise # otherwise, OSError is likely due to g++ or clang++ not being in the path
     if os.path.exists(output): return output
   raise NativeOptimizerCreationException()

 def build_libcxx(src_dirname, lib_filename, files):
   o_s = []
   for src in files:
     o = in_temp(src + '.o')
     srcfile = shared.path_from_root(src_dirname, src)
     execute([shared.PYTHON, shared.EMXX, srcfile, '-o', o, '-std=c++11'] + lib_opts, stdout=stdout, stderr=stderr)
     o_s.append(o)
   shared.Building.link(o_s, in_temp(lib_filename))
   return in_temp(lib_filename)

 def create_gl():
     prev_cxx = os.environ.get("EMMAKEN_CXX")
     if prev_cxx:
         os.environ["EMMAKEN_CXX"] = ""
     o = in_temp("gl.o")
     execute([shared.PYTHON, shared.EMCC, shared.path_from_root("system", "lib", "gl.c"), "-o", o])
     if prev_cxx:
         os.environ["EMMAKEN_CXX"] = prev_cxx
     return o

 def build_libcxx(src_dirname, lib_filename, files, lib_opts):
   o_s = []
   commands = []
   for src in files:
     o = in_temp(src + '.o')
     srcfile = shared.path_from_root(src_dirname, src)
     commands.append([shared.PYTHON, shared.EMXX, srcfile, '-o', o, '-std=c++11'] + default_opts + lib_opts)
     o_s.append(o)
   run_commands(commands)
   shared.Building.link(o_s, in_temp(lib_filename))
   return in_temp(lib_filename)

 def apply_libc(need):
   # libc needs some sign correction. # If we are in mode 0, switch to 2. We will add our lines
   try:
     if shared.Settings.CORRECT_SIGNS == 0: raise Exception('we need to change to 2')
   except: # we fail if equal to 0 - so we need to switch to 2 - or if CORRECT_SIGNS is not even in Settings
     shared.Settings.CORRECT_SIGNS = 2
   if shared.Settings.CORRECT_SIGNS == 2:
     shared.Settings.CORRECT_SIGNS_LINES = [shared.path_from_root('src', 'dlmalloc.c') + ':' + str(i+4) for i in [4816, 4191, 4246, 4199, 4205, 4235, 4227]]
   # If we are in mode 1, we are correcting everything anyhow. If we are in mode 3, we will be corrected
   # so all is well anyhow too.
   return True

 def create_optimizer():
   shared.logging.debug('building native optimizer')
   output = shared.Cache.get_path('optimizer.exe')
   shared.try_delete(output)
   errs = []
   for compiler in [shared.CLANG, 'g++', 'clang++']: # try our clang first, otherwise hope for a system compiler in the path
     shared.logging.debug('  using ' + compiler)
     out, err = subprocess.Popen([compiler, shared.path_from_root('tools', 'optimizer', 'optimizer.cpp'), '-O3', '-std=c++11', '-fno-exceptions', '-fno-rtti', '-o', output], stderr=subprocess.PIPE).communicate()
     # for profiling/debugging: '-g', '-fno-omit-frame-pointer'
     if os.path.exists(output): return output
     errs.append(err)
   raise Exception('failed to build native optimizer, errors from each attempt: ' + '\n=================\n'.join(errs))

 def create_optimizer():
     shared.logging.debug("building native optimizer: " + name)
     output = shared.Cache.get_path(name)
     shared.try_delete(output)
     for compiler in [
         shared.CLANG,
         "g++",
         "clang++",
     ]:  # try our clang first, otherwise hope for a system compiler in the path
         shared.logging.debug("  using " + compiler)
         try:
             out, err = subprocess.Popen(
                 [
                     compiler,
                     shared.path_from_root("tools", "optimizer", "parser.cpp"),
                     shared.path_from_root("tools", "optimizer", "simple_ast.cpp"),
                     shared.path_from_root("tools", "optimizer", "optimizer.cpp"),
                     shared.path_from_root("tools", "optimizer", "optimizer-main.cpp"),
                     "-O3",
                     "-std=c++11",
                     "-fno-exceptions",
                     "-fno-rtti",
                     "-o",
                     output,
                 ]
                 + args,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE,
             ).communicate()
             outs.append(out)
             errs.append(err)
         except OSError:
             if compiler == shared.CLANG:
                 raise  # otherwise, OSError is likely due to g++ or clang++ not being in the path
         if os.path.exists(output):
             return output
     raise NativeOptimizerCreationException()

 def build_libcxx(src_dirname, lib_filename, files, lib_opts, has_noexcept_version=False):
   o_s = []
   commands = []
   opts = default_opts + lib_opts
   if has_noexcept_version and shared.Settings.DISABLE_EXCEPTION_CATCHING:
     opts += ['-fno-exceptions']
   for src in files:
     o = in_temp(src + '.o')
     srcfile = shared.path_from_root(src_dirname, src)
     commands.append([shared.PYTHON, shared.EMXX, srcfile, '-o', o, '-std=c++11'] + opts)
     o_s.append(o)
   run_commands(commands)
   if lib_filename.endswith('.bc'):
     shared.Building.link(o_s, in_temp(lib_filename))
   elif lib_filename.endswith('.a'):
     shared.Building.emar('cr', in_temp(lib_filename), o_s)
   else:
     raise Exception('unknown suffix ' + lib_filename)
   return in_temp(lib_filename)

while 1:
  print 'Tried %d, notes: %s' % (tried, notes)
  print '1) Generate C'
  shared.execute([CSMITH, '--no-volatiles', '--no-math64', '--no-packed-struct'],# +
                 #['--max-block-depth', '2', '--max-block-size', '2', '--max-expr-complexity', '2', '--max-funcs', '2'],
                 stdout=open(filename + '.c', 'w'))
  #shutil.copyfile(filename + '.c', 'testcase%d.c' % tried)
  print '1) Generate C... %.2f K of C source' % (len(open(filename + '.c').read())/1024.)

  tried += 1

  print '2) Compile natively'
  shared.try_delete(filename)
  shared.execute([shared.CLANG_CC, '-O2', filename + '.c', '-o', filename + '1'] + CSMITH_CFLAGS, stderr=PIPE) #  + shared.EMSDK_OPTS
  shared.execute([shared.CLANG_CC, '-O2', '-emit-llvm', '-c', '-Xclang', '-triple=i386-pc-linux-gnu', filename + '.c', '-o', filename + '.bc'] + CSMITH_CFLAGS + shared.EMSDK_OPTS, stderr=PIPE)
  shared.execute([shared.path_from_root('tools', 'nativize_llvm.py'), filename + '.bc'], stdout=PIPE, stderr=PIPE)
  shutil.move(filename + '.bc.run', filename + '2')
  shared.execute([shared.CLANG_CC, filename + '.c', '-o', filename + '3'] + CSMITH_CFLAGS, stderr=PIPE)
  print '3) Run natively'
  try:
    correct1 = shared.timeout_run(Popen([filename + '1'], stdout=PIPE, stderr=PIPE), 3)
    if 'Segmentation fault' in correct1 or len(correct1) < 10: raise Exception('segfault')
    correct2 = shared.timeout_run(Popen([filename + '2'], stdout=PIPE, stderr=PIPE), 3)
    if 'Segmentation fault' in correct2 or len(correct2) < 10: raise Exception('segfault')
    correct3 = shared.timeout_run(Popen([filename + '3'], stdout=PIPE, stderr=PIPE), 3)
    if 'Segmentation fault' in correct3 or len(correct3) < 10: raise Exception('segfault')
    if correct1 != correct3: raise Exception('clang opts change result')
  except Exception, e:
    print 'Failed or infinite looping in native, skipping', e
    notes['invalid'] += 1
    continue