static void emit_ucmp(const struct lp_build_tgsi_action *action,
		      struct lp_build_tgsi_context *bld_base,
		      struct lp_build_emit_data *emit_data)
{
	LLVMBuilderRef builder = bld_base->base.gallivm->builder;

	LLVMValueRef arg0 = LLVMBuildBitCast(builder, emit_data->args[0],
					     bld_base->uint_bld.elem_type, "");

	LLVMValueRef v = LLVMBuildICmp(builder, LLVMIntNE, arg0,
				       bld_base->uint_bld.zero, "");

	emit_data->output[emit_data->chan] =
		LLVMBuildSelect(builder, v, emit_data->args[1], emit_data->args[2], "");
}

/**
 * Return (a & b)
 */
LLVMValueRef
lp_build_and(struct lp_build_context *bld, LLVMValueRef a, LLVMValueRef b)
{
   LLVMBuilderRef builder = bld->gallivm->builder;
   const struct lp_type type = bld->type;
   LLVMValueRef res;

   assert(lp_check_value(type, a));
   assert(lp_check_value(type, b));

   /* can't do bitwise ops on floating-point values */
   if (type.floating) {
      a = LLVMBuildBitCast(builder, a, bld->int_vec_type, "");
      b = LLVMBuildBitCast(builder, b, bld->int_vec_type, "");
   }

   res = LLVMBuildAnd(builder, a, b, "");

   if (type.floating) {
      res = LLVMBuildBitCast(builder, res, bld->vec_type, "");
   }

   return res;
}

static void trace_known(compile_t* c, LLVMValueRef ctx, LLVMValueRef object,
  ast_t* type, bool immutable)
{
  reachable_type_t* t = reach_type(c->reachable, type);

  // If this type has no trace function, don't try to recurse in the runtime.
  if(t->trace_fn != NULL)
  {
    // Cast the object to an object pointer.
    LLVMValueRef args[4];
    args[0] = ctx;
    args[1] = LLVMBuildBitCast(c->builder, object, c->object_ptr, "");
    args[2] = t->trace_fn;
    args[3] = LLVMConstInt(c->i32, immutable, false);

    gencall_runtime(c, "pony_traceobject", args, 4, "");
  } else {
    // Cast the object to a void pointer.
    LLVMValueRef args[2];
    args[0] = ctx;
    args[1] = LLVMBuildBitCast(c->builder, object, c->void_ptr, "");
    gencall_runtime(c, "pony_trace", args, 2, "");
  }
}

/**
 * Perform the occlusion test and increase the counter.
 * Test the depth mask. Add the number of channel which has none zero mask
 * into the occlusion counter. e.g. maskvalue is {-1, -1, -1, -1}.
 * The counter will add 4.
 *
 * \param type holds element type of the mask vector.
 * \param maskvalue is the depth test mask.
 * \param counter is a pointer of the uint32 counter.
 */
static void
lp_build_occlusion_count(LLVMBuilderRef builder,
                         struct lp_type type,
                         LLVMValueRef maskvalue,
                         LLVMValueRef counter)
{
   LLVMValueRef countmask = lp_build_const_int_vec(type, 1);
   LLVMValueRef countv = LLVMBuildAnd(builder, maskvalue, countmask, "countv");
   LLVMTypeRef i8v16 = LLVMVectorType(LLVMInt8Type(), 16);
   LLVMValueRef counti = LLVMBuildBitCast(builder, countv, i8v16, "counti");
   LLVMValueRef maskarray[4] = {
      LLVMConstInt(LLVMInt32Type(), 0, 0),
      LLVMConstInt(LLVMInt32Type(), 4, 0),
      LLVMConstInt(LLVMInt32Type(), 8, 0),
      LLVMConstInt(LLVMInt32Type(), 12, 0),
   };
   LLVMValueRef shufflemask = LLVMConstVector(maskarray, 4);
   LLVMValueRef shufflev =  LLVMBuildShuffleVector(builder, counti, LLVMGetUndef(i8v16), shufflemask, "shufflev");
   LLVMValueRef shuffle = LLVMBuildBitCast(builder, shufflev, LLVMInt32Type(), "shuffle");
   LLVMValueRef count = lp_build_intrinsic_unary(builder, "llvm.ctpop.i32", LLVMInt32Type(), shuffle);
   LLVMValueRef orig = LLVMBuildLoad(builder, counter, "orig");
   LLVMValueRef incr = LLVMBuildAdd(builder, orig, count, "incr");
   LLVMBuildStore(builder, incr, counter);
}

static void trace_array_elements(compile_t* c, reach_type_t* t,
  LLVMValueRef ctx, LLVMValueRef object, LLVMValueRef pointer)
{
  // Get the type argument for the array. This will be used to generate the
  // per-element trace call.
  ast_t* typeargs = ast_childidx(t->ast, 2);
  ast_t* typearg = ast_child(typeargs);

  if(!gentrace_needed(typearg))
    return;

  reach_type_t* t_elem = reach_type(c->reach, typearg);
  pointer = LLVMBuildBitCast(c->builder, pointer,
    LLVMPointerType(t_elem->use_type, 0), "");

  LLVMBasicBlockRef entry_block = LLVMGetInsertBlock(c->builder);
  LLVMBasicBlockRef cond_block = codegen_block(c, "cond");
  LLVMBasicBlockRef body_block = codegen_block(c, "body");
  LLVMBasicBlockRef post_block = codegen_block(c, "post");

  // Read the size.
  LLVMValueRef size = field_value(c, object, 1);
  LLVMBuildBr(c->builder, cond_block);

  // While the index is less than the size, trace an element. The initial
  // index when coming from the entry block is zero.
  LLVMPositionBuilderAtEnd(c->builder, cond_block);
  LLVMValueRef phi = LLVMBuildPhi(c->builder, c->intptr, "");
  LLVMValueRef zero = LLVMConstInt(c->intptr, 0, false);
  LLVMAddIncoming(phi, &zero, &entry_block, 1);
  LLVMValueRef test = LLVMBuildICmp(c->builder, LLVMIntULT, phi, size, "");
  LLVMBuildCondBr(c->builder, test, body_block, post_block);

  // The phi node is the index. Get the element and trace it.
  LLVMPositionBuilderAtEnd(c->builder, body_block);
  LLVMValueRef elem_ptr = LLVMBuildGEP(c->builder, pointer, &phi, 1, "elem");
  LLVMValueRef elem = LLVMBuildLoad(c->builder, elem_ptr, "");
  gentrace(c, ctx, elem, typearg);

  // Add one to the phi node and branch back to the cond block.
  LLVMValueRef one = LLVMConstInt(c->intptr, 1, false);
  LLVMValueRef inc = LLVMBuildAdd(c->builder, phi, one, "");
  body_block = LLVMGetInsertBlock(c->builder);
  LLVMAddIncoming(phi, &inc, &body_block, 1);
  LLVMBuildBr(c->builder, cond_block);

  LLVMPositionBuilderAtEnd(c->builder, post_block);
}

LLVMValueRef gencall_allocstruct(compile_t* c, gentype_t* g)
{
  // Disable debug anchor
  dwarf_location(&c->dwarf, NULL);

  // We explicitly want a boxed version.
  // Get the size of the structure.
  size_t size = (size_t)LLVMABISizeOfType(c->target_data, g->structure);

  // Get the finaliser, if there is one.
  const char* final = genname_finalise(g->type_name);
  LLVMValueRef final_fun = LLVMGetNamedFunction(c->module, final);

  // Allocate the object.
  LLVMValueRef args[3];
  args[0] = codegen_ctx(c);

  LLVMValueRef result;

  if(final_fun == NULL)
  {
    if(size <= HEAP_MAX)
    {
      uint32_t index = ponyint_heap_index(size);
      args[1] = LLVMConstInt(c->i32, index, false);
      result = gencall_runtime(c, "pony_alloc_small", args, 2, "");
    } else {
      args[1] = LLVMConstInt(c->intptr, size, false);
      result = gencall_runtime(c, "pony_alloc_large", args, 2, "");
    }
  } else {
    args[1] = LLVMConstInt(c->intptr, size, false);
    args[2] = LLVMConstBitCast(final_fun, c->final_fn);
    result = gencall_runtime(c, "pony_alloc_final", args, 3, "");
  }

  result = LLVMBuildBitCast(c->builder, result, g->structure_ptr, "");

  // Set the descriptor.
  if(g->underlying != TK_STRUCT)
  {
    LLVMValueRef desc_ptr = LLVMBuildStructGEP(c->builder, result, 0, "");
    LLVMBuildStore(c->builder, g->desc, desc_ptr);
  }

  return result;
}

static LLVMValueRef 
translateIdLval(SymbolTable *TyTable, SymbolTable *ValTable, ASTNode *Node) {
  Type *IdType = (Type*) symTableFind(ValTable, Node->Value);
  LLVMValueRef IdValue;

  if (IdType->EscapedLevel > 0) {
    LLVMTypeRef LLVMType = getLLVMTypeFromType(TyTable, IdType);
    LLVMValueRef EVPtr   = getEscapedVar(ValTable, Node->Value, Node->EscapedLevel);
    LLVMValueRef EVLoad  = LLVMBuildLoad(Builder, EVPtr, "");

    IdValue = LLVMBuildBitCast(Builder, EVLoad, LLVMPointerType(LLVMType, 0), "");
  } else {
    IdValue = resolveAliasId(ValTable, Node->Value, &toValName, &symTableFindLocal);
  }

  return IdValue;
}

static void pointer_delete(compile_t* c, reach_type_t* t, reach_type_t* t_elem)
{
  FIND_METHOD("_delete");

  LLVMTypeRef params[3];
  params[0] = t->use_type;
  params[1] = c->intptr;
  params[2] = c->intptr;
  start_function(c, m, t_elem->use_type, params, 3);

  // Set up a constant integer for the allocation size.
  size_t size = (size_t)LLVMABISizeOfType(c->target_data, t_elem->use_type);
  LLVMValueRef l_size = LLVMConstInt(c->intptr, size, false);

  LLVMValueRef ptr = LLVMGetParam(m->func, 0);
  LLVMValueRef n = LLVMGetParam(m->func, 1);
  LLVMValueRef len = LLVMGetParam(m->func, 2);

  LLVMValueRef elem_ptr = LLVMBuildBitCast(c->builder, ptr,
    LLVMPointerType(t_elem->use_type, 0), "");
  LLVMValueRef result = LLVMBuildLoad(c->builder, elem_ptr, "");

  LLVMValueRef dst = LLVMBuildPtrToInt(c->builder, elem_ptr, c->intptr, "");
  LLVMValueRef offset = LLVMBuildMul(c->builder, n, l_size, "");
  LLVMValueRef src = LLVMBuildAdd(c->builder, dst, offset, "");
  LLVMValueRef elen = LLVMBuildMul(c->builder, len, l_size, "");

  LLVMValueRef args[5];
  args[0] = LLVMBuildIntToPtr(c->builder, dst, c->void_ptr, "");
  args[1] = LLVMBuildIntToPtr(c->builder, src, c->void_ptr, "");
  args[2] = elen;
  args[3] = LLVMConstInt(c->i32, 1, false);
  args[4] = LLVMConstInt(c->i1, 0, false);

  // llvm.memmove.*(ptr, ptr + (n * sizeof(elem)), len * sizeof(elem))
  if(target_is_ilp32(c->opt->triple))
  {
    gencall_runtime(c, "llvm.memmove.p0i8.p0i8.i32", args, 5, "");
  } else {
    gencall_runtime(c, "llvm.memmove.p0i8.p0i8.i64", args, 5, "");
  }

  // Return ptr[0].
  LLVMBuildRet(c->builder, result);
  codegen_finishfun(c);
}

LLVMValueRef ac_build_cvt_pkrtz_f16(struct ac_llvm_context *ctx,
				    LLVMValueRef args[2])
{
	if (HAVE_LLVM >= 0x0500) {
		LLVMTypeRef v2f16 =
			LLVMVectorType(LLVMHalfTypeInContext(ctx->context), 2);
		LLVMValueRef res =
			ac_build_intrinsic(ctx, "llvm.amdgcn.cvt.pkrtz",
					   v2f16, args, 2,
					   AC_FUNC_ATTR_READNONE);
		return LLVMBuildBitCast(ctx->builder, res, ctx->i32, "");
	}

	return ac_build_intrinsic(ctx, "llvm.SI.packf16", ctx->i32, args, 2,
				  AC_FUNC_ATTR_READNONE |
				  AC_FUNC_ATTR_LEGACY);
}

static LLVMValueRef
rgb_to_rgba_aos(struct gallivm_state *gallivm,
                unsigned n,
                LLVMValueRef r, LLVMValueRef g, LLVMValueRef b)
{
   LLVMBuilderRef builder = gallivm->builder;
   struct lp_type type;
   LLVMValueRef a;
   LLVMValueRef rgba;

   memset(&type, 0, sizeof type);
   type.sign = TRUE;
   type.width = 32;
   type.length = n;

   assert(lp_check_value(type, r));
   assert(lp_check_value(type, g));
   assert(lp_check_value(type, b));

   /*
    * Make a 4 x unorm8 vector
    */

#ifdef PIPE_ARCH_LITTLE_ENDIAN
   r = r;
   g = LLVMBuildShl(builder, g, lp_build_const_int_vec(gallivm, type, 8), "");
   b = LLVMBuildShl(builder, b, lp_build_const_int_vec(gallivm, type, 16), "");
   a = lp_build_const_int_vec(gallivm, type, 0xff000000);
#else
   r = LLVMBuildShl(builder, r, lp_build_const_int_vec(gallivm, type, 24), "");
   g = LLVMBuildShl(builder, g, lp_build_const_int_vec(gallivm, type, 16), "");
   b = LLVMBuildShl(builder, b, lp_build_const_int_vec(gallivm, type, 8), "");
   a = lp_build_const_int_vec(gallivm, type, 0x000000ff);
#endif

   rgba = r;
   rgba = LLVMBuildOr(builder, rgba, g, "");
   rgba = LLVMBuildOr(builder, rgba, b, "");
   rgba = LLVMBuildOr(builder, rgba, a, "");

   rgba = LLVMBuildBitCast(builder, rgba,
                           LLVMVectorType(LLVMInt8TypeInContext(gallivm->context), 4*n), "");

   return rgba;
}

/**
 * lp_build_assert.
 *
 * Build an assertion in LLVM IR by building a function call to the
 * lp_assert() function above.
 *
 * \param condition should be an 'i1' or 'i32' value
 * \param msg  a string to print if the assertion fails.
 */
LLVMValueRef
lp_build_assert(LLVMBuilderRef builder, LLVMValueRef condition,
                const char *msg)
{
   LLVMModuleRef module;
   LLVMTypeRef arg_types[2];
   LLVMValueRef msg_string, assert_func, params[2], r;

   module = LLVMGetGlobalParent(LLVMGetBasicBlockParent(
                            LLVMGetInsertBlock(builder)));

   msg_string = lp_build_const_string_variable(module, msg, strlen(msg) + 1);

   arg_types[0] = LLVMInt32Type();
   arg_types[1] = LLVMPointerType(LLVMInt8Type(), 0);

   /* lookup the lp_assert function */
   assert_func = LLVMGetNamedFunction(module, "lp_assert");

   /* Create the assertion function if not found */
   if (!assert_func) {
      LLVMTypeRef func_type =
         LLVMFunctionType(LLVMVoidType(), arg_types, 2, 0);

      assert_func = LLVMAddFunction(module, "lp_assert", func_type);
      LLVMSetFunctionCallConv(assert_func, LLVMCCallConv);
      LLVMSetLinkage(assert_func, LLVMExternalLinkage);
      LLVMAddGlobalMapping(lp_build_engine, assert_func,
                           func_to_pointer((func_pointer)lp_assert));
   }
   assert(assert_func);

   /* build function call param list */
   params[0] = LLVMBuildZExt(builder, condition, arg_types[0], "");
   params[1] = LLVMBuildBitCast(builder, msg_string, arg_types[1], "");

   /* check arg types */
   assert(LLVMTypeOf(params[0]) == arg_types[0]);
   assert(LLVMTypeOf(params[1]) == arg_types[1]);

   r = LLVMBuildCall(builder, assert_func, params, 2, "");

   return r;
}

/**
 * Transpose from AOS <-> SOA
 *
 * @param single_type_lp   type of pixels
 * @param src              the 4 * n pixel input
 * @param dst              the 4 * n pixel output
 */
void
lp_build_transpose_aos(struct gallivm_state *gallivm,
                       struct lp_type single_type_lp,
                       const LLVMValueRef src[4],
                       LLVMValueRef dst[4])
{
   struct lp_type double_type_lp = single_type_lp;
   LLVMTypeRef single_type;
   LLVMTypeRef double_type;
   LLVMValueRef t0, t1, t2, t3;

   double_type_lp.length >>= 1;
   double_type_lp.width  <<= 1;

   double_type = lp_build_vec_type(gallivm, double_type_lp);
   single_type = lp_build_vec_type(gallivm, single_type_lp);

   /* Interleave x, y, z, w -> xy and zw */
   t0 = lp_build_interleave2_half(gallivm, single_type_lp, src[0], src[1], 0);
   t1 = lp_build_interleave2_half(gallivm, single_type_lp, src[2], src[3], 0);
   t2 = lp_build_interleave2_half(gallivm, single_type_lp, src[0], src[1], 1);
   t3 = lp_build_interleave2_half(gallivm, single_type_lp, src[2], src[3], 1);

   /* Cast to double width type for second interleave */
   t0 = LLVMBuildBitCast(gallivm->builder, t0, double_type, "t0");
   t1 = LLVMBuildBitCast(gallivm->builder, t1, double_type, "t1");
   t2 = LLVMBuildBitCast(gallivm->builder, t2, double_type, "t2");
   t3 = LLVMBuildBitCast(gallivm->builder, t3, double_type, "t3");

   /* Interleave xy, zw -> xyzw */
   dst[0] = lp_build_interleave2_half(gallivm, double_type_lp, t0, t1, 0);
   dst[1] = lp_build_interleave2_half(gallivm, double_type_lp, t0, t1, 1);
   dst[2] = lp_build_interleave2_half(gallivm, double_type_lp, t2, t3, 0);
   dst[3] = lp_build_interleave2_half(gallivm, double_type_lp, t2, t3, 1);

   /* Cast back to original single width type */
   dst[0] = LLVMBuildBitCast(gallivm->builder, dst[0], single_type, "dst0");
   dst[1] = LLVMBuildBitCast(gallivm->builder, dst[1], single_type, "dst1");
   dst[2] = LLVMBuildBitCast(gallivm->builder, dst[2], single_type, "dst2");
   dst[3] = LLVMBuildBitCast(gallivm->builder, dst[3], single_type, "dst3");
}

LLVMValueRef gen_assign_cast(compile_t* c, LLVMTypeRef l_type,
  LLVMValueRef r_value, ast_t* type)
{
  if(r_value <= GEN_NOVALUE)
    return r_value;

  LLVMTypeRef r_type = LLVMTypeOf(r_value);

  if(r_type == l_type)
    return r_value;

  switch(LLVMGetTypeKind(l_type))
  {
    case LLVMIntegerTypeKind:
    case LLVMHalfTypeKind:
    case LLVMFloatTypeKind:
    case LLVMDoubleTypeKind:
      assert(LLVMGetTypeKind(r_type) == LLVMPointerTypeKind);
      return gen_unbox(c, type, r_value);

    case LLVMPointerTypeKind:
      r_value = gen_box(c, type, r_value);

      if(r_value == NULL)
        return NULL;

      return LLVMBuildBitCast(c->builder, r_value, l_type, "");

    case LLVMStructTypeKind:
      if(LLVMGetTypeKind(r_type) == LLVMPointerTypeKind)
      {
        r_value = gen_unbox(c, type, r_value);
        assert(LLVMGetTypeKind(LLVMTypeOf(r_value)) == LLVMStructTypeKind);
      }

      return assign_to_tuple(c, l_type, r_value, type);

    default: {}
  }

  assert(0);
  return NULL;
}

static LLVMValueRef cast_ffi_arg(compile_t* c, ffi_decl_t* decl, ast_t* ast,
  LLVMValueRef arg, LLVMTypeRef param, const char* name)
{
  if(arg == NULL)
    return NULL;

  LLVMTypeRef arg_type = LLVMTypeOf(arg);

  if(param == arg_type)
    return arg;

  if((LLVMABISizeOfType(c->target_data, param) !=
    LLVMABISizeOfType(c->target_data, arg_type)))
  {
    report_ffi_type_err(c, decl, ast, name);
    return NULL;
  }

  switch(LLVMGetTypeKind(param))
  {
    case LLVMPointerTypeKind:
      if(LLVMGetTypeKind(arg_type) == LLVMIntegerTypeKind)
        return LLVMBuildIntToPtr(c->builder, arg, param, "");
      else
        return LLVMBuildBitCast(c->builder, arg, param, "");

    case LLVMIntegerTypeKind:
      if(LLVMGetTypeKind(arg_type) == LLVMPointerTypeKind)
        return LLVMBuildPtrToInt(c->builder, arg, param, "");

      break;

    case LLVMStructTypeKind:
      pony_assert(LLVMGetTypeKind(arg_type) == LLVMStructTypeKind);
      return arg;

    default: {}
  }

  pony_assert(false);
  return NULL;
}

void genprim_array_trace(compile_t* c, reach_type_t* t)
{
  codegen_startfun(c, t->trace_fn, NULL, NULL);
  LLVMSetFunctionCallConv(t->trace_fn, LLVMCCallConv);
  LLVMValueRef ctx = LLVMGetParam(t->trace_fn, 0);
  LLVMValueRef arg = LLVMGetParam(t->trace_fn, 1);

  // Read the base pointer.
  LLVMValueRef object = LLVMBuildBitCast(c->builder, arg, t->use_type, "");
  LLVMValueRef pointer = field_value(c, object, 3);

  // Trace the base pointer.
  LLVMValueRef args[2];
  args[0] = ctx;
  args[1] = pointer;
  gencall_runtime(c, "pony_trace", args, 2, "");

  trace_array_elements(c, t, ctx, object, pointer);
  LLVMBuildRetVoid(c->builder);
  codegen_finishfun(c);
}

void genprim_array_serialise_trace(compile_t* c, reach_type_t* t)
{
  // Generate the serialise_trace function.
  t->serialise_trace_fn = codegen_addfun(c, genname_serialise_trace(t->name),
    c->trace_type);

  codegen_startfun(c, t->serialise_trace_fn, NULL, NULL);
  LLVMSetFunctionCallConv(t->serialise_trace_fn, LLVMCCallConv);
  LLVMSetLinkage(t->serialise_trace_fn, LLVMExternalLinkage);

  LLVMValueRef ctx = LLVMGetParam(t->serialise_trace_fn, 0);
  LLVMValueRef arg = LLVMGetParam(t->serialise_trace_fn, 1);
  LLVMValueRef object = LLVMBuildBitCast(c->builder, arg, t->use_type, "");

  // Read the size.
  LLVMValueRef size = field_value(c, object, 1);

  // Calculate the size of the element type.
  ast_t* typeargs = ast_childidx(t->ast, 2);
  ast_t* typearg = ast_child(typeargs);
  reach_type_t* t_elem = reach_type(c->reach, typearg);

  size_t abisize = (size_t)LLVMABISizeOfType(c->target_data, t_elem->use_type);
  LLVMValueRef l_size = LLVMConstInt(c->intptr, abisize, false);

  // Reserve space for the array elements.
  LLVMValueRef pointer = field_value(c, object, 3);

  LLVMValueRef args[3];
  args[0] = ctx;
  args[1] = pointer;
  args[2] = LLVMBuildMul(c->builder, size, l_size, "");
  gencall_runtime(c, "pony_serialise_reserve", args, 3, "");

  // Trace the array elements.
  trace_array_elements(c, t, ctx, object, pointer);

  LLVMBuildRetVoid(c->builder);
  codegen_finishfun(c);
}

LLVMValueRef gen_call(struct node *ast)
{
	LLVMValueRef func, *arg_list = NULL;
	struct node *n;
	int arg_count, i;

	func = LLVMBuildBitCast(builder,
			rvalue_to_lvalue(codegen(ast->one)),
			LLVMPointerType(TYPE_FUNC, 0),
			"");

	arg_count = count_chain(ast->two);
	arg_list = calloc(sizeof(LLVMValueRef), arg_count);

	if (arg_count > 0 && arg_list == NULL)
		generror("out of memory");

	for (i = 0, n = ast->two; i < arg_count; i++, n = n->two)
		arg_list[arg_count - i - 1] = codegen(n->one);

	return LLVMBuildCall(builder, func, arg_list, arg_count, "");
}

static void pointer_update(compile_t* c, reach_type_t* t, reach_type_t* t_elem)
{
  FIND_METHOD("_update");

  LLVMTypeRef params[3];
  params[0] = t->use_type;
  params[1] = c->intptr;
  params[2] = t_elem->use_type;
  start_function(c, m, t_elem->use_type, params, 3);

  LLVMValueRef ptr = LLVMGetParam(m->func, 0);
  LLVMValueRef index = LLVMGetParam(m->func, 1);

  LLVMValueRef elem_ptr = LLVMBuildBitCast(c->builder, ptr,
    LLVMPointerType(t_elem->use_type, 0), "");
  LLVMValueRef loc = LLVMBuildGEP(c->builder, elem_ptr, &index, 1, "");
  LLVMValueRef result = LLVMBuildLoad(c->builder, loc, "");
  LLVMBuildStore(c->builder, LLVMGetParam(m->func, 2), loc);

  LLVMBuildRet(c->builder, result);
  codegen_finishfun(c);
}

/**
 * Convert a vector of rgba8 values into 32bit wide SoA vectors.
 *
 * \param dst_type  The desired return type. For pure integer formats
 *                  this should be a 32bit wide int or uint vector type,
 *                  otherwise a float vector type.
 *
 * \param packed    The rgba8 values to pack.
 *
 * \param rgba      The 4 SoA return vectors.
 */
void
lp_build_rgba8_to_fi32_soa(struct gallivm_state *gallivm,
                           struct lp_type dst_type,
                           LLVMValueRef packed,
                           LLVMValueRef *rgba)
{
   LLVMBuilderRef builder = gallivm->builder;
   LLVMValueRef mask = lp_build_const_int_vec(gallivm, dst_type, 0xff);
   unsigned chan;

   /* XXX technically shouldn't use that for uint dst_type */
   packed = LLVMBuildBitCast(builder, packed,
                             lp_build_int_vec_type(gallivm, dst_type), "");

   /* Decode the input vector components */
   for (chan = 0; chan < 4; ++chan) {
#ifdef PIPE_ARCH_LITTLE_ENDIAN
      unsigned start = chan*8;
#else
      unsigned start = (3-chan)*8;
#endif
      unsigned stop = start + 8;
      LLVMValueRef input;

      input = packed;

      if (start)
         input = LLVMBuildLShr(builder, input,
                               lp_build_const_int_vec(gallivm, dst_type, start), "");

      if (stop < 32)
         input = LLVMBuildAnd(builder, input, mask, "");

      if (dst_type.floating)
         input = lp_build_unsigned_norm_to_float(gallivm, 8, dst_type, input);

      rgba[chan] = input;
   }
}

/**
 * Gather elements from scatter positions in memory into a single vector.
 *
 * @param src_width src element width
 * @param dst_width result element width (source will be expanded to fit)
 * @param length length of the offsets,
 * @param base_ptr base pointer, should be a i8 pointer type.
 * @param offsets vector with offsets
 */
LLVMValueRef
lp_build_gather(LLVMBuilderRef builder,
                unsigned length,
                unsigned src_width,
                unsigned dst_width,
                LLVMValueRef base_ptr,
                LLVMValueRef offsets)
{
   LLVMTypeRef src_type = LLVMIntType(src_width);
   LLVMTypeRef src_ptr_type = LLVMPointerType(src_type, 0);
   LLVMTypeRef dst_elem_type = LLVMIntType(dst_width);
   LLVMTypeRef dst_vec_type = LLVMVectorType(dst_elem_type, length);
   LLVMValueRef res;
   unsigned i;

   res = LLVMGetUndef(dst_vec_type);
   for(i = 0; i < length; ++i) {
      LLVMValueRef index = LLVMConstInt(LLVMInt32Type(), i, 0);
      LLVMValueRef elem_offset;
      LLVMValueRef elem_ptr;
      LLVMValueRef elem;

      elem_offset = LLVMBuildExtractElement(builder, offsets, index, "");
      elem_ptr = LLVMBuildGEP(builder, base_ptr, &elem_offset, 1, "");
      elem_ptr = LLVMBuildBitCast(builder, elem_ptr, src_ptr_type, "");
      elem = LLVMBuildLoad(builder, elem_ptr, "");

      assert(src_width <= dst_width);
      if(src_width > dst_width)
         elem = LLVMBuildTrunc(builder, elem, dst_elem_type, "");
      if(src_width < dst_width)
         elem = LLVMBuildZExt(builder, elem, dst_elem_type, "");

      res = LLVMBuildInsertElement(builder, res, elem, index, "");
   }

   return res;
}