brw_eu_emit.c

/*
 Copyright (C) Intel Corp.  2006.  All Rights Reserved.
 Intel funded Tungsten Graphics to
 develop this 3D driver.

 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the
 "Software"), to deal in the Software without restriction, including
 without limitation the rights to use, copy, modify, merge, publish,
 distribute, sublicense, and/or sell copies of the Software, and to
 permit persons to whom the Software is furnished to do so, subject to
 the following conditions:

 The above copyright notice and this permission notice (including the
 next paragraph) shall be included in all copies or substantial
 portions of the Software.

 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

 **********************************************************************/
 /*
  * Authors:
  *   Keith Whitwell <[email protected]>
  */


#include "brw_eu_defines.h"
#include "brw_eu.h"

#include "util/ralloc.h"

/**
 * Prior to Sandybridge, the SEND instruction accepted non-MRF source
 * registers, implicitly moving the operand to a message register.
 *
 * On Sandybridge, this is no longer the case.  This function performs the
 * explicit move; it should be called before emitting a SEND instruction.
 */
void
gfx6_resolve_implied_move(struct brw_codegen *p,
			  struct brw_reg *src,
			  unsigned msg_reg_nr)
{
   const struct intel_device_info *devinfo = p->devinfo;
   if (devinfo->ver < 6)
      return;

   if (src->file == BRW_MESSAGE_REGISTER_FILE)
      return;

   if (src->file != BRW_ARCHITECTURE_REGISTER_FILE || src->nr != BRW_ARF_NULL) {
      assert(devinfo->ver < 12);
      brw_push_insn_state(p);
      brw_set_default_exec_size(p, BRW_EXECUTE_8);
      brw_set_default_mask_control(p, BRW_MASK_DISABLE);
      brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
      brw_MOV(p, retype(brw_message_reg(msg_reg_nr), BRW_REGISTER_TYPE_UD),
	      retype(*src, BRW_REGISTER_TYPE_UD));
      brw_pop_insn_state(p);
   }
   *src = brw_message_reg(msg_reg_nr);
}

static void
gfx7_convert_mrf_to_grf(struct brw_codegen *p, struct brw_reg *reg)
{
   /* From the Ivybridge PRM, Volume 4 Part 3, page 218 ("send"):
    * "The send with EOT should use register space R112-R127 for <src>. This is
    *  to enable loading of a new thread into the same slot while the message
    *  with EOT for current thread is pending dispatch."
    *
    * Since we're pretending to have 16 MRFs anyway, we may as well use the
    * registers required for messages with EOT.
    */
   const struct intel_device_info *devinfo = p->devinfo;
   if (devinfo->ver >= 7 && reg->file == BRW_MESSAGE_REGISTER_FILE) {
      reg->file = BRW_GENERAL_REGISTER_FILE;
      reg->nr += GFX7_MRF_HACK_START;
   }
}

void
brw_set_dest(struct brw_codegen *p, brw_inst *inst, struct brw_reg dest)
{
   const struct intel_device_info *devinfo = p->devinfo;

   if (dest.file == BRW_MESSAGE_REGISTER_FILE)
      assert((dest.nr & ~BRW_MRF_COMPR4) < BRW_MAX_MRF(devinfo->ver));
   else if (dest.file == BRW_GENERAL_REGISTER_FILE)
      assert(dest.nr < 128);

   /* The hardware has a restriction where a destination of size Byte with
    * a stride of 1 is only allowed for a packed byte MOV. For any other
    * instruction, the stride must be at least 2, even when the destination
    * is the NULL register.
    */
   if (dest.file == BRW_ARCHITECTURE_REGISTER_FILE &&
       dest.nr == BRW_ARF_NULL &&
       type_sz(dest.type) == 1 &&
       dest.hstride == BRW_HORIZONTAL_STRIDE_1) {
      dest.hstride = BRW_HORIZONTAL_STRIDE_2;
   }

   gfx7_convert_mrf_to_grf(p, &dest);

   if (devinfo->ver >= 12 &&
       (brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SEND ||
        brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDC)) {
      assert(dest.file == BRW_GENERAL_REGISTER_FILE ||
             dest.file == BRW_ARCHITECTURE_REGISTER_FILE);
      assert(dest.address_mode == BRW_ADDRESS_DIRECT);
      assert(dest.subnr == 0);
      assert(brw_inst_exec_size(devinfo, inst) == BRW_EXECUTE_1 ||
             (dest.hstride == BRW_HORIZONTAL_STRIDE_1 &&
              dest.vstride == dest.width + 1));
      assert(!dest.negate && !dest.abs);
      brw_inst_set_dst_reg_file(devinfo, inst, dest.file);
      brw_inst_set_dst_da_reg_nr(devinfo, inst, dest.nr);

   } else if (brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDS ||
              brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDSC) {
      assert(devinfo->ver < 12);
      assert(dest.file == BRW_GENERAL_REGISTER_FILE ||
             dest.file == BRW_ARCHITECTURE_REGISTER_FILE);
      assert(dest.address_mode == BRW_ADDRESS_DIRECT);
      assert(dest.subnr % 16 == 0);
      assert(dest.hstride == BRW_HORIZONTAL_STRIDE_1 &&
             dest.vstride == dest.width + 1);
      assert(!dest.negate && !dest.abs);
      brw_inst_set_dst_da_reg_nr(devinfo, inst, dest.nr);
      brw_inst_set_dst_da16_subreg_nr(devinfo, inst, dest.subnr / 16);
      brw_inst_set_send_dst_reg_file(devinfo, inst, dest.file);
   } else {
      brw_inst_set_dst_file_type(devinfo, inst, dest.file, dest.type);
      brw_inst_set_dst_address_mode(devinfo, inst, dest.address_mode);

      if (dest.address_mode == BRW_ADDRESS_DIRECT) {
         brw_inst_set_dst_da_reg_nr(devinfo, inst, dest.nr);

         if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) {
            brw_inst_set_dst_da1_subreg_nr(devinfo, inst, dest.subnr);
            if (dest.hstride == BRW_HORIZONTAL_STRIDE_0)
               dest.hstride = BRW_HORIZONTAL_STRIDE_1;
            brw_inst_set_dst_hstride(devinfo, inst, dest.hstride);
         } else {
            brw_inst_set_dst_da16_subreg_nr(devinfo, inst, dest.subnr / 16);
            brw_inst_set_da16_writemask(devinfo, inst, dest.writemask);
            if (dest.file == BRW_GENERAL_REGISTER_FILE ||
                dest.file == BRW_MESSAGE_REGISTER_FILE) {
               assert(dest.writemask != 0);
            }
            /* From the Ivybridge PRM, Vol 4, Part 3, Section 5.2.4.1:
             *    Although Dst.HorzStride is a don't care for Align16, HW needs
             *    this to be programmed as "01".
             */
            brw_inst_set_dst_hstride(devinfo, inst, 1);
         }
      } else {
         brw_inst_set_dst_ia_subreg_nr(devinfo, inst, dest.subnr);

         /* These are different sizes in align1 vs align16:
          */
         if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) {
            brw_inst_set_dst_ia1_addr_imm(devinfo, inst,
                                          dest.indirect_offset);
            if (dest.hstride == BRW_HORIZONTAL_STRIDE_0)
               dest.hstride = BRW_HORIZONTAL_STRIDE_1;
            brw_inst_set_dst_hstride(devinfo, inst, dest.hstride);
         } else {
            brw_inst_set_dst_ia16_addr_imm(devinfo, inst,
                                           dest.indirect_offset);
            /* even ignored in da16, still need to set as '01' */
            brw_inst_set_dst_hstride(devinfo, inst, 1);
         }
      }
   }

   /* Generators should set a default exec_size of either 8 (SIMD4x2 or SIMD8)
    * or 16 (SIMD16), as that's normally correct.  However, when dealing with
    * small registers, it can be useful for us to automatically reduce it to
    * match the register size.
    */
   if (p->automatic_exec_sizes) {
      /*
       * In platforms that support fp64 we can emit instructions with a width
       * of 4 that need two SIMD8 registers and an exec_size of 8 or 16. In
       * these cases we need to make sure that these instructions have their
       * exec sizes set properly when they are emitted and we can't rely on
       * this code to fix it.
       */
      bool fix_exec_size;
      if (devinfo->ver >= 6)
         fix_exec_size = dest.width < BRW_EXECUTE_4;
      else
         fix_exec_size = dest.width < BRW_EXECUTE_8;

      if (fix_exec_size)
         brw_inst_set_exec_size(devinfo, inst, dest.width);
   }
}

void
brw_set_src0(struct brw_codegen *p, brw_inst *inst, struct brw_reg reg)
{
   const struct intel_device_info *devinfo = p->devinfo;

   if (reg.file == BRW_MESSAGE_REGISTER_FILE)
      assert((reg.nr & ~BRW_MRF_COMPR4) < BRW_MAX_MRF(devinfo->ver));
   else if (reg.file == BRW_GENERAL_REGISTER_FILE)
      assert(reg.nr < 128);

   gfx7_convert_mrf_to_grf(p, &reg);

   if (devinfo->ver >= 6 &&
       (brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SEND ||
        brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDC ||
        brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDS ||
        brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDSC)) {
      /* Any source modifiers or regions will be ignored, since this just
       * identifies the MRF/GRF to start reading the message contents from.
       * Check for some likely failures.
       */
      assert(!reg.negate);
      assert(!reg.abs);
      assert(reg.address_mode == BRW_ADDRESS_DIRECT);
   }

   if (devinfo->ver >= 12 &&
       (brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SEND ||
        brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDC)) {
      assert(reg.file != BRW_IMMEDIATE_VALUE);
      assert(reg.address_mode == BRW_ADDRESS_DIRECT);
      assert(reg.subnr == 0);
      assert(has_scalar_region(reg) ||
             (reg.hstride == BRW_HORIZONTAL_STRIDE_1 &&
              reg.vstride == reg.width + 1));
      assert(!reg.negate && !reg.abs);
      brw_inst_set_send_src0_reg_file(devinfo, inst, reg.file);
      brw_inst_set_src0_da_reg_nr(devinfo, inst, reg.nr);

   } else if (brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDS ||
              brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDSC) {
      assert(reg.file == BRW_GENERAL_REGISTER_FILE);
      assert(reg.address_mode == BRW_ADDRESS_DIRECT);
      assert(reg.subnr % 16 == 0);
      assert(has_scalar_region(reg) ||
             (reg.hstride == BRW_HORIZONTAL_STRIDE_1 &&
              reg.vstride == reg.width + 1));
      assert(!reg.negate && !reg.abs);
      brw_inst_set_src0_da_reg_nr(devinfo, inst, reg.nr);
      brw_inst_set_src0_da16_subreg_nr(devinfo, inst, reg.subnr / 16);
   } else {
      brw_inst_set_src0_file_type(devinfo, inst, reg.file, reg.type);
      brw_inst_set_src0_abs(devinfo, inst, reg.abs);
      brw_inst_set_src0_negate(devinfo, inst, reg.negate);
      brw_inst_set_src0_address_mode(devinfo, inst, reg.address_mode);

      if (reg.file == BRW_IMMEDIATE_VALUE) {
         if (reg.type == BRW_REGISTER_TYPE_DF ||
             brw_inst_opcode(devinfo, inst) == BRW_OPCODE_DIM)
            brw_inst_set_imm_df(devinfo, inst, reg.df);
         else if (reg.type == BRW_REGISTER_TYPE_UQ ||
                  reg.type == BRW_REGISTER_TYPE_Q)
            brw_inst_set_imm_uq(devinfo, inst, reg.u64);
         else
            brw_inst_set_imm_ud(devinfo, inst, reg.ud);

         if (devinfo->ver < 12 && type_sz(reg.type) < 8) {
            brw_inst_set_src1_reg_file(devinfo, inst,
                                       BRW_ARCHITECTURE_REGISTER_FILE);
            brw_inst_set_src1_reg_hw_type(devinfo, inst,
                                          brw_inst_src0_reg_hw_type(devinfo, inst));
         }
      } else {
         if (reg.address_mode == BRW_ADDRESS_DIRECT) {
            brw_inst_set_src0_da_reg_nr(devinfo, inst, reg.nr);
            if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) {
                brw_inst_set_src0_da1_subreg_nr(devinfo, inst, reg.subnr);
            } else {
               brw_inst_set_src0_da16_subreg_nr(devinfo, inst, reg.subnr / 16);
            }
         } else {
            brw_inst_set_src0_ia_subreg_nr(devinfo, inst, reg.subnr);

            if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) {
               brw_inst_set_src0_ia1_addr_imm(devinfo, inst, reg.indirect_offset);
            } else {
               brw_inst_set_src0_ia16_addr_imm(devinfo, inst, reg.indirect_offset);
            }
         }

         if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) {
            if (reg.width == BRW_WIDTH_1 &&
                brw_inst_exec_size(devinfo, inst) == BRW_EXECUTE_1) {
               brw_inst_set_src0_hstride(devinfo, inst, BRW_HORIZONTAL_STRIDE_0);
               brw_inst_set_src0_width(devinfo, inst, BRW_WIDTH_1);
               brw_inst_set_src0_vstride(devinfo, inst, BRW_VERTICAL_STRIDE_0);
            } else {
               brw_inst_set_src0_hstride(devinfo, inst, reg.hstride);
               brw_inst_set_src0_width(devinfo, inst, reg.width);
               brw_inst_set_src0_vstride(devinfo, inst, reg.vstride);
            }
         } else {
            brw_inst_set_src0_da16_swiz_x(devinfo, inst,
               BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_X));
            brw_inst_set_src0_da16_swiz_y(devinfo, inst,
               BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_Y));
            brw_inst_set_src0_da16_swiz_z(devinfo, inst,
               BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_Z));
            brw_inst_set_src0_da16_swiz_w(devinfo, inst,
               BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_W));

            if (reg.vstride == BRW_VERTICAL_STRIDE_8) {
               /* This is an oddity of the fact we're using the same
                * descriptions for registers in align_16 as align_1:
                */
               brw_inst_set_src0_vstride(devinfo, inst, BRW_VERTICAL_STRIDE_4);
            } else if (devinfo->verx10 == 70 &&
                       reg.type == BRW_REGISTER_TYPE_DF &&
                       reg.vstride == BRW_VERTICAL_STRIDE_2) {
               /* From SNB PRM:
                *
                * "For Align16 access mode, only encodings of 0000 and 0011
                *  are allowed. Other codes are reserved."
                *
                * Presumably the DevSNB behavior applies to IVB as well.
                */
               brw_inst_set_src0_vstride(devinfo, inst, BRW_VERTICAL_STRIDE_4);
            } else {
               brw_inst_set_src0_vstride(devinfo, inst, reg.vstride);
            }
         }
      }
   }
}


void
brw_set_src1(struct brw_codegen *p, brw_inst *inst, struct brw_reg reg)
{
   const struct intel_device_info *devinfo = p->devinfo;

   if (reg.file == BRW_GENERAL_REGISTER_FILE)
      assert(reg.nr < 128);

   if (brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDS ||
       brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDSC ||
       (devinfo->ver >= 12 &&
        (brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SEND ||
         brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDC))) {
      assert(reg.file == BRW_GENERAL_REGISTER_FILE ||
             reg.file == BRW_ARCHITECTURE_REGISTER_FILE);
      assert(reg.address_mode == BRW_ADDRESS_DIRECT);
      assert(reg.subnr == 0);
      assert(has_scalar_region(reg) ||
             (reg.hstride == BRW_HORIZONTAL_STRIDE_1 &&
              reg.vstride == reg.width + 1));
      assert(!reg.negate && !reg.abs);
      brw_inst_set_send_src1_reg_nr(devinfo, inst, reg.nr);
      brw_inst_set_send_src1_reg_file(devinfo, inst, reg.file);
   } else {
      /* From the IVB PRM Vol. 4, Pt. 3, Section 3.3.3.5:
       *
       *    "Accumulator registers may be accessed explicitly as src0
       *    operands only."
       */
      assert(reg.file != BRW_ARCHITECTURE_REGISTER_FILE ||
             reg.nr != BRW_ARF_ACCUMULATOR);

      gfx7_convert_mrf_to_grf(p, &reg);
      assert(reg.file != BRW_MESSAGE_REGISTER_FILE);

      brw_inst_set_src1_file_type(devinfo, inst, reg.file, reg.type);
      brw_inst_set_src1_abs(devinfo, inst, reg.abs);
      brw_inst_set_src1_negate(devinfo, inst, reg.negate);

      /* Only src1 can be immediate in two-argument instructions.
       */
      assert(brw_inst_src0_reg_file(devinfo, inst) != BRW_IMMEDIATE_VALUE);

      if (reg.file == BRW_IMMEDIATE_VALUE) {
         /* two-argument instructions can only use 32-bit immediates */
         assert(type_sz(reg.type) < 8);
         brw_inst_set_imm_ud(devinfo, inst, reg.ud);
      } else {
         /* This is a hardware restriction, which may or may not be lifted
          * in the future:
          */
         assert (reg.address_mode == BRW_ADDRESS_DIRECT);
         /* assert (reg.file == BRW_GENERAL_REGISTER_FILE); */

         brw_inst_set_src1_da_reg_nr(devinfo, inst, reg.nr);
         if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) {
            brw_inst_set_src1_da1_subreg_nr(devinfo, inst, reg.subnr);
         } else {
            brw_inst_set_src1_da16_subreg_nr(devinfo, inst, reg.subnr / 16);
         }

         if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) {
            if (reg.width == BRW_WIDTH_1 &&
                brw_inst_exec_size(devinfo, inst) == BRW_EXECUTE_1) {
               brw_inst_set_src1_hstride(devinfo, inst, BRW_HORIZONTAL_STRIDE_0);
               brw_inst_set_src1_width(devinfo, inst, BRW_WIDTH_1);
               brw_inst_set_src1_vstride(devinfo, inst, BRW_VERTICAL_STRIDE_0);
            } else {
               brw_inst_set_src1_hstride(devinfo, inst, reg.hstride);
               brw_inst_set_src1_width(devinfo, inst, reg.width);
               brw_inst_set_src1_vstride(devinfo, inst, reg.vstride);
            }
         } else {
            brw_inst_set_src1_da16_swiz_x(devinfo, inst,
               BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_X));
            brw_inst_set_src1_da16_swiz_y(devinfo, inst,
               BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_Y));
            brw_inst_set_src1_da16_swiz_z(devinfo, inst,
               BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_Z));
            brw_inst_set_src1_da16_swiz_w(devinfo, inst,
               BRW_GET_SWZ(reg.swizzle, BRW_CHANNEL_W));

            if (reg.vstride == BRW_VERTICAL_STRIDE_8) {
               /* This is an oddity of the fact we're using the same
                * descriptions for registers in align_16 as align_1:
                */
               brw_inst_set_src1_vstride(devinfo, inst, BRW_VERTICAL_STRIDE_4);
            } else if (devinfo->verx10 == 70 &&
                       reg.type == BRW_REGISTER_TYPE_DF &&
                       reg.vstride == BRW_VERTICAL_STRIDE_2) {
               /* From SNB PRM:
                *
                * "For Align16 access mode, only encodings of 0000 and 0011
                *  are allowed. Other codes are reserved."
                *
                * Presumably the DevSNB behavior applies to IVB as well.
                */
               brw_inst_set_src1_vstride(devinfo, inst, BRW_VERTICAL_STRIDE_4);
            } else {
               brw_inst_set_src1_vstride(devinfo, inst, reg.vstride);
            }
         }
      }
   }
}

/**
 * Specify the descriptor and extended descriptor immediate for a SEND(C)
 * message instruction.
 */
void
brw_set_desc_ex(struct brw_codegen *p, brw_inst *inst,
                unsigned desc, unsigned ex_desc)
{
   const struct intel_device_info *devinfo = p->devinfo;
   assert(brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SEND ||
          brw_inst_opcode(devinfo, inst) == BRW_OPCODE_SENDC);
   if (devinfo->ver < 12)
      brw_inst_set_src1_file_type(devinfo, inst,
                                  BRW_IMMEDIATE_VALUE, BRW_REGISTER_TYPE_UD);
   brw_inst_set_send_desc(devinfo, inst, desc);
   if (devinfo->ver >= 9)
      brw_inst_set_send_ex_desc(devinfo, inst, ex_desc);
}

static void brw_set_math_message( struct brw_codegen *p,
				  brw_inst *inst,
				  unsigned function,
				  unsigned integer_type,
				  bool low_precision,
				  unsigned dataType )
{
   const struct intel_device_info *devinfo = p->devinfo;
   unsigned msg_length;
   unsigned response_length;

   /* Infer message length from the function */
   switch (function) {
   case BRW_MATH_FUNCTION_POW:
   case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT:
   case BRW_MATH_FUNCTION_INT_DIV_REMAINDER:
   case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER:
      msg_length = 2;
      break;
   default:
      msg_length = 1;
      break;
   }

   /* Infer response length from the function */
   switch (function) {
   case BRW_MATH_FUNCTION_SINCOS:
   case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER:
      response_length = 2;
      break;
   default:
      response_length = 1;
      break;
   }

   brw_set_desc(p, inst, brw_message_desc(
                   devinfo, msg_length, response_length, false));

   brw_inst_set_sfid(devinfo, inst, BRW_SFID_MATH);
   brw_inst_set_math_msg_function(devinfo, inst, function);
   brw_inst_set_math_msg_signed_int(devinfo, inst, integer_type);
   brw_inst_set_math_msg_precision(devinfo, inst, low_precision);
   brw_inst_set_math_msg_saturate(devinfo, inst, brw_inst_saturate(devinfo, inst));
   brw_inst_set_math_msg_data_type(devinfo, inst, dataType);
   brw_inst_set_saturate(devinfo, inst, 0);
}


static void brw_set_ff_sync_message(struct brw_codegen *p,
				    brw_inst *insn,
				    bool allocate,
				    unsigned response_length,
				    bool end_of_thread)
{
   const struct intel_device_info *devinfo = p->devinfo;

   brw_set_desc(p, insn, brw_message_desc(
                   devinfo, 1, response_length, true));

   brw_inst_set_sfid(devinfo, insn, BRW_SFID_URB);
   brw_inst_set_eot(devinfo, insn, end_of_thread);
   brw_inst_set_urb_opcode(devinfo, insn, 1); /* FF_SYNC */
   brw_inst_set_urb_allocate(devinfo, insn, allocate);
   /* The following fields are not used by FF_SYNC: */
   brw_inst_set_urb_global_offset(devinfo, insn, 0);
   brw_inst_set_urb_swizzle_control(devinfo, insn, 0);
   brw_inst_set_urb_used(devinfo, insn, 0);
   brw_inst_set_urb_complete(devinfo, insn, 0);
}

static void brw_set_urb_message( struct brw_codegen *p,
				 brw_inst *insn,
                                 enum brw_urb_write_flags flags,
				 unsigned msg_length,
				 unsigned response_length,
				 unsigned offset,
				 unsigned swizzle_control )
{
   const struct intel_device_info *devinfo = p->devinfo;

   assert(devinfo->ver < 7 || swizzle_control != BRW_URB_SWIZZLE_TRANSPOSE);
   assert(devinfo->ver < 7 || !(flags & BRW_URB_WRITE_ALLOCATE));
   assert(devinfo->ver >= 7 || !(flags & BRW_URB_WRITE_PER_SLOT_OFFSET));

   brw_set_desc(p, insn, brw_message_desc(
                   devinfo, msg_length, response_length, true));

   brw_inst_set_sfid(devinfo, insn, BRW_SFID_URB);
   brw_inst_set_eot(devinfo, insn, !!(flags & BRW_URB_WRITE_EOT));

   if (flags & BRW_URB_WRITE_OWORD) {
      assert(msg_length == 2); /* header + one OWORD of data */
      brw_inst_set_urb_opcode(devinfo, insn, BRW_URB_OPCODE_WRITE_OWORD);
   } else {
      brw_inst_set_urb_opcode(devinfo, insn, BRW_URB_OPCODE_WRITE_HWORD);
   }

   brw_inst_set_urb_global_offset(devinfo, insn, offset);
   brw_inst_set_urb_swizzle_control(devinfo, insn, swizzle_control);

   if (devinfo->ver < 8) {
      brw_inst_set_urb_complete(devinfo, insn, !!(flags & BRW_URB_WRITE_COMPLETE));
   }

   if (devinfo->ver < 7) {
      brw_inst_set_urb_allocate(devinfo, insn, !!(flags & BRW_URB_WRITE_ALLOCATE));
      brw_inst_set_urb_used(devinfo, insn, !(flags & BRW_URB_WRITE_UNUSED));
   } else {
      brw_inst_set_urb_per_slot_offset(devinfo, insn,
         !!(flags & BRW_URB_WRITE_PER_SLOT_OFFSET));
   }
}

static void
gfx7_set_dp_scratch_message(struct brw_codegen *p,
                            brw_inst *inst,
                            bool write,
                            bool dword,
                            bool invalidate_after_read,
                            unsigned num_regs,
                            unsigned addr_offset,
                            unsigned mlen,
                            unsigned rlen,
                            bool header_present)
{
   const struct intel_device_info *devinfo = p->devinfo;
   assert(num_regs == 1 || num_regs == 2 || num_regs == 4 ||
          (devinfo->ver >= 8 && num_regs == 8));
   const unsigned block_size = (devinfo->ver >= 8 ? util_logbase2(num_regs) :
                                num_regs - 1);

   brw_set_desc(p, inst, brw_message_desc(
                   devinfo, mlen, rlen, header_present));

   brw_inst_set_sfid(devinfo, inst, GFX7_SFID_DATAPORT_DATA_CACHE);
   brw_inst_set_dp_category(devinfo, inst, 1); /* Scratch Block Read/Write msgs */
   brw_inst_set_scratch_read_write(devinfo, inst, write);
   brw_inst_set_scratch_type(devinfo, inst, dword);
   brw_inst_set_scratch_invalidate_after_read(devinfo, inst, invalidate_after_read);
   brw_inst_set_scratch_block_size(devinfo, inst, block_size);
   brw_inst_set_scratch_addr_offset(devinfo, inst, addr_offset);
}

static void
brw_inst_set_state(const struct intel_device_info *devinfo,
                   brw_inst *insn,
                   const struct brw_insn_state *state)
{
   brw_inst_set_exec_size(devinfo, insn, state->exec_size);
   brw_inst_set_group(devinfo, insn, state->group);
   brw_inst_set_compression(devinfo, insn, state->compressed);
   brw_inst_set_access_mode(devinfo, insn, state->access_mode);
   brw_inst_set_mask_control(devinfo, insn, state->mask_control);
   if (devinfo->ver >= 12)
      brw_inst_set_swsb(devinfo, insn, tgl_swsb_encode(devinfo, state->swsb));
   brw_inst_set_saturate(devinfo, insn, state->saturate);
   brw_inst_set_pred_control(devinfo, insn, state->predicate);
   brw_inst_set_pred_inv(devinfo, insn, state->pred_inv);

   if (is_3src(devinfo, brw_inst_opcode(devinfo, insn)) &&
       state->access_mode == BRW_ALIGN_16) {
      brw_inst_set_3src_a16_flag_subreg_nr(devinfo, insn, state->flag_subreg % 2);
      if (devinfo->ver >= 7)
         brw_inst_set_3src_a16_flag_reg_nr(devinfo, insn, state->flag_subreg / 2);
   } else {
      brw_inst_set_flag_subreg_nr(devinfo, insn, state->flag_subreg % 2);
      if (devinfo->ver >= 7)
         brw_inst_set_flag_reg_nr(devinfo, insn, state->flag_subreg / 2);
   }

   if (devinfo->ver >= 6)
      brw_inst_set_acc_wr_control(devinfo, insn, state->acc_wr_control);
}

static brw_inst *
brw_append_insns(struct brw_codegen *p, unsigned nr_insn, unsigned align)
{
   assert(util_is_power_of_two_or_zero(sizeof(brw_inst)));
   assert(util_is_power_of_two_or_zero(align));
   const unsigned align_insn = MAX2(align / sizeof(brw_inst), 1);
   const unsigned start_insn = ALIGN(p->nr_insn, align_insn);
   const unsigned new_nr_insn = start_insn + nr_insn;

   if (p->store_size < new_nr_insn) {
      p->store_size = util_next_power_of_two(new_nr_insn * sizeof(brw_inst));
      p->store = reralloc(p->mem_ctx, p->store, brw_inst, p->store_size);
   }

   /* Memset any padding due to alignment to 0.  We don't want to be hashing
    * or caching a bunch of random bits we got from a memory allocation.
    */
   if (p->nr_insn < start_insn) {
      memset(&p->store[p->nr_insn], 0,
             (start_insn - p->nr_insn) * sizeof(brw_inst));
   }

   assert(p->next_insn_offset == p->nr_insn * sizeof(brw_inst));
   p->nr_insn = new_nr_insn;
   p->next_insn_offset = new_nr_insn * sizeof(brw_inst);

   return &p->store[start_insn];
}

void
brw_realign(struct brw_codegen *p, unsigned align)
{
   brw_append_insns(p, 0, align);
}

int
brw_append_data(struct brw_codegen *p, void *data,
                unsigned size, unsigned align)
{
   unsigned nr_insn = DIV_ROUND_UP(size, sizeof(brw_inst));
   void *dst = brw_append_insns(p, nr_insn, align);
   memcpy(dst, data, size);

   /* If it's not a whole number of instructions, memset the end */
   if (size < nr_insn * sizeof(brw_inst))
      memset(dst + size, 0, nr_insn * sizeof(brw_inst) - size);

   return dst - (void *)p->store;
}

#define next_insn brw_next_insn
brw_inst *
brw_next_insn(struct brw_codegen *p, unsigned opcode)
{
   const struct intel_device_info *devinfo = p->devinfo;
   brw_inst *insn = brw_append_insns(p, 1, sizeof(brw_inst));

   memset(insn, 0, sizeof(*insn));
   brw_inst_set_opcode(devinfo, insn, opcode);

   /* Apply the default instruction state */
   brw_inst_set_state(devinfo, insn, p->current);

   return insn;
}

void
brw_add_reloc(struct brw_codegen *p, uint32_t id,
              enum brw_shader_reloc_type type,
              uint32_t offset, uint32_t delta)
{
   if (p->num_relocs + 1 > p->reloc_array_size) {
      p->reloc_array_size = MAX2(16, p->reloc_array_size * 2);
      p->relocs = reralloc(p->mem_ctx, p->relocs,
                           struct brw_shader_reloc, p->reloc_array_size);
   }

   p->relocs[p->num_relocs++] = (struct brw_shader_reloc) {
      .id = id,
      .type = type,
      .offset = offset,
      .delta = delta,
   };
}

static brw_inst *
brw_alu1(struct brw_codegen *p, unsigned opcode,
         struct brw_reg dest, struct brw_reg src)
{
   brw_inst *insn = next_insn(p, opcode);
   brw_set_dest(p, insn, dest);
   brw_set_src0(p, insn, src);
   return insn;
}

static brw_inst *
brw_alu2(struct brw_codegen *p, unsigned opcode,
         struct brw_reg dest, struct brw_reg src0, struct brw_reg src1)
{
   /* 64-bit immediates are only supported on 1-src instructions */
   assert(src0.file != BRW_IMMEDIATE_VALUE || type_sz(src0.type) <= 4);
   assert(src1.file != BRW_IMMEDIATE_VALUE || type_sz(src1.type) <= 4);

   brw_inst *insn = next_insn(p, opcode);
   brw_set_dest(p, insn, dest);
   brw_set_src0(p, insn, src0);
   brw_set_src1(p, insn, src1);
   return insn;
}

static int
get_3src_subreg_nr(struct brw_reg reg)
{
   /* Normally, SubRegNum is in bytes (0..31).  However, 3-src instructions
    * use 32-bit units (components 0..7).  Since they only support F/D/UD
    * types, this doesn't lose any flexibility, but uses fewer bits.
    */
   return reg.subnr / 4;
}

static enum gfx10_align1_3src_vertical_stride
to_3src_align1_vstride(const struct intel_device_info *devinfo,
                       enum brw_vertical_stride vstride)
{
   switch (vstride) {
   case BRW_VERTICAL_STRIDE_0:
      return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_0;
   case BRW_VERTICAL_STRIDE_1:
      assert(devinfo->ver >= 12);
      return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_1;
   case BRW_VERTICAL_STRIDE_2:
      assert(devinfo->ver < 12);
      return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_2;
   case BRW_VERTICAL_STRIDE_4:
      return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_4;
   case BRW_VERTICAL_STRIDE_8:
   case BRW_VERTICAL_STRIDE_16:
      return BRW_ALIGN1_3SRC_VERTICAL_STRIDE_8;
   default:
      unreachable("invalid vstride");
   }
}


static enum gfx10_align1_3src_src_horizontal_stride
to_3src_align1_hstride(enum brw_horizontal_stride hstride)
{
   switch (hstride) {
   case BRW_HORIZONTAL_STRIDE_0:
      return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_0;
   case BRW_HORIZONTAL_STRIDE_1:
      return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_1;
   case BRW_HORIZONTAL_STRIDE_2:
      return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_2;
   case BRW_HORIZONTAL_STRIDE_4:
      return BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_4;
   default:
      unreachable("invalid hstride");
   }
}

static brw_inst *
brw_alu3(struct brw_codegen *p, unsigned opcode, struct brw_reg dest,
         struct brw_reg src0, struct brw_reg src1, struct brw_reg src2)
{
   const struct intel_device_info *devinfo = p->devinfo;
   brw_inst *inst = next_insn(p, opcode);

   gfx7_convert_mrf_to_grf(p, &dest);

   assert(dest.nr < 128);

   if (devinfo->ver >= 10)
      assert(!(src0.file == BRW_IMMEDIATE_VALUE &&
               src2.file == BRW_IMMEDIATE_VALUE));

   assert(src0.file == BRW_IMMEDIATE_VALUE || src0.nr < 128);
   assert(src1.file != BRW_IMMEDIATE_VALUE && src1.nr < 128);
   assert(src2.file == BRW_IMMEDIATE_VALUE || src2.nr < 128);
   assert(dest.address_mode == BRW_ADDRESS_DIRECT);
   assert(src0.address_mode == BRW_ADDRESS_DIRECT);
   assert(src1.address_mode == BRW_ADDRESS_DIRECT);
   assert(src2.address_mode == BRW_ADDRESS_DIRECT);

   if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) {
      assert(dest.file == BRW_GENERAL_REGISTER_FILE ||
             dest.file == BRW_ARCHITECTURE_REGISTER_FILE);

      if (devinfo->ver >= 12) {
         brw_inst_set_3src_a1_dst_reg_file(devinfo, inst, dest.file);
         brw_inst_set_3src_dst_reg_nr(devinfo, inst, dest.nr);
      } else {
         if (dest.file == BRW_ARCHITECTURE_REGISTER_FILE) {
            brw_inst_set_3src_a1_dst_reg_file(devinfo, inst,
                                              BRW_ALIGN1_3SRC_ACCUMULATOR);
            brw_inst_set_3src_dst_reg_nr(devinfo, inst, BRW_ARF_ACCUMULATOR);
         } else {
            brw_inst_set_3src_a1_dst_reg_file(devinfo, inst,
                                              BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE);
            brw_inst_set_3src_dst_reg_nr(devinfo, inst, dest.nr);
         }
      }
      brw_inst_set_3src_a1_dst_subreg_nr(devinfo, inst, dest.subnr / 8);

      brw_inst_set_3src_a1_dst_hstride(devinfo, inst, BRW_ALIGN1_3SRC_DST_HORIZONTAL_STRIDE_1);

      if (brw_reg_type_is_floating_point(dest.type)) {
         brw_inst_set_3src_a1_exec_type(devinfo, inst,
                                        BRW_ALIGN1_3SRC_EXEC_TYPE_FLOAT);
      } else {
         brw_inst_set_3src_a1_exec_type(devinfo, inst,
                                        BRW_ALIGN1_3SRC_EXEC_TYPE_INT);
      }

      brw_inst_set_3src_a1_dst_type(devinfo, inst, dest.type);
      brw_inst_set_3src_a1_src0_type(devinfo, inst, src0.type);
      brw_inst_set_3src_a1_src1_type(devinfo, inst, src1.type);
      brw_inst_set_3src_a1_src2_type(devinfo, inst, src2.type);

      if (src0.file == BRW_IMMEDIATE_VALUE) {
         brw_inst_set_3src_a1_src0_imm(devinfo, inst, src0.ud);
      } else {
         brw_inst_set_3src_a1_src0_vstride(
            devinfo, inst, to_3src_align1_vstride(devinfo, src0.vstride));
         brw_inst_set_3src_a1_src0_hstride(devinfo, inst,
                                           to_3src_align1_hstride(src0.hstride));
         brw_inst_set_3src_a1_src0_subreg_nr(devinfo, inst, src0.subnr);
         if (src0.type == BRW_REGISTER_TYPE_NF) {
            brw_inst_set_3src_src0_reg_nr(devinfo, inst, BRW_ARF_ACCUMULATOR);
         } else {
            brw_inst_set_3src_src0_reg_nr(devinfo, inst, src0.nr);
         }
         brw_inst_set_3src_src0_abs(devinfo, inst, src0.abs);
         brw_inst_set_3src_src0_negate(devinfo, inst, src0.negate);
      }
      brw_inst_set_3src_a1_src1_vstride(
         devinfo, inst, to_3src_align1_vstride(devinfo, src1.vstride));
      brw_inst_set_3src_a1_src1_hstride(devinfo, inst,
                                        to_3src_align1_hstride(src1.hstride));

      brw_inst_set_3src_a1_src1_subreg_nr(devinfo, inst, src1.subnr);
      if (src1.file == BRW_ARCHITECTURE_REGISTER_FILE) {
         brw_inst_set_3src_src1_reg_nr(devinfo, inst, BRW_ARF_ACCUMULATOR);
      } else {
         brw_inst_set_3src_src1_reg_nr(devinfo, inst, src1.nr);
      }
      brw_inst_set_3src_src1_abs(devinfo, inst, src1.abs);
      brw_inst_set_3src_src1_negate(devinfo, inst, src1.negate);

      if (src2.file == BRW_IMMEDIATE_VALUE) {
         brw_inst_set_3src_a1_src2_imm(devinfo, inst, src2.ud);
      } else {
         brw_inst_set_3src_a1_src2_hstride(devinfo, inst,
                                           to_3src_align1_hstride(src2.hstride));
         /* no vstride on src2 */
         brw_inst_set_3src_a1_src2_subreg_nr(devinfo, inst, src2.subnr);
         brw_inst_set_3src_src2_reg_nr(devinfo, inst, src2.nr);
         brw_inst_set_3src_src2_abs(devinfo, inst, src2.abs);
         brw_inst_set_3src_src2_negate(devinfo, inst, src2.negate);
      }

      assert(src0.file == BRW_GENERAL_REGISTER_FILE ||
             src0.file == BRW_IMMEDIATE_VALUE ||
             (src0.file == BRW_ARCHITECTURE_REGISTER_FILE &&
              src0.type == BRW_REGISTER_TYPE_NF));
      assert(src1.file == BRW_GENERAL_REGISTER_FILE ||
             src1.file == BRW_ARCHITECTURE_REGISTER_FILE);
      assert(src2.file == BRW_GENERAL_REGISTER_FILE ||
             src2.file == BRW_IMMEDIATE_VALUE);

      if (devinfo->ver >= 12) {
         if (src0.file == BRW_IMMEDIATE_VALUE) {
            brw_inst_set_3src_a1_src0_is_imm(devinfo, inst, 1);
         } else {
            brw_inst_set_3src_a1_src0_reg_file(devinfo, inst, src0.file);
         }

         brw_inst_set_3src_a1_src1_reg_file(devinfo, inst, src1.file);

         if (src2.file == BRW_IMMEDIATE_VALUE) {
            brw_inst_set_3src_a1_src2_is_imm(devinfo, inst, 1);
         } else {
            brw_inst_set_3src_a1_src2_reg_file(devinfo, inst, src2.file);
         }
      } else {
         brw_inst_set_3src_a1_src0_reg_file(devinfo, inst,
                                            src0.file == BRW_GENERAL_REGISTER_FILE ?
                                            BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE :
                                            BRW_ALIGN1_3SRC_IMMEDIATE_VALUE);
         brw_inst_set_3src_a1_src1_reg_file(devinfo, inst,
                                            src1.file == BRW_GENERAL_REGISTER_FILE ?
                                            BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE :
                                            BRW_ALIGN1_3SRC_ACCUMULATOR);
         brw_inst_set_3src_a1_src2_reg_file(devinfo, inst,
                                            src2.file == BRW_GENERAL_REGISTER_FILE ?
                                            BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE :
                                            BRW_ALIGN1_3SRC_IMMEDIATE_VALUE);
      }

   } else {
      assert(dest.file == BRW_GENERAL_REGISTER_FILE ||
             dest.file == BRW_MESSAGE_REGISTER_FILE);
      assert(dest.type == BRW_REGISTER_TYPE_F  ||
             dest.type == BRW_REGISTER_TYPE_DF ||
             dest.type == BRW_REGISTER_TYPE_D  ||
             dest.type == BRW_REGISTER_TYPE_UD ||
             (dest.type == BRW_REGISTER_TYPE_HF && devinfo->ver >= 8));
      if (devinfo->ver == 6) {
         brw_inst_set_3src_a16_dst_reg_file(devinfo, inst,
                                            dest.file == BRW_MESSAGE_REGISTER_FILE);
      }
      brw_inst_set_3src_dst_reg_nr(devinfo, inst, dest.nr);
      brw_inst_set_3src_a16_dst_subreg_nr(devinfo, inst, dest.subnr / 4);
      brw_inst_set_3src_a16_dst_writemask(devinfo, inst, dest.writemask);

      assert(src0.file == BRW_GENERAL_REGISTER_FILE);
      brw_inst_set_3src_a16_src0_swizzle(devinfo, inst, src0.swizzle);
      brw_inst_set_3src_a16_src0_subreg_nr(devinfo, inst, get_3src_subreg_nr(src0));
      brw_inst_set_3src_src0_reg_nr(devinfo, inst, src0.nr);
      brw_inst_set_3src_src0_abs(devinfo, inst, src0.abs);
      brw_inst_set_3src_src0_negate(devinfo, inst, src0.negate);
      brw_inst_set_3src_a16_src0_rep_ctrl(devinfo, inst,
                                          src0.vstride == BRW_VERTICAL_STRIDE_0);

      assert(src1.file == BRW_GENERAL_REGISTER_FILE);
      brw_inst_set_3src_a16_src1_swizzle(devinfo, inst, src1.swizzle);
      brw_inst_set_3src_a16_src1_subreg_nr(devinfo, inst, get_3src_subreg_nr(src1));
      brw_inst_set_3src_src1_reg_nr(devinfo, inst, src1.nr);
      brw_inst_set_3src_src1_abs(devinfo, inst, src1.abs);
      brw_inst_set_3src_src1_negate(devinfo, inst, src1.negate);
      brw_inst_set_3src_a16_src1_rep_ctrl(devinfo, inst,
                                          src1.vstride == BRW_VERTICAL_STRIDE_0);

      assert(src2.file == BRW_GENERAL_REGISTER_FILE);
      brw_inst_set_3src_a16_src2_swizzle(devinfo, inst, src2.swizzle);
      brw_inst_set_3src_a16_src2_subreg_nr(devinfo, inst, get_3src_subreg_nr(src2));
      brw_inst_set_3src_src2_reg_nr(devinfo, inst, src2.nr);
      brw_inst_set_3src_src2_abs(devinfo, inst, src2.abs);
      brw_inst_set_3src_src2_negate(devinfo, inst, src2.negate);
      brw_inst_set_3src_a16_src2_rep_ctrl(devinfo, inst,
                                          src2.vstride == BRW_VERTICAL_STRIDE_0);

      if (devinfo->ver >= 7) {
         /* Set both the source and destination types based on dest.type,
          * ignoring the source register types.  The MAD and LRP emitters ensure
          * that all four types are float.  The BFE and BFI2 emitters, however,
          * may send us mixed D and UD types and want us to ignore that and use
          * the destination type.
          */
         brw_inst_set_3src_a16_src_type(devinfo, inst, dest.type);
         brw_inst_set_3src_a16_dst_type(devinfo, inst, dest.type);

         /* From the Bspec, 3D Media GPGPU, Instruction fields, srcType:
          *
          *    "Three source instructions can use operands with mixed-mode
          *     precision. When SrcType field is set to :f or :hf it defines
          *     precision for source 0 only, and fields Src1Type and Src2Type
          *     define precision for other source operands:
          *
          *     0b = :f. Single precision Float (32-bit).
          *     1b = :hf. Half precision Float (16-bit)."
          */
         if (src1.type == BRW_REGISTER_TYPE_HF)
            brw_inst_set_3src_a16_src1_type(devinfo, inst, 1);

         if (src2.type == BRW_REGISTER_TYPE_HF)
            brw_inst_set_3src_a16_src2_type(devinfo, inst, 1);
      }
   }

   return inst;
}


/***********************************************************************
 * Convenience routines.
 */
#define ALU1(OP)					\
brw_inst *brw_##OP(struct brw_codegen *p,		\
	      struct brw_reg dest,			\
	      struct brw_reg src0)   			\
{							\
   return brw_alu1(p, BRW_OPCODE_##OP, dest, src0);    	\
}

#define ALU2(OP)					\
brw_inst *brw_##OP(struct brw_codegen *p,		\
	      struct brw_reg dest,			\
	      struct brw_reg src0,			\
	      struct brw_reg src1)   			\
{							\
   return brw_alu2(p, BRW_OPCODE_##OP, dest, src0, src1);	\
}

#define ALU3(OP)					\
brw_inst *brw_##OP(struct brw_codegen *p,		\
	      struct brw_reg dest,			\
	      struct brw_reg src0,			\
	      struct brw_reg src1,			\
	      struct brw_reg src2)   			\
{                                                       \
   if (p->current->access_mode == BRW_ALIGN_16) {       \
      if (src0.vstride == BRW_VERTICAL_STRIDE_0)        \
         src0.swizzle = BRW_SWIZZLE_XXXX;               \
      if (src1.vstride == BRW_VERTICAL_STRIDE_0)        \
         src1.swizzle = BRW_SWIZZLE_XXXX;               \
      if (src2.vstride == BRW_VERTICAL_STRIDE_0)        \
         src2.swizzle = BRW_SWIZZLE_XXXX;               \
   }                                                    \
   return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2);	\
}

#define ALU3F(OP)                                               \
brw_inst *brw_##OP(struct brw_codegen *p,         \
                                 struct brw_reg dest,           \
                                 struct brw_reg src0,           \
                                 struct brw_reg src1,           \
                                 struct brw_reg src2)           \
{                                                               \
   assert(dest.type == BRW_REGISTER_TYPE_F ||                   \
          dest.type == BRW_REGISTER_TYPE_DF);                   \
   if (dest.type == BRW_REGISTER_TYPE_F) {                      \
      assert(src0.type == BRW_REGISTER_TYPE_F);                 \
      assert(src1.type == BRW_REGISTER_TYPE_F);                 \
      assert(src2.type == BRW_REGISTER_TYPE_F);                 \
   } else if (dest.type == BRW_REGISTER_TYPE_DF) {              \
      assert(src0.type == BRW_REGISTER_TYPE_DF);                \
      assert(src1.type == BRW_REGISTER_TYPE_DF);                \
      assert(src2.type == BRW_REGISTER_TYPE_DF);                \
   }                                                            \
                                                                \
   if (p->current->access_mode == BRW_ALIGN_16) {               \
      if (src0.vstride == BRW_VERTICAL_STRIDE_0)                \
         src0.swizzle = BRW_SWIZZLE_XXXX;                       \
      if (src1.vstride == BRW_VERTICAL_STRIDE_0)                \
         src1.swizzle = BRW_SWIZZLE_XXXX;                       \
      if (src2.vstride == BRW_VERTICAL_STRIDE_0)                \
         src2.swizzle = BRW_SWIZZLE_XXXX;                       \
   }                                                            \
   return brw_alu3(p, BRW_OPCODE_##OP, dest, src0, src1, src2); \
}

ALU2(SEL)
ALU1(NOT)
ALU2(AND)
ALU2(OR)
ALU2(XOR)
ALU2(SHR)
ALU2(SHL)
ALU1(DIM)
ALU2(ASR)
ALU2(ROL)
ALU2(ROR)
ALU3(CSEL)
ALU1(FRC)
ALU1(RNDD)
ALU1(RNDE)
ALU1(RNDU)
ALU1(RNDZ)
ALU2(MAC)
ALU2(MACH)
ALU1(LZD)
ALU2(DP4)
ALU2(DPH)
ALU2(DP3)
ALU2(DP2)
ALU3(MAD)
ALU3F(LRP)
ALU1(BFREV)
ALU3(BFE)
ALU2(BFI1)
ALU3(BFI2)
ALU1(FBH)
ALU1(FBL)
ALU1(CBIT)
ALU2(ADDC)
ALU2(SUBB)

brw_inst *
brw_MOV(struct brw_codegen *p, struct brw_reg dest, struct brw_reg src0)
{
   const struct intel_device_info *devinfo = p->devinfo;

   /* When converting F->DF on IVB/BYT, every odd source channel is ignored.
    * To avoid the problems that causes, we use an <X,2,0> source region to
    * read each element twice.
    */
   if (devinfo->verx10 == 70 &&
       brw_get_default_access_mode(p) == BRW_ALIGN_1 &&
       dest.type == BRW_REGISTER_TYPE_DF &&
       (src0.type == BRW_REGISTER_TYPE_F ||
        src0.type == BRW_REGISTER_TYPE_D ||
        src0.type == BRW_REGISTER_TYPE_UD) &&
       !has_scalar_region(src0)) {
      assert(src0.vstride == src0.width + src0.hstride);
      src0.vstride = src0.hstride;
      src0.width = BRW_WIDTH_2;
      src0.hstride = BRW_HORIZONTAL_STRIDE_0;
   }

   return brw_alu1(p, BRW_OPCODE_MOV, dest, src0);
}

brw_inst *
brw_ADD(struct brw_codegen *p, struct brw_reg dest,
        struct brw_reg src0, struct brw_reg src1)
{
   /* 6.2.2: add */
   if (src0.type == BRW_REGISTER_TYPE_F ||
       (src0.file == BRW_IMMEDIATE_VALUE &&
	src0.type == BRW_REGISTER_TYPE_VF)) {
      assert(src1.type != BRW_REGISTER_TYPE_UD);
      assert(src1.type != BRW_REGISTER_TYPE_D);
   }

   if (src1.type == BRW_REGISTER_TYPE_F ||
       (src1.file == BRW_IMMEDIATE_VALUE &&
	src1.type == BRW_REGISTER_TYPE_VF)) {
      assert(src0.type != BRW_REGISTER_TYPE_UD);
      assert(src0.type != BRW_REGISTER_TYPE_D);
   }

   return brw_alu2(p, BRW_OPCODE_ADD, dest, src0, src1);
}

brw_inst *
brw_AVG(struct brw_codegen *p, struct brw_reg dest,
        struct brw_reg src0, struct brw_reg src1)
{
   assert(dest.type == src0.type);
   assert(src0.type == src1.type);
   switch (src0.type) {
   case BRW_REGISTER_TYPE_B:
   case BRW_REGISTER_TYPE_UB:
   case BRW_REGISTER_TYPE_W:
   case BRW_REGISTER_TYPE_UW:
   case BRW_REGISTER_TYPE_D:
   case BRW_REGISTER_TYPE_UD:
      break;
   default:
      unreachable("Bad type for brw_AVG");
   }

   return brw_alu2(p, BRW_OPCODE_AVG, dest, src0, src1);
}

brw_inst *
brw_MUL(struct brw_codegen *p, struct brw_reg dest,
        struct brw_reg src0, struct brw_reg src1)
{
   /* 6.32.38: mul */
   if (src0.type == BRW_REGISTER_TYPE_D ||
       src0.type == BRW_REGISTER_TYPE_UD ||
       src1.type == BRW_REGISTER_TYPE_D ||
       src1.type == BRW_REGISTER_TYPE_UD) {
      assert(dest.type != BRW_REGISTER_TYPE_F);
   }

   if (src0.type == BRW_REGISTER_TYPE_F ||
       (src0.file == BRW_IMMEDIATE_VALUE &&
	src0.type == BRW_REGISTER_TYPE_VF)) {
      assert(src1.type != BRW_REGISTER_TYPE_UD);
      assert(src1.type != BRW_REGISTER_TYPE_D);
   }

   if (src1.type == BRW_REGISTER_TYPE_F ||
       (src1.file == BRW_IMMEDIATE_VALUE &&
	src1.type == BRW_REGISTER_TYPE_VF)) {
      assert(src0.type != BRW_REGISTER_TYPE_UD);
      assert(src0.type != BRW_REGISTER_TYPE_D);
   }

   assert(src0.file != BRW_ARCHITECTURE_REGISTER_FILE ||
	  src0.nr != BRW_ARF_ACCUMULATOR);
   assert(src1.file != BRW_ARCHITECTURE_REGISTER_FILE ||
	  src1.nr != BRW_ARF_ACCUMULATOR);

   return brw_alu2(p, BRW_OPCODE_MUL, dest, src0, src1);
}

brw_inst *
brw_LINE(struct brw_codegen *p, struct brw_reg dest,
         struct brw_reg src0, struct brw_reg src1)
{
   src0.vstride = BRW_VERTICAL_STRIDE_0;
   src0.width = BRW_WIDTH_1;
   src0.hstride = BRW_HORIZONTAL_STRIDE_0;
   return brw_alu2(p, BRW_OPCODE_LINE, dest, src0, src1);
}

brw_inst *
brw_PLN(struct brw_codegen *p, struct brw_reg dest,
        struct brw_reg src0, struct brw_reg src1)
{
   src0.vstride = BRW_VERTICAL_STRIDE_0;
   src0.width = BRW_WIDTH_1;
   src0.hstride = BRW_HORIZONTAL_STRIDE_0;
   src1.vstride = BRW_VERTICAL_STRIDE_8;
   src1.width = BRW_WIDTH_8;
   src1.hstride = BRW_HORIZONTAL_STRIDE_1;
   return brw_alu2(p, BRW_OPCODE_PLN, dest, src0, src1);
}

brw_inst *
brw_F32TO16(struct brw_codegen *p, struct brw_reg dst, struct brw_reg src)
{
   const struct intel_device_info *devinfo = p->devinfo;
   const bool align16 = brw_get_default_access_mode(p) == BRW_ALIGN_16;
   /* The F32TO16 instruction doesn't support 32-bit destination types in
    * Align1 mode, and neither does the Gfx8 implementation in terms of a
    * converting MOV.  Gfx7 does zero out the high 16 bits in Align16 mode as
    * an undocumented feature.
    */
   const bool needs_zero_fill = (dst.type == BRW_REGISTER_TYPE_UD &&
                                 (!align16 || devinfo->ver >= 8));
   brw_inst *inst;

   if (align16) {
      assert(dst.type == BRW_REGISTER_TYPE_UD);
   } else {
      assert(dst.type == BRW_REGISTER_TYPE_UD ||
             dst.type == BRW_REGISTER_TYPE_W ||
             dst.type == BRW_REGISTER_TYPE_UW ||
             dst.type == BRW_REGISTER_TYPE_HF);
   }

   brw_push_insn_state(p);

   if (needs_zero_fill) {
      brw_set_default_access_mode(p, BRW_ALIGN_1);
      dst = spread(retype(dst, BRW_REGISTER_TYPE_W), 2);
   }

   if (devinfo->ver >= 8) {
      inst = brw_MOV(p, retype(dst, BRW_REGISTER_TYPE_HF), src);
   } else {
      assert(devinfo->ver == 7);
      inst = brw_alu1(p, BRW_OPCODE_F32TO16, dst, src);
   }

   if (needs_zero_fill) {
      if (devinfo->ver < 12)
         brw_inst_set_no_dd_clear(devinfo, inst, true);
      brw_set_default_swsb(p, tgl_swsb_null());
      inst = brw_MOV(p, suboffset(dst, 1), brw_imm_w(0));
      if (devinfo->ver < 12)
         brw_inst_set_no_dd_check(devinfo, inst, true);
   }

   brw_pop_insn_state(p);
   return inst;
}

brw_inst *
brw_F16TO32(struct brw_codegen *p, struct brw_reg dst, struct brw_reg src)
{
   const struct intel_device_info *devinfo = p->devinfo;
   bool align16 = brw_get_default_access_mode(p) == BRW_ALIGN_16;

   if (align16) {
      assert(src.type == BRW_REGISTER_TYPE_UD);
   } else {
      /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32:
       *
       *   Because this instruction does not have a 16-bit floating-point
       *   type, the source data type must be Word (W). The destination type
       *   must be F (Float).
       */
      if (src.type == BRW_REGISTER_TYPE_UD)
         src = spread(retype(src, BRW_REGISTER_TYPE_W), 2);

      assert(src.type == BRW_REGISTER_TYPE_W ||
             src.type == BRW_REGISTER_TYPE_UW ||
             src.type == BRW_REGISTER_TYPE_HF);
   }

   if (devinfo->ver >= 8) {
      return brw_MOV(p, dst, retype(src, BRW_REGISTER_TYPE_HF));
   } else {
      assert(devinfo->ver == 7);
      return brw_alu1(p, BRW_OPCODE_F16TO32, dst, src);
   }
}


void brw_NOP(struct brw_codegen *p)
{
   brw_inst *insn = next_insn(p, BRW_OPCODE_NOP);
   memset(insn, 0, sizeof(*insn));
   brw_inst_set_opcode(p->devinfo, insn, BRW_OPCODE_NOP);
}

void brw_SYNC(struct brw_codegen *p, enum tgl_sync_function func)
{
   brw_inst *insn = next_insn(p, BRW_OPCODE_SYNC);
   brw_inst_set_cond_modifier(p->devinfo, insn, func);
}

/***********************************************************************
 * Comparisons, if/else/endif
 */

brw_inst *
brw_JMPI(struct brw_codegen *p, struct brw_reg index,
         unsigned predicate_control)
{
   const struct intel_device_info *devinfo = p->devinfo;
   struct brw_reg ip = brw_ip_reg();
   brw_inst *inst = brw_alu2(p, BRW_OPCODE_JMPI, ip, ip, index);

   brw_inst_set_exec_size(devinfo, inst, BRW_EXECUTE_1);
   brw_inst_set_qtr_control(devinfo, inst, BRW_COMPRESSION_NONE);
   brw_inst_set_mask_control(devinfo, inst, BRW_MASK_DISABLE);
   brw_inst_set_pred_control(devinfo, inst, predicate_control);

   return inst;
}

static void
push_if_stack(struct brw_codegen *p, brw_inst *inst)
{
   p->if_stack[p->if_stack_depth] = inst - p->store;

   p->if_stack_depth++;
   if (p->if_stack_array_size <= p->if_stack_depth) {
      p->if_stack_array_size *= 2;
      p->if_stack = reralloc(p->mem_ctx, p->if_stack, int,
			     p->if_stack_array_size);
   }
}

static brw_inst *
pop_if_stack(struct brw_codegen *p)
{
   p->if_stack_depth--;
   return &p->store[p->if_stack[p->if_stack_depth]];
}

static void
push_loop_stack(struct brw_codegen *p, brw_inst *inst)
{
   if (p->loop_stack_array_size <= (p->loop_stack_depth + 1)) {
      p->loop_stack_array_size *= 2;
      p->loop_stack = reralloc(p->mem_ctx, p->loop_stack, int,
			       p->loop_stack_array_size);
      p->if_depth_in_loop = reralloc(p->mem_ctx, p->if_depth_in_loop, int,
				     p->loop_stack_array_size);
   }

   p->loop_stack[p->loop_stack_depth] = inst - p->store;
   p->loop_stack_depth++;
   p->if_depth_in_loop[p->loop_stack_depth] = 0;
}

static brw_inst *
get_inner_do_insn(struct brw_codegen *p)
{
   return &p->store[p->loop_stack[p->loop_stack_depth - 1]];
}

/* EU takes the value from the flag register and pushes it onto some
 * sort of a stack (presumably merging with any flag value already on
 * the stack).  Within an if block, the flags at the top of the stack
 * control execution on each channel of the unit, eg. on each of the
 * 16 pixel values in our wm programs.
 *
 * When the matching 'else' instruction is reached (presumably by
 * countdown of the instruction count patched in by our ELSE/ENDIF
 * functions), the relevant flags are inverted.
 *
 * When the matching 'endif' instruction is reached, the flags are
 * popped off.  If the stack is now empty, normal execution resumes.
 */
brw_inst *
brw_IF(struct brw_codegen *p, unsigned execute_size)
{
   const struct intel_device_info *devinfo = p->devinfo;
   brw_inst *insn;

   insn = next_insn(p, BRW_OPCODE_IF);

   /* Override the defaults for this instruction:
    */
   if (devinfo->ver < 6) {
      brw_set_dest(p, insn, brw_ip_reg());
      brw_set_src0(p, insn, brw_ip_reg());
      brw_set_src1(p, insn, brw_imm_d(0x0));
   } else if (devinfo->ver == 6) {
      brw_set_dest(p, insn, brw_imm_w(0));
      brw_inst_set_gfx6_jump_count(devinfo, insn, 0);
      brw_set_src0(p, insn, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D)));
      brw_set_src1(p, insn, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D)));
   } else if (devinfo->ver == 7) {
      brw_set_dest(p, insn, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D)));
      brw_set_src0(p, insn, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D)));
      brw_set_src1(p, insn, brw_imm_w(0));
      brw_inst_set_jip(devinfo, insn, 0);
      brw_inst_set_uip(devinfo, insn, 0);
   } else {
      brw_set_dest(p, insn, vec1(retype(brw_null_reg(), BRW_REGISTER_TYPE_D)));
      if (devinfo->ver < 12)
         brw_set_src0(p, insn, brw_imm_d(0));
      brw_inst_set_jip(devinfo, insn, 0);
      brw_inst_set_uip(devinfo, insn, 0);
   }

   brw_inst_set_exec_size(devinfo, insn, execute_size);
   brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE);
   brw_inst_set_pred_control(devinfo, insn, BRW_PREDICATE_NORMAL);
   brw_inst_set_mask_control(devinfo, insn, BRW_MASK_ENABLE);
   if (!p->single_program_flow && devinfo->ver < 6)
      brw_inst_set_thread_control(devinfo, insn, BRW_THREAD_SWITCH);

   push_if_stack(p, insn);
   p->if_depth_in_loop[p->loop_stack_depth]++;
   return insn;
}

/* This function is only used for gfx6-style IF instructions with an
 * embedded comparison (conditional modifier).  It is not used on gfx7.
 */
brw_inst *
gfx6_IF(struct brw_codegen *p, enum brw_conditional_mod conditional,
	struct brw_reg src0, struct brw_reg src1)
{
   const struct intel_device_info *devinfo = p->devinfo;
   brw_inst *insn;

   insn = next_insn(p, BRW_OPCODE_IF);

   brw_set_dest(p, insn, brw_imm_w(0));
   brw_inst_set_exec_size(devinfo, insn, brw_get_default_exec_size(p));
   brw_inst_set_gfx6_jump_count(devinfo, insn, 0);
   brw_set_src0(p, insn, src0);
   brw_set_src1(p, insn, src1);

   assert(brw_inst_qtr_control(devinfo, insn) == BRW_COMPRESSION_NONE);
   assert(brw_inst_pred_control(devinfo, insn) == BRW_PREDICATE_NONE);
   brw_inst_set_cond_modifier(devinfo, insn, conditional);

   push_if_stack(p, insn);
   return insn;
}

/**
 * In single-program-flow (SPF) mode, convert IF and ELSE into ADDs.
 */
static void
convert_IF_ELSE_to_ADD(struct brw_codegen *p,
                       brw_inst *if_inst, brw_inst *else_inst)
{
   const struct intel_device_info *devinfo = p->devinfo;

   /* The next instruction (where the ENDIF would be, if it existed) */
   brw_inst *next_inst = &p->store[p->nr_insn];

   assert(p->single_program_flow);
   assert(if_inst != NULL && brw_inst_opcode(devinfo, if_inst) == BRW_OPCODE_IF);
   assert(else_inst == NULL || brw_inst_opcode(devinfo, else_inst) == BRW_OPCODE_ELSE);
   assert(brw_inst_exec_size(devinfo, if_inst) == BRW_EXECUTE_1);

   /* Convert IF to an ADD instruction that moves the instruction pointer
    * to the first instruction of the ELSE block.  If there is no ELSE
    * block, point to where ENDIF would be.  Reverse the predicate.
    *
    * There's no need to execute an ENDIF since we don't need to do any
    * stack operations, and if we're currently executing, we just want to
    * continue normally.
    */
   brw_inst_set_opcode(devinfo, if_inst, BRW_OPCODE_ADD);
   brw_inst_set_pred_inv(devinfo, if_inst, true);

   if (else_inst != NULL) {
      /* Convert ELSE to an ADD instruction that points where the ENDIF
       * would be.
       */
      brw_inst_set_opcode(devinfo, else_inst, BRW_OPCODE_ADD);

      brw_inst_set_imm_ud(devinfo, if_inst, (else_inst - if_inst + 1) * 16);
      brw_inst_set_imm_ud(devinfo, else_inst, (next_inst - else_inst) * 16);
   } else {
      brw_inst_set_imm_ud(devinfo, if_inst, (next_inst - if_inst) * 16);
   }
}

/**
 * Patch IF and ELSE instructions with appropriate jump targets.
 */
static void
patch_IF_ELSE(struct brw_codegen *p,
              brw_inst *if_inst, brw_inst *else_inst, brw_inst *endif_inst)
{
   const struct intel_device_info *devinfo = p->devinfo;

   /* We shouldn't be patching IF and ELSE instructions in single program flow
    * mode when gen < 6, because in single program flow mode on those
    * platforms, we convert flow control instructions to conditional ADDs that
    * operate on IP (see brw_ENDIF).
    *
    * However, on Gfx6, writing to IP doesn't work in single program flow mode
    * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
    * not be updated by non-flow control instructions.").  And on later
    * platforms, there is no significant benefit to converting control flow
    * instructions to conditional ADDs.  So we do patch IF and ELSE
    * instructions in single program flow mode on those platforms.
    */
   if (devinfo->ver < 6)
      assert(!p->single_program_flow);

   assert(if_inst != NULL && brw_inst_opcode(devinfo, if_inst) == BRW_OPCODE_IF);
   assert(endif_inst != NULL);
   assert(else_inst == NULL || brw_inst_opcode(devinfo, else_inst) == BRW_OPCODE_ELSE);

   unsigned br = brw_jump_scale(devinfo);

   assert(brw_inst_opcode(devinfo, endif_inst) == BRW_OPCODE_ENDIF);
   brw_inst_set_exec_size(devinfo, endif_inst, brw_inst_exec_size(devinfo, if_inst));

   if (else_inst == NULL) {
      /* Patch IF -> ENDIF */
      if (devinfo->ver < 6) {
	 /* Turn it into an IFF, which means no mask stack operations for
	  * all-false and jumping past the ENDIF.
	  */
         brw_inst_set_opcode(devinfo, if_inst, BRW_OPCODE_IFF);
         brw_inst_set_gfx4_jump_count(devinfo, if_inst,
                                      br * (endif_inst - if_inst + 1));
         brw_inst_set_gfx4_pop_count(devinfo, if_inst, 0);
      } else if (devinfo->ver == 6) {
	 /* As of gfx6, there is no IFF and IF must point to the ENDIF. */
         brw_inst_set_gfx6_jump_count(devinfo, if_inst, br*(endif_inst - if_inst));
      } else {
         brw_inst_set_uip(devinfo, if_inst, br * (endif_inst - if_inst));
         brw_inst_set_jip(devinfo, if_inst, br * (endif_inst - if_inst));
      }
   } else {
      brw_inst_set_exec_size(devinfo, else_inst, brw_inst_exec_size(devinfo, if_inst));

      /* Patch IF -> ELSE */
      if (devinfo->ver < 6) {
         brw_inst_set_gfx4_jump_count(devinfo, if_inst,
                                      br * (else_inst - if_inst));
         brw_inst_set_gfx4_pop_count(devinfo, if_inst, 0);
      } else if (devinfo->ver == 6) {
         brw_inst_set_gfx6_jump_count(devinfo, if_inst,
                                      br * (else_inst - if_inst + 1));
      }

      /* Patch ELSE -> ENDIF */
      if (devinfo->ver < 6) {
	 /* BRW_OPCODE_ELSE pre-gfx6 should point just past the
	  * matching ENDIF.
	  */
         brw_inst_set_gfx4_jump_count(devinfo, else_inst,
                                      br * (endif_inst - else_inst + 1));
         brw_inst_set_gfx4_pop_count(devinfo, else_inst, 1);
      } else if (devinfo->ver == 6) {
	 /* BRW_OPCODE_ELSE on gfx6 should point to the matching ENDIF. */
         brw_inst_set_gfx6_jump_count(devinfo, else_inst,
                                      br * (endif_inst - else_inst));
      } else {
	 /* The IF instruction's JIP should point just past the ELSE */
         brw_inst_set_jip(devinfo, if_inst, br * (else_inst - if_inst + 1));
	 /* The IF instruction's UIP and ELSE's JIP should point to ENDIF */
         brw_inst_set_uip(devinfo, if_inst, br * (endif_inst - if_inst));
         brw_inst_set_jip(devinfo, else_inst, br * (endif_inst - else_inst));
         if (devinfo->ver >= 8) {
            /* Since we don't set branch_ctrl, the ELSE's JIP and UIP both
             * should point to ENDIF.
             */
            brw_inst_set_uip(devinfo, else_inst, br * (endif_inst - else_inst));
         }
      }
   }
}

void
brw_ELSE(struct brw_codegen *p)
{
   const struct intel_device_info *devinfo = p->devinfo;
   brw_inst *insn;

   insn = next_insn(p, BRW_OPCODE_ELSE);

   if (devinfo->ver < 6) {
      brw_set_dest(p, insn, brw_ip_reg());
      brw_set_src0(p, insn, brw_ip_reg());
      brw_set_src1(p, insn, brw_imm_d(0x0));
   } else if (devinfo->ver == 6) {
      brw_set_dest(p, insn, brw_imm_w(0));
      brw_inst_set_gfx6_jump_count(devinfo, insn, 0);
      brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
      brw_set_src1(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
   } else if (devinfo->ver == 7) {
      brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
      brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
      brw_set_src1(p, insn, brw_imm_w(0));
      brw_inst_set_jip(devinfo, insn, 0);
      brw_inst_set_uip(devinfo, insn, 0);
   } else {
      brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
      if (devinfo->ver < 12)
         brw_set_src0(p, insn, brw_imm_d(0));
      brw_inst_set_jip(devinfo, insn, 0);
      brw_inst_set_uip(devinfo, insn, 0);
   }

   brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE);
   brw_inst_set_mask_control(devinfo, insn, BRW_MASK_ENABLE);
   if (!p->single_program_flow && devinfo->ver < 6)
      brw_inst_set_thread_control(devinfo, insn, BRW_THREAD_SWITCH);

   push_if_stack(p, insn);
}

void
brw_ENDIF(struct brw_codegen *p)
{
   const struct intel_device_info *devinfo = p->devinfo;
   brw_inst *insn = NULL;
   brw_inst *else_inst = NULL;
   brw_inst *if_inst = NULL;
   brw_inst *tmp;
   bool emit_endif = true;

   /* In single program flow mode, we can express IF and ELSE instructions
    * equivalently as ADD instructions that operate on IP.  On platforms prior
    * to Gfx6, flow control instructions cause an implied thread switch, so
    * this is a significant savings.
    *
    * However, on Gfx6, writing to IP doesn't work in single program flow mode
    * (see the SandyBridge PRM, Volume 4 part 2, p79: "When SPF is ON, IP may
    * not be updated by non-flow control instructions.").  And on later
    * platforms, there is no significant benefit to converting control flow
    * instructions to conditional ADDs.  So we only do this trick on Gfx4 and
    * Gfx5.
    */
   if (devinfo->ver < 6 && p->single_program_flow)
      emit_endif = false;

   /*
    * A single next_insn() may change the base address of instruction store
    * memory(p->store), so call it first before referencing the instruction
    * store pointer from an index
    */
   if (emit_endif)
      insn = next_insn(p, BRW_OPCODE_ENDIF);

   /* Pop the IF and (optional) ELSE instructions from the stack */
   p->if_depth_in_loop[p->loop_stack_depth]--;
   tmp = pop_if_stack(p);
   if (brw_inst_opcode(devinfo, tmp) == BRW_OPCODE_ELSE) {
      else_inst = tmp;
      tmp = pop_if_stack(p);
   }
   if_inst = tmp;

   if (!emit_endif) {
      /* ENDIF is useless; don't bother emitting it. */
      convert_IF_ELSE_to_ADD(p, if_inst, else_inst);
      return;
   }

   if (devinfo->ver < 6) {
      brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
      brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
      brw_set_src1(p, insn, brw_imm_d(0x0));
   } else if (devinfo->ver == 6) {
      brw_set_dest(p, insn, brw_imm_w(0));
      brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
      brw_set_src1(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
   } else if (devinfo->ver == 7) {
      brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
      brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
      brw_set_src1(p, insn, brw_imm_w(0));
   } else {
      brw_set_src0(p, insn, brw_imm_d(0));
   }

   brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE);
   brw_inst_set_mask_control(devinfo, insn, BRW_MASK_ENABLE);
   if (devinfo->ver < 6)
      brw_inst_set_thread_control(devinfo, insn, BRW_THREAD_SWITCH);

   /* Also pop item off the stack in the endif instruction: */
   if (devinfo->ver < 6) {
      brw_inst_set_gfx4_jump_count(devinfo, insn, 0);
      brw_inst_set_gfx4_pop_count(devinfo, insn, 1);
   } else if (devinfo->ver == 6) {
      brw_inst_set_gfx6_jump_count(devinfo, insn, 2);
   } else {
      brw_inst_set_jip(devinfo, insn, 2);
   }
   patch_IF_ELSE(p, if_inst, else_inst, insn);
}

brw_inst *
brw_BREAK(struct brw_codegen *p)
{
   const struct intel_device_info *devinfo = p->devinfo;
   brw_inst *insn;

   insn = next_insn(p, BRW_OPCODE_BREAK);
   if (devinfo->ver >= 8) {
      brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
      brw_set_src0(p, insn, brw_imm_d(0x0));
   } else if (devinfo->ver >= 6) {
      brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
      brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
      brw_set_src1(p, insn, brw_imm_d(0x0));
   } else {
      brw_set_dest(p, insn, brw_ip_reg());
      brw_set_src0(p, insn, brw_ip_reg());
      brw_set_src1(p, insn, brw_imm_d(0x0));
      brw_inst_set_gfx4_pop_count(devinfo, insn,
                                  p->if_depth_in_loop[p->loop_stack_depth]);
   }
   brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE);
   brw_inst_set_exec_size(devinfo, insn, brw_get_default_exec_size(p));

   return insn;
}

brw_inst *
brw_CONT(struct brw_codegen *p)
{
   const struct intel_device_info *devinfo = p->devinfo;
   brw_inst *insn;

   insn = next_insn(p, BRW_OPCODE_CONTINUE);
   brw_set_dest(p, insn, brw_ip_reg());
   if (devinfo->ver >= 8) {
      brw_set_src0(p, insn, brw_imm_d(0x0));
   } else {
      brw_set_src0(p, insn, brw_ip_reg());
      brw_set_src1(p, insn, brw_imm_d(0x0));
   }

   if (devinfo->ver < 6) {
      brw_inst_set_gfx4_pop_count(devinfo, insn,
                                  p->if_depth_in_loop[p->loop_stack_depth]);
   }
   brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE);
   brw_inst_set_exec_size(devinfo, insn, brw_get_default_exec_size(p));
   return insn;
}

brw_inst *
brw_HALT(struct brw_codegen *p)
{
   const struct intel_device_info *devinfo = p->devinfo;
   brw_inst *insn;

   insn = next_insn(p, BRW_OPCODE_HALT);
   brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
   if (devinfo->ver < 6) {
      /* From the Gfx4 PRM:
       *
       *    "IP register must be put (for example, by the assembler) at <dst>
       *    and <src0> locations.
       */
      brw_set_dest(p, insn, brw_ip_reg());
      brw_set_src0(p, insn, brw_ip_reg());
      brw_set_src1(p, insn, brw_imm_d(0x0)); /* exitcode updated later. */
   } else if (devinfo->ver < 8) {
      brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
      brw_set_src1(p, insn, brw_imm_d(0x0)); /* UIP and JIP, updated later. */
   } else if (devinfo->ver < 12) {
      brw_set_src0(p, insn, brw_imm_d(0x0));
   }

   brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE);
   brw_inst_set_exec_size(devinfo, insn, brw_get_default_exec_size(p));
   return insn;
}

/* DO/WHILE loop:
 *
 * The DO/WHILE is just an unterminated loop -- break or continue are
 * used for control within the loop.  We have a few ways they can be
 * done.
 *
 * For uniform control flow, the WHILE is just a jump, so ADD ip, ip,
 * jip and no DO instruction.
 *
 * For non-uniform control flow pre-gfx6, there's a DO instruction to
 * push the mask, and a WHILE to jump back, and BREAK to get out and
 * pop the mask.
 *
 * For gfx6, there's no more mask stack, so no need for DO.  WHILE
 * just points back to the first instruction of the loop.
 */
brw_inst *
brw_DO(struct brw_codegen *p, unsigned execute_size)
{
   const struct intel_device_info *devinfo = p->devinfo;

   if (devinfo->ver >= 6 || p->single_program_flow) {
      push_loop_stack(p, &p->store[p->nr_insn]);
      return &p->store[p->nr_insn];
   } else {
      brw_inst *insn = next_insn(p, BRW_OPCODE_DO);

      push_loop_stack(p, insn);

      /* Override the defaults for this instruction:
       */
      brw_set_dest(p, insn, brw_null_reg());
      brw_set_src0(p, insn, brw_null_reg());
      brw_set_src1(p, insn, brw_null_reg());

      brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE);
      brw_inst_set_exec_size(devinfo, insn, execute_size);
      brw_inst_set_pred_control(devinfo, insn, BRW_PREDICATE_NONE);

      return insn;
   }
}

/**
 * For pre-gfx6, we patch BREAK/CONT instructions to point at the WHILE
 * instruction here.
 *
 * For gfx6+, see brw_set_uip_jip(), which doesn't care so much about the loop
 * nesting, since it can always just point to the end of the block/current loop.
 */
static void
brw_patch_break_cont(struct brw_codegen *p, brw_inst *while_inst)
{
   const struct intel_device_info *devinfo = p->devinfo;
   brw_inst *do_inst = get_inner_do_insn(p);
   brw_inst *inst;
   unsigned br = brw_jump_scale(devinfo);

   assert(devinfo->ver < 6);

   for (inst = while_inst - 1; inst != do_inst; inst--) {
      /* If the jump count is != 0, that means that this instruction has already
       * been patched because it's part of a loop inside of the one we're
       * patching.
       */
      if (brw_inst_opcode(devinfo, inst) == BRW_OPCODE_BREAK &&
          brw_inst_gfx4_jump_count(devinfo, inst) == 0) {
         brw_inst_set_gfx4_jump_count(devinfo, inst, br*((while_inst - inst) + 1));
      } else if (brw_inst_opcode(devinfo, inst) == BRW_OPCODE_CONTINUE &&
                 brw_inst_gfx4_jump_count(devinfo, inst) == 0) {
         brw_inst_set_gfx4_jump_count(devinfo, inst, br * (while_inst - inst));
      }
   }
}

brw_inst *
brw_WHILE(struct brw_codegen *p)
{
   const struct intel_device_info *devinfo = p->devinfo;
   brw_inst *insn, *do_insn;
   unsigned br = brw_jump_scale(devinfo);

   if (devinfo->ver >= 6) {
      insn = next_insn(p, BRW_OPCODE_WHILE);
      do_insn = get_inner_do_insn(p);

      if (devinfo->ver >= 8) {
         brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
         if (devinfo->ver < 12)
            brw_set_src0(p, insn, brw_imm_d(0));
         brw_inst_set_jip(devinfo, insn, br * (do_insn - insn));
      } else if (devinfo->ver == 7) {
         brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
         brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
         brw_set_src1(p, insn, brw_imm_w(0));
         brw_inst_set_jip(devinfo, insn, br * (do_insn - insn));
      } else {
         brw_set_dest(p, insn, brw_imm_w(0));
         brw_inst_set_gfx6_jump_count(devinfo, insn, br * (do_insn - insn));
         brw_set_src0(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
         brw_set_src1(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_D));
      }

      brw_inst_set_exec_size(devinfo, insn, brw_get_default_exec_size(p));

   } else {
      if (p->single_program_flow) {
	 insn = next_insn(p, BRW_OPCODE_ADD);
         do_insn = get_inner_do_insn(p);

	 brw_set_dest(p, insn, brw_ip_reg());
	 brw_set_src0(p, insn, brw_ip_reg());
	 brw_set_src1(p, insn, brw_imm_d((do_insn - insn) * 16));
         brw_inst_set_exec_size(devinfo, insn, BRW_EXECUTE_1);
      } else {
	 insn = next_insn(p, BRW_OPCODE_WHILE);
         do_insn = get_inner_do_insn(p);

         assert(brw_inst_opcode(devinfo, do_insn) == BRW_OPCODE_DO);

	 brw_set_dest(p, insn, brw_ip_reg());
	 brw_set_src0(p, insn, brw_ip_reg());
	 brw_set_src1(p, insn, brw_imm_d(0));

         brw_inst_set_exec_size(devinfo, insn, brw_inst_exec_size(devinfo, do_insn));
         brw_inst_set_gfx4_jump_count(devinfo, insn, br * (do_insn - insn + 1));
         brw_inst_set_gfx4_pop_count(devinfo, insn, 0);

	 brw_patch_break_cont(p, insn);
      }
   }
   brw_inst_set_qtr_control(devinfo, insn, BRW_COMPRESSION_NONE);

   p->loop_stack_depth--;

   return insn;
}

/* FORWARD JUMPS:
 */
void brw_land_fwd_jump(struct brw_codegen *p, int jmp_insn_idx)
{
   const struct intel_device_info *devinfo = p->devinfo;
   brw_inst *jmp_insn = &p->store[jmp_insn_idx];
   unsigned jmpi = 1;

   if (devinfo->ver >= 5)
      jmpi = 2;

   assert(brw_inst_opcode(devinfo, jmp_insn) == BRW_OPCODE_JMPI);
   assert(brw_inst_src1_reg_file(devinfo, jmp_insn) == BRW_IMMEDIATE_VALUE);

   brw_inst_set_gfx4_jump_count(devinfo, jmp_insn,
                                jmpi * (p->nr_insn - jmp_insn_idx - 1));
}

/* To integrate with the above, it makes sense that the comparison
 * instruction should populate the flag register.  It might be simpler
 * just to use the flag reg for most WM tasks?
 */
void brw_CMP(struct brw_codegen *p,
	     struct brw_reg dest,
	     unsigned conditional,
	     struct brw_reg src0,
	     struct brw_reg src1)
{
   const struct intel_device_info *devinfo = p->devinfo;
   brw_inst *insn = next_insn(p, BRW_OPCODE_CMP);

   brw_inst_set_cond_modifier(devinfo, insn, conditional);
   brw_set_dest(p, insn, dest);
   brw_set_src0(p, insn, src0);
   brw_set_src1(p, insn, src1);

   /* Item WaCMPInstNullDstForcesThreadSwitch in the Haswell Bspec workarounds
    * page says:
    *    "Any CMP instruction with a null destination must use a {switch}."
    *
    * It also applies to other Gfx7 platforms (IVB, BYT) even though it isn't
    * mentioned on their work-arounds pages.
    */
   if (devinfo->ver == 7) {
      if (dest.file == BRW_ARCHITECTURE_REGISTER_FILE &&
          dest.nr == BRW_ARF_NULL) {
         brw_inst_set_thread_control(devinfo, insn, BRW_THREAD_SWITCH);
      }
   }
}

void brw_CMPN(struct brw_codegen *p,
              struct brw_reg dest,
              unsigned conditional,
              struct brw_reg src0,
              struct brw_reg src1)
{
   const struct intel_device_info *devinfo = p->devinfo;
   brw_inst *insn = next_insn(p, BRW_OPCODE_CMPN);

   brw_inst_set_cond_modifier(devinfo, insn, conditional);
   brw_set_dest(p, insn, dest);
   brw_set_src0(p, insn, src0);
   brw_set_src1(p, insn, src1);

   /* Page 166 of the Ivy Bridge PRM Volume 4 part 3 (Execution Unit ISA)
    * says:
    *
    *    If the destination is the null register, the {Switch} instruction
    *    option must be used.
    *
    * Page 77 of the Haswell PRM Volume 2b contains the same text.
    */
   if (devinfo->ver == 7) {
      if (dest.file == BRW_ARCHITECTURE_REGISTER_FILE &&
          dest.nr == BRW_ARF_NULL) {
         brw_inst_set_thread_control(devinfo, insn, BRW_THREAD_SWITCH);
      }
   }
}

/***********************************************************************
 * Helpers for the various SEND message types:
 */

/** Extended math function, float[8].
 */
void gfx4_math(struct brw_codegen *p,
	       struct brw_reg dest,
	       unsigned function,
	       unsigned msg_reg_nr,
	       struct brw_reg src,
	       unsigned precision )
{
   const struct intel_device_info *devinfo = p->devinfo;
   brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);
   unsigned data_type;
   if (has_scalar_region(src)) {
      data_type = BRW_MATH_DATA_SCALAR;
   } else {
      data_type = BRW_MATH_DATA_VECTOR;
   }

   assert(devinfo->ver < 6);

   /* Example code doesn't set predicate_control for send
    * instructions.
    */
   brw_inst_set_pred_control(devinfo, insn, 0);
   brw_inst_set_base_mrf(devinfo, insn, msg_reg_nr);

   brw_set_dest(p, insn, dest);
   brw_set_src0(p, insn, src);
   brw_set_math_message(p,
                        insn,
                        function,
                        src.type == BRW_REGISTER_TYPE_D,
                        precision,
                        data_type);
}

void gfx6_math(struct brw_codegen *p,
	       struct brw_reg dest,
	       unsigned function,
	       struct brw_reg src0,
	       struct brw_reg src1)
{
   const struct intel_device_info *devinfo = p->devinfo;
   brw_inst *insn = next_insn(p, BRW_OPCODE_MATH);

   assert(devinfo->ver >= 6);

   assert(dest.file == BRW_GENERAL_REGISTER_FILE ||
          (devinfo->ver >= 7 && dest.file == BRW_MESSAGE_REGISTER_FILE));

   assert(dest.hstride == BRW_HORIZONTAL_STRIDE_1);
   if (devinfo->ver == 6) {
      assert(src0.hstride == BRW_HORIZONTAL_STRIDE_1);
      assert(src1.hstride == BRW_HORIZONTAL_STRIDE_1);
   }

   if (function == BRW_MATH_FUNCTION_INT_DIV_QUOTIENT ||
       function == BRW_MATH_FUNCTION_INT_DIV_REMAINDER ||
       function == BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER) {
      assert(src0.type != BRW_REGISTER_TYPE_F);
      assert(src1.type != BRW_REGISTER_TYPE_F);
      assert(src1.file == BRW_GENERAL_REGISTER_FILE ||
             (devinfo->ver >= 8 && src1.file == BRW_IMMEDIATE_VALUE));
   } else {
      assert(src0.type == BRW_REGISTER_TYPE_F ||
             (src0.type == BRW_REGISTER_TYPE_HF && devinfo->ver >= 9));
      assert(src1.type == BRW_REGISTER_TYPE_F ||
             (src1.type == BRW_REGISTER_TYPE_HF && devinfo->ver >= 9));
   }

   /* Source modifiers are ignored for extended math instructions on Gfx6. */
   if (devinfo->ver == 6) {
      assert(!src0.negate);
      assert(!src0.abs);
      assert(!src1.negate);
      assert(!src1.abs);
   }

   brw_inst_set_math_function(devinfo, insn, function);

   brw_set_dest(p, insn, dest);
   brw_set_src0(p, insn, src0);
   brw_set_src1(p, insn, src1);
}

/**
 * Return the right surface index to access the thread scratch space using
 * stateless dataport messages.
 */
unsigned
brw_scratch_surface_idx(const struct brw_codegen *p)
{
   /* The scratch space is thread-local so IA coherency is unnecessary. */
   if (p->devinfo->ver >= 8)
      return GFX8_BTI_STATELESS_NON_COHERENT;
   else
      return BRW_BTI_STATELESS;
}

/**
 * Write a block of OWORDs (half a GRF each) from the scratch buffer,
 * using a constant offset per channel.
 *
 * The offset must be aligned to oword size (16 bytes).  Used for
 * register spilling.
 */
void brw_oword_block_write_scratch(struct brw_codegen *p,
				   struct brw_reg mrf,
				   int num_regs,
				   unsigned offset)
{
   const struct intel_device_info *devinfo = p->devinfo;
   const unsigned target_cache =
      (devinfo->ver >= 7 ? GFX7_SFID_DATAPORT_DATA_CACHE :
       devinfo->ver >= 6 ? GFX6_SFID_DATAPORT_RENDER_CACHE :
       BRW_SFID_DATAPORT_WRITE);
   const struct tgl_swsb swsb = brw_get_default_swsb(p);
   uint32_t msg_type;

   if (devinfo->ver >= 6)
      offset /= 16;

   mrf = retype(mrf, BRW_REGISTER_TYPE_UD);

   const unsigned mlen = 1 + num_regs;

   /* Set up the message header.  This is g0, with g0.2 filled with
    * the offset.  We don't want to leave our offset around in g0 or
    * it'll screw up texture samples, so set it up inside the message
    * reg.
    */
   {
      brw_push_insn_state(p);
      brw_set_default_exec_size(p, BRW_EXECUTE_8);
      brw_set_default_mask_control(p, BRW_MASK_DISABLE);
      brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
      brw_set_default_swsb(p, tgl_swsb_src_dep(swsb));

      brw_MOV(p, mrf, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));

      /* set message header global offset field (reg 0, element 2) */
      brw_set_default_exec_size(p, BRW_EXECUTE_1);
      brw_set_default_swsb(p, tgl_swsb_null());
      brw_MOV(p,
	      retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE,
				  mrf.nr,
				  2), BRW_REGISTER_TYPE_UD),
	      brw_imm_ud(offset));

      brw_pop_insn_state(p);
      brw_set_default_swsb(p, tgl_swsb_dst_dep(swsb, 1));
   }

   {
      struct brw_reg dest;
      brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);
      int send_commit_msg;
      struct brw_reg src_header = retype(brw_vec8_grf(0, 0),
					 BRW_REGISTER_TYPE_UW);

      brw_inst_set_sfid(devinfo, insn, target_cache);
      brw_inst_set_compression(devinfo, insn, false);

      if (brw_inst_exec_size(devinfo, insn) >= 16)
	 src_header = vec16(src_header);

      assert(brw_inst_pred_control(devinfo, insn) == BRW_PREDICATE_NONE);
      if (devinfo->ver < 6)
         brw_inst_set_base_mrf(devinfo, insn, mrf.nr);

      /* Until gfx6, writes followed by reads from the same location
       * are not guaranteed to be ordered unless write_commit is set.
       * If set, then a no-op write is issued to the destination
       * register to set a dependency, and a read from the destination
       * can be used to ensure the ordering.
       *
       * For gfx6, only writes between different threads need ordering
       * protection.  Our use of DP writes is all about register
       * spilling within a thread.
       */
      if (devinfo->ver >= 6) {
	 dest = retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW);
	 send_commit_msg = 0;
      } else {
	 dest = src_header;
	 send_commit_msg = 1;
      }

      brw_set_dest(p, insn, dest);
      if (devinfo->ver >= 6) {
	 brw_set_src0(p, insn, mrf);
      } else {
	 brw_set_src0(p, insn, brw_null_reg());
      }

      if (devinfo->ver >= 6)
	 msg_type = GFX6_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE;
      else
	 msg_type = BRW_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE;

      brw_set_desc(p, insn,
                   brw_message_desc(devinfo, mlen, send_commit_msg, true) |
                   brw_dp_write_desc(devinfo, brw_scratch_surface_idx(p),
                                     BRW_DATAPORT_OWORD_BLOCK_DWORDS(num_regs * 8),
                                     msg_type, send_commit_msg));
   }
}


/**
 * Read a block of owords (half a GRF each) from the scratch buffer
 * using a constant index per channel.
 *
 * Offset must be aligned to oword size (16 bytes).  Used for register
 * spilling.
 */
void
brw_oword_block_read_scratch(struct brw_codegen *p,
			     struct brw_reg dest,
			     struct brw_reg mrf,
			     int num_regs,
			     unsigned offset)
{
   const struct intel_device_info *devinfo = p->devinfo;
   const struct tgl_swsb swsb = brw_get_default_swsb(p);

   if (devinfo->ver >= 6)
      offset /= 16;

   if (p->devinfo->ver >= 7) {
      /* On gen 7 and above, we no longer have message registers and we can
       * send from any register we want.  By using the destination register
       * for the message, we guarantee that the implied message write won't
       * accidentally overwrite anything.  This has been a problem because
       * the MRF registers and source for the final FB write are both fixed
       * and may overlap.
       */
      mrf = retype(dest, BRW_REGISTER_TYPE_UD);
   } else {
      mrf = retype(mrf, BRW_REGISTER_TYPE_UD);
   }
   dest = retype(dest, BRW_REGISTER_TYPE_UW);

   const unsigned rlen = num_regs;
   const unsigned target_cache =
      (devinfo->ver >= 7 ? GFX7_SFID_DATAPORT_DATA_CACHE :
       devinfo->ver >= 6 ? GFX6_SFID_DATAPORT_RENDER_CACHE :
       BRW_SFID_DATAPORT_READ);

   {
      brw_push_insn_state(p);
      brw_set_default_swsb(p, tgl_swsb_src_dep(swsb));
      brw_set_default_exec_size(p, BRW_EXECUTE_8);
      brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
      brw_set_default_mask_control(p, BRW_MASK_DISABLE);

      brw_MOV(p, mrf, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));

      /* set message header global offset field (reg 0, element 2) */
      brw_set_default_exec_size(p, BRW_EXECUTE_1);
      brw_set_default_swsb(p, tgl_swsb_null());
      brw_MOV(p, get_element_ud(mrf, 2), brw_imm_ud(offset));

      brw_pop_insn_state(p);
      brw_set_default_swsb(p, tgl_swsb_dst_dep(swsb, 1));
   }

   {
      brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);

      brw_inst_set_sfid(devinfo, insn, target_cache);
      assert(brw_inst_pred_control(devinfo, insn) == 0);
      brw_inst_set_compression(devinfo, insn, false);

      brw_set_dest(p, insn, dest);	/* UW? */
      if (devinfo->ver >= 6) {
	 brw_set_src0(p, insn, mrf);
      } else {
	 brw_set_src0(p, insn, brw_null_reg());
         brw_inst_set_base_mrf(devinfo, insn, mrf.nr);
      }

      brw_set_desc(p, insn,
                   brw_message_desc(devinfo, 1, rlen, true) |
                   brw_dp_read_desc(devinfo, brw_scratch_surface_idx(p),
                                    BRW_DATAPORT_OWORD_BLOCK_DWORDS(num_regs * 8),
                                    BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ,
                                    BRW_DATAPORT_READ_TARGET_RENDER_CACHE));
   }
}

void
gfx7_block_read_scratch(struct brw_codegen *p,
                        struct brw_reg dest,
                        int num_regs,
                        unsigned offset)
{
   brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);
   assert(brw_inst_pred_control(p->devinfo, insn) == BRW_PREDICATE_NONE);

   brw_set_dest(p, insn, retype(dest, BRW_REGISTER_TYPE_UW));

   /* The HW requires that the header is present; this is to get the g0.5
    * scratch offset.
    */
   brw_set_src0(p, insn, brw_vec8_grf(0, 0));

   /* According to the docs, offset is "A 12-bit HWord offset into the memory
    * Immediate Memory buffer as specified by binding table 0xFF."  An HWORD
    * is 32 bytes, which happens to be the size of a register.
    */
   offset /= REG_SIZE;
   assert(offset < (1 << 12));

   gfx7_set_dp_scratch_message(p, insn,
                               false, /* scratch read */
                               false, /* OWords */
                               false, /* invalidate after read */
                               num_regs,
                               offset,
                               1,        /* mlen: just g0 */
                               num_regs, /* rlen */
                               true);    /* header present */
}

/**
 * Read float[4] vectors from the data port constant cache.
 * Location (in buffer) should be a multiple of 16.
 * Used for fetching shader constants.
 */
void brw_oword_block_read(struct brw_codegen *p,
			  struct brw_reg dest,
			  struct brw_reg mrf,
			  uint32_t offset,
			  uint32_t bind_table_index)
{
   const struct intel_device_info *devinfo = p->devinfo;
   const unsigned target_cache =
      (devinfo->ver >= 6 ? GFX6_SFID_DATAPORT_CONSTANT_CACHE :
       BRW_SFID_DATAPORT_READ);
   const unsigned exec_size = 1 << brw_get_default_exec_size(p);
   const struct tgl_swsb swsb = brw_get_default_swsb(p);

   /* On newer hardware, offset is in units of owords. */
   if (devinfo->ver >= 6)
      offset /= 16;

   mrf = retype(mrf, BRW_REGISTER_TYPE_UD);

   brw_push_insn_state(p);
   brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
   brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
   brw_set_default_mask_control(p, BRW_MASK_DISABLE);

   brw_push_insn_state(p);
   brw_set_default_exec_size(p, BRW_EXECUTE_8);
   brw_set_default_swsb(p, tgl_swsb_src_dep(swsb));
   brw_MOV(p, mrf, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));

   /* set message header global offset field (reg 0, element 2) */
   brw_set_default_exec_size(p, BRW_EXECUTE_1);
   brw_set_default_swsb(p, tgl_swsb_null());
   brw_MOV(p,
	   retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE,
			       mrf.nr,
			       2), BRW_REGISTER_TYPE_UD),
	   brw_imm_ud(offset));
   brw_pop_insn_state(p);

   brw_set_default_swsb(p, tgl_swsb_dst_dep(swsb, 1));

   brw_inst *insn = next_insn(p, BRW_OPCODE_SEND);

   brw_inst_set_sfid(devinfo, insn, target_cache);

   /* cast dest to a uword[8] vector */
   dest = retype(vec8(dest), BRW_REGISTER_TYPE_UW);

   brw_set_dest(p, insn, dest);
   if (devinfo->ver >= 6) {
      brw_set_src0(p, insn, mrf);
   } else {
      brw_set_src0(p, insn, brw_null_reg());
      brw_inst_set_base_mrf(devinfo, insn, mrf.nr);
   }

   brw_set_desc(p, insn,
                brw_message_desc(devinfo, 1, DIV_ROUND_UP(exec_size, 8), true) |
                brw_dp_read_desc(devinfo, bind_table_index,
                                 BRW_DATAPORT_OWORD_BLOCK_DWORDS(exec_size),
                                 BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ,
                                 BRW_DATAPORT_READ_TARGET_DATA_CACHE));

   brw_pop_insn_state(p);
}

brw_inst *
brw_fb_WRITE(struct brw_codegen *p,
             struct brw_reg payload,
             struct brw_reg implied_header,
             unsigned msg_control,
             unsigned binding_table_index,
             unsigned msg_length,
             unsigned response_length,
             bool eot,
             bool last_render_target,
             bool header_present)
{
   const struct intel_device_info *devinfo = p->devinfo;
   const unsigned target_cache =
      (devinfo->ver >= 6 ? GFX6_SFID_DATAPORT_RENDER_CACHE :
       BRW_SFID_DATAPORT_WRITE);
   brw_inst *insn;
   struct brw_reg dest, src0;

   if (brw_get_default_exec_size(p) >= BRW_EXECUTE_16)
      dest = retype(vec16(brw_null_reg()), BRW_REGISTER_TYPE_UW);
   else
      dest = retype(vec8(brw_null_reg()), BRW_REGISTER_TYPE_UW);

   if (devinfo->ver >= 6) {
      insn = next_insn(p, BRW_OPCODE_SENDC);
   } else {
      insn = next_insn(p, BRW_OPCODE_SEND);
   }
   brw_inst_set_sfid(devinfo, insn, target_cache);
   brw_inst_set_compression(devinfo, insn, false);

   if (devinfo->ver >= 6) {
      /* headerless version, just submit color payload */
      src0 = payload;
   } else {
      assert(payload.file == BRW_MESSAGE_REGISTER_FILE);
      brw_inst_set_base_mrf(devinfo, insn, payload.nr);
      src0 = implied_header;
   }

   brw_set_dest(p, insn, dest);
   brw_set_src0(p, insn, src0);
   brw_set_desc(p, insn,
                brw_message_desc(devinfo, msg_length, response_length,
                                 header_present) |
                brw_fb_write_desc(devinfo, binding_table_index, msg_control,
                                  last_render_target,
                                  false /* coarse_write */));
   brw_inst_set_eot(devinfo, insn, eot);

   return insn;
}

brw_inst *
gfx9_fb_READ(struct brw_codegen *p,
             struct brw_reg dst,
             struct brw_reg payload,
             unsigned binding_table_index,
             unsigned msg_length,
             unsigned response_length,
             bool per_sample)
{
   const struct intel_device_info *devinfo = p->devinfo;
   assert(devinfo->ver >= 9);
   brw_inst *insn = next_insn(p, BRW_OPCODE_SENDC);

   brw_inst_set_sfid(devinfo, insn, GFX6_SFID_DATAPORT_RENDER_CACHE);
   brw_set_dest(p, insn, dst);
   brw_set_src0(p, insn, payload);
   brw_set_desc(
      p, insn,
      brw_message_desc(devinfo, msg_length, response_length, true) |
      brw_fb_read_desc(devinfo, binding_table_index, 0 /* msg_control */,
                       1 << brw_get_default_exec_size(p), per_sample));
   brw_inst_set_rt_slot_group(devinfo, insn, brw_get_default_group(p) / 16);

   return insn;
}

/**
 * Texture sample instruction.
 * Note: the msg_type plus msg_length values determine exactly what kind
 * of sampling operation is performed.  See volume 4, page 161 of docs.
 */
void brw_SAMPLE(struct brw_codegen *p,
		struct brw_reg dest,
		unsigned msg_reg_nr,
		struct brw_reg src0,
		unsigned binding_table_index,
		unsigned sampler,
		unsigned msg_type,
		unsigned response_length,
		unsigned msg_length,
		unsigned header_present,
		unsigned simd_mode,
		unsigned return_format)
{
   const struct intel_device_info *devinfo = p->devinfo;
   brw_inst *insn;

   if (msg_reg_nr != -1)
      gfx6_resolve_implied_move(p, &src0, msg_reg_nr);

   insn = next_insn(p, BRW_OPCODE_SEND);
   brw_inst_set_sfid(devinfo, insn, BRW_SFID_SAMPLER);
   brw_inst_set_pred_control(devinfo, insn, BRW_PREDICATE_NONE); /* XXX */

   /* From the 965 PRM (volume 4, part 1, section 14.2.41):
    *
    *    "Instruction compression is not allowed for this instruction (that
    *     is, send). The hardware behavior is undefined if this instruction is
    *     set as compressed. However, compress control can be set to "SecHalf"
    *     to affect the EMask generation."
    *
    * No similar wording is found in later PRMs, but there are examples
    * utilizing send with SecHalf.  More importantly, SIMD8 sampler messages
    * are allowed in SIMD16 mode and they could not work without SecHalf.  For
    * these reasons, we allow BRW_COMPRESSION_2NDHALF here.
    */
   brw_inst_set_compression(devinfo, insn, false);

   if (devinfo->ver < 6)
      brw_inst_set_base_mrf(devinfo, insn, msg_reg_nr);

   brw_set_dest(p, insn, dest);
   brw_set_src0(p, insn, src0);
   brw_set_desc(p, insn,
                brw_message_desc(devinfo, msg_length, response_length,
                                 header_present) |
                brw_sampler_desc(devinfo, binding_table_index, sampler,
                                 msg_type, simd_mode, return_format));
}

/* Adjust the message header's sampler state pointer to
 * select the correct group of 16 samplers.
 */
void brw_adjust_sampler_state_pointer(struct brw_codegen *p,
                                      struct brw_reg header,
                                      struct brw_reg sampler_index)
{
   /* The "Sampler Index" field can only store values between 0 and 15.
    * However, we can add an offset to the "Sampler State Pointer"
    * field, effectively selecting a different set of 16 samplers.
    *
    * The "Sampler State Pointer" needs to be aligned to a 32-byte
    * offset, and each sampler state is only 16-bytes, so we can't
    * exclusively use the offset - we have to use both.
    */

   const struct intel_device_info *devinfo = p->devinfo;

   if (sampler_index.file == BRW_IMMEDIATE_VALUE) {
      const int sampler_state_size = 16; /* 16 bytes */
      uint32_t sampler = sampler_index.ud;

      if (sampler >= 16) {
         assert(devinfo->verx10 >= 75);
         brw_ADD(p,
                 get_element_ud(header, 3),
                 get_element_ud(brw_vec8_grf(0, 0), 3),
                 brw_imm_ud(16 * (sampler / 16) * sampler_state_size));
      }
   } else {
      /* Non-const sampler array indexing case */
      if (devinfo->verx10 <= 70) {
         return;
      }

      struct brw_reg temp = get_element_ud(header, 3);

      brw_push_insn_state(p);
      brw_AND(p, temp, get_element_ud(sampler_index, 0), brw_imm_ud(0x0f0));
      brw_set_default_swsb(p, tgl_swsb_regdist(1));
      brw_SHL(p, temp, temp, brw_imm_ud(4));
      brw_ADD(p,
              get_element_ud(header, 3),
              get_element_ud(brw_vec8_grf(0, 0), 3),
              temp);
      brw_pop_insn_state(p);
   }
}

/* All these variables are pretty confusing - we might be better off
 * using bitmasks and macros for this, in the old style.  Or perhaps
 * just having the caller instantiate the fields in dword3 itself.
 */
void brw_urb_WRITE(struct brw_codegen *p,
		   struct brw_reg dest,
		   unsigned msg_reg_nr,
		   struct brw_reg src0,
                   enum brw_urb_write_flags flags,
		   unsigned msg_length,
		   unsigned response_length,
		   unsigned offset,
		   unsigned swizzle)
{
   const struct intel_device_info *devinfo = p->devinfo;
   brw_inst *insn;

   gfx6_resolve_implied_move(p, &src0, msg_reg_nr);

   if (devinfo->ver >= 7 && !(flags & BRW_URB_WRITE_USE_CHANNEL_MASKS)) {
      /* Enable Channel Masks in the URB_WRITE_HWORD message header */
      brw_push_insn_state(p);
      brw_set_default_access_mode(p, BRW_ALIGN_1);
      brw_set_default_mask_control(p, BRW_MASK_DISABLE);
      brw_set_default_exec_size(p, BRW_EXECUTE_1);
      brw_OR(p, retype(brw_vec1_reg(BRW_MESSAGE_REGISTER_FILE, msg_reg_nr, 5),
		       BRW_REGISTER_TYPE_UD),
	        retype(brw_vec1_grf(0, 5), BRW_REGISTER_TYPE_UD),
		brw_imm_ud(0xff00));
      brw_pop_insn_state(p);
   }

   insn = next_insn(p, BRW_OPCODE_SEND);

   assert(msg_length < BRW_MAX_MRF(devinfo->ver));

   brw_set_dest(p, insn, dest);
   brw_set_src0(p, insn, src0);
   brw_set_src1(p, insn, brw_imm_d(0));

   if (devinfo->ver < 6)
      brw_inst_set_base_mrf(devinfo, insn, msg_reg_nr);

   brw_set_urb_message(p,
		       insn,
		       flags,
		       msg_length,
		       response_length,
		       offset,
		       swizzle);
}

void
brw_send_indirect_message(struct brw_codegen *p,
                          unsigned sfid,
                          struct brw_reg dst,
                          struct brw_reg payload,
                          struct brw_reg desc,
                          unsigned desc_imm,
                          bool eot)
{
   const struct intel_device_info *devinfo = p->devinfo;
   struct brw_inst *send;

   dst = retype(dst, BRW_REGISTER_TYPE_UW);

   assert(desc.type == BRW_REGISTER_TYPE_UD);

   if (desc.file == BRW_IMMEDIATE_VALUE) {
      send = next_insn(p, BRW_OPCODE_SEND);
      brw_set_src0(p, send, retype(payload, BRW_REGISTER_TYPE_UD));
      brw_set_desc(p, send, desc.ud | desc_imm);
   } else {
      const struct tgl_swsb swsb = brw_get_default_swsb(p);
      struct brw_reg addr = retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD);

      brw_push_insn_state(p);
      brw_set_default_access_mode(p, BRW_ALIGN_1);
      brw_set_default_mask_control(p, BRW_MASK_DISABLE);
      brw_set_default_exec_size(p, BRW_EXECUTE_1);
      brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
      brw_set_default_swsb(p, tgl_swsb_src_dep(swsb));

      /* Load the indirect descriptor to an address register using OR so the
       * caller can specify additional descriptor bits with the desc_imm
       * immediate.
       */
      brw_OR(p, addr, desc, brw_imm_ud(desc_imm));

      brw_pop_insn_state(p);

      brw_set_default_swsb(p, tgl_swsb_dst_dep(swsb, 1));
      send = next_insn(p, BRW_OPCODE_SEND);
      brw_set_src0(p, send, retype(payload, BRW_REGISTER_TYPE_UD));

      if (devinfo->ver >= 12)
         brw_inst_set_send_sel_reg32_desc(devinfo, send, true);
      else
         brw_set_src1(p, send, addr);
   }

   brw_set_dest(p, send, dst);
   brw_inst_set_sfid(devinfo, send, sfid);
   brw_inst_set_eot(devinfo, send, eot);
}

void
brw_send_indirect_split_message(struct brw_codegen *p,
                                unsigned sfid,
                                struct brw_reg dst,
                                struct brw_reg payload0,
                                struct brw_reg payload1,
                                struct brw_reg desc,
                                unsigned desc_imm,
                                struct brw_reg ex_desc,
                                unsigned ex_desc_imm,
                                bool eot)
{
   const struct intel_device_info *devinfo = p->devinfo;
   struct brw_inst *send;

   dst = retype(dst, BRW_REGISTER_TYPE_UW);

   assert(desc.type == BRW_REGISTER_TYPE_UD);

   if (desc.file == BRW_IMMEDIATE_VALUE) {
      desc.ud |= desc_imm;
   } else {
      const struct tgl_swsb swsb = brw_get_default_swsb(p);
      struct brw_reg addr = retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD);

      brw_push_insn_state(p);
      brw_set_default_access_mode(p, BRW_ALIGN_1);
      brw_set_default_mask_control(p, BRW_MASK_DISABLE);
      brw_set_default_exec_size(p, BRW_EXECUTE_1);
      brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
      brw_set_default_swsb(p, tgl_swsb_src_dep(swsb));

      /* Load the indirect descriptor to an address register using OR so the
       * caller can specify additional descriptor bits with the desc_imm
       * immediate.
       */
      brw_OR(p, addr, desc, brw_imm_ud(desc_imm));

      brw_pop_insn_state(p);
      desc = addr;

      brw_set_default_swsb(p, tgl_swsb_dst_dep(swsb, 1));
   }

   if (ex_desc.file == BRW_IMMEDIATE_VALUE &&
       (devinfo->ver >= 12 ||
        ((ex_desc.ud | ex_desc_imm) & INTEL_MASK(15, 12)) == 0)) {
      ex_desc.ud |= ex_desc_imm;
   } else {
      const struct tgl_swsb swsb = brw_get_default_swsb(p);
      struct brw_reg addr = retype(brw_address_reg(2), BRW_REGISTER_TYPE_UD);

      brw_push_insn_state(p);
      brw_set_default_access_mode(p, BRW_ALIGN_1);
      brw_set_default_mask_control(p, BRW_MASK_DISABLE);
      brw_set_default_exec_size(p, BRW_EXECUTE_1);
      brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
      brw_set_default_swsb(p, tgl_swsb_src_dep(swsb));

      /* Load the indirect extended descriptor to an address register using OR
       * so the caller can specify additional descriptor bits with the
       * desc_imm immediate.
       *
       * Even though the instruction dispatcher always pulls the SFID and EOT
       * fields from the instruction itself, actual external unit which
       * processes the message gets the SFID and EOT from the extended
       * descriptor which comes from the address register.  If we don't OR
       * those two bits in, the external unit may get confused and hang.
       */
      unsigned imm_part = ex_desc_imm | sfid | eot << 5;

      if (ex_desc.file == BRW_IMMEDIATE_VALUE) {
         /* ex_desc bits 15:12 don't exist in the instruction encoding prior
          * to Gfx12, so we may have fallen back to an indirect extended
          * descriptor.
          */
         brw_MOV(p, addr, brw_imm_ud(ex_desc.ud | imm_part));
      } else {
         brw_OR(p, addr, ex_desc, brw_imm_ud(imm_part));
      }

      brw_pop_insn_state(p);
      ex_desc = addr;

      brw_set_default_swsb(p, tgl_swsb_dst_dep(swsb, 1));
   }

   send = next_insn(p, devinfo->ver >= 12 ? BRW_OPCODE_SEND : BRW_OPCODE_SENDS);
   brw_set_dest(p, send, dst);
   brw_set_src0(p, send, retype(payload0, BRW_REGISTER_TYPE_UD));
   brw_set_src1(p, send, retype(payload1, BRW_REGISTER_TYPE_UD));

   if (desc.file == BRW_IMMEDIATE_VALUE) {
      brw_inst_set_send_sel_reg32_desc(devinfo, send, 0);
      brw_inst_set_send_desc(devinfo, send, desc.ud);
   } else {
      assert(desc.file == BRW_ARCHITECTURE_REGISTER_FILE);
      assert(desc.nr == BRW_ARF_ADDRESS);
      assert(desc.subnr == 0);
      brw_inst_set_send_sel_reg32_desc(devinfo, send, 1);
   }

   if (ex_desc.file == BRW_IMMEDIATE_VALUE) {
      brw_inst_set_send_sel_reg32_ex_desc(devinfo, send, 0);
      brw_inst_set_sends_ex_desc(devinfo, send, ex_desc.ud);
   } else {
      assert(ex_desc.file == BRW_ARCHITECTURE_REGISTER_FILE);
      assert(ex_desc.nr == BRW_ARF_ADDRESS);
      assert((ex_desc.subnr & 0x3) == 0);
      brw_inst_set_send_sel_reg32_ex_desc(devinfo, send, 1);
      brw_inst_set_send_ex_desc_ia_subreg_nr(devinfo, send, ex_desc.subnr >> 2);
   }

   brw_inst_set_sfid(devinfo, send, sfid);
   brw_inst_set_eot(devinfo, send, eot);
}

static void
brw_send_indirect_surface_message(struct brw_codegen *p,
                                  unsigned sfid,
                                  struct brw_reg dst,
                                  struct brw_reg payload,
                                  struct brw_reg surface,
                                  unsigned desc_imm)
{
   if (surface.file != BRW_IMMEDIATE_VALUE) {
      const struct tgl_swsb swsb = brw_get_default_swsb(p);
      struct brw_reg addr = retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD);

      brw_push_insn_state(p);
      brw_set_default_access_mode(p, BRW_ALIGN_1);
      brw_set_default_mask_control(p, BRW_MASK_DISABLE);
      brw_set_default_exec_size(p, BRW_EXECUTE_1);
      brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
      brw_set_default_swsb(p, tgl_swsb_src_dep(swsb));

      /* Mask out invalid bits from the surface index to avoid hangs e.g. when
       * some surface array is accessed out of bounds.
       */
      brw_AND(p, addr,
              suboffset(vec1(retype(surface, BRW_REGISTER_TYPE_UD)),
                        BRW_GET_SWZ(surface.swizzle, 0)),
              brw_imm_ud(0xff));

      brw_pop_insn_state(p);

      surface = addr;
      brw_set_default_swsb(p, tgl_swsb_dst_dep(swsb, 1));
   }

   brw_send_indirect_message(p, sfid, dst, payload, surface, desc_imm, false);
}

static bool
while_jumps_before_offset(const struct intel_device_info *devinfo,
                          brw_inst *insn, int while_offset, int start_offset)
{
   int scale = 16 / brw_jump_scale(devinfo);
   int jip = devinfo->ver == 6 ? brw_inst_gfx6_jump_count(devinfo, insn)
                               : brw_inst_jip(devinfo, insn);
   assert(jip < 0);
   return while_offset + jip * scale <= start_offset;
}


static int
brw_find_next_block_end(struct brw_codegen *p, int start_offset)
{
   int offset;
   void *store = p->store;
   const struct intel_device_info *devinfo = p->devinfo;

   int depth = 0;

   for (offset = next_offset(devinfo, store, start_offset);
        offset < p->next_insn_offset;
        offset = next_offset(devinfo, store, offset)) {
      brw_inst *insn = store + offset;

      switch (brw_inst_opcode(devinfo, insn)) {
      case BRW_OPCODE_IF:
         depth++;
         break;
      case BRW_OPCODE_ENDIF:
         if (depth == 0)
            return offset;
         depth--;
         break;
      case BRW_OPCODE_WHILE:
         /* If the while doesn't jump before our instruction, it's the end
          * of a sibling do...while loop.  Ignore it.
          */
         if (!while_jumps_before_offset(devinfo, insn, offset, start_offset))
            continue;
         FALLTHROUGH;
      case BRW_OPCODE_ELSE:
      case BRW_OPCODE_HALT:
         if (depth == 0)
            return offset;
         break;
      default:
         break;
      }
   }

   return 0;
}

/* There is no DO instruction on gfx6, so to find the end of the loop
 * we have to see if the loop is jumping back before our start
 * instruction.
 */
static int
brw_find_loop_end(struct brw_codegen *p, int start_offset)
{
   const struct intel_device_info *devinfo = p->devinfo;
   int offset;
   void *store = p->store;

   assert(devinfo->ver >= 6);

   /* Always start after the instruction (such as a WHILE) we're trying to fix
    * up.
    */
   for (offset = next_offset(devinfo, store, start_offset);
        offset < p->next_insn_offset;
        offset = next_offset(devinfo, store, offset)) {
      brw_inst *insn = store + offset;

      if (brw_inst_opcode(devinfo, insn) == BRW_OPCODE_WHILE) {
	 if (while_jumps_before_offset(devinfo, insn, offset, start_offset))
	    return offset;
      }
   }
   assert(!"not reached");
   return start_offset;
}

/* After program generation, go back and update the UIP and JIP of
 * BREAK, CONT, and HALT instructions to their correct locations.
 */
void
brw_set_uip_jip(struct brw_codegen *p, int start_offset)
{
   const struct intel_device_info *devinfo = p->devinfo;
   int offset;
   int br = brw_jump_scale(devinfo);
   int scale = 16 / br;
   void *store = p->store;

   if (devinfo->ver < 6)
      return;

   for (offset = start_offset; offset < p->next_insn_offset; offset += 16) {
      brw_inst *insn = store + offset;
      assert(brw_inst_cmpt_control(devinfo, insn) == 0);

      int block_end_offset = brw_find_next_block_end(p, offset);
      switch (brw_inst_opcode(devinfo, insn)) {
      case BRW_OPCODE_BREAK:
         assert(block_end_offset != 0);
         brw_inst_set_jip(devinfo, insn, (block_end_offset - offset) / scale);
	 /* Gfx7 UIP points to WHILE; Gfx6 points just after it */
         brw_inst_set_uip(devinfo, insn,
	    (brw_find_loop_end(p, offset) - offset +
             (devinfo->ver == 6 ? 16 : 0)) / scale);
	 break;
      case BRW_OPCODE_CONTINUE:
         assert(block_end_offset != 0);
         brw_inst_set_jip(devinfo, insn, (block_end_offset - offset) / scale);
         brw_inst_set_uip(devinfo, insn,
            (brw_find_loop_end(p, offset) - offset) / scale);

         assert(brw_inst_uip(devinfo, insn) != 0);
         assert(brw_inst_jip(devinfo, insn) != 0);
	 break;

      case BRW_OPCODE_ENDIF: {
         int32_t jump = (block_end_offset == 0) ?
                        1 * br : (block_end_offset - offset) / scale;
         if (devinfo->ver >= 7)
            brw_inst_set_jip(devinfo, insn, jump);
         else
            brw_inst_set_gfx6_jump_count(devinfo, insn, jump);
	 break;
      }

      case BRW_OPCODE_HALT:
	 /* From the Sandy Bridge PRM (volume 4, part 2, section 8.3.19):
	  *
	  *    "In case of the halt instruction not inside any conditional
	  *     code block, the value of <JIP> and <UIP> should be the
	  *     same. In case of the halt instruction inside conditional code
	  *     block, the <UIP> should be the end of the program, and the
	  *     <JIP> should be end of the most inner conditional code block."
	  *
	  * The uip will have already been set by whoever set up the
	  * instruction.
	  */
	 if (block_end_offset == 0) {
            brw_inst_set_jip(devinfo, insn, brw_inst_uip(devinfo, insn));
	 } else {
            brw_inst_set_jip(devinfo, insn, (block_end_offset - offset) / scale);
	 }
         assert(brw_inst_uip(devinfo, insn) != 0);
         assert(brw_inst_jip(devinfo, insn) != 0);
	 break;

      default:
         break;
      }
   }
}

void brw_ff_sync(struct brw_codegen *p,
		   struct brw_reg dest,
		   unsigned msg_reg_nr,
		   struct brw_reg src0,
		   bool allocate,
		   unsigned response_length,
		   bool eot)
{
   const struct intel_device_info *devinfo = p->devinfo;
   brw_inst *insn;

   gfx6_resolve_implied_move(p, &src0, msg_reg_nr);

   insn = next_insn(p, BRW_OPCODE_SEND);
   brw_set_dest(p, insn, dest);
   brw_set_src0(p, insn, src0);
   brw_set_src1(p, insn, brw_imm_d(0));

   if (devinfo->ver < 6)
      brw_inst_set_base_mrf(devinfo, insn, msg_reg_nr);

   brw_set_ff_sync_message(p,
			   insn,
			   allocate,
			   response_length,
			   eot);
}

/**
 * Emit the SEND instruction necessary to generate stream output data on Gfx6
 * (for transform feedback).
 *
 * If send_commit_msg is true, this is the last piece of stream output data
 * from this thread, so send the data as a committed write.  According to the
 * Sandy Bridge PRM (volume 2 part 1, section 4.5.1):
 *
 *   "Prior to End of Thread with a URB_WRITE, the kernel must ensure all
 *   writes are complete by sending the final write as a committed write."
 */
void
brw_svb_write(struct brw_codegen *p,
              struct brw_reg dest,
              unsigned msg_reg_nr,
              struct brw_reg src0,
              unsigned binding_table_index,
              bool   send_commit_msg)
{
   const struct intel_device_info *devinfo = p->devinfo;
   assert(devinfo->ver == 6);
   const unsigned target_cache = GFX6_SFID_DATAPORT_RENDER_CACHE;
   brw_inst *insn;

   gfx6_resolve_implied_move(p, &src0, msg_reg_nr);

   insn = next_insn(p, BRW_OPCODE_SEND);
   brw_inst_set_sfid(devinfo, insn, target_cache);
   brw_set_dest(p, insn, dest);
   brw_set_src0(p, insn, src0);
   brw_set_desc(p, insn,
                brw_message_desc(devinfo, 1, send_commit_msg, true) |
                brw_dp_write_desc(devinfo, binding_table_index,
                                  0, /* msg_control: ignored */
                                  GFX6_DATAPORT_WRITE_MESSAGE_STREAMED_VB_WRITE,
                                  send_commit_msg)); /* send_commit_msg */
}

static unsigned
brw_surface_payload_size(unsigned num_channels,
                         unsigned exec_size /**< 0 for SIMD4x2 */)
{
   if (exec_size == 0)
      return 1; /* SIMD4x2 */
   else if (exec_size <= 8)
      return num_channels;
   else
      return 2 * num_channels;
}

void
brw_untyped_atomic(struct brw_codegen *p,
                   struct brw_reg dst,
                   struct brw_reg payload,
                   struct brw_reg surface,
                   unsigned atomic_op,
                   unsigned msg_length,
                   bool response_expected,
                   bool header_present)
{
   const struct intel_device_info *devinfo = p->devinfo;
   const unsigned sfid = (devinfo->verx10 >= 75 ?
                          HSW_SFID_DATAPORT_DATA_CACHE_1 :
                          GFX7_SFID_DATAPORT_DATA_CACHE);
   const bool align1 = brw_get_default_access_mode(p) == BRW_ALIGN_1;
   /* SIMD4x2 untyped atomic instructions only exist on HSW+ */
   const bool has_simd4x2 = devinfo->verx10 >= 75;
   const unsigned exec_size = align1 ? 1 << brw_get_default_exec_size(p) :
                              has_simd4x2 ? 0 : 8;
   const unsigned response_length =
      brw_surface_payload_size(response_expected, exec_size);
   const unsigned desc =
      brw_message_desc(devinfo, msg_length, response_length, header_present) |
      brw_dp_untyped_atomic_desc(devinfo, exec_size, atomic_op,
                                 response_expected);
   /* Mask out unused components -- This is especially important in Align16
    * mode on generations that don't have native support for SIMD4x2 atomics,
    * because unused but enabled components will cause the dataport to perform
    * additional atomic operations on the addresses that happen to be in the
    * uninitialized Y, Z and W coordinates of the payload.
    */
   const unsigned mask = align1 ? WRITEMASK_XYZW : WRITEMASK_X;

   brw_send_indirect_surface_message(p, sfid, brw_writemask(dst, mask),
                                     payload, surface, desc);
}

void
brw_untyped_surface_read(struct brw_codegen *p,
                         struct brw_reg dst,
                         struct brw_reg payload,
                         struct brw_reg surface,
                         unsigned msg_length,
                         unsigned num_channels)
{
   const struct intel_device_info *devinfo = p->devinfo;
   const unsigned sfid = (devinfo->verx10 >= 75 ?
                          HSW_SFID_DATAPORT_DATA_CACHE_1 :
                          GFX7_SFID_DATAPORT_DATA_CACHE);
   const bool align1 = brw_get_default_access_mode(p) == BRW_ALIGN_1;
   const unsigned exec_size = align1 ? 1 << brw_get_default_exec_size(p) : 0;
   const unsigned response_length =
      brw_surface_payload_size(num_channels, exec_size);
   const unsigned desc =
      brw_message_desc(devinfo, msg_length, response_length, false) |
      brw_dp_untyped_surface_rw_desc(devinfo, exec_size, num_channels, false);

   brw_send_indirect_surface_message(p, sfid, dst, payload, surface, desc);
}

void
brw_untyped_surface_write(struct brw_codegen *p,
                          struct brw_reg payload,
                          struct brw_reg surface,
                          unsigned msg_length,
                          unsigned num_channels,
                          bool header_present)
{
   const struct intel_device_info *devinfo = p->devinfo;
   const unsigned sfid = (devinfo->verx10 >= 75 ?
                          HSW_SFID_DATAPORT_DATA_CACHE_1 :
                          GFX7_SFID_DATAPORT_DATA_CACHE);
   const bool align1 = brw_get_default_access_mode(p) == BRW_ALIGN_1;
   /* SIMD4x2 untyped surface write instructions only exist on HSW+ */
   const bool has_simd4x2 = devinfo->verx10 >= 75;
   const unsigned exec_size = align1 ? 1 << brw_get_default_exec_size(p) :
                              has_simd4x2 ? 0 : 8;
   const unsigned desc =
      brw_message_desc(devinfo, msg_length, 0, header_present) |
      brw_dp_untyped_surface_rw_desc(devinfo, exec_size, num_channels, true);
   /* Mask out unused components -- See comment in brw_untyped_atomic(). */
   const unsigned mask = !has_simd4x2 && !align1 ? WRITEMASK_X : WRITEMASK_XYZW;

   brw_send_indirect_surface_message(p, sfid, brw_writemask(brw_null_reg(), mask),
                                     payload, surface, desc);
}

static void
brw_set_memory_fence_message(struct brw_codegen *p,
                             struct brw_inst *insn,
                             enum brw_message_target sfid,
                             bool commit_enable,
                             unsigned bti)
{
   const struct intel_device_info *devinfo = p->devinfo;

   brw_set_desc(p, insn, brw_message_desc(
                   devinfo, 1, (commit_enable ? 1 : 0), true));

   brw_inst_set_sfid(devinfo, insn, sfid);

   switch (sfid) {
   case GFX6_SFID_DATAPORT_RENDER_CACHE:
      brw_inst_set_dp_msg_type(devinfo, insn, GFX7_DATAPORT_RC_MEMORY_FENCE);
      break;
   case GFX7_SFID_DATAPORT_DATA_CACHE:
      brw_inst_set_dp_msg_type(devinfo, insn, GFX7_DATAPORT_DC_MEMORY_FENCE);
      break;
   default:
      unreachable("Not reached");
   }

   if (commit_enable)
      brw_inst_set_dp_msg_control(devinfo, insn, 1 << 5);

   assert(devinfo->ver >= 11 || bti == 0);
   brw_inst_set_binding_table_index(devinfo, insn, bti);
}

static void
gfx12_set_memory_fence_message(struct brw_codegen *p,
                               struct brw_inst *insn,
                               enum brw_message_target sfid)
{
   const unsigned mlen = 1; /* g0 header */
    /* Completion signaled by write to register. No data returned. */
   const unsigned rlen = 1;

   brw_inst_set_sfid(p->devinfo, insn, sfid);

   enum lsc_fence_scope scope = LSC_FENCE_THREADGROUP;
   enum lsc_flush_type flush_type = LSC_FLUSH_TYPE_NONE;

   if (sfid == GFX12_SFID_TGM) {
      scope = LSC_FENCE_GPU;
      flush_type = LSC_FLUSH_TYPE_EVICT;
   }

   brw_set_desc(p, insn, lsc_fence_msg_desc(p->devinfo, scope,
                                            flush_type, false) |
                         brw_message_desc(p->devinfo, mlen, rlen, false));
}

void
brw_memory_fence(struct brw_codegen *p,
                 struct brw_reg dst,
                 struct brw_reg src,
                 enum opcode send_op,
                 enum brw_message_target sfid,
                 bool commit_enable,
                 unsigned bti)
{
   const struct intel_device_info *devinfo = p->devinfo;

   dst = retype(vec1(dst), BRW_REGISTER_TYPE_UW);
   src = retype(vec1(src), BRW_REGISTER_TYPE_UD);

   /* Set dst as destination for dependency tracking, the MEMORY_FENCE
    * message doesn't write anything back.
    */
   struct brw_inst *insn = next_insn(p, send_op);
   brw_inst_set_mask_control(devinfo, insn, BRW_MASK_DISABLE);
   brw_inst_set_exec_size(devinfo, insn, BRW_EXECUTE_1);
   brw_set_dest(p, insn, dst);
   brw_set_src0(p, insn, src);

   /* All DG2 hardware requires LSC for fence messages, even A-step */
   if (devinfo->has_lsc)
      gfx12_set_memory_fence_message(p, insn, sfid);
   else
      brw_set_memory_fence_message(p, insn, sfid, commit_enable, bti);
}

void
brw_pixel_interpolator_query(struct brw_codegen *p,
                             struct brw_reg dest,
                             struct brw_reg mrf,
                             bool noperspective,
                             bool coarse_pixel_rate,
                             unsigned mode,
                             struct brw_reg data,
                             unsigned msg_length,
                             unsigned response_length)
{
   const struct intel_device_info *devinfo = p->devinfo;
   const uint16_t exec_size = brw_get_default_exec_size(p);
   const unsigned slot_group = brw_get_default_group(p) / 16;
   const unsigned simd_mode = (exec_size == BRW_EXECUTE_16);
   const unsigned desc =
      brw_message_desc(devinfo, msg_length, response_length, false) |
      brw_pixel_interp_desc(devinfo, mode, noperspective, coarse_pixel_rate,
                            simd_mode, slot_group);

   /* brw_send_indirect_message will automatically use a direct send message
    * if data is actually immediate.
    */
   brw_send_indirect_message(p,
                             GFX7_SFID_PIXEL_INTERPOLATOR,
                             dest,
                             mrf,
                             vec1(data),
                             desc,
                             false);
}

void
brw_find_live_channel(struct brw_codegen *p, struct brw_reg dst,
                      struct brw_reg mask)
{
   const struct intel_device_info *devinfo = p->devinfo;
   const unsigned exec_size = 1 << brw_get_default_exec_size(p);
   const unsigned qtr_control = brw_get_default_group(p) / 8;
   brw_inst *inst;

   assert(devinfo->ver >= 7);
   assert(mask.type == BRW_REGISTER_TYPE_UD);

   brw_push_insn_state(p);

   /* The flag register is only used on Gfx7 in align1 mode, so avoid setting
    * unnecessary bits in the instruction words, get the information we need
    * and reset the default flag register. This allows more instructions to be
    * compacted.
    */
   const unsigned flag_subreg = p->current->flag_subreg;
   brw_set_default_flag_reg(p, 0, 0);

   if (brw_get_default_access_mode(p) == BRW_ALIGN_1) {
      brw_set_default_mask_control(p, BRW_MASK_DISABLE);

      if (devinfo->ver >= 8) {
         /* Getting the first active channel index is easy on Gfx8: Just find
          * the first bit set in the execution mask.  The register exists on
          * HSW already but it reads back as all ones when the current
          * instruction has execution masking disabled, so it's kind of
          * useless.
          */
         struct brw_reg exec_mask =
            retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD);

         brw_set_default_exec_size(p, BRW_EXECUTE_1);
         if (mask.file != BRW_IMMEDIATE_VALUE || mask.ud != 0xffffffff) {
            /* Unfortunately, ce0 does not take into account the thread
             * dispatch mask, which may be a problem in cases where it's not
             * tightly packed (i.e. it doesn't have the form '2^n - 1' for
             * some n).  Combine ce0 with the given dispatch (or vector) mask
             * to mask off those channels which were never dispatched by the
             * hardware.
             */
            brw_SHR(p, vec1(dst), mask, brw_imm_ud(qtr_control * 8));
            brw_set_default_swsb(p, tgl_swsb_regdist(1));
            brw_AND(p, vec1(dst), exec_mask, vec1(dst));
            exec_mask = vec1(dst);
         }

         /* Quarter control has the effect of magically shifting the value of
          * ce0 so you'll get the first active channel relative to the
          * specified quarter control as result.
          */
         inst = brw_FBL(p, vec1(dst), exec_mask);
      } else {
         const struct brw_reg flag = brw_flag_subreg(flag_subreg);

         brw_set_default_exec_size(p, BRW_EXECUTE_1);
         brw_MOV(p, retype(flag, BRW_REGISTER_TYPE_UD), brw_imm_ud(0));

         /* Run enough instructions returning zero with execution masking and
          * a conditional modifier enabled in order to get the full execution
          * mask in f1.0.  We could use a single 32-wide move here if it
          * weren't because of the hardware bug that causes channel enables to
          * be applied incorrectly to the second half of 32-wide instructions
          * on Gfx7.
          */
         const unsigned lower_size = MIN2(16, exec_size);
         for (unsigned i = 0; i < exec_size / lower_size; i++) {
            inst = brw_MOV(p, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW),
                           brw_imm_uw(0));
            brw_inst_set_mask_control(devinfo, inst, BRW_MASK_ENABLE);
            brw_inst_set_group(devinfo, inst, lower_size * i + 8 * qtr_control);
            brw_inst_set_cond_modifier(devinfo, inst, BRW_CONDITIONAL_Z);
            brw_inst_set_exec_size(devinfo, inst, cvt(lower_size) - 1);
            brw_inst_set_flag_reg_nr(devinfo, inst, flag_subreg / 2);
            brw_inst_set_flag_subreg_nr(devinfo, inst, flag_subreg % 2);
         }

         /* Find the first bit set in the exec_size-wide portion of the flag
          * register that was updated by the last sequence of MOV
          * instructions.
          */
         const enum brw_reg_type type = brw_int_type(exec_size / 8, false);
         brw_set_default_exec_size(p, BRW_EXECUTE_1);
         brw_FBL(p, vec1(dst), byte_offset(retype(flag, type), qtr_control));
      }
   } else {
      brw_set_default_mask_control(p, BRW_MASK_DISABLE);

      if (devinfo->ver >= 8 &&
          mask.file == BRW_IMMEDIATE_VALUE && mask.ud == 0xffffffff) {
         /* In SIMD4x2 mode the first active channel index is just the
          * negation of the first bit of the mask register.  Note that ce0
          * doesn't take into account the dispatch mask, so the Gfx7 path
          * should be used instead unless you have the guarantee that the
          * dispatch mask is tightly packed (i.e. it has the form '2^n - 1'
          * for some n).
          */
         inst = brw_AND(p, brw_writemask(dst, WRITEMASK_X),
                        negate(retype(brw_mask_reg(0), BRW_REGISTER_TYPE_UD)),
                        brw_imm_ud(1));

      } else {
         /* Overwrite the destination without and with execution masking to
          * find out which of the channels is active.
          */
         brw_push_insn_state(p);
         brw_set_default_exec_size(p, BRW_EXECUTE_4);
         brw_MOV(p, brw_writemask(vec4(dst), WRITEMASK_X),
                 brw_imm_ud(1));

         inst = brw_MOV(p, brw_writemask(vec4(dst), WRITEMASK_X),
                        brw_imm_ud(0));
         brw_pop_insn_state(p);
         brw_inst_set_mask_control(devinfo, inst, BRW_MASK_ENABLE);
      }
   }

   brw_pop_insn_state(p);
}

void
brw_broadcast(struct brw_codegen *p,
              struct brw_reg dst,
              struct brw_reg src,
              struct brw_reg idx)
{
   const struct intel_device_info *devinfo = p->devinfo;
   const bool align1 = brw_get_default_access_mode(p) == BRW_ALIGN_1;
   brw_inst *inst;

   brw_push_insn_state(p);
   brw_set_default_mask_control(p, BRW_MASK_DISABLE);
   brw_set_default_exec_size(p, align1 ? BRW_EXECUTE_1 : BRW_EXECUTE_4);

   assert(src.file == BRW_GENERAL_REGISTER_FILE &&
          src.address_mode == BRW_ADDRESS_DIRECT);
   assert(!src.abs && !src.negate);
   assert(src.type == dst.type);

   if ((src.vstride == 0 && (src.hstride == 0 || !align1)) ||
       idx.file == BRW_IMMEDIATE_VALUE) {
      /* Trivial, the source is already uniform or the index is a constant.
       * We will typically not get here if the optimizer is doing its job, but
       * asserting would be mean.
       */
      const unsigned i = idx.file == BRW_IMMEDIATE_VALUE ? idx.ud : 0;
      src = align1 ? stride(suboffset(src, i), 0, 1, 0) :
                     stride(suboffset(src, 4 * i), 0, 4, 1);

      if (type_sz(src.type) > 4 && !devinfo->has_64bit_float) {
         brw_MOV(p, subscript(dst, BRW_REGISTER_TYPE_D, 0),
                    subscript(src, BRW_REGISTER_TYPE_D, 0));
         brw_set_default_swsb(p, tgl_swsb_null());
         brw_MOV(p, subscript(dst, BRW_REGISTER_TYPE_D, 1),
                    subscript(src, BRW_REGISTER_TYPE_D, 1));
      } else {
         brw_MOV(p, dst, src);
      }
   } else {
      /* From the Haswell PRM section "Register Region Restrictions":
       *
       *    "The lower bits of the AddressImmediate must not overflow to
       *    change the register address.  The lower 5 bits of Address
       *    Immediate when added to lower 5 bits of address register gives
       *    the sub-register offset. The upper bits of Address Immediate
       *    when added to upper bits of address register gives the register
       *    address. Any overflow from sub-register offset is dropped."
       *
       * Fortunately, for broadcast, we never have a sub-register offset so
       * this isn't an issue.
       */
      assert(src.subnr == 0);

      if (align1) {
         const struct brw_reg addr =
            retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD);
         unsigned offset = src.nr * REG_SIZE + src.subnr;
         /* Limit in bytes of the signed indirect addressing immediate. */
         const unsigned limit = 512;

         brw_push_insn_state(p);
         brw_set_default_mask_control(p, BRW_MASK_DISABLE);
         brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);

         /* Take into account the component size and horizontal stride. */
         assert(src.vstride == src.hstride + src.width);
         brw_SHL(p, addr, vec1(idx),
                 brw_imm_ud(util_logbase2(type_sz(src.type)) +
                            src.hstride - 1));

         /* We can only address up to limit bytes using the indirect
          * addressing immediate, account for the difference if the source
          * register is above this limit.
          */
         if (offset >= limit) {
            brw_set_default_swsb(p, tgl_swsb_regdist(1));
            brw_ADD(p, addr, addr, brw_imm_ud(offset - offset % limit));
            offset = offset % limit;
         }

         brw_pop_insn_state(p);

         brw_set_default_swsb(p, tgl_swsb_regdist(1));

         /* Use indirect addressing to fetch the specified component. */
         if (type_sz(src.type) > 4 &&
             (devinfo->is_cherryview || intel_device_info_is_9lp(devinfo) ||
              !devinfo->has_64bit_float)) {
            /* From the Cherryview PRM Vol 7. "Register Region Restrictions":
             *
             *    "When source or destination datatype is 64b or operation is
             *    integer DWord multiply, indirect addressing must not be
             *    used."
             *
             * To work around both of this issue, we do two integer MOVs
             * insead of one 64-bit MOV.  Because no double value should ever
             * cross a register boundary, it's safe to use the immediate
             * offset in the indirect here to handle adding 4 bytes to the
             * offset and avoid the extra ADD to the register file.
             */
            brw_MOV(p, subscript(dst, BRW_REGISTER_TYPE_D, 0),
                       retype(brw_vec1_indirect(addr.subnr, offset),
                              BRW_REGISTER_TYPE_D));
            brw_set_default_swsb(p, tgl_swsb_null());
            brw_MOV(p, subscript(dst, BRW_REGISTER_TYPE_D, 1),
                       retype(brw_vec1_indirect(addr.subnr, offset + 4),
                              BRW_REGISTER_TYPE_D));
         } else {
            brw_MOV(p, dst,
                    retype(brw_vec1_indirect(addr.subnr, offset), src.type));
         }
      } else {
         /* In SIMD4x2 mode the index can be either zero or one, replicate it
          * to all bits of a flag register,
          */
         inst = brw_MOV(p,
                        brw_null_reg(),
                        stride(brw_swizzle(idx, BRW_SWIZZLE_XXXX), 4, 4, 1));
         brw_inst_set_pred_control(devinfo, inst, BRW_PREDICATE_NONE);
         brw_inst_set_cond_modifier(devinfo, inst, BRW_CONDITIONAL_NZ);
         brw_inst_set_flag_reg_nr(devinfo, inst, 1);

         /* and use predicated SEL to pick the right channel. */
         inst = brw_SEL(p, dst,
                        stride(suboffset(src, 4), 4, 4, 1),
                        stride(src, 4, 4, 1));
         brw_inst_set_pred_control(devinfo, inst, BRW_PREDICATE_NORMAL);
         brw_inst_set_flag_reg_nr(devinfo, inst, 1);
      }
   }

   brw_pop_insn_state(p);
}

/**
 * This instruction is generated as a single-channel align1 instruction by
 * both the VS and FS stages when using INTEL_DEBUG=shader_time.
 *
 * We can't use the typed atomic op in the FS because that has the execution
 * mask ANDed with the pixel mask, but we just want to write the one dword for
 * all the pixels.
 *
 * We don't use the SIMD4x2 atomic ops in the VS because want to just write
 * one u32.  So we use the same untyped atomic write message as the pixel
 * shader.
 *
 * The untyped atomic operation requires a BUFFER surface type with RAW
 * format, and is only accessible through the legacy DATA_CACHE dataport
 * messages.
 */
void brw_shader_time_add(struct brw_codegen *p,
                         struct brw_reg payload,
                         uint32_t surf_index)
{
   const struct intel_device_info *devinfo = p->devinfo;
   const unsigned sfid = (devinfo->verx10 >= 75 ?
                          HSW_SFID_DATAPORT_DATA_CACHE_1 :
                          GFX7_SFID_DATAPORT_DATA_CACHE);
   assert(devinfo->ver >= 7);

   brw_push_insn_state(p);
   brw_set_default_access_mode(p, BRW_ALIGN_1);
   brw_set_default_mask_control(p, BRW_MASK_DISABLE);
   brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
   brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);

   /* We use brw_vec1_reg and unmasked because we want to increment the given
    * offset only once.
    */
   brw_set_dest(p, send, brw_vec1_reg(BRW_ARCHITECTURE_REGISTER_FILE,
                                      BRW_ARF_NULL, 0));
   brw_set_src0(p, send, brw_vec1_reg(payload.file,
                                      payload.nr, 0));
   brw_set_desc(p, send, (brw_message_desc(devinfo, 2, 0, false) |
                          brw_dp_untyped_atomic_desc(devinfo, 1, BRW_AOP_ADD,
                                                     false)));

   brw_inst_set_sfid(devinfo, send, sfid);
   brw_inst_set_binding_table_index(devinfo, send, surf_index);

   brw_pop_insn_state(p);
}


/**
 * Emit the SEND message for a barrier
 */
void
brw_barrier(struct brw_codegen *p, struct brw_reg src)
{
   const struct intel_device_info *devinfo = p->devinfo;
   struct brw_inst *inst;

   assert(devinfo->ver >= 7);

   brw_push_insn_state(p);
   brw_set_default_access_mode(p, BRW_ALIGN_1);
   inst = next_insn(p, BRW_OPCODE_SEND);
   brw_set_dest(p, inst, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW));
   brw_set_src0(p, inst, src);
   brw_set_src1(p, inst, brw_null_reg());
   brw_set_desc(p, inst, brw_message_desc(devinfo, 1, 0, false));

   brw_inst_set_sfid(devinfo, inst, BRW_SFID_MESSAGE_GATEWAY);
   brw_inst_set_gateway_subfuncid(devinfo, inst,
                                  BRW_MESSAGE_GATEWAY_SFID_BARRIER_MSG);

   brw_inst_set_mask_control(devinfo, inst, BRW_MASK_DISABLE);
   brw_pop_insn_state(p);
}


/**
 * Emit the wait instruction for a barrier
 */
void
brw_WAIT(struct brw_codegen *p)
{
   const struct intel_device_info *devinfo = p->devinfo;
   struct brw_inst *insn;

   struct brw_reg src = brw_notification_reg();

   insn = next_insn(p, BRW_OPCODE_WAIT);
   brw_set_dest(p, insn, src);
   brw_set_src0(p, insn, src);
   brw_set_src1(p, insn, brw_null_reg());

   brw_inst_set_exec_size(devinfo, insn, BRW_EXECUTE_1);
   brw_inst_set_mask_control(devinfo, insn, BRW_MASK_DISABLE);
}

void
brw_float_controls_mode(struct brw_codegen *p,
                        unsigned mode, unsigned mask)
{
   /* From the Skylake PRM, Volume 7, page 760:
    *  "Implementation Restriction on Register Access: When the control
    *   register is used as an explicit source and/or destination, hardware
    *   does not ensure execution pipeline coherency. Software must set the
    *   thread control field to ‘switch’ for an instruction that uses
    *   control register as an explicit operand."
    *
    * On Gfx12+ this is implemented in terms of SWSB annotations instead.
    */
   brw_set_default_swsb(p, tgl_swsb_regdist(1));

   brw_inst *inst = brw_AND(p, brw_cr0_reg(0), brw_cr0_reg(0),
                            brw_imm_ud(~mask));
   brw_inst_set_exec_size(p->devinfo, inst, BRW_EXECUTE_1);
   if (p->devinfo->ver < 12)
      brw_inst_set_thread_control(p->devinfo, inst, BRW_THREAD_SWITCH);

   if (mode) {
      brw_inst *inst_or = brw_OR(p, brw_cr0_reg(0), brw_cr0_reg(0),
                                 brw_imm_ud(mode));
      brw_inst_set_exec_size(p->devinfo, inst_or, BRW_EXECUTE_1);
      if (p->devinfo->ver < 12)
         brw_inst_set_thread_control(p->devinfo, inst_or, BRW_THREAD_SWITCH);
   }

   if (p->devinfo->ver >= 12)
      brw_SYNC(p, TGL_SYNC_NOP);
}

void
brw_update_reloc_imm(const struct intel_device_info *devinfo,
                     brw_inst *inst,
                     uint32_t value)
{
   /* Sanity check that the instruction is a MOV of an immediate */
   assert(brw_inst_opcode(devinfo, inst) == BRW_OPCODE_MOV);
   assert(brw_inst_src0_reg_file(devinfo, inst) == BRW_IMMEDIATE_VALUE);

   /* If it was compacted, we can't safely rewrite */
   assert(brw_inst_cmpt_control(devinfo, inst) == 0);

   brw_inst_set_imm_ud(devinfo, inst, value);
}

/* A default value for constants that will be patched at run-time.
 * We pick an arbitrary value that prevents instruction compaction.
 */
#define DEFAULT_PATCH_IMM 0x4a7cc037

void
brw_MOV_reloc_imm(struct brw_codegen *p,
                  struct brw_reg dst,
                  enum brw_reg_type src_type,
                  uint32_t id)
{
   assert(type_sz(src_type) == 4);
   assert(type_sz(dst.type) == 4);

   brw_add_reloc(p, id, BRW_SHADER_RELOC_TYPE_MOV_IMM,
                 p->next_insn_offset, 0);

   brw_MOV(p, dst, retype(brw_imm_ud(DEFAULT_PATCH_IMM), src_type));
}