Path: blob/21.2-virgl/src/amd/compiler/aco_builder_h.py
4550 views
1template = """\2/*3* Copyright (c) 2019 Valve Corporation4*5* Permission is hereby granted, free of charge, to any person obtaining a6* copy of this software and associated documentation files (the "Software"),7* to deal in the Software without restriction, including without limitation8* the rights to use, copy, modify, merge, publish, distribute, sublicense,9* and/or sell copies of the Software, and to permit persons to whom the10* Software is furnished to do so, subject to the following conditions:11*12* The above copyright notice and this permission notice (including the next13* paragraph) shall be included in all copies or substantial portions of the14* Software.15*16* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR17* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,18* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL19* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER20* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING21* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS22* IN THE SOFTWARE.23*24* This file was generated by aco_builder_h.py25*/2627#ifndef _ACO_BUILDER_28#define _ACO_BUILDER_2930#include "aco_ir.h"3132namespace aco {33enum dpp_ctrl {34_dpp_quad_perm = 0x000,35_dpp_row_sl = 0x100,36_dpp_row_sr = 0x110,37_dpp_row_rr = 0x120,38dpp_wf_sl1 = 0x130,39dpp_wf_rl1 = 0x134,40dpp_wf_sr1 = 0x138,41dpp_wf_rr1 = 0x13C,42dpp_row_mirror = 0x140,43dpp_row_half_mirror = 0x141,44dpp_row_bcast15 = 0x142,45dpp_row_bcast31 = 0x14346};4748inline dpp_ctrl49dpp_quad_perm(unsigned lane0, unsigned lane1, unsigned lane2, unsigned lane3)50{51assert(lane0 < 4 && lane1 < 4 && lane2 < 4 && lane3 < 4);52return (dpp_ctrl)(lane0 | (lane1 << 2) | (lane2 << 4) | (lane3 << 6));53}5455inline dpp_ctrl56dpp_row_sl(unsigned amount)57{58assert(amount > 0 && amount < 16);59return (dpp_ctrl)(((unsigned) _dpp_row_sl) | amount);60}6162inline dpp_ctrl63dpp_row_sr(unsigned amount)64{65assert(amount > 0 && amount < 16);66return (dpp_ctrl)(((unsigned) _dpp_row_sr) | amount);67}6869inline dpp_ctrl70dpp_row_rr(unsigned amount)71{72assert(amount > 0 && amount < 16);73return (dpp_ctrl)(((unsigned) _dpp_row_rr) | amount);74}7576inline unsigned77ds_pattern_bitmode(unsigned and_mask, unsigned or_mask, unsigned xor_mask)78{79assert(and_mask < 32 && or_mask < 32 && xor_mask < 32);80return and_mask | (or_mask << 5) | (xor_mask << 10);81}8283aco_ptr<Instruction> create_s_mov(Definition dst, Operand src);8485enum sendmsg {86sendmsg_none = 0,87_sendmsg_gs = 2,88_sendmsg_gs_done = 3,89sendmsg_save_wave = 4,90sendmsg_stall_wave_gen = 5,91sendmsg_halt_waves = 6,92sendmsg_ordered_ps_done = 7,93sendmsg_early_prim_dealloc = 8,94sendmsg_gs_alloc_req = 9,95sendmsg_id_mask = 0xf,96};9798inline sendmsg99sendmsg_gs(bool cut, bool emit, unsigned stream)100{101assert(stream < 4);102return (sendmsg)((unsigned)_sendmsg_gs | (cut << 4) | (emit << 5) | (stream << 8));103}104105inline sendmsg106sendmsg_gs_done(bool cut, bool emit, unsigned stream)107{108assert(stream < 4);109return (sendmsg)((unsigned)_sendmsg_gs_done | (cut << 4) | (emit << 5) | (stream << 8));110}111112class Builder {113public:114struct Result {115Instruction *instr;116117Result(Instruction *instr_) : instr(instr_) {}118119operator Instruction *() const {120return instr;121}122123operator Temp() const {124return instr->definitions[0].getTemp();125}126127operator Operand() const {128return Operand((Temp)*this);129}130131Definition& def(unsigned index) const {132return instr->definitions[index];133}134135aco_ptr<Instruction> get_ptr() const {136return aco_ptr<Instruction>(instr);137}138};139140struct Op {141Operand op;142Op(Temp tmp) : op(tmp) {}143Op(Operand op_) : op(op_) {}144Op(Result res) : op((Temp)res) {}145};146147enum WaveSpecificOpcode {148s_cselect = (unsigned) aco_opcode::s_cselect_b64,149s_cmp_lg = (unsigned) aco_opcode::s_cmp_lg_u64,150s_and = (unsigned) aco_opcode::s_and_b64,151s_andn2 = (unsigned) aco_opcode::s_andn2_b64,152s_or = (unsigned) aco_opcode::s_or_b64,153s_orn2 = (unsigned) aco_opcode::s_orn2_b64,154s_not = (unsigned) aco_opcode::s_not_b64,155s_mov = (unsigned) aco_opcode::s_mov_b64,156s_wqm = (unsigned) aco_opcode::s_wqm_b64,157s_and_saveexec = (unsigned) aco_opcode::s_and_saveexec_b64,158s_or_saveexec = (unsigned) aco_opcode::s_or_saveexec_b64,159s_xnor = (unsigned) aco_opcode::s_xnor_b64,160s_xor = (unsigned) aco_opcode::s_xor_b64,161s_bcnt1_i32 = (unsigned) aco_opcode::s_bcnt1_i32_b64,162s_bitcmp1 = (unsigned) aco_opcode::s_bitcmp1_b64,163s_ff1_i32 = (unsigned) aco_opcode::s_ff1_i32_b64,164s_flbit_i32 = (unsigned) aco_opcode::s_flbit_i32_b64,165s_lshl = (unsigned) aco_opcode::s_lshl_b64,166};167168Program *program;169bool use_iterator;170bool start; // only when use_iterator == false171RegClass lm;172173std::vector<aco_ptr<Instruction>> *instructions;174std::vector<aco_ptr<Instruction>>::iterator it;175bool is_precise = false;176bool is_nuw = false;177178Builder(Program *pgm) : program(pgm), use_iterator(false), start(false), lm(pgm ? pgm->lane_mask : s2), instructions(NULL) {}179Builder(Program *pgm, Block *block) : program(pgm), use_iterator(false), start(false), lm(pgm ? pgm->lane_mask : s2), instructions(&block->instructions) {}180Builder(Program *pgm, std::vector<aco_ptr<Instruction>> *instrs) : program(pgm), use_iterator(false), start(false), lm(pgm ? pgm->lane_mask : s2), instructions(instrs) {}181182Builder precise() const {183Builder res = *this;184res.is_precise = true;185return res;186};187188Builder nuw() const {189Builder res = *this;190res.is_nuw = true;191return res;192}193194void moveEnd(Block *block) {195instructions = &block->instructions;196}197198void reset() {199use_iterator = false;200start = false;201instructions = NULL;202}203204void reset(Block *block) {205use_iterator = false;206start = false;207instructions = &block->instructions;208}209210void reset(std::vector<aco_ptr<Instruction>> *instrs) {211use_iterator = false;212start = false;213instructions = instrs;214}215216void reset(std::vector<aco_ptr<Instruction>> *instrs, std::vector<aco_ptr<Instruction>>::iterator instr_it) {217use_iterator = true;218start = false;219instructions = instrs;220it = instr_it;221}222223Result insert(aco_ptr<Instruction> instr) {224Instruction *instr_ptr = instr.get();225if (instructions) {226if (use_iterator) {227it = instructions->emplace(it, std::move(instr));228it = std::next(it);229} else if (!start) {230instructions->emplace_back(std::move(instr));231} else {232instructions->emplace(instructions->begin(), std::move(instr));233}234}235return Result(instr_ptr);236}237238Result insert(Instruction* instr) {239if (instructions) {240if (use_iterator) {241it = instructions->emplace(it, aco_ptr<Instruction>(instr));242it = std::next(it);243} else if (!start) {244instructions->emplace_back(aco_ptr<Instruction>(instr));245} else {246instructions->emplace(instructions->begin(), aco_ptr<Instruction>(instr));247}248}249return Result(instr);250}251252Temp tmp(RegClass rc) {253return program->allocateTmp(rc);254}255256Temp tmp(RegType type, unsigned size) {257return tmp(RegClass(type, size));258}259260Definition def(RegClass rc) {261return Definition(program->allocateTmp(rc));262}263264Definition def(RegType type, unsigned size) {265return def(RegClass(type, size));266}267268Definition def(RegClass rc, PhysReg reg) {269return Definition(program->allocateId(rc), reg, rc);270}271272inline aco_opcode w64or32(WaveSpecificOpcode opcode) const {273if (program->wave_size == 64)274return (aco_opcode) opcode;275276switch (opcode) {277case s_cselect:278return aco_opcode::s_cselect_b32;279case s_cmp_lg:280return aco_opcode::s_cmp_lg_u32;281case s_and:282return aco_opcode::s_and_b32;283case s_andn2:284return aco_opcode::s_andn2_b32;285case s_or:286return aco_opcode::s_or_b32;287case s_orn2:288return aco_opcode::s_orn2_b32;289case s_not:290return aco_opcode::s_not_b32;291case s_mov:292return aco_opcode::s_mov_b32;293case s_wqm:294return aco_opcode::s_wqm_b32;295case s_and_saveexec:296return aco_opcode::s_and_saveexec_b32;297case s_or_saveexec:298return aco_opcode::s_or_saveexec_b32;299case s_xnor:300return aco_opcode::s_xnor_b32;301case s_xor:302return aco_opcode::s_xor_b32;303case s_bcnt1_i32:304return aco_opcode::s_bcnt1_i32_b32;305case s_bitcmp1:306return aco_opcode::s_bitcmp1_b32;307case s_ff1_i32:308return aco_opcode::s_ff1_i32_b32;309case s_flbit_i32:310return aco_opcode::s_flbit_i32_b32;311case s_lshl:312return aco_opcode::s_lshl_b32;313default:314unreachable("Unsupported wave specific opcode.");315}316}317318% for fixed in ['m0', 'vcc', 'exec', 'scc']:319Operand ${fixed}(Temp tmp) {320% if fixed == 'vcc' or fixed == 'exec':321//vcc_hi and exec_hi can still be used in wave32322assert(tmp.type() == RegType::sgpr && tmp.bytes() <= 8);323% endif324Operand op(tmp);325op.setFixed(aco::${fixed});326return op;327}328329Definition ${fixed}(Definition def) {330% if fixed == 'vcc' or fixed == 'exec':331//vcc_hi and exec_hi can still be used in wave32332assert(def.regClass().type() == RegType::sgpr && def.bytes() <= 8);333% endif334def.setFixed(aco::${fixed});335return def;336}337338Definition hint_${fixed}(Definition def) {339% if fixed == 'vcc' or fixed == 'exec':340//vcc_hi and exec_hi can still be used in wave32341assert(def.regClass().type() == RegType::sgpr && def.bytes() <= 8);342% endif343def.setHint(aco::${fixed});344return def;345}346347Definition hint_${fixed}(RegClass rc) {348return hint_${fixed}(def(rc));349}350351% endfor352353Operand set16bit(Operand op) {354op.set16bit(true);355return op;356}357358Operand set24bit(Operand op) {359op.set24bit(true);360return op;361}362363/* hand-written helpers */364Temp as_uniform(Op op)365{366assert(op.op.isTemp());367if (op.op.getTemp().type() == RegType::vgpr)368return pseudo(aco_opcode::p_as_uniform, def(RegType::sgpr, op.op.size()), op);369else370return op.op.getTemp();371}372373Result v_mul_imm(Definition dst, Temp tmp, uint32_t imm, bool bits24=false)374{375assert(tmp.type() == RegType::vgpr);376bool has_lshl_add = program->chip_class >= GFX9;377/* v_mul_lo_u32 has 1.6x the latency of most VALU on GFX10 (8 vs 5 cycles),378* compared to 4x the latency on <GFX10. */379unsigned mul_cost = program->chip_class >= GFX10 ? 1 : (4 + Operand::c32(imm).isLiteral());380if (imm == 0) {381return copy(dst, Operand::zero());382} else if (imm == 1) {383return copy(dst, Operand(tmp));384} else if (util_is_power_of_two_or_zero(imm)) {385return vop2(aco_opcode::v_lshlrev_b32, dst, Operand::c32(ffs(imm) - 1u), tmp);386} else if (bits24) {387return vop2(aco_opcode::v_mul_u32_u24, dst, Operand::c32(imm), tmp);388} else if (util_is_power_of_two_nonzero(imm - 1u)) {389return vadd32(dst, vop2(aco_opcode::v_lshlrev_b32, def(v1), Operand::c32(ffs(imm - 1u) - 1u), tmp), tmp);390} else if (mul_cost > 2 && util_is_power_of_two_nonzero(imm + 1u)) {391return vsub32(dst, vop2(aco_opcode::v_lshlrev_b32, def(v1), Operand::c32(ffs(imm + 1u) - 1u), tmp), tmp);392}393394unsigned instrs_required = util_bitcount(imm);395if (!has_lshl_add) {396instrs_required = util_bitcount(imm) - (imm & 0x1); /* shifts */397instrs_required += util_bitcount(imm) - 1; /* additions */398}399if (instrs_required < mul_cost) {400Result res(NULL);401Temp cur;402while (imm) {403unsigned shift = u_bit_scan(&imm);404Definition tmp_dst = imm ? def(v1) : dst;405406if (shift && cur.id())407res = vadd32(Definition(tmp_dst), vop2(aco_opcode::v_lshlrev_b32, def(v1), Operand::c32(shift), tmp), cur);408else if (shift)409res = vop2(aco_opcode::v_lshlrev_b32, Definition(tmp_dst), Operand::c32(shift), tmp);410else if (cur.id())411res = vadd32(Definition(tmp_dst), tmp, cur);412else413tmp_dst = Definition(tmp);414415cur = tmp_dst.getTemp();416}417return res;418}419420Temp imm_tmp = copy(def(s1), Operand::c32(imm));421return vop3(aco_opcode::v_mul_lo_u32, dst, imm_tmp, tmp);422}423424Result v_mul24_imm(Definition dst, Temp tmp, uint32_t imm)425{426return v_mul_imm(dst, tmp, imm, true);427}428429Result copy(Definition dst, Op op) {430return pseudo(aco_opcode::p_parallelcopy, dst, op);431}432433Result vadd32(Definition dst, Op a, Op b, bool carry_out=false, Op carry_in=Op(Operand(s2)), bool post_ra=false) {434if (!b.op.isTemp() || b.op.regClass().type() != RegType::vgpr)435std::swap(a, b);436if (!post_ra && (!b.op.hasRegClass() || b.op.regClass().type() == RegType::sgpr))437b = copy(def(v1), b);438439if (!carry_in.op.isUndefined())440return vop2(aco_opcode::v_addc_co_u32, Definition(dst), hint_vcc(def(lm)), a, b, carry_in);441else if (program->chip_class >= GFX10 && carry_out)442return vop3(aco_opcode::v_add_co_u32_e64, Definition(dst), def(lm), a, b);443else if (program->chip_class < GFX9 || carry_out)444return vop2(aco_opcode::v_add_co_u32, Definition(dst), hint_vcc(def(lm)), a, b);445else446return vop2(aco_opcode::v_add_u32, Definition(dst), a, b);447}448449Result vsub32(Definition dst, Op a, Op b, bool carry_out=false, Op borrow=Op(Operand(s2)))450{451if (!borrow.op.isUndefined() || program->chip_class < GFX9)452carry_out = true;453454bool reverse = !b.op.isTemp() || b.op.regClass().type() != RegType::vgpr;455if (reverse)456std::swap(a, b);457if (!b.op.hasRegClass() || b.op.regClass().type() == RegType::sgpr)458b = copy(def(v1), b);459460aco_opcode op;461Temp carry;462if (carry_out) {463carry = tmp(s2);464if (borrow.op.isUndefined())465op = reverse ? aco_opcode::v_subrev_co_u32 : aco_opcode::v_sub_co_u32;466else467op = reverse ? aco_opcode::v_subbrev_co_u32 : aco_opcode::v_subb_co_u32;468} else {469op = reverse ? aco_opcode::v_subrev_u32 : aco_opcode::v_sub_u32;470}471bool vop3 = false;472if (program->chip_class >= GFX10 && op == aco_opcode::v_subrev_co_u32) {473vop3 = true;474op = aco_opcode::v_subrev_co_u32_e64;475} else if (program->chip_class >= GFX10 && op == aco_opcode::v_sub_co_u32) {476vop3 = true;477op = aco_opcode::v_sub_co_u32_e64;478}479480int num_ops = borrow.op.isUndefined() ? 2 : 3;481int num_defs = carry_out ? 2 : 1;482aco_ptr<Instruction> sub;483if (vop3)484sub.reset(create_instruction<VOP3_instruction>(op, Format::VOP3, num_ops, num_defs));485else486sub.reset(create_instruction<VOP2_instruction>(op, Format::VOP2, num_ops, num_defs));487sub->operands[0] = a.op;488sub->operands[1] = b.op;489if (!borrow.op.isUndefined())490sub->operands[2] = borrow.op;491sub->definitions[0] = dst;492if (carry_out) {493sub->definitions[1] = Definition(carry);494sub->definitions[1].setHint(aco::vcc);495}496return insert(std::move(sub));497}498499Result readlane(Definition dst, Op vsrc, Op lane)500{501if (program->chip_class >= GFX8)502return vop3(aco_opcode::v_readlane_b32_e64, dst, vsrc, lane);503else504return vop2(aco_opcode::v_readlane_b32, dst, vsrc, lane);505}506Result writelane(Definition dst, Op val, Op lane, Op vsrc) {507if (program->chip_class >= GFX8)508return vop3(aco_opcode::v_writelane_b32_e64, dst, val, lane, vsrc);509else510return vop2(aco_opcode::v_writelane_b32, dst, val, lane, vsrc);511}512<%513import itertools514formats = [("pseudo", [Format.PSEUDO], 'Pseudo_instruction', list(itertools.product(range(5), range(6))) + [(8, 1), (1, 8)]),515("sop1", [Format.SOP1], 'SOP1_instruction', [(0, 1), (1, 0), (1, 1), (2, 1), (3, 2)]),516("sop2", [Format.SOP2], 'SOP2_instruction', itertools.product([1, 2], [2, 3])),517("sopk", [Format.SOPK], 'SOPK_instruction', itertools.product([0, 1, 2], [0, 1])),518("sopp", [Format.SOPP], 'SOPP_instruction', itertools.product([0, 1], [0, 1])),519("sopc", [Format.SOPC], 'SOPC_instruction', [(1, 2)]),520("smem", [Format.SMEM], 'SMEM_instruction', [(0, 4), (0, 3), (1, 0), (1, 3), (1, 2), (0, 0)]),521("ds", [Format.DS], 'DS_instruction', [(1, 1), (1, 2), (0, 3), (0, 4)]),522("mubuf", [Format.MUBUF], 'MUBUF_instruction', [(0, 4), (1, 3)]),523("mtbuf", [Format.MTBUF], 'MTBUF_instruction', [(0, 4), (1, 3)]),524("mimg", [Format.MIMG], 'MIMG_instruction', itertools.product([0, 1], [3, 4, 5, 6, 7])),525("exp", [Format.EXP], 'Export_instruction', [(0, 4)]),526("branch", [Format.PSEUDO_BRANCH], 'Pseudo_branch_instruction', itertools.product([1], [0, 1])),527("barrier", [Format.PSEUDO_BARRIER], 'Pseudo_barrier_instruction', [(0, 0)]),528("reduction", [Format.PSEUDO_REDUCTION], 'Pseudo_reduction_instruction', [(3, 2)]),529("vop1", [Format.VOP1], 'VOP1_instruction', [(0, 0), (1, 1), (2, 2)]),530("vop1_sdwa", [Format.VOP1, Format.SDWA], 'SDWA_instruction', [(1, 1)]),531("vop2", [Format.VOP2], 'VOP2_instruction', itertools.product([1, 2], [2, 3])),532("vop2_sdwa", [Format.VOP2, Format.SDWA], 'SDWA_instruction', itertools.product([1, 2], [2, 3])),533("vopc", [Format.VOPC], 'VOPC_instruction', itertools.product([1, 2], [2])),534("vopc_sdwa", [Format.VOPC, Format.SDWA], 'SDWA_instruction', itertools.product([1, 2], [2])),535("vop3", [Format.VOP3], 'VOP3_instruction', [(1, 3), (1, 2), (1, 1), (2, 2)]),536("vop3p", [Format.VOP3P], 'VOP3P_instruction', [(1, 2), (1, 3)]),537("vintrp", [Format.VINTRP], 'Interp_instruction', [(1, 2), (1, 3)]),538("vop1_dpp", [Format.VOP1, Format.DPP], 'DPP_instruction', [(1, 1)]),539("vop2_dpp", [Format.VOP2, Format.DPP], 'DPP_instruction', itertools.product([1, 2], [2, 3])),540("vopc_dpp", [Format.VOPC, Format.DPP], 'DPP_instruction', itertools.product([1, 2], [2])),541("vop1_e64", [Format.VOP1, Format.VOP3], 'VOP3_instruction', itertools.product([1], [1])),542("vop2_e64", [Format.VOP2, Format.VOP3], 'VOP3_instruction', itertools.product([1, 2], [2, 3])),543("vopc_e64", [Format.VOPC, Format.VOP3], 'VOP3_instruction', itertools.product([1, 2], [2])),544("flat", [Format.FLAT], 'FLAT_instruction', [(0, 3), (1, 2)]),545("global", [Format.GLOBAL], 'FLAT_instruction', [(0, 3), (1, 2)])]546formats = [(f if len(f) == 5 else f + ('',)) for f in formats]547%>\\548% for name, formats, struct, shapes, extra_field_setup in formats:549% for num_definitions, num_operands in shapes:550<%551args = ['aco_opcode opcode']552for i in range(num_definitions):553args.append('Definition def%d' % i)554for i in range(num_operands):555args.append('Op op%d' % i)556for f in formats:557args += f.get_builder_field_decls()558%>\\559560Result ${name}(${', '.join(args)})561{562${struct} *instr = create_instruction<${struct}>(opcode, (Format)(${'|'.join('(int)Format::%s' % f.name for f in formats)}), ${num_operands}, ${num_definitions});563% for i in range(num_definitions):564instr->definitions[${i}] = def${i};565instr->definitions[${i}].setPrecise(is_precise);566instr->definitions[${i}].setNUW(is_nuw);567% endfor568% for i in range(num_operands):569instr->operands[${i}] = op${i}.op;570% endfor571% for f in formats:572% for dest, field_name in zip(f.get_builder_field_dests(), f.get_builder_field_names()):573instr->${dest} = ${field_name};574% endfor575${f.get_builder_initialization(num_operands)}576% endfor577${extra_field_setup}578return insert(instr);579}580581% if name == 'sop1' or name == 'sop2' or name == 'sopc':582<%583args[0] = 'WaveSpecificOpcode opcode'584params = []585for i in range(num_definitions):586params.append('def%d' % i)587for i in range(num_operands):588params.append('op%d' % i)589%>\\590591inline Result ${name}(${', '.join(args)})592{593return ${name}(w64or32(opcode), ${', '.join(params)});594}595596% endif597% endfor598% endfor599};600601} // namespace aco602603#endif /* _ACO_BUILDER_ */"""604605from aco_opcodes import opcodes, Format606from mako.template import Template607608print(Template(template).render(opcodes=opcodes, Format=Format))609610611