radeon_emulate_loops.c

/*
 * Copyright 2010 Tom Stellard <[email protected]>
 *
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial
 * portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 */

/**
 * \file
 */

#include <stdio.h>
#include "c99_math.h"
#include "radeon_emulate_loops.h"
#include "radeon_compiler.h"
#include "radeon_compiler_util.h"
#include "radeon_dataflow.h"

#define VERBOSE 0

#define DBG(...) do { if (VERBOSE) fprintf(stderr, __VA_ARGS__); } while(0)

struct const_value {
	struct radeon_compiler * C;
	struct rc_src_register * Src;
	float Value;
	int HasValue;
};

struct count_inst {
	struct radeon_compiler * C;
	int Index;
	rc_swizzle Swz;
	float Amount;
	int Unknown;
	unsigned BranchDepth;
};

static unsigned int loop_max_possible_iterations(struct radeon_compiler *c,
			struct loop_info * loop)
{
	unsigned int total_i = rc_recompute_ips(c);
	unsigned int loop_i = (loop->EndLoop->IP - loop->BeginLoop->IP) - 1;
	/* +1 because the program already has one iteration of the loop. */
	return 1 + ((c->max_alu_insts - total_i) / loop_i);
}

static void unroll_loop(struct radeon_compiler * c, struct loop_info * loop,
						unsigned int iterations)
{
	unsigned int i;
	struct rc_instruction * ptr;
	struct rc_instruction * first = loop->BeginLoop->Next;
	struct rc_instruction * last = loop->EndLoop->Prev;
	struct rc_instruction * append_to = last;
	rc_remove_instruction(loop->BeginLoop);
	rc_remove_instruction(loop->EndLoop);
	for( i = 1; i < iterations; i++){
		for(ptr = first; ptr != last->Next; ptr = ptr->Next){
			struct rc_instruction *new = rc_alloc_instruction(c);
			memcpy(new, ptr, sizeof(struct rc_instruction));
			rc_insert_instruction(append_to, new);
			append_to = new;
		}
	}
}


static void update_const_value(void * data, struct rc_instruction * inst,
		rc_register_file file, unsigned int index, unsigned int mask)
{
	struct const_value * value = data;
	if(value->Src->File != file ||
	   value->Src->Index != index ||
	   !(1 << GET_SWZ(value->Src->Swizzle, 0) & mask)){
		return;
	}
	switch(inst->U.I.Opcode){
	case RC_OPCODE_MOV:
		if(!rc_src_reg_is_immediate(value->C, inst->U.I.SrcReg[0].File,
						inst->U.I.SrcReg[0].Index)){
			return;
		}
		value->HasValue = 1;
		value->Value =
			rc_get_constant_value(value->C,
					      inst->U.I.SrcReg[0].Index,
					      inst->U.I.SrcReg[0].Swizzle,
					      inst->U.I.SrcReg[0].Negate, 0);
		break;
	}
}

static void get_incr_amount(void * data, struct rc_instruction * inst,
		rc_register_file file, unsigned int index, unsigned int mask)
{
	struct count_inst * count_inst = data;
	int amnt_src_index;
	const struct rc_opcode_info * opcode;
	float amount;

	if(file != RC_FILE_TEMPORARY ||
	   count_inst->Index != index ||
	   (1 << GET_SWZ(count_inst->Swz,0) != mask)){
		return;
	}

	/* XXX: Give up if the counter is modified within an IF block.  We
	 * could handle this case with better analysis. */
	if (count_inst->BranchDepth > 0) {
		count_inst->Unknown = 1;
		return;
	}

	/* Find the index of the counter register. */
	opcode = rc_get_opcode_info(inst->U.I.Opcode);
	if(opcode->NumSrcRegs != 2){
		count_inst->Unknown = 1;
		return;
	}
	if(inst->U.I.SrcReg[0].File == RC_FILE_TEMPORARY &&
	   inst->U.I.SrcReg[0].Index == count_inst->Index &&
	   inst->U.I.SrcReg[0].Swizzle == count_inst->Swz){
		amnt_src_index = 1;
	} else if( inst->U.I.SrcReg[1].File == RC_FILE_TEMPORARY &&
		   inst->U.I.SrcReg[1].Index == count_inst->Index &&
		   inst->U.I.SrcReg[1].Swizzle == count_inst->Swz){
		amnt_src_index = 0;
	}
	else{
		count_inst->Unknown = 1;
		return;
	}
	if(rc_src_reg_is_immediate(count_inst->C,
				inst->U.I.SrcReg[amnt_src_index].File,
				inst->U.I.SrcReg[amnt_src_index].Index)){
		amount = rc_get_constant_value(count_inst->C,
				inst->U.I.SrcReg[amnt_src_index].Index,
				inst->U.I.SrcReg[amnt_src_index].Swizzle,
				inst->U.I.SrcReg[amnt_src_index].Negate, 0);
	}
	else{
		count_inst->Unknown = 1 ;
		return;
	}
	switch(inst->U.I.Opcode){
	case RC_OPCODE_ADD:
		count_inst->Amount += amount;
		break;
	case RC_OPCODE_SUB:
		if(amnt_src_index == 0){
			count_inst->Unknown = 0;
			return;
		}
		count_inst->Amount -= amount;
		break;
	default:
		count_inst->Unknown = 1;
		return;
	}
}

/**
 * If c->max_alu_inst is -1, then all eligible loops will be unrolled regardless
 * of how many iterations they have.
 */
static int try_unroll_loop(struct radeon_compiler * c, struct loop_info * loop)
{
	int end_loops;
	int iterations;
	struct count_inst count_inst;
	float limit_value;
	struct rc_src_register * counter;
	struct rc_src_register * limit;
	struct const_value counter_value;
	struct rc_instruction * inst;

	/* Find the counter and the upper limit */

	if(rc_src_reg_is_immediate(c, loop->Cond->U.I.SrcReg[0].File,
					loop->Cond->U.I.SrcReg[0].Index)){
		limit = &loop->Cond->U.I.SrcReg[0];
		counter = &loop->Cond->U.I.SrcReg[1];
	}
	else if(rc_src_reg_is_immediate(c, loop->Cond->U.I.SrcReg[1].File,
					loop->Cond->U.I.SrcReg[1].Index)){
		limit = &loop->Cond->U.I.SrcReg[1];
		counter = &loop->Cond->U.I.SrcReg[0];
	}
	else{
		DBG("No constant limit.\n");
		return 0;
	}

	/* Find the initial value of the counter */
	counter_value.Src = counter;
	counter_value.Value = 0.0f;
	counter_value.HasValue = 0;
	counter_value.C = c;
	for(inst = c->Program.Instructions.Next; inst != loop->BeginLoop;
							inst = inst->Next){
		rc_for_all_writes_mask(inst, update_const_value, &counter_value);
	}
	if(!counter_value.HasValue){
		DBG("Initial counter value cannot be determined.\n");
		return 0;
	}
	DBG("Initial counter value is %f\n", counter_value.Value);
	/* Determine how the counter is modified each loop */
	count_inst.C = c;
	count_inst.Index = counter->Index;
	count_inst.Swz = counter->Swizzle;
	count_inst.Amount = 0.0f;
	count_inst.Unknown = 0;
	count_inst.BranchDepth = 0;
	end_loops = 1;
	for(inst = loop->BeginLoop->Next; end_loops > 0; inst = inst->Next){
		switch(inst->U.I.Opcode){
		/* XXX In the future we might want to try to unroll nested
		 * loops here.*/
		case RC_OPCODE_BGNLOOP:
			end_loops++;
			break;
		case RC_OPCODE_ENDLOOP:
			loop->EndLoop = inst;
			end_loops--;
			break;
		case RC_OPCODE_BRK:
			/* Don't unroll loops if it has a BRK instruction
			 * other one used when testing the main conditional
			 * of the loop. */

			/* Make sure we haven't entered a nested loops. */
			if(inst != loop->Brk && end_loops == 1) {
				return 0;
			}
			break;
		case RC_OPCODE_IF:
			count_inst.BranchDepth++;
			break;
		case RC_OPCODE_ENDIF:
			count_inst.BranchDepth--;
			break;
		default:
			rc_for_all_writes_mask(inst, get_incr_amount, &count_inst);
			if(count_inst.Unknown){
				return 0;
			}
			break;
		}
	}
	/* Infinite loop */
	if(count_inst.Amount == 0.0f){
		return 0;
	}
	DBG("Counter is increased by %f each iteration.\n", count_inst.Amount);
	/* Calculate the number of iterations of this loop.  Keeping this
	 * simple, since we only support increment and decrement loops.
	 */
	limit_value = rc_get_constant_value(c, limit->Index, limit->Swizzle,
							limit->Negate, 0);
	DBG("Limit is %f.\n", limit_value);
	/* The iteration calculations are opposite of what you would expect.
	 * In a normal loop, if the condition is met, then loop continues, but
	 * with our loops, if the condition is met, the is exited. */
	switch(loop->Cond->U.I.Opcode){
	case RC_OPCODE_SGE:
	case RC_OPCODE_SLE:
		iterations = (int) ceilf((limit_value - counter_value.Value) /
							count_inst.Amount);
		break;

	case RC_OPCODE_SGT:
	case RC_OPCODE_SLT:
		iterations = (int) floorf((limit_value - counter_value.Value) /
							count_inst.Amount) + 1;
		break;
	default:
		return 0;
	}

	if (c->max_alu_insts > 0
		&& iterations > loop_max_possible_iterations(c, loop)) {
		return 0;
	}

	DBG("Loop will have %d iterations.\n", iterations);

	/* Prepare loop for unrolling */
	rc_remove_instruction(loop->Cond);
	rc_remove_instruction(loop->If);
	rc_remove_instruction(loop->Brk);
	rc_remove_instruction(loop->EndIf);

	unroll_loop(c, loop, iterations);
	loop->EndLoop = NULL;
	return 1;
}

/**
 * @param c
 * @param loop
 * @param inst A pointer to a BGNLOOP instruction.
 * @return 1 if all of the members of loop where set.
 * @return 0 if there was an error and some members of loop are still NULL.
 */
static int build_loop_info(struct radeon_compiler * c, struct loop_info * loop,
						struct rc_instruction * inst)
{
	struct rc_instruction * ptr;

	if(inst->U.I.Opcode != RC_OPCODE_BGNLOOP){
		rc_error(c, "%s: expected BGNLOOP", __FUNCTION__);
		return 0;
	}

	memset(loop, 0, sizeof(struct loop_info));

	loop->BeginLoop = inst;

	for(ptr = loop->BeginLoop->Next; !loop->EndLoop; ptr = ptr->Next) {

		if (ptr == &c->Program.Instructions) {
			rc_error(c, "%s: BGNLOOP without an ENDLOOP.\n",
								__FUNCTION__);
			return 0;
		}

		switch(ptr->U.I.Opcode){
		case RC_OPCODE_BGNLOOP:
		{
			/* Nested loop, skip ahead to the end. */
			unsigned int loop_depth = 1;
			for(ptr = ptr->Next; ptr != &c->Program.Instructions;
							ptr = ptr->Next){
				if (ptr->U.I.Opcode == RC_OPCODE_BGNLOOP) {
					loop_depth++;
				} else if (ptr->U.I.Opcode == RC_OPCODE_ENDLOOP) {
					if (!--loop_depth) {
						break;
					}
				}
			}
			if (ptr == &c->Program.Instructions) {
				rc_error(c, "%s: BGNLOOP without an ENDLOOP\n",
								__FUNCTION__);
					return 0;
			}
			break;
		}
		case RC_OPCODE_BRK:
		{
			struct rc_src_register *src;
			if(ptr->Next->U.I.Opcode != RC_OPCODE_ENDIF
					|| ptr->Prev->U.I.Opcode != RC_OPCODE_IF
					|| loop->Brk){
				continue;
			}
			loop->Brk = ptr;
			loop->If = ptr->Prev;
			loop->EndIf = ptr->Next;
			src = &loop->If->U.I.SrcReg[0];

			for (loop->Cond = loop->If->Prev;
				loop->Cond->U.I.Opcode != RC_OPCODE_BGNLOOP;
				loop->Cond = loop->Cond->Prev) {

				const struct rc_dst_register *dst = &loop->Cond->U.I.DstReg;
				if (dst->File == src->File &&
						dst->Index == src->Index &&
						dst->WriteMask & (rc_swizzle_to_writemask(src->Swizzle))) {
					if (loop->Cond->U.I.Opcode == RC_OPCODE_CMP) {
						src = &loop->Cond->U.I.SrcReg[0];
						continue;
					}
					break;
				}
			}

			if (loop->Cond->U.I.Opcode == RC_OPCODE_BGNLOOP) {
				rc_error(c, "%s: Cannot find condition for if\n", __FUNCTION__);
				return 0;
			}
			break;
		}
		case RC_OPCODE_ENDLOOP:
			loop->EndLoop = ptr;
			break;
		}
	}

	if (loop->BeginLoop && loop->Brk && loop->If && loop->EndIf
					&& loop->Cond && loop->EndLoop) {
		return 1;
	}
	return 0;
}

/**
 * This function prepares a loop to be unrolled by converting it into an if
 * statement.  Here is an outline of the conversion process:
 * BGNLOOP;                         	-> BGNLOOP;
 * <Additional conditional code>	-> <Additional conditional code>
 * SGE/SLT temp[0], temp[1], temp[2];	-> SLT/SGE temp[0], temp[1], temp[2];
 * IF temp[0];                      	-> IF temp[0];
 * BRK;                             	->
 * ENDIF;                           	-> <Loop Body>
 * <Loop Body>                      	-> ENDIF;
 * ENDLOOP;                         	-> ENDLOOP
 *
 * @param inst A pointer to a BGNLOOP instruction.
 * @return 1 for success, 0 for failure
 */
static int transform_loop(struct emulate_loop_state * s,
						struct rc_instruction * inst)
{
	struct loop_info * loop;

	memory_pool_array_reserve(&s->C->Pool, struct loop_info,
			s->Loops, s->LoopCount, s->LoopReserved, 1);

	loop = &s->Loops[s->LoopCount++];

	if (!build_loop_info(s->C, loop, inst)) {
		rc_error(s->C, "Failed to build loop info\n");
		return 0;
	}

	if(try_unroll_loop(s->C, loop)){
		return 1;
	}

	/* Reverse the conditional instruction */
	switch(loop->Cond->U.I.Opcode){
	case RC_OPCODE_SGE:
		loop->Cond->U.I.Opcode = RC_OPCODE_SLT;
		break;
	case RC_OPCODE_SLT:
		loop->Cond->U.I.Opcode = RC_OPCODE_SGE;
		break;
	case RC_OPCODE_SLE:
		loop->Cond->U.I.Opcode = RC_OPCODE_SGT;
		break;
	case RC_OPCODE_SGT:
		loop->Cond->U.I.Opcode = RC_OPCODE_SLE;
		break;
	case RC_OPCODE_SEQ:
		loop->Cond->U.I.Opcode = RC_OPCODE_SNE;
		break;
	case RC_OPCODE_SNE:
		loop->Cond->U.I.Opcode = RC_OPCODE_SEQ;
		break;
	default:
		rc_error(s->C, "loop->Cond is not a conditional.\n");
		return 0;
	}

	/* Prepare the loop to be emulated */
	rc_remove_instruction(loop->Brk);
	rc_remove_instruction(loop->EndIf);
	rc_insert_instruction(loop->EndLoop->Prev, loop->EndIf);
	return 1;
}

void rc_transform_loops(struct radeon_compiler *c, void *user)
{
	struct emulate_loop_state * s = &c->loop_state;
	struct rc_instruction * ptr;

	memset(s, 0, sizeof(struct emulate_loop_state));
	s->C = c;
	for(ptr = s->C->Program.Instructions.Next;
			ptr != &s->C->Program.Instructions; ptr = ptr->Next) {
		if(ptr->Type == RC_INSTRUCTION_NORMAL &&
					ptr->U.I.Opcode == RC_OPCODE_BGNLOOP){
			if (!transform_loop(s, ptr))
				return;
		}
	}
}

void rc_unroll_loops(struct radeon_compiler *c, void *user)
{
	struct rc_instruction * inst;
	struct loop_info loop;

	for(inst = c->Program.Instructions.Next;
			inst != &c->Program.Instructions; inst = inst->Next) {

		if (inst->U.I.Opcode == RC_OPCODE_BGNLOOP) {
			if (build_loop_info(c, &loop, inst)) {
				try_unroll_loop(c, &loop);
			}
		}
	}
}

void rc_emulate_loops(struct radeon_compiler *c, void *user)
{
	struct emulate_loop_state * s = &c->loop_state;
	int i;
	/* Iterate backwards of the list of loops so that loops that nested
	 * loops are unrolled first.
	 */
	for( i = s->LoopCount - 1; i >= 0; i-- ){
		unsigned int iterations;

		if(!s->Loops[i].EndLoop){
			continue;
		}
		iterations = loop_max_possible_iterations(s->C, &s->Loops[i]);
		unroll_loop(s->C, &s->Loops[i], iterations);
	}
}