path: root/src/gallium/drivers/etnaviv/etnaviv_nir.c

/*
 * Copyright (c) 2018 Etnaviv Project
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sub license,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Authors:
 *    Philipp Zabel <p.zabel@pengutronix.de>
 *    Michael Tretter <m.tretter@pengutronix.de>
 */

#include "etnaviv_nir.h"

#include "etnaviv_compiler.h"
#include "etnaviv_debug.h"
#include "etnaviv_shader.h"

#include "compiler/nir/nir.h"
#include "compiler/nir/nir_control_flow.h"
#include "nir/tgsi_to_nir.h"
#include "util/register_allocate.h"

static const nir_shader_compiler_options options = {
   .fuse_ffma = true,
   .lower_flrp32 = true,
   .lower_fpow = true,
   .lower_sub = true,
   .fdot_replicates = true,
   .lower_extract_byte = true,
   .lower_extract_word = true,
   .max_unroll_iterations = 32,
};

const nir_shader_compiler_options *
etna_get_compiler_options()
{
   if (etna_mesa_debug & ETNA_DBG_NIR)
      return &options;
   return NULL;
}

struct nir_shader *
etna_tgsi_to_nir(const struct tgsi_token *tokens) {
   return tgsi_to_nir(tokens, &options);
}

static void
etna_optimize_loop(nir_shader *s)
{
   bool progress;

   do {
      progress = false;

      NIR_PASS_V(s, nir_lower_to_source_mods);
      NIR_PASS_V(s, nir_lower_vars_to_ssa);
      NIR_PASS(progress, s, nir_copy_prop);
      NIR_PASS(progress, s, nir_opt_dce);
      NIR_PASS(progress, s, nir_opt_dead_cf);
      NIR_PASS(progress, s, nir_opt_cse);
      NIR_PASS(progress, s, nir_opt_algebraic);
      NIR_PASS(progress, s, nir_opt_constant_folding);
      NIR_PASS(progress, s, nir_opt_undef);
      NIR_PASS(progress, s, nir_opt_remove_phis);
   } while (progress);
}

/* Move const loads, input load intrinsics, and uniform load intrinsics to the
 * beginning of the function implementation.
 *
 * Input loads are moved to the beginning to guarantee that the SSA covers the
 * lifetime of the input register value. Const loads are moved since their
 * SSAs can be sources to the input loads. Uniform loads are moved to the
 * beginning for decorative purposes only, these will be folded into their
 * users later.  Moving uniform loads and const loads to the beginning of the
 * function, and ahead of input loads is safe because their SSAs will never be
 * replaced with registers.
 */
static void
etna_move_load_intrinsics(nir_shader *shader)
{
   nir_foreach_function(function, shader) {
      nir_block *start_block = nir_start_block(function->impl);
      nir_instr *first_instr = nir_block_first_instr(start_block);

      nir_foreach_block(block, function->impl) {
         nir_foreach_instr_safe(instr, block) {
            if (instr->type == nir_instr_type_intrinsic) {
               nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
               if (intr->intrinsic != nir_intrinsic_load_input &&
                   intr->intrinsic != nir_intrinsic_load_uniform)
                  continue;
            } else if (instr->type != nir_instr_type_load_const) {
               continue;
            }

            if (instr == first_instr) {
               first_instr = nir_instr_next(instr);
            } else {
               /* We don't use nir_instr_remove here to keep uses and defs */
               exec_node_remove(&instr->node);
               exec_node_insert_node_before(&first_instr->node, &instr->node);
               instr->block = first_instr->block;
            }
         }
      }

      nir_metadata_preserve(function->impl, nir_metadata_block_index);
   }
}

/* Insert a mov to a new SSA directly before instr, replacing src */
static void
insert_mov(nir_instr *instr, nir_src *src, nir_shader *shader)
{
   unsigned num_components, max_swizzle;

   nir_alu_instr *mov = nir_alu_instr_create(shader, nir_op_fmov);
   nir_src_copy(&mov->src[0].src, src, &mov->instr);

   num_components = nir_src_num_components(mov->src[0].src);
   max_swizzle = num_components - 1;
   mov->src[0].swizzle[0] = 0;
   mov->src[0].swizzle[1] = MIN2(max_swizzle, 1);
   mov->src[0].swizzle[2] = MIN2(max_swizzle, 2);
   mov->src[0].swizzle[3] = MIN2(max_swizzle, 3);
   mov->dest.write_mask = (1 << num_components) - 1;
   nir_ssa_dest_init(&mov->instr, &mov->dest.dest, 4, 32, NULL);
   nir_instr_rewrite_src(instr, src,
                         nir_src_for_ssa(&mov->dest.dest.ssa));
   nir_instr_insert_before(instr, &mov->instr);
}

/* Check if there is an input load intrinsic instruction with a destination
 * SSA that would be assigned to reg and interferes with the given SSA.
 */
static bool
interferes_with_input_load_dest(nir_ssa_def *ssa, int reg,
                                nir_function_impl *impl)
{
   nir_foreach_instr(instr, nir_start_block(impl)) {
      if (instr->type != nir_instr_type_intrinsic)
         continue;

      nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
      if (intr->intrinsic != nir_intrinsic_load_input)
         continue;

      int base = nir_intrinsic_base(intr);
      nir_const_value *const_offset = nir_src_as_const_value(intr->src[0]);
      int offset = const_offset->u32[0];

      if (base != reg)
         continue;

      assert(offset == 0);
      assert(intr->dest.is_ssa);

      if (ssa == &intr->dest.ssa)
         continue;

      if (nir_ssa_defs_interfere(ssa, &intr->dest.ssa))
         return true;
   }

   return false;
}

/* Split output store intrinsics into a mov to a temporary SSA and the actual
 * store, if necessary, that is, if the output store source SSA interferes with
 * an input load destination SSA that would be forced to the same register.
 *
 * This must be done after etna_move_load_intrinsics moved load intrinsics to
 * the beginning of the function, and should be done after the optimization
 * loop. There must be no copy propagation optimization step between this pass
 * and the register assignment.
 */
static void
etna_lower_store_intrinsics(nir_shader *shader)
{
   nir_foreach_function(function, shader) {
      nir_metadata_require(function->impl, nir_metadata_live_ssa_defs);
      nir_foreach_block(block, function->impl) {
         nir_foreach_instr(instr, block) {
            if (instr->type != nir_instr_type_intrinsic)
               continue;

            nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
            if (intr->intrinsic != nir_intrinsic_store_output)
               continue;

            int base = nir_intrinsic_base(intr);
            nir_const_value *const_offset = nir_src_as_const_value(intr->src[1]);
            int offset = const_offset->u32[0];

            assert(offset == 0);
            assert(intr->src[0].is_ssa);

            if (interferes_with_input_load_dest(intr->src[0].ssa, base,
                                                function->impl))
               insert_mov(&intr->instr, &intr->src[0], shader);
         }
      }
      nir_metadata_preserve(function->impl,
                            nir_metadata_block_index | nir_metadata_dominance);
   }
}

/* Return the destination SSA if it should be replaced with a global register,
 * or NULL.
 */
static nir_ssa_def *
etna_instr_replaceable_ssa_dest(nir_instr *instr)
{
   if (instr->type == nir_instr_type_load_const)
      return NULL;

   if (instr->type == nir_instr_type_intrinsic) {
      nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);

      /* Skip (constant and uniform) load intrinsics. We'll fold those
       * into the using instructions later in the code emitter.
       */
      if (intr->intrinsic == nir_intrinsic_load_input)
         return intr->dest.is_ssa ? &intr->dest.ssa : NULL;
      else
         return NULL;
   }

   if (instr->type == nir_instr_type_alu) {
      nir_alu_instr *alu = nir_instr_as_alu(instr);

      return alu->dest.dest.is_ssa ? &alu->dest.dest.ssa : NULL;
   }

   if (instr->type == nir_instr_type_tex) {
      nir_tex_instr *tex = nir_instr_as_tex(instr);

      return tex->dest.is_ssa ? &tex->dest.ssa : NULL;
   }

   return NULL;
}

/* Return the NIR global register corresponding to a given temporary register,
 * creating it if necessary.
 */
static nir_register *
etna_ensure_temporary(nir_shader *shader, int index)
{
   nir_foreach_register(reg, &shader->registers) {
      if (reg->index == index)
         return reg;
   }

   nir_register *reg = nir_global_reg_create(shader);
   reg->num_components = 4;
   reg->num_array_elems = 0;
   reg->bit_size = 32;
   reg->index = index;
   if (shader->reg_alloc < index + 1)
      shader->reg_alloc = index + 1;
   reg->name = NULL;

   return reg;
}

/* Assign global registers, with index numbers corresponding to the
 * 64-register temporary register file available to each shader invocation.
 *
 * This expects all load intrinsics to be moved to the start of the function,
 * and all store intrinsics to be moved to the end of the function already, so
 * that interference between input, output, and temporary values is described
 * correctly.
 */
static void
etna_assign_registers(nir_shader *shader)
{
   struct ra_regs *regs = ra_alloc_reg_set(NULL, 64, false);
   int class = ra_alloc_reg_class(regs);
   unsigned int **q_values;
   unsigned int *input_reg;
   unsigned int *output_reg;

   /* Input/output registers only have to be assigned manually to the beginning
    * of the temporary register range in the fragment shader. Otherwise the
    * register allocator may choose freely.
    */
   bool preassign_io = shader->info.stage == MESA_SHADER_FRAGMENT;

   if (!preassign_io) {
      input_reg = ralloc_array(NULL, unsigned, shader->num_inputs);
      output_reg = ralloc_array(NULL, unsigned, shader->num_outputs);
   }

   /* A single register file with 64 registers is available to each running
    * shader, with no conflicts between them.
    */
   for (int r = 0; r < 64; r++)
      ra_class_add_reg(regs, class, r);
   q_values = ralloc_array(regs, unsigned *, 1);
   q_values[0] = rzalloc_array(q_values, unsigned, 1);
   q_values[0][0] = 0;
   ra_set_finalize(regs, q_values);

   nir_foreach_function(function, shader) {
      nir_metadata_require(function->impl, nir_metadata_live_ssa_defs);

      int count = 0;

      /* Count SSA assignments to be replaced with registers. */
      nir_foreach_block(block, function->impl) {
         nir_foreach_instr(instr, block) {
            if (etna_instr_replaceable_ssa_dest(instr))
               count++;
         }
      }

      nir_ssa_def **ssa_defs = ralloc_array(NULL, nir_ssa_def *, count);

      /* Collect SSA assignments to be replaced with registers */
      int i = 0;
      nir_foreach_block(block, function->impl) {
         nir_foreach_instr(instr, block) {
            nir_ssa_def *ssa = etna_instr_replaceable_ssa_dest(instr);

            if (ssa)
               ssa_defs[i++] = ssa;
         }
      }

      struct ra_graph *g = ra_alloc_interference_graph(regs, count);

      /* Collect SSA interference information and force input loads to
       * the correct registers in the fragment shader.
       */
      for (i = 0; i < count; i++) {
         nir_ssa_def *ssa = ssa_defs[i];

         for (int j = 0; j < i; j++) {
            if (nir_ssa_defs_interfere(ssa, ssa_defs[j]))
               ra_add_node_interference(g, i, j);
         }

         if (preassign_io) {
            nir_instr *instr = ssa->parent_instr;

            if (instr->type != nir_instr_type_intrinsic)
               continue;

            nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
            if (intr->intrinsic != nir_intrinsic_load_input)
               continue;

            int base = nir_intrinsic_base(intr);
            nir_const_value *const_offset = nir_src_as_const_value(intr->src[0]);
            int offset = const_offset->u32[0];

            assert(offset == 0);

            ra_set_node_reg(g, i, base);
         }
      }

      if (preassign_io) {
         /* Force output stores to the correct registers */
         nir_foreach_block(block, function->impl) {
            nir_foreach_instr(instr, block) {
               if (instr->type != nir_instr_type_intrinsic)
                  continue;

               nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);

               if (intr->intrinsic != nir_intrinsic_store_output)
                  continue;

               int base = nir_intrinsic_base(intr);
               nir_const_value *const_offset = nir_src_as_const_value(intr->src[1]);
               int offset = const_offset->u32[0];

               assert(offset == 0);
               assert(intr->src[0].is_ssa);

               /* Find the replaceable SSA used as source */
               for (i = 0; i < count; i++) {
                  if (ssa_defs[i] == intr->src[0].ssa)
                     ra_set_node_reg(g, i, base);
               }
            }
         }
      }

      /* Allocate registers */
      bool ok = ra_allocate(g);
      assert(ok);

      /* Replace SSA assignments with allocated registers */
      for (i = 0; i < count; i++) {
         int r = ra_get_node_reg(g, i);
         nir_register *reg = etna_ensure_temporary(shader, r);
         nir_ssa_def *ssa = ssa_defs[i];

         nir_ssa_def_rewrite_uses(ssa, nir_src_for_reg(reg));
         assert(list_empty(&ssa->uses) && list_empty(&ssa->if_uses));

         nir_instr *instr = ssa->parent_instr;

         if (instr->type == nir_instr_type_alu) {
            nir_alu_instr *alu = nir_instr_as_alu(instr);

            nir_instr_rewrite_dest(&alu->instr, &alu->dest.dest,
                                   nir_dest_for_reg(reg));
         } else if (instr->type == nir_instr_type_tex) {
            nir_tex_instr *tex = nir_instr_as_tex(instr);

            nir_instr_rewrite_dest(&tex->instr, &tex->dest,
                                   nir_dest_for_reg(reg));
         } else if (instr->type == nir_instr_type_intrinsic) {
            nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);

            if (intr->intrinsic == nir_intrinsic_load_input) {
               nir_instr_rewrite_dest(&intr->instr, &intr->dest,
                                      nir_dest_for_reg(reg));
               if (!preassign_io) {
                  int base = nir_intrinsic_base(intr);

                  input_reg[base] = reg->index;
                  nir_intrinsic_set_base(intr, reg->index);
               }
            }
         }
      }

      if (!preassign_io) {
         /* Update output store base from assigned register */
         nir_foreach_block(block, function->impl) {
            nir_foreach_instr(instr, block) {
               if (instr->type != nir_instr_type_intrinsic)
                  continue;

               nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);

               if (intr->intrinsic != nir_intrinsic_store_output)
                  continue;

               assert(!intr->src[0].is_ssa);
               nir_register *reg = intr->src[0].reg.reg;

               int base = nir_intrinsic_base(intr);
               output_reg[base] = reg->index;

               nir_intrinsic_set_base(intr, reg->index);
            }
         }
      }

      ralloc_free(g);
      ralloc_free(ssa_defs);
      nir_metadata_preserve(function->impl,
                            nir_metadata_block_index | nir_metadata_dominance);
   }

   if (!preassign_io) {
      nir_foreach_variable(var, &shader->inputs)
         var->data.driver_location = input_reg[var->data.driver_location];
      nir_foreach_variable(var, &shader->outputs)
         var->data.driver_location = output_reg[var->data.driver_location];

      ralloc_free(output_reg);
      ralloc_free(input_reg);
   }

   ralloc_free(regs);
}

/* Remove input_load and output_store intrinsics after global register
 * allocation. After the SSA destinations are replaced, these contain no useful
 * information anymore.
 *
 * This cleanup step should be called after etna_assign_registers.
 */
static void
etna_remove_io_intrinsics(nir_shader *shader)
{
   nir_foreach_function(function, shader) {
      nir_foreach_block(block, function->impl) {
         nir_foreach_instr_safe(instr, block) {
            if (instr->type != nir_instr_type_intrinsic)
               continue;

            nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);

            if (intr->intrinsic == nir_intrinsic_load_input) {
               assert(!intr->dest.is_ssa && intr->dest.reg.reg->is_global);

               int base = nir_intrinsic_base(intr);
               nir_const_value *const_offset = nir_src_as_const_value(intr->src[0]);
               int offset = const_offset->u32[0];

               assert(base == intr->dest.reg.reg->index);
               assert(offset == 0);

               nir_instr_remove(instr);
            } else if (intr->intrinsic == nir_intrinsic_store_output) {
               assert(!intr->src[0].is_ssa && intr->src[0].reg.reg->is_global);

               int base = nir_intrinsic_base(intr);
               nir_const_value *const_offset = nir_src_as_const_value(intr->src[1]);
               int offset = const_offset->u32[0];

               assert(base == intr->src[0].reg.reg->index);
               assert(offset == 0);

               nir_instr_remove(instr);
            }
         }
      }
      nir_metadata_preserve(function->impl, nir_metadata_block_index);
   }
}

struct nir_shader *
etna_optimize_nir(struct etna_shader *shader,
                  struct nir_shader *s,
                  const struct etna_shader_key *key)
{
   NIR_PASS_V(s, nir_copy_prop);

   NIR_PASS_V(s, nir_opt_global_to_local);
   NIR_PASS_V(s, nir_lower_regs_to_ssa);
   NIR_PASS_V(s, etna_move_load_intrinsics);

   etna_optimize_loop(s);

   NIR_PASS_V(s, nir_remove_dead_variables, nir_var_local);

   /* Introduces additional movs to avoid input/output register conflicts */
   NIR_PASS_V(s, etna_lower_store_intrinsics);

   NIR_PASS_V(s, nir_convert_from_ssa, true);
   NIR_PASS_V(s, etna_assign_registers);
   NIR_PASS_V(s, etna_remove_io_intrinsics);

   NIR_PASS_V(s, nir_opt_dce);

   /* Do this after register assignment to avoid creating temporary registers
    * that cause suboptimal register assignment.
    */
   NIR_PASS_V(s, nir_move_vec_src_uses_to_dest);
   NIR_PASS_V(s, nir_lower_vec_to_movs);

   nir_sweep(s);

   if (DBG_ENABLED(ETNA_DBG_DUMP_SHADERS)) {
      printf("=============[ NIR ]=============\n");
      nir_print_shader(s, stdout);
      printf("---------------------------------\n");
   }

   return s;
}