anv: Implement VK_EXT_inline_uniform_block

Reviewed-by: Lionel Landwerlin <lionel.g.landwerlin@intel.com>

src/intel/vulkan/anv_cmd_buffer.c

@@ -30,6 +30,7 @@
 #include "anv_private.h"
 
 #include "vk_format_info.h"
+#include "vk_util.h"
 
 /** \file anv_cmd_buffer.c
  *
@@ -1109,6 +1110,19 @@ void anv_CmdPushDescriptorSetKHR(
          }
          break;
 
+      case VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT: {
+         const VkWriteDescriptorSetInlineUniformBlockEXT *inline_write =
+            vk_find_struct_const(write->pNext,
+                                 WRITE_DESCRIPTOR_SET_INLINE_UNIFORM_BLOCK_EXT);
+         assert(inline_write->dataSize == write->descriptorCount);
+         anv_descriptor_set_write_inline_uniform_data(cmd_buffer->device, set,
+                                                      write->dstBinding,
+                                                      inline_write->pData,
+                                                      write->dstArrayElement,
+                                                      inline_write->dataSize);
+         break;
+      }
+
       default:
          break;
       }

src/intel/vulkan/anv_descriptor_set.c

@@ -28,6 +28,7 @@
 #include <fcntl.h>
 
 #include "util/mesa-sha1.h"
+#include "vk_util.h"
 
 #include "anv_private.h"
 
@@ -75,6 +76,10 @@ anv_descriptor_data_for_type(const struct anv_physical_device *device,
       data = ANV_DESCRIPTOR_SURFACE_STATE;
       break;
 
+   case VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT:
+      data = ANV_DESCRIPTOR_INLINE_UNIFORM;
+      break;
+
    default:
       unreachable("Unsupported descriptor type");
    }
@@ -92,6 +97,11 @@ anv_descriptor_data_size(enum anv_descriptor_data data)
 unsigned
 anv_descriptor_size(const struct anv_descriptor_set_binding_layout *layout)
 {
+   if (layout->data & ANV_DESCRIPTOR_INLINE_UNIFORM) {
+      assert(layout->data == ANV_DESCRIPTOR_INLINE_UNIFORM);
+      return layout->array_size;
+   }
+
    return anv_descriptor_data_size(layout->data);
 }
 
@@ -106,6 +116,7 @@ unsigned
 anv_descriptor_type_size(const struct anv_physical_device *pdevice,
                          VkDescriptorType type)
 {
+   assert(type != VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT);
    return anv_descriptor_data_size(anv_descriptor_data_for_type(pdevice, type));
 }
 
@@ -295,9 +306,19 @@ VkResult anv_CreateDescriptorSetLayout(
          break;
       }
 
-      set_layout->binding[b].descriptor_offset = descriptor_buffer_size;
-      descriptor_buffer_size += anv_descriptor_size(&set_layout->binding[b]) *
-                                binding->descriptorCount;
+      if (binding->descriptorType ==
+          VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT) {
+         /* Inline uniform blocks are specified to use the descriptor array
+          * size as the size in bytes of the block.
+          */
+         descriptor_buffer_size = align_u32(descriptor_buffer_size, 32);
+         set_layout->binding[b].descriptor_offset = descriptor_buffer_size;
+         descriptor_buffer_size += binding->descriptorCount;
+      } else {
+         set_layout->binding[b].descriptor_offset = descriptor_buffer_size;
+         descriptor_buffer_size += anv_descriptor_size(&set_layout->binding[b]) *
+                                   binding->descriptorCount;
+      }
 
       set_layout->shader_stages |= binding->stageFlags;
    }
@@ -472,6 +493,10 @@ VkResult anv_CreateDescriptorPool(
    ANV_FROM_HANDLE(anv_device, device, _device);
    struct anv_descriptor_pool *pool;
 
+   const VkDescriptorPoolInlineUniformBlockCreateInfoEXT *inline_info =
+      vk_find_struct_const(pCreateInfo->pNext,
+                           DESCRIPTOR_POOL_INLINE_UNIFORM_BLOCK_CREATE_INFO_EXT);
+
    uint32_t descriptor_count = 0;
    uint32_t buffer_view_count = 0;
    uint32_t descriptor_bo_size = 0;
@@ -485,6 +510,16 @@ VkResult anv_CreateDescriptorPool(
 
       unsigned desc_data_size = anv_descriptor_data_size(desc_data) *
                                 pCreateInfo->pPoolSizes[i].descriptorCount;
+
+      if (pCreateInfo->pPoolSizes[i].type ==
+          VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT) {
+         /* Inline uniform blocks are specified to use the descriptor array
+          * size as the size in bytes of the block.
+          */
+         assert(inline_info);
+         desc_data_size += pCreateInfo->pPoolSizes[i].descriptorCount;
+      }
+
       descriptor_bo_size += desc_data_size;
 
       descriptor_count += pCreateInfo->pPoolSizes[i].descriptorCount;
@@ -499,6 +534,9 @@ VkResult anv_CreateDescriptorPool(
     */
    descriptor_bo_size += 32 * pCreateInfo->maxSets;
    descriptor_bo_size = ALIGN(descriptor_bo_size, 4096);
+   /* We align inline uniform blocks to 32B */
+   if (inline_info)
+      descriptor_bo_size += 32 * inline_info->maxInlineUniformBlockBindings;
 
    const size_t pool_size =
       pCreateInfo->maxSets * sizeof(struct anv_descriptor_set) +
@@ -971,6 +1009,24 @@ anv_descriptor_set_write_buffer(struct anv_device *device,
    }
 }
 
+void
+anv_descriptor_set_write_inline_uniform_data(struct anv_device *device,
+                                             struct anv_descriptor_set *set,
+                                             uint32_t binding,
+                                             const void *data,
+                                             size_t offset,
+                                             size_t size)
+{
+   const struct anv_descriptor_set_binding_layout *bind_layout =
+      &set->layout->binding[binding];
+
+   assert(bind_layout->data & ANV_DESCRIPTOR_INLINE_UNIFORM);
+
+   void *desc_map = set->desc_mem.map + bind_layout->descriptor_offset;
+
+   memcpy(desc_map + offset, data, size);
+}
+
 void anv_UpdateDescriptorSets(
     VkDevice                                    _device,
     uint32_t                                    descriptorWriteCount,
@@ -1033,6 +1089,19 @@ void anv_UpdateDescriptorSets(
          }
          break;
 
+      case VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT: {
+         const VkWriteDescriptorSetInlineUniformBlockEXT *inline_write =
+            vk_find_struct_const(write->pNext,
+                                 WRITE_DESCRIPTOR_SET_INLINE_UNIFORM_BLOCK_EXT);
+         assert(inline_write->dataSize == write->descriptorCount);
+         anv_descriptor_set_write_inline_uniform_data(device, set,
+                                                      write->dstBinding,
+                                                      inline_write->pData,
+                                                      write->dstArrayElement,
+                                                      inline_write->dataSize);
+         break;
+      }
+
       default:
          break;
       }
@@ -1058,14 +1127,23 @@ void anv_UpdateDescriptorSets(
       for (uint32_t j = 0; j < copy->descriptorCount; j++)
          dst_desc[j] = src_desc[j];
 
-      unsigned desc_size = anv_descriptor_size(src_layout);
-      if (desc_size > 0) {
-         assert(desc_size == anv_descriptor_size(dst_layout));
+      if (src_layout->data & ANV_DESCRIPTOR_INLINE_UNIFORM) {
+         assert(src_layout->data == ANV_DESCRIPTOR_INLINE_UNIFORM);
          memcpy(dst->desc_mem.map + dst_layout->descriptor_offset +
-                                    copy->dstArrayElement * desc_size,
+                                    copy->dstArrayElement,
                 src->desc_mem.map + src_layout->descriptor_offset +
-                                    copy->srcArrayElement * desc_size,
-                copy->descriptorCount * desc_size);
+                                    copy->srcArrayElement,
+                copy->descriptorCount);
+      } else {
+         unsigned desc_size = anv_descriptor_size(src_layout);
+         if (desc_size > 0) {
+            assert(desc_size == anv_descriptor_size(dst_layout));
+            memcpy(dst->desc_mem.map + dst_layout->descriptor_offset +
+                                       copy->dstArrayElement * desc_size,
+                   src->desc_mem.map + src_layout->descriptor_offset +
+                                       copy->srcArrayElement * desc_size,
+                   copy->descriptorCount * desc_size);
+         }
       }
    }
 }
@@ -1135,6 +1213,14 @@ anv_descriptor_set_write_template(struct anv_device *device,
          }
          break;
 
+      case VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT:
+         anv_descriptor_set_write_inline_uniform_data(device, set,
+                                                      entry->binding,
+                                                      data + entry->offset,
+                                                      entry->array_element,
+                                                      entry->array_count);
+         break;
+
       default:
          break;
       }

src/intel/vulkan/anv_device.c

@@ -950,6 +950,14 @@ void anv_GetPhysicalDeviceFeatures2(
          break;
       }
 
+      case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_INLINE_UNIFORM_BLOCK_FEATURES_EXT: {
+         VkPhysicalDeviceInlineUniformBlockFeaturesEXT *features =
+            (VkPhysicalDeviceInlineUniformBlockFeaturesEXT *)ext;
+         features->inlineUniformBlock = true;
+         features->descriptorBindingInlineUniformBlockUpdateAfterBind = false;
+         break;
+      }
+
       case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MULTIVIEW_FEATURES: {
          VkPhysicalDeviceMultiviewFeatures *features =
             (VkPhysicalDeviceMultiviewFeatures *)ext;
@@ -1240,6 +1248,21 @@ void anv_GetPhysicalDeviceProperties2(
          break;
       }
 
+      case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_INLINE_UNIFORM_BLOCK_PROPERTIES_EXT: {
+         VkPhysicalDeviceInlineUniformBlockPropertiesEXT *props =
+            (VkPhysicalDeviceInlineUniformBlockPropertiesEXT *)ext;
+         props->maxInlineUniformBlockSize = MAX_INLINE_UNIFORM_BLOCK_SIZE;
+         props->maxPerStageDescriptorInlineUniformBlocks =
+            MAX_INLINE_UNIFORM_BLOCK_DESCRIPTORS;
+         props->maxPerStageDescriptorUpdateAfterBindInlineUniformBlocks =
+            MAX_INLINE_UNIFORM_BLOCK_DESCRIPTORS;
+         props->maxDescriptorSetInlineUniformBlocks =
+            MAX_INLINE_UNIFORM_BLOCK_DESCRIPTORS;
+         props->maxDescriptorSetUpdateAfterBindInlineUniformBlocks =
+            MAX_INLINE_UNIFORM_BLOCK_DESCRIPTORS;
+         break;
+      }
+
       case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_3_PROPERTIES: {
          VkPhysicalDeviceMaintenance3Properties *props =
             (VkPhysicalDeviceMaintenance3Properties *)ext;

src/intel/vulkan/anv_extensions.py

@@ -125,6 +125,7 @@ EXTENSIONS = [
     Extension('VK_EXT_external_memory_dma_buf',           1, True),
     Extension('VK_EXT_global_priority',                   1,
               'device->has_context_priority'),
+    Extension('VK_EXT_inline_uniform_block',              1, True),
     Extension('VK_EXT_pci_bus_info',                      2, True),
     Extension('VK_EXT_post_depth_coverage',               1, 'device->info.gen >= 9'),
     Extension('VK_EXT_sampler_filter_minmax',             1, 'device->info.gen >= 9'),

src/intel/vulkan/anv_nir_apply_pipeline_layout.c

@@ -150,18 +150,28 @@ lower_res_index_intrinsic(nir_intrinsic_instr *intrin,
    uint32_t set = nir_intrinsic_desc_set(intrin);
    uint32_t binding = nir_intrinsic_binding(intrin);
 
+   const struct anv_descriptor_set_binding_layout *bind_layout =
+      &state->layout->set[set].layout->binding[binding];
+
    uint32_t surface_index = state->set[set].surface_offsets[binding];
-   uint32_t array_size =
-      state->layout->set[set].layout->binding[binding].array_size;
+   uint32_t array_size = bind_layout->array_size;
 
    nir_ssa_def *array_index = nir_ssa_for_src(b, intrin->src[0], 1);
    if (nir_src_is_const(intrin->src[0]) || state->add_bounds_checks)
       array_index = nir_umin(b, array_index, nir_imm_int(b, array_size - 1));
 
-   /* We're using nir_address_format_vk_index_offset */
-   nir_ssa_def *index =
-      nir_vec2(b, nir_iadd_imm(b, array_index, surface_index),
-                  nir_imm_int(b, 0));
+   nir_ssa_def *index;
+   if (bind_layout->data & ANV_DESCRIPTOR_INLINE_UNIFORM) {
+      /* This is an inline uniform block.  Just reference the descriptor set
+       * and use the descriptor offset as the base.
+       */
+      index = nir_imm_ivec2(b, state->set[set].desc_offset,
+                               bind_layout->descriptor_offset);
+   } else {
+      /* We're using nir_address_format_vk_index_offset */
+      index = nir_vec2(b, nir_iadd_imm(b, array_index, surface_index),
+                          nir_imm_int(b, 0));
+   }
 
    assert(intrin->dest.is_ssa);
    nir_ssa_def_rewrite_uses(&intrin->dest.ssa, nir_src_for_ssa(index));

src/intel/vulkan/anv_private.h

@@ -163,6 +163,8 @@ struct gen_l3_config;
 #define MAX_IMAGES 64
 #define MAX_GEN8_IMAGES 8
 #define MAX_PUSH_DESCRIPTORS 32 /* Minimum requirement */
+#define MAX_INLINE_UNIFORM_BLOCK_SIZE 4096
+#define MAX_INLINE_UNIFORM_BLOCK_DESCRIPTORS 32
 
 /* The kernel relocation API has a limitation of a 32-bit delta value
  * applied to the address before it is written which, in spite of it being
@@ -1507,6 +1509,8 @@ enum anv_descriptor_data {
    ANV_DESCRIPTOR_BUFFER_VIEW    = (1 << 2),
    /** The descriptor contains auxiliary image layout data */
    ANV_DESCRIPTOR_IMAGE_PARAM    = (1 << 3),
+   /** The descriptor contains auxiliary image layout data */
+   ANV_DESCRIPTOR_INLINE_UNIFORM = (1 << 4),
 };
 
 struct anv_descriptor_set_binding_layout {
@@ -1518,7 +1522,9 @@ struct anv_descriptor_set_binding_layout {
    /* Bitfield representing the type of data this descriptor contains */
    enum anv_descriptor_data data;
 
-   /* Number of array elements in this binding */
+   /* Number of array elements in this binding (or size in bytes for inline
+    * uniform data)
+    */
    uint16_t array_size;
 
    /* Index into the flattend descriptor set */
@@ -1732,6 +1738,13 @@ anv_descriptor_set_write_buffer(struct anv_device *device,
                                 uint32_t element,
                                 VkDeviceSize offset,
                                 VkDeviceSize range);
+void
+anv_descriptor_set_write_inline_uniform_data(struct anv_device *device,
+                                             struct anv_descriptor_set *set,
+                                             uint32_t binding,
+                                             const void *data,
+                                             size_t offset,
+                                             size_t size);
 
 void
 anv_descriptor_set_write_template(struct anv_device *device,