i965: Add blorp-based texture upload and download paths

v1 (Topi Pohjolainen): original patch.

v2 (Topi Pohjolainen):
   - Fix return value (s/MESA_FORMAT_NONE/false/) (Anuj)
   - Move _mesa_tex_format_from_format_and_type() just
     in the end avoiding additional if-block (Anuj)
   - Explain better the array alignment restriction (Anuj)
   - Do not bail out in case of gl_pixelstore_attrib::ImageHeight,
     it is handled by _mesa_image_offset() automatically (Ken).
   - Support 1D_ARRAY by flipping depth, width and y, z (Ken).

v3 (Topi Pohjolainen):
   - Contrary to v2, do not try to handle
     gl_pixelstore_attrib::ImageHeight. Currently there are no
     tests in piglit or cts for it. One could possibly copy or
     modify tests/texturing/texsubimage.c. There, however, seems
     to be number of corner cases to consider. Moreover, current
     meta path applies the packing height for both source and
     targets when determining the offset. This would probably
     require re-visiting also.

v4 (Topi Pohjolainen): Rebased on top of merged drm-bacon

v5 (Jason Ekstrand):
   - Move to brw_blorp.c
   - Significant refactoring
   - Fixed 1-D array textures
   - Simplified handling of PBOs vs. CPU data.
   - Handle gl_pixelstore_attrib::ImageHeight.  It turns out there are
     piglit tests that cover this. The original version was failing them
     because of an error in the way it handled 1-D array textures.
   - Add support for texture download

v6 (Kenneth Graunke): Rebase fixes:
   - Use intel_miptree_check_level_layer instead of deleted fields
   - Update for mesa_format_supports_render[] rename.
   - Pass 'false' (read-only) to intel_bufferobj_buffer

v7 (Kenneth Graunke):
   - Fix brw_blorp_download_miptree to pass 'false' (not read only) for
     the destination buffer (caught by Chris Wilson).
   - Fix blorp_get_client_bo to pass intel_bufferobj_buffer !read_only
     for the 'writable' parameter instead of 'false' (caught by Jason).
   - Support GL_BGR, GL_BGRA, GL_BGRA_INTEGER, GL_BGR_INTEGER, allowing
     us to use this for ReadPixels on the window system buffer (caught
     by Chris Wilson).
   - Fix y-flipping bugs in download path (exposed by BGRA support).
   - Fix false vs. NULL return value in blorp_get_client_bo.

Signed-off-by: Jason Ekstrand <jason@jlekstrand.net>
Reviewed-by: Topi Pohjolainen <topi.pohjolainen@intel.com>

src/mesa/drivers/dri/i965/brw_blorp.c

@@ -24,7 +24,10 @@
 #include "main/context.h"
 #include "main/teximage.h"
 #include "main/blend.h"
+#include "main/bufferobj.h"
+#include "main/enums.h"
 #include "main/fbobject.h"
+#include "main/image.h"
 #include "main/renderbuffer.h"
 #include "main/glformats.h"
 
@@ -33,6 +36,7 @@
 #include "brw_defines.h"
 #include "brw_meta_util.h"
 #include "brw_state.h"
+#include "intel_buffer_objects.h"
 #include "intel_fbo.h"
 #include "common/gen_debug.h"
 
@@ -758,6 +762,344 @@ brw_blorp_framebuffer(struct brw_context *brw,
    return mask;
 }
 
+static struct brw_bo *
+blorp_get_client_bo(struct brw_context *brw,
+                    unsigned w, unsigned h, unsigned d,
+                    GLenum target, GLenum format, GLenum type,
+                    const void *pixels,
+                    const struct gl_pixelstore_attrib *packing,
+                    uint32_t *offset_out, uint32_t *row_stride_out,
+                    uint32_t *image_stride_out, bool read_only)
+{
+   /* Account for SKIP_PIXELS, SKIP_ROWS, ALIGNMENT, and SKIP_IMAGES */
+   const GLuint dims = _mesa_get_texture_dimensions(target);
+   const uint32_t first_pixel = _mesa_image_offset(dims, packing, w, h,
+                                                   format, type, 0, 0, 0);
+   const uint32_t last_pixel =  _mesa_image_offset(dims, packing, w, h,
+                                                   format, type,
+                                                   d - 1, h - 1, w);
+   const uint32_t stride = _mesa_image_row_stride(packing, w, format, type);
+   const uint32_t cpp = _mesa_bytes_per_pixel(format, type);
+   const uint32_t size = last_pixel - first_pixel;
+
+   *row_stride_out = stride;
+   *image_stride_out = _mesa_image_image_stride(packing, w, h, format, type);
+
+   if (_mesa_is_bufferobj(packing->BufferObj)) {
+      const uint32_t offset = first_pixel + (intptr_t)pixels;
+      if (!read_only && ((offset % cpp) || (stride % cpp))) {
+         perf_debug("Bad PBO alignment; fallback to CPU mapping\n");
+         return NULL;
+      }
+
+      /* This is a user-provided PBO. We just need to get the BO out */
+      struct intel_buffer_object *intel_pbo =
+         intel_buffer_object(packing->BufferObj);
+      struct brw_bo *bo =
+         intel_bufferobj_buffer(brw, intel_pbo, offset, size, !read_only);
+
+      /* We take a reference to the BO so that the caller can just always
+       * unref without having to worry about whether it's a user PBO or one
+       * we created.
+       */
+      brw_bo_reference(bo);
+
+      *offset_out = offset;
+      return bo;
+   } else {
+      /* Someone should have already checked that there is data to upload. */
+      assert(pixels);
+
+      /* Creating a temp buffer currently only works for upload */
+      assert(read_only);
+
+      /* This is not a user-provided PBO.  Instead, pixels is a pointer to CPU
+       * data which we need to copy into a BO.
+       */
+      struct brw_bo *bo =
+         brw_bo_alloc(brw->bufmgr, "tmp_tex_subimage_src", size, 64);
+      if (bo == NULL) {
+         perf_debug("intel_texsubimage: temp bo creation failed: size = %u\n",
+                    size);
+         return NULL;
+      }
+
+      if (brw_bo_subdata(bo, 0, size, pixels + first_pixel)) {
+         perf_debug("intel_texsubimage: temp bo upload failed\n");
+         brw_bo_unreference(bo);
+         return NULL;
+      }
+
+      *offset_out = 0;
+      return bo;
+   }
+}
+
+/* Consider all the restrictions and determine the format of the source. */
+static mesa_format
+blorp_get_client_format(struct brw_context *brw,
+                        GLenum format, GLenum type,
+                        const struct gl_pixelstore_attrib *packing)
+{
+   if (brw->ctx._ImageTransferState)
+      return MESA_FORMAT_NONE;
+
+   if (packing->SwapBytes || packing->LsbFirst || packing->Invert) {
+      perf_debug("intel_texsubimage_blorp: unsupported gl_pixelstore_attrib\n");
+      return MESA_FORMAT_NONE;
+   }
+
+   if (format != GL_RED &&
+       format != GL_RG &&
+       format != GL_RGB &&
+       format != GL_BGR &&
+       format != GL_RGBA &&
+       format != GL_BGRA &&
+       format != GL_ALPHA &&
+       format != GL_RED_INTEGER &&
+       format != GL_RG_INTEGER &&
+       format != GL_RGB_INTEGER &&
+       format != GL_BGR_INTEGER &&
+       format != GL_RGBA_INTEGER &&
+       format != GL_BGRA_INTEGER) {
+      perf_debug("intel_texsubimage_blorp: %s not supported",
+                 _mesa_enum_to_string(format));
+      return MESA_FORMAT_NONE;
+   }
+
+   return _mesa_tex_format_from_format_and_type(&brw->ctx, format, type);
+}
+
+static bool
+need_signed_unsigned_int_conversion(mesa_format src_format,
+                                    mesa_format dst_format)
+{
+   const GLenum src_type = _mesa_get_format_datatype(src_format);
+   const GLenum dst_type = _mesa_get_format_datatype(dst_format);
+   return (src_type == GL_INT && dst_type == GL_UNSIGNED_INT) ||
+          (src_type == GL_UNSIGNED_INT && dst_type == GL_INT);
+}
+
+bool
+brw_blorp_upload_miptree(struct brw_context *brw,
+                         struct intel_mipmap_tree *dst_mt,
+                         mesa_format dst_format,
+                         uint32_t level, uint32_t x, uint32_t y, uint32_t z,
+                         uint32_t width, uint32_t height, uint32_t depth,
+                         GLenum target, GLenum format, GLenum type,
+                         const void *pixels,
+                         const struct gl_pixelstore_attrib *packing)
+{
+   const mesa_format src_format =
+      blorp_get_client_format(brw, format, type, packing);
+   if (src_format == MESA_FORMAT_NONE)
+      return false;
+
+   if (!brw->mesa_format_supports_render[dst_format]) {
+      perf_debug("intel_texsubimage: can't use %s as render target\n",
+                 _mesa_get_format_name(dst_format));
+      return false;
+   }
+
+   /* This function relies on blorp_blit to upload the pixel data to the
+    * miptree.  But, blorp_blit doesn't support signed to unsigned or
+    * unsigned to signed integer conversions.
+    */
+   if (need_signed_unsigned_int_conversion(src_format, dst_format))
+      return false;
+
+   uint32_t src_offset, src_row_stride, src_image_stride;
+   struct brw_bo *src_bo =
+      blorp_get_client_bo(brw, width, height, depth,
+                          target, format, type, pixels, packing,
+                          &src_offset, &src_row_stride,
+                          &src_image_stride, true);
+   if (src_bo == NULL)
+      return false;
+
+   /* Now that source is offset to correct starting point, adjust the
+    * given dimensions to treat 1D arrays as 2D.
+    */
+   if (target == GL_TEXTURE_1D_ARRAY) {
+      assert(depth == 1);
+      assert(z == 0);
+      depth = height;
+      height = 1;
+      z = y;
+      y = 0;
+      src_image_stride = src_row_stride;
+   }
+
+   intel_miptree_check_level_layer(dst_mt, level, z + depth - 1);
+
+   bool result = false;
+
+   /* Blit slice-by-slice creating a single-slice miptree for each layer. Even
+    * in case of linear buffers hardware wants image arrays to be aligned by
+    * four rows. This way hardware only gets one image at a time and any
+    * source alignment will do.
+    */
+   for (unsigned i = 0; i < depth; ++i) {
+      struct intel_mipmap_tree *src_mt = intel_miptree_create_for_bo(
+                                            brw, src_bo, src_format,
+                                            src_offset + i * src_image_stride,
+                                            width, height, 1,
+                                            src_row_stride, 0);
+
+      if (!src_mt) {
+         perf_debug("intel_texsubimage: miptree creation for src failed\n");
+         goto err;
+      }
+
+      /* In case exact match is needed, copy using equivalent UINT formats
+       * preventing hardware from changing presentation for SNORM -1.
+       */
+      if (src_mt->format == dst_format) {
+         brw_blorp_copy_miptrees(brw, src_mt, 0, 0,
+                                 dst_mt, level, z + i,
+                                 0, 0, x, y, width, height);
+      } else {
+         brw_blorp_blit_miptrees(brw, src_mt, 0, 0,
+                                 src_format, SWIZZLE_XYZW,
+                                 dst_mt, level, z + i,
+                                 dst_format,
+                                 0, 0, width, height,
+                                 x, y, x + width, y + height,
+                                 GL_NEAREST, false, false, false, false);
+      }
+
+      intel_miptree_release(&src_mt);
+   }
+
+   result = true;
+
+err:
+   brw_bo_unreference(src_bo);
+
+   return result;
+}
+
+bool
+brw_blorp_download_miptree(struct brw_context *brw,
+                           struct intel_mipmap_tree *src_mt,
+                           mesa_format src_format, uint32_t src_swizzle,
+                           uint32_t level, uint32_t x, uint32_t y, uint32_t z,
+                           uint32_t width, uint32_t height, uint32_t depth,
+                           GLenum target, GLenum format, GLenum type,
+                           bool y_flip, const void *pixels,
+                           const struct gl_pixelstore_attrib *packing)
+{
+   const mesa_format dst_format =
+      blorp_get_client_format(brw, format, type, packing);
+   if (dst_format == MESA_FORMAT_NONE)
+      return false;
+
+   if (!brw->mesa_format_supports_render[dst_format]) {
+      perf_debug("intel_texsubimage: can't use %s as render target\n",
+                 _mesa_get_format_name(dst_format));
+      return false;
+   }
+
+   /* This function relies on blorp_blit to download the pixel data from the
+    * miptree. But, blorp_blit doesn't support signed to unsigned or unsigned
+    * to signed integer conversions.
+    */
+   if (need_signed_unsigned_int_conversion(src_format, dst_format))
+      return false;
+
+   /* We can't fetch from LUMINANCE or intensity as that would require a
+    * non-trivial swizzle.
+    */
+   switch (_mesa_get_format_base_format(src_format)) {
+   case GL_LUMINANCE:
+   case GL_LUMINANCE_ALPHA:
+   case GL_INTENSITY:
+      return false;
+   default:
+      break;
+   }
+
+   /* This pass only works for PBOs */
+   assert(_mesa_is_bufferobj(packing->BufferObj));
+
+   uint32_t dst_offset, dst_row_stride, dst_image_stride;
+   struct brw_bo *dst_bo =
+      blorp_get_client_bo(brw, width, height, depth,
+                          target, format, type, pixels, packing,
+                          &dst_offset, &dst_row_stride,
+                          &dst_image_stride, false);
+   if (dst_bo == NULL)
+      return false;
+
+   /* Now that source is offset to correct starting point, adjust the
+    * given dimensions to treat 1D arrays as 2D.
+    */
+   if (target == GL_TEXTURE_1D_ARRAY) {
+      assert(depth == 1);
+      assert(z == 0);
+      depth = height;
+      height = 1;
+      z = y;
+      y = 0;
+      dst_image_stride = dst_row_stride;
+   }
+
+   intel_miptree_check_level_layer(src_mt, level, z + depth - 1);
+
+   int y0 = y;
+   int y1 = y + height;
+   if (y_flip) {
+      apply_y_flip(&y0, &y1, minify(src_mt->surf.phys_level0_sa.height,
+                                    level - src_mt->first_level));
+   }
+
+   bool result = false;
+
+   /* Blit slice-by-slice creating a single-slice miptree for each layer. Even
+    * in case of linear buffers hardware wants image arrays to be aligned by
+    * four rows. This way hardware only gets one image at a time and any
+    * source alignment will do.
+    */
+   for (unsigned i = 0; i < depth; ++i) {
+      struct intel_mipmap_tree *dst_mt = intel_miptree_create_for_bo(
+                                            brw, dst_bo, dst_format,
+                                            dst_offset + i * dst_image_stride,
+                                            width, height, 1,
+                                            dst_row_stride, 0);
+
+      if (!dst_mt) {
+         perf_debug("intel_texsubimage: miptree creation for src failed\n");
+         goto err;
+      }
+
+      /* In case exact match is needed, copy using equivalent UINT formats
+       * preventing hardware from changing presentation for SNORM -1.
+       */
+      if (dst_mt->format == src_format && !y_flip &&
+          src_swizzle == SWIZZLE_XYZW) {
+         brw_blorp_copy_miptrees(brw, src_mt, level, z + i,
+                                 dst_mt, 0, 0,
+                                 x, y, 0, 0, width, height);
+      } else {
+         brw_blorp_blit_miptrees(brw, src_mt, level, z + i,
+                                 src_format, src_swizzle,
+                                 dst_mt, 0, 0, dst_format,
+                                 x, y0, x + width, y1,
+                                 0, 0, width, height,
+                                 GL_NEAREST, false, y_flip, false, false);
+      }
+
+      intel_miptree_release(&dst_mt);
+   }
+
+   result = true;
+
+err:
+   brw_bo_unreference(dst_bo);
+
+   return result;
+}
+
 static bool
 set_write_disables(const struct intel_renderbuffer *irb,
                    const GLubyte *color_mask, bool *color_write_disable)

src/mesa/drivers/dri/i965/brw_blorp.h

@@ -67,6 +67,26 @@ brw_blorp_copy_buffers(struct brw_context *brw,
                        unsigned dst_offset,
                        unsigned size);
 
+bool
+brw_blorp_upload_miptree(struct brw_context *brw,
+                         struct intel_mipmap_tree *dst_mt,
+                         mesa_format dst_format,
+                         uint32_t level, uint32_t x, uint32_t y, uint32_t z,
+                         uint32_t width, uint32_t height, uint32_t depth,
+                         GLenum target, GLenum format, GLenum type,
+                         const void *pixels,
+                         const struct gl_pixelstore_attrib *packing);
+
+bool
+brw_blorp_download_miptree(struct brw_context *brw,
+                           struct intel_mipmap_tree *src_mt,
+                           mesa_format src_format, uint32_t src_swizzle,
+                           uint32_t level, uint32_t x, uint32_t y, uint32_t z,
+                           uint32_t width, uint32_t height, uint32_t depth,
+                           GLenum target, GLenum format, GLenum type,
+                           bool y_flip, const void *pixels,
+                           const struct gl_pixelstore_attrib *packing);
+
 void
 brw_blorp_clear_color(struct brw_context *brw, struct gl_framebuffer *fb,
                       GLbitfield mask, bool partial_clear, bool encode_srgb);