anv/allocator: Enable snooping on block pool and anv_bo_pool BOs.

We are not going to use userptr for anv block pool BOs anymore. However,
so far we have been relying on the fact that userptr BOs are snooped on
non-llc platforms. Let's make sure that the block pool BOs are still
snooped, and we can also remove the clflush'ing that we do on all state
buffers.

And since we plan to remove the flushes, set the anv_bo_pool BOs to
cached (snooped on non-LLC platforms) too. For LLC platforms, they are
all cached by default, so this becomes a no-op.

v5:
 - Add snooping to anv_bo_pool BOs too (Jason).
 - Remove anv_gem_set_domain.

Reviewed-by: Jason Ekstrand <jason@jlekstrand.net>

src/intel/vulkan/anv_allocator.c

@@ -546,19 +546,17 @@ anv_block_pool_expand_range(struct anv_block_pool *pool,
    cleanup->size = size;
    cleanup->gem_handle = gem_handle;
 
-#if 0
    /* Regular objects are created I915_CACHING_CACHED on LLC platforms and
     * I915_CACHING_NONE on non-LLC platforms. However, userptr objects are
     * always created as I915_CACHING_CACHED, which on non-LLC means
-    * snooped. That can be useful but comes with a bit of overheard.  Since
-    * we're eplicitly clflushing and don't want the overhead we need to turn
-    * it off. */
-   if (!pool->device->info.has_llc) {
-      anv_gem_set_caching(pool->device, gem_handle, I915_CACHING_NONE);
-      anv_gem_set_domain(pool->device, gem_handle,
-                         I915_GEM_DOMAIN_GTT, I915_GEM_DOMAIN_GTT);
-   }
-#endif
+    * snooped.
+    *
+    * On platforms that support softpin, we are not going to use userptr
+    * anymore, but we still want to rely on the snooped states. So make sure
+    * everything is set to I915_CACHING_CACHED.
+    */
+   if (!pool->device->info.has_llc)
+      anv_gem_set_caching(pool->device, gem_handle, I915_CACHING_CACHED);
 
    /* Now that we successfull allocated everything, we can write the new
     * center_bo_offset back into pool. */
@@ -1405,6 +1403,14 @@ anv_bo_pool_alloc(struct anv_bo_pool *pool, struct anv_bo *bo, uint32_t size)
       return vk_error(VK_ERROR_MEMORY_MAP_FAILED);
    }
 
+   /* We are removing the state flushes, so lets make sure that these buffers
+    * are cached/snooped.
+    */
+   if (!pool->device->info.has_llc) {
+      anv_gem_set_caching(pool->device, new_bo.gem_handle,
+                          I915_CACHING_CACHED);
+   }
+
    *bo = new_bo;
 
    VG(VALGRIND_MEMPOOL_ALLOC(pool, bo->map, size));