linux/drivers/gpu/drm/nouveau/nouveau_gem.c
Linus Torvalds 1d36dffa5d drm for 5.11-rc1
core:
 - documentation updates
 - deprecate DRM_FORMAT_MOD_NONE
 - atomic crtc enable/disable rework
 - GEM convert drivers to gem object functions
 - remove SCATTER_LIST_MAX_SEGMENT
 
 sched:
 - avoid infinite waits
 
 ttm:
 - remove AGP support
 - don't modify caching for swapout
 - ttm pinning rework
 - major TTM reworks
 - new backend allocator
 - multihop support
 
 vram-helper:
 - top down BO placement fix
 - TTM changes
 - GEM object support
 
 displayport:
 - DP 2.0 DPCD prep work
 - DP MST extended DPCD caps
 
 fbdev:
 - mark as orphaned
 
 amdgpu:
 - Initial Vangogh support
 - Green Sardine support
 - Dimgrey Cavefish support
 - SG display support for renoir
 - SMU7 improvements
 - gfx9+ modiifier support
 - CI BACO fixes
 
 radeon:
 - expose voltage via hwmon on SUMO
 
 amdkfd:
 - fix unique id handling
 
 i915:
 - more DG1 enablement
 - bigjoiner support
 - integer scaling filter support
 - async flip support
 - ICL+ DSI command mode
 - Improve display shutdown
 - Display refactoring
 - eLLC machine fbdev loading fix
 - dma scatterlist fixes
 - TGL hang fixes
 - eLLC display buffer caching on SKL+
 - MOCS PTE seeting for gen9+
 
 msm:
 - Shutdown hook
 - GPU cooling device support
 - DSI 7nm and 10nm phy/pll updates
 - sm8150/sm2850 DPU support
 - GEM locking re-work
 - LLCC system cache support
 
 aspeed:
 - sysfs output config support
 
 ast:
 - LUT fix
 - new display mode
 
 gma500:
 - remove 2d framebuffer accel
 
 panfrost:
 - move gpu reset to a worker
 
 exynos:
 - new HDMI mode support
 
 mediatek:
 - MT8167 support
 - yaml bindings
 - MIPI DSI phy code moved
 
 etnaviv:
 - new perf counter
 - more lockdep annotation
 
 hibmc:
 - i2c DDC support
 
 ingenic:
 - pixel clock reset fix
 - reserved memory support
 - allow both DMA channels at once
 - different pixel format support
 - 30/24/8-bit palette modes
 
 tilcdc:
 - don't keep vblank irq enabled
 
 vc4:
 - new maintainer added
 - DSI registration fix
 
 virtio:
 - blob resource support
 - host visible and cross-device support
 - uuid api support
 -----BEGIN PGP SIGNATURE-----
 
 iQIcBAABAgAGBQJf0upGAAoJEAx081l5xIa+1EoP/2OkZnl5d9S26qPja15EoRFl
 S69OjNci331Br9Y111jD2OCtyqA7w3ppnvCmzpHOBK1IZjhkxOVNC6PSUFSV4M3V
 oVOxZK0KaMHpLU2p90NbURWHa2TOktj7IWb9FrhPaEeBECbFuORZ2TbloFhaoyyt
 9auEAwqYRPgF8CSYOjQGGZJ85MQN4ImExTdY13+BZgQlGLiSPHfpnLVJ1Q5TPt6A
 BLgcU/DFcqOZqyjeu+CuA+LZSHjHeVJxTOGRX65PoTtU3Xus8TRZ/qL4r8e6mAI1
 boFLmsevvQlzaQ9GFohc+l9QR/dtnm6SpZxuEelewh7sQvsz2GI+SNF+OHcwHCph
 TYIEtyZNaz1bf7ip75FGbhEVaWh2PUMn3zkGlYt+zqAtznYB+dFPc31hhuVn3o5X
 c8UwLDUUJLzTePKPZ0UtzIu4Gm2RYTyRsnUAP0OKP/0WaZRyxnoQMYm5Llg7RBe0
 5ZJSWjJPBlv1YMWAHQ0YMZ+MhnFE8k4eV/8WfBQnb2INosgzKfJXEmu6ffAkPqSq
 jxBsrVQwtOMF2P9VEfdQDv3fs0GKDuZN5ezTFuW59Dt4VYfCUe2FTssSwFBIp5X9
 erPJ/nk883rcI6F0PdArNYvWpwPlVSDJyfTxQbYYxVAf8X1ARJCU3PT6iBnGO3i4
 d5tveSc8HoOXr4W3eIjn
 =c9rl
 -----END PGP SIGNATURE-----

Merge tag 'drm-next-2020-12-11' of git://anongit.freedesktop.org/drm/drm

Pull drm updates from Dave Airlie:
 "Not a huge amount of big things here, AMD has support for a few new HW
  variants (vangogh, green sardine, dimgrey cavefish), Intel has some
  more DG1 enablement. We have a few big reworks of the TTM layers and
  interfaces, GEM and atomic internal API reworks cross tree. fbdev is
  marked orphaned in here as well to reflect the current reality.

  core:
   - documentation updates
   - deprecate DRM_FORMAT_MOD_NONE
   - atomic crtc enable/disable rework
   - GEM convert drivers to gem object functions
   - remove SCATTER_LIST_MAX_SEGMENT

  sched:
   - avoid infinite waits

  ttm:
   - remove AGP support
   - don't modify caching for swapout
   - ttm pinning rework
   - major TTM reworks
   - new backend allocator
   - multihop support

  vram-helper:
   - top down BO placement fix
   - TTM changes
   - GEM object support

  displayport:
   - DP 2.0 DPCD prep work
   - DP MST extended DPCD caps

  fbdev:
   - mark as orphaned

  amdgpu:
   - Initial Vangogh support
   - Green Sardine support
   - Dimgrey Cavefish support
   - SG display support for renoir
   - SMU7 improvements
   - gfx9+ modiifier support
   - CI BACO fixes

  radeon:
   - expose voltage via hwmon on SUMO

  amdkfd:
   - fix unique id handling

  i915:
   - more DG1 enablement
   - bigjoiner support
   - integer scaling filter support
   - async flip support
   - ICL+ DSI command mode
   - Improve display shutdown
   - Display refactoring
   - eLLC machine fbdev loading fix
   - dma scatterlist fixes
   - TGL hang fixes
   - eLLC display buffer caching on SKL+
   - MOCS PTE seeting for gen9+

  msm:
   - Shutdown hook
   - GPU cooling device support
   - DSI 7nm and 10nm phy/pll updates
   - sm8150/sm2850 DPU support
   - GEM locking re-work
   - LLCC system cache support

  aspeed:
   - sysfs output config support

  ast:
   - LUT fix
   - new display mode

  gma500:
   - remove 2d framebuffer accel

  panfrost:
   - move gpu reset to a worker

  exynos:
   - new HDMI mode support

  mediatek:
   - MT8167 support
   - yaml bindings
   - MIPI DSI phy code moved

  etnaviv:
   - new perf counter
   - more lockdep annotation

  hibmc:
   - i2c DDC support

  ingenic:
   - pixel clock reset fix
   - reserved memory support
   - allow both DMA channels at once
   - different pixel format support
   - 30/24/8-bit palette modes

  tilcdc:
   - don't keep vblank irq enabled

  vc4:
   - new maintainer added
   - DSI registration fix

  virtio:
   - blob resource support
   - host visible and cross-device support
   - uuid api support"

* tag 'drm-next-2020-12-11' of git://anongit.freedesktop.org/drm/drm: (1754 commits)
  drm/amdgpu: Initialise drm_gem_object_funcs for imported BOs
  drm/amdgpu: fix size calculation with stolen vga memory
  drm/amdgpu: remove amdgpu_ttm_late_init  and amdgpu_bo_late_init
  drm/amdgpu: free the pre-OS console framebuffer after the first modeset
  drm/amdgpu: enable runtime pm using BACO on CI dGPUs
  drm/amdgpu/cik: enable BACO reset on Bonaire
  drm/amd/pm: update smu10.h WORKLOAD_PPLIB setting for raven
  drm/amd/pm: remove one unsupported smu function for vangogh
  drm/amd/display: setup system context for APUs
  drm/amd/display: add S/G support for Vangogh
  drm/amdkfd: Fix leak in dmabuf import
  drm/amdgpu: use AMDGPU_NUM_VMID when possible
  drm/amdgpu: fix sdma instance fw version and feature version init
  drm/amd/pm: update driver if version for dimgrey_cavefish
  drm/amd/display: 3.2.115
  drm/amd/display: [FW Promotion] Release 0.0.45
  drm/amd/display: Revert DCN2.1 dram_clock_change_latency update
  drm/amd/display: Enable gpu_vm_support for dcn3.01
  drm/amd/display: Fixed the audio noise during mode switching with HDCP mode on
  drm/amd/display: Add wm table for Renoir
  ...
2020-12-14 11:07:56 -08:00

981 lines
25 KiB
C

/*
* Copyright (C) 2008 Ben Skeggs.
* All Rights Reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice (including the
* next paragraph) shall be included in all copies or substantial
* portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
* IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*
*/
#include <drm/drm_gem_ttm_helper.h>
#include "nouveau_drv.h"
#include "nouveau_dma.h"
#include "nouveau_fence.h"
#include "nouveau_abi16.h"
#include "nouveau_ttm.h"
#include "nouveau_gem.h"
#include "nouveau_mem.h"
#include "nouveau_vmm.h"
#include <nvif/class.h>
#include <nvif/push206e.h>
void
nouveau_gem_object_del(struct drm_gem_object *gem)
{
struct nouveau_bo *nvbo = nouveau_gem_object(gem);
struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
struct device *dev = drm->dev->dev;
int ret;
ret = pm_runtime_get_sync(dev);
if (WARN_ON(ret < 0 && ret != -EACCES)) {
pm_runtime_put_autosuspend(dev);
return;
}
if (gem->import_attach)
drm_prime_gem_destroy(gem, nvbo->bo.sg);
ttm_bo_put(&nvbo->bo);
pm_runtime_mark_last_busy(dev);
pm_runtime_put_autosuspend(dev);
}
int
nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv)
{
struct nouveau_cli *cli = nouveau_cli(file_priv);
struct nouveau_bo *nvbo = nouveau_gem_object(gem);
struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
struct device *dev = drm->dev->dev;
struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm;
struct nouveau_vma *vma;
int ret;
if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50)
return 0;
ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL);
if (ret)
return ret;
ret = pm_runtime_get_sync(dev);
if (ret < 0 && ret != -EACCES) {
pm_runtime_put_autosuspend(dev);
goto out;
}
ret = nouveau_vma_new(nvbo, vmm, &vma);
pm_runtime_mark_last_busy(dev);
pm_runtime_put_autosuspend(dev);
out:
ttm_bo_unreserve(&nvbo->bo);
return ret;
}
struct nouveau_gem_object_unmap {
struct nouveau_cli_work work;
struct nouveau_vma *vma;
};
static void
nouveau_gem_object_delete(struct nouveau_vma *vma)
{
nouveau_fence_unref(&vma->fence);
nouveau_vma_del(&vma);
}
static void
nouveau_gem_object_delete_work(struct nouveau_cli_work *w)
{
struct nouveau_gem_object_unmap *work =
container_of(w, typeof(*work), work);
nouveau_gem_object_delete(work->vma);
kfree(work);
}
static void
nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma)
{
struct dma_fence *fence = vma->fence ? &vma->fence->base : NULL;
struct nouveau_gem_object_unmap *work;
list_del_init(&vma->head);
if (!fence) {
nouveau_gem_object_delete(vma);
return;
}
if (!(work = kmalloc(sizeof(*work), GFP_KERNEL))) {
WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0);
nouveau_gem_object_delete(vma);
return;
}
work->work.func = nouveau_gem_object_delete_work;
work->vma = vma;
nouveau_cli_work_queue(vma->vmm->cli, fence, &work->work);
}
void
nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv)
{
struct nouveau_cli *cli = nouveau_cli(file_priv);
struct nouveau_bo *nvbo = nouveau_gem_object(gem);
struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
struct device *dev = drm->dev->dev;
struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : & cli->vmm;
struct nouveau_vma *vma;
int ret;
if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50)
return;
ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL);
if (ret)
return;
vma = nouveau_vma_find(nvbo, vmm);
if (vma) {
if (--vma->refs == 0) {
ret = pm_runtime_get_sync(dev);
if (!WARN_ON(ret < 0 && ret != -EACCES)) {
nouveau_gem_object_unmap(nvbo, vma);
pm_runtime_mark_last_busy(dev);
}
pm_runtime_put_autosuspend(dev);
}
}
ttm_bo_unreserve(&nvbo->bo);
}
const struct drm_gem_object_funcs nouveau_gem_object_funcs = {
.free = nouveau_gem_object_del,
.open = nouveau_gem_object_open,
.close = nouveau_gem_object_close,
.pin = nouveau_gem_prime_pin,
.unpin = nouveau_gem_prime_unpin,
.get_sg_table = nouveau_gem_prime_get_sg_table,
.vmap = drm_gem_ttm_vmap,
.vunmap = drm_gem_ttm_vunmap,
};
int
nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain,
uint32_t tile_mode, uint32_t tile_flags,
struct nouveau_bo **pnvbo)
{
struct nouveau_drm *drm = cli->drm;
struct nouveau_bo *nvbo;
int ret;
if (!(domain & (NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART)))
domain |= NOUVEAU_GEM_DOMAIN_CPU;
nvbo = nouveau_bo_alloc(cli, &size, &align, domain, tile_mode,
tile_flags);
if (IS_ERR(nvbo))
return PTR_ERR(nvbo);
nvbo->bo.base.funcs = &nouveau_gem_object_funcs;
/* Initialize the embedded gem-object. We return a single gem-reference
* to the caller, instead of a normal nouveau_bo ttm reference. */
ret = drm_gem_object_init(drm->dev, &nvbo->bo.base, size);
if (ret) {
drm_gem_object_release(&nvbo->bo.base);
kfree(nvbo);
return ret;
}
ret = nouveau_bo_init(nvbo, size, align, domain, NULL, NULL);
if (ret) {
nouveau_bo_ref(NULL, &nvbo);
return ret;
}
/* we restrict allowed domains on nv50+ to only the types
* that were requested at creation time. not possibly on
* earlier chips without busting the ABI.
*/
nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM |
NOUVEAU_GEM_DOMAIN_GART;
if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA)
nvbo->valid_domains &= domain;
*pnvbo = nvbo;
return 0;
}
static int
nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem,
struct drm_nouveau_gem_info *rep)
{
struct nouveau_cli *cli = nouveau_cli(file_priv);
struct nouveau_bo *nvbo = nouveau_gem_object(gem);
struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm;
struct nouveau_vma *vma;
if (is_power_of_2(nvbo->valid_domains))
rep->domain = nvbo->valid_domains;
else if (nvbo->bo.mem.mem_type == TTM_PL_TT)
rep->domain = NOUVEAU_GEM_DOMAIN_GART;
else
rep->domain = NOUVEAU_GEM_DOMAIN_VRAM;
rep->offset = nvbo->offset;
if (vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) {
vma = nouveau_vma_find(nvbo, vmm);
if (!vma)
return -EINVAL;
rep->offset = vma->addr;
}
rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT;
rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.base.vma_node);
rep->tile_mode = nvbo->mode;
rep->tile_flags = nvbo->contig ? 0 : NOUVEAU_GEM_TILE_NONCONTIG;
if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI)
rep->tile_flags |= nvbo->kind << 8;
else
if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA)
rep->tile_flags |= nvbo->kind << 8 | nvbo->comp << 16;
else
rep->tile_flags |= nvbo->zeta;
return 0;
}
int
nouveau_gem_ioctl_new(struct drm_device *dev, void *data,
struct drm_file *file_priv)
{
struct nouveau_cli *cli = nouveau_cli(file_priv);
struct drm_nouveau_gem_new *req = data;
struct nouveau_bo *nvbo = NULL;
int ret = 0;
ret = nouveau_gem_new(cli, req->info.size, req->align,
req->info.domain, req->info.tile_mode,
req->info.tile_flags, &nvbo);
if (ret)
return ret;
ret = drm_gem_handle_create(file_priv, &nvbo->bo.base,
&req->info.handle);
if (ret == 0) {
ret = nouveau_gem_info(file_priv, &nvbo->bo.base, &req->info);
if (ret)
drm_gem_handle_delete(file_priv, req->info.handle);
}
/* drop reference from allocate - handle holds it now */
drm_gem_object_put(&nvbo->bo.base);
return ret;
}
static int
nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains,
uint32_t write_domains, uint32_t valid_domains)
{
struct nouveau_bo *nvbo = nouveau_gem_object(gem);
struct ttm_buffer_object *bo = &nvbo->bo;
uint32_t domains = valid_domains & nvbo->valid_domains &
(write_domains ? write_domains : read_domains);
uint32_t pref_domains = 0;;
if (!domains)
return -EINVAL;
valid_domains &= ~(NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART);
if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) &&
bo->mem.mem_type == TTM_PL_VRAM)
pref_domains |= NOUVEAU_GEM_DOMAIN_VRAM;
else if ((domains & NOUVEAU_GEM_DOMAIN_GART) &&
bo->mem.mem_type == TTM_PL_TT)
pref_domains |= NOUVEAU_GEM_DOMAIN_GART;
else if (domains & NOUVEAU_GEM_DOMAIN_VRAM)
pref_domains |= NOUVEAU_GEM_DOMAIN_VRAM;
else
pref_domains |= NOUVEAU_GEM_DOMAIN_GART;
nouveau_bo_placement_set(nvbo, pref_domains, valid_domains);
return 0;
}
struct validate_op {
struct list_head list;
struct ww_acquire_ctx ticket;
};
static void
validate_fini_no_ticket(struct validate_op *op, struct nouveau_channel *chan,
struct nouveau_fence *fence,
struct drm_nouveau_gem_pushbuf_bo *pbbo)
{
struct nouveau_bo *nvbo;
struct drm_nouveau_gem_pushbuf_bo *b;
while (!list_empty(&op->list)) {
nvbo = list_entry(op->list.next, struct nouveau_bo, entry);
b = &pbbo[nvbo->pbbo_index];
if (likely(fence)) {
nouveau_bo_fence(nvbo, fence, !!b->write_domains);
if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) {
struct nouveau_vma *vma =
(void *)(unsigned long)b->user_priv;
nouveau_fence_unref(&vma->fence);
dma_fence_get(&fence->base);
vma->fence = fence;
}
}
if (unlikely(nvbo->validate_mapped)) {
ttm_bo_kunmap(&nvbo->kmap);
nvbo->validate_mapped = false;
}
list_del(&nvbo->entry);
nvbo->reserved_by = NULL;
ttm_bo_unreserve(&nvbo->bo);
drm_gem_object_put(&nvbo->bo.base);
}
}
static void
validate_fini(struct validate_op *op, struct nouveau_channel *chan,
struct nouveau_fence *fence,
struct drm_nouveau_gem_pushbuf_bo *pbbo)
{
validate_fini_no_ticket(op, chan, fence, pbbo);
ww_acquire_fini(&op->ticket);
}
static int
validate_init(struct nouveau_channel *chan, struct drm_file *file_priv,
struct drm_nouveau_gem_pushbuf_bo *pbbo,
int nr_buffers, struct validate_op *op)
{
struct nouveau_cli *cli = nouveau_cli(file_priv);
int trycnt = 0;
int ret = -EINVAL, i;
struct nouveau_bo *res_bo = NULL;
LIST_HEAD(gart_list);
LIST_HEAD(vram_list);
LIST_HEAD(both_list);
ww_acquire_init(&op->ticket, &reservation_ww_class);
retry:
if (++trycnt > 100000) {
NV_PRINTK(err, cli, "%s failed and gave up.\n", __func__);
return -EINVAL;
}
for (i = 0; i < nr_buffers; i++) {
struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i];
struct drm_gem_object *gem;
struct nouveau_bo *nvbo;
gem = drm_gem_object_lookup(file_priv, b->handle);
if (!gem) {
NV_PRINTK(err, cli, "Unknown handle 0x%08x\n", b->handle);
ret = -ENOENT;
break;
}
nvbo = nouveau_gem_object(gem);
if (nvbo == res_bo) {
res_bo = NULL;
drm_gem_object_put(gem);
continue;
}
if (nvbo->reserved_by && nvbo->reserved_by == file_priv) {
NV_PRINTK(err, cli, "multiple instances of buffer %d on "
"validation list\n", b->handle);
drm_gem_object_put(gem);
ret = -EINVAL;
break;
}
ret = ttm_bo_reserve(&nvbo->bo, true, false, &op->ticket);
if (ret) {
list_splice_tail_init(&vram_list, &op->list);
list_splice_tail_init(&gart_list, &op->list);
list_splice_tail_init(&both_list, &op->list);
validate_fini_no_ticket(op, chan, NULL, NULL);
if (unlikely(ret == -EDEADLK)) {
ret = ttm_bo_reserve_slowpath(&nvbo->bo, true,
&op->ticket);
if (!ret)
res_bo = nvbo;
}
if (unlikely(ret)) {
if (ret != -ERESTARTSYS)
NV_PRINTK(err, cli, "fail reserve\n");
break;
}
}
if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) {
struct nouveau_vmm *vmm = chan->vmm;
struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm);
if (!vma) {
NV_PRINTK(err, cli, "vma not found!\n");
ret = -EINVAL;
break;
}
b->user_priv = (uint64_t)(unsigned long)vma;
} else {
b->user_priv = (uint64_t)(unsigned long)nvbo;
}
nvbo->reserved_by = file_priv;
nvbo->pbbo_index = i;
if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) &&
(b->valid_domains & NOUVEAU_GEM_DOMAIN_GART))
list_add_tail(&nvbo->entry, &both_list);
else
if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM)
list_add_tail(&nvbo->entry, &vram_list);
else
if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)
list_add_tail(&nvbo->entry, &gart_list);
else {
NV_PRINTK(err, cli, "invalid valid domains: 0x%08x\n",
b->valid_domains);
list_add_tail(&nvbo->entry, &both_list);
ret = -EINVAL;
break;
}
if (nvbo == res_bo)
goto retry;
}
ww_acquire_done(&op->ticket);
list_splice_tail(&vram_list, &op->list);
list_splice_tail(&gart_list, &op->list);
list_splice_tail(&both_list, &op->list);
if (ret)
validate_fini(op, chan, NULL, NULL);
return ret;
}
static int
validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli,
struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo)
{
struct nouveau_drm *drm = chan->drm;
struct nouveau_bo *nvbo;
int ret, relocs = 0;
list_for_each_entry(nvbo, list, entry) {
struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index];
ret = nouveau_gem_set_domain(&nvbo->bo.base, b->read_domains,
b->write_domains,
b->valid_domains);
if (unlikely(ret)) {
NV_PRINTK(err, cli, "fail set_domain\n");
return ret;
}
ret = nouveau_bo_validate(nvbo, true, false);
if (unlikely(ret)) {
if (ret != -ERESTARTSYS)
NV_PRINTK(err, cli, "fail ttm_validate\n");
return ret;
}
ret = nouveau_fence_sync(nvbo, chan, !!b->write_domains, true);
if (unlikely(ret)) {
if (ret != -ERESTARTSYS)
NV_PRINTK(err, cli, "fail post-validate sync\n");
return ret;
}
if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) {
if (nvbo->offset == b->presumed.offset &&
((nvbo->bo.mem.mem_type == TTM_PL_VRAM &&
b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) ||
(nvbo->bo.mem.mem_type == TTM_PL_TT &&
b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART)))
continue;
if (nvbo->bo.mem.mem_type == TTM_PL_TT)
b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART;
else
b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM;
b->presumed.offset = nvbo->offset;
b->presumed.valid = 0;
relocs++;
}
}
return relocs;
}
static int
nouveau_gem_pushbuf_validate(struct nouveau_channel *chan,
struct drm_file *file_priv,
struct drm_nouveau_gem_pushbuf_bo *pbbo,
int nr_buffers,
struct validate_op *op, bool *apply_relocs)
{
struct nouveau_cli *cli = nouveau_cli(file_priv);
int ret;
INIT_LIST_HEAD(&op->list);
if (nr_buffers == 0)
return 0;
ret = validate_init(chan, file_priv, pbbo, nr_buffers, op);
if (unlikely(ret)) {
if (ret != -ERESTARTSYS)
NV_PRINTK(err, cli, "validate_init\n");
return ret;
}
ret = validate_list(chan, cli, &op->list, pbbo);
if (unlikely(ret < 0)) {
if (ret != -ERESTARTSYS)
NV_PRINTK(err, cli, "validating bo list\n");
validate_fini(op, chan, NULL, NULL);
return ret;
} else if (ret > 0) {
*apply_relocs = true;
}
return 0;
}
static inline void
u_free(void *addr)
{
kvfree(addr);
}
static inline void *
u_memcpya(uint64_t user, unsigned nmemb, unsigned size)
{
void *mem;
void __user *userptr = (void __force __user *)(uintptr_t)user;
size *= nmemb;
mem = kvmalloc(size, GFP_KERNEL);
if (!mem)
return ERR_PTR(-ENOMEM);
if (copy_from_user(mem, userptr, size)) {
u_free(mem);
return ERR_PTR(-EFAULT);
}
return mem;
}
static int
nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli,
struct drm_nouveau_gem_pushbuf *req,
struct drm_nouveau_gem_pushbuf_reloc *reloc,
struct drm_nouveau_gem_pushbuf_bo *bo)
{
int ret = 0;
unsigned i;
for (i = 0; i < req->nr_relocs; i++) {
struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i];
struct drm_nouveau_gem_pushbuf_bo *b;
struct nouveau_bo *nvbo;
uint32_t data;
if (unlikely(r->bo_index >= req->nr_buffers)) {
NV_PRINTK(err, cli, "reloc bo index invalid\n");
ret = -EINVAL;
break;
}
b = &bo[r->bo_index];
if (b->presumed.valid)
continue;
if (unlikely(r->reloc_bo_index >= req->nr_buffers)) {
NV_PRINTK(err, cli, "reloc container bo index invalid\n");
ret = -EINVAL;
break;
}
nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv;
if (unlikely(r->reloc_bo_offset + 4 >
nvbo->bo.mem.num_pages << PAGE_SHIFT)) {
NV_PRINTK(err, cli, "reloc outside of bo\n");
ret = -EINVAL;
break;
}
if (!nvbo->kmap.virtual) {
ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages,
&nvbo->kmap);
if (ret) {
NV_PRINTK(err, cli, "failed kmap for reloc\n");
break;
}
nvbo->validate_mapped = true;
}
if (r->flags & NOUVEAU_GEM_RELOC_LOW)
data = b->presumed.offset + r->data;
else
if (r->flags & NOUVEAU_GEM_RELOC_HIGH)
data = (b->presumed.offset + r->data) >> 32;
else
data = r->data;
if (r->flags & NOUVEAU_GEM_RELOC_OR) {
if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART)
data |= r->tor;
else
data |= r->vor;
}
ret = ttm_bo_wait(&nvbo->bo, false, false);
if (ret) {
NV_PRINTK(err, cli, "reloc wait_idle failed: %d\n", ret);
break;
}
nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data);
}
return ret;
}
int
nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
struct drm_file *file_priv)
{
struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv);
struct nouveau_cli *cli = nouveau_cli(file_priv);
struct nouveau_abi16_chan *temp;
struct nouveau_drm *drm = nouveau_drm(dev);
struct drm_nouveau_gem_pushbuf *req = data;
struct drm_nouveau_gem_pushbuf_push *push;
struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL;
struct drm_nouveau_gem_pushbuf_bo *bo;
struct nouveau_channel *chan = NULL;
struct validate_op op;
struct nouveau_fence *fence = NULL;
int i, j, ret = 0;
bool do_reloc = false, sync = false;
if (unlikely(!abi16))
return -ENOMEM;
list_for_each_entry(temp, &abi16->channels, head) {
if (temp->chan->chid == req->channel) {
chan = temp->chan;
break;
}
}
if (!chan)
return nouveau_abi16_put(abi16, -ENOENT);
if (unlikely(atomic_read(&chan->killed)))
return nouveau_abi16_put(abi16, -ENODEV);
sync = req->vram_available & NOUVEAU_GEM_PUSHBUF_SYNC;
req->vram_available = drm->gem.vram_available;
req->gart_available = drm->gem.gart_available;
if (unlikely(req->nr_push == 0))
goto out_next;
if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) {
NV_PRINTK(err, cli, "pushbuf push count exceeds limit: %d max %d\n",
req->nr_push, NOUVEAU_GEM_MAX_PUSH);
return nouveau_abi16_put(abi16, -EINVAL);
}
if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) {
NV_PRINTK(err, cli, "pushbuf bo count exceeds limit: %d max %d\n",
req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS);
return nouveau_abi16_put(abi16, -EINVAL);
}
if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) {
NV_PRINTK(err, cli, "pushbuf reloc count exceeds limit: %d max %d\n",
req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS);
return nouveau_abi16_put(abi16, -EINVAL);
}
push = u_memcpya(req->push, req->nr_push, sizeof(*push));
if (IS_ERR(push))
return nouveau_abi16_put(abi16, PTR_ERR(push));
bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo));
if (IS_ERR(bo)) {
u_free(push);
return nouveau_abi16_put(abi16, PTR_ERR(bo));
}
/* Ensure all push buffers are on validate list */
for (i = 0; i < req->nr_push; i++) {
if (push[i].bo_index >= req->nr_buffers) {
NV_PRINTK(err, cli, "push %d buffer not in list\n", i);
ret = -EINVAL;
goto out_prevalid;
}
}
/* Validate buffer list */
revalidate:
ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo,
req->nr_buffers, &op, &do_reloc);
if (ret) {
if (ret != -ERESTARTSYS)
NV_PRINTK(err, cli, "validate: %d\n", ret);
goto out_prevalid;
}
/* Apply any relocations that are required */
if (do_reloc) {
if (!reloc) {
validate_fini(&op, chan, NULL, bo);
reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc));
if (IS_ERR(reloc)) {
ret = PTR_ERR(reloc);
goto out_prevalid;
}
goto revalidate;
}
ret = nouveau_gem_pushbuf_reloc_apply(cli, req, reloc, bo);
if (ret) {
NV_PRINTK(err, cli, "reloc apply: %d\n", ret);
goto out;
}
}
if (chan->dma.ib_max) {
ret = nouveau_dma_wait(chan, req->nr_push + 1, 16);
if (ret) {
NV_PRINTK(err, cli, "nv50cal_space: %d\n", ret);
goto out;
}
for (i = 0; i < req->nr_push; i++) {
struct nouveau_vma *vma = (void *)(unsigned long)
bo[push[i].bo_index].user_priv;
nv50_dma_push(chan, vma->addr + push[i].offset,
push[i].length);
}
} else
if (drm->client.device.info.chipset >= 0x25) {
ret = PUSH_WAIT(chan->chan.push, req->nr_push * 2);
if (ret) {
NV_PRINTK(err, cli, "cal_space: %d\n", ret);
goto out;
}
for (i = 0; i < req->nr_push; i++) {
struct nouveau_bo *nvbo = (void *)(unsigned long)
bo[push[i].bo_index].user_priv;
PUSH_CALL(chan->chan.push, nvbo->offset + push[i].offset);
PUSH_DATA(chan->chan.push, 0);
}
} else {
ret = PUSH_WAIT(chan->chan.push, req->nr_push * (2 + NOUVEAU_DMA_SKIPS));
if (ret) {
NV_PRINTK(err, cli, "jmp_space: %d\n", ret);
goto out;
}
for (i = 0; i < req->nr_push; i++) {
struct nouveau_bo *nvbo = (void *)(unsigned long)
bo[push[i].bo_index].user_priv;
uint32_t cmd;
cmd = chan->push.addr + ((chan->dma.cur + 2) << 2);
cmd |= 0x20000000;
if (unlikely(cmd != req->suffix0)) {
if (!nvbo->kmap.virtual) {
ret = ttm_bo_kmap(&nvbo->bo, 0,
nvbo->bo.mem.
num_pages,
&nvbo->kmap);
if (ret) {
WIND_RING(chan);
goto out;
}
nvbo->validate_mapped = true;
}
nouveau_bo_wr32(nvbo, (push[i].offset +
push[i].length - 8) / 4, cmd);
}
PUSH_JUMP(chan->chan.push, nvbo->offset + push[i].offset);
PUSH_DATA(chan->chan.push, 0);
for (j = 0; j < NOUVEAU_DMA_SKIPS; j++)
PUSH_DATA(chan->chan.push, 0);
}
}
ret = nouveau_fence_new(chan, false, &fence);
if (ret) {
NV_PRINTK(err, cli, "error fencing pushbuf: %d\n", ret);
WIND_RING(chan);
goto out;
}
if (sync) {
if (!(ret = nouveau_fence_wait(fence, false, false))) {
if ((ret = dma_fence_get_status(&fence->base)) == 1)
ret = 0;
}
}
out:
validate_fini(&op, chan, fence, bo);
nouveau_fence_unref(&fence);
if (do_reloc) {
struct drm_nouveau_gem_pushbuf_bo __user *upbbo =
u64_to_user_ptr(req->buffers);
for (i = 0; i < req->nr_buffers; i++) {
if (bo[i].presumed.valid)
continue;
if (copy_to_user(&upbbo[i].presumed, &bo[i].presumed,
sizeof(bo[i].presumed))) {
ret = -EFAULT;
break;
}
}
}
out_prevalid:
if (!IS_ERR(reloc))
u_free(reloc);
u_free(bo);
u_free(push);
out_next:
if (chan->dma.ib_max) {
req->suffix0 = 0x00000000;
req->suffix1 = 0x00000000;
} else
if (drm->client.device.info.chipset >= 0x25) {
req->suffix0 = 0x00020000;
req->suffix1 = 0x00000000;
} else {
req->suffix0 = 0x20000000 |
(chan->push.addr + ((chan->dma.cur + 2) << 2));
req->suffix1 = 0x00000000;
}
return nouveau_abi16_put(abi16, ret);
}
int
nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data,
struct drm_file *file_priv)
{
struct drm_nouveau_gem_cpu_prep *req = data;
struct drm_gem_object *gem;
struct nouveau_bo *nvbo;
bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT);
bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE);
long lret;
int ret;
gem = drm_gem_object_lookup(file_priv, req->handle);
if (!gem)
return -ENOENT;
nvbo = nouveau_gem_object(gem);
lret = dma_resv_wait_timeout_rcu(nvbo->bo.base.resv, write, true,
no_wait ? 0 : 30 * HZ);
if (!lret)
ret = -EBUSY;
else if (lret > 0)
ret = 0;
else
ret = lret;
nouveau_bo_sync_for_cpu(nvbo);
drm_gem_object_put(gem);
return ret;
}
int
nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data,
struct drm_file *file_priv)
{
struct drm_nouveau_gem_cpu_fini *req = data;
struct drm_gem_object *gem;
struct nouveau_bo *nvbo;
gem = drm_gem_object_lookup(file_priv, req->handle);
if (!gem)
return -ENOENT;
nvbo = nouveau_gem_object(gem);
nouveau_bo_sync_for_device(nvbo);
drm_gem_object_put(gem);
return 0;
}
int
nouveau_gem_ioctl_info(struct drm_device *dev, void *data,
struct drm_file *file_priv)
{
struct drm_nouveau_gem_info *req = data;
struct drm_gem_object *gem;
int ret;
gem = drm_gem_object_lookup(file_priv, req->handle);
if (!gem)
return -ENOENT;
ret = nouveau_gem_info(file_priv, gem, req);
drm_gem_object_put(gem);
return ret;
}