blob: 92821167707927891bca4d336d8d9133e8c47c1d [file] [log] [blame]
/*
* Intel MIC Platform Software Stack (MPSS)
*
* Copyright(c) 2015 Intel Corporation.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License, version 2, as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* Intel SCIF driver.
*
*/
#include "scif_main.h"
/*
* struct scif_vma_info - Information about a remote memory mapping
* created via scif_mmap(..)
* @vma: VM area struct
* @list: link to list of active vmas
*/
struct scif_vma_info {
struct vm_area_struct *vma;
struct list_head list;
};
void scif_recv_munmap(struct scif_dev *scifdev, struct scifmsg *msg)
{
struct scif_rma_req req;
struct scif_window *window = NULL;
struct scif_window *recv_window =
(struct scif_window *)msg->payload[0];
struct scif_endpt *ep;
ep = (struct scif_endpt *)recv_window->ep;
req.out_window = &window;
req.offset = recv_window->offset;
req.prot = recv_window->prot;
req.nr_bytes = recv_window->nr_pages << PAGE_SHIFT;
req.type = SCIF_WINDOW_FULL;
req.head = &ep->rma_info.reg_list;
msg->payload[0] = ep->remote_ep;
mutex_lock(&ep->rma_info.rma_lock);
/* Does a valid window exist? */
if (scif_query_window(&req)) {
dev_err(&scifdev->sdev->dev,
"%s %d -ENXIO\n", __func__, __LINE__);
msg->uop = SCIF_UNREGISTER_ACK;
goto error;
}
scif_put_window(window, window->nr_pages);
if (!window->ref_count) {
atomic_inc(&ep->rma_info.tw_refcount);
ep->rma_info.async_list_del = 1;
list_del_init(&window->list);
scif_free_window_offset(ep, window, window->offset);
}
error:
mutex_unlock(&ep->rma_info.rma_lock);
if (window && !window->ref_count)
scif_queue_for_cleanup(window, &scif_info.rma);
}
/*
* Remove valid remote memory mappings created via scif_mmap(..) from the
* process address space since the remote node is lost
*/
static void __scif_zap_mmaps(struct scif_endpt *ep)
{
struct list_head *item;
struct scif_vma_info *info;
struct vm_area_struct *vma;
unsigned long size;
spin_lock(&ep->lock);
list_for_each(item, &ep->rma_info.vma_list) {
info = list_entry(item, struct scif_vma_info, list);
vma = info->vma;
size = vma->vm_end - vma->vm_start;
zap_vma_ptes(vma, vma->vm_start, size);
dev_dbg(scif_info.mdev.this_device,
"%s ep %p zap vma %p size 0x%lx\n",
__func__, ep, info->vma, size);
}
spin_unlock(&ep->lock);
}
/*
* Traverse the list of endpoints for a particular remote node and
* zap valid remote memory mappings since the remote node is lost
*/
static void _scif_zap_mmaps(int node, struct list_head *head)
{
struct scif_endpt *ep;
struct list_head *item;
mutex_lock(&scif_info.connlock);
list_for_each(item, head) {
ep = list_entry(item, struct scif_endpt, list);
if (ep->remote_dev->node == node)
__scif_zap_mmaps(ep);
}
mutex_unlock(&scif_info.connlock);
}
/*
* Wrapper for removing remote memory mappings for a particular node. This API
* is called by peer nodes as part of handling a lost node.
*/
void scif_zap_mmaps(int node)
{
_scif_zap_mmaps(node, &scif_info.connected);
_scif_zap_mmaps(node, &scif_info.disconnected);
}
/*
* This API is only called while handling a lost node:
* a) Remote node is dead.
* b) Remote memory mappings have been zapped
* So we can traverse the remote_reg_list without any locks. Since
* the window has not yet been unregistered we can drop the ref count
* and queue it to the cleanup thread.
*/
static void __scif_cleanup_rma_for_zombies(struct scif_endpt *ep)
{
struct list_head *pos, *tmp;
struct scif_window *window;
list_for_each_safe(pos, tmp, &ep->rma_info.remote_reg_list) {
window = list_entry(pos, struct scif_window, list);
if (window->ref_count)
scif_put_window(window, window->nr_pages);
else
dev_err(scif_info.mdev.this_device,
"%s %d unexpected\n",
__func__, __LINE__);
if (!window->ref_count) {
atomic_inc(&ep->rma_info.tw_refcount);
list_del_init(&window->list);
scif_queue_for_cleanup(window, &scif_info.rma);
}
}
}
/* Cleanup remote registration lists for zombie endpoints */
void scif_cleanup_rma_for_zombies(int node)
{
struct scif_endpt *ep;
struct list_head *item;
mutex_lock(&scif_info.eplock);
list_for_each(item, &scif_info.zombie) {
ep = list_entry(item, struct scif_endpt, list);
if (ep->remote_dev && ep->remote_dev->node == node)
__scif_cleanup_rma_for_zombies(ep);
}
mutex_unlock(&scif_info.eplock);
flush_work(&scif_info.misc_work);
}
/* Insert the VMA into the per endpoint VMA list */
static int scif_insert_vma(struct scif_endpt *ep, struct vm_area_struct *vma)
{
struct scif_vma_info *info;
int err = 0;
info = kzalloc(sizeof(*info), GFP_KERNEL);
if (!info) {
err = -ENOMEM;
goto done;
}
info->vma = vma;
spin_lock(&ep->lock);
list_add_tail(&info->list, &ep->rma_info.vma_list);
spin_unlock(&ep->lock);
done:
return err;
}
/* Delete the VMA from the per endpoint VMA list */
static void scif_delete_vma(struct scif_endpt *ep, struct vm_area_struct *vma)
{
struct list_head *item;
struct scif_vma_info *info;
spin_lock(&ep->lock);
list_for_each(item, &ep->rma_info.vma_list) {
info = list_entry(item, struct scif_vma_info, list);
if (info->vma == vma) {
list_del(&info->list);
kfree(info);
break;
}
}
spin_unlock(&ep->lock);
}
static phys_addr_t scif_get_phys(phys_addr_t phys, struct scif_endpt *ep)
{
struct scif_dev *scifdev = (struct scif_dev *)ep->remote_dev;
struct scif_hw_dev *sdev = scifdev->sdev;
phys_addr_t out_phys, apt_base = 0;
/*
* If the DMA address is card relative then we need to add the
* aperture base for mmap to work correctly
*/
if (!scifdev_self(scifdev) && sdev->aper && sdev->card_rel_da)
apt_base = sdev->aper->pa;
out_phys = apt_base + phys;
return out_phys;
}
int scif_get_pages(scif_epd_t epd, off_t offset, size_t len,
struct scif_range **pages)
{
struct scif_endpt *ep = (struct scif_endpt *)epd;
struct scif_rma_req req;
struct scif_window *window = NULL;
int nr_pages, err, i;
dev_dbg(scif_info.mdev.this_device,
"SCIFAPI get_pinned_pages: ep %p offset 0x%lx len 0x%lx\n",
ep, offset, len);
err = scif_verify_epd(ep);
if (err)
return err;
if (!len || (offset < 0) ||
(offset + len < offset) ||
(ALIGN(offset, PAGE_SIZE) != offset) ||
(ALIGN(len, PAGE_SIZE) != len))
return -EINVAL;
nr_pages = len >> PAGE_SHIFT;
req.out_window = &window;
req.offset = offset;
req.prot = 0;
req.nr_bytes = len;
req.type = SCIF_WINDOW_SINGLE;
req.head = &ep->rma_info.remote_reg_list;
mutex_lock(&ep->rma_info.rma_lock);
/* Does a valid window exist? */
err = scif_query_window(&req);
if (err) {
dev_err(&ep->remote_dev->sdev->dev,
"%s %d err %d\n", __func__, __LINE__, err);
goto error;
}
/* Allocate scif_range */
*pages = kzalloc(sizeof(**pages), GFP_KERNEL);
if (!*pages) {
err = -ENOMEM;
goto error;
}
/* Allocate phys addr array */
(*pages)->phys_addr = scif_zalloc(nr_pages * sizeof(dma_addr_t));
if (!((*pages)->phys_addr)) {
err = -ENOMEM;
goto error;
}
if (scif_is_mgmt_node() && !scifdev_self(ep->remote_dev)) {
/* Allocate virtual address array */
((*pages)->va = scif_zalloc(nr_pages * sizeof(void *)));
if (!(*pages)->va) {
err = -ENOMEM;
goto error;
}
}
/* Populate the values */
(*pages)->cookie = window;
(*pages)->nr_pages = nr_pages;
(*pages)->prot_flags = window->prot;
for (i = 0; i < nr_pages; i++) {
(*pages)->phys_addr[i] =
__scif_off_to_dma_addr(window, offset +
(i * PAGE_SIZE));
(*pages)->phys_addr[i] = scif_get_phys((*pages)->phys_addr[i],
ep);
if (scif_is_mgmt_node() && !scifdev_self(ep->remote_dev))
(*pages)->va[i] =
ep->remote_dev->sdev->aper->va +
(*pages)->phys_addr[i] -
ep->remote_dev->sdev->aper->pa;
}
scif_get_window(window, nr_pages);
error:
mutex_unlock(&ep->rma_info.rma_lock);
if (err) {
if (*pages) {
scif_free((*pages)->phys_addr,
nr_pages * sizeof(dma_addr_t));
scif_free((*pages)->va,
nr_pages * sizeof(void *));
kfree(*pages);
*pages = NULL;
}
dev_err(&ep->remote_dev->sdev->dev,
"%s %d err %d\n", __func__, __LINE__, err);
}
return err;
}
EXPORT_SYMBOL_GPL(scif_get_pages);
int scif_put_pages(struct scif_range *pages)
{
struct scif_endpt *ep;
struct scif_window *window;
struct scifmsg msg;
if (!pages || !pages->cookie)
return -EINVAL;
window = pages->cookie;
if (!window || window->magic != SCIFEP_MAGIC)
return -EINVAL;
ep = (struct scif_endpt *)window->ep;
/*
* If the state is SCIFEP_CONNECTED or SCIFEP_DISCONNECTED then the
* callee should be allowed to release references to the pages,
* else the endpoint was not connected in the first place,
* hence the ENOTCONN.
*/
if (ep->state != SCIFEP_CONNECTED && ep->state != SCIFEP_DISCONNECTED)
return -ENOTCONN;
mutex_lock(&ep->rma_info.rma_lock);
scif_put_window(window, pages->nr_pages);
/* Initiate window destruction if ref count is zero */
if (!window->ref_count) {
list_del(&window->list);
mutex_unlock(&ep->rma_info.rma_lock);
scif_drain_dma_intr(ep->remote_dev->sdev,
ep->rma_info.dma_chan);
/* Inform the peer about this window being destroyed. */
msg.uop = SCIF_MUNMAP;
msg.src = ep->port;
msg.payload[0] = window->peer_window;
/* No error handling for notification messages */
scif_nodeqp_send(ep->remote_dev, &msg);
/* Destroy this window from the peer's registered AS */
scif_destroy_remote_window(window);
} else {
mutex_unlock(&ep->rma_info.rma_lock);
}
scif_free(pages->phys_addr, pages->nr_pages * sizeof(dma_addr_t));
scif_free(pages->va, pages->nr_pages * sizeof(void *));
kfree(pages);
return 0;
}
EXPORT_SYMBOL_GPL(scif_put_pages);
/*
* scif_rma_list_mmap:
*
* Traverse the remote registration list starting from start_window:
* 1) Create VtoP mappings via remap_pfn_range(..)
* 2) Once step 1) and 2) complete successfully then traverse the range of
* windows again and bump the reference count.
* RMA lock must be held.
*/
static int scif_rma_list_mmap(struct scif_window *start_window, s64 offset,
int nr_pages, struct vm_area_struct *vma)
{
s64 end_offset, loop_offset = offset;
struct scif_window *window = start_window;
int loop_nr_pages, nr_pages_left = nr_pages;
struct scif_endpt *ep = (struct scif_endpt *)start_window->ep;
struct list_head *head = &ep->rma_info.remote_reg_list;
int i, err = 0;
dma_addr_t phys_addr;
struct scif_window_iter src_win_iter;
size_t contig_bytes = 0;
might_sleep();
list_for_each_entry_from(window, head, list) {
end_offset = window->offset +
(window->nr_pages << PAGE_SHIFT);
loop_nr_pages = min_t(int,
(end_offset - loop_offset) >> PAGE_SHIFT,
nr_pages_left);
scif_init_window_iter(window, &src_win_iter);
for (i = 0; i < loop_nr_pages; i++) {
phys_addr = scif_off_to_dma_addr(window, loop_offset,
&contig_bytes,
&src_win_iter);
phys_addr = scif_get_phys(phys_addr, ep);
err = remap_pfn_range(vma,
vma->vm_start +
loop_offset - offset,
phys_addr >> PAGE_SHIFT,
PAGE_SIZE,
vma->vm_page_prot);
if (err)
goto error;
loop_offset += PAGE_SIZE;
}
nr_pages_left -= loop_nr_pages;
if (!nr_pages_left)
break;
}
/*
* No more failures expected. Bump up the ref count for all
* the windows. Another traversal from start_window required
* for handling errors encountered across windows during
* remap_pfn_range(..).
*/
loop_offset = offset;
nr_pages_left = nr_pages;
window = start_window;
head = &ep->rma_info.remote_reg_list;
list_for_each_entry_from(window, head, list) {
end_offset = window->offset +
(window->nr_pages << PAGE_SHIFT);
loop_nr_pages = min_t(int,
(end_offset - loop_offset) >> PAGE_SHIFT,
nr_pages_left);
scif_get_window(window, loop_nr_pages);
nr_pages_left -= loop_nr_pages;
loop_offset += (loop_nr_pages << PAGE_SHIFT);
if (!nr_pages_left)
break;
}
error:
if (err)
dev_err(scif_info.mdev.this_device,
"%s %d err %d\n", __func__, __LINE__, err);
return err;
}
/*
* scif_rma_list_munmap:
*
* Traverse the remote registration list starting from window:
* 1) Decrement ref count.
* 2) If the ref count drops to zero then send a SCIF_MUNMAP message to peer.
* RMA lock must be held.
*/
static void scif_rma_list_munmap(struct scif_window *start_window,
s64 offset, int nr_pages)
{
struct scifmsg msg;
s64 loop_offset = offset, end_offset;
int loop_nr_pages, nr_pages_left = nr_pages;
struct scif_endpt *ep = (struct scif_endpt *)start_window->ep;
struct list_head *head = &ep->rma_info.remote_reg_list;
struct scif_window *window = start_window, *_window;
msg.uop = SCIF_MUNMAP;
msg.src = ep->port;
loop_offset = offset;
nr_pages_left = nr_pages;
list_for_each_entry_safe_from(window, _window, head, list) {
end_offset = window->offset +
(window->nr_pages << PAGE_SHIFT);
loop_nr_pages = min_t(int,
(end_offset - loop_offset) >> PAGE_SHIFT,
nr_pages_left);
scif_put_window(window, loop_nr_pages);
if (!window->ref_count) {
struct scif_dev *rdev = ep->remote_dev;
scif_drain_dma_intr(rdev->sdev,
ep->rma_info.dma_chan);
/* Inform the peer about this munmap */
msg.payload[0] = window->peer_window;
/* No error handling for Notification messages. */
scif_nodeqp_send(ep->remote_dev, &msg);
list_del(&window->list);
/* Destroy this window from the peer's registered AS */
scif_destroy_remote_window(window);
}
nr_pages_left -= loop_nr_pages;
loop_offset += (loop_nr_pages << PAGE_SHIFT);
if (!nr_pages_left)
break;
}
}
/*
* The private data field of each VMA used to mmap a remote window
* points to an instance of struct vma_pvt
*/
struct vma_pvt {
struct scif_endpt *ep; /* End point for remote window */
s64 offset; /* offset within remote window */
bool valid_offset; /* offset is valid only if the original
* mmap request was for a single page
* else the offset within the vma is
* the correct offset
*/
struct kref ref;
};
static void vma_pvt_release(struct kref *ref)
{
struct vma_pvt *vmapvt = container_of(ref, struct vma_pvt, ref);
kfree(vmapvt);
}
/**
* scif_vma_open - VMA open driver callback
* @vma: VMM memory area.
* The open method is called by the kernel to allow the subsystem implementing
* the VMA to initialize the area. This method is invoked any time a new
* reference to the VMA is made (when a process forks, for example).
* The one exception happens when the VMA is first created by mmap;
* in this case, the driver's mmap method is called instead.
* This function is also invoked when an existing VMA is split by the kernel
* due to a call to munmap on a subset of the VMA resulting in two VMAs.
* The kernel invokes this function only on one of the two VMAs.
*/
static void scif_vma_open(struct vm_area_struct *vma)
{
struct vma_pvt *vmapvt = vma->vm_private_data;
dev_dbg(scif_info.mdev.this_device,
"SCIFAPI vma open: vma_start 0x%lx vma_end 0x%lx\n",
vma->vm_start, vma->vm_end);
scif_insert_vma(vmapvt->ep, vma);
kref_get(&vmapvt->ref);
}
/**
* scif_munmap - VMA close driver callback.
* @vma: VMM memory area.
* When an area is destroyed, the kernel calls its close operation.
* Note that there's no usage count associated with VMA's; the area
* is opened and closed exactly once by each process that uses it.
*/
static void scif_munmap(struct vm_area_struct *vma)
{
struct scif_endpt *ep;
struct vma_pvt *vmapvt = vma->vm_private_data;
int nr_pages = vma_pages(vma);
s64 offset;
struct scif_rma_req req;
struct scif_window *window = NULL;
int err;
might_sleep();
dev_dbg(scif_info.mdev.this_device,
"SCIFAPI munmap: vma_start 0x%lx vma_end 0x%lx\n",
vma->vm_start, vma->vm_end);
ep = vmapvt->ep;
offset = vmapvt->valid_offset ? vmapvt->offset :
(vma->vm_pgoff) << PAGE_SHIFT;
dev_dbg(scif_info.mdev.this_device,
"SCIFAPI munmap: ep %p nr_pages 0x%x offset 0x%llx\n",
ep, nr_pages, offset);
req.out_window = &window;
req.offset = offset;
req.nr_bytes = vma->vm_end - vma->vm_start;
req.prot = vma->vm_flags & (VM_READ | VM_WRITE);
req.type = SCIF_WINDOW_PARTIAL;
req.head = &ep->rma_info.remote_reg_list;
mutex_lock(&ep->rma_info.rma_lock);
err = scif_query_window(&req);
if (err)
dev_err(scif_info.mdev.this_device,
"%s %d err %d\n", __func__, __LINE__, err);
else
scif_rma_list_munmap(window, offset, nr_pages);
mutex_unlock(&ep->rma_info.rma_lock);
/*
* The kernel probably zeroes these out but we still want
* to clean up our own mess just in case.
*/
vma->vm_ops = NULL;
vma->vm_private_data = NULL;
kref_put(&vmapvt->ref, vma_pvt_release);
scif_delete_vma(ep, vma);
}
static const struct vm_operations_struct scif_vm_ops = {
.open = scif_vma_open,
.close = scif_munmap,
};
/**
* scif_mmap - Map pages in virtual address space to a remote window.
* @vma: VMM memory area.
* @epd: endpoint descriptor
*
* Return: Upon successful completion, scif_mmap() returns zero
* else an apt error is returned as documented in scif.h
*/
int scif_mmap(struct vm_area_struct *vma, scif_epd_t epd)
{
struct scif_rma_req req;
struct scif_window *window = NULL;
struct scif_endpt *ep = (struct scif_endpt *)epd;
s64 start_offset = vma->vm_pgoff << PAGE_SHIFT;
int nr_pages = vma_pages(vma);
int err;
struct vma_pvt *vmapvt;
dev_dbg(scif_info.mdev.this_device,
"SCIFAPI mmap: ep %p start_offset 0x%llx nr_pages 0x%x\n",
ep, start_offset, nr_pages);
err = scif_verify_epd(ep);
if (err)
return err;
might_sleep();
err = scif_insert_vma(ep, vma);
if (err)
return err;
vmapvt = kzalloc(sizeof(*vmapvt), GFP_KERNEL);
if (!vmapvt) {
scif_delete_vma(ep, vma);
return -ENOMEM;
}
vmapvt->ep = ep;
kref_init(&vmapvt->ref);
req.out_window = &window;
req.offset = start_offset;
req.nr_bytes = vma->vm_end - vma->vm_start;
req.prot = vma->vm_flags & (VM_READ | VM_WRITE);
req.type = SCIF_WINDOW_PARTIAL;
req.head = &ep->rma_info.remote_reg_list;
mutex_lock(&ep->rma_info.rma_lock);
/* Does a valid window exist? */
err = scif_query_window(&req);
if (err) {
dev_err(&ep->remote_dev->sdev->dev,
"%s %d err %d\n", __func__, __LINE__, err);
goto error_unlock;
}
/* Default prot for loopback */
if (!scifdev_self(ep->remote_dev))
vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
/*
* VM_DONTCOPY - Do not copy this vma on fork
* VM_DONTEXPAND - Cannot expand with mremap()
* VM_RESERVED - Count as reserved_vm like IO
* VM_PFNMAP - Page-ranges managed without "struct page"
* VM_IO - Memory mapped I/O or similar
*
* We do not want to copy this VMA automatically on a fork(),
* expand this VMA due to mremap() or swap out these pages since
* the VMA is actually backed by physical pages in the remote
* node's physical memory and not via a struct page.
*/
vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP;
if (!scifdev_self(ep->remote_dev))
vma->vm_flags |= VM_IO | VM_PFNMAP;
/* Map this range of windows */
err = scif_rma_list_mmap(window, start_offset, nr_pages, vma);
if (err) {
dev_err(&ep->remote_dev->sdev->dev,
"%s %d err %d\n", __func__, __LINE__, err);
goto error_unlock;
}
/* Set up the driver call back */
vma->vm_ops = &scif_vm_ops;
vma->vm_private_data = vmapvt;
error_unlock:
mutex_unlock(&ep->rma_info.rma_lock);
if (err) {
kfree(vmapvt);
dev_err(&ep->remote_dev->sdev->dev,
"%s %d err %d\n", __func__, __LINE__, err);
scif_delete_vma(ep, vma);
}
return err;
}