2016-05-19 00:15:08 +08:00
|
|
|
/*
|
|
|
|
* Copyright(c) 2016 Intel Corporation. All rights reserved.
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
* it under the terms of version 2 of the GNU General Public License as
|
|
|
|
* published by the Free Software Foundation.
|
|
|
|
*
|
|
|
|
* This program is distributed in the hope that it will be useful, but
|
|
|
|
* WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
* General Public License for more details.
|
|
|
|
*/
|
|
|
|
#include <linux/pagemap.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/device.h>
|
2017-02-05 23:03:58 +08:00
|
|
|
#include <linux/magic.h>
|
2016-07-25 12:55:45 +08:00
|
|
|
#include <linux/mount.h>
|
2016-05-19 00:15:08 +08:00
|
|
|
#include <linux/pfn_t.h>
|
2016-07-25 12:55:45 +08:00
|
|
|
#include <linux/hash.h>
|
2016-07-25 06:55:42 +08:00
|
|
|
#include <linux/cdev.h>
|
2016-05-19 00:15:08 +08:00
|
|
|
#include <linux/slab.h>
|
|
|
|
#include <linux/dax.h>
|
|
|
|
#include <linux/fs.h>
|
|
|
|
#include <linux/mm.h>
|
2016-08-07 07:05:06 +08:00
|
|
|
#include "dax.h"
|
2016-05-19 00:15:08 +08:00
|
|
|
|
2016-07-25 06:55:42 +08:00
|
|
|
static dev_t dax_devt;
|
2016-05-19 00:15:08 +08:00
|
|
|
static struct class *dax_class;
|
|
|
|
static DEFINE_IDA(dax_minor_ida);
|
2016-07-25 06:55:42 +08:00
|
|
|
static int nr_dax = CONFIG_NR_DEV_DAX;
|
|
|
|
module_param(nr_dax, int, S_IRUGO);
|
2016-07-25 12:55:45 +08:00
|
|
|
static struct vfsmount *dax_mnt;
|
|
|
|
static struct kmem_cache *dax_cache __read_mostly;
|
|
|
|
static struct super_block *dax_superblock __read_mostly;
|
2016-07-25 06:55:42 +08:00
|
|
|
MODULE_PARM_DESC(nr_dax, "max number of device-dax instances");
|
2016-05-19 00:15:08 +08:00
|
|
|
|
|
|
|
/**
|
|
|
|
* struct dax_region - mapping infrastructure for dax devices
|
|
|
|
* @id: kernel-wide unique region for a memory range
|
|
|
|
* @base: linear address corresponding to @res
|
|
|
|
* @kref: to pin while other agents have a need to do lookups
|
|
|
|
* @dev: parent device backing this region
|
|
|
|
* @align: allocation and mapping alignment for child dax devices
|
|
|
|
* @res: physical address range of the region
|
|
|
|
* @pfn_flags: identify whether the pfns are paged back or not
|
|
|
|
*/
|
|
|
|
struct dax_region {
|
|
|
|
int id;
|
|
|
|
struct ida ida;
|
|
|
|
void *base;
|
|
|
|
struct kref kref;
|
|
|
|
struct device *dev;
|
|
|
|
unsigned int align;
|
|
|
|
struct resource res;
|
|
|
|
unsigned long pfn_flags;
|
|
|
|
};
|
|
|
|
|
|
|
|
/**
|
|
|
|
* struct dax_dev - subdivision of a dax region
|
|
|
|
* @region - parent region
|
|
|
|
* @dev - device backing the character device
|
2016-07-25 06:55:42 +08:00
|
|
|
* @cdev - core chardev data
|
2016-05-15 03:20:44 +08:00
|
|
|
* @alive - !alive + rcu grace period == no new mappings can be established
|
2016-05-19 00:15:08 +08:00
|
|
|
* @id - child id in the region
|
|
|
|
* @num_resources - number of physical address extents in this device
|
|
|
|
* @res - array of physical address ranges
|
|
|
|
*/
|
|
|
|
struct dax_dev {
|
|
|
|
struct dax_region *region;
|
2016-07-25 12:55:45 +08:00
|
|
|
struct inode *inode;
|
2016-08-11 15:41:51 +08:00
|
|
|
struct device dev;
|
2016-07-25 06:55:42 +08:00
|
|
|
struct cdev cdev;
|
2016-05-15 03:20:44 +08:00
|
|
|
bool alive;
|
2016-05-19 00:15:08 +08:00
|
|
|
int id;
|
|
|
|
int num_resources;
|
|
|
|
struct resource res[0];
|
|
|
|
};
|
|
|
|
|
2016-12-18 06:50:04 +08:00
|
|
|
static ssize_t id_show(struct device *dev,
|
|
|
|
struct device_attribute *attr, char *buf)
|
|
|
|
{
|
|
|
|
struct dax_region *dax_region;
|
|
|
|
ssize_t rc = -ENXIO;
|
|
|
|
|
|
|
|
device_lock(dev);
|
|
|
|
dax_region = dev_get_drvdata(dev);
|
|
|
|
if (dax_region)
|
|
|
|
rc = sprintf(buf, "%d\n", dax_region->id);
|
|
|
|
device_unlock(dev);
|
|
|
|
|
|
|
|
return rc;
|
|
|
|
}
|
|
|
|
static DEVICE_ATTR_RO(id);
|
|
|
|
|
|
|
|
static ssize_t region_size_show(struct device *dev,
|
|
|
|
struct device_attribute *attr, char *buf)
|
|
|
|
{
|
|
|
|
struct dax_region *dax_region;
|
|
|
|
ssize_t rc = -ENXIO;
|
|
|
|
|
|
|
|
device_lock(dev);
|
|
|
|
dax_region = dev_get_drvdata(dev);
|
|
|
|
if (dax_region)
|
|
|
|
rc = sprintf(buf, "%llu\n", (unsigned long long)
|
|
|
|
resource_size(&dax_region->res));
|
|
|
|
device_unlock(dev);
|
|
|
|
|
|
|
|
return rc;
|
|
|
|
}
|
|
|
|
static struct device_attribute dev_attr_region_size = __ATTR(size, 0444,
|
|
|
|
region_size_show, NULL);
|
|
|
|
|
|
|
|
static ssize_t align_show(struct device *dev,
|
|
|
|
struct device_attribute *attr, char *buf)
|
|
|
|
{
|
|
|
|
struct dax_region *dax_region;
|
|
|
|
ssize_t rc = -ENXIO;
|
|
|
|
|
|
|
|
device_lock(dev);
|
|
|
|
dax_region = dev_get_drvdata(dev);
|
|
|
|
if (dax_region)
|
|
|
|
rc = sprintf(buf, "%u\n", dax_region->align);
|
|
|
|
device_unlock(dev);
|
|
|
|
|
|
|
|
return rc;
|
|
|
|
}
|
|
|
|
static DEVICE_ATTR_RO(align);
|
|
|
|
|
|
|
|
static struct attribute *dax_region_attributes[] = {
|
|
|
|
&dev_attr_region_size.attr,
|
|
|
|
&dev_attr_align.attr,
|
|
|
|
&dev_attr_id.attr,
|
|
|
|
NULL,
|
|
|
|
};
|
|
|
|
|
|
|
|
static const struct attribute_group dax_region_attribute_group = {
|
|
|
|
.name = "dax_region",
|
|
|
|
.attrs = dax_region_attributes,
|
|
|
|
};
|
|
|
|
|
|
|
|
static const struct attribute_group *dax_region_attribute_groups[] = {
|
|
|
|
&dax_region_attribute_group,
|
|
|
|
NULL,
|
|
|
|
};
|
|
|
|
|
2016-07-25 12:55:45 +08:00
|
|
|
static struct inode *dax_alloc_inode(struct super_block *sb)
|
2016-05-19 00:15:08 +08:00
|
|
|
{
|
2016-07-25 12:55:45 +08:00
|
|
|
return kmem_cache_alloc(dax_cache, GFP_KERNEL);
|
|
|
|
}
|
2016-05-19 00:15:08 +08:00
|
|
|
|
2016-07-25 12:55:45 +08:00
|
|
|
static void dax_i_callback(struct rcu_head *head)
|
|
|
|
{
|
|
|
|
struct inode *inode = container_of(head, struct inode, i_rcu);
|
|
|
|
|
|
|
|
kmem_cache_free(dax_cache, inode);
|
2016-05-19 00:15:08 +08:00
|
|
|
}
|
|
|
|
|
2016-07-25 12:55:45 +08:00
|
|
|
static void dax_destroy_inode(struct inode *inode)
|
2016-05-19 00:15:08 +08:00
|
|
|
{
|
2016-07-25 12:55:45 +08:00
|
|
|
call_rcu(&inode->i_rcu, dax_i_callback);
|
2016-05-19 00:15:08 +08:00
|
|
|
}
|
|
|
|
|
2016-07-25 12:55:45 +08:00
|
|
|
static const struct super_operations dax_sops = {
|
|
|
|
.statfs = simple_statfs,
|
|
|
|
.alloc_inode = dax_alloc_inode,
|
|
|
|
.destroy_inode = dax_destroy_inode,
|
|
|
|
.drop_inode = generic_delete_inode,
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct dentry *dax_mount(struct file_system_type *fs_type,
|
|
|
|
int flags, const char *dev_name, void *data)
|
2016-05-19 00:15:08 +08:00
|
|
|
{
|
2016-07-25 12:55:45 +08:00
|
|
|
return mount_pseudo(fs_type, "dax:", &dax_sops, NULL, DAXFS_MAGIC);
|
|
|
|
}
|
2016-05-19 00:15:08 +08:00
|
|
|
|
2016-07-25 12:55:45 +08:00
|
|
|
static struct file_system_type dax_type = {
|
|
|
|
.name = "dax",
|
|
|
|
.mount = dax_mount,
|
|
|
|
.kill_sb = kill_anon_super,
|
|
|
|
};
|
|
|
|
|
|
|
|
static int dax_test(struct inode *inode, void *data)
|
|
|
|
{
|
|
|
|
return inode->i_cdev == data;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int dax_set(struct inode *inode, void *data)
|
|
|
|
{
|
|
|
|
inode->i_cdev = data;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct inode *dax_inode_get(struct cdev *cdev, dev_t devt)
|
|
|
|
{
|
|
|
|
struct inode *inode;
|
|
|
|
|
|
|
|
inode = iget5_locked(dax_superblock, hash_32(devt + DAXFS_MAGIC, 31),
|
|
|
|
dax_test, dax_set, cdev);
|
|
|
|
|
|
|
|
if (!inode)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
if (inode->i_state & I_NEW) {
|
|
|
|
inode->i_mode = S_IFCHR;
|
|
|
|
inode->i_flags = S_DAX;
|
|
|
|
inode->i_rdev = devt;
|
|
|
|
mapping_set_gfp_mask(&inode->i_data, GFP_USER);
|
|
|
|
unlock_new_inode(inode);
|
|
|
|
}
|
|
|
|
return inode;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void init_once(void *inode)
|
|
|
|
{
|
|
|
|
inode_init_once(inode);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int dax_inode_init(void)
|
|
|
|
{
|
|
|
|
int rc;
|
|
|
|
|
|
|
|
dax_cache = kmem_cache_create("dax_cache", sizeof(struct inode), 0,
|
|
|
|
(SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT|
|
|
|
|
SLAB_MEM_SPREAD|SLAB_ACCOUNT),
|
|
|
|
init_once);
|
|
|
|
if (!dax_cache)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
rc = register_filesystem(&dax_type);
|
|
|
|
if (rc)
|
|
|
|
goto err_register_fs;
|
|
|
|
|
|
|
|
dax_mnt = kern_mount(&dax_type);
|
|
|
|
if (IS_ERR(dax_mnt)) {
|
|
|
|
rc = PTR_ERR(dax_mnt);
|
|
|
|
goto err_mount;
|
|
|
|
}
|
|
|
|
dax_superblock = dax_mnt->mnt_sb;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
err_mount:
|
|
|
|
unregister_filesystem(&dax_type);
|
|
|
|
err_register_fs:
|
|
|
|
kmem_cache_destroy(dax_cache);
|
|
|
|
|
|
|
|
return rc;
|
2016-05-19 00:15:08 +08:00
|
|
|
}
|
|
|
|
|
2016-07-25 12:55:45 +08:00
|
|
|
static void dax_inode_exit(void)
|
|
|
|
{
|
|
|
|
kern_unmount(dax_mnt);
|
|
|
|
unregister_filesystem(&dax_type);
|
|
|
|
kmem_cache_destroy(dax_cache);
|
|
|
|
}
|
|
|
|
|
2016-05-19 00:15:08 +08:00
|
|
|
static void dax_region_free(struct kref *kref)
|
|
|
|
{
|
|
|
|
struct dax_region *dax_region;
|
|
|
|
|
|
|
|
dax_region = container_of(kref, struct dax_region, kref);
|
|
|
|
kfree(dax_region);
|
|
|
|
}
|
|
|
|
|
|
|
|
void dax_region_put(struct dax_region *dax_region)
|
|
|
|
{
|
|
|
|
kref_put(&dax_region->kref, dax_region_free);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(dax_region_put);
|
|
|
|
|
2016-12-18 06:50:04 +08:00
|
|
|
static void dax_region_unregister(void *region)
|
|
|
|
{
|
|
|
|
struct dax_region *dax_region = region;
|
|
|
|
|
|
|
|
sysfs_remove_groups(&dax_region->dev->kobj,
|
|
|
|
dax_region_attribute_groups);
|
|
|
|
dax_region_put(dax_region);
|
|
|
|
}
|
|
|
|
|
2016-05-19 00:15:08 +08:00
|
|
|
struct dax_region *alloc_dax_region(struct device *parent, int region_id,
|
|
|
|
struct resource *res, unsigned int align, void *addr,
|
|
|
|
unsigned long pfn_flags)
|
|
|
|
{
|
|
|
|
struct dax_region *dax_region;
|
|
|
|
|
2016-12-18 06:50:04 +08:00
|
|
|
/*
|
|
|
|
* The DAX core assumes that it can store its private data in
|
|
|
|
* parent->driver_data. This WARN is a reminder / safeguard for
|
|
|
|
* developers of device-dax drivers.
|
|
|
|
*/
|
|
|
|
if (dev_get_drvdata(parent)) {
|
|
|
|
dev_WARN(parent, "dax core failed to setup private data\n");
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2016-07-20 07:17:58 +08:00
|
|
|
if (!IS_ALIGNED(res->start, align)
|
|
|
|
|| !IS_ALIGNED(resource_size(res), align))
|
|
|
|
return NULL;
|
2016-05-19 00:15:08 +08:00
|
|
|
|
2016-07-20 07:17:58 +08:00
|
|
|
dax_region = kzalloc(sizeof(*dax_region), GFP_KERNEL);
|
2016-05-19 00:15:08 +08:00
|
|
|
if (!dax_region)
|
|
|
|
return NULL;
|
|
|
|
|
2016-12-18 06:50:04 +08:00
|
|
|
dev_set_drvdata(parent, dax_region);
|
2016-05-19 00:15:08 +08:00
|
|
|
memcpy(&dax_region->res, res, sizeof(*res));
|
|
|
|
dax_region->pfn_flags = pfn_flags;
|
|
|
|
kref_init(&dax_region->kref);
|
|
|
|
dax_region->id = region_id;
|
|
|
|
ida_init(&dax_region->ida);
|
|
|
|
dax_region->align = align;
|
|
|
|
dax_region->dev = parent;
|
|
|
|
dax_region->base = addr;
|
2016-12-18 06:50:04 +08:00
|
|
|
if (sysfs_create_groups(&parent->kobj, dax_region_attribute_groups)) {
|
|
|
|
kfree(dax_region);
|
|
|
|
return NULL;;
|
|
|
|
}
|
2016-05-19 00:15:08 +08:00
|
|
|
|
2016-12-18 06:50:04 +08:00
|
|
|
kref_get(&dax_region->kref);
|
|
|
|
if (devm_add_action_or_reset(parent, dax_region_unregister, dax_region))
|
|
|
|
return NULL;
|
2016-05-19 00:15:08 +08:00
|
|
|
return dax_region;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(alloc_dax_region);
|
|
|
|
|
2016-08-11 15:41:51 +08:00
|
|
|
static struct dax_dev *to_dax_dev(struct device *dev)
|
|
|
|
{
|
|
|
|
return container_of(dev, struct dax_dev, dev);
|
|
|
|
}
|
|
|
|
|
2016-05-19 00:15:08 +08:00
|
|
|
static ssize_t size_show(struct device *dev,
|
|
|
|
struct device_attribute *attr, char *buf)
|
|
|
|
{
|
2016-08-11 15:41:51 +08:00
|
|
|
struct dax_dev *dax_dev = to_dax_dev(dev);
|
2016-05-19 00:15:08 +08:00
|
|
|
unsigned long long size = 0;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < dax_dev->num_resources; i++)
|
|
|
|
size += resource_size(&dax_dev->res[i]);
|
|
|
|
|
|
|
|
return sprintf(buf, "%llu\n", size);
|
|
|
|
}
|
|
|
|
static DEVICE_ATTR_RO(size);
|
|
|
|
|
|
|
|
static struct attribute *dax_device_attributes[] = {
|
|
|
|
&dev_attr_size.attr,
|
|
|
|
NULL,
|
|
|
|
};
|
|
|
|
|
|
|
|
static const struct attribute_group dax_device_attribute_group = {
|
|
|
|
.attrs = dax_device_attributes,
|
|
|
|
};
|
|
|
|
|
|
|
|
static const struct attribute_group *dax_attribute_groups[] = {
|
|
|
|
&dax_device_attribute_group,
|
|
|
|
NULL,
|
|
|
|
};
|
|
|
|
|
2016-05-15 03:20:44 +08:00
|
|
|
static int check_vma(struct dax_dev *dax_dev, struct vm_area_struct *vma,
|
|
|
|
const char *func)
|
|
|
|
{
|
|
|
|
struct dax_region *dax_region = dax_dev->region;
|
2016-08-11 15:41:51 +08:00
|
|
|
struct device *dev = &dax_dev->dev;
|
2016-05-15 03:20:44 +08:00
|
|
|
unsigned long mask;
|
|
|
|
|
|
|
|
if (!dax_dev->alive)
|
|
|
|
return -ENXIO;
|
|
|
|
|
2016-11-17 01:00:38 +08:00
|
|
|
/* prevent private mappings from being established */
|
2016-12-07 09:03:35 +08:00
|
|
|
if ((vma->vm_flags & VM_MAYSHARE) != VM_MAYSHARE) {
|
2016-05-15 03:20:44 +08:00
|
|
|
dev_info(dev, "%s: %s: fail, attempted private mapping\n",
|
|
|
|
current->comm, func);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
mask = dax_region->align - 1;
|
|
|
|
if (vma->vm_start & mask || vma->vm_end & mask) {
|
|
|
|
dev_info(dev, "%s: %s: fail, unaligned vma (%#lx - %#lx, %#lx)\n",
|
|
|
|
current->comm, func, vma->vm_start, vma->vm_end,
|
|
|
|
mask);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) == PFN_DEV
|
|
|
|
&& (vma->vm_flags & VM_DONTCOPY) == 0) {
|
|
|
|
dev_info(dev, "%s: %s: fail, dax range requires MADV_DONTFORK\n",
|
|
|
|
current->comm, func);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!vma_is_dax(vma)) {
|
|
|
|
dev_info(dev, "%s: %s: fail, vma is not DAX capable\n",
|
|
|
|
current->comm, func);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static phys_addr_t pgoff_to_phys(struct dax_dev *dax_dev, pgoff_t pgoff,
|
|
|
|
unsigned long size)
|
|
|
|
{
|
|
|
|
struct resource *res;
|
|
|
|
phys_addr_t phys;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < dax_dev->num_resources; i++) {
|
|
|
|
res = &dax_dev->res[i];
|
|
|
|
phys = pgoff * PAGE_SIZE + res->start;
|
|
|
|
if (phys >= res->start && phys <= res->end)
|
|
|
|
break;
|
|
|
|
pgoff -= PHYS_PFN(resource_size(res));
|
|
|
|
}
|
|
|
|
|
|
|
|
if (i < dax_dev->num_resources) {
|
|
|
|
res = &dax_dev->res[i];
|
|
|
|
if (phys + size - 1 <= res->end)
|
|
|
|
return phys;
|
|
|
|
}
|
|
|
|
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
mm,fs,dax: change ->pmd_fault to ->huge_fault
Patch series "1G transparent hugepage support for device dax", v2.
The following series implements support for 1G trasparent hugepage on
x86 for device dax. The bulk of the code was written by Mathew Wilcox a
while back supporting transparent 1G hugepage for fs DAX. I have
forward ported the relevant bits to 4.10-rc. The current submission has
only the necessary code to support device DAX.
Comments from Dan Williams: So the motivation and intended user of this
functionality mirrors the motivation and users of 1GB page support in
hugetlbfs. Given expected capacities of persistent memory devices an
in-memory database may want to reduce tlb pressure beyond what they can
already achieve with 2MB mappings of a device-dax file. We have
customer feedback to that effect as Willy mentioned in his previous
version of these patches [1].
[1]: https://lkml.org/lkml/2016/1/31/52
Comments from Nilesh @ Oracle:
There are applications which have a process model; and if you assume
10,000 processes attempting to mmap all the 6TB memory available on a
server; we are looking at the following:
processes : 10,000
memory : 6TB
pte @ 4k page size: 8 bytes / 4K of memory * #processes = 6TB / 4k * 8 * 10000 = 1.5GB * 80000 = 120,000GB
pmd @ 2M page size: 120,000 / 512 = ~240GB
pud @ 1G page size: 240GB / 512 = ~480MB
As you can see with 2M pages, this system will use up an exorbitant
amount of DRAM to hold the page tables; but the 1G pages finally brings
it down to a reasonable level. Memory sizes will keep increasing; so
this number will keep increasing.
An argument can be made to convert the applications from process model
to thread model, but in the real world that may not be always practical.
Hopefully this helps explain the use case where this is valuable.
This patch (of 3):
In preparation for adding the ability to handle PUD pages, convert
vm_operations_struct.pmd_fault to vm_operations_struct.huge_fault. The
vm_fault structure is extended to include a union of the different page
table pointers that may be needed, and three flag bits are reserved to
indicate which type of pointer is in the union.
[ross.zwisler@linux.intel.com: remove unused function ext4_dax_huge_fault()]
Link: http://lkml.kernel.org/r/1485813172-7284-1-git-send-email-ross.zwisler@linux.intel.com
[dave.jiang@intel.com: clear PMD or PUD size flags when in fall through path]
Link: http://lkml.kernel.org/r/148589842696.5820.16078080610311444794.stgit@djiang5-desk3.ch.intel.com
Link: http://lkml.kernel.org/r/148545058784.17912.6353162518188733642.stgit@djiang5-desk3.ch.intel.com
Signed-off-by: Matthew Wilcox <mawilcox@microsoft.com>
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
Signed-off-by: Ross Zwisler <ross.zwisler@linux.intel.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Jan Kara <jack@suse.com>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Nilesh Choudhury <nilesh.choudhury@oracle.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Dave Jiang <dave.jiang@intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2017-02-25 06:56:59 +08:00
|
|
|
static int __dax_dev_pte_fault(struct dax_dev *dax_dev, struct vm_fault *vmf)
|
2016-05-15 03:20:44 +08:00
|
|
|
{
|
2016-08-11 15:41:51 +08:00
|
|
|
struct device *dev = &dax_dev->dev;
|
2016-05-15 03:20:44 +08:00
|
|
|
struct dax_region *dax_region;
|
|
|
|
int rc = VM_FAULT_SIGBUS;
|
|
|
|
phys_addr_t phys;
|
|
|
|
pfn_t pfn;
|
|
|
|
|
2017-02-25 06:56:41 +08:00
|
|
|
if (check_vma(dax_dev, vmf->vma, __func__))
|
2016-05-15 03:20:44 +08:00
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
|
|
|
|
dax_region = dax_dev->region;
|
|
|
|
if (dax_region->align > PAGE_SIZE) {
|
|
|
|
dev_dbg(dev, "%s: alignment > fault size\n", __func__);
|
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
}
|
|
|
|
|
|
|
|
phys = pgoff_to_phys(dax_dev, vmf->pgoff, PAGE_SIZE);
|
|
|
|
if (phys == -1) {
|
|
|
|
dev_dbg(dev, "%s: phys_to_pgoff(%#lx) failed\n", __func__,
|
|
|
|
vmf->pgoff);
|
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
}
|
|
|
|
|
|
|
|
pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
|
|
|
|
|
2017-02-25 06:56:41 +08:00
|
|
|
rc = vm_insert_mixed(vmf->vma, vmf->address, pfn);
|
2016-05-15 03:20:44 +08:00
|
|
|
|
|
|
|
if (rc == -ENOMEM)
|
|
|
|
return VM_FAULT_OOM;
|
|
|
|
if (rc < 0 && rc != -EBUSY)
|
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
|
|
|
|
return VM_FAULT_NOPAGE;
|
|
|
|
}
|
|
|
|
|
2017-02-23 07:40:06 +08:00
|
|
|
static int __dax_dev_pmd_fault(struct dax_dev *dax_dev, struct vm_fault *vmf)
|
2016-05-15 03:20:44 +08:00
|
|
|
{
|
2017-02-23 07:40:03 +08:00
|
|
|
unsigned long pmd_addr = vmf->address & PMD_MASK;
|
2016-08-11 15:41:51 +08:00
|
|
|
struct device *dev = &dax_dev->dev;
|
2016-05-15 03:20:44 +08:00
|
|
|
struct dax_region *dax_region;
|
|
|
|
phys_addr_t phys;
|
|
|
|
pgoff_t pgoff;
|
|
|
|
pfn_t pfn;
|
|
|
|
|
2017-02-23 07:40:06 +08:00
|
|
|
if (check_vma(dax_dev, vmf->vma, __func__))
|
2016-05-15 03:20:44 +08:00
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
|
|
|
|
dax_region = dax_dev->region;
|
|
|
|
if (dax_region->align > PMD_SIZE) {
|
|
|
|
dev_dbg(dev, "%s: alignment > fault size\n", __func__);
|
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* dax pmd mappings require pfn_t_devmap() */
|
|
|
|
if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) != (PFN_DEV|PFN_MAP)) {
|
|
|
|
dev_dbg(dev, "%s: alignment > fault size\n", __func__);
|
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
}
|
|
|
|
|
2017-02-23 07:40:06 +08:00
|
|
|
pgoff = linear_page_index(vmf->vma, pmd_addr);
|
2016-09-04 01:36:00 +08:00
|
|
|
phys = pgoff_to_phys(dax_dev, pgoff, PMD_SIZE);
|
2016-05-15 03:20:44 +08:00
|
|
|
if (phys == -1) {
|
|
|
|
dev_dbg(dev, "%s: phys_to_pgoff(%#lx) failed\n", __func__,
|
|
|
|
pgoff);
|
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
}
|
|
|
|
|
|
|
|
pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
|
|
|
|
|
2017-02-23 07:40:06 +08:00
|
|
|
return vmf_insert_pfn_pmd(vmf->vma, vmf->address, vmf->pmd, pfn,
|
2017-02-23 07:40:03 +08:00
|
|
|
vmf->flags & FAULT_FLAG_WRITE);
|
2016-05-15 03:20:44 +08:00
|
|
|
}
|
|
|
|
|
2017-02-25 06:57:05 +08:00
|
|
|
#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
|
|
|
|
static int __dax_dev_pud_fault(struct dax_dev *dax_dev, struct vm_fault *vmf)
|
|
|
|
{
|
|
|
|
unsigned long pud_addr = vmf->address & PUD_MASK;
|
|
|
|
struct device *dev = &dax_dev->dev;
|
|
|
|
struct dax_region *dax_region;
|
|
|
|
phys_addr_t phys;
|
|
|
|
pgoff_t pgoff;
|
|
|
|
pfn_t pfn;
|
|
|
|
|
|
|
|
if (check_vma(dax_dev, vmf->vma, __func__))
|
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
|
|
|
|
dax_region = dax_dev->region;
|
|
|
|
if (dax_region->align > PUD_SIZE) {
|
|
|
|
dev_dbg(dev, "%s: alignment > fault size\n", __func__);
|
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* dax pud mappings require pfn_t_devmap() */
|
|
|
|
if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) != (PFN_DEV|PFN_MAP)) {
|
|
|
|
dev_dbg(dev, "%s: alignment > fault size\n", __func__);
|
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
}
|
|
|
|
|
|
|
|
pgoff = linear_page_index(vmf->vma, pud_addr);
|
|
|
|
phys = pgoff_to_phys(dax_dev, pgoff, PUD_SIZE);
|
|
|
|
if (phys == -1) {
|
|
|
|
dev_dbg(dev, "%s: phys_to_pgoff(%#lx) failed\n", __func__,
|
|
|
|
pgoff);
|
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
}
|
|
|
|
|
|
|
|
pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
|
|
|
|
|
|
|
|
return vmf_insert_pfn_pud(vmf->vma, vmf->address, vmf->pud, pfn,
|
|
|
|
vmf->flags & FAULT_FLAG_WRITE);
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
static int __dax_dev_pud_fault(struct dax_dev *dax_dev, struct vm_fault *vmf)
|
|
|
|
{
|
|
|
|
return VM_FAULT_FALLBACK;
|
|
|
|
}
|
|
|
|
#endif /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
|
|
|
|
|
2017-02-25 06:57:08 +08:00
|
|
|
static int dax_dev_huge_fault(struct vm_fault *vmf,
|
|
|
|
enum page_entry_size pe_size)
|
2016-05-15 03:20:44 +08:00
|
|
|
{
|
|
|
|
int rc;
|
2017-02-23 07:40:06 +08:00
|
|
|
struct file *filp = vmf->vma->vm_file;
|
2016-05-15 03:20:44 +08:00
|
|
|
struct dax_dev *dax_dev = filp->private_data;
|
|
|
|
|
2016-08-11 15:41:51 +08:00
|
|
|
dev_dbg(&dax_dev->dev, "%s: %s: %s (%#lx - %#lx)\n", __func__,
|
2017-02-23 07:40:03 +08:00
|
|
|
current->comm, (vmf->flags & FAULT_FLAG_WRITE)
|
2017-02-23 07:40:06 +08:00
|
|
|
? "write" : "read",
|
|
|
|
vmf->vma->vm_start, vmf->vma->vm_end);
|
2016-05-15 03:20:44 +08:00
|
|
|
|
|
|
|
rcu_read_lock();
|
2017-02-25 06:57:08 +08:00
|
|
|
switch (pe_size) {
|
|
|
|
case PE_SIZE_PTE:
|
mm,fs,dax: change ->pmd_fault to ->huge_fault
Patch series "1G transparent hugepage support for device dax", v2.
The following series implements support for 1G trasparent hugepage on
x86 for device dax. The bulk of the code was written by Mathew Wilcox a
while back supporting transparent 1G hugepage for fs DAX. I have
forward ported the relevant bits to 4.10-rc. The current submission has
only the necessary code to support device DAX.
Comments from Dan Williams: So the motivation and intended user of this
functionality mirrors the motivation and users of 1GB page support in
hugetlbfs. Given expected capacities of persistent memory devices an
in-memory database may want to reduce tlb pressure beyond what they can
already achieve with 2MB mappings of a device-dax file. We have
customer feedback to that effect as Willy mentioned in his previous
version of these patches [1].
[1]: https://lkml.org/lkml/2016/1/31/52
Comments from Nilesh @ Oracle:
There are applications which have a process model; and if you assume
10,000 processes attempting to mmap all the 6TB memory available on a
server; we are looking at the following:
processes : 10,000
memory : 6TB
pte @ 4k page size: 8 bytes / 4K of memory * #processes = 6TB / 4k * 8 * 10000 = 1.5GB * 80000 = 120,000GB
pmd @ 2M page size: 120,000 / 512 = ~240GB
pud @ 1G page size: 240GB / 512 = ~480MB
As you can see with 2M pages, this system will use up an exorbitant
amount of DRAM to hold the page tables; but the 1G pages finally brings
it down to a reasonable level. Memory sizes will keep increasing; so
this number will keep increasing.
An argument can be made to convert the applications from process model
to thread model, but in the real world that may not be always practical.
Hopefully this helps explain the use case where this is valuable.
This patch (of 3):
In preparation for adding the ability to handle PUD pages, convert
vm_operations_struct.pmd_fault to vm_operations_struct.huge_fault. The
vm_fault structure is extended to include a union of the different page
table pointers that may be needed, and three flag bits are reserved to
indicate which type of pointer is in the union.
[ross.zwisler@linux.intel.com: remove unused function ext4_dax_huge_fault()]
Link: http://lkml.kernel.org/r/1485813172-7284-1-git-send-email-ross.zwisler@linux.intel.com
[dave.jiang@intel.com: clear PMD or PUD size flags when in fall through path]
Link: http://lkml.kernel.org/r/148589842696.5820.16078080610311444794.stgit@djiang5-desk3.ch.intel.com
Link: http://lkml.kernel.org/r/148545058784.17912.6353162518188733642.stgit@djiang5-desk3.ch.intel.com
Signed-off-by: Matthew Wilcox <mawilcox@microsoft.com>
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
Signed-off-by: Ross Zwisler <ross.zwisler@linux.intel.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Jan Kara <jack@suse.com>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Nilesh Choudhury <nilesh.choudhury@oracle.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Dave Jiang <dave.jiang@intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2017-02-25 06:56:59 +08:00
|
|
|
rc = __dax_dev_pte_fault(dax_dev, vmf);
|
|
|
|
break;
|
2017-02-25 06:57:08 +08:00
|
|
|
case PE_SIZE_PMD:
|
mm,fs,dax: change ->pmd_fault to ->huge_fault
Patch series "1G transparent hugepage support for device dax", v2.
The following series implements support for 1G trasparent hugepage on
x86 for device dax. The bulk of the code was written by Mathew Wilcox a
while back supporting transparent 1G hugepage for fs DAX. I have
forward ported the relevant bits to 4.10-rc. The current submission has
only the necessary code to support device DAX.
Comments from Dan Williams: So the motivation and intended user of this
functionality mirrors the motivation and users of 1GB page support in
hugetlbfs. Given expected capacities of persistent memory devices an
in-memory database may want to reduce tlb pressure beyond what they can
already achieve with 2MB mappings of a device-dax file. We have
customer feedback to that effect as Willy mentioned in his previous
version of these patches [1].
[1]: https://lkml.org/lkml/2016/1/31/52
Comments from Nilesh @ Oracle:
There are applications which have a process model; and if you assume
10,000 processes attempting to mmap all the 6TB memory available on a
server; we are looking at the following:
processes : 10,000
memory : 6TB
pte @ 4k page size: 8 bytes / 4K of memory * #processes = 6TB / 4k * 8 * 10000 = 1.5GB * 80000 = 120,000GB
pmd @ 2M page size: 120,000 / 512 = ~240GB
pud @ 1G page size: 240GB / 512 = ~480MB
As you can see with 2M pages, this system will use up an exorbitant
amount of DRAM to hold the page tables; but the 1G pages finally brings
it down to a reasonable level. Memory sizes will keep increasing; so
this number will keep increasing.
An argument can be made to convert the applications from process model
to thread model, but in the real world that may not be always practical.
Hopefully this helps explain the use case where this is valuable.
This patch (of 3):
In preparation for adding the ability to handle PUD pages, convert
vm_operations_struct.pmd_fault to vm_operations_struct.huge_fault. The
vm_fault structure is extended to include a union of the different page
table pointers that may be needed, and three flag bits are reserved to
indicate which type of pointer is in the union.
[ross.zwisler@linux.intel.com: remove unused function ext4_dax_huge_fault()]
Link: http://lkml.kernel.org/r/1485813172-7284-1-git-send-email-ross.zwisler@linux.intel.com
[dave.jiang@intel.com: clear PMD or PUD size flags when in fall through path]
Link: http://lkml.kernel.org/r/148589842696.5820.16078080610311444794.stgit@djiang5-desk3.ch.intel.com
Link: http://lkml.kernel.org/r/148545058784.17912.6353162518188733642.stgit@djiang5-desk3.ch.intel.com
Signed-off-by: Matthew Wilcox <mawilcox@microsoft.com>
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
Signed-off-by: Ross Zwisler <ross.zwisler@linux.intel.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Jan Kara <jack@suse.com>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Nilesh Choudhury <nilesh.choudhury@oracle.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Dave Jiang <dave.jiang@intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2017-02-25 06:56:59 +08:00
|
|
|
rc = __dax_dev_pmd_fault(dax_dev, vmf);
|
2017-02-25 06:57:05 +08:00
|
|
|
break;
|
2017-02-25 06:57:08 +08:00
|
|
|
case PE_SIZE_PUD:
|
2017-02-25 06:57:05 +08:00
|
|
|
rc = __dax_dev_pud_fault(dax_dev, vmf);
|
mm,fs,dax: change ->pmd_fault to ->huge_fault
Patch series "1G transparent hugepage support for device dax", v2.
The following series implements support for 1G trasparent hugepage on
x86 for device dax. The bulk of the code was written by Mathew Wilcox a
while back supporting transparent 1G hugepage for fs DAX. I have
forward ported the relevant bits to 4.10-rc. The current submission has
only the necessary code to support device DAX.
Comments from Dan Williams: So the motivation and intended user of this
functionality mirrors the motivation and users of 1GB page support in
hugetlbfs. Given expected capacities of persistent memory devices an
in-memory database may want to reduce tlb pressure beyond what they can
already achieve with 2MB mappings of a device-dax file. We have
customer feedback to that effect as Willy mentioned in his previous
version of these patches [1].
[1]: https://lkml.org/lkml/2016/1/31/52
Comments from Nilesh @ Oracle:
There are applications which have a process model; and if you assume
10,000 processes attempting to mmap all the 6TB memory available on a
server; we are looking at the following:
processes : 10,000
memory : 6TB
pte @ 4k page size: 8 bytes / 4K of memory * #processes = 6TB / 4k * 8 * 10000 = 1.5GB * 80000 = 120,000GB
pmd @ 2M page size: 120,000 / 512 = ~240GB
pud @ 1G page size: 240GB / 512 = ~480MB
As you can see with 2M pages, this system will use up an exorbitant
amount of DRAM to hold the page tables; but the 1G pages finally brings
it down to a reasonable level. Memory sizes will keep increasing; so
this number will keep increasing.
An argument can be made to convert the applications from process model
to thread model, but in the real world that may not be always practical.
Hopefully this helps explain the use case where this is valuable.
This patch (of 3):
In preparation for adding the ability to handle PUD pages, convert
vm_operations_struct.pmd_fault to vm_operations_struct.huge_fault. The
vm_fault structure is extended to include a union of the different page
table pointers that may be needed, and three flag bits are reserved to
indicate which type of pointer is in the union.
[ross.zwisler@linux.intel.com: remove unused function ext4_dax_huge_fault()]
Link: http://lkml.kernel.org/r/1485813172-7284-1-git-send-email-ross.zwisler@linux.intel.com
[dave.jiang@intel.com: clear PMD or PUD size flags when in fall through path]
Link: http://lkml.kernel.org/r/148589842696.5820.16078080610311444794.stgit@djiang5-desk3.ch.intel.com
Link: http://lkml.kernel.org/r/148545058784.17912.6353162518188733642.stgit@djiang5-desk3.ch.intel.com
Signed-off-by: Matthew Wilcox <mawilcox@microsoft.com>
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
Signed-off-by: Ross Zwisler <ross.zwisler@linux.intel.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Jan Kara <jack@suse.com>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Nilesh Choudhury <nilesh.choudhury@oracle.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Dave Jiang <dave.jiang@intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2017-02-25 06:56:59 +08:00
|
|
|
break;
|
|
|
|
default:
|
|
|
|
return VM_FAULT_FALLBACK;
|
|
|
|
}
|
2016-05-15 03:20:44 +08:00
|
|
|
rcu_read_unlock();
|
|
|
|
|
|
|
|
return rc;
|
|
|
|
}
|
|
|
|
|
2017-02-25 06:57:08 +08:00
|
|
|
static int dax_dev_fault(struct vm_fault *vmf)
|
|
|
|
{
|
|
|
|
return dax_dev_huge_fault(vmf, PE_SIZE_PTE);
|
|
|
|
}
|
|
|
|
|
2016-05-15 03:20:44 +08:00
|
|
|
static const struct vm_operations_struct dax_dev_vm_ops = {
|
|
|
|
.fault = dax_dev_fault,
|
2017-02-25 06:57:08 +08:00
|
|
|
.huge_fault = dax_dev_huge_fault,
|
2016-05-15 03:20:44 +08:00
|
|
|
};
|
|
|
|
|
2016-08-11 15:38:03 +08:00
|
|
|
static int dax_mmap(struct file *filp, struct vm_area_struct *vma)
|
2016-05-15 03:20:44 +08:00
|
|
|
{
|
|
|
|
struct dax_dev *dax_dev = filp->private_data;
|
|
|
|
int rc;
|
|
|
|
|
2016-08-11 15:41:51 +08:00
|
|
|
dev_dbg(&dax_dev->dev, "%s\n", __func__);
|
2016-05-15 03:20:44 +08:00
|
|
|
|
|
|
|
rc = check_vma(dax_dev, vma, __func__);
|
|
|
|
if (rc)
|
|
|
|
return rc;
|
|
|
|
|
|
|
|
vma->vm_ops = &dax_dev_vm_ops;
|
|
|
|
vma->vm_flags |= VM_MIXEDMAP | VM_HUGEPAGE;
|
|
|
|
return 0;
|
2016-08-07 23:23:56 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/* return an unmapped area aligned to the dax region specified alignment */
|
2016-08-11 15:38:03 +08:00
|
|
|
static unsigned long dax_get_unmapped_area(struct file *filp,
|
2016-08-07 23:23:56 +08:00
|
|
|
unsigned long addr, unsigned long len, unsigned long pgoff,
|
|
|
|
unsigned long flags)
|
|
|
|
{
|
|
|
|
unsigned long off, off_end, off_align, len_align, addr_align, align;
|
|
|
|
struct dax_dev *dax_dev = filp ? filp->private_data : NULL;
|
|
|
|
struct dax_region *dax_region;
|
|
|
|
|
|
|
|
if (!dax_dev || addr)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
dax_region = dax_dev->region;
|
|
|
|
align = dax_region->align;
|
|
|
|
off = pgoff << PAGE_SHIFT;
|
|
|
|
off_end = off + len;
|
|
|
|
off_align = round_up(off, align);
|
|
|
|
|
|
|
|
if ((off_end <= off_align) || ((off_end - off_align) < align))
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
len_align = len + align;
|
|
|
|
if ((off + len_align) < off)
|
|
|
|
goto out;
|
2016-05-15 03:20:44 +08:00
|
|
|
|
2016-08-07 23:23:56 +08:00
|
|
|
addr_align = current->mm->get_unmapped_area(filp, addr, len_align,
|
|
|
|
pgoff, flags);
|
|
|
|
if (!IS_ERR_VALUE(addr_align)) {
|
|
|
|
addr_align += (off - addr_align) & (align - 1);
|
|
|
|
return addr_align;
|
|
|
|
}
|
|
|
|
out:
|
|
|
|
return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags);
|
|
|
|
}
|
|
|
|
|
2016-08-11 15:38:03 +08:00
|
|
|
static int dax_open(struct inode *inode, struct file *filp)
|
2016-08-07 23:23:56 +08:00
|
|
|
{
|
2016-07-25 06:55:42 +08:00
|
|
|
struct dax_dev *dax_dev;
|
2016-08-07 23:23:56 +08:00
|
|
|
|
2016-07-25 06:55:42 +08:00
|
|
|
dax_dev = container_of(inode->i_cdev, struct dax_dev, cdev);
|
|
|
|
dev_dbg(&dax_dev->dev, "%s\n", __func__);
|
2016-07-25 12:55:45 +08:00
|
|
|
inode->i_mapping = dax_dev->inode->i_mapping;
|
|
|
|
inode->i_mapping->host = dax_dev->inode;
|
|
|
|
filp->f_mapping = inode->i_mapping;
|
2016-08-11 15:41:51 +08:00
|
|
|
filp->private_data = dax_dev;
|
|
|
|
inode->i_flags = S_DAX;
|
2016-08-07 23:23:56 +08:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
2016-05-15 03:20:44 +08:00
|
|
|
|
2016-08-11 15:38:03 +08:00
|
|
|
static int dax_release(struct inode *inode, struct file *filp)
|
2016-08-07 23:23:56 +08:00
|
|
|
{
|
|
|
|
struct dax_dev *dax_dev = filp->private_data;
|
|
|
|
|
2016-07-25 06:55:42 +08:00
|
|
|
dev_dbg(&dax_dev->dev, "%s\n", __func__);
|
2016-08-07 23:23:56 +08:00
|
|
|
return 0;
|
2016-05-15 03:20:44 +08:00
|
|
|
}
|
|
|
|
|
2016-05-19 00:15:08 +08:00
|
|
|
static const struct file_operations dax_fops = {
|
|
|
|
.llseek = noop_llseek,
|
|
|
|
.owner = THIS_MODULE,
|
2016-08-11 15:38:03 +08:00
|
|
|
.open = dax_open,
|
|
|
|
.release = dax_release,
|
|
|
|
.get_unmapped_area = dax_get_unmapped_area,
|
|
|
|
.mmap = dax_mmap,
|
2016-05-19 00:15:08 +08:00
|
|
|
};
|
|
|
|
|
2016-08-11 15:41:51 +08:00
|
|
|
static void dax_dev_release(struct device *dev)
|
2016-08-07 23:23:56 +08:00
|
|
|
{
|
2016-08-11 15:41:51 +08:00
|
|
|
struct dax_dev *dax_dev = to_dax_dev(dev);
|
2016-08-07 23:23:56 +08:00
|
|
|
struct dax_region *dax_region = dax_dev->region;
|
|
|
|
|
2016-08-11 15:41:51 +08:00
|
|
|
ida_simple_remove(&dax_region->ida, dax_dev->id);
|
|
|
|
ida_simple_remove(&dax_minor_ida, MINOR(dev->devt));
|
|
|
|
dax_region_put(dax_region);
|
2016-07-25 12:55:45 +08:00
|
|
|
iput(dax_dev->inode);
|
2016-08-11 15:41:51 +08:00
|
|
|
kfree(dax_dev);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void unregister_dax_dev(void *dev)
|
|
|
|
{
|
|
|
|
struct dax_dev *dax_dev = to_dax_dev(dev);
|
2016-07-25 06:55:42 +08:00
|
|
|
struct cdev *cdev = &dax_dev->cdev;
|
2016-08-11 15:41:51 +08:00
|
|
|
|
2016-08-07 23:23:56 +08:00
|
|
|
dev_dbg(dev, "%s\n", __func__);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Note, rcu is not protecting the liveness of dax_dev, rcu is
|
|
|
|
* ensuring that any fault handlers that might have seen
|
|
|
|
* dax_dev->alive == true, have completed. Any fault handlers
|
|
|
|
* that start after synchronize_rcu() has started will abort
|
|
|
|
* upon seeing dax_dev->alive == false.
|
|
|
|
*/
|
|
|
|
dax_dev->alive = false;
|
|
|
|
synchronize_rcu();
|
2016-08-05 07:53:50 +08:00
|
|
|
unmap_mapping_range(dax_dev->inode->i_mapping, 0, 0, 1);
|
2016-07-25 06:55:42 +08:00
|
|
|
cdev_del(cdev);
|
2016-08-07 23:23:56 +08:00
|
|
|
device_unregister(dev);
|
|
|
|
}
|
|
|
|
|
2016-07-20 08:51:40 +08:00
|
|
|
struct dax_dev *devm_create_dax_dev(struct dax_region *dax_region,
|
|
|
|
struct resource *res, int count)
|
2016-08-07 23:23:56 +08:00
|
|
|
{
|
|
|
|
struct device *parent = dax_region->dev;
|
|
|
|
struct dax_dev *dax_dev;
|
2016-07-20 07:17:58 +08:00
|
|
|
int rc = 0, minor, i;
|
2016-08-07 23:23:56 +08:00
|
|
|
struct device *dev;
|
2016-07-25 06:55:42 +08:00
|
|
|
struct cdev *cdev;
|
2016-08-07 23:23:56 +08:00
|
|
|
dev_t dev_t;
|
|
|
|
|
|
|
|
dax_dev = kzalloc(sizeof(*dax_dev) + sizeof(*res) * count, GFP_KERNEL);
|
|
|
|
if (!dax_dev)
|
2016-07-20 08:51:40 +08:00
|
|
|
return ERR_PTR(-ENOMEM);
|
2016-08-07 23:23:56 +08:00
|
|
|
|
2016-07-20 07:17:58 +08:00
|
|
|
for (i = 0; i < count; i++) {
|
|
|
|
if (!IS_ALIGNED(res[i].start, dax_region->align)
|
|
|
|
|| !IS_ALIGNED(resource_size(&res[i]),
|
|
|
|
dax_region->align)) {
|
|
|
|
rc = -EINVAL;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
dax_dev->res[i].start = res[i].start;
|
|
|
|
dax_dev->res[i].end = res[i].end;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (i < count)
|
|
|
|
goto err_id;
|
|
|
|
|
2016-08-07 23:23:56 +08:00
|
|
|
dax_dev->id = ida_simple_get(&dax_region->ida, 0, 0, GFP_KERNEL);
|
|
|
|
if (dax_dev->id < 0) {
|
|
|
|
rc = dax_dev->id;
|
|
|
|
goto err_id;
|
|
|
|
}
|
|
|
|
|
|
|
|
minor = ida_simple_get(&dax_minor_ida, 0, 0, GFP_KERNEL);
|
|
|
|
if (minor < 0) {
|
|
|
|
rc = minor;
|
|
|
|
goto err_minor;
|
|
|
|
}
|
|
|
|
|
2016-09-08 21:53:28 +08:00
|
|
|
dev_t = MKDEV(MAJOR(dax_devt), minor);
|
|
|
|
dev = &dax_dev->dev;
|
2016-07-25 12:55:45 +08:00
|
|
|
dax_dev->inode = dax_inode_get(&dax_dev->cdev, dev_t);
|
|
|
|
if (!dax_dev->inode) {
|
|
|
|
rc = -ENOMEM;
|
|
|
|
goto err_inode;
|
|
|
|
}
|
|
|
|
|
2016-07-25 06:55:42 +08:00
|
|
|
/* device_initialize() so cdev can reference kobj parent */
|
2016-08-11 15:41:51 +08:00
|
|
|
device_initialize(dev);
|
2016-07-25 06:55:42 +08:00
|
|
|
|
|
|
|
cdev = &dax_dev->cdev;
|
|
|
|
cdev_init(cdev, &dax_fops);
|
|
|
|
cdev->owner = parent->driver->owner;
|
|
|
|
cdev->kobj.parent = &dev->kobj;
|
|
|
|
rc = cdev_add(&dax_dev->cdev, dev_t, 1);
|
|
|
|
if (rc)
|
|
|
|
goto err_cdev;
|
|
|
|
|
|
|
|
/* from here on we're committed to teardown via dax_dev_release() */
|
|
|
|
dax_dev->num_resources = count;
|
|
|
|
dax_dev->alive = true;
|
|
|
|
dax_dev->region = dax_region;
|
|
|
|
kref_get(&dax_region->kref);
|
|
|
|
|
2016-08-11 15:41:51 +08:00
|
|
|
dev->devt = dev_t;
|
|
|
|
dev->class = dax_class;
|
|
|
|
dev->parent = parent;
|
|
|
|
dev->groups = dax_attribute_groups;
|
|
|
|
dev->release = dax_dev_release;
|
|
|
|
dev_set_name(dev, "dax%d.%d", dax_region->id, dax_dev->id);
|
|
|
|
rc = device_add(dev);
|
|
|
|
if (rc) {
|
|
|
|
put_device(dev);
|
2016-07-20 08:51:40 +08:00
|
|
|
return ERR_PTR(rc);
|
2016-08-11 15:41:51 +08:00
|
|
|
}
|
2016-08-07 23:23:56 +08:00
|
|
|
|
2016-07-20 08:51:40 +08:00
|
|
|
rc = devm_add_action_or_reset(dax_region->dev, unregister_dax_dev, dev);
|
|
|
|
if (rc)
|
|
|
|
return ERR_PTR(rc);
|
|
|
|
|
|
|
|
return dax_dev;
|
2016-08-07 23:23:56 +08:00
|
|
|
|
2016-07-25 06:55:42 +08:00
|
|
|
err_cdev:
|
2016-07-25 12:55:45 +08:00
|
|
|
iput(dax_dev->inode);
|
|
|
|
err_inode:
|
2016-07-25 06:55:42 +08:00
|
|
|
ida_simple_remove(&dax_minor_ida, minor);
|
2016-08-07 23:23:56 +08:00
|
|
|
err_minor:
|
|
|
|
ida_simple_remove(&dax_region->ida, dax_dev->id);
|
|
|
|
err_id:
|
2016-08-11 15:41:51 +08:00
|
|
|
kfree(dax_dev);
|
2016-08-07 23:23:56 +08:00
|
|
|
|
2016-07-20 08:51:40 +08:00
|
|
|
return ERR_PTR(rc);
|
2016-08-07 23:23:56 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(devm_create_dax_dev);
|
|
|
|
|
2016-05-19 00:15:08 +08:00
|
|
|
static int __init dax_init(void)
|
|
|
|
{
|
|
|
|
int rc;
|
|
|
|
|
2016-07-25 12:55:45 +08:00
|
|
|
rc = dax_inode_init();
|
|
|
|
if (rc)
|
2016-05-19 00:15:08 +08:00
|
|
|
return rc;
|
2016-07-25 12:55:45 +08:00
|
|
|
|
2016-07-25 06:55:42 +08:00
|
|
|
nr_dax = max(nr_dax, 256);
|
|
|
|
rc = alloc_chrdev_region(&dax_devt, 0, nr_dax, "dax");
|
|
|
|
if (rc)
|
2016-07-25 12:55:45 +08:00
|
|
|
goto err_chrdev;
|
2016-05-19 00:15:08 +08:00
|
|
|
|
|
|
|
dax_class = class_create(THIS_MODULE, "dax");
|
|
|
|
if (IS_ERR(dax_class)) {
|
2016-07-25 12:55:45 +08:00
|
|
|
rc = PTR_ERR(dax_class);
|
|
|
|
goto err_class;
|
2016-05-19 00:15:08 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
2016-07-25 12:55:45 +08:00
|
|
|
|
|
|
|
err_class:
|
|
|
|
unregister_chrdev_region(dax_devt, nr_dax);
|
|
|
|
err_chrdev:
|
|
|
|
dax_inode_exit();
|
|
|
|
return rc;
|
2016-05-19 00:15:08 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static void __exit dax_exit(void)
|
|
|
|
{
|
|
|
|
class_destroy(dax_class);
|
2016-07-25 06:55:42 +08:00
|
|
|
unregister_chrdev_region(dax_devt, nr_dax);
|
2016-05-19 00:15:08 +08:00
|
|
|
ida_destroy(&dax_minor_ida);
|
2016-07-25 12:55:45 +08:00
|
|
|
dax_inode_exit();
|
2016-05-19 00:15:08 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
MODULE_AUTHOR("Intel Corporation");
|
|
|
|
MODULE_LICENSE("GPL v2");
|
|
|
|
subsys_initcall(dax_init);
|
|
|
|
module_exit(dax_exit);
|