2020-04-22 01:04:05 +08:00
|
|
|
.. SPDX-License-Identifier: GPL-2.0
|
|
|
|
|
|
|
|
=================================================
|
|
|
|
Using RCU hlist_nulls to protect list and objects
|
|
|
|
=================================================
|
|
|
|
|
|
|
|
This section describes how to use hlist_nulls to
|
|
|
|
protect read-mostly linked lists and
|
|
|
|
objects using SLAB_TYPESAFE_BY_RCU allocations.
|
|
|
|
|
2022-03-30 22:41:00 +08:00
|
|
|
Please read the basics in listRCU.rst.
|
2020-04-22 01:04:05 +08:00
|
|
|
|
2020-04-22 01:04:11 +08:00
|
|
|
Using 'nulls'
|
|
|
|
=============
|
|
|
|
|
2020-04-22 01:04:05 +08:00
|
|
|
Using special makers (called 'nulls') is a convenient way
|
2022-11-05 05:03:15 +08:00
|
|
|
to solve following problem.
|
2020-04-22 01:04:05 +08:00
|
|
|
|
2022-11-05 05:03:15 +08:00
|
|
|
Without 'nulls', a typical RCU linked list managing objects which are
|
|
|
|
allocated with SLAB_TYPESAFE_BY_RCU kmem_cache can use the following
|
2023-06-17 07:36:24 +08:00
|
|
|
algorithms. Following examples assume 'obj' is a pointer to such
|
|
|
|
objects, which is having below type.
|
|
|
|
|
|
|
|
::
|
|
|
|
|
|
|
|
struct object {
|
|
|
|
struct hlist_node obj_node;
|
|
|
|
atomic_t refcnt;
|
|
|
|
unsigned int key;
|
|
|
|
};
|
2020-04-22 01:04:05 +08:00
|
|
|
|
2022-11-05 05:03:15 +08:00
|
|
|
1) Lookup algorithm
|
|
|
|
-------------------
|
2020-04-22 01:04:05 +08:00
|
|
|
|
|
|
|
::
|
|
|
|
|
|
|
|
begin:
|
2023-06-14 02:24:31 +08:00
|
|
|
rcu_read_lock();
|
2020-04-22 01:04:05 +08:00
|
|
|
obj = lockless_lookup(key);
|
|
|
|
if (obj) {
|
2023-06-13 08:57:01 +08:00
|
|
|
if (!try_get_ref(obj)) { // might fail for free objects
|
|
|
|
rcu_read_unlock();
|
2020-04-22 01:04:05 +08:00
|
|
|
goto begin;
|
2023-06-13 08:57:01 +08:00
|
|
|
}
|
2020-04-22 01:04:05 +08:00
|
|
|
/*
|
|
|
|
* Because a writer could delete object, and a writer could
|
|
|
|
* reuse these object before the RCU grace period, we
|
|
|
|
* must check key after getting the reference on object
|
|
|
|
*/
|
|
|
|
if (obj->key != key) { // not the object we expected
|
|
|
|
put_ref(obj);
|
2022-11-05 05:03:15 +08:00
|
|
|
rcu_read_unlock();
|
2020-04-22 01:04:05 +08:00
|
|
|
goto begin;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
|
|
|
Beware that lockless_lookup(key) cannot use traditional hlist_for_each_entry_rcu()
|
|
|
|
but a version with an additional memory barrier (smp_rmb())
|
|
|
|
|
|
|
|
::
|
|
|
|
|
|
|
|
lockless_lookup(key)
|
|
|
|
{
|
|
|
|
struct hlist_node *node, *next;
|
|
|
|
for (pos = rcu_dereference((head)->first);
|
2022-11-05 05:03:15 +08:00
|
|
|
pos && ({ next = pos->next; smp_rmb(); prefetch(next); 1; }) &&
|
2023-06-17 07:36:25 +08:00
|
|
|
({ obj = hlist_entry(pos, typeof(*obj), obj_node); 1; });
|
2022-11-05 05:03:15 +08:00
|
|
|
pos = rcu_dereference(next))
|
2020-04-22 01:04:05 +08:00
|
|
|
if (obj->key == key)
|
|
|
|
return obj;
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
And note the traditional hlist_for_each_entry_rcu() misses this smp_rmb()::
|
|
|
|
|
|
|
|
struct hlist_node *node;
|
|
|
|
for (pos = rcu_dereference((head)->first);
|
2022-11-05 05:03:15 +08:00
|
|
|
pos && ({ prefetch(pos->next); 1; }) &&
|
2023-06-17 07:36:25 +08:00
|
|
|
({ obj = hlist_entry(pos, typeof(*obj), obj_node); 1; });
|
2022-11-05 05:03:15 +08:00
|
|
|
pos = rcu_dereference(pos->next))
|
2023-06-14 02:24:31 +08:00
|
|
|
if (obj->key == key)
|
|
|
|
return obj;
|
2020-04-22 01:04:05 +08:00
|
|
|
return NULL;
|
|
|
|
|
|
|
|
Quoting Corey Minyard::
|
|
|
|
|
|
|
|
"If the object is moved from one list to another list in-between the
|
|
|
|
time the hash is calculated and the next field is accessed, and the
|
|
|
|
object has moved to the end of a new list, the traversal will not
|
|
|
|
complete properly on the list it should have, since the object will
|
|
|
|
be on the end of the new list and there's not a way to tell it's on a
|
|
|
|
new list and restart the list traversal. I think that this can be
|
|
|
|
solved by pre-fetching the "next" field (with proper barriers) before
|
|
|
|
checking the key."
|
|
|
|
|
2022-11-05 05:03:15 +08:00
|
|
|
2) Insertion algorithm
|
|
|
|
----------------------
|
2020-04-22 01:04:05 +08:00
|
|
|
|
2023-06-17 07:36:25 +08:00
|
|
|
We need to make sure a reader cannot read the new 'obj->obj_node.next' value
|
2022-11-05 05:03:15 +08:00
|
|
|
and previous value of 'obj->key'. Otherwise, an item could be deleted
|
2020-04-22 01:04:05 +08:00
|
|
|
from a chain, and inserted into another chain. If new chain was empty
|
2022-11-05 05:03:15 +08:00
|
|
|
before the move, 'next' pointer is NULL, and lockless reader can not
|
|
|
|
detect the fact that it missed following items in original chain.
|
2020-04-22 01:04:05 +08:00
|
|
|
|
|
|
|
::
|
|
|
|
|
|
|
|
/*
|
2022-11-05 05:03:15 +08:00
|
|
|
* Please note that new inserts are done at the head of list,
|
|
|
|
* not in the middle or end.
|
|
|
|
*/
|
2020-04-22 01:04:05 +08:00
|
|
|
obj = kmem_cache_alloc(...);
|
|
|
|
lock_chain(); // typically a spin_lock()
|
|
|
|
obj->key = key;
|
2022-11-05 05:03:15 +08:00
|
|
|
atomic_set_release(&obj->refcnt, 1); // key before refcnt
|
2020-04-22 01:04:05 +08:00
|
|
|
hlist_add_head_rcu(&obj->obj_node, list);
|
|
|
|
unlock_chain(); // typically a spin_unlock()
|
|
|
|
|
|
|
|
|
2022-11-05 05:03:15 +08:00
|
|
|
3) Removal algorithm
|
|
|
|
--------------------
|
|
|
|
|
2020-04-22 01:04:05 +08:00
|
|
|
Nothing special here, we can use a standard RCU hlist deletion.
|
|
|
|
But thanks to SLAB_TYPESAFE_BY_RCU, beware a deleted object can be reused
|
|
|
|
very very fast (before the end of RCU grace period)
|
|
|
|
|
|
|
|
::
|
|
|
|
|
|
|
|
if (put_last_reference_on(obj) {
|
|
|
|
lock_chain(); // typically a spin_lock()
|
|
|
|
hlist_del_init_rcu(&obj->obj_node);
|
|
|
|
unlock_chain(); // typically a spin_unlock()
|
|
|
|
kmem_cache_free(cachep, obj);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
--------------------------------------------------------------------------
|
|
|
|
|
2020-04-22 01:04:11 +08:00
|
|
|
Avoiding extra smp_rmb()
|
|
|
|
========================
|
|
|
|
|
2023-06-17 07:36:26 +08:00
|
|
|
With hlist_nulls we can avoid extra smp_rmb() in lockless_lookup().
|
2020-04-22 01:04:05 +08:00
|
|
|
|
|
|
|
For example, if we choose to store the slot number as the 'nulls'
|
|
|
|
end-of-list marker for each slot of the hash table, we can detect
|
|
|
|
a race (some writer did a delete and/or a move of an object
|
|
|
|
to another chain) checking the final 'nulls' value if
|
|
|
|
the lookup met the end of chain. If final 'nulls' value
|
|
|
|
is not the slot number, then we must restart the lookup at
|
|
|
|
the beginning. If the object was moved to the same chain,
|
2022-11-05 05:03:15 +08:00
|
|
|
then the reader doesn't care: It might occasionally
|
2020-04-22 01:04:05 +08:00
|
|
|
scan the list again without harm.
|
|
|
|
|
2023-06-17 07:36:24 +08:00
|
|
|
Note that using hlist_nulls means the type of 'obj_node' field of
|
|
|
|
'struct object' becomes 'struct hlist_nulls_node'.
|
|
|
|
|
2020-04-22 01:04:05 +08:00
|
|
|
|
2022-11-05 05:03:15 +08:00
|
|
|
1) lookup algorithm
|
|
|
|
-------------------
|
2020-04-22 01:04:05 +08:00
|
|
|
|
|
|
|
::
|
|
|
|
|
|
|
|
head = &table[slot];
|
|
|
|
begin:
|
2022-11-05 05:03:15 +08:00
|
|
|
rcu_read_lock();
|
2023-06-17 07:36:25 +08:00
|
|
|
hlist_nulls_for_each_entry_rcu(obj, node, head, obj_node) {
|
2020-04-22 01:04:05 +08:00
|
|
|
if (obj->key == key) {
|
2022-11-05 05:03:15 +08:00
|
|
|
if (!try_get_ref(obj)) { // might fail for free objects
|
|
|
|
rcu_read_unlock();
|
2020-04-22 01:04:05 +08:00
|
|
|
goto begin;
|
2022-11-05 05:03:15 +08:00
|
|
|
}
|
2020-04-22 01:04:05 +08:00
|
|
|
if (obj->key != key) { // not the object we expected
|
|
|
|
put_ref(obj);
|
2022-11-05 05:03:15 +08:00
|
|
|
rcu_read_unlock();
|
2020-04-22 01:04:05 +08:00
|
|
|
goto begin;
|
|
|
|
}
|
2022-11-05 05:03:15 +08:00
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// If the nulls value we got at the end of this lookup is
|
|
|
|
// not the expected one, we must restart lookup.
|
|
|
|
// We probably met an item that was moved to another chain.
|
|
|
|
if (get_nulls_value(node) != slot) {
|
|
|
|
put_ref(obj);
|
|
|
|
rcu_read_unlock();
|
|
|
|
goto begin;
|
2020-04-22 01:04:05 +08:00
|
|
|
}
|
|
|
|
obj = NULL;
|
|
|
|
|
|
|
|
out:
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
2022-11-05 05:03:15 +08:00
|
|
|
2) Insert algorithm
|
|
|
|
-------------------
|
2020-04-22 01:04:05 +08:00
|
|
|
|
2023-06-17 07:36:26 +08:00
|
|
|
Same to the above one, but uses hlist_nulls_add_head_rcu() instead of
|
|
|
|
hlist_add_head_rcu().
|
|
|
|
|
2020-04-22 01:04:05 +08:00
|
|
|
::
|
|
|
|
|
|
|
|
/*
|
2022-11-05 05:03:15 +08:00
|
|
|
* Please note that new inserts are done at the head of list,
|
|
|
|
* not in the middle or end.
|
|
|
|
*/
|
2020-04-22 01:04:05 +08:00
|
|
|
obj = kmem_cache_alloc(cachep);
|
|
|
|
lock_chain(); // typically a spin_lock()
|
|
|
|
obj->key = key;
|
2022-11-05 05:03:15 +08:00
|
|
|
atomic_set_release(&obj->refcnt, 1); // key before refcnt
|
2020-04-22 01:04:05 +08:00
|
|
|
/*
|
2022-11-05 05:03:15 +08:00
|
|
|
* insert obj in RCU way (readers might be traversing chain)
|
|
|
|
*/
|
2020-04-22 01:04:05 +08:00
|
|
|
hlist_nulls_add_head_rcu(&obj->obj_node, list);
|
|
|
|
unlock_chain(); // typically a spin_unlock()
|