matthieu/frr
1
0
Fork 0
forked from Mirror/frr
Code Pull requests Activity

lib/seqlock: avoid syscalls in no-waiter cases

Browse source 
When we have no contention on the seqlock, we shouldn't incur the cost
of syscalls.

Signed-off-by: David Lamparter <equinox@opensourcerouting.org>
This commit is contained in:
David Lamparter 2019-06-19 12:52:38 +02:00
parent 5e4f10b1e0
commit 6046b690b5
4 changed files with 69 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

56
lib/seqlock.c
View file

@ -103,16 +103,23 @@ void seqlock_wait(struct seqlock *sqlo, seqlock_val_t val)
seqlock_assert_valid(val);
wait_prep(sqlo);
while (1) {
cur = atomic_load_explicit(&sqlo->pos, memory_order_acquire);
if (!(cur & 1))
break;
cal = cur - val - 1;
cur = atomic_load_explicit(&sqlo->pos, memory_order_relaxed);
while (cur & SEQLOCK_HELD) {
cal = SEQLOCK_VAL(cur) - val - 1;
assert(cal < 0x40000000 || cal > 0xc0000000);
if (cal < 0x80000000)
break;
wait_once(sqlo, cur);
if ((cur & SEQLOCK_WAITERS)
|| atomic_compare_exchange_weak_explicit(
&sqlo->pos, &cur, cur | SEQLOCK_WAITERS,
memory_order_relaxed, memory_order_relaxed)) {
wait_once(sqlo, cur | SEQLOCK_WAITERS);
cur = atomic_load_explicit(&sqlo->pos,
memory_order_relaxed);
}
/* else: we failed to swap in cur because it just changed */
}
wait_done(sqlo);
}
@ -123,26 +130,32 @@ bool seqlock_check(struct seqlock *sqlo, seqlock_val_t val)
seqlock_assert_valid(val);
cur = atomic_load_explicit(&sqlo->pos, memory_order_acquire);
if (!(cur & 1))
cur = atomic_load_explicit(&sqlo->pos, memory_order_relaxed);
if (!(cur & SEQLOCK_HELD))
return 1;
cur -= val;
cur = SEQLOCK_VAL(cur) - val - 1;
assert(cur < 0x40000000 || cur > 0xc0000000);
return cur < 0x80000000;
}
void seqlock_acquire_val(struct seqlock *sqlo, seqlock_val_t val)
{
seqlock_val_t prev;
seqlock_assert_valid(val);
atomic_store_explicit(&sqlo->pos, val, memory_order_release);
wait_poke(sqlo);
prev = atomic_exchange_explicit(&sqlo->pos, val, memory_order_relaxed);
if (prev & SEQLOCK_WAITERS)
wait_poke(sqlo);
}
void seqlock_release(struct seqlock *sqlo)
{
atomic_store_explicit(&sqlo->pos, 0, memory_order_release);
wait_poke(sqlo);
seqlock_val_t prev;
prev = atomic_exchange_explicit(&sqlo->pos, 0, memory_order_relaxed);
if (prev & SEQLOCK_WAITERS)
wait_poke(sqlo);
}
void seqlock_init(struct seqlock *sqlo)
@ -154,14 +167,23 @@ void seqlock_init(struct seqlock *sqlo)
seqlock_val_t seqlock_cur(struct seqlock *sqlo)
{
return atomic_load_explicit(&sqlo->pos, memory_order_acquire);
return SEQLOCK_VAL(atomic_load_explicit(&sqlo->pos,
memory_order_relaxed));
}
seqlock_val_t seqlock_bump(struct seqlock *sqlo)
{
seqlock_val_t val;
seqlock_val_t val, cur;
val = atomic_fetch_add_explicit(&sqlo->pos, 2, memory_order_release);
wait_poke(sqlo);
cur = atomic_load_explicit(&sqlo->pos, memory_order_relaxed);
seqlock_assert_valid(cur);
do {
val = SEQLOCK_VAL(cur) + SEQLOCK_INCR;
} while (!atomic_compare_exchange_weak_explicit(&sqlo->pos, &cur, val,
memory_order_relaxed, memory_order_relaxed));
if (cur & SEQLOCK_WAITERS)
wait_poke(sqlo);
return val;
}

10
lib/seqlock.h
View file

@ -54,12 +54,18 @@
*/
/* use sequentially increasing "ticket numbers". lowest bit will always
* be 1 to have a 'cleared' indication (i.e., counts 1,3,5,7,etc. )
* be 1 to have a 'cleared' indication (i.e., counts 1,5,9,13,etc. )
* 2nd lowest bit is used to indicate we have waiters.
*/
typedef _Atomic uint32_t seqlock_ctr_t;
typedef uint32_t seqlock_val_t;
#define seqlock_assert_valid(val) assert(val & 1)
#define seqlock_assert_valid(val) assert((val) & SEQLOCK_HELD)
#define SEQLOCK_HELD (1U << 0)
#define SEQLOCK_WAITERS (1U << 1)
#define SEQLOCK_VAL(n) ((n) & ~SEQLOCK_WAITERS)
#define SEQLOCK_STARTVAL 1U
#define SEQLOCK_INCR 4U
struct seqlock {
/* always used */

10
tests/lib/test_atomlist.c
View file

@ -253,7 +253,7 @@ static void *thr1func(void *arg)
struct testrun *tr;
for (tr = runs; tr; tr = tr->next) {
sv = seqlock_bump(&p->sqlo);
sv = seqlock_bump(&p->sqlo) - SEQLOCK_INCR;
seqlock_wait(&sqlo, sv);
tr->func(offset);
@ -288,14 +288,14 @@ static void run_tr(struct testrun *tr)
size_t c = 0, s = 0, n = 0;
struct item *item, *prev, dummy;
printf("[%02u] %35s %s\n", seqlock_cur(&sqlo) >> 1, "", desc);
printf("[%02u] %35s %s\n", seqlock_cur(&sqlo) >> 2, "", desc);
fflush(stdout);
if (tr->prefill != NOCLEAR)
clear_list(tr->prefill);
monotime(&tv);
sv = seqlock_bump(&sqlo);
sv = seqlock_bump(&sqlo) - SEQLOCK_INCR;
for (size_t i = 0; i < NTHREADS; i++) {
seqlock_wait(&thr[i].sqlo, seqlock_cur(&sqlo));
s += thr[i].counter;
@ -325,7 +325,7 @@ static void run_tr(struct testrun *tr)
assert(c == alist_count(&ahead));
}
printf("\033[1A[%02u] %9"PRId64"us c=%5zu s=%5zu n=%5zu %s\n",
sv >> 1, delta, c, s, n, desc);
sv >> 2, delta, c, s, n, desc);
}
#ifdef BASIC_TESTS
@ -381,7 +381,7 @@ int main(int argc, char **argv)
basic_tests();
seqlock_init(&sqlo);
seqlock_acquire_val(&sqlo, 1);
seqlock_acquire_val(&sqlo, SEQLOCK_STARTVAL);
for (i = 0; i < NTHREADS; i++) {
seqlock_init(&thr[i].sqlo);

34
tests/lib/test_seqlock.c
View file

@ -66,20 +66,20 @@ static void *thr1func(void *arg)
seqlock_wait(&sqlo, 1);
writestr("thr1 @1\n");
seqlock_wait(&sqlo, 3);
writestr("thr1 @3\n");
seqlock_wait(&sqlo, 5);
writestr("thr1 @5\n");
seqlock_wait(&sqlo, 7);
writestr("thr1 @7\n");
seqlock_wait(&sqlo, 9);
writestr("thr1 @9\n");
seqlock_wait(&sqlo, 11);
writestr("thr1 @11\n");
seqlock_wait(&sqlo, 13);
writestr("thr1 @13\n");
seqlock_wait(&sqlo, 17);
writestr("thr1 @17\n");
seqlock_wait(&sqlo, 21);
writestr("thr1 @21\n");
return NULL;
}
@ -95,11 +95,11 @@ int main(int argc, char **argv)
assert(seqlock_cur(&sqlo) == 1);
assert(seqlock_bump(&sqlo) == 1);
assert(seqlock_cur(&sqlo) == 3);
assert(seqlock_bump(&sqlo) == 3);
assert(seqlock_cur(&sqlo) == 5);
assert(seqlock_bump(&sqlo) == 5);
assert(seqlock_bump(&sqlo) == 7);
assert(seqlock_cur(&sqlo) == 9);
assert(seqlock_bump(&sqlo) == 9);
assert(seqlock_bump(&sqlo) == 13);
assert(seqlock_cur(&sqlo) == 17);
assert(seqlock_held(&sqlo));
seqlock_release(&sqlo);
@ -108,16 +108,16 @@ int main(int argc, char **argv)
pthread_create(&thr1, NULL, thr1func, NULL);
sleep(1);
writestr("main @3\n");
seqlock_acquire_val(&sqlo, 3);
writestr("main @5\n");
seqlock_acquire_val(&sqlo, 5);
sleep(2);
writestr("main @5\n");
writestr("main @9\n");
seqlock_bump(&sqlo);
sleep(1);
writestr("main @9\n");
seqlock_acquire_val(&sqlo, 9);
writestr("main @17\n");
seqlock_acquire_val(&sqlo, 17);
sleep(1);
writestr("main @release\n");