Can any lock block the kernel

intro

As a programmer, I spend my coding time exclusively in user space. It's easy to get the impression that if multiple threads need to access a shared resource then a lock should be used to synchronize them and the thread that couldn't acquired that lock should be suspended.
But if you think it a little more deeply in kernel space, you may find out that acquirement and suspend are very heavy logic: how could we guarantee the "acquirement and suspend" itself is atomic. If we need another lock mechanism, isn't this an infinite recursion?

spin_lock: the buck stops here

A typical lock in kernel is mutex_lock which we can check closely. A salient character of its implementation is the ubiquitous spin_lock. Note that both mutex_lock and mutex_unlock depend on, acquiring and releasing, the same lock->wait_lock.

/// @ref: https://github.com/torvalds/linux/blob/08dbfad3f5040f5bdb6c529da20d6d4e81fefd72/kernel/locking/mutex.c#L660
/*
 * Lock a mutex (possibly interruptible), slowpath:
 */
static __always_inline int __sched
__mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclass,
		    struct lockdep_map *nest_lock, unsigned long ip,
		    struct ww_acquire_ctx *ww_ctx, const bool use_ww_ctx)
	__cond_acquires(0, lock)
{
///...
	raw_spin_lock_irqsave(&lock->wait_lock, flags);
	/*
	 * After waiting to acquire the wait_lock, try again.
	 */
///...
	__set_current_state(TASK_RUNNING);
	raw_spin_unlock(&current->blocked_lock);
///...
}
/// @ref: https://github.com/torvalds/linux/blob/08dbfad3f5040f5bdb6c529da20d6d4e81fefd72/kernel/locking/mutex.c#L981
/*
 * Release the lock, slowpath:
 */
static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip)
	__releases(lock)
{
///...
	raw_spin_lock(&current->blocked_lock);
///...
	raw_spin_unlock(&current->blocked_lock);
	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
	if (next) {
		wake_up_process(next);
		put_task_struct(next);
	}
}

What is a spin_lock

In essence, the implementation relies on some basic cpu instruction to conquer the concurrence, like cpxchg or lock inc etc.

The following is a simple emulation of the implementation. we can see it can guarantee synchronize concurrency against 10 threads.

tsecer@harry: cd D:\spare\202608\
> ^C
tsecer@harry: cd /mnt/
c/    d/    wsl/  wslg/
tsecer@harry: cd /mnt/d/spare/202608/
tsecer@harry: cat spinlock_in_userspace.cpp
#include <thread>
#include <unistd.h>
#include <assert.h>
#include <stdio.h>
#include <chrono>

// all threads try to modify this var concurrently
int contend_var;
// indicating whether worker threads can start the loop to modify the contend variable
int loop_ready;
// thread count to modify the contend_var
constexpr int thread_count = 10;
// modification count in thread
constexpr int loop_count = 1000 * 1000;
// indicating that the lock is untaken
constexpr int unlocked_state = 0;
// indicating that the lock is acquired
constexpr int locked_state = 1;
// some functons require a non-const pointer
int mutable_unlocked_state = unlocked_state;

struct Timer
{
    using Clock = std::chrono::high_resolution_clock;
    Clock::time_point start;

    void begin()
    {
        start = Clock::now();
    }

    void end()
    {
        nano_seconds =  std::chrono::duration_cast<std::chrono::nanoseconds>(Clock::now() - start).count();
    }
    long nano_seconds;
} gTimer;

void worker(int *plock)
{
    // wait for loop_ready so threads can execute at almost the same time
    while (!loop_ready);
    int unlocked = 0;
    int loop_var = loop_count;
    while (loop_var-- > 0)
    {
        if (plock)
        {
            // waiting for the lock turning into unlocked and acquire it
            while (!__atomic_compare_exchange_n(plock, &unlocked, locked_state, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST))
            {
                __atomic_store(&unlocked, &mutable_unlocked_state, __ATOMIC_SEQ_CST);
            }
        }
        contend_var += 1;
        if (plock)
        {
            // We must be holding the lock
            assert(*plock == locked_state);
            __atomic_store(plock, &unlocked, __ATOMIC_SEQ_CST);
        }
    }
    gTimer.end();
}

int main(int argc, const char *argv[])
{
    std::thread td[10];
    int lock = 0;
    for (auto &t : td)
    {
        // if argcount larger than 1, lock is used
        t = std::thread(worker, argc > 1 ? &lock : nullptr);
    }
    // wait for all threads being running
    sleep(1);
    // Notify all threads to loop the contend_var
    loop_ready = 1;
    // assume all loops in threads start now
    gTimer.begin();
    for (auto &t : td)
    {
        t.join();
    }
    printf("contend_var %d modify count %d nano_seconds %ld \n",
            contend_var, thread_count * loop_count, gTimer.nano_seconds);
    return 0;
}
tsecer@harry: g++ spinlock_in_userspace.cpp -lpthread
tsecer@harry: ./a.out
contend_var 2023164 modify count 10000000 nano_seconds 33822800
tsecer@harry: ./a.out lock
contend_var 10000000 modify count 10000000 nano_seconds 7313905500
tsecer@harry:

Can kernel really block

What is kernel doing when it idles? Can kernel really have a rest?

idle

For 386, the cpu will keep executing the nop instruction.

/// @ref: https://github.com/torvalds/linux/blob/08dbfad3f5040f5bdb6c529da20d6d4e81fefd72/arch/x86/um/asm/processor.h#L24
/* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */
static __always_inline void rep_nop(void)
{
	__asm__ __volatile__("rep;nop": : :"memory");
}

static __always_inline void cpu_relax(void)
{
	if (time_travel_mode == TT_MODE_INFCPU ||
	    time_travel_mode == TT_MODE_EXTERNAL)
		time_travel_ndelay(1);
	else
		rep_nop();
}

panic

The opposition of idle may be panic. But even in panic, the kernel is still hogging the CPU, there is no real at ease for kernel.

/// @ref: https://github.com/torvalds/linux/blob/08dbfad3f5040f5bdb6c529da20d6d4e81fefd72/kernel/panic.c#L767
/**
 * vpanic - halt the system
 * @fmt: The text string to print
 * @args: Arguments for the format string
 *
 * Display a message, then perform cleanups. This function never returns.
 */
void vpanic(const char *fmt, va_list args)
{
///...
	/*
	 * The final messages may not have been printed if in a context that
	 * defers printing (such as NMI) and irq_work is not available.
	 * Explicitly flush the kernel log buffer one last time.
	 */
	console_flush_on_panic(CONSOLE_FLUSH_PENDING);
	nbcon_atomic_flush_unsafe();

	local_irq_enable();
	for (i = 0; ; i += PANIC_TIMER_STEP) {
		touch_softlockup_watchdog();
		if (i >= i_next) {
			i += panic_blink(state ^= 1);
			i_next = i + 3600 / PANIC_BLINK_SPD;
		}
		mdelay(PANIC_TIMER_STEP);
	}
}

outro

The spin lock implies some constraints on kernel, such as "kernel threads never page fault ". Otherwise, an execution path may take the spin_lock, enter excetion handler, take the very same lock, which could result in a dead lock.