futex同步机制分析之二glibc的实现

在前方中提到过，经常使用的几个线程（进程）同步机制，其实都是调用futex来实现的，下面就从glibc2.8中把源码找出来对一下，看看到底是不是确实是如此，同时，对代码进行一些简单的分析。 
每个细节的实现其实都可以展开来讲成独立的一篇，这里只是介绍同步机制的来龙去脉和实现，就不再对具体的细节进行展开，回头如果有时间，再对具体的每个技术进行专门的分析。 

一、信号灯的实现

int __new_sem_wait (sem_t *sem)
{
  /* We need to check whether we need to act upon a cancellation request here
     because POSIX specifies that cancellation points "shall occur" in
     sem_wait and sem_timedwait, which also means that they need to check
     this regardless whether they block or not (unlike "may occur"
     functions).  See the POSIX Rationale for this requirement: Section
     "Thread Cancellation Overview" [1] and austin group issue #1076 [2]
     for thoughs on why this may be a suboptimal design.
 
     [1] http://pubs.opengroup.org/onlinepubs/9699919799/xrat/V4_xsh_chap02.html
     [2] http://austingroupbugs.net/view.php?id=1076 for thoughts on why this
   \*/
  __pthread_testcancel ();
 
  //处理sem_wait
  if (__new_sem_wait_fast ((struct new_sem *) sem, 0) == 0)
    return 0;
  else
    return __new_sem_wait_slow((struct new_sem *) sem, NULL);
}
versioned_symbol (libpthread, \__new_sem_wait, sem_wait, GLIBC_2_1);
 
//处理新老两种库的sem的生成
#if SHLIB_COMPAT (libpthread, GLIBC_2_0, GLIBC_2_1)
int
attribute_compat_text_section
\__old_sem_wait (sem_t \*sem)
{
  int \*futex = (int \*) sem;
  int err;
 
  //循环判断状态
  do
    {
      if (atomic_decrement_if_positive (futex) > 0)
    return 0;
 
      /* Enable asynchronous cancellation.  Required by the standard.  \*/
      int oldtype = \__pthread_enable_asynccancel ();
 
      /* Always assume the semaphore is shared.  \*/
      err = lll_futex_wait (futex, 0, LLL_SHARED);
 
      /* Disable asynchronous cancellation.  \*/
      \__pthread_disable_asynccancel (oldtype);
    }
  while (err == 0 || err == -EWOULDBLOCK);
 
  \__set_errno (-err);
  return -1;
}
 
compat_symbol (libpthread, \__old_sem_wait, sem_wait, GLIBC_2_0);
#endif

在新的2.8及以上版本中，兼容了老的sem和futex，又重新生成了一个新的sem，这个数据结构体比原来一个单纯的整型多了不少，看一下代码： 

/* Semaphore variable structure.  */
struct new_sem
{
#if \__HAVE_64B_ATOMICS
  /* The data field holds both value (in the least-significant 32 bits) and
     nwaiters.  \*/
# if \__BYTE_ORDER == \__LITTLE_ENDIAN
#  define SEM_VALUE_OFFSET 0
# elif \__BYTE_ORDER == \__BIG_ENDIAN
#  define SEM_VALUE_OFFSET 1
# else
# error Unsupported byte order.
# endif
# define SEM_NWAITERS_SHIFT 32
# define SEM_VALUE_MASK (~(unsigned int)0)
  uint64_t data;
  int private;
  int pad;
#else
# define SEM_VALUE_SHIFT 1
# define SEM_NWAITERS_MASK ((unsigned int)1)
  unsigned int value;
  int private;
  int pad;
  unsigned int nwaiters;
#endif
};
 
struct old_sem
{
  unsigned int value;
};

不过怎么说呢，原理仍然是一样的。不过需要注意的是，网上很多的资料仍然是对老的进行的分析，所以在看代码时要与时俱进。 

二、mutex实现

再看一下mutex的LOCK实现源码： 

static int
__pthread_mutex_lock_full (pthread_mutex_t *mutex)
{
/* We cannot acquire the mutex nor has its owner died.  Thus, try
      to block using futexes.  Set FUTEX_WAITERS if necessary so that
      other threads are aware that there are potentially threads
      blocked on the futex.  Restart if oldval changed in the
      meantime.  \*/
   if ((oldval & FUTEX_WAITERS) == 0)
     {
       //前文提到过的CAS
       if (atomic_compare_and_exchange_bool_acq (&mutex->\__data.\__lock,
             oldval | FUTEX_WAITERS,
             oldval)
     != 0)
   {
     oldval = mutex->\__data.\__lock;
     continue;
   }
       oldval |= FUTEX_WAITERS;
     }
 
   /* It is now possible that we share the FUTEX_WAITERS flag with
      another thread; therefore, update assume_other_futex_waiters so
      that we do not forget about this when handling other cases
      above and thus do not cause lost wake-ups.  \*/
   assume_other_futex_waiters |= FUTEX_WAITERS;
 
   /* Block using the futex and reload current lock value.  \*/
   lll_futex_wait (&mutex->\__data.\__lock, oldval,
       PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
   oldval = mutex->\__data.\__lock;//前方提到参变量
 }
 
     /* We have acquired the mutex; check if it is still consistent.  */
     //此处即为一致性检查
     if (\__builtin_expect (mutex->\__data.\__owner
         == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 {
   /* This mutex is now not recoverable.  \*/
   mutex->__data.__count = 0;
   int private = PTHREAD_ROBUST_MUTEX_PSHARED (mutex);
   lll_unlock (mutex->__data.__lock, private);
   /* FIXME This violates the mutex destruction requirements.  See
      \__pthread_mutex_unlock_full.  \*/
   THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
   return ENOTRECOVERABLE;
 }
}

三、条件变量的实现

glibc中条件变量的实现如下： 

#define __lll_cond_lock(futex, private)                                 \
  ((void)                                                               \
   ({                                                                   \
     int \*\__futex = (futex);                                            \
     if (__glibc_unlikely (atomic_exchange_acq (\__futex, 2) != 0))      \
       __lll_lock_wait (\__futex, private);                              \
   }))
#define lll_cond_lock(futex, private) __lll_cond_lock (&(futex), private)
 
static void __attribute__ ((unused))
__condvar_acquire_lock (pthread_cond_t *cond, int private)
{
  unsigned int s = atomic_load_relaxed (&cond->\__data.\__g1_orig_size);
  while ((s & 3) == 0)
    {
      if (atomic_compare_exchange_weak_acquire (&cond->\__data.\__g1_orig_size,
      &s, s | 1))
    return;
      /* TODO Spinning and back-off.  \*/
    }
  /* We can't change from not acquired to acquired, so try to change to
     acquired-with-futex-wake-request and do a futex wait if we cannot change
     from not acquired.  \*/
  while (1)
    {
      while ((s & 3) != 2)
    {
      if (atomic_compare_exchange_weak_acquire
          (&cond->__data.__g1_orig_size, &s, (s & ~(unsigned int) 3) | 2))
        {
          if ((s & 3) == 0)
        return;
          break;
        }
      /* TODO Back off.  \*/
    }
      futex_wait_simple (&cond->__data.__g1_orig_size,
      (s & ~(unsigned int) 3) | 2, private);
      /* Reload so we see a recent value.  \*/
      s = atomic_load_relaxed (&cond->__data.__g1_orig_size);
    }
}
 
/* See __condvar_acquire_lock.  */
static void __attribute__ ((unused))
__condvar_release_lock (pthread_cond_t *cond, int private)
{
  if ((atomic_fetch_and_release (&cond->__data.__g1_orig_size,
                 ~(unsigned int) 3) & 3)
      == 2)
    futex_wake (&cond->__data.__g1_orig_size, 1, private);
}

四、系统调用

其实所有的调用，最终都要落到系统调用中来： 

#define lll_futex_syscall(nargs, futexp, op, ...)                       \
  ({                                                                    \
    INTERNAL_SYSCALL_DECL (\__err);                                      \
    long int \__ret = INTERNAL_SYSCALL (futex, \__err, nargs, futexp, op, \
                       \__VA_ARGS__);                    \
    (__glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (\__ret, \__err))         \
     ? -INTERNAL_SYSCALL_ERRNO (\__ret, \__err) : 0);                     \
  })
 
#define lll_futex_wait(futexp, val, private) \
  lll_futex_timed_wait (futexp, val, NULL, private)
 
#define lll_futex_timed_wait(futexp, val, timeout, private)     \
  lll_futex_syscall (4, futexp,                                 \
             __lll_private_flag (FUTEX_WAIT, private),  \
             val, timeout)
 
#define lll_futex_timed_wait_bitset(futexp, val, timeout, clockbit, private) \
  lll_futex_syscall (6, futexp,                                         \
             __lll_private_flag (FUTEX_WAIT_BITSET | (clockbit), \
                     private),                      \
             val, timeout, NULL /* Unused.  */,                 \
             FUTEX_BITSET_MATCH_ANY)
 
#define lll_futex_wake(futexp, nr, private)                             \
  lll_futex_syscall (4, futexp,                                         \
             __lll_private_flag (FUTEX_WAKE, private), nr, 0)
 
/* Returns non-zero if error happened, zero if success.  */
#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val, private) \
  lll_futex_syscall (6, futexp,                                         \
             __lll_private_flag (FUTEX_CMP_REQUEUE, private),   \
             nr_wake, nr_move, mutex, val)
 
/* Returns non-zero if error happened, zero if success.  */
#define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2, private) \
  lll_futex_syscall (6, futexp,                                         \
             __lll_private_flag (FUTEX_WAKE_OP, private),       \
             nr_wake, nr_wake2, futexp2,                        \
             FUTEX_OP_CLEAR_WAKE_IF_GT_ONE)
 
/* Priority Inheritance support.  */
#define lll_futex_wait_requeue_pi(futexp, val, mutex, private) \
  lll_futex_timed_wait_requeue_pi (futexp, val, NULL, 0, mutex, private)
 
#define lll_futex_timed_wait_requeue_pi(futexp, val, timeout, clockbit, \
                    mutex, private)                 \
  lll_futex_syscall (5, futexp,                                         \
             __lll_private_flag (FUTEX_WAIT_REQUEUE_PI          \
                     | (clockbit), private),        \
             val, timeout, mutex)
 
 
#define lll_futex_cmp_requeue_pi(futexp, nr_wake, nr_move, mutex,       \
                 val, private)                          \
  lll_futex_syscall (6, futexp,                                         \
             __lll_private_flag (FUTEX_CMP_REQUEUE_PI,          \
                     private),                      \
             nr_wake, nr_move, mutex, val)

futex又可以分类成以下几部分： 
normal futex：通用的futex. 
pi-futex:pi协议的futex. 
requeue-pi:使用队列的pi协议futex. 
robust-futex:robust规则的futex. 
lowlevellock提供三种锁的机制，一是基于0或1的互斥的锁规则，二是基于robust futex定义的锁规则，三是用于condition重新对临界区上锁的操作。在前文的参数介绍中也有类似的介绍。 
这些代码最终又会调用下一篇分析中的内核部分代码。这样，就可以从应用到Glibc库再到内核把futex看得清清楚楚。