Unix Prog: Thread Attributes

1. Some Thread limitations

 ubuntu@ip-172-31-23-227:~$ less /usr/include/x86_64-linux-gnu/bits/local_lim.h  
 ......  
 /* The number of data keys per process. */  
 #define _POSIX_THREAD_KEYS_MAX 128  
 /* This is the value this implementation supports. */  
 #define PTHREAD_KEYS_MAX    1024  
 ......  
 /* Controlling the iterations of destructors for thread-specific data. */  
 #define _POSIX_THREAD_DESTRUCTOR_ITERATIONS   4  
 /* Number of iterations this implementation does. */  
 #define PTHREAD_DESTRUCTOR_ITERATIONS  _POSIX_THREAD_DESTRUCTOR_ITERATIONS  
 ......  
 /* The number of threads per process. */  
 #define _POSIX_THREAD_THREADS_MAX    64  
 /* We have no predefined limit on the number of threads. */  
 #undef PTHREAD_THREADS_MAX  
 ......  
 /* Minimum size for a thread. We are free to choose a reasonable value. */  
 #define PTHREAD_STACK_MIN    16384  
 ......  

2. Thread Attributes

System Definitions:

 ubuntu@ip-172-31-23-227:~$ less /usr/include/pthread.h  
 ......  
 /* Initialize thread attribute *ATTR with default attributes  
   (detachstate is PTHREAD_JOINABLE, scheduling policy is SCHED_OTHER,  
   no user-provided stack). */  
 extern int pthread_attr_init (pthread_attr_t *__attr) __THROW __nonnull ((1));  
   
 /* Destroy thread attribute *ATTR. */  
 extern int pthread_attr_destroy (pthread_attr_t *__attr)  
    __THROW __nonnull ((1));  
 ......  

We can use the pthread_attr_t structure to modify the default attributes, and associate these attributes with threads that we create.

We use the pthread_attr_init function to initialize pthread_attr_t data structure.

pthread_attr_t structure is supposed to be opaque to applications. Application should use API to query and set the attribute. This helps improve the portability.

3. Attribute -- Detachable State

System Definitions:

 ubuntu@ip-172-31-23-227:~$ less /usr/include/pthread.h  
 ......  
 /* Get detach state attribute. */  
 extern int pthread_attr_getdetachstate (const pthread_attr_t *__attr,  
                     int *__detachstate)  
    __THROW __nonnull ((1, 2));  
   
 /* Set detach state attribute. */  
 extern int pthread_attr_setdetachstate (pthread_attr_t *__attr,  
                     int __detachstate)  
    __THROW __nonnull ((1));  
 ......  
/* Detach state.  */
enum
{
  PTHREAD_CREATE_JOINABLE,
#define PTHREAD_CREATE_JOINABLE PTHREAD_CREATE_JOINABLE
  PTHREAD_CREATE_DETACHED
#define PTHREAD_CREATE_DETACHED PTHREAD_CREATE_DETACHED
};
......

Thread with detachable state: allow the operating system to recliam the thread's resources when thread exits.

The second parameter in above functions can be either PTHREAD_CREATE_JOINABLE or PTHREAD_CREATE_DETACHED.

Example:

 #include<stdio.h>  
 #include<stdlib.h>  
 #include<unistd.h>  
 #include<pthread.h>  
   
 void* thd_fn(void *vd)  
 {  
  // sleep for 1 second to let main thread run pthread_attr_destroy firstly  
  sleep(1);  
  printf("Hello world!\n");  
 }  
   
 int main(int argc, char* argv[])  
 {  
  int err;  
  pthread_t tid;  
  pthread_attr_t attr;  
   
  // Initialize attributes  
  err = pthread_attr_init(&attr);  
  if(err != 0) {  
   printf("pthread_attr_init error!\n");  
   exit(err);  
  }  
   
  err = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);  
   
  // Create and launch the thread  
  if(err == 0)  
   err = pthread_create(&tid, &attr, thd_fn, NULL);  
  pthread_attr_destroy(&attr);  
   
  // Sleep for 2 seconds to let thread finish  
  sleep(2);  
  return err;  
 }  

shell:
The program launched the thread firstly, the thread wait for 1 second, during this period, main thread finish calling pthread_attr_destroy and start waiting for 2 seconds. During this period, the thread finish and return.
This indicates that, pthread_attr_destroy is called before thread finish is ok.

 ubuntu@ip-172-31-23-227:~$ ./thread.out  
 Hello world!  

4. Other Attributes API:
The usage is same as above example for detachable state.

System Definition:

 ubuntu@ip-172-31-23-227:~$ less /usr/include/pthread.h  
 ......  
 /* Return the previously set address for the stack. */  
 extern int pthread_attr_getstackaddr (const pthread_attr_t *__restrict  
                    __attr, void **__restrict __stackaddr)  
    __THROW __nonnull ((1, 2)) __attribute_deprecated__;  
   
 /* Set the starting address of the stack of the thread to be created.  
   Depending on whether the stack grows up or down the value must either  
   be higher or lower than all the address in the memory block. The  
   minimal size of the block must be PTHREAD_STACK_MIN. */  
 extern int pthread_attr_setstackaddr (pthread_attr_t *__attr,  
                    void *__stackaddr)  
    __THROW __nonnull ((1)) __attribute_deprecated__;  
   
 /* Return the currently used minimal stack size. */  
 extern int pthread_attr_getstacksize (const pthread_attr_t *__restrict  
                    __attr, size_t *__restrict __stacksize)  
    __THROW __nonnull ((1, 2));  
   
 /* Add information about the minimum stack size needed for the thread  
   to be started. This size must never be less than PTHREAD_STACK_MIN  
   and must also not exceed the system limits. */  
 extern int pthread_attr_setstacksize (pthread_attr_t *__attr,  
                    size_t __stacksize)  
    __THROW __nonnull ((1));  
   
 #ifdef __USE_XOPEN2K  
 /* Return the previously set address for the stack. */  
 extern int pthread_attr_getstack (const pthread_attr_t *__restrict __attr,  
                  void **__restrict __stackaddr,  
                  size_t *__restrict __stacksize)  
    __THROW __nonnull ((1, 2, 3));  
   
 /* The following two interfaces are intended to replace the last two. They  
   require setting the address as well as the size since only setting the  
   address will make the implementation on some architectures impossible. */  
 extern int pthread_attr_setstack (pthread_attr_t *__attr, void *__stackaddr,  
                  size_t __stacksize) __THROW __nonnull ((1));  
 ......  

Within the process, the virtual address space is shared by all threads.
If you run out of virtual address space for thread stacks, you can use malloc or mmap to allocate space for an alternate stack and use pthread_attr_setstack to change the stack location of threads you create.

The guardsize thread attribute controls the size of the memory extent after the end of the thread's stack to protect against stack overflow. If we change the stackaddr thread attribute, the system assumes that we will be managing our stacks and disables stack guard buffers, just as if we had set the guardsize thread attribute to 0.

If the thread's stack pointer overflow into the guard area, the application will receive an error, possibly with a signal.

Definition:

 ubuntu@ip-172-31-23-227:~$ less /usr/include/pthread.h  
 ......  
 /* Get the size of the guard area created for stack overflow protection. */  
 extern int pthread_attr_getguardsize (const pthread_attr_t *__attr,  
                    size_t *__guardsize)  
    __THROW __nonnull ((1, 2));  
   
 /* Set the size of the guard area created for stack overflow protection. */  
 extern int pthread_attr_setguardsize (pthread_attr_t *__attr,  
                    size_t __guardsize)  
    __THROW __nonnull ((1));  
 ......  

Concurrency level:
Concurrency level indicates how many user threads can be mapped to several kernel facilities simultaneously. Unix allows user to get and set the concurrency level to adjust the process performance, but different implementations varies. OS has to right to ignore user's request. If return value or setup value is 0, then thread's concurrency level is controlled completely by OS.

Definition:

 ubuntu@ip-172-31-23-227:~$ less /usr/include/pthread.h  
 ......  
 /* Determine level of concurrency. */  
 extern int pthread_getconcurrency (void) __THROW;  
   
 /* Set new concurrency level to LEVEL. */  
 extern int pthread_setconcurrency (int __level) __THROW;  
 ......  

Unix Prog: Condition Variables

1. Condition Variables Concept
A thread must first lock the mutex to change the condition state. Other threads will not notice the change until they acquire the mutex, because the mutex must be locked to able to evaluate the condition.

2. Condition Variable Definitions:

 ubuntu@ip-172-31-23-227:~$ less /usr/include/pthread.h  
 ......  
 /* Initialize condition variable COND using attributes ATTR, or use  
   the default values if later is NULL. */  
 extern int pthread_cond_init (pthread_cond_t *__restrict __cond,  
                const pthread_condattr_t *__restrict __cond_attr)  
    __THROW __nonnull ((1));  
   
 /* Destroy condition variable COND. */  
 extern int pthread_cond_destroy (pthread_cond_t *__cond)  
    __THROW __nonnull ((1));  
   
 /* Wake up one thread waiting for condition variable COND. */  
 extern int pthread_cond_signal (pthread_cond_t *__cond)  
    __THROWNL __nonnull ((1));  
   
 /* Wake up all threads waiting for condition variables COND. */  
 extern int pthread_cond_broadcast (pthread_cond_t *__cond)  
    __THROWNL __nonnull ((1));  
   
 /* Wait for condition variable COND to be signaled or broadcast.  
   MUTEX is assumed to be locked before.  
   
   This function is a cancellation point and therefore not marked with  
   __THROW. */  
 extern int pthread_cond_wait (pthread_cond_t *__restrict __cond,  
                pthread_mutex_t *__restrict __mutex)  
    __nonnull ((1, 2));  
   
 /* Wait for condition variable COND to be signaled or broadcast until  
   ABSTIME. MUTEX is assumed to be locked before. ABSTIME is an  
   absolute time specification; zero is the beginning of the epoch  
   (00:00:00 GMT, January 1, 1970).  
   
   This function is a cancellation point and therefore not marked with  
   __THROW. */  
 extern int pthread_cond_timedwait (pthread_cond_t *__restrict __cond,  
                   pthread_mutex_t *__restrict __mutex,  
                   const struct timespec *__restrict __abstime)  
    __nonnull ((1, 2, 3));  
 ......  

Before a condition variable is used, it must firstly be initialized. We can assign the constant PTHREAD_COND_INITIALIZER or use pthread_cond_init to initialize it.

Before freeing the memory, we can use the pthread_cond_destroy function to reclaim the storage.

We use pthread_cond_wait to wait for a condition to be true.  The caller passes it locked to the function, which then atomically places the calling thread on the list of threads waiting for the condition and unlocks the mutex. When pthread_cond_wait returns, the mutex is again locked.

pthread_cond_timedwait function works the same as the pthread_cond_wait function with the addition of the timeout. With struct timespec, we specify how long we are willing to wait as an ABSOLUTE time instead of relative time.

pthread_cond_signal function will wake up one thread waiting on a condition, pthread_cond_broadcast function will wake up all threads waiting on a condition.

struct timespec:

 ubuntu@ip-172-31-23-227:~$ less /usr/include/linux/time.h  
 ......  
 struct timespec {  
     __kernel_time_t tv_sec;         /* seconds */  
     long      tv_nsec;        /* nanoseconds */  
 };  
 ......  

We can use following code to get the timespec:

3. Example for getting the timespec:

 #include<stdio.h>  
 #include<stdlib.h>  
 #include<sys/time.h>  
 #include<unistd.h>  
   
 void maketimeout(struct timespec *tsp, long minutes)  
 {  
  struct timeval now;  
   
  gettimeofday(&now, NULL);  
   
  tsp->tv_sec = now.tv_sec;  
  tsp->tv_nsec = now.tv_usec*1000; // tv_usec is microseconds, tv_nsec is nano seconds  
  tsp->tv_sec += minutes * 60;  
   
 }  
   
 int main(int argc, char* argv[])  
 {  
  struct timespec tsp;  
  maketimeout(&tsp, 10);  
   
  printf("tsp->tv_sec: %ld\n", tsp.tv_sec);  
  printf("tsp->tv_nsec: %ld\n", tsp.tv_nsec);  
   
  exit(0);  
 }  

4. Example of condition variables:
cond.c:

 #include<stdio.h>  
 #include<stdlib.h>  
 #include<unistd.h>  
 #include<pthread.h>  
   
 struct msg {  
  struct msg *m_next;  
  char *text;  
 };  
   
 struct msg *workq;  
   
 pthread_cond_t qready = PTHREAD_COND_INITIALIZER;  
 pthread_mutex_t qlock = PTHREAD_MUTEX_INITIALIZER;  
   
 void* process_msg(void* vd)  
 {  
  struct msg *mp;  
   
  // The thread launching this function will keep processing message  
  for(;;) {  
   pthread_mutex_lock(&qlock);  
   
   // wait for the enqueue_messsage thread to broadcast signal  
   // Once pthread_cond_wait blocks, qlock mutex is released, then  
   // enqueue_message thread could re-lock the mutex and insert the message  
   while(workq == NULL)  
    pthread_cond_wait(&qready, &qlock);  
   mp = workq;  
   workq = mp->m_next;  
   pthread_mutex_unlock(&qlock);  
   /* ......process message...... */  
   printf("message: %s\n", mp->text);  
  }  
 }  
   
 void enqueue_msg(struct msg *mp)  
 {  
  // Lock the mutex and enqueue message to the head of queue  
  pthread_mutex_lock(&qlock);  
  mp->m_next = workq;  
  workq = mp;  
  pthread_mutex_unlock(&qlock);  
   
  // Unlock the mutex then broadcast signal to allow one thread  
  // wake up who is waiting for the signal on cond qready  
  pthread_cond_signal(&qready);  
 }  
   
 int main(int argc, char* argv[])  
 {  
  // Launch the thread including process_msg, which will block  
  // on pthread_cond_wait, then wait for the signal from main thread  
  pthread_t tid;  
  if(pthread_create(&tid, NULL, process_msg, NULL) != 0) {  
   printf("pthread_create failed.\n");  
   exit(1);  
  }  
   
  // Main thread build one struct message, then call enqueue_msg to  
  // insert message and broadcast the signal  
  sleep(1);  
  struct msg smsg;  
  smsg.text = "Hello world!";  
  enqueue_msg(&smsg);  
  sleep(1);  
   
  exit(0);  
 }  

shell:
After receiving the signal, the process_message thread get out of pthread_cond_wait and processed the message successfully.

 ubuntu@ip-172-31-23-227:~$ ./cond.out  
 message: Hello world!  

Unix Prog: Reader-Writer Lock

1. Reader-Writer Locks

Three states for reader_writer locks:
1) locks in read mode
2) locks in write mode
3) unlocked mode

Only one thread can be in lock-in-write mode, but multiple threads can coexist i n lock-in-read mode.

1) If the lock is write-locked, no threads can gain the lock until it is released
2) If the lock is read-locked, threads attempting to lock it in read mode are given access, but threads attempting to lock it in write mode will be blocked.
3) read-writer usually blocks additional readers if a lock is already held in read mode and a thread is blocked trying to acquire the lock in write mode. This prevents a constant stream of readers from starving waiting writers.

Read-writer locks are well suited for situations in which data structures are read more often than they are modified.

2. System Call Definitions:
read-writer locks must be initialized before use and destroyed before freeing their underlying memory:

 ubuntu@ip-172-31-23-227:~$ less /usr/include/pthread.h  
 ......  
 /* Initialize read-write lock RWLOCK using attributes ATTR, or use  
   the default values if later is NULL. */  
 extern int pthread_rwlock_init (pthread_rwlock_t *__restrict __rwlock,  
                 const pthread_rwlockattr_t *__restrict  
                 __attr) __THROW __nonnull ((1));  
   
 /* Destroy read-write lock RWLOCK. */  
 extern int pthread_rwlock_destroy (pthread_rwlock_t *__rwlock)  
    __THROW __nonnull ((1));  
   
 /* Acquire read lock for RWLOCK. */  
 extern int pthread_rwlock_rdlock (pthread_rwlock_t *__rwlock)  
    __THROWNL __nonnull ((1));  
   
 /* Try to acquire read lock for RWLOCK. */  
 extern int pthread_rwlock_tryrdlock (pthread_rwlock_t *__rwlock)  
  __THROWNL __nonnull ((1));  
   
 /* Acquire write lock for RWLOCK. */  
 extern int pthread_rwlock_wrlock (pthread_rwlock_t *__rwlock)  
    __THROWNL __nonnull ((1));  
   
 /* Try to acquire write lock for RWLOCK. */  
 extern int pthread_rwlock_trywrlock (pthread_rwlock_t *__rwlock)  
    __THROWNL __nonnull ((1));  
   
 /* Unlock RWLOCK. */  
 extern int pthread_rwlock_unlock (pthread_rwlock_t *__rwlock)  
    __THROWNL __nonnull ((1));  
 ......  

To lock a reader-writer lock in read mode, we call pthread_rwlock_rdlock.
To write-lock mode, we call pthread_rwlock_wrlock
To unlock, we call pthread_rwlock_unlock

Similar from pthread_mutex_trylock, pthread_rwlock_tryrdlock and pthread_rwlock_trywrlock,  they never blocked, but if the lock is not locked, it will lock it.

trylock is very useful for avoding the deadlock, since it never block.

2. Example:
rwlock.c:

 #include<stdio.h>  
 #include<stdlib.h>  
 #include<unistd.h>  
 #include<pthread.h>  
   
 struct job {  
  struct job *j_next;  
  struct job *j_prev;  
  pthread_t j_id;  
 };  
   
 struct queue {  
  struct job *q_head;  
  struct job *q_tail;  
  pthread_rwlock_t q_lock;  
 };  
   
 // Initialize the queue by setting up the head, tail pointer  
 // And also init the reader-writer lock  
 int queue_init(struct queue *qp)  
 {  
  int err;  
  qp->q_head = NULL;  
  qp->q_tail = NULL;  
  err = pthread_rwlock_init(&qp->q_lock, NULL);  
  if(err != 0)  
   return err;  
   
  return 0;  
 }  
   
 void job_insert(struct queue *qp, struct job *jp)  
 {  
  // Lock into write-mode  
  pthread_rwlock_wrlock(&qp->q_lock);  
   
  // Insert the job in front of the head of queue  
  jp->j_next = qp->q_head;  
  jp->j_prev = NULL;  
   
  if(qp->q_head != NULL)  
   qp->q_head->j_prev = jp;  
  else  
   qp->q_tail = jp;  
  qp->q_head = jp;  
   
  // Unlock  
  pthread_rwlock_unlock(&qp->q_lock);  
 }  
   
 void job_append(struct queue *qp, struct job *jp)  
 {  
  // Lock into write-mode  
  pthread_rwlock_wrlock(&qp->q_lock);  
   
  // Append the job to the tail of queue  
  jp->j_next = NULL;  
  jp->j_prev = qp->q_tail;  
  if(qp->q_tail != NULL)  
   qp->q_tail->j_next = jp;  
  else  
   qp->q_head = jp;  
  qp->q_tail = jp;  
   
  // Unlock  
  pthread_rwlock_unlock(&qp->q_lock);  
 }  
   
 void job_remove(struct queue *qp, struct job *jp)  
 {  
  // Lock into the write mode  
  pthread_rwlock_wrlock(&qp->q_lock);  
   
  // remove the job from the queue  
  if(jp == qp->q_head) {  
   qp->q_head = jp->j_next;  
   if(qp->q_tail == jp)  
    qp->q_tail = NULL;  
   else  
    jp->j_next->j_prev = jp->j_prev;  
  }  
  else if(jp == qp->q_tail) {  
   qp->q_tail = jp->j_prev;  
   if(qp->q_head == jp)  
    qp->q_head = NULL;  
   else  
    jp->j_prev->j_next = jp->j_next;  
  }  
  else {  
   jp->j_prev->j_next = jp->j_next;  
   jp->j_next->j_prev = jp->j_prev;  
  }  
   
  // Unlock  
  pthread_rwlock_unlock(&qp->q_lock);  
 }  
   
 struct job *job_find(struct queue *qp, pthread_t id)  
 {  
  struct job *jp;  
   
  if(pthread_rwlock_rdlock(&qp->q_lock) != 0)  
   return NULL;  
   
  for(jp = qp->q_head; jp != NULL; jp = jp->j_next)  
   if(pthread_equal(jp->j_id, id))  
    break;  
   
  pthread_rwlock_unlock(&qp->q_lock);  
  return jp;  
 }  
   
 int main(int argc, char* argv[])  
 {  
  struct queue que;  
  struct job job;  
   
  queue_init(&que);  
  job_insert(&que, &job);  
  job_remove(&que, &job);  
   
  exit(0);  
 }  

Every time when the thread operate the global struct queue, we use read-write lock to lock. In this way, we make global struct queue to be thread safe.

Unix Prog: Deadlock Avoidance(2)

1. Simplified Example:
In prevoious example, hashlock mutex is used to control the modification for hash table, struct foo lock mutex f_lock is used to control the modification of the local struct foo. We can change it to "use hashlock mutex to control everything" to considerably reduce the complexity of the code, but worsen the performance of program, since it is possible that many threads are blocked by one mutex: hashlock.

deadlock.c:

 #include<stdio.h>  
 #include<stdlib.h>  
 #include<unistd.h>  
 #include<pthread.h>  
   
 #define NHASH 29  
 #define HASH(fp) (((unsigned long)fp) % NHASH)  
 struct foo *fh[NHASH];  
   
 pthread_mutex_t hashlock = PTHREAD_MUTEX_INITIALIZER;  
   
 struct foo {  
  int f_count;  
  pthread_mutex_t f_lock;  
  struct foo *f_next;  
  int f_id;  
 };  
   
 struct foo *foo_alloc(int id)  
 {  
  struct foo *fp;  
  int idx;  
   
  // Create the struct foo and insert into hash table  
  if ((fp = malloc(sizeof(struct foo))) != NULL) {  
   fp->f_count = 1;  
   fp->f_id = id;  
   if(pthread_mutex_init(&fp->f_lock, NULL) != 0) {  
    free(fp);  
    return NULL;  
   }  
   idx = HASH(fp);  
   pthread_mutex_lock(&hashlock);  
   fp->f_next = fh[idx];  
   fh[idx] = fp;  
   
   // We lock the f_lock here, and don't unlock it, this doesn't  
   // affect anything  
   // 1) no other threads can come here again  
   // 2) it is impossible to malloc a struct while locking on the  
   //  same mutex  
   pthread_mutex_lock(&fp->f_lock);  
   pthread_mutex_unlock(&hashlock);  
  }  
   
  return (fp);  
 }  
   
 void foo_hold(struct foo *fp)  
 {  
  // Right now hash lock is used to control the modification  
  // of any struct foo  
  pthread_mutex_lock(&hashlock);  
  fp->f_count++;  
  pthread_mutex_unlock(&hashlock);  
 }  
   
 struct foo * foo_find(int id)  
 {  
  struct foo *fp;  
  int idx;  
   
  pthread_mutex_lock(&hashlock);  
   
  // Find the struct with specified id  
  for(idx = 0; idx < NHASH; idx++) {  
   for(fp = fh[idx]; fp != NULL; fp = fp->f_next) {  
    if(fp->f_id == id) {  
     // Note that we are still using hashlock to control the  
     // modification of struct foo  
     fp->f_count++;  
     pthread_mutex_unlock(&hashlock);  
     return fp;  
    }  
   }  
  }  
   
  // If not found the struct, return NULL  
  pthread_mutex_unlock(&hashlock);  
  return NULL;  
 }  
   
 void foo_rele(struct foo*fp)  
 {  
  struct foo *tfp;  
  int idx;  
   
  pthread_mutex_lock(&hashlock);  
  // We only use the haslock to control modification of  
  // any struct  
  if(--fp->f_count == 0) {  
   idx = HASH(fp);  
   tfp = fh[idx];  
   if(tfp == fp) {  
    fh[idx] = fp->f_next;  
   } else {  
    while(tfp->f_next != fp)  
     tfp = tfp->f_next;  
    tfp->f_next = fp->f_next;  
   }  
   
   pthread_mutex_unlock(&hashlock);  
   pthread_mutex_destroy(&fp->f_lock);  
   free(fp);  
  }  
  else {  
   pthread_mutex_unlock(&hashlock);  
  }  
 }  
   
 int main(int argc, char* argv[])  
 {  
  struct foo *fp1;  
  struct foo *fp2;  
  fp1 = foo_alloc(1);  
  fp2 = foo_alloc(2);  
   
  foo_rele(fp1);  
  foo_rele(fp2);  
  exit(0);  
 }  

trade-off for multithreading programming:
If locking granularity is too coarse, you end up with too many threads blocking behind the same mutex, while little improvement possible from concurrency.
If locking granularity is too fine, then you suffer bad performance from excess locking overhead, and you end up with complex code.

Unix Prog: Deadlock Avoidance(1)

1. Deadlock Occur
1) One thread try to lock same mutex twice
2) One thread locks mutex a while waiting for unlocking of mutex b.
Another thread locks mutex b while waiting for unlocking of mutex a.

Solution:
1) Don't try to lock same mutex more than one time.
2) Always lock mutex with same order at all threads!
always lock mutex a then mutex b in above example

2. Example avoiding the deadlock:
deadlock.c:

 #include<stdio.h>  
 #include<stdlib.h>  
 #include<unistd.h>  
 #include<pthread.h>  
   
 #define NHASH 29  
 #define HASH(fp) (((unsigned long)fp) % NHASH)  
 struct foo *fh[NHASH];  
   
 pthread_mutex_t hashlock = PTHREAD_MUTEX_INITIALIZER;  
   
 struct foo {  
  int f_count;  
  pthread_mutex_t f_lock;  
  struct foo *f_next;  
  int f_id;  
 };  
   
 struct foo *foo_alloc(int id)  
 {  
  struct foo *fp;  
  int idx;  
   
  // Create the struct foo and insert into hash table  
  if ((fp = malloc(sizeof(struct foo))) != NULL) {  
   fp->f_count = 1;  
   fp->f_id = id;  
   if(pthread_mutex_init(&fp->f_lock, NULL) != 0) {  
    free(fp);  
    return NULL;  
   }  
   
   // Call hash function to get the hash index  
   // Insert the struct into hash table  
   // Insert the struct into the front of list  
   idx = HASH(fp);  
   pthread_mutex_lock(&hashlock);  
   fp->f_next = fh[idx];  
   fh[idx] = fp;  
   
   // After inserting into the hash table, struct foo is inside  
   // global data structure. It is possible that other threads may  
   // access it. At this time, we need lock the mutex. Also, we honor  
   // the order of locking hashlcok firstly then f_lock in struct foo  
   pthread_mutex_lock(&fp->f_lock);  
   pthread_mutex_unlock(&hashlock);  
   /* ... cotinue initialization ... */  
   pthread_mutex_unlock(&fp->f_lock);  
  }  
   
  return (fp);  
 }  
   
 void foo_hold(struct foo *fp)  
 {  
  pthread_mutex_lock(&fp->f_lock);  
  fp->f_count++;  
  pthread_mutex_unlock(&fp->f_lock);  
 }  
   
 struct foo * foo_find(int id)  
 {  
  struct foo *fp;  
  int idx;  
   
  pthread_mutex_lock(&hashlock);  
   
  // Find the struct with specified id  
  for(idx = 0; idx < NHASH; idx++) {  
   for(fp = fh[idx]; fp != NULL; fp = fp->f_next) {  
    if(fp->f_id == id) {  
     // if found the struct, call foo_hold, note that we are still  
     // honoring the order of locking hashlock firstly then f_lock  
     // to avoid deadlock  
     foo_hold(fp);  
     pthread_mutex_unlock(&hashlock);  
     return fp;  
    }  
   }  
  }  
   
  // If not found the struct, return NULL  
  pthread_mutex_unlock(&hashlock);  
  return NULL;  
 }  
   
 void foo_rele(struct foo*fp)  
 {  
  struct foo *tfp;  
  int idx;  
   
  // Before accessing the struct in global data structure, we need  
  // to acquire the lock firstly  
  pthread_mutex_lock(&fp->f_lock);  
  if(fp->f_count == 1) {  
   // After confirming that f_count = 1, we plan to remove it from  
   // the hash table. Then we need to lock hashlock mutex to modify  
   // the hash table. But the problem here is: we need to unlock  
   // f_lock firstly, then lock hashlock, to honor the order that  
   // we should always lock hashlock firstly then lock f_lock to avoid  
   // dead lock.  
   
   // After we unlock f_lock, and then lock the hashlock, and then  
   // lock f_lock, during this minor time period, other threads are  
   // possible to modify the struct. so we need to re-confirm whether  
   // f_count is still 1, if not, reduce the reference count and return  
   pthread_mutex_unlock(&fp->f_lock);  
   pthread_mutex_lock(&hashlock);  
   pthread_mutex_lock(&fp->f_lock);  
   if(fp->f_count != 1) {  
    fp->f_count--;  
    pthread_mutex_unlock(&fp->f_lock);  
    pthread_mutex_unlock(&hashlock);  
    return;  
   }  
   
   //If reference count is still 1, then we proceed to remove it from list  
   idx = HASH(fp);  
   tfp = fh[idx];  
   if(tfp == fp) {  
    fh[idx] = fp->f_next;  
   } else {  
    while(tfp->f_next != fp)  
     tfp = tfp->f_next;  
    tfp->f_next = fp->f_next;  
   }  
   
   //Release lock the struct foo  
   pthread_mutex_unlock(&hashlock);  
   pthread_mutex_unlock(&fp->f_lock);  
   pthread_mutex_destroy(&fp->f_lock);  
   free(fp);  
  }  
  else {  
   fp->f_count--;  
   pthread_mutex_unlock(&fp->f_lock);  
  }  
 }  
   
 int main(int argc, char* argv[])  
 {  
  struct foo *fp1;  
  struct foo *fp2;  
  fp1 = foo_alloc(1);  
  fp2 = foo_alloc(2);  
  fp1 = foo_find(1);  
   
  foo_rele(fp1);  
  foo_rele(fp1);  
  foo_rele(fp2);  
  exit(0);  
 }  

With all methods used in above example, we made struct foo in hash table to be thread safety. It won't be deadlocked in any situation.

Unix Prog: Mutexes

1. mutex
A mutex is basically a lock that we set(lock) before accessing a shared resource and release(unlock) when we're done. While it is set, any other thread tries to set it will block until we release it.

system call definition:

 ubuntu@ip-172-31-23-227:~$ less /usr/include/pthread.h  
 ......  
 /* Initialize a mutex. */  
 extern int pthread_mutex_init (pthread_mutex_t *__mutex,  
                 const pthread_mutexattr_t *__mutexattr)  
    __THROW __nonnull ((1));  
   
 /* Destroy a mutex. */  
 extern int pthread_mutex_destroy (pthread_mutex_t *__mutex)  
    __THROW __nonnull ((1));  
   
 /* Try locking a mutex. */  
 extern int pthread_mutex_trylock (pthread_mutex_t *__mutex)  
    __THROWNL __nonnull ((1));  
   
 /* Lock a mutex. */  
 extern int pthread_mutex_lock (pthread_mutex_t *__mutex)  
    __THROWNL __nonnull ((1));  
   
 /* Unlock a mutex. */  
 extern int pthread_mutex_unlock (pthread_mutex_t *__mutex)  
    __THROWNL __nonnull ((1));  
 ......  

Before using a mutex, we need to either initialize it to PTHREAD_MUTEX_INITIALIZER or call pthread_mutex_init.

If we allocate the mutex dynamically, when we need to call pthread_mutex_destroy before freeing the memory.

pthread_mutex_trylock behaves conditionally:
1) if mutex is locked, it will return EBUSY without blocking, without locking the mutex.
2) if mutex is not locked, it will lock the mutex without blocking.

2. Example:

 #include<stdio.h>  
 #include<stdlib.h>  
 #include<pthread.h>  
   
 int value = 0;  
 pthread_mutex_t lock;  
   
 void* thr_fn1(void *arg)  
 {  
  pthread_mutex_trylock(&lock);  
  printf("thread 1 trylocked\n");  
  value++;  
  sleep(3);  
  printf("thread 1 unlocked\n");  
  pthread_mutex_unlock(&lock);  
 }  
   
 int main(int argc, char* argv[])  
 {  
  // Initialize mutex  
  if(pthread_mutex_init(&lock, NULL) != 0) {  
   printf("mutex init failed.\n");  
   exit(1);  
  }  
   
  // launch the thread 1  
  pthread_t tid1;  
  if(pthread_create(&tid1, NULL, thr_fn1, NULL) != 0) {  
   printf("pthread_create failed.\n");  
   exit(2);  
  }  
   
  // main thread sleep 2 seconds to let thread 1 run firstly  
  sleep(2);  
   
  printf("waiting for thread 1 lock...\n");  
  pthread_mutex_lock(&lock);  
  printf("main thread locked\n");  
  value++;  
  pthread_mutex_unlock(&lock);  
   
  printf("value = %d\n", value);  
  exit(0);  
 }  

shell:
Firstly, main thread fell asleep to let thread 1 run firstly. Thread 1 "trylock" the mutex, and sleep 3 seconds. During this period, main thread waked up and and lock the mutex, but at this time, thread 1 still doesn't release the lock.
After 1 second, thread 1 wake up and release the lock. Main thread acquire the lock.

 ubuntu@ip-172-31-23-227:~$ ./trylock.out  
 thread 1 trylocked  
 waiting for thread 1 lock...  
 thread 1 unlocked  
 main thread locked  
 value = 2  

===============================================================

 #include<stdio.h>  
 #include<stdlib.h>  
 #include<pthread.h>  
   
 int value = 0;  
 pthread_mutex_t lock;  
   
 void* thr_fn1(void *arg)  
 {  
  if(pthread_mutex_trylock(&lock) == 0) {  
   printf("thread 1 trylocked\n");  
   value++;  
   printf("thread 1 unlocked\n");  
   pthread_mutex_unlock(&lock);  
  }  
  else {  
   printf("thread trylock failed, but it doesn't block!\n");  
  }  
 }  
   
 int main(int argc, char* argv[])  
 {  
  // Initialize mutex  
  if(pthread_mutex_init(&lock, NULL) != 0) {  
   printf("mutex init failed.\n");  
   exit(1);  
  }  
   
  // Main thread lock the mutex  
  pthread_mutex_lock(&lock);  
  printf("main thread locked\n");  
  value++;  
   
  // launch the thread 1  
  pthread_t tid1;  
  if(pthread_create(&tid1, NULL, thr_fn1, NULL) != 0) {  
   printf("pthread_create failed.\n");  
   exit(2);  
  }  
   
  // Sleep 2 seconds to let thread 1 execute  
  sleep(2);  
  pthread_mutex_unlock(&lock);  
   
  printf("value = %d\n", value);  
  exit(0);  
 }  

shell:
Firstly, main thread lock the mutex. And then launch the thread 1, who will trylock the mutex, but at this time, mutex is already locked by main thread, so trylock failed, but it doesn't block there, it keep executing and output:
"thread trylock failed, but it doesn't block!". Then thread 1 is finished.

After 2 seconds, main thread wake up and release the mutex.

 ubuntu@ip-172-31-23-227:~$ ./trylock.out  
 main thread locked  
 thread trylock failed, but it doesn't block!  
 value = 1  

Unix Prog: Thread Synchronization

1. Thread Synchronization Examples

If the "write" operation needs multiple memory cycles, then when multiple threads are operating same variables, the thread 2 may read inconsistent value.

At above example, thread 1 is reading the variable, then writing the new value. After the first step of write, at this time, thread 2 read the value of the variable, which is a inconsistent value.

Adding the lock can solve the problem:

Every time when a thread needs to access the thread, it needs to acquire the lock, after the operation is done, thread needs to release the lock to let other threads operate the lock.

At above example, thread 2 acquire the lock firstly to read the variable, then release the lock. And then thread 1 acquire the lock to read and write a new value, during this period, thread 2 can not read the value of variable until thread 1 release its lock.

When multiple threads are modifying the variable value at the same time, inconsistency may also occur:

At above example, both thread 1 and thread 2 are going to increase the i's value, so we expect i should be increased by 2, but in the end i is creased by 1.

Unix Prog: Thread Termination(2)

1. pthread_cancel, pthread_cleanup_push, pthread_cleanup_popup

Definition:

 ubuntu@ip-172-31-23-227:~$ less /usr/include/pthread.h  
 ......  
 /* Cancel THREAD immediately or at the next possibility. */  
 extern int pthread_cancel (pthread_t __th);  
 ......  
 /* Install a cleanup handler: ROUTINE will be called with arguments ARG  
   when the thread is canceled or calls pthread_exit. ROUTINE will also  
   be called with arguments ARG when the matching pthread_cleanup_pop  
   is executed with non-zero EXECUTE argument.  
   
   pthread_cleanup_push and pthread_cleanup_pop are macros and must always  
   be used in matching pairs at the same nesting level of braces. */  
 # define pthread_cleanup_push(routine, arg) \  
  do {                                    \  
   __pthread_cleanup_class __clframe (routine, arg)  
   
 /* Remove a cleanup handler installed by the matching pthread_cleanup_push.  
   If EXECUTE is non-zero, the handler function is called. */  
 # define pthread_cleanup_pop(execute) \  
   __clframe.__setdoit (execute);                      \  
 ......  

By default, pthread_cancel will make the thread with id specified by __th behave as if it had called pthread_exit with an argument of PTHREAD_CANCELED. And calling thread just make the request, it doesn't wait.

We can push cleanup handler into a stack, and when we call pthread_cleanup_pop, handler will be called in the reversed order when putting in.

Clean up handler will be called when:
1) makes a call to pthread_exit
2) responds to cancellation request
3) makes a call to pthread_cleanup_pop with a non-zero execute argument
(Note: if execute argument is not 0, then the handler is not called, but handler will still be removed from stack)

Example:

 #include<stdio.h>  
 #include<stdlib.h>  
 #include<unistd.h>  
 #include<string.h>  
 #include<pthread.h>  
   
 void cleanup(void *arg)  
 {  
  printf("cleanup: %s\n", (char*)arg);  
 }  
   
 void* thr_fn1(void* arg)  
 {  
  // Push the clean up handlers  
  printf("thread 1 start.\n");  
  pthread_cleanup_push(cleanup, (void*)"Thread 1 first handler!\n");  
  pthread_cleanup_push(cleanup, (void*)"Thread 1 second handler!\n");  
  printf("thread 1 push complete.\n");  
   
  if(arg) return ((void*) 1);  
   
  // Note: we have to provide the pthread_cleanup_pop here, otherwise  
  // code won't compile. Since pthread_cleanup_push is defined as macro,  
  // and after expansion, each pthread_cleanup_push needs one pthread_cleanup_pop  
  // at the same level.  
   
  pthread_cleanup_pop(0);  
  pthread_cleanup_pop(0);  
  return((void*)1);  
 }  
   
 void* thr_fn2(void* arg)  
 {  
  // Push the clean up handlers  
  printf("thread 2 start.\n");  
  pthread_cleanup_push(cleanup, (void*)"Thread 2 first handler!\n");  
  pthread_cleanup_push(cleanup, (void*)"Thread 2 second handler!\n");  
  printf("thread 2 push complete.\n");  
   
  if(arg) pthread_exit((void*)2);  
   
  pthread_cleanup_pop(0);  
  pthread_cleanup_pop(0);  
  return((void*)1);  
 }  
   
 int main(int argc, char* argv[])  
 {  
  int err;  
  pthread_t tid1, tid2;  
  void *tret;  
   
  // Launch the 1st thread  
  err = pthread_create(&tid1, NULL, thr_fn1, (void*)1);  
  if(err != 0) {  
   printf("can't create thread 1: %s\n", strerror(err));  
   exit(1);  
  }  
   
  // Launch the 2nd thread  
  err = pthread_create(&tid2, NULL, thr_fn2, (void*)1);  
  if(err != 0) {  
   printf("can't create thread 2: %s\n", strerror(err));  
   exit(1);  
  }  
   
  // Wait for the 1st thread finish  
  err = pthread_join(tid1, &tret);  
  if(err != 0) {  
   printf("can't join with thread 1: %s\n", strerror(err));  
   exit(2);  
  }  
   
  printf("thread 1 exit code: %ld\n", (long)tret);  
   
  // Wait for the 2nd thread finish  
  err = pthread_join(tid2, &tret);  
  if(err != 0) {  
   printf("can't join with thread 2: %s\n", strerror(err));  
   exit(2);  
  }  
   
  printf("thread 2 exit code: %ld\n", (long)tret);  
  exit(0);  
 }  

shell:
1) After we launching the thread 1 and thread 2. By OS scheduler, thread 2 runs firstly. When thread 2 calls pthread_exit, it calls clean up handler in the reversed way.
2) Then thread 1 runs. But it doesn't call clean up handler since it is using "return" instead of pthread_exit.
3) Then the main program use pthread_join to get the thread 1 's exit code
4) Lastly the main program use pthread_join to get the thread 2 's exit code

 ubuntu@ip-172-31-23-227:~$ ./thread.out  
 thread 2 start.  
 thread 2 push complete.  
 cleanup: Thread 2 second handler!  
   
 cleanup: Thread 2 first handler!  
   
 thread 1 start.  
 thread 1 push complete.  
 thread 1 exit code: 1  
 thread 2 exit code: 2  

2. pthread_detach
When a thread is created, by default it is joinable thread. This means that after the thread finishes, the calling thread still need to use pthread_join to ask os to reclaim the storage and do the clean up jobs.
If the thread is created with detachable state, then after it finishes, it will immediately reclaim all storage and do the clean up job.

We can call pthread_detach to make a thread into detached state to ask os reclaim its storage and do cleanup jobs after thread is finished.

Definition:

 ubuntu@ip-172-31-23-227:~$ less /usr/include/pthread.h  
 ......  
 /* Indicate that the thread TH is never to be joined with PTHREAD_JOIN.  
   The resources of TH will therefore be freed immediately when it  
   terminates, instead of waiting for another thread to perform PTHREAD_JOIN  
   on it. */  
 extern int pthread_detach (pthread_t __th) __THROW;  
 ......  

Unix Prog: Thread Termination(1)

1. Ways to terminate a thread
1) The thread simply return from the start routine. The return value is the exit code.
2) The thread can be cancelled by another thread in the process.
3) The thread can call pthread_exit to terminate itself.

2. pthread_exit, pthread_join
Definition:

 ubuntu@ip-172-31-23-227:~$ less /usr/include/pthread.h  
 ......  
 /* Terminate calling thread.  
   
   The registered cleanup handlers are called via exception handling  
   so we cannot mark this function with __THROW.*/  
 extern void pthread_exit (void *__retval) __attribute__ ((__noreturn__));  
   
 /* Make calling thread wait for termination of the thread TH. The  
   exit status of the thread is stored in *THREAD_RETURN, if THREAD_RETURN  
   is not NULL.  
   
   This function is a cancellation point and therefore not marked with  
   __THROW. */  
 extern int pthread_join (pthread_t __th, void **__thread_return);  
 ......  

pthread_join will automatically put the thread into detached state, then thread's storage can be reclaimed immediately. If the thread is already in the detached state, then pthread_join normally fail, returning EINVAL.

Example:
thread.c:

 #include<stdio.h>  
 #include<stdlib.h>  
 #include<pthread.h>  
 #include<string.h>  
 #include<unistd.h>  
   
 void* thr_fn1(void *arg)  
 {  
  printf("thread 1 returning \n");  
  return((void*)1);  
 }  
   
 void* thr_fn2(void *arg)  
 {  
  printf("thread 2 exiting \n");  
  pthread_exit((void*)2);  
 }  
   
 int main(int argc, char* argv[])  
 {  
  int err;  
  pthread_t tid1, tid2;  
  void *tret;  
   
  // Create the first thread, the first thread starts running  
  err = pthread_create(&tid1, NULL, thr_fn1, NULL);  
  if(err != 0) {  
   printf("can't create thread 1: %s\n", strerror(err));  
   exit(1);  
  }  
   
  // Create the second thread, the second thread starts running  
  err = pthread_create(&tid2, NULL, thr_fn2, NULL);  
  if(err != 0) {  
   printf("can't create thread 2: %s\n", strerror(err));  
   exit(1);  
  }  
   
  // Calling thread, waits for the first thread, if it exits,  
  // Catch the return code of it  
  err = pthread_join(tid1, &tret);  
  if(err != 0) {  
   printf("can't join with thread 1: %s\n", strerror(err));  
   exit(2);  
  }  
  printf("thread 1 return code: %ld\n", (long)tret);  
   
  sleep(2);  
   
  // Calling thread, waits for the second thread, if it exits,  
  // Catch the exit code of it  
  err = pthread_join(tid2, &tret);  
  if(err != 0) {  
   printf("can't join with thread 2: %s\n", strerror(err));  
   exit(2);  
  }  
  printf("thread 2 exit code: %ld\n", (long)tret);  
   
  exit(0);  
 }  

shell:
After running the program, it outputted the first 3 lines. And then after 2 seconds, it outputted last line.
This proves that even if the newly created thread finished running, it is NOT in detached state.It is still there waiting for the pthread_join to pick up its exit code.

 ubuntu@ip-172-31-23-227:~$ ./thread.out  
 thread 2 exiting  
 thread 1 returning  
 thread 1 return code: 1  
 thread 2 exit code: 2  

3. thread "shared" memory overlapping
Thread can return the address of struct to return more complex information, but we need to make sure that the struct is not allocated from stack. Otherwise, after exiting from the thread, the struct's memory maybe written by sth else.

Bad Example:
thread.c:

 #include<stdio.h>  
 #include<stdlib.h>  
 #include<pthread.h>  
 #include<unistd.h>  
 #include<string.h>  
   
 struct foo {  
  int a, b, c, d;  
 };  
   
 void printfoo(const char *s, const struct foo *fp)  
 {  
  printf("%s", s);  
  printf(" structure at 0x%lx\n", (unsigned long)fp);  
  printf(" foo.a = %d\n", fp->a);  
  printf(" foo.b = %d\n", fp->b);  
  printf(" foo.c = %d\n", fp->c);  
  printf(" foo.d = %d\n", fp->d);  
 }  
   
 void *thr_fn1(void *arg)  
 {  
  struct foo foo = {1, 2, 3, 4};  
  printfoo("thread 1:\n", &foo);  
  pthread_exit((void*)&foo);  
 }  
   
 void *thr_fn2(void *arg)  
 {  
  printf("thread 2: ID is %d\n", (int)pthread_self());  
  pthread_exit((void*)0);  
 }  
   
 int main(int argc, char* argv[])  
 {  
  int err;  
  pthread_t tid1, tid2;  
  struct foo *fp;  
   
  // Launch the first thread  
  err = pthread_create(&tid1, NULL, thr_fn1, NULL);  
  if(err != 0) {  
   printf("Can't create thread 1: %s\n", strerror(err));  
   exit(1);  
  }  
   
  // Wait for the first thread finish  
  err = pthread_join(tid1, (void*)&fp);  
  if(err != 0) {  
   printf("Can't join with thread 1: %s\n", strerror(err));  
   exit(2);  
  }  
   
  sleep(1);  
   
  // Launch the second thread  
  printf("Parent starting the second thread\n");  
  err = pthread_create(&tid2, NULL, thr_fn2, NULL);  
  if(err != 0) {  
   printf("Can't create thread 2: %s\n", strerror(err));  
   exit(1);  
  }  
   
  sleep(1);  
  printfoo("parent: \n", fp);  
  exit(0);  
 }  

shell:
After launching the program, it started the first thread, to print out the local struct foo. And then it returned the address of local struct foo.
Next, the program launched the 2nd thread.
Finally, the program print out the struct foo based on the address returned from first thread. But the value is corrupted there(note the address remain the same), this indicate that. pthread_join make first thread's storage immediately reclaimed, and then used for sth else, which corrupted the data there.

 ubuntu@ip-172-31-23-227:~$ ./thread.out  
 thread 1:  
  structure at 0x7ffc17928f00  
  foo.a = 1  
  foo.b = 2  
  foo.c = 3  
  foo.d = 4  
 Parent starting the second thread  
 thread 2: ID is 395482880  
 parent:  
  structure at 0x7ffc17928f00  
  foo.a = 395482880  
  foo.b = 32764  
  foo.c = 395482880  
  foo.d = 32764  

Unix Prog: Thread Creation

1. pthread_create

When program starts, it only has one thread. Additional thread can be created by system call: pthread_create.

Definition:

 ubuntu@ip-172-31-23-227:~$ less /usr/include/pthread.h  
 ......  
 /* Create a new thread, starting with execution of START-ROUTINE  
   getting passed ARG. Creation attributed come from ATTR. The new  
   handle is stored in *NEWTHREAD. */  
 extern int pthread_create (pthread_t *__restrict __newthread,  
               const pthread_attr_t *__restrict __attr,  
               void *(*__start_routine) (void *),  
               void *__restrict __arg) __THROWNL __nonnull ((1, 3));  
 ......  

1st argument is used to populate the newly created thread id
2nd argument is used to setup the thread attribute
3rd argument is used to setup the starting routine(function)
4th argument is used to setup the routine arguments. If there is more than one argument, need to provide the address of structure containing all arguments.

Note:
When a thread is created, there is no guarantee who runs firstly, calling thread or newly created thread.

The newly created thread will inherits the calling thread's floating-point environment and signal mask, but pending signals are cleared.

2. Example
thread.c:

 #include<stdio.h>  
 #include<stdlib.h>  
 #include<unistd.h>  
 #include<string.h>  
 #include<pthread.h>  
   
 pthread_t ntid;  
   
 void printids(const char *s)  
 {  
  pid_t pid;  
  pthread_t tid;  
   
  pid = getpid();  
  tid = pthread_self();  
  printf("%s pid %u tid %u (0x%x)\n", s, (unsigned int)pid, (unsigned int)tid, (unsigned int)tid);  
 }  
   
 void* thr_fn(void *arg)  
 {  
  printids("new thread: ");  
  return((void *) 0);  
 }  
   
 int main(int argc, char* argv[])  
 {  
  int err;  
   
  // Create the new thread, populate the thread id on ntid,  
  // launch the function thr_fn as the starting routine  
  err = pthread_create(&ntid, NULL, thr_fn, NULL);  
  if(err != 0) {  
   printf("pthread_create error: %s!\n", strerror(err));  
   exit(1);  
  }  
   
  // Print information about main thread, but there is no guarantee  
  // that following output will appear firstly, maybe new thread is executed firstly  
  printids("main thread: ");  
   
  // Main thread sleep for one second to wait for new thread  
  // Otherwise if main thread is finished firstly, before the new thread  
  // is executed, the entire process is terminated.  
  sleep(1);  
  exit(0);  
 }  

shell:
1) We need to provide the pthread library, otherwise pthread_create will not be located.
2) main thread is executed firstly, and then new thread.

 ubuntu@ip-172-31-23-227:~$ gcc -g -pthread thread.c -o thread.out  
 ubuntu@ip-172-31-23-227:~$ ./thread.out  
 main thread: pid 29582 tid 2301646656 (0x89305740)  
 new thread: pid 29582 tid 2293282560 (0x88b0b700)  

Note: the new thread used pthread_self() to get its thread id, instead of referring to global "ntid". Because there is no guarantee main thread, new thread who will run firstly. It is possible that when new thread run, the ntid is still not populated.

In some systems, if pid is different, then that operating system use "clone" system call when calling the "pthread_create", and 2 processes are using some ways to share the execution context.

3. Unix Libraries
When using gcc, if the function or API declared in header files are already defined under gcc lib, then we need to do nothing. Otherwise, we need to specify our own library (path).

The upper part is GNU library, and the lower part is gcc lib.
We need to specify -pthread when using gcc to link, because pthread_create is defined inside libpthread.a which is part of gnu libs.

 ubuntu@ip-172-31-23-227:~$ sudo find /usr -name *.a  
 /usr/lib/libhgfs.a  
 /usr/lib/libguestlib.a  
 /usr/lib/x86_64-linux-gnu/librpcsvc.a  
 /usr/lib/x86_64-linux-gnu/libieee.a  
 /usr/lib/x86_64-linux-gnu/libBrokenLocale.a  
 /usr/lib/x86_64-linux-gnu/libanl.a  
 /usr/lib/x86_64-linux-gnu/libc_nonshared.a  
 /usr/lib/x86_64-linux-gnu/libmcheck.a  
 /usr/lib/x86_64-linux-gnu/libpthread.a  
 /usr/lib/x86_64-linux-gnu/libnsl.a  
 /usr/lib/x86_64-linux-gnu/libdl.a  
 /usr/lib/x86_64-linux-gnu/libresolv.a  
 /usr/lib/x86_64-linux-gnu/libg.a  
 /usr/lib/x86_64-linux-gnu/libpthread_nonshared.a  
 /usr/lib/x86_64-linux-gnu/libc.a  
 /usr/lib/x86_64-linux-gnu/libcrypt.a  
 /usr/lib/x86_64-linux-gnu/libutil.a  
 /usr/lib/x86_64-linux-gnu/librt.a  
 /usr/lib/x86_64-linux-gnu/libm.a  
 /usr/lib/gcc/x86_64-linux-gnu/4.8/libquadmath.a  
 /usr/lib/gcc/x86_64-linux-gnu/4.8/libssp_nonshared.a  
 /usr/lib/gcc/x86_64-linux-gnu/4.8/libitm.a  
 /usr/lib/gcc/x86_64-linux-gnu/4.8/libgcc_eh.a  
 /usr/lib/gcc/x86_64-linux-gnu/4.8/libasan.a  
 /usr/lib/gcc/x86_64-linux-gnu/4.8/libgcc.a  
 /usr/lib/gcc/x86_64-linux-gnu/4.8/libgomp.a  
 /usr/lib/gcc/x86_64-linux-gnu/4.8/libgcov.a  
 /usr/lib/gcc/x86_64-linux-gnu/4.8/libtsan.a  
 /usr/lib/gcc/x86_64-linux-gnu/4.8/libbacktrace.a  
 /usr/lib/gcc/x86_64-linux-gnu/4.8/libatomic.a  
 /usr/lib/libvmtools.a  

Unix Prog: thread identification

1. Thread Concepts

With multiple threads, we can design our program to do more than one thing at a time within a single process.
1) We can assign each thread to handle each event type within the process.
2) Threads automatically have the access to the same memory address space and file descriptors
3) Problems can be partitioned and overall throughput can be improved
4) Interactive program can improve the response time by using multiple threads separate calculation and user IO

The benefits of multi-threading  can be realized even if with uni-processor.

Thread contains context information within the process, including:
1) thread id
2) a set of register values
3) a stack
4) scheduling priority and policy
5) signal mask
6) errno variable
7) thread specific data

2. Thread Identification
Unix use type "pthread_t" to identify the thread, and UNIX implementations are allowed to implement it with struct or integers. So there is no portable way to print out the pthread_t type.

Definition:
In current system, it is unsigned long int.

 ubuntu@ip-172-31-23-227:~$ less /usr/include/x86_64-linux-gnu/bits/pthreadtypes.h  
 ......  
 /* Thread identifiers. The structure of the attribute type is not  
   exposed on purpose. */  
 typedef unsigned long int pthread_t;  
 ......  

And UNIX provide common interface to compare two pthread ids:
If returning 0, means not equal; if returning non-0, means equal

 ubuntu@ip-172-31-23-227:~$ less /usr/include/pthread.h  
 ......  
 /* Compare two thread identifiers. */  
 extern int pthread_equal (pthread_t __thread1, pthread_t __thread2)  
  __THROW __attribute__ ((__const__));  
 ......  

And the way to get current pthread id:

 ubuntu@ip-172-31-23-227:~$ less /usr/include/pthread.h  
 ......  
 /* Obtain the identifier of the current thread. */  
 extern pthread_t pthread_self (void) __THROW __attribute__ ((__const__));  
 ......  

3. Example:
thread.c:

 #include<stdio.h>  
 #include<stdlib.h>  
 #include<unistd.h>  
 #include<pthread.h>  
   
 int main(int argc, char* argv[])  
 {  
  pthread_t tid = 0;  
  printf("Current thread id: %ld\n", pthread_self());  
  if(pthread_equal(tid, pthread_self()) != 0) {  
   printf("Current thread id is equal to 0\n");  
  }  
  else {  
   printf("Current thread id is not equal to 0\n");  
  }  
  exit(0);  
 }  

shell:

 ubuntu@ip-172-31-23-227:~$ ./thread.out  
 Current thread id: 0  
 Current thread id is equal to 0  

Unix Prog: Signal Reference APIs

1. sys_siglist

Definition:

 ubuntu@ip-172-31-23-227:~$ less /usr/include/signal.h  
 ......  
 /* Names of the signals. This variable exists only for compatibility.  
   Use `strsignal' instead (see <string.h>). */  
 extern const char *const _sys_siglist[_NSIG];  
 extern const char *const sys_siglist[_NSIG];  
 ......  

Example:
siglist.c:

 #include<stdio.h>  
 #include<stdlib.h>  
 #include<unistd.h>  
 #include<signal.h>  
   
 int main(int argc, char* argv[])  
 {  
  printf("SIGCONT: %s\n", sys_siglist[SIGCONT]);  
  printf("SIGCHLD: %s\n", sys_siglist[SIGCHLD]);  
  exit(0);  
 }  

shell:

 ubuntu@ip-172-31-23-227:~$ ./siglist.out  
 SIGCONT: Continued  
 SIGCHLD: Child exited  

2. psignal

Definition:

 ubuntu@ip-172-31-23-227:~$ less /usr/include/signal.h  
 ......  
 /* Print a message describing the meaning of the given signal number. */  
 extern void psignal (int __sig, const char *__s);  
 ......  

Example:
psignal.c:

 #include<stdio.h>  
 #include<stdlib.h>  
 #include<unistd.h>  
 #include<signal.h>  
   
 int main(int argc, char* argv[])  
 {  
  psignal(SIGCONT, "SIGCONT:");  
  psignal(SIGCHLD, "SIGCHLD:");  
   
  exit(0);  
 }  

shell:

 ubuntu@ip-172-31-23-227:~$ ./psignal.out  
 SIGCONT:: Continued  
 SIGCHLD:: Child exited  

3. strsignal

System Definition:

 ubuntu@ip-172-31-23-227:~$ less /usr/include/string.h  
 ......  
 /* Return a string describing the meaning of the signal number in SIG. */  
 extern char *strsignal (int __sig) __THROW;  
 ......  

Example:
strsig.c:

 #include<stdio.h>  
 #include<stdlib.h>  
 #include<string.h>  
 #include<signal.h>  
   
 int main(int argc, char* argv[])  
 {  
  char *p = strsignal(SIGCONT);  
  printf("SIGCONT: %s\n", p);  
   
  p = strsignal(SIGCHLD);  
  printf("SIGCHLD: %s\n", p);  
   
  exit(0);  
 }  

shell:

 ubuntu@ip-172-31-23-227:~$ ./strsig.out  
 SIGCONT: Continued  
 SIGCHLD: Child exited  

4. sig2str, str2sig

Definition:

Unix Prog: Job-Control Signals

1. Job-Control Signals

SIGCHLD, child process has stopped or continued.
SIGCONT, continue process, if stopped.
SIGSTOP, stop signal(can't be caught or ignored)
SIGTSTP, Interactive stop signal
SIGTTIN, Read from controlling terminal by member of a background process group
SIGTTOU, Write to controlling terminal by member of a background process group

Except for SIGCHLD, most applications don't handle the job-control signals, the shell itself will handle them. For example:

1) When typing the suspend character(ctrl-z), SIGTSTP is sent to all processes in the foreground process group.
2) When telling the shell to resume a job in the foreground or background, the shell sends all the processes in the job a SIGCONT signal.
3) If SIGTTIN or SIGTTOU is delivered to a process by shell, the process is stopped by default, and the job-control shell recognizes this and notifies us.

Interaction of job-control signals:
When any of the four stop signals(SIGTSTP, SIGSTOP, SIGTTIN, or SIGTTOU) is generated for a process, any pending SIGCONT signal for that process is discarded.

When SIGCONT signal is generated, any pending stop signals are discarded.

2. Example of job-control signal handling:

tstp.c:

 #include<stdio.h>  
 #include<stdlib.h>  
 #include<unistd.h>  
 #include<signal.h>  
   
 #define BUFFSIZE 1024  
   
 void sig_tstp(int signo)  
 {  
  sigset_t mask;  
   
  printf("\ncatch the stop signal. \n");  
   
  /* Before stopping the process, we can use code over here to re-setup  
   * the terminal. So user could do other work after this interactive app  
   * is stopped  
   */  
   
  // Unblock the SIGTSTP signal  
  sigemptyset(&mask);  
  sigaddset(&mask, SIGTSTP);  
  sigprocmask(SIG_UNBLOCK, &mask, NULL);  
   
  // Set the SIGTSTP signal back to default way, stop the process  
  signal(SIGTSTP, SIG_DFL);  
  kill(getpid(), SIGTSTP);  
   
  // Then the process get stopped, it won't return back from kill until  
  // receiving one SIGCONT signal  
   
  signal(SIGTSTP, sig_tstp);  
   
  /* After the process is continued, we can use code over there to re-clean  
   * the screen. So user could continue using the interactive app  
   */  
  printf("\ncatch the continue signal. \n");  
 }  
   
 int main(int argc, char* argv[])  
 {  
  int n;  
  char buf[BUFFSIZE];  
   
  // only catch the SIGTSTP if we're running with a job-control shell  
  /* Note, init process will initialize the handler of SIGTSTP to be SIG_IGN,  
   * and job-control shell will re-setup it to be SIG_DFL)  
   */  
  if(signal(SIGTSTP, SIG_IGN) == SIG_DFL)  
   signal(SIGTSTP, sig_tstp);  
   
  while((n = read(STDIN_FILENO, buf, BUFFSIZE)) > 0)  
   if(write(STDOUT_FILENO, buf, n) != n) {  
    printf("write error!\n");  
    exit(1);  
   }  
   
  if(n < 0) {  
   printf("read error!\n");  
   exit(2);  
  }  
   
  exit(0);  
 }  

shell:
Run the tstp.out, after typing the Ctrl Z to trigger the SIGTSTP signal, the signal handler catch the stop signal and stuck on "kill(getpid(), SIGTSTP)".
Run the fg command, to send SIGCONT signal, then "kill" system call return and re-setup the SIGTSTP handler.

 ubuntu@ip-172-31-23-227:~$ ./tstp.out  
 Hello world!  
 Hello world!  
 ^Z  
 catch the stop signal.  
   
 [2]+ Stopped         ./tstp.out  
 ubuntu@ip-172-31-23-227:~$ fg  
 ./tstp.out  
   
 catch the continue signal.  
 Hello world!  
 Hello world!  
 ^C  

Unix Prog: sleep function and signals

1. Implement sleep using sigalrm

sleep.c:

 #include<stdio.h>  
 #include<stdlib.h>  
 #include<unistd.h>  
 #include<signal.h>  
   
 void sig_alrm(int signo)  
 {}  
   
 unsigned int sleep(unsigned int nsecs)  
 {  
  struct sigaction newact, oldact;  
  sigset_t newmask, oldmask, suspmask;  
  unsigned int unslept;  
   
  // Setup the SIGALRM handler  
  newact.sa_handler = sig_alrm;  
  sigemptyset(&newact.sa_mask);  
  newact.sa_flags = 0;  
  sigaction(SIGALRM, &newact, &oldact);  
   
  // Block SIGALRM  
  sigemptyset(&newmask);  
  sigaddset(&newmask, SIGALRM);  
  sigprocmask(SIG_BLOCK, &newmask, &oldmask);  
   
  alarm(nsecs);  
   
  // Make suspend mask, make sure SIGALRM is unblocked  
  suspmask = oldmask;  
  sigdelset(&suspmask, SIGALRM);  
  sigsuspend(&suspmask);  
   
  // Restore SIGALRM action, restore signal mask  
  unslept = alarm(0);  
  sigaction(SIGALRM, &oldact, NULL);  
   
  sigprocmask(SIG_SETMASK, &oldmask, NULL);  
  return(unslept);  
 }  
   
 int main(int argc, char* argv[])  
 {  
  sleep(3);  
 }  

2. Problem of "sleep with SIGALRM"
Since it use alarm function to generate the SIGALRM signal.
But what if user is trying to call more than one alarm function across different system calls. At some operating system, the second alarm function will cancel the previous one.
But in above case, we are using sigsuspend to stop the process, so there is no way to run another sleep function with alarm which can cancel previous alarm.

Unix Prog: Return Value of System

test.c:

 #include<stdio.h>  
 #include<stdlib.h>  
 #include<unistd.h>  
 #include<signal.h>  
   
 int main(int argc, char* argv[])  
 {  
  sleep(5);  
  return 88;  
 }  

system.c:

 #include<stdio.h>  
 #include<stdlib.h>  
 #include<unistd.h>  
   
 int main(int argc, char* argv[])  
 {  
  int status = system("./test.out");  
  printf("status code: %d\n", status);  
  exit(0);  
 }  

1. Execute from the shell
1) Run the test.out directly from the shell, the return value is one returned in test.c
2) Run the test.out, and interrupt it with Ctrl C. The return value is the one assigned by system when meeting with interrupt signal to terminate the process

 ubuntu@ip-172-31-23-227:~$ ./test.out  
 ubuntu@ip-172-31-23-227:~$ echo $?  
 88  
 ubuntu@ip-172-31-23-227:~$ ./test.out  
 ^C  
 ubuntu@ip-172-31-23-227:~$ echo $?  
 130  

2. Execute by the shell
Same as above

 ubuntu@ip-172-31-23-227:~$ echo $?  
 88  
 ubuntu@ip-172-31-23-227:~$ sh -c "./test.out"  
 ^C  
 ubuntu@ip-172-31-23-227:~$ echo $?  
 130  

3. Execute by system function
1) Run the system.out to run test.out with system function, and the value returned is not the one returned in test.c. Instead, it is the value returned by "shell", launched by system.
2) Run the system.out to run test.out with system function, and interrupt it in the middle. Then the value returned is not the one returned in test.c. Instead, it is the value returned by "shell", launched by system, and it is the value of 2(SIGINT number).

 ubuntu@ip-172-31-23-227:~$ ./system.out  
 status code: 22528  
 ubuntu@ip-172-31-23-227:~$ ./system.out  
 ^Cstatus code: 2  

Unix Prog: system function and signals

1. Signals from system

system.c:

 #include<stdio.h>  
 #include<stdlib.h>  
 #include<signal.h>  
 #include<unistd.h>  
   
 void sig_int(int signo)  
 {  
  printf("caught SIGINT\n");  
 }  
   
 void sig_chld(int signo)  
 {  
  printf("caught SIGCHLD\n");  
 }  
   
 int main(int argc, char* argv[])  
 {  
  if(signal(SIGINT, sig_int) == SIG_ERR) {  
   printf("signal error!\n");  
   exit(1);  
  }  
   
  if(signal(SIGCHLD, sig_chld) == SIG_ERR) {  
   printf("signal error!\n");  
   exit(1);  
  }  
   
  if(system("date") < 0) {  
   printf("system() error!\n");  
   exit(2);  
  }  
   
  exit(0);  
 }  

shell:
After system exits, child process sent the SIGCHLD back to parent process.
This is how system is implemented at current system. But actually system should block SIGCHLD at best.

 ubuntu@ip-172-31-23-227:~$ ./system.out  
 Sun Oct 19 15:53:33 UTC 2014  
 caught SIGCHLD  

2. system implementation
system.c:

 #include<stdio.h>  
 #include<stdlib.h>  
 #include<unistd.h>  
 #include<signal.h>  
 #include<sys/wait.h>  
 #include<errno.h>  
   
 int my_system(const char* cmdstring)  
 {  
  pid_t pid;  
  int status;  
  struct sigaction ignore, saveintr, savequit;  
  sigset_t chldmask, savemask;  
   
  if(cmdstring == NULL) return 1;  
   
  // Setup sigaction for SIGINT, and SIGQUIT to be SIG_IGN  
  ignore.sa_handler = SIG_IGN;  
  sigemptyset(&ignore.sa_mask);  
  ignore.sa_flags = 0;  
  if(sigaction(SIGINT, &ignore, &saveintr) < 0)  
   return -1;  
  if(sigaction(SIGQUIT, &ignore, &savequit) < 0)  
   return -1;  
   
  // Block the SIGCHLD signal  
  sigemptyset(&chldmask);  
  sigaddset(&chldmask, SIGCHLD);  
  if(sigprocmask(SIG_BLOCK, &chldmask, &savemask) < 0)  
   return -1;  
   
  // Launch the child process  
  if((pid = fork()) < 0) {  
   status = -1;  
  } else if (pid == 0) {  
   // In child process, restore the SIGINT, SIGQUIT handler  
   // restore the signal mask. So child process can accept all signals  
   sigaction(SIGINT, &saveintr, NULL);  
   sigaction(SIGQUIT, &savequit, NULL);  
   sigprocmask(SIG_SETMASK, &savemask, NULL);  
   
   execl("/bin/sh", "sh", "-c", cmdstring, (char*) 0);  
   _exit(127);  
  } else {  
   // In parent process, wait for the child process complete  
   while(waitpid(pid, &status, 0) < 0) {  
    if(errno != EINTR) {  
     status = -1;  
     break;  
    }  
   }  
  }  
   
  // Restore the SIGINT, SIGQUIT action, and restore the signal mask  
  if(sigaction(SIGINT, &saveintr, NULL) < 0)  
   return -1;  
  if(sigaction(SIGQUIT, &savequit, NULL) < 0)  
   return -1;  
  if(sigprocmask(SIG_SETMASK, &savemask, NULL) < 0)  
   return -1;  
   
  return status;  
 }  
   
 int main(int argc, char* argv[])  
 {  
  my_system("date");  
  exit(0);  
 }  

shell:

 ubuntu@ip-172-31-23-227:~$ ./system.out  
 Sun Oct 19 18:19:50 UTC 2014