boost_1_45_0/tools/build/v2/engine/src/boehm_gc/pthread_support.c

/* 
 * Copyright (c) 1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1998 by Fergus Henderson.  All rights reserved.
 * Copyright (c) 2000-2005 by Hewlett-Packard Company.  All rights reserved.
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose,  provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */
/*
 * Support code originally for LinuxThreads, the clone()-based kernel
 * thread package for Linux which is included in libc6.
 *
 * This code no doubt makes some assumptions beyond what is
 * guaranteed by the pthread standard, though it now does
 * very little of that.  It now also supports NPTL, and many
 * other Posix thread implementations.  We are trying to merge
 * all flavors of pthread dupport code into this file.
 */
 /* DG/UX ix86 support <takis@xfree86.org> */
/*
 * Linux_threads.c now also includes some code to support HPUX and
 * OSF1 (Compaq Tru64 Unix, really).  The OSF1 support is based on Eric Benson's
 * patch.
 *
 * Eric also suggested an alternate basis for a lock implementation in
 * his code:
 * + #elif defined(OSF1)
 * +    unsigned long GC_allocate_lock = 0;
 * +    msemaphore GC_allocate_semaphore;
 * + #  define GC_TRY_LOCK() \
 * +    ((msem_lock(&GC_allocate_semaphore, MSEM_IF_NOWAIT) == 0) \
 * +     ? (GC_allocate_lock = 1) \
 * +     : 0)
 * + #  define GC_LOCK_TAKEN GC_allocate_lock
 */

/*#define DEBUG_THREADS 1*/

# include "private/pthread_support.h"

# if defined(GC_PTHREADS) && !defined(GC_WIN32_THREADS)

# if defined(GC_DGUX386_THREADS) && !defined(_POSIX4A_DRAFT10_SOURCE)
#   define _POSIX4A_DRAFT10_SOURCE 1
# endif

# if defined(GC_DGUX386_THREADS) && !defined(_USING_POSIX4A_DRAFT10)
#   define _USING_POSIX4A_DRAFT10 1
# endif

# include <stdlib.h>
# include <pthread.h>
# include <sched.h>
# include <time.h>
# include <errno.h>
# include <unistd.h>
# include <sys/mman.h>
# include <sys/time.h>
# include <sys/types.h>
# include <sys/stat.h>
# include <fcntl.h>
# include <signal.h>

# include "gc_inline.h"

#if defined(GC_DARWIN_THREADS)
# include "private/darwin_semaphore.h"
#else
# include <semaphore.h>
#endif /* !GC_DARWIN_THREADS */

#if defined(GC_DARWIN_THREADS) || defined(GC_FREEBSD_THREADS)
# include <sys/sysctl.h>
#endif /* GC_DARWIN_THREADS */

#if defined(GC_NETBSD_THREADS)
# include <sys/param.h>
# include <sys/sysctl.h>
#endif        /* GC_NETBSD_THREADS */

/* Allocator lock definitions.		*/
#if !defined(USE_SPIN_LOCK)
  pthread_mutex_t GC_allocate_ml = PTHREAD_MUTEX_INITIALIZER;
#endif
unsigned long GC_lock_holder = NO_THREAD;
		/* Used only for assertions, and to prevent	 */
		/* recursive reentry in the system call wrapper. */

#if defined(GC_DGUX386_THREADS)
# include <sys/dg_sys_info.h>
# include <sys/_int_psem.h>
  /* sem_t is an uint in DG/UX */
  typedef unsigned int  sem_t;
#endif /* GC_DGUX386_THREADS */

#ifndef __GNUC__
#   define __inline__
#endif

/* Undefine macros used to redirect pthread primitives. */
# undef pthread_create
# if !defined(GC_DARWIN_THREADS)
#   undef pthread_sigmask
# endif
# undef pthread_join
# undef pthread_detach
# if defined(GC_OSF1_THREADS) && defined(_PTHREAD_USE_MANGLED_NAMES_) \
     && !defined(_PTHREAD_USE_PTDNAM_)
  /* Restore the original mangled names on Tru64 UNIX.  */
#   define pthread_create __pthread_create
#   define pthread_join __pthread_join
#   define pthread_detach __pthread_detach
# endif

#ifdef GC_USE_LD_WRAP
#   define WRAP_FUNC(f) __wrap_##f
#   define REAL_FUNC(f) __real_##f
#else
#   ifdef GC_USE_DLOPEN_WRAP
#     include <dlfcn.h>
#     define WRAP_FUNC(f) f
#     define REAL_FUNC(f) GC_real_##f
      /* We define both GC_f and plain f to be the wrapped function.	*/
      /* In that way plain calls work, as do calls from files that	*/
      /* included gc.h, wich redefined f to GC_f.			*/
      /* FIXME: Needs work for DARWIN and True64 (OSF1) */
      typedef int (* GC_pthread_create_t)(pthread_t *, const pthread_attr_t *,
		      		          void * (*)(void *), void *);
      static GC_pthread_create_t GC_real_pthread_create;
      typedef int (* GC_pthread_sigmask_t)(int, const sigset_t *, sigset_t *);
      static GC_pthread_sigmask_t GC_real_pthread_sigmask;
      typedef int (* GC_pthread_join_t)(pthread_t, void **);
      static GC_pthread_join_t GC_real_pthread_join;
      typedef int (* GC_pthread_detach_t)(pthread_t);
      static GC_pthread_detach_t GC_real_pthread_detach;
#   else
#     define WRAP_FUNC(f) GC_##f
#     if !defined(GC_DGUX386_THREADS)
#       define REAL_FUNC(f) f
#     else /* GC_DGUX386_THREADS */
#       define REAL_FUNC(f) __d10_##f
#     endif /* GC_DGUX386_THREADS */
#   endif
#endif

#if defined(GC_USE_DL_WRAP) || defined(GC_USE_DLOPEN_WRAP)
/* Define GC_ functions as aliases for the plain ones, which will	*/
/* be intercepted.  This allows files which include gc.h, and hence	*/
/* generate referemces to the GC_ symbols, to see the right symbols.	*/
      int GC_pthread_create(pthread_t * t, const pthread_attr_t * a,
		         void * (* fn)(void *), void * arg) {
	  return pthread_create(t, a, fn, arg);
      }
      int GC_pthread_sigmask(int how, const sigset_t *mask, sigset_t *old) {
	  return pthread_sigmask(how, mask, old);
      }
      int GC_pthread_join(pthread_t t, void **res) {
	  return pthread_join(t, res);
      }
      int GC_pthread_detach(pthread_t t) {
	  return pthread_detach(t);
      }
#endif /* Linker-based interception. */

#ifdef GC_USE_DLOPEN_WRAP
  static GC_bool GC_syms_initialized = FALSE;

  void GC_init_real_syms(void)
  {
    void *dl_handle;
#   define LIBPTHREAD_NAME "libpthread.so.0"
#   define LIBPTHREAD_NAME_LEN 16 /* incl. trailing 0 */
    size_t len = LIBPTHREAD_NAME_LEN - 1;
    char namebuf[LIBPTHREAD_NAME_LEN];
    static char *libpthread_name = LIBPTHREAD_NAME;

    if (GC_syms_initialized) return;
#   ifdef RTLD_NEXT
      dl_handle = RTLD_NEXT;
#   else
      dl_handle = dlopen(libpthread_name, RTLD_LAZY);
      if (NULL == dl_handle) {
        while (isdigit(libpthread_name[len-1])) --len;
        if (libpthread_name[len-1] == '.') --len;
        memcpy(namebuf, libpthread_name, len);
        namebuf[len] = '\0';
        dl_handle = dlopen(namebuf, RTLD_LAZY);
      }
      if (NULL == dl_handle) ABORT("Couldn't open libpthread\n");
#   endif
    GC_real_pthread_create = (GC_pthread_create_t)
	    			dlsym(dl_handle, "pthread_create");
    GC_real_pthread_sigmask = (GC_pthread_sigmask_t)
	    			dlsym(dl_handle, "pthread_sigmask");
    GC_real_pthread_join = (GC_pthread_join_t)
	    			dlsym(dl_handle, "pthread_join");
    GC_real_pthread_detach = (GC_pthread_detach_t)
	    			dlsym(dl_handle, "pthread_detach");
    GC_syms_initialized = TRUE;
  }

# define INIT_REAL_SYMS() if (!GC_syms_initialized) GC_init_real_syms();
#else
# define INIT_REAL_SYMS()
#endif

void GC_thr_init(void);

static GC_bool parallel_initialized = FALSE;

GC_bool GC_need_to_lock = FALSE;

void GC_init_parallel(void);

long GC_nprocs = 1;	/* Number of processors.  We may not have	*/
			/* access to all of them, but this is as good	*/
			/* a guess as any ...				*/

#ifdef THREAD_LOCAL_ALLOC
/* We must explicitly mark ptrfree and gcj free lists, since the free 	*/
/* list links wouldn't otherwise be found.  We also set them in the 	*/
/* normal free lists, since that involves touching less memory than if	*/
/* we scanned them normally.						*/
void GC_mark_thread_local_free_lists(void)
{
    int i;
    GC_thread p;
    
    for (i = 0; i < THREAD_TABLE_SZ; ++i) {
      for (p = GC_threads[i]; 0 != p; p = p -> next) {
	GC_mark_thread_local_fls_for(&(p->tlfs));
      }
    }
}

#if defined(GC_ASSERTIONS)
    /* Check that all thread-local free-lists are completely marked.	*/
    /* also check that thread-specific-data structures are marked.	*/
    void GC_check_tls(void) {
	int i;
	GC_thread p;
	
	for (i = 0; i < THREAD_TABLE_SZ; ++i) {
	  for (p = GC_threads[i]; 0 != p; p = p -> next) {
	    GC_check_tls_for(&(p->tlfs));
	  }
	}
#       if defined(USE_CUSTOM_SPECIFIC)
	  if (GC_thread_key != 0)
	    GC_check_tsd_marks(GC_thread_key);
#	endif 
    }
#endif /* GC_ASSERTIONS */

#endif /* Thread_local_alloc */

#ifdef PARALLEL_MARK

# ifndef MAX_MARKERS
#   define MAX_MARKERS 16
# endif

static ptr_t marker_sp[MAX_MARKERS] = {0};
#ifdef IA64
  static ptr_t marker_bsp[MAX_MARKERS] = {0};
#endif

void * GC_mark_thread(void * id)
{
  word my_mark_no = 0;

  marker_sp[(word)id] = GC_approx_sp();
# ifdef IA64
    marker_bsp[(word)id] = GC_save_regs_in_stack();
# endif
  for (;; ++my_mark_no) {
    /* GC_mark_no is passed only to allow GC_help_marker to terminate	*/
    /* promptly.  This is important if it were called from the signal	*/
    /* handler or from the GC lock acquisition code.  Under Linux, it's	*/
    /* not safe to call it from a signal handler, since it uses mutexes	*/
    /* and condition variables.  Since it is called only here, the 	*/
    /* argument is unnecessary.						*/
    if (my_mark_no < GC_mark_no || my_mark_no > GC_mark_no + 2) {
	/* resynchronize if we get far off, e.g. because GC_mark_no	*/
	/* wrapped.							*/
	my_mark_no = GC_mark_no;
    }
#   ifdef DEBUG_THREADS
	GC_printf("Starting mark helper for mark number %lu\n", my_mark_no);
#   endif
    GC_help_marker(my_mark_no);
  }
}

extern long GC_markers;		/* Number of mark threads we would	*/
				/* like to have.  Includes the 		*/
				/* initiating thread.			*/

pthread_t GC_mark_threads[MAX_MARKERS];

#define PTHREAD_CREATE REAL_FUNC(pthread_create)

static void start_mark_threads(void)
{
    unsigned i;
    pthread_attr_t attr;

    if (GC_markers > MAX_MARKERS) {
	WARN("Limiting number of mark threads\n", 0);
	GC_markers = MAX_MARKERS;
    }
    if (0 != pthread_attr_init(&attr)) ABORT("pthread_attr_init failed");
	
    if (0 != pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED))
	ABORT("pthread_attr_setdetachstate failed");

#   if defined(HPUX) || defined(GC_DGUX386_THREADS)
      /* Default stack size is usually too small: fix it. */
      /* Otherwise marker threads or GC may run out of	  */
      /* space.						  */
#     define MIN_STACK_SIZE (8*HBLKSIZE*sizeof(word))
      {
	size_t old_size;
	int code;

        if (pthread_attr_getstacksize(&attr, &old_size) != 0)
	  ABORT("pthread_attr_getstacksize failed\n");
	if (old_size < MIN_STACK_SIZE) {
	  if (pthread_attr_setstacksize(&attr, MIN_STACK_SIZE) != 0)
		  ABORT("pthread_attr_setstacksize failed\n");
	}
      }
#   endif /* HPUX || GC_DGUX386_THREADS */
    if (GC_print_stats) {
	GC_log_printf("Starting %ld marker threads\n", GC_markers - 1);
    }
    for (i = 0; i < GC_markers - 1; ++i) {
      if (0 != PTHREAD_CREATE(GC_mark_threads + i, &attr,
			      GC_mark_thread, (void *)(word)i)) {
	WARN("Marker thread creation failed, errno = %ld.\n", errno);
      }
    }
}

#endif /* PARALLEL_MARK */

GC_bool GC_thr_initialized = FALSE;

volatile GC_thread GC_threads[THREAD_TABLE_SZ];

void GC_push_thread_structures(void)
{
    GC_ASSERT(I_HOLD_LOCK());
    GC_push_all((ptr_t)(GC_threads), (ptr_t)(GC_threads)+sizeof(GC_threads));
#   if defined(THREAD_LOCAL_ALLOC)
      GC_push_all((ptr_t)(&GC_thread_key),
	  (ptr_t)(&GC_thread_key)+sizeof(&GC_thread_key));
#   endif
}

/* It may not be safe to allocate when we register the first thread.	*/
static struct GC_Thread_Rep first_thread;

/* Add a thread to GC_threads.  We assume it wasn't already there.	*/
/* Caller holds allocation lock.					*/
GC_thread GC_new_thread(pthread_t id)
{
    int hv = NUMERIC_THREAD_ID(id) % THREAD_TABLE_SZ;
    GC_thread result;
    static GC_bool first_thread_used = FALSE;
    
    GC_ASSERT(I_HOLD_LOCK());
    if (!first_thread_used) {
    	result = &first_thread;
    	first_thread_used = TRUE;
    } else {
        result = (struct GC_Thread_Rep *)
        	 GC_INTERNAL_MALLOC(sizeof(struct GC_Thread_Rep), NORMAL);
	GC_ASSERT(result -> flags == 0);
    }
    if (result == 0) return(0);
    result -> id = id;
    result -> next = GC_threads[hv];
    GC_threads[hv] = result;
    GC_ASSERT(result -> flags == 0 && result -> thread_blocked == 0);
    return(result);
}

/* Delete a thread from GC_threads.  We assume it is there.	*/
/* (The code intentionally traps if it wasn't.)			*/
void GC_delete_thread(pthread_t id)
{
    int hv = NUMERIC_THREAD_ID(id) % THREAD_TABLE_SZ;
    register GC_thread p = GC_threads[hv];
    register GC_thread prev = 0;
    
    GC_ASSERT(I_HOLD_LOCK());
    while (!THREAD_EQUAL(p -> id, id)) {
        prev = p;
        p = p -> next;
    }
    if (prev == 0) {
        GC_threads[hv] = p -> next;
    } else {
        prev -> next = p -> next;
    }
#   ifdef GC_DARWIN_THREADS
	mach_port_deallocate(mach_task_self(), p->stop_info.mach_thread);
#   endif
    GC_INTERNAL_FREE(p);
}

/* If a thread has been joined, but we have not yet		*/
/* been notified, then there may be more than one thread 	*/
/* in the table with the same pthread id.			*/
/* This is OK, but we need a way to delete a specific one.	*/
void GC_delete_gc_thread(GC_thread gc_id)
{
    pthread_t id = gc_id -> id;
    int hv = NUMERIC_THREAD_ID(id) % THREAD_TABLE_SZ;
    register GC_thread p = GC_threads[hv];
    register GC_thread prev = 0;

    GC_ASSERT(I_HOLD_LOCK());
    while (p != gc_id) {
        prev = p;
        p = p -> next;
    }
    if (prev == 0) {
        GC_threads[hv] = p -> next;
    } else {
        prev -> next = p -> next;
    }
#   ifdef GC_DARWIN_THREADS
	mach_port_deallocate(mach_task_self(), p->stop_info.mach_thread);
#   endif
    GC_INTERNAL_FREE(p);
}

/* Return a GC_thread corresponding to a given pthread_t.	*/
/* Returns 0 if it's not there.					*/
/* Caller holds  allocation lock or otherwise inhibits 		*/
/* updates.							*/
/* If there is more than one thread with the given id we 	*/
/* return the most recent one.					*/
GC_thread GC_lookup_thread(pthread_t id)
{
    int hv = NUMERIC_THREAD_ID(id) % THREAD_TABLE_SZ;
    register GC_thread p = GC_threads[hv];
    
    while (p != 0 && !THREAD_EQUAL(p -> id, id)) p = p -> next;
    return(p);
}

#ifdef HANDLE_FORK
/* Remove all entries from the GC_threads table, except the	*/
/* one for the current thread.  We need to do this in the child	*/
/* process after a fork(), since only the current thread 	*/
/* survives in the child.					*/
void GC_remove_all_threads_but_me(void)
{
    pthread_t self = pthread_self();
    int hv;
    GC_thread p, next, me;

    for (hv = 0; hv < THREAD_TABLE_SZ; ++hv) {
      me = 0;
      for (p = GC_threads[hv]; 0 != p; p = next) {
	next = p -> next;
	if (THREAD_EQUAL(p -> id, self)) {
	  me = p;
	  p -> next = 0;
	} else {
#	  ifdef THREAD_LOCAL_ALLOC
	    if (!(p -> flags & FINISHED)) {
	      GC_destroy_thread_local(&(p->tlfs));
	    }
#	  endif /* THREAD_LOCAL_ALLOC */
	  if (p != &first_thread) GC_INTERNAL_FREE(p);
	}
      }
      GC_threads[hv] = me;
    }
}
#endif /* HANDLE_FORK */

#ifdef USE_PROC_FOR_LIBRARIES
GC_bool GC_segment_is_thread_stack(ptr_t lo, ptr_t hi)
{
    int i;
    GC_thread p;
    
    GC_ASSERT(I_HOLD_LOCK());
#   ifdef PARALLEL_MARK
      for (i = 0; i < GC_markers; ++i) {
	if (marker_sp[i] > lo & marker_sp[i] < hi) return TRUE;
#       ifdef IA64
	  if (marker_bsp[i] > lo & marker_bsp[i] < hi) return TRUE;
#	endif
      }
#   endif
    for (i = 0; i < THREAD_TABLE_SZ; i++) {
      for (p = GC_threads[i]; p != 0; p = p -> next) {
	if (0 != p -> stack_end) {
#	  ifdef STACK_GROWS_UP
            if (p -> stack_end >= lo && p -> stack_end < hi) return TRUE;
#	  else /* STACK_GROWS_DOWN */
            if (p -> stack_end > lo && p -> stack_end <= hi) return TRUE;
#	  endif
	}
      }
    }
    return FALSE;
}
#endif /* USE_PROC_FOR_LIBRARIES */

#ifdef IA64
/* Find the largest stack_base smaller than bound.  May be used	*/
/* to find the boundary between a register stack and adjacent	*/
/* immediately preceding memory stack.				*/
ptr_t GC_greatest_stack_base_below(ptr_t bound)
{
    int i;
    GC_thread p;
    ptr_t result = 0;
    
    GC_ASSERT(I_HOLD_LOCK());
#   ifdef PARALLEL_MARK
      for (i = 0; i < GC_markers; ++i) {
	if (marker_sp[i] > result && marker_sp[i] < bound)
	  result = marker_sp[i];
      }
#   endif
    for (i = 0; i < THREAD_TABLE_SZ; i++) {
      for (p = GC_threads[i]; p != 0; p = p -> next) {
	if (p -> stack_end > result && p -> stack_end < bound) {
	  result = p -> stack_end;
	}
      }
    }
    return result;
}
#endif /* IA64 */

#ifdef GC_LINUX_THREADS
/* Return the number of processors, or i<= 0 if it can't be determined.	*/
int GC_get_nprocs(void)
{
    /* Should be "return sysconf(_SC_NPROCESSORS_ONLN);" but that	*/
    /* appears to be buggy in many cases.				*/
    /* We look for lines "cpu<n>" in /proc/stat.			*/
#   define STAT_BUF_SIZE 4096
#   define STAT_READ read
	/* If read is wrapped, this may need to be redefined to call 	*/
	/* the real one.						*/
    char stat_buf[STAT_BUF_SIZE];
    int f;
    word result = 1;
	/* Some old kernels only have a single "cpu nnnn ..."	*/
	/* entry in /proc/stat.  We identify those as 		*/
	/* uniprocessors.					*/
    size_t i, len = 0;

    f = open("/proc/stat", O_RDONLY);
    if (f < 0 || (len = STAT_READ(f, stat_buf, STAT_BUF_SIZE)) < 100) {
	WARN("Couldn't read /proc/stat\n", 0);
	return -1;
    }
    for (i = 0; i < len - 100; ++i) {
        if (stat_buf[i] == '\n' && stat_buf[i+1] == 'c'
	    && stat_buf[i+2] == 'p' && stat_buf[i+3] == 'u') {
	    int cpu_no = atoi(stat_buf + i + 4);
	    if (cpu_no >= result) result = cpu_no + 1;
	}
    }
    close(f);
    return result;
}
#endif /* GC_LINUX_THREADS */

/* We hold the GC lock.  Wait until an in-progress GC has finished.	*/
/* Repeatedly RELEASES GC LOCK in order to wait.			*/
/* If wait_for_all is true, then we exit with the GC lock held and no	*/
/* collection in progress; otherwise we just wait for the current GC	*/
/* to finish.								*/
extern GC_bool GC_collection_in_progress(void);
void GC_wait_for_gc_completion(GC_bool wait_for_all)
{
    GC_ASSERT(I_HOLD_LOCK());
    if (GC_incremental && GC_collection_in_progress()) {
	int old_gc_no = GC_gc_no;

	/* Make sure that no part of our stack is still on the mark stack, */
	/* since it's about to be unmapped.				   */
	while (GC_incremental && GC_collection_in_progress()
	       && (wait_for_all || old_gc_no == GC_gc_no)) {
	    ENTER_GC();
	    GC_in_thread_creation = TRUE;
            GC_collect_a_little_inner(1);
	    GC_in_thread_creation = FALSE;
	    EXIT_GC();
	    UNLOCK();
	    sched_yield();
	    LOCK();
	}
    }
}

#ifdef HANDLE_FORK
/* Procedures called before and after a fork.  The goal here is to make */
/* it safe to call GC_malloc() in a forked child.  It's unclear that is	*/
/* attainable, since the single UNIX spec seems to imply that one 	*/
/* should only call async-signal-safe functions, and we probably can't	*/
/* quite guarantee that.  But we give it our best shot.  (That same	*/
/* spec also implies that it's not safe to call the system malloc	*/
/* between fork() and exec().  Thus we're doing no worse than it.	*/

/* Called before a fork()		*/
void GC_fork_prepare_proc(void)
{
    /* Acquire all relevant locks, so that after releasing the locks	*/
    /* the child will see a consistent state in which monitor 		*/
    /* invariants hold.	 Unfortunately, we can't acquire libc locks	*/
    /* we might need, and there seems to be no guarantee that libc	*/
    /* must install a suitable fork handler.				*/
    /* Wait for an ongoing GC to finish, since we can't finish it in	*/
    /* the (one remaining thread in) the child.				*/
      LOCK();
#     if defined(PARALLEL_MARK) || defined(THREAD_LOCAL_ALLOC)
        GC_wait_for_reclaim();
#     endif
      GC_wait_for_gc_completion(TRUE);
#     if defined(PARALLEL_MARK) || defined(THREAD_LOCAL_ALLOC)
        GC_acquire_mark_lock();
#     endif
}

/* Called in parent after a fork()	*/
void GC_fork_parent_proc(void)
{
#   if defined(PARALLEL_MARK) || defined(THREAD_LOCAL_ALLOC)
      GC_release_mark_lock();
#   endif
    UNLOCK();
}

/* Called in child after a fork()	*/
void GC_fork_child_proc(void)
{
    /* Clean up the thread table, so that just our thread is left. */
#   if defined(PARALLEL_MARK) || defined(THREAD_LOCAL_ALLOC)
      GC_release_mark_lock();
#   endif
    GC_remove_all_threads_but_me();
#   ifdef PARALLEL_MARK
      /* Turn off parallel marking in the child, since we are probably 	*/
      /* just going to exec, and we would have to restart mark threads.	*/
        GC_markers = 1;
        GC_parallel = FALSE;
#   endif /* PARALLEL_MARK */
    UNLOCK();
}
#endif /* HANDLE_FORK */

#if defined(GC_DGUX386_THREADS)
/* Return the number of processors, or i<= 0 if it can't be determined. */
int GC_get_nprocs(void)
{
    /* <takis@XFree86.Org> */
    int numCpus;
    struct dg_sys_info_pm_info pm_sysinfo;
    int status =0;

    status = dg_sys_info((long int *) &pm_sysinfo,
	DG_SYS_INFO_PM_INFO_TYPE, DG_SYS_INFO_PM_CURRENT_VERSION);
    if (status < 0)
       /* set -1 for error */
       numCpus = -1;
    else
      /* Active CPUs */
      numCpus = pm_sysinfo.idle_vp_count;

#  ifdef DEBUG_THREADS
    GC_printf("Number of active CPUs in this system: %d\n", numCpus);
#  endif
    return(numCpus);
}
#endif /* GC_DGUX386_THREADS */

#if defined(GC_NETBSD_THREADS)
static int get_ncpu(void)
{
    int mib[] = {CTL_HW,HW_NCPU};
    int res;
    size_t len = sizeof(res);

    sysctl(mib, sizeof(mib)/sizeof(int), &res, &len, NULL, 0);
    return res;
}
#endif	/* GC_NETBSD_THREADS */

# if defined(GC_LINUX_THREADS) && defined(INCLUDE_LINUX_THREAD_DESCR)
__thread int dummy_thread_local;
# endif

/* We hold the allocation lock.	*/
void GC_thr_init(void)
{
#   ifndef GC_DARWIN_THREADS
        int dummy;
#   endif
    GC_thread t;

    if (GC_thr_initialized) return;
    GC_thr_initialized = TRUE;
    
#   ifdef HANDLE_FORK
      /* Prepare for a possible fork.	*/
        pthread_atfork(GC_fork_prepare_proc, GC_fork_parent_proc,
	  	       GC_fork_child_proc);
#   endif /* HANDLE_FORK */
#   if defined(INCLUDE_LINUX_THREAD_DESCR)
      /* Explicitly register the region including the address 		*/
      /* of a thread local variable.  This should included thread	*/
      /* locals for the main thread, except for those allocated		*/
      /* in response to dlopen calls.					*/  
	{
	  ptr_t thread_local_addr = (ptr_t)(&dummy_thread_local);
	  ptr_t main_thread_start, main_thread_end;
          if (!GC_enclosing_mapping(thread_local_addr, &main_thread_start,
				    &main_thread_end)) {
	    ABORT("Failed to find mapping for main thread thread locals");
	  }
	  GC_add_roots_inner(main_thread_start, main_thread_end, FALSE);
	}
#   endif
    /* Add the initial thread, so we can stop it.	*/
      t = GC_new_thread(pthread_self());
#     ifdef GC_DARWIN_THREADS
         t -> stop_info.mach_thread = mach_thread_self();
#     else
         t -> stop_info.stack_ptr = (ptr_t)(&dummy);
#     endif
      t -> flags = DETACHED | MAIN_THREAD;

    GC_stop_init();

    /* Set GC_nprocs.  */
      {
	char * nprocs_string = GETENV("GC_NPROCS");
	GC_nprocs = -1;
	if (nprocs_string != NULL) GC_nprocs = atoi(nprocs_string);
      }
      if (GC_nprocs <= 0) {
#       if defined(GC_HPUX_THREADS)
	  GC_nprocs = pthread_num_processors_np();
#       endif
#	if defined(GC_OSF1_THREADS) || defined(GC_AIX_THREADS) \
	   || defined(GC_SOLARIS_THREADS)
	  GC_nprocs = sysconf(_SC_NPROCESSORS_ONLN);
	  if (GC_nprocs <= 0) GC_nprocs = 1;
#	endif
#       if defined(GC_IRIX_THREADS)
	  GC_nprocs = sysconf(_SC_NPROC_ONLN);
	  if (GC_nprocs <= 0) GC_nprocs = 1;
#       endif
#       if defined(GC_NETBSD_THREADS)
	  GC_nprocs = get_ncpu();
#       endif
#       if defined(GC_DARWIN_THREADS) || defined(GC_FREEBSD_THREADS)
	  int ncpus = 1;
	  size_t len = sizeof(ncpus);
	  sysctl((int[2]) {CTL_HW, HW_NCPU}, 2, &ncpus, &len, NULL, 0);
	  GC_nprocs = ncpus;
#       endif
#	if defined(GC_LINUX_THREADS) || defined(GC_DGUX386_THREADS)
          GC_nprocs = GC_get_nprocs();
#	endif
#       if defined(GC_GNU_THREADS)
	  if (GC_nprocs <= 0) GC_nprocs = 1;
#       endif
      }
      if (GC_nprocs <= 0) {
	WARN("GC_get_nprocs() returned %ld\n", GC_nprocs);
	GC_nprocs = 2;
#	ifdef PARALLEL_MARK
	  GC_markers = 1;
#	endif
      } else {
#	ifdef PARALLEL_MARK
          {
	    char * markers_string = GETENV("GC_MARKERS");
	    if (markers_string != NULL) {
	      GC_markers = atoi(markers_string);
	    } else {
	      GC_markers = GC_nprocs;
	    }
          }
#	endif
      }
#   ifdef PARALLEL_MARK
      if (GC_print_stats) {
          GC_log_printf("Number of processors = %ld, "
		 "number of marker threads = %ld\n", GC_nprocs, GC_markers);
      }
      if (GC_markers == 1) {
	GC_parallel = FALSE;
	if (GC_print_stats) {
	    GC_log_printf(
		"Single marker thread, turning off parallel marking\n");
	}
      } else {
	GC_parallel = TRUE;
	/* Disable true incremental collection, but generational is OK.	*/
	GC_time_limit = GC_TIME_UNLIMITED;
      }
      /* If we are using a parallel marker, actually start helper threads.  */
        if (GC_parallel) start_mark_threads();
#   endif
}


/* Perform all initializations, including those that	*/
/* may require allocation.				*/
/* Called without allocation lock.			*/
/* Must be called before a second thread is created.	*/
/* Did we say it's called without the allocation lock?	*/
void GC_init_parallel(void)
{
    if (parallel_initialized) return;
    parallel_initialized = TRUE;

    /* GC_init() calls us back, so set flag first.	*/
    if (!GC_is_initialized) GC_init();
    /* Initialize thread local free lists if used.	*/
#   if defined(THREAD_LOCAL_ALLOC)
      LOCK();
      GC_init_thread_local(&(GC_lookup_thread(pthread_self())->tlfs));
      UNLOCK();
#   endif
}


#if !defined(GC_DARWIN_THREADS)
int WRAP_FUNC(pthread_sigmask)(int how, const sigset_t *set, sigset_t *oset)
{
    sigset_t fudged_set;
    
    INIT_REAL_SYMS();
    if (set != NULL && (how == SIG_BLOCK || how == SIG_SETMASK)) {
        fudged_set = *set;
        sigdelset(&fudged_set, SIG_SUSPEND);
        set = &fudged_set;
    }
    return(REAL_FUNC(pthread_sigmask)(how, set, oset));
}
#endif /* !GC_DARWIN_THREADS */

/* Wrapper for functions that are likely to block for an appreciable	*/
/* length of time.							*/

struct blocking_data {
    void (*fn)(void *);
    void *arg;
};

static void GC_do_blocking_inner(ptr_t data, void * context) {
    struct blocking_data * d = (struct blocking_data *) data;
    GC_thread me;
    LOCK();
    me = GC_lookup_thread(pthread_self());
    GC_ASSERT(!(me -> thread_blocked));
#   ifdef SPARC
	me -> stop_info.stack_ptr = GC_save_regs_in_stack();
#   elif !defined(GC_DARWIN_THREADS)
	me -> stop_info.stack_ptr = GC_approx_sp();
#   endif
#   ifdef IA64
	me -> backing_store_ptr = GC_save_regs_in_stack();
#   endif
    me -> thread_blocked = TRUE;
    /* Save context here if we want to support precise stack marking */
    UNLOCK();
    (d -> fn)(d -> arg);
    LOCK();   /* This will block if the world is stopped.	*/
    me -> thread_blocked = FALSE;
    UNLOCK();
}

void GC_do_blocking(void (*fn)(void *), void *arg) {
    struct blocking_data my_data;

    my_data.fn = fn;
    my_data.arg = arg;
    GC_with_callee_saves_pushed(GC_do_blocking_inner, (ptr_t)(&my_data));
}
    
struct start_info {
    void *(*start_routine)(void *);
    void *arg;
    word flags;
    sem_t registered;   	/* 1 ==> in our thread table, but 	*/
				/* parent hasn't yet noticed.		*/
};

int GC_unregister_my_thread(void)
{
    GC_thread me;

    LOCK();
    /* Wait for any GC that may be marking from our stack to	*/
    /* complete before we remove this thread.			*/
    GC_wait_for_gc_completion(FALSE);
    me = GC_lookup_thread(pthread_self());
#   if defined(THREAD_LOCAL_ALLOC)
      GC_destroy_thread_local(&(me->tlfs));
#   endif
    if (me -> flags & DETACHED) {
    	GC_delete_thread(pthread_self());
    } else {
	me -> flags |= FINISHED;
    }
#   if defined(THREAD_LOCAL_ALLOC)
      GC_remove_specific(GC_thread_key);
#   endif
    UNLOCK();
    return GC_SUCCESS;
}

/* Called at thread exit.				*/
/* Never called for main thread.  That's OK, since it	*/
/* results in at most a tiny one-time leak.  And 	*/
/* linuxthreads doesn't reclaim the main threads 	*/
/* resources or id anyway.				*/
void GC_thread_exit_proc(void *arg)
{
    GC_unregister_my_thread();
}

int WRAP_FUNC(pthread_join)(pthread_t thread, void **retval)
{
    int result;
    GC_thread thread_gc_id;
    
    INIT_REAL_SYMS();
    LOCK();
    thread_gc_id = GC_lookup_thread(thread);
    /* This is guaranteed to be the intended one, since the thread id	*/
    /* cant have been recycled by pthreads.				*/
    UNLOCK();
    result = REAL_FUNC(pthread_join)(thread, retval);
# if defined (GC_FREEBSD_THREADS)
    /* On FreeBSD, the wrapped pthread_join() sometimes returns (what
       appears to be) a spurious EINTR which caused the test and real code
       to gratuitously fail.  Having looked at system pthread library source
       code, I see how this return code may be generated.  In one path of
       code, pthread_join() just returns the errno setting of the thread
       being joined.  This does not match the POSIX specification or the
       local man pages thus I have taken the liberty to catch this one
       spurious return value properly conditionalized on GC_FREEBSD_THREADS. */
    if (result == EINTR) result = 0;
# endif
    if (result == 0) {
        LOCK();
        /* Here the pthread thread id may have been recycled. */
        GC_delete_gc_thread(thread_gc_id);
        UNLOCK();
    }
    return result;
}

int
WRAP_FUNC(pthread_detach)(pthread_t thread)
{
    int result;
    GC_thread thread_gc_id;
    
    INIT_REAL_SYMS();
    LOCK();
    thread_gc_id = GC_lookup_thread(thread);
    UNLOCK();
    result = REAL_FUNC(pthread_detach)(thread);
    if (result == 0) {
      LOCK();
      thread_gc_id -> flags |= DETACHED;
      /* Here the pthread thread id may have been recycled. */
      if (thread_gc_id -> flags & FINISHED) {
        GC_delete_gc_thread(thread_gc_id);
      }
      UNLOCK();
    }
    return result;
}

GC_bool GC_in_thread_creation = FALSE;  /* Protected by allocation lock. */

GC_thread GC_register_my_thread_inner(struct GC_stack_base *sb,
				      pthread_t my_pthread)
{
    GC_thread me;

    GC_in_thread_creation = TRUE; /* OK to collect from unknown thread. */
    me = GC_new_thread(my_pthread);
    GC_in_thread_creation = FALSE;
#   ifdef GC_DARWIN_THREADS
      me -> stop_info.mach_thread = mach_thread_self();
#   else
      me -> stop_info.stack_ptr = sb -> mem_base;
#   endif
    me -> stack_end = sb -> mem_base;
#   ifdef IA64
      me -> backing_store_end = sb -> reg_base;
#   endif /* IA64 */
    return me;
}

int GC_register_my_thread(struct GC_stack_base *sb)
{
    pthread_t my_pthread = pthread_self();
    GC_thread me;

    LOCK();
    me = GC_lookup_thread(my_pthread);
    if (0 == me) {
        me = GC_register_my_thread_inner(sb, my_pthread);
	me -> flags |= DETACHED;
    	  /* Treat as detached, since we do not need to worry about	*/
    	  /* pointer results.						*/
	UNLOCK();
        return GC_SUCCESS;
    } else {
	UNLOCK();
	return GC_DUPLICATE;
    }
}

void * GC_inner_start_routine(struct GC_stack_base *sb, void * arg)
{
    struct start_info * si = arg;
    void * result;
    GC_thread me;
    pthread_t my_pthread;
    void *(*start)(void *);
    void *start_arg;

    my_pthread = pthread_self();
#   ifdef DEBUG_THREADS
        GC_printf("Starting thread 0x%x\n", (unsigned)my_pthread);
        GC_printf("pid = %ld\n", (long) getpid());
        GC_printf("sp = 0x%lx\n", (long) &arg);
#   endif
    LOCK();
    me = GC_register_my_thread_inner(sb, my_pthread);
    me -> flags = si -> flags;
    UNLOCK();
    start = si -> start_routine;
#   ifdef DEBUG_THREADS
	GC_printf("start_routine = %p\n", (void *)start);
#   endif
    start_arg = si -> arg;
    sem_post(&(si -> registered));	/* Last action on si.	*/
    					/* OK to deallocate.	*/
    pthread_cleanup_push(GC_thread_exit_proc, 0);
#   if defined(THREAD_LOCAL_ALLOC)
 	LOCK();
        GC_init_thread_local(&(me->tlfs));
	UNLOCK();
#   endif
    result = (*start)(start_arg);
#   if DEBUG_THREADS
        GC_printf("Finishing thread 0x%x\n", (unsigned)pthread_self());
#   endif
    me -> status = result;
    pthread_cleanup_pop(1);
    /* Cleanup acquires lock, ensuring that we can't exit		*/
    /* while a collection that thinks we're alive is trying to stop     */
    /* us.								*/
    return(result);
}

void * GC_start_routine(void * arg)
{
#   ifdef INCLUDE_LINUX_THREAD_DESCR
      struct GC_stack_base sb;

#     ifdef REDIRECT_MALLOC
      	/* GC_get_stack_base may call pthread_getattr_np, which can 	*/
        /* unfortunately call realloc, which may allocate from an	*/
        /* unregistered thread.  This is unpleasant, since it might	*/ 
        /* force heap growth.						*/
        GC_disable();
#     endif
      if (GC_get_stack_base(&sb) != GC_SUCCESS)
	ABORT("Failed to get thread stack base.");
#     ifdef REDIRECT_MALLOC
        GC_enable();
#     endif
      return GC_inner_start_routine(&sb, arg);
#   else
      return GC_call_with_stack_base(GC_inner_start_routine, arg);
#   endif
}

int
WRAP_FUNC(pthread_create)(pthread_t *new_thread,
		  const pthread_attr_t *attr,
                  void *(*start_routine)(void *), void *arg)
{
    int result;
    int detachstate;
    word my_flags = 0;
    struct start_info * si; 
	/* This is otherwise saved only in an area mmapped by the thread */
	/* library, which isn't visible to the collector.		 */
 
    /* We resist the temptation to muck with the stack size here,	*/
    /* even if the default is unreasonably small.  That's the client's	*/
    /* responsibility.							*/

    INIT_REAL_SYMS();
    LOCK();
    si = (struct start_info *)GC_INTERNAL_MALLOC(sizeof(struct start_info),
						 NORMAL);
    UNLOCK();
    if (!parallel_initialized) GC_init_parallel();
    if (0 == si) return(ENOMEM);
    sem_init(&(si -> registered), 0, 0);
    si -> start_routine = start_routine;
    si -> arg = arg;
    LOCK();
    if (!GC_thr_initialized) GC_thr_init();
#   ifdef GC_ASSERTIONS
      {
	size_t stack_size = 0;
	if (NULL != attr) {
	   pthread_attr_getstacksize(attr, &stack_size);
	}
	if (0 == stack_size) {
	   pthread_attr_t my_attr;
	   pthread_attr_init(&my_attr);
	   pthread_attr_getstacksize(&my_attr, &stack_size);
	}
	/* On Solaris 10, with default attr initialization, 	*/
	/* stack_size remains 0.  Fudge it.			*/
	if (0 == stack_size) {
#	    ifndef SOLARIS
	      WARN("Failed to get stack size for assertion checking\n", 0);
#	    endif
	    stack_size = 1000000;
	}
#       ifdef PARALLEL_MARK
	  GC_ASSERT(stack_size >= (8*HBLKSIZE*sizeof(word)));
#       else
          /* FreeBSD-5.3/Alpha: default pthread stack is 64K, 	*/
	  /* HBLKSIZE=8192, sizeof(word)=8			*/
	  GC_ASSERT(stack_size >= 65536);
#       endif
	/* Our threads may need to do some work for the GC.	*/
	/* Ridiculously small threads won't work, and they	*/
	/* probably wouldn't work anyway.			*/
      }
#   endif
    if (NULL == attr) {
	detachstate = PTHREAD_CREATE_JOINABLE;
    } else { 
        pthread_attr_getdetachstate(attr, &detachstate);
    }
    if (PTHREAD_CREATE_DETACHED == detachstate) my_flags |= DETACHED;
    si -> flags = my_flags;
    UNLOCK();
#   ifdef DEBUG_THREADS
        GC_printf("About to start new thread from thread 0x%x\n",
		  (unsigned)pthread_self());
#   endif
    GC_need_to_lock = TRUE;

    result = REAL_FUNC(pthread_create)(new_thread, attr, GC_start_routine, si);

#   ifdef DEBUG_THREADS
        GC_printf("Started thread 0x%x\n", (unsigned)(*new_thread));
#   endif
    /* Wait until child has been added to the thread table.		*/
    /* This also ensures that we hold onto si until the child is done	*/
    /* with it.  Thus it doesn't matter whether it is otherwise		*/
    /* visible to the collector.					*/
    if (0 == result) {
	while (0 != sem_wait(&(si -> registered))) {
            if (EINTR != errno) ABORT("sem_wait failed");
	}
    }
    sem_destroy(&(si -> registered));
    LOCK();
    GC_INTERNAL_FREE(si);
    UNLOCK();

    return(result);
}

/* Spend a few cycles in a way that can't introduce contention with	*/
/* othre threads.							*/
void GC_pause(void)
{
    int i;
#   if !defined(__GNUC__) || defined(__INTEL_COMPILER)
      volatile word dummy = 0;
#   endif

    for (i = 0; i < 10; ++i) { 
#     if defined(__GNUC__) && !defined(__INTEL_COMPILER)
        __asm__ __volatile__ (" " : : : "memory");
#     else
	/* Something that's unlikely to be optimized away. */
	GC_noop(++dummy);
#     endif
    }
}
    
#define SPIN_MAX 128	/* Maximum number of calls to GC_pause before	*/
			/* give up.					*/

volatile GC_bool GC_collecting = 0;
			/* A hint that we're in the collector and       */
                        /* holding the allocation lock for an           */
                        /* extended period.                             */

#if !defined(USE_SPIN_LOCK) || defined(PARALLEL_MARK)
/* If we don't want to use the below spinlock implementation, either	*/
/* because we don't have a GC_test_and_set implementation, or because 	*/
/* we don't want to risk sleeping, we can still try spinning on 	*/
/* pthread_mutex_trylock for a while.  This appears to be very		*/
/* beneficial in many cases.						*/
/* I suspect that under high contention this is nearly always better	*/
/* than the spin lock.  But it's a bit slower on a uniprocessor.	*/
/* Hence we still default to the spin lock.				*/
/* This is also used to acquire the mark lock for the parallel		*/
/* marker.								*/

/* Here we use a strict exponential backoff scheme.  I don't know 	*/
/* whether that's better or worse than the above.  We eventually 	*/
/* yield by calling pthread_mutex_lock(); it never makes sense to	*/
/* explicitly sleep.							*/

#define LOCK_STATS
#ifdef LOCK_STATS
  unsigned long GC_spin_count = 0;
  unsigned long GC_block_count = 0;
  unsigned long GC_unlocked_count = 0;
#endif

void GC_generic_lock(pthread_mutex_t * lock)
{
#ifndef NO_PTHREAD_TRYLOCK
    unsigned pause_length = 1;
    unsigned i;
    
    if (0 == pthread_mutex_trylock(lock)) {
#       ifdef LOCK_STATS
	    ++GC_unlocked_count;
#       endif
	return;
    }
    for (; pause_length <= SPIN_MAX; pause_length <<= 1) {
	for (i = 0; i < pause_length; ++i) {
	    GC_pause();
	}
        switch(pthread_mutex_trylock(lock)) {
	    case 0:
#		ifdef LOCK_STATS
		    ++GC_spin_count;
#		endif
		return;
	    case EBUSY:
		break;
	    default:
		ABORT("Unexpected error from pthread_mutex_trylock");
        }
    }
#endif /* !NO_PTHREAD_TRYLOCK */
#   ifdef LOCK_STATS
	++GC_block_count;
#   endif
    pthread_mutex_lock(lock);
}

#endif /* !USE_SPIN_LOCK || PARALLEL_MARK */

#if defined(USE_SPIN_LOCK)

/* Reasonably fast spin locks.  Basically the same implementation */
/* as STL alloc.h.  This isn't really the right way to do this.   */
/* but until the POSIX scheduling mess gets straightened out ...  */

volatile AO_TS_t GC_allocate_lock = 0;


void GC_lock(void)
{
#   define low_spin_max 30  /* spin cycles if we suspect uniprocessor */
#   define high_spin_max SPIN_MAX /* spin cycles for multiprocessor */
    static unsigned spin_max = low_spin_max;
    unsigned my_spin_max;
    static unsigned last_spins = 0;
    unsigned my_last_spins;
    int i;

    if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_CLEAR) {
        return;
    }
    my_spin_max = spin_max;
    my_last_spins = last_spins;
    for (i = 0; i < my_spin_max; i++) {
        if (GC_collecting || GC_nprocs == 1) goto yield;
        if (i < my_last_spins/2) {
            GC_pause();
            continue;
        }
        if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_CLEAR) {
	    /*
             * got it!
             * Spinning worked.  Thus we're probably not being scheduled
             * against the other process with which we were contending.
             * Thus it makes sense to spin longer the next time.
	     */
            last_spins = i;
            spin_max = high_spin_max;
            return;
        }
    }
    /* We are probably being scheduled against the other process.  Sleep. */
    spin_max = low_spin_max;
yield:
    for (i = 0;; ++i) {
        if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_CLEAR) {
            return;
        }
#       define SLEEP_THRESHOLD 12
		/* Under Linux very short sleeps tend to wait until	*/
		/* the current time quantum expires.  On old Linux	*/
		/* kernels nanosleep(<= 2ms) just spins under Linux.    */
		/* (Under 2.4, this happens only for real-time		*/
		/* processes.)  We want to minimize both behaviors	*/
		/* here.						*/
        if (i < SLEEP_THRESHOLD) {
            sched_yield();
	} else {
	    struct timespec ts;
	
	    if (i > 24) i = 24;
			/* Don't wait for more than about 15msecs, even	*/
			/* under extreme contention.			*/
	    ts.tv_sec = 0;
	    ts.tv_nsec = 1 << i;
	    nanosleep(&ts, 0);
	}
    }
}

#else  /* !USE_SPINLOCK */
void GC_lock(void)
{
#ifndef NO_PTHREAD_TRYLOCK
    if (1 == GC_nprocs || GC_collecting) {
	pthread_mutex_lock(&GC_allocate_ml);
    } else {
        GC_generic_lock(&GC_allocate_ml);
    }
#else  /* !NO_PTHREAD_TRYLOCK */
    pthread_mutex_lock(&GC_allocate_ml);
#endif /* !NO_PTHREAD_TRYLOCK */
}

#endif /* !USE_SPINLOCK */

#if defined(PARALLEL_MARK) || defined(THREAD_LOCAL_ALLOC)

#ifdef GC_ASSERTIONS
  unsigned long GC_mark_lock_holder = NO_THREAD;
#endif

#if 0
  /* Ugly workaround for a linux threads bug in the final versions      */
  /* of glibc2.1.  Pthread_mutex_trylock sets the mutex owner           */
  /* field even when it fails to acquire the mutex.  This causes        */
  /* pthread_cond_wait to die.  Remove for glibc2.2.                    */
  /* According to the man page, we should use                           */
  /* PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP, but that isn't actually   */
  /* defined.                                                           */
  static pthread_mutex_t mark_mutex =
        {0, 0, 0, PTHREAD_MUTEX_ERRORCHECK_NP, {0, 0}};
#else
  static pthread_mutex_t mark_mutex = PTHREAD_MUTEX_INITIALIZER;
#endif

static pthread_cond_t builder_cv = PTHREAD_COND_INITIALIZER;

void GC_acquire_mark_lock(void)
{
/*
    if (pthread_mutex_lock(&mark_mutex) != 0) {
	ABORT("pthread_mutex_lock failed");
    }
*/
    GC_generic_lock(&mark_mutex);
#   ifdef GC_ASSERTIONS
	GC_mark_lock_holder = NUMERIC_THREAD_ID(pthread_self());
#   endif
}

void GC_release_mark_lock(void)
{
    GC_ASSERT(GC_mark_lock_holder == NUMERIC_THREAD_ID(pthread_self()));
#   ifdef GC_ASSERTIONS
	GC_mark_lock_holder = NO_THREAD;
#   endif
    if (pthread_mutex_unlock(&mark_mutex) != 0) {
	ABORT("pthread_mutex_unlock failed");
    }
}

/* Collector must wait for a freelist builders for 2 reasons:		*/
/* 1) Mark bits may still be getting examined without lock.		*/
/* 2) Partial free lists referenced only by locals may not be scanned 	*/
/*    correctly, e.g. if they contain "pointer-free" objects, since the	*/
/*    free-list link may be ignored.					*/
void GC_wait_builder(void)
{
    GC_ASSERT(GC_mark_lock_holder == NUMERIC_THREAD_ID(pthread_self()));
#   ifdef GC_ASSERTIONS
	GC_mark_lock_holder = NO_THREAD;
#   endif
    if (pthread_cond_wait(&builder_cv, &mark_mutex) != 0) {
	ABORT("pthread_cond_wait failed");
    }
    GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
#   ifdef GC_ASSERTIONS
	GC_mark_lock_holder = NUMERIC_THREAD_ID(pthread_self());
#   endif
}

void GC_wait_for_reclaim(void)
{
    GC_acquire_mark_lock();
    while (GC_fl_builder_count > 0) {
	GC_wait_builder();
    }
    GC_release_mark_lock();
}

void GC_notify_all_builder(void)
{
    GC_ASSERT(GC_mark_lock_holder == NUMERIC_THREAD_ID(pthread_self()));
    if (pthread_cond_broadcast(&builder_cv) != 0) {
	ABORT("pthread_cond_broadcast failed");
    }
}

#endif /* PARALLEL_MARK || THREAD_LOCAL_ALLOC */

#ifdef PARALLEL_MARK

static pthread_cond_t mark_cv = PTHREAD_COND_INITIALIZER;

void GC_wait_marker(void)
{
    GC_ASSERT(GC_mark_lock_holder == NUMERIC_THREAD_ID(pthread_self()));
#   ifdef GC_ASSERTIONS
	GC_mark_lock_holder = NO_THREAD;
#   endif
    if (pthread_cond_wait(&mark_cv, &mark_mutex) != 0) {
	ABORT("pthread_cond_wait failed");
    }
    GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
#   ifdef GC_ASSERTIONS
	GC_mark_lock_holder = NUMERIC_THREAD_ID(pthread_self());
#   endif
}

void GC_notify_all_marker(void)
{
    if (pthread_cond_broadcast(&mark_cv) != 0) {
	ABORT("pthread_cond_broadcast failed");
    }
}

#endif /* PARALLEL_MARK */

# endif /* GC_LINUX_THREADS and friends */