00001 #ifndef _QTHREAD_H_
00002 #define _QTHREAD_H_
00003
00004 #include <errno.h>
00005
00006 #include <qthread/qthread-int.h>
00007 #include <qthread/common.h>
00008
00009 #include <string.h>
00010
00011 #ifdef QTHREAD_NEEDS_IA64INTRIN
00012 # ifdef HAVE_IA64INTRIN_H
00013 # include <ia64intrin.h>
00014 # elif defined(HAVE_IA32INTRIN_H)
00015 # include <ia32intrin.h>
00016 # endif
00017 #endif
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029
00030
00031
00032
00033 #define QTHREAD_REDUNDANT 1
00034 #define QTHREAD_SUCCESS 0
00035 #define QTHREAD_BADARGS -1
00036 #define QTHREAD_PTHREAD_ERROR -2
00037 #define QTHREAD_NOT_ALLOWED -3
00038 #define QTHREAD_MALLOC_ERROR ENOMEM
00039 #define QTHREAD_THIRD_PARTY_ERROR -4
00040 #define QTHREAD_TIMEOUT -5
00041 #ifdef EOVERFLOW
00042 # define QTHREAD_OVERFLOW EOVERFLOW
00043 #else
00044 # define QTHREAD_OVERFLOW -6
00045 #endif
00046 #define NO_SHEPHERD ((qthread_shepherd_id_t)-1)
00047
00048 #ifdef __cplusplus
00049 #define Q_STARTCXX extern "C" {
00050 #define Q_ENDCXX }
00051 #else
00052 #define Q_STARTCXX
00053 #define Q_ENDCXX
00054 #endif
00055
00056 #ifdef QTHREAD_ALIGNEDDATA_ALLOWED
00057 # define Q_ALIGNED(x) __attribute__((aligned(x)))
00058 #endif
00059
00060 Q_STARTCXX
00061
00062
00063
00064
00065
00066
00067 #if QTHREAD_SIZEOF_ALIGNED_T == 4
00068 typedef uint32_t Q_ALIGNED(QTHREAD_ALIGNMENT_ALIGNED_T) aligned_t;
00069 typedef int32_t Q_ALIGNED(QTHREAD_ALIGNMENT_ALIGNED_T) saligned_t;
00070 #elif QTHREAD_SIZEOF_ALIGNED_T == 8
00071 typedef uint64_t Q_ALIGNED(QTHREAD_ALIGNMENT_ALIGNED_T) aligned_t;
00072 typedef int64_t Q_ALIGNED(QTHREAD_ALIGNMENT_ALIGNED_T) saligned_t;
00073 #else
00074 #error "Don't know type for sizeof aligned_t"
00075 #endif
00076
00077 typedef struct _syncvar_s {
00078 volatile union {
00079 volatile uint64_t w;
00080 volatile struct {
00081 #ifdef BITFIELD_ORDER_FORWARD
00082 volatile uint64_t data : 60;
00083 volatile unsigned state : 3;
00084 volatile unsigned lock : 1;
00085 #else
00086 volatile unsigned lock : 1;
00087 volatile unsigned state : 3;
00088 volatile uint64_t data : 60;
00089 #endif
00090 } s;
00091 } u;
00092 } syncvar_t;
00093 #define SYNCVAR_STATIC_INITIALIZER { { 0 } }
00094 Q_ENDCXX
00095
00096 #ifdef QTHREAD_SST_PRIMITIVES
00097 # include <qthread/qthread-sst.h>
00098 #else
00099
00100 Q_STARTCXX
00101 typedef struct qthread_s qthread_t;
00102 typedef unsigned int qthread_shepherd_id_t;
00103
00104
00105 typedef aligned_t(*qthread_f) (qthread_t * me, void *arg);
00106
00107
00108
00109
00110
00111
00112
00113 int qthread_init(qthread_shepherd_id_t nshepherds);
00114 int qthread_initialize(void);
00115
00116
00117
00118
00119
00120
00121
00122 void qthread_finalize(void);
00123
00124
00125
00126
00127
00128 int qthread_disable_shepherd(const qthread_shepherd_id_t shep);
00129 void qthread_enable_shepherd(const qthread_shepherd_id_t shep);
00130
00131
00132
00133
00134
00135
00136 void qthread_yield(qthread_t * me);
00137
00138
00139
00140 qthread_t *qthread_self(void);
00141
00142
00143
00144
00145
00146
00147
00148
00149
00150
00151
00152 int qthread_fork(const qthread_f f, const void *const arg, aligned_t * ret);
00153 int qthread_fork_syncvar(const qthread_f f, const void *const arg, syncvar_t * ret);
00154 int qthread_fork_to(const qthread_f f, const void *const arg, aligned_t * ret,
00155 const qthread_shepherd_id_t shepherd);
00156 int qthread_fork_syncvar_to(const qthread_f f, const void *const arg, syncvar_t * ret,
00157 const qthread_shepherd_id_t shepherd);
00158
00159
00160
00161
00162
00163
00164
00165
00166
00167 qthread_t *qthread_prepare(const qthread_f f, const void *const arg,
00168 aligned_t * ret);
00169 qthread_t *qthread_prepare_for(const qthread_f f, const void *const arg,
00170 aligned_t * ret,
00171 const qthread_shepherd_id_t shepherd);
00172
00173 int qthread_schedule(qthread_t * t);
00174 int qthread_schedule_on(qthread_t * t, const qthread_shepherd_id_t shepherd);
00175
00176
00177 int qthread_migrate_to(qthread_t * me, const qthread_shepherd_id_t shepherd);
00178
00179
00180 int qthread_debuglevel(int);
00181
00182
00183
00184 unsigned qthread_id(const qthread_t * t);
00185 qthread_shepherd_id_t qthread_shep(const qthread_t * t);
00186 size_t qthread_stackleft(const qthread_t * t);
00187 aligned_t *qthread_retloc(const qthread_t * t);
00188 int qthread_shep_ok(const qthread_t * t);
00189
00190
00191 int qthread_distance(const qthread_shepherd_id_t src,
00192 const qthread_shepherd_id_t dest);
00193
00194
00195 const qthread_shepherd_id_t *qthread_sorted_sheps(const qthread_t * t);
00196 const qthread_shepherd_id_t *qthread_sorted_sheps_remote(const
00197 qthread_shepherd_id_t
00198 src);
00199
00200 qthread_shepherd_id_t qthread_num_shepherds(void);
00201
00202
00203
00204
00205
00206
00207
00208
00209
00210
00211
00212
00213
00214
00215
00216
00217
00218
00219 int qthread_feb_status(const aligned_t * addr);
00220 int qthread_syncvar_status(syncvar_t *const v);
00221
00222
00223
00224
00225
00226
00227 int qthread_empty(qthread_t * me, const aligned_t * dest);
00228 int qthread_syncvar_empty(qthread_t * restrict const me, syncvar_t * restrict const dest);
00229 int qthread_fill(qthread_t * me, const aligned_t * dest);
00230 int qthread_syncvar_fill(qthread_t * restrict const me, syncvar_t * restrict const dest);
00231
00232
00233
00234
00235
00236
00237
00238
00239
00240
00241
00242
00243
00244
00245
00246 int qthread_writeEF(qthread_t * me, aligned_t * restrict const dest,
00247 const aligned_t * restrict const src);
00248 int qthread_writeEF_const(qthread_t * me, aligned_t * const dest,
00249 const aligned_t src);
00250 int qthread_syncvar_writeEF(qthread_t *restrict me, syncvar_t *restrict
00251 const dest, const uint64_t *restrict const src);
00252 int qthread_syncvar_writeEF_const(qthread_t *restrict me, syncvar_t *restrict
00253 const dest, const uint64_t src);
00254
00255
00256
00257
00258
00259
00260
00261
00262
00263
00264
00265
00266
00267
00268
00269 int qthread_writeF(qthread_t * me, aligned_t * restrict const dest,
00270 const aligned_t * restrict const src);
00271 int qthread_writeF_const(qthread_t * me, aligned_t * const dest,
00272 const aligned_t src);
00273 int qthread_syncvar_writeF(qthread_t *restrict me, syncvar_t *restrict
00274 const dest, const uint64_t *restrict const src);
00275 int qthread_syncvar_writeF_const(qthread_t *restrict me, syncvar_t *restrict
00276 const dest, const uint64_t src);
00277
00278
00279
00280
00281
00282
00283
00284
00285
00286
00287
00288
00289
00290
00291
00292 int qthread_readFF(qthread_t * me, aligned_t * const dest,
00293 const aligned_t * const src);
00294 int qthread_syncvar_readFF(qthread_t * restrict const me, uint64_t * restrict const dest,
00295 syncvar_t * restrict const src);
00296
00297
00298
00299
00300
00301
00302
00303
00304
00305
00306
00307
00308
00309
00310
00311 int qthread_readFE(qthread_t * me, aligned_t * const dest,
00312 const aligned_t * const src);
00313 int qthread_syncvar_readFE(qthread_t * restrict const me, uint64_t * restrict const dest,
00314 syncvar_t * restrict const src);
00315
00316
00317
00318
00319
00320
00321
00322
00323
00324
00325
00326 int qthread_lock(qthread_t * me, const aligned_t * a);
00327 int qthread_unlock(qthread_t * me, const aligned_t * a);
00328
00329 #if defined(QTHREAD_MUTEX_INCREMENT) || (QTHREAD_ASSEMBLY_ARCH == QTHREAD_POWERPC32)
00330 uint32_t qthread_incr32_(volatile uint32_t *, const int32_t);
00331 uint64_t qthread_incr64_(volatile uint64_t *, const int64_t);
00332 float qthread_fincr_(volatile float *, const float);
00333 double qthread_dincr_(volatile double *, const double);
00334 uint32_t qthread_cas32_(volatile uint32_t *, const uint32_t, const uint32_t);
00335 uint64_t qthread_cas64_(volatile uint64_t *, const uint64_t, const uint64_t);
00336 #endif
00337
00338
00339
00340
00341
00342
00343
00344
00345 static QINLINE float qthread_fincr(volatile float *operand, const float incr)
00346 {
00347 #if defined (QTHREAD_MUTEX_INCREMENT)
00348 return qthread_fincr_(operand,incr);
00349 #else
00350 # if (QTHREAD_ASSEMBLY_ARCH == QTHREAD_POWERPC32) || (QTHREAD_ASSEMBLY_ARCH == QTHREAD_POWERPC64)
00351 union
00352 {
00353 float f;
00354 uint32_t i;
00355 } retval;
00356 register float incremented_value;
00357 register uint32_t scratch_int;
00358 uint32_t conversion_memory = conversion_memory;
00359 __asm__ __volatile__("1:\n\t"
00360 "lwarx %0,0,%4\n\t"
00361
00362 "stw %0,%2\n\t"
00363 "lfs %1,%2\n\t"
00364
00365 "fadds %1,%1,%5\n\t"
00366
00367 "stfs %1,%2\n\t"
00368 "lwz %3,%2\n\t"
00369
00370 "stwcx. %3,0,%4\n\t"
00371 "bne- 1b\n\t"
00372 "isync"
00373 :"=&b" (retval.i),
00374 "=&f" (incremented_value),
00375 "=m" (conversion_memory),
00376 "=&r" (scratch_int)
00377 :"r" (operand),
00378 "f" (incr)
00379 :"cc", "memory");
00380
00381 return retval.f;
00382 # elif (QTHREAD_ASSEMBLY_ARCH == QTHREAD_SPARCV9_64) || (QTHREAD_ASSEMBLY_ARCH == QTHREAD_SPARCV9_32)
00383 union
00384 {
00385 float f;
00386 uint32_t i;
00387 } oldval, newval;
00388
00389
00390 do {
00391
00392
00393
00394
00395
00396
00397
00398
00399 oldval.f = *operand;
00400 newval.f = oldval.f + incr;
00401 __asm__ __volatile__
00402 ("membar #StoreStore|#LoadStore|#StoreLoad|#LoadLoad\n\t"
00403 "cas [%1], %2, %0"
00404 :"+r"(newval.i)
00405 :"r" (operand), "r"(oldval.i)
00406 :"cc", "memory");
00407 } while (oldval.i != newval.i);
00408 return oldval.f;
00409 # elif (QTHREAD_ASSEMBLY_ARCH == QTHREAD_IA64)
00410 union
00411 {
00412 float f;
00413 uint32_t i;
00414 } oldval, newval, res;
00415
00416 do {
00417 oldval.f = *operand;
00418 newval.f = oldval.f + incr;
00419 __asm__ __volatile__("mov ar.ccv=%0;;"::"rO"(oldval.i));
00420 __asm__ __volatile__("cmpxchg4.acq %0=[%1],%2,ar.ccv"
00421 :"=r"(res.i)
00422 :"r" (operand), "r"(newval.i)
00423 :"memory");
00424 } while (res.i != oldval.i);
00425 return oldval.f;
00426 # elif (QTHREAD_ASSEMBLY_ARCH == QTHREAD_AMD64) || (QTHREAD_ASSEMBLY_ARCH == QTHREAD_IA32)
00427 union
00428 {
00429 float f;
00430 uint32_t i;
00431 } oldval, newval, retval;
00432
00433 do {
00434 oldval.f = *operand;
00435 newval.f = oldval.f + incr;
00436 __asm__ __volatile__("lock; cmpxchg %1, (%2)"
00437 :"=a"(retval.i)
00438 :"r" (newval.i),
00439 "r"(operand),
00440 "0"(oldval.i)
00441 :"cc", "memory");
00442 } while (retval.i != oldval.i);
00443 return oldval.f;
00444 # else
00445 # error Unsupported assembly architecture for qthread_fincr
00446 # endif
00447 #endif
00448 }
00449
00450 static QINLINE double qthread_dincr(volatile double *operand,
00451 const double incr)
00452 {
00453 #if defined(QTHREAD_MUTEX_INCREMENT) || (QTHREAD_ASSEMBLY_ARCH == QTHREAD_POWERPC32)
00454 return qthread_dincr_(operand, incr);
00455 #else
00456 # if (QTHREAD_ASSEMBLY_ARCH == QTHREAD_POWERPC64)
00457 register uint64_t scratch_int;
00458 register double incremented_value;
00459 union
00460 {
00461 uint64_t i;
00462 double d;
00463 } retval;
00464 uint64_t conversion_memory = conversion_memory;
00465 __asm__ __volatile__("1:\n\t"
00466 "ldarx %0,0,%4\n\t"
00467
00468 "std %0,%2\n\t"
00469 "lfd %1,%2\n\t"
00470
00471 "fadd %1,%1,%5\n\t"
00472
00473 "stfd %1,%2\n\t"
00474 "ld %3,%2\n\t"
00475
00476 "stdcx. %3,0,%4\n\t"
00477 "bne- 1b\n\t"
00478 "isync"
00479 :"=&b" (retval.i),
00480 "=&f" (incremented_value),
00481 "=m" (conversion_memory),
00482 "=r&" (scratch_int)
00483 :"r" (operand),
00484 "f" (incr)
00485 :"cc", "memory");
00486
00487 return retval.d;
00488 # elif (QTHREAD_ASSEMBLY_ARCH == QTHREAD_SPARCV9_32)
00489 double oldval, newval;
00490
00491 newval = *operand;
00492 do {
00493
00494
00495 register uint64_t tmp1 = tmp1;
00496 register uint64_t tmp2 = tmp2;
00497
00498 oldval = newval;
00499 newval = oldval + incr;
00500 __asm__ __volatile__("ldx %0, %1\n\t"
00501 "ldx %4, %2\n\t"
00502 "membar #StoreStore|#LoadStore|#StoreLoad|#LoadLoad\n\t"
00503 "casx [%3], %2, %1\n\t"
00504 "stx %1, %0"
00505
00506
00507 :"=m" (newval), "=&h"(tmp1), "=&h"(tmp2)
00508 :"r" (operand), "m"(oldval)
00509 :"memory");
00510 } while (oldval != newval);
00511 return oldval;
00512 # elif (QTHREAD_ASSEMBLY_ARCH == QTHREAD_SPARCV9_64)
00513 union
00514 {
00515 uint64_t i;
00516 double d;
00517 } oldval, newval;
00518
00519
00520 do {
00521
00522
00523
00524
00525
00526
00527
00528
00529 oldval.d = *operand;
00530 newval.d = oldval.d + incr;
00531 __asm__ __volatile__
00532 ("membar #StoreStore|#LoadStore|#StoreLoad|#LoadLoad\n\t"
00533 "casx [%1], %2, %0"
00534 :"+r"(newval.i)
00535 :"r"(operand), "r"(oldval.i)
00536 :"memory");
00537 } while (oldval.d != newval.d);
00538 return oldval.d;
00539 # elif (QTHREAD_ASSEMBLY_ARCH == QTHREAD_IA64)
00540 union
00541 {
00542 uint64_t i;
00543 double d;
00544 } oldval, newval, res;
00545
00546 do {
00547 oldval.d = *operand;
00548 newval.d = oldval.d + incr;
00549 __asm__ __volatile__("mov ar.ccv=%0;;"::"rO"(oldval.i));
00550 __asm__ __volatile__("cmpxchg8.acq %0=[%1],%2,ar.ccv"
00551 :"=r"(res.i)
00552 :"r" (operand), "r"(newval.i)
00553 :"memory");
00554 } while (res.i != oldval.i);
00555 return oldval.d;
00556
00557 # elif (QTHREAD_ASSEMBLY_ARCH == QTHREAD_AMD64)
00558 union
00559 {
00560 double d;
00561 uint64_t i;
00562 } oldval, newval, retval;
00563
00564 do {
00565 oldval.d = *operand;
00566 newval.d = oldval.d + incr;
00567 # ifdef __PGI
00568 __asm__ __volatile__("lock; cmpxchgq %1, (%2)\n\t"
00569 "mov %%rax,(%0)"
00570 ::"r"(&retval.i),
00571 "r"(newval.i), "r"(operand),
00572 "a"(oldval.i)
00573 :"memory");
00574 # else
00575 __asm__ __volatile__("lock; cmpxchgq %1, (%2)"
00576 :"=a"(retval.i)
00577 :"r"(newval.i), "r"(operand),
00578 "0"(oldval.i)
00579 :"memory");
00580 # endif
00581 } while (retval.i != oldval.i);
00582 return oldval.d;
00583
00584 # elif (QTHREAD_ASSEMBLY_ARCH == QTHREAD_IA32)
00585 union
00586 {
00587 double d;
00588 uint64_t i;
00589 struct
00590 {
00591
00592
00593 uint32_t l;
00594 uint32_t h;
00595 } s;
00596 } oldval, newval;
00597 register char test;
00598
00599 do {
00600 #ifdef __PIC__
00601
00602 # define QTHREAD_PIC_PREFIX "xchg %%ebx, %4\n\t"
00603
00604 # define QTHREAD_PIC_SUFFIX "\n\txchg %%ebx, %4"
00605 # define QTHREAD_PIC_REG_4 "r"
00606 #else
00607 # define QTHREAD_PIC_PREFIX
00608 # define QTHREAD_PIC_SUFFIX
00609 # define QTHREAD_PIC_REG_4 "b"
00610 #endif
00611 oldval.d = *operand;
00612 newval.d = oldval.d + incr;
00613
00614
00615
00616
00617
00618
00619
00620
00621
00622
00623
00624
00625
00626
00627
00628
00629
00630
00631
00632 __asm__ __volatile__(QTHREAD_PIC_PREFIX
00633 "lock; cmpxchg8b (%1)\n\t"
00634 "setne %0"
00635 QTHREAD_PIC_SUFFIX
00636 :"=q"(test)
00637 :"r"(operand),
00638 "a"(oldval.s.l),
00639 "d"(oldval.s.h),
00640 QTHREAD_PIC_REG_4(newval.s.l),
00641 "c"(newval.s.h)
00642 :"memory");
00643 } while (test);
00644 return oldval.d;
00645
00646 # else
00647 # error Unimplemented assembly architecture for qthread_dincr
00648 # endif
00649 #endif
00650 }
00651
00652 static QINLINE uint32_t qthread_incr32(volatile uint32_t * operand,
00653 const uint32_t incr)
00654 {
00655 #ifdef QTHREAD_MUTEX_INCREMENT
00656 return qthread_incr32_(operand,incr);
00657 #else
00658 # if (QTHREAD_ASSEMBLY_ARCH == QTHREAD_POWERPC32) || \
00659 (QTHREAD_ASSEMBLY_ARCH == QTHREAD_POWERPC64)
00660 uint32_t retval;
00661 register unsigned int incrd = incrd;
00662 __asm__ __volatile__("1:\tlwarx %0,0,%2\n\t"
00663 "add %1,%0,%3\n\t"
00664 "stwcx. %1,0,%2\n\t"
00665 "bne- 1b\n\t"
00666 "isync"
00667 :"=&b" (retval), "=&r"(incrd)
00668 :"r" (operand), "r"(incr)
00669 :"cc", "memory");
00670
00671 return retval;
00672 # elif (QTHREAD_ASSEMBLY_ARCH == QTHREAD_SPARCV9_32) || \
00673 (QTHREAD_ASSEMBLY_ARCH == QTHREAD_SPARCV9_64)
00674 register uint32_t oldval, newval;
00675
00676
00677 do {
00678
00679
00680
00681
00682
00683
00684
00685
00686 oldval = *operand;
00687 newval = oldval + incr;
00688
00689
00690 __asm__ __volatile__
00691 ("membar #StoreStore|#LoadStore|#StoreLoad|#LoadLoad\n\t"
00692 "cas [%1] , %2, %0"
00693 :"+r" (newval)
00694 :"r" (operand), "r"(oldval)
00695 :"cc", "memory");
00696 } while (oldval != newval);
00697 return oldval;
00698 # elif (QTHREAD_ASSEMBLY_ARCH == QTHREAD_IA64)
00699 uint32_t res;
00700
00701 if (incr == 1) {
00702 asm volatile ("fetchadd4.rel %0=[%1],1"
00703 :"=r" (res)
00704 :"r" (operand));
00705 } else {
00706 uint32_t old, newval;
00707
00708 do {
00709 old = *operand;
00710 newval = old + incr;
00711 asm volatile ("mov ar.ccv=%0;;":
00712 :"rO" (old));
00713
00714
00715 asm volatile ("cmpxchg4.acq %0=[%1],%2,ar.ccv"
00716 :"=r"(res)
00717 :"r" (operand), "r"(newval)
00718 :"memory");
00719 } while (res != old);
00720 }
00721 return res;
00722 # elif (QTHREAD_ASSEMBLY_ARCH == QTHREAD_IA32) || \
00723 (QTHREAD_ASSEMBLY_ARCH == QTHREAD_AMD64)
00724
00725 uint32_t retval = incr;
00726 __asm__ __volatile__ ("lock ; xaddl %0, (%1);"
00727 :"+r" (retval)
00728 :"r" (operand)
00729 :"memory");
00730
00731 return retval;
00732 # else
00733 # error Unimplemented assembly architecture for qthread_incr32
00734 # endif
00735 #endif
00736 }
00737
00738 static QINLINE uint64_t qthread_incr64(volatile uint64_t * operand,
00739 const uint64_t incr)
00740 {
00741 #if defined(QTHREAD_MUTEX_INCREMENT) || \
00742 (QTHREAD_ASSEMBLY_ARCH == QTHREAD_POWERPC32) || \
00743 (QTHREAD_ASSEMBLY_ARCH == QTHREAD_SPARCV9_32)
00744 return qthread_incr64_(operand,incr);
00745 #else
00746 # if (QTHREAD_ASSEMBLY_ARCH == QTHREAD_POWERPC64)
00747 uint64_t retval;
00748 register uint64_t incrd = incrd;
00749
00750 asm volatile ("1:\tldarx %0,0,%2\n\t"
00751 "add %1,%0,%3\n\t"
00752 "stdcx. %1,0,%2\n\t"
00753 "bne- 1b\n\t"
00754 "isync"
00755 :"=&b" (retval), "=&r"(incrd)
00756 :"r" (operand), "r"(incr)
00757 :"cc", "memory");
00758
00759 return retval;
00760 # elif (QTHREAD_ASSEMBLY_ARCH == QTHREAD_SPARCV9_32)
00761 uint64_t oldval, newval = *operand;
00762
00763 do {
00764
00765
00766 register uint64_t tmp1 = tmp1;
00767 register uint64_t tmp2 = tmp2;
00768
00769 oldval = newval;
00770 newval += incr;
00771
00772
00773 __asm__ __volatile__("ldx %0, %1\n\t"
00774 "ldx %4, %2\n\t"
00775 "membar #StoreStore|#LoadStore|#StoreLoad|#LoadLoad\n\t"
00776 "casx [%3] , %2, %1\n\t"
00777 "stx %1, %0"
00778
00779
00780 :"=m" (newval), "=&h"(tmp1), "=&h"(tmp2)
00781 :"r" (operand), "m"(oldval)
00782 :"cc", "memory");
00783 } while (oldval != newval);
00784 return oldval;
00785 # elif (QTHREAD_ASSEMBLY_ARCH == QTHREAD_SPARCV9_64)
00786 register uint64_t oldval, newval;
00787
00788 # ifdef QTHREAD_ATOMIC_CAS
00789 newval = *operand;
00790 do {
00791 oldval = newval;
00792 newval = __sync_val_compare_and_swap(operand, oldval, oldval + incr);
00793 } while (oldval != newval);
00794 # else
00795 do {
00796
00797
00798
00799
00800
00801
00802
00803
00804 oldval = *operand;
00805 newval = oldval + incr;
00806
00807
00808 __asm__ __volatile__
00809 ("membar #StoreStore|#LoadStore|#StoreLoad|#LoadLoad\n\t"
00810 "casx [%1] , %2, %0"
00811 :"+r"(newval)
00812 :"r" (operand), "r"(oldval)
00813 :"cc", "memory");
00814 } while (oldval != newval);
00815 # endif
00816 return oldval;
00817 # elif (QTHREAD_ASSEMBLY_ARCH == QTHREAD_IA64)
00818 uint64_t res;
00819
00820 if (incr == 1) {
00821 asm volatile ("fetchadd8.rel %0=%1,1"
00822 :"=r" (res)
00823 :"m" (*operand));
00824 } else {
00825 uint64_t old, newval;
00826
00827 do {
00828 old = *operand;
00829 newval = old + incr;
00830 asm volatile ("mov ar.ccv=%0;;":
00831 :"rO" (old));
00832
00833
00834 asm volatile ("cmpxchg8.acq %0=[%1],%2,ar.ccv"
00835 :"=r" (res)
00836 :"r" (operand), "r"(newval)
00837 :"memory");
00838 } while (res != old);
00839 }
00840 return res;
00841 # elif (QTHREAD_ASSEMBLY_ARCH == QTHREAD_IA32)
00842 union
00843 {
00844 uint64_t i;
00845 struct
00846 {
00847
00848
00849 uint32_t l;
00850 uint32_t h;
00851 } s;
00852 } oldval, newval;
00853 register char test;
00854
00855 do {
00856 #ifndef QTHREAD_PIC_PREFIX
00857 # ifdef __PIC__
00858
00859
00860 # define QTHREAD_PIC_PREFIX "xchg %%ebx, %4\n\t"
00861
00862 # define QTHREAD_PIC_SUFFIX "\n\txchg %%ebx, %4"
00863 # define QTHREAD_PIC_REG_4 "r"
00864 # else
00865 # define QTHREAD_PIC_PREFIX
00866 # define QTHREAD_PIC_SUFFIX
00867 # define QTHREAD_PIC_REG_4 "b"
00868 # endif
00869 #endif
00870 oldval.i = *operand;
00871 newval.i = oldval.i + incr;
00872
00873
00874
00875
00876
00877
00878
00879
00880
00881
00882
00883
00884
00885
00886
00887
00888
00889
00890
00891 __asm__ __volatile__(QTHREAD_PIC_PREFIX
00892 "lock; cmpxchg8b (%1)\n\t"
00893 "setne %0"
00894 QTHREAD_PIC_SUFFIX
00895 :"=q"(test)
00896 :"r" (operand),
00897 "a"(oldval.s.l),
00898 "d"(oldval.s.h),
00899 QTHREAD_PIC_REG_4(newval.s.l),
00900 "c"(newval.s.h)
00901 :"memory");
00902 } while (test);
00903 return oldval.i;
00904 # elif (QTHREAD_ASSEMBLY_ARCH == QTHREAD_AMD64)
00905 uint64_t retval = incr;
00906
00907 # ifdef __PGI
00908
00909
00910 __asm__ __volatile__ ("lock xaddq %0, (%2)\n\t"
00911 "mov %0,(%1)"
00912 ::"r" (incr),
00913 "r"(&retval),
00914 "r" (operand)
00915 :"memory");
00916 # else
00917 __asm__ __volatile__ ("lock ; xaddq %0, (%1);"
00918 :"+r" (retval)
00919 :"r" (operand)
00920 :"memory");
00921 # endif
00922
00923 return retval;
00924 # else
00925 # error Unimplemented assembly architecture for qthread_incr64
00926 # endif
00927 #endif
00928 }
00929
00930 static QINLINE unsigned long qthread_incr_xx(
00931 volatile void *addr,
00932 const long int incr,
00933 const size_t length)
00934 {
00935 switch (length) {
00936 case 4:
00937 return qthread_incr32((volatile uint32_t *)addr, incr);
00938 case 8:
00939 return qthread_incr64((volatile uint64_t *)addr, incr);
00940 default:
00941
00942
00943 *(int *)(0) = 0;
00944 }
00945 return 0;
00946 }
00947
00948 #if ! defined(QTHREAD_ATOMIC_CAS) || defined(QTHREAD_MUTEX_INCREMENT)
00949 static QINLINE uint32_t qthread_cas32(volatile uint32_t * operand,
00950 const uint32_t oldval,
00951 const uint32_t newval)
00952 {
00953 #ifdef QTHREAD_MUTEX_INCREMENT // XXX: this is only valid if you don't read *operand without the lock
00954 return qthread_cas32_(operand,oldval,newval);
00955 #else
00956 # if (QTHREAD_ASSEMBLY_ARCH == QTHREAD_POWERPC32) || \
00957 (QTHREAD_ASSEMBLY_ARCH == QTHREAD_POWERPC64)
00958 register uint32_t result;
00959 __asm__ __volatile__ ("1:\n\t"
00960 "lwarx %0,0,%3\n\t"
00961 "cmpw %0,%1\n\t"
00962 "bne 2f\n\t"
00963 "stwcx. %2,0,%3\n\t"
00964 "bne- 1b\n"
00965 "2:"
00966 "isync"
00967 :"=&b" (result)
00968 :"r"(oldval), "r"(newval), "r"(operand)
00969 :"cc", "memory");
00970 return result;
00971 # elif (QTHREAD_ASSEMBLY_ARCH == QTHREAD_SPARCV9_32) || \
00972 (QTHREAD_ASSEMBLY_ARCH == QTHREAD_SPARCV9_64)
00973 register uint32_t newv = newval;
00974 __asm__ __volatile__
00975 ("membar #StoreStore|#LoadStore|#StoreLoad|#LoadLoad\n\t"
00976 "cas [%1], %2, %0"
00977 : "+r" (newv)
00978 : "r" (operand), "r"(oldval)
00979 : "cc", "memory");
00980 return newv;
00981 # elif (QTHREAD_ASSEMBLY_ARCH == QTHREAD_IA64)
00982 register uint32_t retval;
00983 __asm__ __volatile__ ("mov ar.ccv=%0;;": :"rO" (oldval));
00984 __asm__ __volatile__ ("cmpxchg4.acq %0=[%1],%2,ar.ccv"
00985 :"=r"(retval)
00986 :"r"(operand), "r"(newval)
00987 :"memory");
00988 return retval;
00989 # elif (QTHREAD_ASSEMBLY_ARCH == QTHREAD_AMD64) || \
00990 (QTHREAD_ASSEMBLY_ARCH == QTHREAD_IA32)
00991 uint32_t retval;
00992
00993
00994
00995
00996
00997 __asm__ __volatile__ ("lock; cmpxchg %1,(%2)"
00998 : "=&a"(retval)
00999 : "r"(newval), "r" (operand),
01000 "0"(oldval)
01001 :"cc","memory");
01002 return retval;
01003 # else
01004 # error Unimplemented assembly architecture for qthread_cas32
01005 # endif
01006 #endif
01007 }
01008
01009 static QINLINE uint64_t qthread_cas64(volatile uint64_t * operand,
01010 const uint64_t oldval,
01011 const uint64_t newval)
01012 {
01013 #ifdef QTHREAD_MUTEX_INCREMENT
01014 return qthread_cas64_(operand, oldval, newval);
01015 #else
01016 # if (QTHREAD_ASSEMBLY_ARCH == QTHREAD_POWERPC64)
01017 register uint64_t result;
01018 __asm__ __volatile__ ("1:\n\t"
01019 "ldarx %0,0,%3\n\t"
01020 "cmpw %0,%1\n\t"
01021 "bne 2f\n\t"
01022 "stdcx. %2,0,%3\n\t"
01023 "bne- 1b\n"
01024 "2:"
01025 "isync"
01026 :"=&b" (result)
01027 :"r"(oldval), "r"(newval), "r"(operand)
01028 :"cc", "memory");
01029 return result;
01030 # elif (QTHREAD_ASSEMBLY_ARCH == QTHREAD_SPARCV9_32)
01031 register uint64_t tmp1=tmp1;
01032 register uint64_t tmp2=tmp2;
01033 uint64_t newv = newval;
01034 __asm__ __volatile__
01035 ("ldx %0, %1\n\t"
01036 "ldx %4, %2\n\t"
01037 "membar #StoreStore|#LoadStore|#StoreLoad|#LoadLoad\n\t"
01038 "casx [%3], %2, %1\n\t"
01039 "stx %1, %0"
01040
01041
01042 : "+m" (newv), "=&h" (tmp1), "=&h"(tmp2)
01043 : "r" (operand), "m"(oldval)
01044 : "cc", "memory");
01045 return newv;
01046 # elif (QTHREAD_ASSEMBLY_ARCH == QTHREAD_SPARCV9_64)
01047 register uint64_t newv = newval;
01048 __asm__ __volatile__
01049 ("membar #StoreStore|#LoadStore|#StoreLoad|#LoadLoad\n\t"
01050 "casx [%1], %2, %0"
01051 : "+r" (newv)
01052 : "r" (operand), "r"(oldval)
01053 : "cc", "memory");
01054 return newv;
01055 # elif (QTHREAD_ASSEMBLY_ARCH == QTHREAD_IA64)
01056 register uint32_t retval;
01057 __asm__ __volatile__ ("mov ar.ccv=%0;;": :"rO" (oldval));
01058 __asm__ __volatile__ ("cmpxchg8.acq %0=[%1],%2,ar.ccv"
01059 :"=r"(retval)
01060 :"r"(operand), "r"(newval)
01061 :"memory");
01062 return retval;
01063 # elif (QTHREAD_ASSEMBLY_ARCH == QTHREAD_IA32)
01064 union {
01065 uint64_t i;
01066 struct {
01067
01068
01069 uint32_t l;
01070 uint32_t h;
01071 } s;
01072 } oldv, newv, ret;
01073 oldv.i = oldval;
01074 newv.i = newval;
01075 #ifndef QTHREAD_PIC_PREFIX
01076 #ifdef __PIC__
01077
01078 # define QTHREAD_PIC_PREFIX "xchg %%ebx, %4\n\t"
01079
01080 # define QTHREAD_PIC_SUFFIX "\n\txchg %%ebx, %4"
01081 # define QTHREAD_PIC_REG_4 "r"
01082 #else
01083 # define QTHREAD_PIC_PREFIX
01084 # define QTHREAD_PIC_SUFFIX
01085 # define QTHREAD_PIC_REG_4 "b"
01086 #endif
01087 #endif
01088
01089 __asm__ __volatile__ (
01090 QTHREAD_PIC_PREFIX
01091 "lock; cmpxchg8b (%2)"
01092 QTHREAD_PIC_SUFFIX
01093 :"=a"(ret.s.l),
01094 "=d"(ret.s.h)
01095 :"r"(operand),
01096 "a"(oldv.s.l),
01097 QTHREAD_PIC_REG_4(newv.s.l),
01098 "d"(oldv.s.h),
01099 "c"(newv.s.h)
01100 :"memory");
01101 return ret.i;
01102 # elif (QTHREAD_ASSEMBLY_ARCH == QTHREAD_AMD64)
01103
01104
01105
01106
01107
01108 # ifdef __PGI
01109
01110
01111 uint64_t retval;
01112 __asm__ __volatile__ ("lock cmpxchg %1,(%2)\n\t"
01113 "mov %%rax,(%0)"
01114 ::"r"(&retval), "r"(newval), "r"(operand),
01115 "a"(oldval)
01116 :"cc","memory");
01117 return retval;
01118 # else
01119 uint64_t retval;
01120 __asm__ __volatile__ ("lock; cmpxchg %1,(%2)"
01121 : "=a"(retval)
01122 : "r"(newval), "r" (operand),
01123 "a"(oldval)
01124 :"cc","memory");
01125 return retval;
01126 # endif
01127 # elif (QTHREAD_ASSEMBLY_ARCH == QTHREAD_POWERPC32)
01128
01129
01130 uint64_t retval;
01131 qthread_t *me = qthread_self();
01132
01133 qthread_lock(me, (aligned_t*)operand);
01134 retval = *operand;
01135 if (retval == oldval) {
01136 *operand = newval;
01137 }
01138 qthread_unlock(me, (aligned_t*)operand);
01139 return retval;
01140 # else
01141 # error Unimplemented assembly architecture for qthread_cas64
01142 # endif
01143 #endif
01144 }
01145
01146 static QINLINE aligned_t qthread_cas_xx(
01147 volatile aligned_t * addr,
01148 const aligned_t oldval,
01149 const aligned_t newval,
01150 const size_t length)
01151 {
01152 switch (length) {
01153 case 4:
01154 return qthread_cas32((volatile uint32_t *)addr, oldval, newval);
01155 case 8:
01156 return qthread_cas64((volatile uint64_t *)addr, oldval, newval);
01157 default:
01158
01159
01160 *(int *)(0) = 0;
01161 }
01162 return 0;
01163 }
01164
01165 static QINLINE void *qthread_cas_ptr_(
01166 void *volatile *const addr,
01167 void *const oldval,
01168 void *const newval)
01169 {
01170 #if (SIZEOF_VOIDP == 4)
01171 return (void *)(uintptr_t) qthread_cas32((volatile uint32_t *)
01172 addr, (uint32_t) (uintptr_t)
01173 oldval, (uint32_t) (uintptr_t)
01174 newval);
01175 #elif (SIZEOF_VOIDP == 8)
01176 return (void *)(uintptr_t) qthread_cas64((volatile uint64_t *)
01177 addr, (uint64_t) (uintptr_t)
01178 oldval, (uint64_t) (uintptr_t)
01179 newval);
01180 #else
01181 #error The size of void* either could not be determined, or is very unusual.
01182
01183
01184 *(int *)(0) = 0;
01185 return NULL;
01186 #endif
01187 }
01188 #endif
01189
01190 #ifdef QTHREAD_ATOMIC_CAS
01191 # define qthread_cas(ADDR, OLDV, NEWV) \
01192 __sync_val_compare_and_swap((ADDR), (OLDV), (NEWV))
01193 # define qthread_cas_ptr(ADDR, OLDV, NEWV) \
01194 (void*)__sync_val_compare_and_swap((ADDR), (OLDV), (NEWV))
01195 #else
01196 # define qthread_cas(ADDR, OLDV, NEWV) \
01197 qthread_cas_xx((volatile aligned_t*)(ADDR), (aligned_t)(OLDV), (aligned_t)(NEWV), sizeof(*(ADDR)))
01198 # ifdef QTHREAD_ATOMIC_CAS_PTR
01199 # define qthread_cas_ptr(ADDR, OLDV, NEWV) \
01200 (void*)__sync_val_compare_and_swap((ADDR), (OLDV), (NEWV))
01201 # else
01202 # define qthread_cas_ptr(ADDR, OLDV, NEWV) \
01203 qthread_cas_ptr_((void*volatile*const)(ADDR), (void*const)(OLDV), (void*const)(NEWV))
01204 # endif
01205 #endif
01206
01207 #ifdef QTHREAD_USE_ROSE_EXTENSIONS
01208 extern int __qthreads_temp;
01209 void qthread_reset_forCount(qthread_t *);
01210 #endif
01211
01212 Q_ENDCXX
01213
01214 #ifndef __cplusplus
01215
01216 # if defined(QTHREAD_ATOMIC_INCR) && !defined(QTHREAD_MUTEX_INCREMENT)
01217 # define qthread_incr( ADDR, INCVAL ) \
01218 __sync_fetch_and_add(ADDR, INCVAL)
01219 # else
01220 # define qthread_incr( ADDR, INCVAL ) \
01221 qthread_incr_xx( (volatile void*)(ADDR), (long int)(INCVAL), sizeof(*(ADDR)) )
01222 # endif
01223
01224 #else
01225 # include <qthread/qthread.hpp>
01226 #endif
01227
01228 #endif
01229
01230 #endif