13 #include "kmp_atomic.h" 16 typedef unsigned char uchar;
17 typedef unsigned short ushort;
565 #ifndef KMP_GOMP_COMPAT 566 int __kmp_atomic_mode = 1;
568 int __kmp_atomic_mode = 2;
574 kmp_atomic_lock_t __kmp_atomic_lock;
576 kmp_atomic_lock_t __kmp_atomic_lock_1i;
578 kmp_atomic_lock_t __kmp_atomic_lock_2i;
580 kmp_atomic_lock_t __kmp_atomic_lock_4i;
582 kmp_atomic_lock_t __kmp_atomic_lock_4r;
584 kmp_atomic_lock_t __kmp_atomic_lock_8i;
586 kmp_atomic_lock_t __kmp_atomic_lock_8r;
588 kmp_atomic_lock_t __kmp_atomic_lock_8c;
590 kmp_atomic_lock_t __kmp_atomic_lock_10r;
592 kmp_atomic_lock_t __kmp_atomic_lock_16r;
594 kmp_atomic_lock_t __kmp_atomic_lock_16c;
596 kmp_atomic_lock_t __kmp_atomic_lock_20c;
598 kmp_atomic_lock_t __kmp_atomic_lock_32c;
605 #define KMP_ATOMIC_VOLATILE volatile 607 #if (KMP_ARCH_X86) && KMP_HAVE_QUAD 609 static inline Quad_a4_t operator+(Quad_a4_t &lhs, Quad_a4_t &rhs) {
610 return lhs.q + rhs.q;
612 static inline Quad_a4_t operator-(Quad_a4_t &lhs, Quad_a4_t &rhs) {
613 return lhs.q - rhs.q;
615 static inline Quad_a4_t operator*(Quad_a4_t &lhs, Quad_a4_t &rhs) {
616 return lhs.q * rhs.q;
618 static inline Quad_a4_t operator/(Quad_a4_t &lhs, Quad_a4_t &rhs) {
619 return lhs.q / rhs.q;
621 static inline bool operator<(Quad_a4_t &lhs, Quad_a4_t &rhs) {
622 return lhs.q < rhs.q;
624 static inline bool operator>(Quad_a4_t &lhs, Quad_a4_t &rhs) {
625 return lhs.q > rhs.q;
628 static inline Quad_a16_t operator+(Quad_a16_t &lhs, Quad_a16_t &rhs) {
629 return lhs.q + rhs.q;
631 static inline Quad_a16_t operator-(Quad_a16_t &lhs, Quad_a16_t &rhs) {
632 return lhs.q - rhs.q;
634 static inline Quad_a16_t operator*(Quad_a16_t &lhs, Quad_a16_t &rhs) {
635 return lhs.q * rhs.q;
637 static inline Quad_a16_t operator/(Quad_a16_t &lhs, Quad_a16_t &rhs) {
638 return lhs.q / rhs.q;
640 static inline bool operator<(Quad_a16_t &lhs, Quad_a16_t &rhs) {
641 return lhs.q < rhs.q;
643 static inline bool operator>(Quad_a16_t &lhs, Quad_a16_t &rhs) {
644 return lhs.q > rhs.q;
647 static inline kmp_cmplx128_a4_t operator+(kmp_cmplx128_a4_t &lhs,
648 kmp_cmplx128_a4_t &rhs) {
649 return lhs.q + rhs.q;
651 static inline kmp_cmplx128_a4_t operator-(kmp_cmplx128_a4_t &lhs,
652 kmp_cmplx128_a4_t &rhs) {
653 return lhs.q - rhs.q;
655 static inline kmp_cmplx128_a4_t operator*(kmp_cmplx128_a4_t &lhs,
656 kmp_cmplx128_a4_t &rhs) {
657 return lhs.q * rhs.q;
659 static inline kmp_cmplx128_a4_t operator/(kmp_cmplx128_a4_t &lhs,
660 kmp_cmplx128_a4_t &rhs) {
661 return lhs.q / rhs.q;
664 static inline kmp_cmplx128_a16_t operator+(kmp_cmplx128_a16_t &lhs,
665 kmp_cmplx128_a16_t &rhs) {
666 return lhs.q + rhs.q;
668 static inline kmp_cmplx128_a16_t operator-(kmp_cmplx128_a16_t &lhs,
669 kmp_cmplx128_a16_t &rhs) {
670 return lhs.q - rhs.q;
672 static inline kmp_cmplx128_a16_t operator*(kmp_cmplx128_a16_t &lhs,
673 kmp_cmplx128_a16_t &rhs) {
674 return lhs.q * rhs.q;
676 static inline kmp_cmplx128_a16_t operator/(kmp_cmplx128_a16_t &lhs,
677 kmp_cmplx128_a16_t &rhs) {
678 return lhs.q / rhs.q;
681 #endif // (KMP_ARCH_X86) && KMP_HAVE_QUAD 688 #define KMP_CHECK_GTID \ 689 if (gtid == KMP_GTID_UNKNOWN) { \ 690 gtid = __kmp_entry_gtid(); \ 691 } // check and get gtid when needed 698 #define ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, RET_TYPE) \ 699 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \ 700 TYPE *lhs, TYPE rhs) { \ 701 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 702 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid)); 706 #define ATOMIC_LOCK0 __kmp_atomic_lock // all types, for Gnu compat 707 #define ATOMIC_LOCK1i __kmp_atomic_lock_1i // char 708 #define ATOMIC_LOCK2i __kmp_atomic_lock_2i // short 709 #define ATOMIC_LOCK4i __kmp_atomic_lock_4i // long int 710 #define ATOMIC_LOCK4r __kmp_atomic_lock_4r // float 711 #define ATOMIC_LOCK8i __kmp_atomic_lock_8i // long long int 712 #define ATOMIC_LOCK8r __kmp_atomic_lock_8r // double 713 #define ATOMIC_LOCK8c __kmp_atomic_lock_8c // float complex 714 #define ATOMIC_LOCK10r __kmp_atomic_lock_10r // long double 715 #define ATOMIC_LOCK16r __kmp_atomic_lock_16r // _Quad 716 #define ATOMIC_LOCK16c __kmp_atomic_lock_16c // double complex 717 #define ATOMIC_LOCK20c __kmp_atomic_lock_20c // long double complex 718 #define ATOMIC_LOCK32c __kmp_atomic_lock_32c // _Quad complex 726 #define OP_CRITICAL(OP, LCK_ID) \ 727 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 731 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); 733 #define OP_UPDATE_CRITICAL(TYPE, OP, LCK_ID) \ 734 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 735 (*lhs) = (TYPE)((*lhs)OP rhs); \ 736 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); 760 #ifdef KMP_GOMP_COMPAT 761 #define OP_GOMP_CRITICAL(OP, FLAG) \ 762 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 764 OP_CRITICAL(OP, 0); \ 768 #define OP_UPDATE_GOMP_CRITICAL(TYPE, OP, FLAG) \ 769 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 771 OP_UPDATE_CRITICAL(TYPE, OP, 0); \ 775 #define OP_GOMP_CRITICAL(OP, FLAG) 776 #define OP_UPDATE_GOMP_CRITICAL(TYPE, OP, FLAG) 780 #define KMP_DO_PAUSE _mm_delay_32(1) 790 #define OP_CMPXCHG(TYPE, BITS, OP) \ 792 TYPE old_value, new_value; \ 793 old_value = *(TYPE volatile *)lhs; \ 794 new_value = (TYPE)(old_value OP rhs); \ 795 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 796 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 797 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ 800 old_value = *(TYPE volatile *)lhs; \ 801 new_value = (TYPE)(old_value OP rhs); \ 812 #define OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \ 816 kmp_int##BITS *vvv; \ 818 struct _sss old_value, new_value; \ 819 old_value.vvv = (kmp_int##BITS *)&old_value.cmp; \ 820 new_value.vvv = (kmp_int##BITS *)&new_value.cmp; \ 821 *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \ 822 new_value.cmp = (TYPE)(old_value.cmp OP rhs); \ 823 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 824 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) old_value.vvv, \ 825 *VOLATILE_CAST(kmp_int##BITS *) new_value.vvv)) { \ 828 *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \ 829 new_value.cmp = (TYPE)(old_value.cmp OP rhs); \ 833 #endif // USE_CMPXCHG_FIX 835 #if KMP_OS_WINDOWS && (KMP_ARCH_AARCH64 || KMP_ARCH_ARM) 839 #define OP_CMPXCHG(TYPE, BITS, OP) \ 843 kmp_int##BITS *vvv; \ 845 struct _sss old_value, new_value; \ 846 old_value.vvv = (kmp_int##BITS *)&old_value.cmp; \ 847 new_value.vvv = (kmp_int##BITS *)&new_value.cmp; \ 848 *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \ 849 new_value.cmp = old_value.cmp OP rhs; \ 850 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 851 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) old_value.vvv, \ 852 *VOLATILE_CAST(kmp_int##BITS *) new_value.vvv)) { \ 855 *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \ 856 new_value.cmp = old_value.cmp OP rhs; \ 860 #undef OP_UPDATE_CRITICAL 861 #define OP_UPDATE_CRITICAL(TYPE, OP, LCK_ID) \ 862 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 863 (*lhs) = (*lhs)OP rhs; \ 864 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); 866 #endif // KMP_OS_WINDOWS && (KMP_ARCH_AARCH64 || KMP_ARCH_ARM) 868 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 872 #define ATOMIC_FIXED_ADD(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 874 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 875 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \ 877 KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \ 880 #define ATOMIC_CMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 882 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 883 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \ 884 OP_CMPXCHG(TYPE, BITS, OP) \ 889 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, \ 891 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 892 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \ 893 OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \ 896 #endif // USE_CMPXCHG_FIX 901 #define ATOMIC_FIXED_ADD(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 903 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 904 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \ 905 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 907 KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \ 910 OP_UPDATE_CRITICAL(TYPE, OP, \ 915 #define ATOMIC_CMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 917 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 918 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \ 919 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 920 OP_CMPXCHG(TYPE, BITS, OP) \ 923 OP_UPDATE_CRITICAL(TYPE, OP, \ 930 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, \ 932 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 933 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \ 934 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 935 OP_CMPXCHG(TYPE, BITS, OP) \ 938 OP_UPDATE_CRITICAL(TYPE, OP, \ 943 #endif // USE_CMPXCHG_FIX 947 ATOMIC_FIXED_ADD(fixed4, add, kmp_int32, 32, +, 4i, 3,
949 ATOMIC_FIXED_ADD(fixed4, sub, kmp_int32, 32, -, 4i, 3,
952 ATOMIC_CMPXCHG(float4, add, kmp_real32, 32, +, 4r, 3,
954 ATOMIC_CMPXCHG(float4, sub, kmp_real32, 32, -, 4r, 3,
958 ATOMIC_FIXED_ADD(fixed8, add, kmp_int64, 64, +, 8i, 7,
960 ATOMIC_FIXED_ADD(fixed8, sub, kmp_int64, 64, -, 8i, 7,
963 ATOMIC_CMPXCHG(float8, add, kmp_real64, 64, +, 8r, 7,
965 ATOMIC_CMPXCHG(float8, sub, kmp_real64, 64, -, 8r, 7,
983 ATOMIC_CMPXCHG(fixed1, add, kmp_int8, 8, +, 1i, 0,
985 ATOMIC_CMPXCHG(fixed1, andb, kmp_int8, 8, &, 1i, 0,
987 ATOMIC_CMPXCHG(fixed1, div, kmp_int8, 8, /, 1i, 0,
989 ATOMIC_CMPXCHG(fixed1u, div, kmp_uint8, 8, /, 1i, 0,
991 ATOMIC_CMPXCHG(fixed1, mul, kmp_int8, 8, *, 1i, 0,
993 ATOMIC_CMPXCHG(fixed1, orb, kmp_int8, 8, |, 1i, 0,
995 ATOMIC_CMPXCHG(fixed1, shl, kmp_int8, 8, <<, 1i, 0,
997 ATOMIC_CMPXCHG(fixed1, shr, kmp_int8, 8, >>, 1i, 0,
999 ATOMIC_CMPXCHG(fixed1u, shr, kmp_uint8, 8, >>, 1i, 0,
1001 ATOMIC_CMPXCHG(fixed1, sub, kmp_int8, 8, -, 1i, 0,
1003 ATOMIC_CMPXCHG(fixed1, xor, kmp_int8, 8, ^, 1i, 0,
1005 ATOMIC_CMPXCHG(fixed2, add, kmp_int16, 16, +, 2i, 1,
1007 ATOMIC_CMPXCHG(fixed2, andb, kmp_int16, 16, &, 2i, 1,
1009 ATOMIC_CMPXCHG(fixed2, div, kmp_int16, 16, /, 2i, 1,
1011 ATOMIC_CMPXCHG(fixed2u, div, kmp_uint16, 16, /, 2i, 1,
1013 ATOMIC_CMPXCHG(fixed2, mul, kmp_int16, 16, *, 2i, 1,
1015 ATOMIC_CMPXCHG(fixed2, orb, kmp_int16, 16, |, 2i, 1,
1017 ATOMIC_CMPXCHG(fixed2, shl, kmp_int16, 16, <<, 2i, 1,
1019 ATOMIC_CMPXCHG(fixed2, shr, kmp_int16, 16, >>, 2i, 1,
1021 ATOMIC_CMPXCHG(fixed2u, shr, kmp_uint16, 16, >>, 2i, 1,
1023 ATOMIC_CMPXCHG(fixed2, sub, kmp_int16, 16, -, 2i, 1,
1025 ATOMIC_CMPXCHG(fixed2, xor, kmp_int16, 16, ^, 2i, 1,
1027 ATOMIC_CMPXCHG(fixed4, andb, kmp_int32, 32, &, 4i, 3,
1029 ATOMIC_CMPXCHG(fixed4, div, kmp_int32, 32, /, 4i, 3,
1031 ATOMIC_CMPXCHG(fixed4u, div, kmp_uint32, 32, /, 4i, 3,
1033 ATOMIC_CMPXCHG(fixed4, mul, kmp_int32, 32, *, 4i, 3,
1035 ATOMIC_CMPXCHG(fixed4, orb, kmp_int32, 32, |, 4i, 3,
1037 ATOMIC_CMPXCHG(fixed4, shl, kmp_int32, 32, <<, 4i, 3,
1039 ATOMIC_CMPXCHG(fixed4, shr, kmp_int32, 32, >>, 4i, 3,
1041 ATOMIC_CMPXCHG(fixed4u, shr, kmp_uint32, 32, >>, 4i, 3,
1043 ATOMIC_CMPXCHG(fixed4, xor, kmp_int32, 32, ^, 4i, 3,
1045 ATOMIC_CMPXCHG(fixed8, andb, kmp_int64, 64, &, 8i, 7,
1047 ATOMIC_CMPXCHG(fixed8, div, kmp_int64, 64, /, 8i, 7,
1049 ATOMIC_CMPXCHG(fixed8u, div, kmp_uint64, 64, /, 8i, 7,
1051 ATOMIC_CMPXCHG(fixed8, mul, kmp_int64, 64, *, 8i, 7,
1053 ATOMIC_CMPXCHG(fixed8, orb, kmp_int64, 64, |, 8i, 7,
1055 ATOMIC_CMPXCHG(fixed8, shl, kmp_int64, 64, <<, 8i, 7,
1057 ATOMIC_CMPXCHG(fixed8, shr, kmp_int64, 64, >>, 8i, 7,
1059 ATOMIC_CMPXCHG(fixed8u, shr, kmp_uint64, 64, >>, 8i, 7,
1061 ATOMIC_CMPXCHG(fixed8, xor, kmp_int64, 64, ^, 8i, 7,
1063 ATOMIC_CMPXCHG(float4, div, kmp_real32, 32, /, 4r, 3,
1065 ATOMIC_CMPXCHG(float4, mul, kmp_real32, 32, *, 4r, 3,
1067 ATOMIC_CMPXCHG(float8, div, kmp_real64, 64, /, 8r, 7,
1069 ATOMIC_CMPXCHG(float8, mul, kmp_real64, 64, *, 8r, 7,
1079 #define ATOMIC_CRIT_L(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 1080 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1081 OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \ 1082 OP_CRITICAL(= *lhs OP, LCK_ID) \ 1085 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1089 #define ATOMIC_CMPX_L(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, GOMP_FLAG) \ 1090 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1091 OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \ 1092 OP_CMPXCHG(TYPE, BITS, OP) \ 1098 #define ATOMIC_CMPX_L(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, GOMP_FLAG) \ 1099 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1100 OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \ 1101 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 1102 OP_CMPXCHG(TYPE, BITS, OP) \ 1105 OP_CRITICAL(= *lhs OP, LCK_ID) \ 1110 ATOMIC_CMPX_L(fixed1, andl,
char, 8, &&, 1i, 0,
1112 ATOMIC_CMPX_L(fixed1, orl,
char, 8, ||, 1i, 0,
1114 ATOMIC_CMPX_L(fixed2, andl,
short, 16, &&, 2i, 1,
1116 ATOMIC_CMPX_L(fixed2, orl,
short, 16, ||, 2i, 1,
1118 ATOMIC_CMPX_L(fixed4, andl, kmp_int32, 32, &&, 4i, 3,
1120 ATOMIC_CMPX_L(fixed4, orl, kmp_int32, 32, ||, 4i, 3,
1122 ATOMIC_CMPX_L(fixed8, andl, kmp_int64, 64, &&, 8i, 7,
1124 ATOMIC_CMPX_L(fixed8, orl, kmp_int64, 64, ||, 8i, 7,
1136 #define MIN_MAX_CRITSECT(OP, LCK_ID) \ 1137 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 1139 if (*lhs OP rhs) { \ 1142 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); 1145 #ifdef KMP_GOMP_COMPAT 1146 #define GOMP_MIN_MAX_CRITSECT(OP, FLAG) \ 1147 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 1149 MIN_MAX_CRITSECT(OP, 0); \ 1153 #define GOMP_MIN_MAX_CRITSECT(OP, FLAG) 1157 #define MIN_MAX_CMPXCHG(TYPE, BITS, OP) \ 1159 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 1162 old_value = temp_val; \ 1163 while (old_value OP rhs && \ 1164 !KMP_COMPARE_AND_STORE_ACQ##BITS( \ 1165 (kmp_int##BITS *)lhs, \ 1166 *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 1167 *VOLATILE_CAST(kmp_int##BITS *) & rhs)) { \ 1169 old_value = temp_val; \ 1175 #define MIN_MAX_CRITICAL(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 1176 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1177 if (*lhs OP rhs) { \ 1178 GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \ 1179 MIN_MAX_CRITSECT(OP, LCK_ID) \ 1183 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1187 #define MIN_MAX_COMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 1189 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1190 if (*lhs OP rhs) { \ 1191 GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \ 1192 MIN_MAX_CMPXCHG(TYPE, BITS, OP) \ 1199 #define MIN_MAX_COMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 1201 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1202 if (*lhs OP rhs) { \ 1203 GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \ 1204 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 1205 MIN_MAX_CMPXCHG(TYPE, BITS, OP) \ 1208 MIN_MAX_CRITSECT(OP, LCK_ID) \ 1214 MIN_MAX_COMPXCHG(fixed1, max,
char, 8, <, 1i, 0,
1216 MIN_MAX_COMPXCHG(fixed1, min,
char, 8, >, 1i, 0,
1218 MIN_MAX_COMPXCHG(fixed2, max,
short, 16, <, 2i, 1,
1220 MIN_MAX_COMPXCHG(fixed2, min,
short, 16, >, 2i, 1,
1222 MIN_MAX_COMPXCHG(fixed4, max, kmp_int32, 32, <, 4i, 3,
1224 MIN_MAX_COMPXCHG(fixed4, min, kmp_int32, 32, >, 4i, 3,
1226 MIN_MAX_COMPXCHG(fixed8, max, kmp_int64, 64, <, 8i, 7,
1228 MIN_MAX_COMPXCHG(fixed8, min, kmp_int64, 64, >, 8i, 7,
1230 MIN_MAX_COMPXCHG(float4, max, kmp_real32, 32, <, 4r, 3,
1232 MIN_MAX_COMPXCHG(float4, min, kmp_real32, 32, >, 4r, 3,
1234 MIN_MAX_COMPXCHG(float8, max, kmp_real64, 64, <, 8r, 7,
1236 MIN_MAX_COMPXCHG(float8, min, kmp_real64, 64, >, 8r, 7,
1238 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1239 MIN_MAX_CRITICAL(float10, max,
long double, <, 10r,
1241 MIN_MAX_CRITICAL(float10, min,
long double, >, 10r,
1243 #endif // KMP_ARCH_X86 || KMP_ARCH_X86_64 1245 MIN_MAX_CRITICAL(float16, max, QUAD_LEGACY, <, 16r,
1247 MIN_MAX_CRITICAL(float16, min, QUAD_LEGACY, >, 16r,
1250 MIN_MAX_CRITICAL(float16, max_a16, Quad_a16_t, <, 16r,
1252 MIN_MAX_CRITICAL(float16, min_a16, Quad_a16_t, >, 16r,
1254 #endif // (KMP_ARCH_X86) 1255 #endif // KMP_HAVE_QUAD 1259 #define ATOMIC_CRIT_EQV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 1260 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1261 OP_GOMP_CRITICAL(^= (TYPE) ~, GOMP_FLAG) \ 1262 OP_CRITICAL(^= (TYPE) ~, LCK_ID) \ 1266 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1269 #define ATOMIC_CMPX_EQV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 1271 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1272 OP_GOMP_CRITICAL(^= (TYPE) ~, GOMP_FLAG) \ 1273 OP_CMPXCHG(TYPE, BITS, OP) \ 1279 #define ATOMIC_CMPX_EQV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 1281 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1282 OP_GOMP_CRITICAL(^= (TYPE) ~, GOMP_FLAG) \ 1283 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 1284 OP_CMPXCHG(TYPE, BITS, OP) \ 1287 OP_CRITICAL(^= (TYPE) ~, LCK_ID) \ 1292 ATOMIC_CMPXCHG(fixed1, neqv, kmp_int8, 8, ^, 1i, 0,
1294 ATOMIC_CMPXCHG(fixed2, neqv, kmp_int16, 16, ^, 2i, 1,
1296 ATOMIC_CMPXCHG(fixed4, neqv, kmp_int32, 32, ^, 4i, 3,
1298 ATOMIC_CMPXCHG(fixed8, neqv, kmp_int64, 64, ^, 8i, 7,
1300 ATOMIC_CMPX_EQV(fixed1, eqv, kmp_int8, 8, ^~, 1i, 0,
1302 ATOMIC_CMPX_EQV(fixed2, eqv, kmp_int16, 16, ^~, 2i, 1,
1304 ATOMIC_CMPX_EQV(fixed4, eqv, kmp_int32, 32, ^~, 4i, 3,
1306 ATOMIC_CMPX_EQV(fixed8, eqv, kmp_int64, 64, ^~, 8i, 7,
1315 #define ATOMIC_CRITICAL(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 1316 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1317 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \ 1318 OP_UPDATE_CRITICAL(TYPE, OP, LCK_ID) \ 1322 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1324 ATOMIC_CRITICAL(float10, add,
long double, +, 10r,
1326 ATOMIC_CRITICAL(float10, sub,
long double, -, 10r,
1328 ATOMIC_CRITICAL(float10, mul,
long double, *, 10r,
1330 ATOMIC_CRITICAL(float10, div,
long double, /, 10r,
1332 #endif // KMP_ARCH_X86 || KMP_ARCH_X86_64 1335 ATOMIC_CRITICAL(float16, add, QUAD_LEGACY, +, 16r,
1337 ATOMIC_CRITICAL(float16, sub, QUAD_LEGACY, -, 16r,
1339 ATOMIC_CRITICAL(float16, mul, QUAD_LEGACY, *, 16r,
1341 ATOMIC_CRITICAL(float16, div, QUAD_LEGACY, /, 16r,
1344 ATOMIC_CRITICAL(float16, add_a16, Quad_a16_t, +, 16r,
1346 ATOMIC_CRITICAL(float16, sub_a16, Quad_a16_t, -, 16r,
1348 ATOMIC_CRITICAL(float16, mul_a16, Quad_a16_t, *, 16r,
1350 ATOMIC_CRITICAL(float16, div_a16, Quad_a16_t, /, 16r,
1352 #endif // (KMP_ARCH_X86) 1353 #endif // KMP_HAVE_QUAD 1358 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, add, kmp_cmplx32, 64, +, 8c, 7,
1360 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, sub, kmp_cmplx32, 64, -, 8c, 7,
1362 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, mul, kmp_cmplx32, 64, *, 8c, 7,
1364 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, div, kmp_cmplx32, 64, /, 8c, 7,
1368 ATOMIC_CRITICAL(cmplx4, add, kmp_cmplx32, +, 8c, 1)
1369 ATOMIC_CRITICAL(cmplx4, sub, kmp_cmplx32, -, 8c, 1)
1370 ATOMIC_CRITICAL(cmplx4, mul, kmp_cmplx32, *, 8c, 1)
1371 ATOMIC_CRITICAL(cmplx4, div, kmp_cmplx32, /, 8c, 1)
1372 #endif // USE_CMPXCHG_FIX 1374 ATOMIC_CRITICAL(cmplx8, add, kmp_cmplx64, +, 16c, 1)
1375 ATOMIC_CRITICAL(cmplx8, sub, kmp_cmplx64, -, 16c, 1)
1376 ATOMIC_CRITICAL(cmplx8, mul, kmp_cmplx64, *, 16c, 1)
1377 ATOMIC_CRITICAL(cmplx8, div, kmp_cmplx64, /, 16c, 1)
1378 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1379 ATOMIC_CRITICAL(cmplx10, add, kmp_cmplx80, +, 20c,
1381 ATOMIC_CRITICAL(cmplx10, sub, kmp_cmplx80, -, 20c,
1383 ATOMIC_CRITICAL(cmplx10, mul, kmp_cmplx80, *, 20c,
1385 ATOMIC_CRITICAL(cmplx10, div, kmp_cmplx80, /, 20c,
1387 #endif // KMP_ARCH_X86 || KMP_ARCH_X86_64 1389 ATOMIC_CRITICAL(cmplx16, add, CPLX128_LEG, +, 32c,
1391 ATOMIC_CRITICAL(cmplx16, sub, CPLX128_LEG, -, 32c,
1393 ATOMIC_CRITICAL(cmplx16, mul, CPLX128_LEG, *, 32c,
1395 ATOMIC_CRITICAL(cmplx16, div, CPLX128_LEG, /, 32c,
1398 ATOMIC_CRITICAL(cmplx16, add_a16, kmp_cmplx128_a16_t, +, 32c,
1400 ATOMIC_CRITICAL(cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c,
1402 ATOMIC_CRITICAL(cmplx16, mul_a16, kmp_cmplx128_a16_t, *, 32c,
1404 ATOMIC_CRITICAL(cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c,
1406 #endif // (KMP_ARCH_X86) 1407 #endif // KMP_HAVE_QUAD 1411 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1419 #define OP_CRITICAL_REV(TYPE, OP, LCK_ID) \ 1420 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 1422 (*lhs) = (TYPE)((rhs)OP(*lhs)); \ 1424 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); 1426 #ifdef KMP_GOMP_COMPAT 1427 #define OP_GOMP_CRITICAL_REV(TYPE, OP, FLAG) \ 1428 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 1430 OP_CRITICAL_REV(TYPE, OP, 0); \ 1435 #define OP_GOMP_CRITICAL_REV(TYPE, OP, FLAG) 1443 #define ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, RET_TYPE) \ 1444 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_rev(ident_t *id_ref, int gtid, \ 1445 TYPE *lhs, TYPE rhs) { \ 1446 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 1447 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_rev: T#%d\n", gtid)); 1456 #define OP_CMPXCHG_REV(TYPE, BITS, OP) \ 1458 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 1459 TYPE old_value, new_value; \ 1461 old_value = temp_val; \ 1462 new_value = (TYPE)(rhs OP old_value); \ 1463 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 1464 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 1465 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ 1469 old_value = temp_val; \ 1470 new_value = (TYPE)(rhs OP old_value); \ 1475 #define ATOMIC_CMPXCHG_REV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, GOMP_FLAG) \ 1476 ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, void) \ 1477 OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \ 1478 OP_CMPXCHG_REV(TYPE, BITS, OP) \ 1495 ATOMIC_CMPXCHG_REV(fixed1, div, kmp_int8, 8, /, 1i,
1497 ATOMIC_CMPXCHG_REV(fixed1u, div, kmp_uint8, 8, /, 1i,
1499 ATOMIC_CMPXCHG_REV(fixed1, shl, kmp_int8, 8, <<, 1i,
1501 ATOMIC_CMPXCHG_REV(fixed1, shr, kmp_int8, 8, >>, 1i,
1503 ATOMIC_CMPXCHG_REV(fixed1u, shr, kmp_uint8, 8, >>, 1i,
1505 ATOMIC_CMPXCHG_REV(fixed1, sub, kmp_int8, 8, -, 1i,
1508 ATOMIC_CMPXCHG_REV(fixed2, div, kmp_int16, 16, /, 2i,
1510 ATOMIC_CMPXCHG_REV(fixed2u, div, kmp_uint16, 16, /, 2i,
1512 ATOMIC_CMPXCHG_REV(fixed2, shl, kmp_int16, 16, <<, 2i,
1514 ATOMIC_CMPXCHG_REV(fixed2, shr, kmp_int16, 16, >>, 2i,
1516 ATOMIC_CMPXCHG_REV(fixed2u, shr, kmp_uint16, 16, >>, 2i,
1518 ATOMIC_CMPXCHG_REV(fixed2, sub, kmp_int16, 16, -, 2i,
1521 ATOMIC_CMPXCHG_REV(fixed4, div, kmp_int32, 32, /, 4i,
1523 ATOMIC_CMPXCHG_REV(fixed4u, div, kmp_uint32, 32, /, 4i,
1525 ATOMIC_CMPXCHG_REV(fixed4, shl, kmp_int32, 32, <<, 4i,
1527 ATOMIC_CMPXCHG_REV(fixed4, shr, kmp_int32, 32, >>, 4i,
1529 ATOMIC_CMPXCHG_REV(fixed4u, shr, kmp_uint32, 32, >>, 4i,
1531 ATOMIC_CMPXCHG_REV(fixed4, sub, kmp_int32, 32, -, 4i,
1534 ATOMIC_CMPXCHG_REV(fixed8, div, kmp_int64, 64, /, 8i,
1536 ATOMIC_CMPXCHG_REV(fixed8u, div, kmp_uint64, 64, /, 8i,
1538 ATOMIC_CMPXCHG_REV(fixed8, shl, kmp_int64, 64, <<, 8i,
1540 ATOMIC_CMPXCHG_REV(fixed8, shr, kmp_int64, 64, >>, 8i,
1542 ATOMIC_CMPXCHG_REV(fixed8u, shr, kmp_uint64, 64, >>, 8i,
1544 ATOMIC_CMPXCHG_REV(fixed8, sub, kmp_int64, 64, -, 8i,
1547 ATOMIC_CMPXCHG_REV(float4, div, kmp_real32, 32, /, 4r,
1549 ATOMIC_CMPXCHG_REV(float4, sub, kmp_real32, 32, -, 4r,
1552 ATOMIC_CMPXCHG_REV(float8, div, kmp_real64, 64, /, 8r,
1554 ATOMIC_CMPXCHG_REV(float8, sub, kmp_real64, 64, -, 8r,
1564 #define ATOMIC_CRITICAL_REV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 1565 ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, void) \ 1566 OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \ 1567 OP_CRITICAL_REV(TYPE, OP, LCK_ID) \ 1572 ATOMIC_CRITICAL_REV(float10, sub,
long double, -, 10r,
1574 ATOMIC_CRITICAL_REV(float10, div,
long double, /, 10r,
1578 ATOMIC_CRITICAL_REV(float16, sub, QUAD_LEGACY, -, 16r,
1580 ATOMIC_CRITICAL_REV(float16, div, QUAD_LEGACY, /, 16r,
1583 ATOMIC_CRITICAL_REV(float16, sub_a16, Quad_a16_t, -, 16r,
1585 ATOMIC_CRITICAL_REV(float16, div_a16, Quad_a16_t, /, 16r,
1587 #endif // KMP_ARCH_X86 1588 #endif // KMP_HAVE_QUAD 1591 ATOMIC_CRITICAL_REV(cmplx4, sub, kmp_cmplx32, -, 8c,
1593 ATOMIC_CRITICAL_REV(cmplx4, div, kmp_cmplx32, /, 8c,
1595 ATOMIC_CRITICAL_REV(cmplx8, sub, kmp_cmplx64, -, 16c,
1597 ATOMIC_CRITICAL_REV(cmplx8, div, kmp_cmplx64, /, 16c,
1599 ATOMIC_CRITICAL_REV(cmplx10, sub, kmp_cmplx80, -, 20c,
1601 ATOMIC_CRITICAL_REV(cmplx10, div, kmp_cmplx80, /, 20c,
1604 ATOMIC_CRITICAL_REV(cmplx16, sub, CPLX128_LEG, -, 32c,
1606 ATOMIC_CRITICAL_REV(cmplx16, div, CPLX128_LEG, /, 32c,
1609 ATOMIC_CRITICAL_REV(cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c,
1611 ATOMIC_CRITICAL_REV(cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c,
1613 #endif // KMP_ARCH_X86 1614 #endif // KMP_HAVE_QUAD 1616 #endif // KMP_ARCH_X86 || KMP_ARCH_X86_64 1630 #define ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1631 void __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( \ 1632 ident_t *id_ref, int gtid, TYPE *lhs, RTYPE rhs) { \ 1633 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 1635 ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", \ 1639 #define ATOMIC_CRITICAL_FP(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, LCK_ID, \ 1641 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1642 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \ 1643 OP_UPDATE_CRITICAL(TYPE, OP, LCK_ID) \ 1647 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1650 #define ATOMIC_CMPXCHG_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \ 1651 LCK_ID, MASK, GOMP_FLAG) \ 1652 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1653 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \ 1654 OP_CMPXCHG(TYPE, BITS, OP) \ 1660 #define ATOMIC_CMPXCHG_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \ 1661 LCK_ID, MASK, GOMP_FLAG) \ 1662 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1663 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \ 1664 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 1665 OP_CMPXCHG(TYPE, BITS, OP) \ 1668 OP_UPDATE_CRITICAL(TYPE, OP, \ 1675 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1677 #define ATOMIC_CMPXCHG_REV_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \ 1678 RTYPE, LCK_ID, MASK, GOMP_FLAG) \ 1679 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1680 OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \ 1681 OP_CMPXCHG_REV(TYPE, BITS, OP) \ 1683 #define ATOMIC_CRITICAL_REV_FP(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \ 1684 LCK_ID, GOMP_FLAG) \ 1685 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1686 OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \ 1687 OP_CRITICAL_REV(TYPE, OP, LCK_ID) \ 1692 ATOMIC_CMPXCHG_MIX(fixed1,
char, mul, 8, *, float8, kmp_real64, 1i, 0,
1694 ATOMIC_CMPXCHG_MIX(fixed1,
char, div, 8, /, float8, kmp_real64, 1i, 0,
1696 ATOMIC_CMPXCHG_MIX(fixed2,
short, mul, 16, *, float8, kmp_real64, 2i, 1,
1698 ATOMIC_CMPXCHG_MIX(fixed2,
short, div, 16, /, float8, kmp_real64, 2i, 1,
1700 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, mul, 32, *, float8, kmp_real64, 4i, 3,
1702 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, div, 32, /, float8, kmp_real64, 4i, 3,
1704 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, mul, 64, *, float8, kmp_real64, 8i, 7,
1706 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, div, 64, /, float8, kmp_real64, 8i, 7,
1708 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, add, 32, +, float8, kmp_real64, 4r, 3,
1710 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, sub, 32, -, float8, kmp_real64, 4r, 3,
1712 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, mul, 32, *, float8, kmp_real64, 4r, 3,
1714 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, div, 32, /, float8, kmp_real64, 4r, 3,
1720 ATOMIC_CMPXCHG_MIX(fixed1,
char, add, 8, +, fp, _Quad, 1i, 0,
1722 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, add, 8, +, fp, _Quad, 1i, 0,
1724 ATOMIC_CMPXCHG_MIX(fixed1,
char, sub, 8, -, fp, _Quad, 1i, 0,
1726 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, sub, 8, -, fp, _Quad, 1i, 0,
1728 ATOMIC_CMPXCHG_MIX(fixed1,
char, mul, 8, *, fp, _Quad, 1i, 0,
1730 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, mul, 8, *, fp, _Quad, 1i, 0,
1732 ATOMIC_CMPXCHG_MIX(fixed1,
char, div, 8, /, fp, _Quad, 1i, 0,
1734 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, div, 8, /, fp, _Quad, 1i, 0,
1737 ATOMIC_CMPXCHG_MIX(fixed2,
short, add, 16, +, fp, _Quad, 2i, 1,
1739 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, add, 16, +, fp, _Quad, 2i, 1,
1741 ATOMIC_CMPXCHG_MIX(fixed2,
short, sub, 16, -, fp, _Quad, 2i, 1,
1743 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, sub, 16, -, fp, _Quad, 2i, 1,
1745 ATOMIC_CMPXCHG_MIX(fixed2,
short, mul, 16, *, fp, _Quad, 2i, 1,
1747 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, mul, 16, *, fp, _Quad, 2i, 1,
1749 ATOMIC_CMPXCHG_MIX(fixed2,
short, div, 16, /, fp, _Quad, 2i, 1,
1751 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, div, 16, /, fp, _Quad, 2i, 1,
1754 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, add, 32, +, fp, _Quad, 4i, 3,
1756 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, add, 32, +, fp, _Quad, 4i, 3,
1758 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, sub, 32, -, fp, _Quad, 4i, 3,
1760 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, sub, 32, -, fp, _Quad, 4i, 3,
1762 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, mul, 32, *, fp, _Quad, 4i, 3,
1764 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, mul, 32, *, fp, _Quad, 4i, 3,
1766 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, div, 32, /, fp, _Quad, 4i, 3,
1768 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, div, 32, /, fp, _Quad, 4i, 3,
1771 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, add, 64, +, fp, _Quad, 8i, 7,
1773 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, add, 64, +, fp, _Quad, 8i, 7,
1775 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, sub, 64, -, fp, _Quad, 8i, 7,
1777 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, sub, 64, -, fp, _Quad, 8i, 7,
1779 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, mul, 64, *, fp, _Quad, 8i, 7,
1781 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, mul, 64, *, fp, _Quad, 8i, 7,
1783 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, div, 64, /, fp, _Quad, 8i, 7,
1785 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, div, 64, /, fp, _Quad, 8i, 7,
1788 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, add, 32, +, fp, _Quad, 4r, 3,
1790 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, sub, 32, -, fp, _Quad, 4r, 3,
1792 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, mul, 32, *, fp, _Quad, 4r, 3,
1794 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, div, 32, /, fp, _Quad, 4r, 3,
1797 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, add, 64, +, fp, _Quad, 8r, 7,
1799 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, sub, 64, -, fp, _Quad, 8r, 7,
1801 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, mul, 64, *, fp, _Quad, 8r, 7,
1803 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, div, 64, /, fp, _Quad, 8r, 7,
1806 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1807 ATOMIC_CRITICAL_FP(float10,
long double, add, +, fp, _Quad, 10r,
1809 ATOMIC_CRITICAL_FP(float10,
long double, sub, -, fp, _Quad, 10r,
1811 ATOMIC_CRITICAL_FP(float10,
long double, mul, *, fp, _Quad, 10r,
1813 ATOMIC_CRITICAL_FP(float10,
long double, div, /, fp, _Quad, 10r,
1817 ATOMIC_CMPXCHG_REV_MIX(fixed1,
char, sub_rev, 8, -, fp, _Quad, 1i, 0,
1819 ATOMIC_CMPXCHG_REV_MIX(fixed1u, uchar, sub_rev, 8, -, fp, _Quad, 1i, 0,
1821 ATOMIC_CMPXCHG_REV_MIX(fixed1,
char, div_rev, 8, /, fp, _Quad, 1i, 0,
1823 ATOMIC_CMPXCHG_REV_MIX(fixed1u, uchar, div_rev, 8, /, fp, _Quad, 1i, 0,
1826 ATOMIC_CMPXCHG_REV_MIX(fixed2,
short, sub_rev, 16, -, fp, _Quad, 2i, 1,
1828 ATOMIC_CMPXCHG_REV_MIX(fixed2u, ushort, sub_rev, 16, -, fp, _Quad, 2i, 1,
1830 ATOMIC_CMPXCHG_REV_MIX(fixed2,
short, div_rev, 16, /, fp, _Quad, 2i, 1,
1832 ATOMIC_CMPXCHG_REV_MIX(fixed2u, ushort, div_rev, 16, /, fp, _Quad, 2i, 1,
1835 ATOMIC_CMPXCHG_REV_MIX(fixed4, kmp_int32, sub_rev, 32, -, fp, _Quad, 4i, 3,
1837 ATOMIC_CMPXCHG_REV_MIX(fixed4u, kmp_uint32, sub_rev, 32, -, fp, _Quad, 4i, 3,
1839 ATOMIC_CMPXCHG_REV_MIX(fixed4, kmp_int32, div_rev, 32, /, fp, _Quad, 4i, 3,
1841 ATOMIC_CMPXCHG_REV_MIX(fixed4u, kmp_uint32, div_rev, 32, /, fp, _Quad, 4i, 3,
1844 ATOMIC_CMPXCHG_REV_MIX(fixed8, kmp_int64, sub_rev, 64, -, fp, _Quad, 8i, 7,
1846 ATOMIC_CMPXCHG_REV_MIX(fixed8u, kmp_uint64, sub_rev, 64, -, fp, _Quad, 8i, 7,
1848 ATOMIC_CMPXCHG_REV_MIX(fixed8, kmp_int64, div_rev, 64, /, fp, _Quad, 8i, 7,
1850 ATOMIC_CMPXCHG_REV_MIX(fixed8u, kmp_uint64, div_rev, 64, /, fp, _Quad, 8i, 7,
1853 ATOMIC_CMPXCHG_REV_MIX(float4, kmp_real32, sub_rev, 32, -, fp, _Quad, 4r, 3,
1855 ATOMIC_CMPXCHG_REV_MIX(float4, kmp_real32, div_rev, 32, /, fp, _Quad, 4r, 3,
1858 ATOMIC_CMPXCHG_REV_MIX(float8, kmp_real64, sub_rev, 64, -, fp, _Quad, 8r, 7,
1860 ATOMIC_CMPXCHG_REV_MIX(float8, kmp_real64, div_rev, 64, /, fp, _Quad, 8r, 7,
1863 ATOMIC_CRITICAL_REV_FP(float10,
long double, sub_rev, -, fp, _Quad, 10r,
1865 ATOMIC_CRITICAL_REV_FP(float10,
long double, div_rev, /, fp, _Quad, 10r,
1869 #endif // KMP_HAVE_QUAD 1871 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1876 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \ 1877 LCK_ID, MASK, GOMP_FLAG) \ 1878 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1879 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \ 1880 OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \ 1884 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \ 1885 LCK_ID, MASK, GOMP_FLAG) \ 1886 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1887 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \ 1888 OP_CMPXCHG(TYPE, BITS, OP) \ 1890 #endif // USE_CMPXCHG_FIX 1894 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \ 1895 LCK_ID, MASK, GOMP_FLAG) \ 1896 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1897 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \ 1898 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 1899 OP_CMPXCHG(TYPE, BITS, OP) \ 1902 OP_UPDATE_CRITICAL(TYPE, OP, \ 1908 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, add, 64, +, cmplx8, kmp_cmplx64, 8c,
1910 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, sub, 64, -, cmplx8, kmp_cmplx64, 8c,
1912 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, mul, 64, *, cmplx8, kmp_cmplx64, 8c,
1914 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, div, 64, /, cmplx8, kmp_cmplx64, 8c,
1928 #define ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, RET_TYPE) \ 1929 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \ 1931 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 1932 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid)); 1943 #define OP_CMPXCHG_READ(TYPE, BITS, OP) \ 1945 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 1948 kmp_int##BITS i_val; \ 1950 union f_i_union old_value; \ 1952 old_value.f_val = temp_val; \ 1953 old_value.i_val = KMP_COMPARE_AND_STORE_RET##BITS( \ 1954 (kmp_int##BITS *)loc, \ 1955 *VOLATILE_CAST(kmp_int##BITS *) & old_value.i_val, \ 1956 *VOLATILE_CAST(kmp_int##BITS *) & old_value.i_val); \ 1957 new_value = old_value.f_val; \ 1967 #define OP_CRITICAL_READ(OP, LCK_ID) \ 1968 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 1970 new_value = (*loc); \ 1972 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); 1975 #ifdef KMP_GOMP_COMPAT 1976 #define OP_GOMP_CRITICAL_READ(OP, FLAG) \ 1977 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 1979 OP_CRITICAL_READ(OP, 0); \ 1983 #define OP_GOMP_CRITICAL_READ(OP, FLAG) 1987 #define ATOMIC_FIXED_READ(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 1988 ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \ 1990 OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \ 1991 new_value = KMP_TEST_THEN_ADD##BITS(loc, OP 0); \ 1995 #define ATOMIC_CMPXCHG_READ(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 1996 ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \ 1998 OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \ 1999 OP_CMPXCHG_READ(TYPE, BITS, OP) \ 2007 #define ATOMIC_CRITICAL_READ(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 2008 ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \ 2010 OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \ 2011 OP_CRITICAL_READ(OP, LCK_ID) \ 2019 #if (KMP_OS_WINDOWS) 2021 #define OP_CRITICAL_READ_WRK(OP, LCK_ID) \ 2022 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2026 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); 2028 #ifdef KMP_GOMP_COMPAT 2029 #define OP_GOMP_CRITICAL_READ_WRK(OP, FLAG) \ 2030 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 2032 OP_CRITICAL_READ_WRK(OP, 0); \ 2035 #define OP_GOMP_CRITICAL_READ_WRK(OP, FLAG) 2038 #define ATOMIC_BEGIN_READ_WRK(TYPE_ID, OP_ID, TYPE) \ 2039 void __kmpc_atomic_##TYPE_ID##_##OP_ID(TYPE *out, ident_t *id_ref, int gtid, \ 2041 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 2042 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid)); 2045 #define ATOMIC_CRITICAL_READ_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 2046 ATOMIC_BEGIN_READ_WRK(TYPE_ID, OP_ID, TYPE) \ 2047 OP_GOMP_CRITICAL_READ_WRK(OP## =, GOMP_FLAG) \ 2048 OP_CRITICAL_READ_WRK(OP, LCK_ID) \ 2051 #endif // KMP_OS_WINDOWS 2055 ATOMIC_FIXED_READ(fixed4, rd, kmp_int32, 32, +, 0)
2056 ATOMIC_FIXED_READ(fixed8, rd, kmp_int64, 64, +,
2058 ATOMIC_CMPXCHG_READ(float4, rd, kmp_real32, 32, +,
2060 ATOMIC_CMPXCHG_READ(float8, rd, kmp_real64, 64, +,
2064 ATOMIC_CMPXCHG_READ(fixed1, rd, kmp_int8, 8, +,
2066 ATOMIC_CMPXCHG_READ(fixed2, rd, kmp_int16, 16, +,
2069 ATOMIC_CRITICAL_READ(float10, rd,
long double, +, 10r,
2072 ATOMIC_CRITICAL_READ(float16, rd, QUAD_LEGACY, +, 16r,
2074 #endif // KMP_HAVE_QUAD 2077 #if (KMP_OS_WINDOWS) 2078 ATOMIC_CRITICAL_READ_WRK(cmplx4, rd, kmp_cmplx32, +, 8c,
2081 ATOMIC_CRITICAL_READ(cmplx4, rd, kmp_cmplx32, +, 8c,
2083 #endif // (KMP_OS_WINDOWS) 2084 ATOMIC_CRITICAL_READ(cmplx8, rd, kmp_cmplx64, +, 16c,
2086 ATOMIC_CRITICAL_READ(cmplx10, rd, kmp_cmplx80, +, 20c,
2089 ATOMIC_CRITICAL_READ(cmplx16, rd, CPLX128_LEG, +, 32c,
2092 ATOMIC_CRITICAL_READ(float16, a16_rd, Quad_a16_t, +, 16r,
2094 ATOMIC_CRITICAL_READ(cmplx16, a16_rd, kmp_cmplx128_a16_t, +, 32c,
2096 #endif // (KMP_ARCH_X86) 2097 #endif // KMP_HAVE_QUAD 2102 #define ATOMIC_XCHG_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2103 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 2104 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \ 2105 KMP_XCHG_FIXED##BITS(lhs, rhs); \ 2108 #define ATOMIC_XCHG_FLOAT_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2109 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 2110 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \ 2111 KMP_XCHG_REAL##BITS(lhs, rhs); \ 2121 #define OP_CMPXCHG_WR(TYPE, BITS, OP) \ 2123 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 2124 TYPE old_value, new_value; \ 2126 old_value = temp_val; \ 2128 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 2129 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 2130 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ 2132 old_value = temp_val; \ 2138 #define ATOMIC_CMPXCHG_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2139 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 2140 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \ 2141 OP_CMPXCHG_WR(TYPE, BITS, OP) \ 2150 #define ATOMIC_CRITICAL_WR(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 2151 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 2152 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \ 2153 OP_CRITICAL(OP, LCK_ID) \ 2157 ATOMIC_XCHG_WR(fixed1, wr, kmp_int8, 8, =,
2159 ATOMIC_XCHG_WR(fixed2, wr, kmp_int16, 16, =,
2161 ATOMIC_XCHG_WR(fixed4, wr, kmp_int32, 32, =,
2164 ATOMIC_CMPXCHG_WR(fixed8, wr, kmp_int64, 64, =,
2167 ATOMIC_XCHG_WR(fixed8, wr, kmp_int64, 64, =,
2169 #endif // (KMP_ARCH_X86) 2171 ATOMIC_XCHG_FLOAT_WR(float4, wr, kmp_real32, 32, =,
2174 ATOMIC_CMPXCHG_WR(float8, wr, kmp_real64, 64, =,
2177 ATOMIC_XCHG_FLOAT_WR(float8, wr, kmp_real64, 64, =,
2179 #endif // (KMP_ARCH_X86) 2181 ATOMIC_CRITICAL_WR(float10, wr,
long double, =, 10r,
2184 ATOMIC_CRITICAL_WR(float16, wr, QUAD_LEGACY, =, 16r,
2186 #endif // KMP_HAVE_QUAD 2187 ATOMIC_CRITICAL_WR(cmplx4, wr, kmp_cmplx32, =, 8c, 1)
2188 ATOMIC_CRITICAL_WR(cmplx8, wr, kmp_cmplx64, =, 16c,
2190 ATOMIC_CRITICAL_WR(cmplx10, wr, kmp_cmplx80, =, 20c,
2193 ATOMIC_CRITICAL_WR(cmplx16, wr, CPLX128_LEG, =, 32c,
2196 ATOMIC_CRITICAL_WR(float16, a16_wr, Quad_a16_t, =, 16r,
2198 ATOMIC_CRITICAL_WR(cmplx16, a16_wr, kmp_cmplx128_a16_t, =, 32c,
2200 #endif // (KMP_ARCH_X86) 2201 #endif // KMP_HAVE_QUAD 2211 #define ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, RET_TYPE) \ 2212 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \ 2213 TYPE *lhs, TYPE rhs, int flag) { \ 2214 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 2215 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid)); 2223 #define OP_CRITICAL_CPT(OP, LCK_ID) \ 2224 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2228 new_value = (*lhs); \ 2230 new_value = (*lhs); \ 2234 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2237 #define OP_UPDATE_CRITICAL_CPT(TYPE, OP, LCK_ID) \ 2238 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2241 (*lhs) = (TYPE)((*lhs)OP rhs); \ 2242 new_value = (*lhs); \ 2244 new_value = (*lhs); \ 2245 (*lhs) = (TYPE)((*lhs)OP rhs); \ 2248 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2252 #ifdef KMP_GOMP_COMPAT 2253 #define OP_GOMP_CRITICAL_CPT(TYPE, OP, FLAG) \ 2254 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 2256 OP_UPDATE_CRITICAL_CPT(TYPE, OP, 0); \ 2259 #define OP_GOMP_CRITICAL_CPT(TYPE, OP, FLAG) 2269 #define OP_CMPXCHG_CPT(TYPE, BITS, OP) \ 2271 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 2272 TYPE old_value, new_value; \ 2274 old_value = temp_val; \ 2275 new_value = (TYPE)(old_value OP rhs); \ 2276 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 2277 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 2278 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ 2280 old_value = temp_val; \ 2281 new_value = (TYPE)(old_value OP rhs); \ 2290 #define ATOMIC_CMPXCHG_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2291 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2294 OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \ 2295 OP_CMPXCHG_CPT(TYPE, BITS, OP) \ 2299 #define ATOMIC_FIXED_ADD_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2300 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2301 TYPE old_value, new_value; \ 2303 OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \ 2305 old_value = KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \ 2307 return old_value OP rhs; \ 2313 ATOMIC_FIXED_ADD_CPT(fixed4, add_cpt, kmp_int32, 32, +,
2315 ATOMIC_FIXED_ADD_CPT(fixed4, sub_cpt, kmp_int32, 32, -,
2317 ATOMIC_FIXED_ADD_CPT(fixed8, add_cpt, kmp_int64, 64, +,
2319 ATOMIC_FIXED_ADD_CPT(fixed8, sub_cpt, kmp_int64, 64, -,
2322 ATOMIC_CMPXCHG_CPT(float4, add_cpt, kmp_real32, 32, +,
2324 ATOMIC_CMPXCHG_CPT(float4, sub_cpt, kmp_real32, 32, -,
2326 ATOMIC_CMPXCHG_CPT(float8, add_cpt, kmp_real64, 64, +,
2328 ATOMIC_CMPXCHG_CPT(float8, sub_cpt, kmp_real64, 64, -,
2343 ATOMIC_CMPXCHG_CPT(fixed1, add_cpt, kmp_int8, 8, +,
2345 ATOMIC_CMPXCHG_CPT(fixed1, andb_cpt, kmp_int8, 8, &,
2347 ATOMIC_CMPXCHG_CPT(fixed1, div_cpt, kmp_int8, 8, /,
2349 ATOMIC_CMPXCHG_CPT(fixed1u, div_cpt, kmp_uint8, 8, /,
2351 ATOMIC_CMPXCHG_CPT(fixed1, mul_cpt, kmp_int8, 8, *,
2353 ATOMIC_CMPXCHG_CPT(fixed1, orb_cpt, kmp_int8, 8, |,
2355 ATOMIC_CMPXCHG_CPT(fixed1, shl_cpt, kmp_int8, 8, <<,
2357 ATOMIC_CMPXCHG_CPT(fixed1, shr_cpt, kmp_int8, 8, >>,
2359 ATOMIC_CMPXCHG_CPT(fixed1u, shr_cpt, kmp_uint8, 8, >>,
2361 ATOMIC_CMPXCHG_CPT(fixed1, sub_cpt, kmp_int8, 8, -,
2363 ATOMIC_CMPXCHG_CPT(fixed1, xor_cpt, kmp_int8, 8, ^,
2365 ATOMIC_CMPXCHG_CPT(fixed2, add_cpt, kmp_int16, 16, +,
2367 ATOMIC_CMPXCHG_CPT(fixed2, andb_cpt, kmp_int16, 16, &,
2369 ATOMIC_CMPXCHG_CPT(fixed2, div_cpt, kmp_int16, 16, /,
2371 ATOMIC_CMPXCHG_CPT(fixed2u, div_cpt, kmp_uint16, 16, /,
2373 ATOMIC_CMPXCHG_CPT(fixed2, mul_cpt, kmp_int16, 16, *,
2375 ATOMIC_CMPXCHG_CPT(fixed2, orb_cpt, kmp_int16, 16, |,
2377 ATOMIC_CMPXCHG_CPT(fixed2, shl_cpt, kmp_int16, 16, <<,
2379 ATOMIC_CMPXCHG_CPT(fixed2, shr_cpt, kmp_int16, 16, >>,
2381 ATOMIC_CMPXCHG_CPT(fixed2u, shr_cpt, kmp_uint16, 16, >>,
2383 ATOMIC_CMPXCHG_CPT(fixed2, sub_cpt, kmp_int16, 16, -,
2385 ATOMIC_CMPXCHG_CPT(fixed2, xor_cpt, kmp_int16, 16, ^,
2387 ATOMIC_CMPXCHG_CPT(fixed4, andb_cpt, kmp_int32, 32, &,
2389 ATOMIC_CMPXCHG_CPT(fixed4, div_cpt, kmp_int32, 32, /,
2391 ATOMIC_CMPXCHG_CPT(fixed4u, div_cpt, kmp_uint32, 32, /,
2393 ATOMIC_CMPXCHG_CPT(fixed4, mul_cpt, kmp_int32, 32, *,
2395 ATOMIC_CMPXCHG_CPT(fixed4, orb_cpt, kmp_int32, 32, |,
2397 ATOMIC_CMPXCHG_CPT(fixed4, shl_cpt, kmp_int32, 32, <<,
2399 ATOMIC_CMPXCHG_CPT(fixed4, shr_cpt, kmp_int32, 32, >>,
2401 ATOMIC_CMPXCHG_CPT(fixed4u, shr_cpt, kmp_uint32, 32, >>,
2403 ATOMIC_CMPXCHG_CPT(fixed4, xor_cpt, kmp_int32, 32, ^,
2405 ATOMIC_CMPXCHG_CPT(fixed8, andb_cpt, kmp_int64, 64, &,
2407 ATOMIC_CMPXCHG_CPT(fixed8, div_cpt, kmp_int64, 64, /,
2409 ATOMIC_CMPXCHG_CPT(fixed8u, div_cpt, kmp_uint64, 64, /,
2411 ATOMIC_CMPXCHG_CPT(fixed8, mul_cpt, kmp_int64, 64, *,
2413 ATOMIC_CMPXCHG_CPT(fixed8, orb_cpt, kmp_int64, 64, |,
2415 ATOMIC_CMPXCHG_CPT(fixed8, shl_cpt, kmp_int64, 64, <<,
2417 ATOMIC_CMPXCHG_CPT(fixed8, shr_cpt, kmp_int64, 64, >>,
2419 ATOMIC_CMPXCHG_CPT(fixed8u, shr_cpt, kmp_uint64, 64, >>,
2421 ATOMIC_CMPXCHG_CPT(fixed8, xor_cpt, kmp_int64, 64, ^,
2423 ATOMIC_CMPXCHG_CPT(float4, div_cpt, kmp_real32, 32, /,
2425 ATOMIC_CMPXCHG_CPT(float4, mul_cpt, kmp_real32, 32, *,
2427 ATOMIC_CMPXCHG_CPT(float8, div_cpt, kmp_real64, 64, /,
2429 ATOMIC_CMPXCHG_CPT(float8, mul_cpt, kmp_real64, 64, *,
2441 #define ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \ 2442 TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( \ 2443 ident_t *id_ref, int gtid, TYPE *lhs, RTYPE rhs, int flag) { \ 2444 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 2446 ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", \ 2450 #define ATOMIC_CMPXCHG_CPT_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \ 2451 RTYPE, LCK_ID, MASK, GOMP_FLAG) \ 2452 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \ 2455 OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \ 2456 OP_CMPXCHG_CPT(TYPE, BITS, OP) \ 2460 #define ATOMIC_CRITICAL_CPT_MIX(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \ 2461 LCK_ID, GOMP_FLAG) \ 2462 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \ 2465 OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \ 2466 OP_UPDATE_CRITICAL_CPT(TYPE, OP, LCK_ID) \ 2469 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, add_cpt, 8, +, fp, _Quad, 1i, 0,
2471 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, add_cpt, 8, +, fp, _Quad, 1i, 0,
2473 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, sub_cpt, 8, -, fp, _Quad, 1i, 0,
2475 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, sub_cpt, 8, -, fp, _Quad, 1i, 0,
2477 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, mul_cpt, 8, *, fp, _Quad, 1i, 0,
2479 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, mul_cpt, 8, *, fp, _Quad, 1i, 0,
2481 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, div_cpt, 8, /, fp, _Quad, 1i, 0,
2483 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, div_cpt, 8, /, fp, _Quad, 1i, 0,
2486 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, add_cpt, 16, +, fp, _Quad, 2i, 1,
2488 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, add_cpt, 16, +, fp, _Quad, 2i, 1,
2490 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, sub_cpt, 16, -, fp, _Quad, 2i, 1,
2492 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, sub_cpt, 16, -, fp, _Quad, 2i, 1,
2494 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, mul_cpt, 16, *, fp, _Quad, 2i, 1,
2496 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, mul_cpt, 16, *, fp, _Quad, 2i, 1,
2498 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, div_cpt, 16, /, fp, _Quad, 2i, 1,
2500 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, div_cpt, 16, /, fp, _Quad, 2i, 1,
2503 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, add_cpt, 32, +, fp, _Quad, 4i, 3,
2505 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, add_cpt, 32, +, fp, _Quad, 4i, 3,
2507 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, sub_cpt, 32, -, fp, _Quad, 4i, 3,
2509 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, sub_cpt, 32, -, fp, _Quad, 4i, 3,
2511 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, mul_cpt, 32, *, fp, _Quad, 4i, 3,
2513 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, mul_cpt, 32, *, fp, _Quad, 4i, 3,
2515 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, div_cpt, 32, /, fp, _Quad, 4i, 3,
2517 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, div_cpt, 32, /, fp, _Quad, 4i, 3,
2520 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, add_cpt, 64, +, fp, _Quad, 8i, 7,
2522 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, add_cpt, 64, +, fp, _Quad, 8i, 7,
2524 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, sub_cpt, 64, -, fp, _Quad, 8i, 7,
2526 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, sub_cpt, 64, -, fp, _Quad, 8i, 7,
2528 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, mul_cpt, 64, *, fp, _Quad, 8i, 7,
2530 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, mul_cpt, 64, *, fp, _Quad, 8i, 7,
2532 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, div_cpt, 64, /, fp, _Quad, 8i, 7,
2534 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, div_cpt, 64, /, fp, _Quad, 8i, 7,
2537 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, add_cpt, 32, +, fp, _Quad, 4r, 3,
2539 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, sub_cpt, 32, -, fp, _Quad, 4r, 3,
2541 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, mul_cpt, 32, *, fp, _Quad, 4r, 3,
2543 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, div_cpt, 32, /, fp, _Quad, 4r, 3,
2546 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, add_cpt, 64, +, fp, _Quad, 8r, 7,
2548 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, sub_cpt, 64, -, fp, _Quad, 8r, 7,
2550 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, mul_cpt, 64, *, fp, _Quad, 8r, 7,
2552 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, div_cpt, 64, /, fp, _Quad, 8r, 7,
2555 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, add_cpt, +, fp, _Quad, 10r,
2557 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, sub_cpt, -, fp, _Quad, 10r,
2559 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, mul_cpt, *, fp, _Quad, 10r,
2561 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, div_cpt, /, fp, _Quad, 10r,
2564 #endif // KMP_HAVE_QUAD 2575 #define OP_CRITICAL_L_CPT(OP, LCK_ID) \ 2576 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2580 (*lhs) = new_value; \ 2582 new_value = (*lhs); \ 2586 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); 2589 #ifdef KMP_GOMP_COMPAT 2590 #define OP_GOMP_CRITICAL_L_CPT(OP, FLAG) \ 2591 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 2593 OP_CRITICAL_L_CPT(OP, 0); \ 2597 #define OP_GOMP_CRITICAL_L_CPT(OP, FLAG) 2602 #define ATOMIC_CMPX_L_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2603 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2606 OP_GOMP_CRITICAL_L_CPT(= *lhs OP, GOMP_FLAG) \ 2607 OP_CMPXCHG_CPT(TYPE, BITS, OP) \ 2610 ATOMIC_CMPX_L_CPT(fixed1, andl_cpt,
char, 8, &&,
2612 ATOMIC_CMPX_L_CPT(fixed1, orl_cpt,
char, 8, ||,
2614 ATOMIC_CMPX_L_CPT(fixed2, andl_cpt,
short, 16, &&,
2616 ATOMIC_CMPX_L_CPT(fixed2, orl_cpt,
short, 16, ||,
2618 ATOMIC_CMPX_L_CPT(fixed4, andl_cpt, kmp_int32, 32, &&,
2620 ATOMIC_CMPX_L_CPT(fixed4, orl_cpt, kmp_int32, 32, ||,
2622 ATOMIC_CMPX_L_CPT(fixed8, andl_cpt, kmp_int64, 64, &&,
2624 ATOMIC_CMPX_L_CPT(fixed8, orl_cpt, kmp_int64, 64, ||,
2636 #define MIN_MAX_CRITSECT_CPT(OP, LCK_ID) \ 2637 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2639 if (*lhs OP rhs) { \ 2645 new_value = old_value; \ 2649 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2653 #ifdef KMP_GOMP_COMPAT 2654 #define GOMP_MIN_MAX_CRITSECT_CPT(OP, FLAG) \ 2655 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 2657 MIN_MAX_CRITSECT_CPT(OP, 0); \ 2660 #define GOMP_MIN_MAX_CRITSECT_CPT(OP, FLAG) 2664 #define MIN_MAX_CMPXCHG_CPT(TYPE, BITS, OP) \ 2666 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 2669 old_value = temp_val; \ 2670 while (old_value OP rhs && \ 2671 !KMP_COMPARE_AND_STORE_ACQ##BITS( \ 2672 (kmp_int##BITS *)lhs, \ 2673 *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 2674 *VOLATILE_CAST(kmp_int##BITS *) & rhs)) { \ 2676 old_value = temp_val; \ 2686 #define MIN_MAX_CRITICAL_CPT(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 2687 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2688 TYPE new_value, old_value; \ 2689 if (*lhs OP rhs) { \ 2690 GOMP_MIN_MAX_CRITSECT_CPT(OP, GOMP_FLAG) \ 2691 MIN_MAX_CRITSECT_CPT(OP, LCK_ID) \ 2696 #define MIN_MAX_COMPXCHG_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2697 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2698 TYPE new_value, old_value; \ 2700 if (*lhs OP rhs) { \ 2701 GOMP_MIN_MAX_CRITSECT_CPT(OP, GOMP_FLAG) \ 2702 MIN_MAX_CMPXCHG_CPT(TYPE, BITS, OP) \ 2707 MIN_MAX_COMPXCHG_CPT(fixed1, max_cpt,
char, 8, <,
2709 MIN_MAX_COMPXCHG_CPT(fixed1, min_cpt,
char, 8, >,
2711 MIN_MAX_COMPXCHG_CPT(fixed2, max_cpt,
short, 16, <,
2713 MIN_MAX_COMPXCHG_CPT(fixed2, min_cpt,
short, 16, >,
2715 MIN_MAX_COMPXCHG_CPT(fixed4, max_cpt, kmp_int32, 32, <,
2717 MIN_MAX_COMPXCHG_CPT(fixed4, min_cpt, kmp_int32, 32, >,
2719 MIN_MAX_COMPXCHG_CPT(fixed8, max_cpt, kmp_int64, 64, <,
2721 MIN_MAX_COMPXCHG_CPT(fixed8, min_cpt, kmp_int64, 64, >,
2723 MIN_MAX_COMPXCHG_CPT(float4, max_cpt, kmp_real32, 32, <,
2725 MIN_MAX_COMPXCHG_CPT(float4, min_cpt, kmp_real32, 32, >,
2727 MIN_MAX_COMPXCHG_CPT(float8, max_cpt, kmp_real64, 64, <,
2729 MIN_MAX_COMPXCHG_CPT(float8, min_cpt, kmp_real64, 64, >,
2731 MIN_MAX_CRITICAL_CPT(float10, max_cpt,
long double, <, 10r,
2733 MIN_MAX_CRITICAL_CPT(float10, min_cpt,
long double, >, 10r,
2736 MIN_MAX_CRITICAL_CPT(float16, max_cpt, QUAD_LEGACY, <, 16r,
2738 MIN_MAX_CRITICAL_CPT(float16, min_cpt, QUAD_LEGACY, >, 16r,
2741 MIN_MAX_CRITICAL_CPT(float16, max_a16_cpt, Quad_a16_t, <, 16r,
2743 MIN_MAX_CRITICAL_CPT(float16, min_a16_cpt, Quad_a16_t, >, 16r,
2745 #endif // (KMP_ARCH_X86) 2746 #endif // KMP_HAVE_QUAD 2749 #ifdef KMP_GOMP_COMPAT 2750 #define OP_GOMP_CRITICAL_EQV_CPT(OP, FLAG) \ 2751 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 2753 OP_CRITICAL_CPT(OP, 0); \ 2756 #define OP_GOMP_CRITICAL_EQV_CPT(OP, FLAG) 2759 #define ATOMIC_CMPX_EQV_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2760 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2763 OP_GOMP_CRITICAL_EQV_CPT(^= (TYPE) ~, GOMP_FLAG) \ 2764 OP_CMPXCHG_CPT(TYPE, BITS, OP) \ 2769 ATOMIC_CMPXCHG_CPT(fixed1, neqv_cpt, kmp_int8, 8, ^,
2771 ATOMIC_CMPXCHG_CPT(fixed2, neqv_cpt, kmp_int16, 16, ^,
2773 ATOMIC_CMPXCHG_CPT(fixed4, neqv_cpt, kmp_int32, 32, ^,
2775 ATOMIC_CMPXCHG_CPT(fixed8, neqv_cpt, kmp_int64, 64, ^,
2777 ATOMIC_CMPX_EQV_CPT(fixed1, eqv_cpt, kmp_int8, 8, ^~,
2779 ATOMIC_CMPX_EQV_CPT(fixed2, eqv_cpt, kmp_int16, 16, ^~,
2781 ATOMIC_CMPX_EQV_CPT(fixed4, eqv_cpt, kmp_int32, 32, ^~,
2783 ATOMIC_CMPX_EQV_CPT(fixed8, eqv_cpt, kmp_int64, 64, ^~,
2792 #define ATOMIC_CRITICAL_CPT(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 2793 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2795 OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \ 2796 OP_UPDATE_CRITICAL_CPT(TYPE, OP, LCK_ID) \ 2802 #define OP_CRITICAL_CPT_WRK(OP, LCK_ID) \ 2803 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2813 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2817 #ifdef KMP_GOMP_COMPAT 2818 #define OP_GOMP_CRITICAL_CPT_WRK(OP, FLAG) \ 2819 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 2821 OP_CRITICAL_CPT_WRK(OP## =, 0); \ 2824 #define OP_GOMP_CRITICAL_CPT_WRK(OP, FLAG) 2828 #define ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \ 2829 void __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, TYPE *lhs, \ 2830 TYPE rhs, TYPE *out, int flag) { \ 2831 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 2832 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid)); 2835 #define ATOMIC_CRITICAL_CPT_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 2836 ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \ 2837 OP_GOMP_CRITICAL_CPT_WRK(OP, GOMP_FLAG) \ 2838 OP_CRITICAL_CPT_WRK(OP## =, LCK_ID) \ 2844 ATOMIC_CRITICAL_CPT(float10, add_cpt,
long double, +, 10r,
2846 ATOMIC_CRITICAL_CPT(float10, sub_cpt,
long double, -, 10r,
2848 ATOMIC_CRITICAL_CPT(float10, mul_cpt,
long double, *, 10r,
2850 ATOMIC_CRITICAL_CPT(float10, div_cpt,
long double, /, 10r,
2854 ATOMIC_CRITICAL_CPT(float16, add_cpt, QUAD_LEGACY, +, 16r,
2856 ATOMIC_CRITICAL_CPT(float16, sub_cpt, QUAD_LEGACY, -, 16r,
2858 ATOMIC_CRITICAL_CPT(float16, mul_cpt, QUAD_LEGACY, *, 16r,
2860 ATOMIC_CRITICAL_CPT(float16, div_cpt, QUAD_LEGACY, /, 16r,
2863 ATOMIC_CRITICAL_CPT(float16, add_a16_cpt, Quad_a16_t, +, 16r,
2865 ATOMIC_CRITICAL_CPT(float16, sub_a16_cpt, Quad_a16_t, -, 16r,
2867 ATOMIC_CRITICAL_CPT(float16, mul_a16_cpt, Quad_a16_t, *, 16r,
2869 ATOMIC_CRITICAL_CPT(float16, div_a16_cpt, Quad_a16_t, /, 16r,
2871 #endif // (KMP_ARCH_X86) 2872 #endif // KMP_HAVE_QUAD 2877 ATOMIC_CRITICAL_CPT_WRK(cmplx4, add_cpt, kmp_cmplx32, +, 8c,
2879 ATOMIC_CRITICAL_CPT_WRK(cmplx4, sub_cpt, kmp_cmplx32, -, 8c,
2881 ATOMIC_CRITICAL_CPT_WRK(cmplx4, mul_cpt, kmp_cmplx32, *, 8c,
2883 ATOMIC_CRITICAL_CPT_WRK(cmplx4, div_cpt, kmp_cmplx32, /, 8c,
2886 ATOMIC_CRITICAL_CPT(cmplx8, add_cpt, kmp_cmplx64, +, 16c,
2888 ATOMIC_CRITICAL_CPT(cmplx8, sub_cpt, kmp_cmplx64, -, 16c,
2890 ATOMIC_CRITICAL_CPT(cmplx8, mul_cpt, kmp_cmplx64, *, 16c,
2892 ATOMIC_CRITICAL_CPT(cmplx8, div_cpt, kmp_cmplx64, /, 16c,
2894 ATOMIC_CRITICAL_CPT(cmplx10, add_cpt, kmp_cmplx80, +, 20c,
2896 ATOMIC_CRITICAL_CPT(cmplx10, sub_cpt, kmp_cmplx80, -, 20c,
2898 ATOMIC_CRITICAL_CPT(cmplx10, mul_cpt, kmp_cmplx80, *, 20c,
2900 ATOMIC_CRITICAL_CPT(cmplx10, div_cpt, kmp_cmplx80, /, 20c,
2903 ATOMIC_CRITICAL_CPT(cmplx16, add_cpt, CPLX128_LEG, +, 32c,
2905 ATOMIC_CRITICAL_CPT(cmplx16, sub_cpt, CPLX128_LEG, -, 32c,
2907 ATOMIC_CRITICAL_CPT(cmplx16, mul_cpt, CPLX128_LEG, *, 32c,
2909 ATOMIC_CRITICAL_CPT(cmplx16, div_cpt, CPLX128_LEG, /, 32c,
2912 ATOMIC_CRITICAL_CPT(cmplx16, add_a16_cpt, kmp_cmplx128_a16_t, +, 32c,
2914 ATOMIC_CRITICAL_CPT(cmplx16, sub_a16_cpt, kmp_cmplx128_a16_t, -, 32c,
2916 ATOMIC_CRITICAL_CPT(cmplx16, mul_a16_cpt, kmp_cmplx128_a16_t, *, 32c,
2918 ATOMIC_CRITICAL_CPT(cmplx16, div_a16_cpt, kmp_cmplx128_a16_t, /, 32c,
2920 #endif // (KMP_ARCH_X86) 2921 #endif // KMP_HAVE_QUAD 2927 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 2934 #define OP_CRITICAL_CPT_REV(TYPE, OP, LCK_ID) \ 2935 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2939 (*lhs) = (TYPE)((rhs)OP(*lhs)); \ 2940 new_value = (*lhs); \ 2942 new_value = (*lhs); \ 2943 (*lhs) = (TYPE)((rhs)OP(*lhs)); \ 2945 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2949 #ifdef KMP_GOMP_COMPAT 2950 #define OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, FLAG) \ 2951 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 2953 OP_CRITICAL_CPT_REV(TYPE, OP, 0); \ 2956 #define OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, FLAG) 2966 #define OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \ 2968 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 2969 TYPE old_value, new_value; \ 2971 old_value = temp_val; \ 2972 new_value = (TYPE)(rhs OP old_value); \ 2973 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 2974 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 2975 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ 2977 old_value = temp_val; \ 2978 new_value = (TYPE)(rhs OP old_value); \ 2987 #define ATOMIC_CMPXCHG_CPT_REV(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2988 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2991 OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) \ 2992 OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \ 2995 ATOMIC_CMPXCHG_CPT_REV(fixed1, div_cpt_rev, kmp_int8, 8, /,
2997 ATOMIC_CMPXCHG_CPT_REV(fixed1u, div_cpt_rev, kmp_uint8, 8, /,
2999 ATOMIC_CMPXCHG_CPT_REV(fixed1, shl_cpt_rev, kmp_int8, 8, <<,
3001 ATOMIC_CMPXCHG_CPT_REV(fixed1, shr_cpt_rev, kmp_int8, 8, >>,
3003 ATOMIC_CMPXCHG_CPT_REV(fixed1u, shr_cpt_rev, kmp_uint8, 8, >>,
3005 ATOMIC_CMPXCHG_CPT_REV(fixed1, sub_cpt_rev, kmp_int8, 8, -,
3007 ATOMIC_CMPXCHG_CPT_REV(fixed2, div_cpt_rev, kmp_int16, 16, /,
3009 ATOMIC_CMPXCHG_CPT_REV(fixed2u, div_cpt_rev, kmp_uint16, 16, /,
3011 ATOMIC_CMPXCHG_CPT_REV(fixed2, shl_cpt_rev, kmp_int16, 16, <<,
3013 ATOMIC_CMPXCHG_CPT_REV(fixed2, shr_cpt_rev, kmp_int16, 16, >>,
3015 ATOMIC_CMPXCHG_CPT_REV(fixed2u, shr_cpt_rev, kmp_uint16, 16, >>,
3017 ATOMIC_CMPXCHG_CPT_REV(fixed2, sub_cpt_rev, kmp_int16, 16, -,
3019 ATOMIC_CMPXCHG_CPT_REV(fixed4, div_cpt_rev, kmp_int32, 32, /,
3021 ATOMIC_CMPXCHG_CPT_REV(fixed4u, div_cpt_rev, kmp_uint32, 32, /,
3023 ATOMIC_CMPXCHG_CPT_REV(fixed4, shl_cpt_rev, kmp_int32, 32, <<,
3025 ATOMIC_CMPXCHG_CPT_REV(fixed4, shr_cpt_rev, kmp_int32, 32, >>,
3027 ATOMIC_CMPXCHG_CPT_REV(fixed4u, shr_cpt_rev, kmp_uint32, 32, >>,
3029 ATOMIC_CMPXCHG_CPT_REV(fixed4, sub_cpt_rev, kmp_int32, 32, -,
3031 ATOMIC_CMPXCHG_CPT_REV(fixed8, div_cpt_rev, kmp_int64, 64, /,
3033 ATOMIC_CMPXCHG_CPT_REV(fixed8u, div_cpt_rev, kmp_uint64, 64, /,
3035 ATOMIC_CMPXCHG_CPT_REV(fixed8, shl_cpt_rev, kmp_int64, 64, <<,
3037 ATOMIC_CMPXCHG_CPT_REV(fixed8, shr_cpt_rev, kmp_int64, 64, >>,
3039 ATOMIC_CMPXCHG_CPT_REV(fixed8u, shr_cpt_rev, kmp_uint64, 64, >>,
3041 ATOMIC_CMPXCHG_CPT_REV(fixed8, sub_cpt_rev, kmp_int64, 64, -,
3043 ATOMIC_CMPXCHG_CPT_REV(float4, div_cpt_rev, kmp_real32, 32, /,
3045 ATOMIC_CMPXCHG_CPT_REV(float4, sub_cpt_rev, kmp_real32, 32, -,
3047 ATOMIC_CMPXCHG_CPT_REV(float8, div_cpt_rev, kmp_real64, 64, /,
3049 ATOMIC_CMPXCHG_CPT_REV(float8, sub_cpt_rev, kmp_real64, 64, -,
3059 #define ATOMIC_CRITICAL_CPT_REV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 3060 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 3063 OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) \ 3064 OP_CRITICAL_CPT_REV(TYPE, OP, LCK_ID) \ 3069 ATOMIC_CRITICAL_CPT_REV(float10, sub_cpt_rev,
long double, -, 10r,
3071 ATOMIC_CRITICAL_CPT_REV(float10, div_cpt_rev,
long double, /, 10r,
3075 ATOMIC_CRITICAL_CPT_REV(float16, sub_cpt_rev, QUAD_LEGACY, -, 16r,
3077 ATOMIC_CRITICAL_CPT_REV(float16, div_cpt_rev, QUAD_LEGACY, /, 16r,
3080 ATOMIC_CRITICAL_CPT_REV(float16, sub_a16_cpt_rev, Quad_a16_t, -, 16r,
3082 ATOMIC_CRITICAL_CPT_REV(float16, div_a16_cpt_rev, Quad_a16_t, /, 16r,
3084 #endif // (KMP_ARCH_X86) 3085 #endif // KMP_HAVE_QUAD 3092 #define OP_CRITICAL_CPT_REV_WRK(OP, LCK_ID) \ 3093 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 3096 (*lhs) = (rhs)OP(*lhs); \ 3100 (*lhs) = (rhs)OP(*lhs); \ 3103 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 3107 #ifdef KMP_GOMP_COMPAT 3108 #define OP_GOMP_CRITICAL_CPT_REV_WRK(OP, FLAG) \ 3109 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 3111 OP_CRITICAL_CPT_REV_WRK(OP, 0); \ 3114 #define OP_GOMP_CRITICAL_CPT_REV_WRK(OP, FLAG) 3118 #define ATOMIC_CRITICAL_CPT_REV_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, \ 3120 ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \ 3121 OP_GOMP_CRITICAL_CPT_REV_WRK(OP, GOMP_FLAG) \ 3122 OP_CRITICAL_CPT_REV_WRK(OP, LCK_ID) \ 3128 ATOMIC_CRITICAL_CPT_REV_WRK(cmplx4, sub_cpt_rev, kmp_cmplx32, -, 8c,
3130 ATOMIC_CRITICAL_CPT_REV_WRK(cmplx4, div_cpt_rev, kmp_cmplx32, /, 8c,
3133 ATOMIC_CRITICAL_CPT_REV(cmplx8, sub_cpt_rev, kmp_cmplx64, -, 16c,
3135 ATOMIC_CRITICAL_CPT_REV(cmplx8, div_cpt_rev, kmp_cmplx64, /, 16c,
3137 ATOMIC_CRITICAL_CPT_REV(cmplx10, sub_cpt_rev, kmp_cmplx80, -, 20c,
3139 ATOMIC_CRITICAL_CPT_REV(cmplx10, div_cpt_rev, kmp_cmplx80, /, 20c,
3142 ATOMIC_CRITICAL_CPT_REV(cmplx16, sub_cpt_rev, CPLX128_LEG, -, 32c,
3144 ATOMIC_CRITICAL_CPT_REV(cmplx16, div_cpt_rev, CPLX128_LEG, /, 32c,
3147 ATOMIC_CRITICAL_CPT_REV(cmplx16, sub_a16_cpt_rev, kmp_cmplx128_a16_t, -, 32c,
3149 ATOMIC_CRITICAL_CPT_REV(cmplx16, div_a16_cpt_rev, kmp_cmplx128_a16_t, /, 32c,
3151 #endif // (KMP_ARCH_X86) 3152 #endif // KMP_HAVE_QUAD 3163 #define ATOMIC_CMPXCHG_CPT_REV_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \ 3164 RTYPE, LCK_ID, MASK, GOMP_FLAG) \ 3165 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \ 3168 OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) \ 3169 OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \ 3173 #define ATOMIC_CRITICAL_CPT_REV_MIX(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \ 3174 LCK_ID, GOMP_FLAG) \ 3175 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \ 3178 OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) \ 3179 OP_CRITICAL_CPT_REV(TYPE, OP, LCK_ID) \ 3182 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1,
char, sub_cpt_rev, 8, -, fp, _Quad, 1i, 0,
3184 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1u, uchar, sub_cpt_rev, 8, -, fp, _Quad, 1i, 0,
3186 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1,
char, div_cpt_rev, 8, /, fp, _Quad, 1i, 0,
3188 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1u, uchar, div_cpt_rev, 8, /, fp, _Quad, 1i, 0,
3191 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2,
short, sub_cpt_rev, 16, -, fp, _Quad, 2i, 1,
3193 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2u, ushort, sub_cpt_rev, 16, -, fp, _Quad, 2i,
3196 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2,
short, div_cpt_rev, 16, /, fp, _Quad, 2i, 1,
3198 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2u, ushort, div_cpt_rev, 16, /, fp, _Quad, 2i,
3202 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4, kmp_int32, sub_cpt_rev, 32, -, fp, _Quad, 4i,
3204 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4u, kmp_uint32, sub_cpt_rev, 32, -, fp, _Quad,
3206 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4, kmp_int32, div_cpt_rev, 32, /, fp, _Quad, 4i,
3208 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4u, kmp_uint32, div_cpt_rev, 32, /, fp, _Quad,
3211 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8, kmp_int64, sub_cpt_rev, 64, -, fp, _Quad, 8i,
3214 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8u, kmp_uint64, sub_cpt_rev, 64, -, fp, _Quad,
3217 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8, kmp_int64, div_cpt_rev, 64, /, fp, _Quad, 8i,
3220 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8u, kmp_uint64, div_cpt_rev, 64, /, fp, _Quad,
3224 ATOMIC_CMPXCHG_CPT_REV_MIX(float4, kmp_real32, sub_cpt_rev, 32, -, fp, _Quad,
3227 ATOMIC_CMPXCHG_CPT_REV_MIX(float4, kmp_real32, div_cpt_rev, 32, /, fp, _Quad,
3231 ATOMIC_CMPXCHG_CPT_REV_MIX(float8, kmp_real64, sub_cpt_rev, 64, -, fp, _Quad,
3234 ATOMIC_CMPXCHG_CPT_REV_MIX(float8, kmp_real64, div_cpt_rev, 64, /, fp, _Quad,
3238 ATOMIC_CRITICAL_CPT_REV_MIX(float10,
long double, sub_cpt_rev, -, fp, _Quad,
3240 ATOMIC_CRITICAL_CPT_REV_MIX(float10,
long double, div_cpt_rev, /, fp, _Quad,
3243 #endif // KMP_HAVE_QUAD 3247 #define ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \ 3248 TYPE __kmpc_atomic_##TYPE_ID##_swp(ident_t *id_ref, int gtid, TYPE *lhs, \ 3250 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 3251 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid)); 3253 #define CRITICAL_SWP(LCK_ID) \ 3254 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 3256 old_value = (*lhs); \ 3259 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 3263 #ifdef KMP_GOMP_COMPAT 3264 #define GOMP_CRITICAL_SWP(FLAG) \ 3265 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 3270 #define GOMP_CRITICAL_SWP(FLAG) 3273 #define ATOMIC_XCHG_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \ 3274 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \ 3276 GOMP_CRITICAL_SWP(GOMP_FLAG) \ 3277 old_value = KMP_XCHG_FIXED##BITS(lhs, rhs); \ 3281 #define ATOMIC_XCHG_FLOAT_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \ 3282 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \ 3284 GOMP_CRITICAL_SWP(GOMP_FLAG) \ 3285 old_value = KMP_XCHG_REAL##BITS(lhs, rhs); \ 3290 #define CMPXCHG_SWP(TYPE, BITS) \ 3292 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 3293 TYPE old_value, new_value; \ 3295 old_value = temp_val; \ 3297 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 3298 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 3299 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ 3301 old_value = temp_val; \ 3308 #define ATOMIC_CMPXCHG_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \ 3309 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \ 3312 GOMP_CRITICAL_SWP(GOMP_FLAG) \ 3313 CMPXCHG_SWP(TYPE, BITS) \ 3316 ATOMIC_XCHG_SWP(fixed1, kmp_int8, 8, KMP_ARCH_X86)
3317 ATOMIC_XCHG_SWP(fixed2, kmp_int16, 16, KMP_ARCH_X86)
3318 ATOMIC_XCHG_SWP(fixed4, kmp_int32, 32, KMP_ARCH_X86)
3320 ATOMIC_XCHG_FLOAT_SWP(float4, kmp_real32, 32,
3324 ATOMIC_CMPXCHG_SWP(fixed8, kmp_int64, 64,
3326 ATOMIC_CMPXCHG_SWP(float8, kmp_real64, 64,
3329 ATOMIC_XCHG_SWP(fixed8, kmp_int64, 64, KMP_ARCH_X86)
3330 ATOMIC_XCHG_FLOAT_SWP(float8, kmp_real64, 64,
3332 #endif // (KMP_ARCH_X86) 3337 #define ATOMIC_CRITICAL_SWP(TYPE_ID, TYPE, LCK_ID, GOMP_FLAG) \ 3338 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \ 3340 GOMP_CRITICAL_SWP(GOMP_FLAG) \ 3341 CRITICAL_SWP(LCK_ID) \ 3349 #define ATOMIC_BEGIN_SWP_WRK(TYPE_ID, TYPE) \ 3350 void __kmpc_atomic_##TYPE_ID##_swp(ident_t *id_ref, int gtid, TYPE *lhs, \ 3351 TYPE rhs, TYPE *out) { \ 3352 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 3353 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid)); 3355 #define CRITICAL_SWP_WRK(LCK_ID) \ 3356 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 3361 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 3365 #ifdef KMP_GOMP_COMPAT 3366 #define GOMP_CRITICAL_SWP_WRK(FLAG) \ 3367 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 3369 CRITICAL_SWP_WRK(0); \ 3372 #define GOMP_CRITICAL_SWP_WRK(FLAG) 3376 #define ATOMIC_CRITICAL_SWP_WRK(TYPE_ID, TYPE, LCK_ID, GOMP_FLAG) \ 3377 ATOMIC_BEGIN_SWP_WRK(TYPE_ID, TYPE) \ 3379 GOMP_CRITICAL_SWP_WRK(GOMP_FLAG) \ 3380 CRITICAL_SWP_WRK(LCK_ID) \ 3384 ATOMIC_CRITICAL_SWP(float10,
long double, 10r, 1)
3386 ATOMIC_CRITICAL_SWP(float16, QUAD_LEGACY, 16r, 1)
3387 #endif // KMP_HAVE_QUAD 3389 ATOMIC_CRITICAL_SWP_WRK(cmplx4, kmp_cmplx32, 8c, 1)
3394 ATOMIC_CRITICAL_SWP(cmplx8, kmp_cmplx64, 16c, 1)
3395 ATOMIC_CRITICAL_SWP(cmplx10, kmp_cmplx80, 20c, 1)
3397 ATOMIC_CRITICAL_SWP(cmplx16, CPLX128_LEG, 32c, 1)
3399 ATOMIC_CRITICAL_SWP(float16_a16, Quad_a16_t, 16r,
3401 ATOMIC_CRITICAL_SWP(cmplx16_a16, kmp_cmplx128_a16_t, 32c,
3403 #endif // (KMP_ARCH_X86) 3404 #endif // KMP_HAVE_QUAD 3408 #endif // KMP_ARCH_X86 || KMP_ARCH_X86_64 3415 void __kmpc_atomic_1(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3416 void (*f)(
void *,
void *,
void *)) {
3417 KMP_DEBUG_ASSERT(__kmp_init_serial);
3420 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
3426 kmp_int8 old_value, new_value;
3428 old_value = *(kmp_int8 *)lhs;
3429 (*f)(&new_value, &old_value, rhs);
3432 while (!KMP_COMPARE_AND_STORE_ACQ8((kmp_int8 *)lhs, *(kmp_int8 *)&old_value,
3433 *(kmp_int8 *)&new_value)) {
3436 old_value = *(kmp_int8 *)lhs;
3437 (*f)(&new_value, &old_value, rhs);
3444 #ifdef KMP_GOMP_COMPAT 3445 if (__kmp_atomic_mode == 2) {
3446 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3449 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_1i, gtid);
3451 (*f)(lhs, lhs, rhs);
3453 #ifdef KMP_GOMP_COMPAT 3454 if (__kmp_atomic_mode == 2) {
3455 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3458 __kmp_release_atomic_lock(&__kmp_atomic_lock_1i, gtid);
3462 void __kmpc_atomic_2(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3463 void (*f)(
void *,
void *,
void *)) {
3465 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
3467 #elif KMP_ARCH_X86 || KMP_ARCH_X86_64
3470 !((kmp_uintptr_t)lhs & 0x1)
3473 kmp_int16 old_value, new_value;
3475 old_value = *(kmp_int16 *)lhs;
3476 (*f)(&new_value, &old_value, rhs);
3479 while (!KMP_COMPARE_AND_STORE_ACQ16(
3480 (kmp_int16 *)lhs, *(kmp_int16 *)&old_value, *(kmp_int16 *)&new_value)) {
3483 old_value = *(kmp_int16 *)lhs;
3484 (*f)(&new_value, &old_value, rhs);
3491 #ifdef KMP_GOMP_COMPAT 3492 if (__kmp_atomic_mode == 2) {
3493 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3496 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_2i, gtid);
3498 (*f)(lhs, lhs, rhs);
3500 #ifdef KMP_GOMP_COMPAT 3501 if (__kmp_atomic_mode == 2) {
3502 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3505 __kmp_release_atomic_lock(&__kmp_atomic_lock_2i, gtid);
3509 void __kmpc_atomic_4(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3510 void (*f)(
void *,
void *,
void *)) {
3511 KMP_DEBUG_ASSERT(__kmp_init_serial);
3516 #
if KMP_ARCH_X86 || KMP_ARCH_X86_64
3519 !((kmp_uintptr_t)lhs & 0x3)
3522 kmp_int32 old_value, new_value;
3524 old_value = *(kmp_int32 *)lhs;
3525 (*f)(&new_value, &old_value, rhs);
3528 while (!KMP_COMPARE_AND_STORE_ACQ32(
3529 (kmp_int32 *)lhs, *(kmp_int32 *)&old_value, *(kmp_int32 *)&new_value)) {
3532 old_value = *(kmp_int32 *)lhs;
3533 (*f)(&new_value, &old_value, rhs);
3541 #ifdef KMP_GOMP_COMPAT 3542 if (__kmp_atomic_mode == 2) {
3543 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3546 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_4i, gtid);
3548 (*f)(lhs, lhs, rhs);
3550 #ifdef KMP_GOMP_COMPAT 3551 if (__kmp_atomic_mode == 2) {
3552 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3555 __kmp_release_atomic_lock(&__kmp_atomic_lock_4i, gtid);
3559 void __kmpc_atomic_8(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3560 void (*f)(
void *,
void *,
void *)) {
3561 KMP_DEBUG_ASSERT(__kmp_init_serial);
3564 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
3566 #elif KMP_ARCH_X86 || KMP_ARCH_X86_64
3569 !((kmp_uintptr_t)lhs & 0x7)
3572 kmp_int64 old_value, new_value;
3574 old_value = *(kmp_int64 *)lhs;
3575 (*f)(&new_value, &old_value, rhs);
3577 while (!KMP_COMPARE_AND_STORE_ACQ64(
3578 (kmp_int64 *)lhs, *(kmp_int64 *)&old_value, *(kmp_int64 *)&new_value)) {
3581 old_value = *(kmp_int64 *)lhs;
3582 (*f)(&new_value, &old_value, rhs);
3590 #ifdef KMP_GOMP_COMPAT 3591 if (__kmp_atomic_mode == 2) {
3592 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3595 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_8i, gtid);
3597 (*f)(lhs, lhs, rhs);
3599 #ifdef KMP_GOMP_COMPAT 3600 if (__kmp_atomic_mode == 2) {
3601 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3604 __kmp_release_atomic_lock(&__kmp_atomic_lock_8i, gtid);
3607 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 3608 void __kmpc_atomic_10(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3609 void (*f)(
void *,
void *,
void *)) {
3610 KMP_DEBUG_ASSERT(__kmp_init_serial);
3612 #ifdef KMP_GOMP_COMPAT 3613 if (__kmp_atomic_mode == 2) {
3614 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3617 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_10r, gtid);
3619 (*f)(lhs, lhs, rhs);
3621 #ifdef KMP_GOMP_COMPAT 3622 if (__kmp_atomic_mode == 2) {
3623 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3626 __kmp_release_atomic_lock(&__kmp_atomic_lock_10r, gtid);
3628 #endif // KMP_ARCH_X86 || KMP_ARCH_X86_64 3630 void __kmpc_atomic_16(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3631 void (*f)(
void *,
void *,
void *)) {
3632 KMP_DEBUG_ASSERT(__kmp_init_serial);
3634 #ifdef KMP_GOMP_COMPAT 3635 if (__kmp_atomic_mode == 2) {
3636 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3639 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_16c, gtid);
3641 (*f)(lhs, lhs, rhs);
3643 #ifdef KMP_GOMP_COMPAT 3644 if (__kmp_atomic_mode == 2) {
3645 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3648 __kmp_release_atomic_lock(&__kmp_atomic_lock_16c, gtid);
3650 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 3651 void __kmpc_atomic_20(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3652 void (*f)(
void *,
void *,
void *)) {
3653 KMP_DEBUG_ASSERT(__kmp_init_serial);
3655 #ifdef KMP_GOMP_COMPAT 3656 if (__kmp_atomic_mode == 2) {
3657 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3660 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_20c, gtid);
3662 (*f)(lhs, lhs, rhs);
3664 #ifdef KMP_GOMP_COMPAT 3665 if (__kmp_atomic_mode == 2) {
3666 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3669 __kmp_release_atomic_lock(&__kmp_atomic_lock_20c, gtid);
3671 #endif // KMP_ARCH_X86 || KMP_ARCH_X86_64 3672 void __kmpc_atomic_32(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3673 void (*f)(
void *,
void *,
void *)) {
3674 KMP_DEBUG_ASSERT(__kmp_init_serial);
3676 #ifdef KMP_GOMP_COMPAT 3677 if (__kmp_atomic_mode == 2) {
3678 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3681 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_32c, gtid);
3683 (*f)(lhs, lhs, rhs);
3685 #ifdef KMP_GOMP_COMPAT 3686 if (__kmp_atomic_mode == 2) {
3687 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3690 __kmp_release_atomic_lock(&__kmp_atomic_lock_32c, gtid);
3696 void __kmpc_atomic_start(
void) {
3697 int gtid = __kmp_entry_gtid();
3698 KA_TRACE(20, (
"__kmpc_atomic_start: T#%d\n", gtid));
3699 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3702 void __kmpc_atomic_end(
void) {
3703 int gtid = __kmp_get_gtid();
3704 KA_TRACE(20, (
"__kmpc_atomic_end: T#%d\n", gtid));
3705 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3708 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 3726 bool __kmpc_atomic_bool_1_cas(
ident_t *loc,
int gtid,
char *x,
char e,
char d) {
3727 return KMP_COMPARE_AND_STORE_ACQ8(x, e, d);
3729 bool __kmpc_atomic_bool_2_cas(
ident_t *loc,
int gtid,
short *x,
short e,
3731 return KMP_COMPARE_AND_STORE_ACQ16(x, e, d);
3733 bool __kmpc_atomic_bool_4_cas(
ident_t *loc,
int gtid, kmp_int32 *x, kmp_int32 e,
3735 return KMP_COMPARE_AND_STORE_ACQ32(x, e, d);
3737 bool __kmpc_atomic_bool_8_cas(
ident_t *loc,
int gtid, kmp_int64 *x, kmp_int64 e,
3739 return KMP_COMPARE_AND_STORE_ACQ64(x, e, d);
3756 char __kmpc_atomic_val_1_cas(
ident_t *loc,
int gtid,
char *x,
char e,
char d) {
3757 return KMP_COMPARE_AND_STORE_RET8(x, e, d);
3759 short __kmpc_atomic_val_2_cas(
ident_t *loc,
int gtid,
short *x,
short e,
3761 return KMP_COMPARE_AND_STORE_RET16(x, e, d);
3763 kmp_int32 __kmpc_atomic_val_4_cas(
ident_t *loc,
int gtid, kmp_int32 *x,
3764 kmp_int32 e, kmp_int32 d) {
3765 return KMP_COMPARE_AND_STORE_RET32(x, e, d);
3767 kmp_int64 __kmpc_atomic_val_8_cas(
ident_t *loc,
int gtid, kmp_int64 *x,
3768 kmp_int64 e, kmp_int64 d) {
3769 return KMP_COMPARE_AND_STORE_RET64(x, e, d);
3788 bool __kmpc_atomic_bool_1_cas_cpt(
ident_t *loc,
int gtid,
char *x,
char e,
3790 char old = KMP_COMPARE_AND_STORE_RET8(x, e, d);
3793 KMP_ASSERT(pv != NULL);
3797 bool __kmpc_atomic_bool_2_cas_cpt(
ident_t *loc,
int gtid,
short *x,
short e,
3798 short d,
short *pv) {
3799 short old = KMP_COMPARE_AND_STORE_RET16(x, e, d);
3802 KMP_ASSERT(pv != NULL);
3806 bool __kmpc_atomic_bool_4_cas_cpt(
ident_t *loc,
int gtid, kmp_int32 *x,
3807 kmp_int32 e, kmp_int32 d, kmp_int32 *pv) {
3808 kmp_int32 old = KMP_COMPARE_AND_STORE_RET32(x, e, d);
3811 KMP_ASSERT(pv != NULL);
3815 bool __kmpc_atomic_bool_8_cas_cpt(
ident_t *loc,
int gtid, kmp_int64 *x,
3816 kmp_int64 e, kmp_int64 d, kmp_int64 *pv) {
3817 kmp_int64 old = KMP_COMPARE_AND_STORE_RET64(x, e, d);
3820 KMP_ASSERT(pv != NULL);
3841 char __kmpc_atomic_val_1_cas_cpt(
ident_t *loc,
int gtid,
char *x,
char e,
3843 char old = KMP_COMPARE_AND_STORE_RET8(x, e, d);
3844 KMP_ASSERT(pv != NULL);
3845 *pv = old == e ? d : old;
3848 short __kmpc_atomic_val_2_cas_cpt(
ident_t *loc,
int gtid,
short *x,
short e,
3849 short d,
short *pv) {
3850 short old = KMP_COMPARE_AND_STORE_RET16(x, e, d);
3851 KMP_ASSERT(pv != NULL);
3852 *pv = old == e ? d : old;
3855 kmp_int32 __kmpc_atomic_val_4_cas_cpt(
ident_t *loc,
int gtid, kmp_int32 *x,
3856 kmp_int32 e, kmp_int32 d, kmp_int32 *pv) {
3857 kmp_int32 old = KMP_COMPARE_AND_STORE_RET32(x, e, d);
3858 KMP_ASSERT(pv != NULL);
3859 *pv = old == e ? d : old;
3862 kmp_int64 __kmpc_atomic_val_8_cas_cpt(
ident_t *loc,
int gtid, kmp_int64 *x,
3863 kmp_int64 e, kmp_int64 d, kmp_int64 *pv) {
3864 kmp_int64 old = KMP_COMPARE_AND_STORE_RET64(x, e, d);
3865 KMP_ASSERT(pv != NULL);
3866 *pv = old == e ? d : old;
3871 #endif // KMP_ARCH_X86 || KMP_ARCH_X86_64