1 /*********************************************************************** 2 * * 3 * This software is part of the ast package * 4 * Copyright (c) 1985-2012 AT&T Intellectual Property * 5 * and is licensed under the * 6 * Eclipse Public License, Version 1.0 * 7 * by AT&T Intellectual Property * 8 * * 9 * A copy of the License is available at * 10 * http://www.eclipse.org/org/documents/epl-v10.html * 11 * (with md5 checksum b35adb5213ca9657e911e9befb180842) * 12 * * 13 * Information and Software Systems Research * 14 * AT&T Research * 15 * Florham Park NJ * 16 * * 17 * Glenn Fowler <gsf@research.att.com> * 18 * David Korn <dgk@research.att.com> * 19 * Phong Vo <kpv@research.att.com> * 20 * * 21 ***********************************************************************/ 22 #pragma prototyped 23 24 #include "asohdr.h" 25 #include "FEATURE/aso" 26 27 #if defined(_UWIN) && defined(_BLD_ast) 28 29 NoN(aso) 30 31 #else 32 33 /* 34 * ast atomic scalar operations 35 * AT&T Research 36 * 37 * cas { 8 16 32 [64] } subset snarfed from the work by 38 * Adam Edgar and Kiem-Phong Vo 2010-10-10 39 * 40 * lock methods and emulations by 41 * Glenn Fowler 2011-11-11 42 * 43 * hopefully stable by 2012-12-12 44 */ 45 46 #if !_PACKAGE_ast 47 48 #if _UWIN 49 50 extern ssize_t sfsprintf(char*, size_t, const char*, ...); 51 52 #else 53 54 #include <stdio.h> 55 56 #define sfsprintf snprintf 57 58 #endif 59 60 #endif 61 62 #if defined(_aso_casptr) && (defined(_aso_cas32) || defined(_aso_cas64)) 63 #define ASO_METHOD (&_aso_meth_intrinsic) 64 #define ASO_LOCKF 0 65 #else 66 #define ASO_METHOD (&_aso_meth_signal) 67 #define ASO_LOCKF _aso_lock_signal 68 #endif 69 70 typedef union 71 { 72 uint8_t c[2]; 73 uint16_t i; 74 } U16_8_t; 75 76 typedef union 77 { 78 uint8_t c[4]; 79 uint32_t i; 80 } U32_8_t; 81 82 typedef union 83 { 84 uint16_t c[2]; 85 uint32_t i; 86 } U32_16_t; 87 88 #ifdef _ast_int8_t 89 90 typedef union 91 { 92 uint8_t c[8]; 93 uint64_t i; 94 } U64_8_t; 95 96 typedef union 97 { 98 uint16_t c[4]; 99 uint64_t i; 100 } U64_16_t; 101 102 typedef union 103 { 104 uint32_t c[2]; 105 uint64_t i; 106 } U64_32_t; 107 108 #endif 109 110 typedef struct State_s 111 { 112 Asometh_t* meth; 113 Asolock_f lockf; 114 Asoerror_f errorf; 115 uintmax_t hung; 116 unsigned int hung2; 117 void* data; 118 pid_t pid; 119 } State_t; 120 121 static unsigned int _aso_data_signal; 122 123 static ssize_t 124 _aso_lock_signal(void* data, ssize_t k, void volatile* p) 125 { 126 if (k >= 0) 127 { 128 _aso_data_signal--; 129 return 0; 130 } 131 while (_aso_data_signal++) 132 _aso_data_signal--; 133 return 1; 134 } 135 136 static Asometh_t _aso_meth_signal = { "signal", ASO_SIGNAL, 0, _aso_lock_signal }; 137 extern Asometh_t _aso_meth_semaphore; 138 extern Asometh_t _aso_meth_fcntl; 139 static Asometh_t _aso_meth_intrinsic = { "intrinsic", ASO_INTRINSIC|ASO_PROCESS|ASO_THREAD|ASO_SIGNAL, 0, 0 }; 140 141 static Asometh_t* method[] = 142 { 143 &_aso_meth_signal, 144 #if defined(_ast_int8_t) && defined(_aso_cas64) || !defined(_ast_int8_t) && defined(_aso_cas32) 145 &_aso_meth_intrinsic, 146 #endif 147 #if _aso_semaphore 148 &_aso_meth_semaphore, 149 #endif 150 #if _aso_fcntl 151 &_aso_meth_fcntl, 152 #endif 153 }; 154 155 static State_t state = 156 { 157 ASO_METHOD, ASO_LOCKF 158 }; 159 160 static int 161 asoerror(int type, const char* format, const char* a, const char* b, long n) 162 { 163 char buf[128]; 164 165 if (b) 166 sfsprintf(buf, sizeof(buf), format, a, b, n); 167 else if (a) 168 sfsprintf(buf, sizeof(buf), format, a, n); 169 else 170 sfsprintf(buf, sizeof(buf), format, n); 171 return state.errorf(type, buf); 172 } 173 174 /* 175 * if type!=0 return lock method for type with name details 176 * else if name!=0 return lock method matching <name>[,<details>] 177 * else return the current lock method 178 * 0 returned on error 179 * 180 * the user visible asometh() calls this function 181 * it allows, e.g., for -ltaso to provide an asometh() intercept 182 * that prepends its ASO_THREAD methods ahead of the _asometh() methods 183 */ 184 185 Asometh_t* 186 _asometh(int type, void* data) 187 { 188 size_t n; 189 int i; 190 char* e; 191 Asometh_t* meth; 192 char* name; 193 194 if (type == ASO_NEXT) 195 { 196 if (!(meth = (Asometh_t*)data)) 197 return method[0]; 198 for (i = 0; i < elementsof(method) - 1; i++) 199 if (meth == method[i]) 200 return method[i+1]; 201 return 0; 202 } 203 if (type) 204 { 205 for (i = 0; i < elementsof(method); i++) 206 if (method[i]->type & type) 207 { 208 method[i]->details = (char*)data; 209 return method[i]; 210 } 211 return 0; 212 } 213 if (!(name = (char*)data)) 214 return state.meth; 215 n = (e = strchr(name, ',')) ? (e - name) : strlen(name); 216 for (i = 0; i < elementsof(method); i++) 217 if (strncmp(name, method[i]->name, n) == 0) 218 { 219 if (e) 220 method[i]->details = e + 1; 221 return method[i]; 222 } 223 return 0; 224 } 225 226 /* 227 * clean up lock method on exit 228 */ 229 230 static void 231 asoexit(void) 232 { 233 if (state.meth && state.meth->initf && state.data && state.pid == getpid()) 234 { 235 state.lockf = ASO_METHOD->lockf; 236 state.meth->initf(state.data, 0); 237 state.data = 0; 238 } 239 } 240 241 /* 242 * initialize lock method 243 */ 244 245 int 246 asoinit(const char* details, Asometh_t* meth, Asodisc_t* disc) 247 { 248 void* data; 249 250 if (disc) 251 { 252 state.errorf = disc->errorf; 253 state.hung2 = disc->hung; 254 state.hung = 1; 255 state.hung <<= state.hung2; 256 state.hung--; 257 } 258 if (!meth) 259 return state.pid != 0; 260 if (!meth->lockf && !(meth->type & ASO_INTRINSIC)) 261 { 262 if (state.errorf) 263 asoerror(ASO_EMETHOD, "%s method has no lock function", meth->name, 0, 0); 264 return -1; 265 } 266 state.lockf = ASO_METHOD->lockf; 267 if (state.meth && state.meth->initf && state.data) 268 { 269 state.meth->initf(state.data, 0); 270 state.data = 0; 271 } 272 if (!meth->initf) 273 data = 0; 274 else if (!(data = meth->initf(0, details ? details : meth->details))) 275 { 276 state.meth = ASO_METHOD; 277 if (state.errorf) 278 asoerror(ASO_EMETHOD, "%s method initialization failed -- reverting to the %s method", meth->name, state.meth->name, 0); 279 return -1; 280 } 281 state.meth = meth; 282 state.data = data; 283 state.lockf = meth->lockf; 284 if (!state.pid) 285 { 286 state.pid = getpid(); 287 atexit(asoexit); 288 } 289 return 0; 290 } 291 292 /* 293 * loop check for hung spin locks 294 * and periodic relinquishing of the processor 295 */ 296 297 int 298 asoloop(uintmax_t rep) 299 { 300 if (state.hung && !(rep & state.hung) && state.errorf) 301 return asoerror(ASO_EHUNG, "spin lock possibly hung after 2^%u attempts", 0, 0, state.hung2); 302 return (rep & ASO_RELAX) ? 0 : asorelax(1); 303 } 304 305 /* 306 * error checking state.lockf() call 307 */ 308 309 static ssize_t 310 lock(void* data, ssize_t k, void volatile* p) 311 { 312 ssize_t r; 313 314 if ((r = state.lockf(data, k, p)) < 0 && state.errorf) 315 asoerror(ASO_EMETHOD, "%s method lock failed", state.meth->name, 0, 0); 316 return r; 317 } 318 319 /* 320 * sync and return "current" value 321 */ 322 323 uint8_t 324 asoget8(uint8_t volatile* p) 325 { 326 int o; 327 328 do 329 { 330 o = *p; 331 } while (asocas8(p, o, o) != o); 332 return o; 333 } 334 335 uint16_t 336 asoget16(uint16_t volatile* p) 337 { 338 int o; 339 340 do 341 { 342 o = *p; 343 } while (asocas16(p, o, o) != o); 344 return o; 345 } 346 347 uint32_t 348 asoget32(uint32_t volatile* p) 349 { 350 uint32_t o; 351 352 do 353 { 354 o = *p; 355 } while (asocas32(p, o, o) != o); 356 return o; 357 } 358 359 #ifdef _ast_int8_t 360 361 uint64_t 362 asoget64(uint64_t volatile* p) 363 { 364 uint64_t o; 365 366 do 367 { 368 o = *p; 369 } while (asocas64(p, o, o) != o); 370 return o; 371 } 372 373 #endif 374 375 void* 376 asogetptr(void volatile* p) 377 { 378 void* o; 379 380 do 381 { 382 o = *(void* volatile*)p; 383 } while (asocasptr(p, o, o) != o); 384 return o; 385 } 386 387 /* 388 * increment and return old value 389 */ 390 391 uint8_t 392 asoinc8(uint8_t volatile* p) 393 { 394 ssize_t k; 395 int o; 396 397 #if defined(_aso_inc8) 398 if (!state.lockf) 399 return _aso_inc8(p); 400 #else 401 if (!state.lockf) 402 { 403 do 404 { 405 o = *p; 406 } while (asocas8(p, o, o + 1) != o); 407 return o; 408 } 409 #endif 410 k = lock(state.data, 0, p); 411 o = (*p)++; 412 lock(state.data, k, p); 413 return o; 414 } 415 416 uint16_t 417 asoinc16(uint16_t volatile* p) 418 { 419 ssize_t k; 420 int o; 421 422 #if defined(_aso_inc16) 423 if (!state.lockf) 424 return _aso_inc16(p); 425 #else 426 if (!state.lockf) 427 { 428 do 429 { 430 o = *p; 431 } while (asocas16(p, o, o + 1) != o); 432 return o; 433 } 434 #endif 435 k = lock(state.data, 0, p); 436 o = (*p)++; 437 lock(state.data, k, p); 438 return o; 439 } 440 441 uint32_t 442 asoinc32(uint32_t volatile* p) 443 { 444 ssize_t k; 445 int o; 446 447 #if defined(_aso_inc32) 448 if (!state.lockf) 449 return _aso_inc32(p); 450 #else 451 if (!state.lockf) 452 { 453 do 454 { 455 o = *p; 456 } while (asocas32(p, o, o + 1) != o); 457 return o; 458 } 459 #endif 460 k = lock(state.data, 0, p); 461 o = (*p)++; 462 lock(state.data, k, p); 463 return o; 464 } 465 466 #ifdef _ast_int8_t 467 468 uint64_t 469 asoinc64(uint64_t volatile* p) 470 { 471 ssize_t k; 472 uint64_t o; 473 474 #if defined(_aso_inc64) 475 if (!state.lockf) 476 return _aso_inc64(p); 477 #else 478 if (!state.lockf) 479 { 480 do 481 { 482 o = *p; 483 } while (asocas64(p, o, o + 1) != o); 484 return o; 485 } 486 #endif 487 k = lock(state.data, 0, p); 488 o = (*p)++; 489 lock(state.data, k, p); 490 return o; 491 } 492 493 #endif 494 495 /* 496 * decrement and return old value 497 */ 498 499 uint8_t 500 asodec8(uint8_t volatile* p) 501 { 502 ssize_t k; 503 int o; 504 505 #if defined(_aso_dec8) 506 if (!state.lockf) 507 return _aso_dec8(p); 508 #else 509 if (!state.lockf) 510 { 511 do 512 { 513 o = *p; 514 } while (asocas8(p, o, o - 1) != o); 515 return o; 516 } 517 #endif 518 k = lock(state.data, 0, p); 519 o = (*p)--; 520 lock(state.data, k, p); 521 return o; 522 } 523 524 uint16_t 525 asodec16(uint16_t volatile* p) 526 { 527 ssize_t k; 528 int o; 529 530 #if defined(_aso_dec16) 531 if (!state.lockf) 532 return _aso_dec16(p); 533 #else 534 if (!state.lockf) 535 { 536 do 537 { 538 o = *p; 539 } while (asocas16(p, o, o - 1) != o); 540 return o; 541 } 542 #endif 543 k = lock(state.data, 0, p); 544 o = (*p)--; 545 lock(state.data, k, p); 546 return o; 547 } 548 549 uint32_t 550 asodec32(uint32_t volatile* p) 551 { 552 ssize_t k; 553 int o; 554 555 #if defined(_aso_dec32) 556 if (!state.lockf) 557 return _aso_dec32(p); 558 #else 559 if (!state.lockf) 560 { 561 do 562 { 563 o = *p; 564 } while (asocas32(p, o, o - 1) != o); 565 return o; 566 } 567 #endif 568 k = lock(state.data, 0, p); 569 o = (*p)--; 570 lock(state.data, k, p); 571 return o; 572 } 573 574 #ifdef _ast_int8_t 575 576 uint64_t 577 asodec64(uint64_t volatile* p) 578 { 579 ssize_t k; 580 uint64_t o; 581 582 #if defined(_aso_dec64) 583 if (!state.lockf) 584 return _aso_dec64(p); 585 #else 586 if (!state.lockf) 587 { 588 do 589 { 590 o = *p; 591 } while (asocas64(p, o, o - 1) != o); 592 return o; 593 } 594 #endif 595 k = lock(state.data, 0, p); 596 o = (*p)--; 597 lock(state.data, k, p); 598 return o; 599 } 600 601 #endif 602 603 /* 604 * { 8 16 32 [64] } compare with old, swap with new if same, and return old value 605 */ 606 607 uint8_t 608 asocas8(uint8_t volatile* p, int o, int n) 609 { 610 ssize_t k; 611 612 #if defined(_aso_cas8) 613 if (!state.lockf) 614 return _aso_cas8(p, o, n); 615 #elif defined(_aso_cas16) 616 if (!state.lockf) 617 { 618 U16_8_t u; 619 U16_8_t v; 620 U16_8_t* a; 621 int s; 622 int i; 623 624 s = (int)(integralof(p) & (sizeof(u.i) - 1)); 625 a = (U16_8_t*)((char*)0 + (integralof(p) & ~(sizeof(u.i) - 1))); 626 for (;;) 627 { 628 u.i = a->i; 629 u.c[s] = o; 630 v.i = u.i; 631 v.c[s] = n; 632 if (_aso_cas16(&a->i, u.i, v.i) == u.i) 633 break; 634 for (i = 0;; i++) 635 if (i >= elementsof(u.c)) 636 return a->c[s]; 637 else if (i != s && u.c[i] != a->c[i]) 638 break; 639 } 640 return o; 641 } 642 #elif defined(_aso_cas32) 643 if (!state.lockf) 644 { 645 U32_8_t u; 646 U32_8_t v; 647 U32_8_t* a; 648 int s; 649 int i; 650 651 s = (int)(integralof(p) & (sizeof(u.i) - 1)); 652 a = (U32_8_t*)((char*)0 + (integralof(p) & ~(sizeof(u.i) - 1))); 653 for (;;) 654 { 655 u.i = a->i; 656 u.c[s] = o; 657 v.i = u.i; 658 v.c[s] = n; 659 if (_aso_cas32(&a->i, u.i, v.i) == u.i) 660 break; 661 for (i = 0;; i++) 662 if (i >= elementsof(u.c)) 663 return a->c[s]; 664 else if (i != s && u.c[i] != a->c[i]) 665 break; 666 } 667 return o; 668 } 669 #elif defined(_aso_cas64) 670 if (!state.lockf) 671 { 672 U64_8_t u; 673 U64_8_t v; 674 U64_8_t* a; 675 int s; 676 int i; 677 678 s = (int)(integralof(p) & (sizeof(u.i) - 1)); 679 a = (U64_8_t*)((char*)0 + (integralof(p) & ~(sizeof(u.i) - 1))); 680 for (;;) 681 { 682 u.i = a->i; 683 u.c[s] = o; 684 v.i = u.i; 685 v.c[s] = n; 686 if (_aso_cas64(&a->i, u.i, v.i) == u.i) 687 break; 688 for (i = 0;; i++) 689 if (i >= elementsof(u.c)) 690 return a->c[s]; 691 else if (i != s && u.c[i] != a->c[i]) 692 break; 693 } 694 return o; 695 } 696 #endif 697 k = lock(state.data, 0, p); 698 if (*p == o) 699 *p = n; 700 else 701 o = *p; 702 lock(state.data, k, p); 703 return o; 704 } 705 706 uint16_t 707 asocas16(uint16_t volatile* p, uint16_t o, uint16_t n) 708 { 709 ssize_t k; 710 711 #if defined(_aso_cas16) 712 if (!state.lockf) 713 return _aso_cas16(p, o, n); 714 #elif defined(_aso_cas32) 715 if (!state.lockf) 716 { 717 U32_16_t u; 718 U32_16_t v; 719 U32_16_t* a; 720 int s; 721 int i; 722 723 s = (int)(integralof(p) & (sizeof(u.i) - 1)) / 2; 724 a = (U32_16_t*)((char*)0 + (integralof(p) & ~(sizeof(u.i) - 1))); 725 for (;;) 726 { 727 u.i = a->i; 728 u.c[s] = o; 729 v.i = u.i; 730 v.c[s] = n; 731 if (_aso_cas32(&a->i, u.i, v.i) == u.i) 732 break; 733 for (i = 0;; i++) 734 if (i >= elementsof(u.c)) 735 return a->c[s]; 736 else if (i != s && u.c[i] != a->c[i]) 737 break; 738 } 739 return o; 740 } 741 #elif defined(_aso_cas64) 742 if (!state.lockf) 743 { 744 U64_16_t u; 745 U64_16_t v; 746 U64_16_t* a; 747 int s; 748 int i; 749 750 s = (int)(integralof(p) & (sizeof(u.i) - 1)) / 2; 751 a = (U64_16_t*)((char*)0 + (integralof(p) & ~(sizeof(u.i) - 1))); 752 for (;;) 753 { 754 u.i = a->i; 755 u.c[s] = o; 756 v.i = u.i; 757 v.c[s] = n; 758 if (_aso_cas64(&a->i, u.i, v.i) == u.i) 759 break; 760 for (i = 0;; i++) 761 if (i >= elementsof(u.c)) 762 return a->c[s]; 763 else if (i != s && u.c[i] != a->c[i]) 764 break; 765 } 766 return o; 767 } 768 #endif 769 k = lock(state.data, 0, p); 770 if (*p == o) 771 *p = n; 772 else 773 o = *p; 774 lock(state.data, k, p); 775 return o; 776 } 777 778 uint32_t 779 asocas32(uint32_t volatile* p, uint32_t o, uint32_t n) 780 { 781 ssize_t k; 782 783 #if defined(_aso_cas32) 784 if (!state.lockf) 785 return _aso_cas32(p, o, n); 786 #elif defined(_aso_cas64) 787 if (!state.lockf) 788 { 789 U64_32_t u; 790 U64_32_t v; 791 U64_32_t* a; 792 int s; 793 int i; 794 795 s = (int)(integralof(p) & (sizeof(u.i) - 1)) / 4; 796 a = (U64_32_t*)((char*)0 + (integralof(p) & ~(sizeof(u.i) - 1))); 797 for (;;) 798 { 799 u.i = a->i; 800 u.c[s] = o; 801 v.i = u.i; 802 v.c[s] = n; 803 if (_aso_cas64(&a->i, u.i, v.i) == u.i) 804 break; 805 for (i = 0;; i++) 806 if (i >= elementsof(u.c)) 807 return a->c[s]; 808 else if (i != s && u.c[i] != a->c[i]) 809 break; 810 } 811 return o; 812 } 813 #endif 814 k = lock(state.data, 0, p); 815 if (*p == o) 816 *p = n; 817 else 818 o = *p; 819 lock(state.data, k, p); 820 return o; 821 } 822 823 #ifdef _ast_int8_t 824 825 uint64_t 826 asocas64(uint64_t volatile* p, uint64_t o, uint64_t n) 827 { 828 ssize_t k; 829 830 #if defined(_aso_cas64) 831 if (!state.lockf) 832 return _aso_cas64(p, o, n); 833 #endif 834 k = lock(state.data, 0, p); 835 if (*p == o) 836 *p = n; 837 else 838 o = *p; 839 lock(state.data, k, p); 840 return o; 841 } 842 843 #endif 844 845 /* 846 * compare with old, swap with new if same, and return old value 847 */ 848 849 void* 850 asocasptr(void volatile* p, void* o, void* n) 851 { 852 ssize_t k; 853 854 #if defined(_aso_casptr) 855 if (!state.lockf) 856 return _aso_casptr((void**)p, o, n); 857 #endif 858 k = lock(state.data, 0, p); 859 if (*(void* volatile*)p == o) 860 *(void* volatile*)p = n; 861 else 862 o = *(void* volatile*)p; 863 lock(state.data, k, p); 864 return o; 865 } 866 867 #endif 868