1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
23 */
24/*
25 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
26 * Use is subject to license terms.
27 */
28
29/*
30 * __vexp: double precision vector exp
31 *
32 * Algorithm:
33 *
34 * Write x = (k + j/256)ln2 + r, where k and j are integers, j >= 0,
35 * and |r| <= ln2/512.  Then exp(x) = 2^k * 2^(j/256) * exp(r).
36 * Compute exp(r) by a polynomial approximation exp(r) ~ 1 + p(r)
37 * where p(r) := r*(1+r*(B1+r*(B2+r*B3))).  From a table, obtain
38 * h and l such that h ~ 2^(j/256) to double precision and h+l
39 * ~ 2^(j/256) to well more than double precision.  Then exp(x)
40 * ~ 2^k * (h + (l + h * p(r))) to about double precision.  Note
41 * that the multiplication by 2^k requires some finagling when
42 * the result might be subnormal.
43 *
44 * Accuracy:
45 *
46 * For normal results, the largest error observed is less than
47 * 0.6 ulps.  For subnormal results, the largest error observed
48 * is 0.737 ulps.
49 */
50
51#include <sys/isa_defs.h>
52
53#ifdef _LITTLE_ENDIAN
54#define	HI(x)	*(1+(int *)&x)
55#define	LO(x)	*(unsigned *)&x
56#define	DBLWORD(x, y)	y, x
57#else
58#define	HI(x)	*(int *)&x
59#define	LO(x)	*(1+(unsigned *)&x)
60#define	DBLWORD(x, y)	x, y
61#endif
62
63#ifdef __RESTRICT
64#define	restrict _Restrict
65#else
66#define	restrict
67#endif
68
69static const double TBL[] = {
70	1.00000000000000000000e+00,  0.00000000000000000000e+00,
71	1.00271127505020252180e+00, -3.63661592869226394432e-17,
72	1.00542990111280272636e+00,  9.49918653545503175702e-17,
73	1.00815589811841754830e+00, -3.25205875608430806089e-17,
74	1.01088928605170047526e+00, -1.52347786033685771763e-17,
75	1.01363008495148942956e+00,  9.28359976818356758749e-18,
76	1.01637831491095309566e+00, -5.77217007319966002766e-17,
77	1.01913399607773791367e+00,  3.60190498225966110587e-17,
78	1.02189714865411662714e+00,  5.10922502897344389359e-17,
79	1.02466779289713572076e+00, -7.56160786848777820704e-17,
80	1.02744594911876374610e+00, -4.95607417464536982418e-17,
81	1.03023163768604097967e+00,  3.31983004108081294377e-17,
82	1.03302487902122841490e+00,  7.60083887402708848935e-18,
83	1.03582569360195719810e+00, -7.80678239133763616702e-17,
84	1.03863410196137873065e+00,  5.99627378885251061843e-17,
85	1.04145012468831610342e+00,  3.78483048028757620966e-17,
86	1.04427378242741375480e+00,  8.55188970553796365958e-17,
87	1.04710509587928979336e+00,  7.27707724310431474861e-17,
88	1.04994408580068721015e+00,  5.59293784812700258637e-17,
89	1.05279077300462642341e+00, -9.62948289902693573942e-17,
90	1.05564517836055715705e+00,  1.75932573877209198414e-18,
91	1.05850732279451276163e+00, -7.15265185663778073796e-17,
92	1.06137722728926209292e+00, -1.19735370853656575649e-17,
93	1.06425491288446449900e+00,  5.07875419861123039357e-17,
94	1.06714040067682369717e+00, -7.89985396684158212226e-17,
95	1.07003371182024187291e+00, -9.93716271128891938112e-17,
96	1.07293486752597555522e+00, -3.83966884335882380671e-18,
97	1.07584388906279104781e+00, -1.00027161511441361125e-17,
98	1.07876079775711986031e+00, -6.65666043605659260344e-17,
99	1.08168561499321524977e+00, -4.78262390299708626556e-17,
100	1.08461836221330920615e+00,  3.16615284581634611576e-17,
101	1.08755906091776965994e+00,  5.40934930782029075923e-18,
102	1.09050773266525768967e+00, -3.04678207981247114697e-17,
103	1.09346439907288583981e+00,  1.44139581472692093420e-17,
104	1.09642908181637688259e+00, -5.91993348444931582405e-17,
105	1.09940180263022191376e+00,  7.17045959970192322483e-17,
106	1.10238258330784089090e+00,  5.26603687157069438656e-17,
107	1.10537144570174117320e+00,  8.23928876050021358995e-17,
108	1.10836841172367872588e+00, -8.78681384518052661558e-17,
109	1.11137350334481754821e+00,  5.56394502666969764311e-17,
110	1.11438674259589243221e+00,  1.04102784568455709549e-16,
111	1.11740815156736927882e+00, -7.97680590262822045601e-17,
112	1.12043775240960674644e+00, -6.20108590655417874998e-17,
113	1.12347556733301989773e+00, -9.69973758898704299544e-17,
114	1.12652161860824184814e+00,  5.16585675879545612073e-17,
115	1.12957592856628807887e+00,  6.71280585872625658758e-17,
116	1.13263851959871919561e+00,  3.23735616673800026374e-17,
117	1.13570941415780546357e+00,  5.06659992612615524241e-17,
118	1.13878863475669156458e+00,  8.91281267602540777782e-17,
119	1.14187620396956157620e+00,  4.65109117753141238741e-17,
120	1.14497214443180417298e+00,  4.64128989217001065651e-17,
121	1.14807647884017893780e+00,  6.89774023662719177044e-17,
122	1.15118922995298267331e+00,  3.25071021886382721198e-17,
123	1.15431042059021593538e+00,  1.04171289462732661865e-16,
124	1.15744007363375112085e+00, -9.12387123113440028710e-17,
125	1.16057821202749877898e+00, -3.26104020541739310553e-17,
126	1.16372485877757747552e+00,  3.82920483692409349872e-17,
127	1.16688003695248165847e+00, -8.79187957999916974198e-17,
128	1.17004376968325018993e+00, -1.84774420179000469438e-18,
129	1.17321608016363732041e+00, -7.28756258658499447915e-17,
130	1.17639699165028122074e+00,  5.55420325421807896277e-17,
131	1.17958652746287584456e+00,  1.00923127751003904354e-16,
132	1.18278471098434101449e+00,  1.54297543007907605845e-17,
133	1.18599156566099384058e+00, -9.20950683529310590495e-18,
134	1.18920711500272102690e+00,  3.98201523146564611098e-17,
135	1.19243138258315117817e+00,  4.39755141560972082715e-17,
136	1.19566439203982732842e+00,  4.61660367048148139743e-17,
137	1.19890616707438057986e+00, -9.80919335600842311848e-17,
138	1.20215673145270307565e+00,  6.64498149925230124489e-17,
139	1.20541610900512385918e+00, -3.35727219326752963448e-17,
140	1.20868432362658162482e+00, -4.74672594522898409739e-17,
141	1.21196139927680124337e+00, -4.89061107752111835732e-17,
142	1.21524735998046895524e+00, -7.71263069268148813091e-17,
143	1.21854222982740845183e+00, -9.00672695836383767487e-17,
144	1.22184603297275762301e+00, -1.06110212114026911612e-16,
145	1.22515879363714552674e+00, -8.90353381426998342947e-17,
146	1.22848053610687002468e+00, -1.89878163130252995312e-17,
147	1.23181128473407586199e+00,  7.38938247161005024655e-17,
148	1.23515106393693341325e+00, -1.07552443443078413783e-16,
149	1.23849989819981654016e+00,  2.76770205557396742995e-17,
150	1.24185781207348400201e+00,  4.65802759183693679123e-17,
151	1.24522483017525797955e+00, -4.67724044984672750044e-17,
152	1.24860097718920481924e+00, -8.26181099902196355046e-17,
153	1.25198627786631622172e+00,  4.83416715246989759959e-17,
154	1.25538075702469109629e+00, -6.71138982129687841853e-18,
155	1.25878443954971652730e+00, -8.42178258773059935677e-17,
156	1.26219735039425073886e+00, -3.08446488747384584900e-17,
157	1.26561951457880628169e+00,  4.25057700345086802072e-17,
158	1.26905095719173321989e+00,  2.66793213134218609523e-18,
159	1.27249170338940276181e+00, -1.05779162672124210291e-17,
160	1.27594177839639200123e+00,  9.91543024421429032951e-17,
161	1.27940120750566932450e+00, -9.75909500835606221035e-17,
162	1.28287001607877826359e+00,  1.71359491824356096814e-17,
163	1.28634822954602556777e+00, -3.41695570693618197638e-17,
164	1.28983587340666572274e+00,  8.94925753089759172195e-17,
165	1.29333297322908946647e+00, -2.97459044313275164581e-17,
166	1.29683955465100964055e+00,  2.53825027948883149593e-17,
167	1.30035564337965059423e+00,  5.67872810280221742200e-17,
168	1.30388126519193581210e+00,  8.64767559826787117946e-17,
169	1.30741644593467731816e+00, -7.33664565287886889230e-17,
170	1.31096121152476441374e+00, -7.18153613551945385697e-17,
171	1.31451558794935463581e+00,  2.26754331510458564505e-17,
172	1.31807960126606404927e+00, -5.45795582714915288619e-17,
173	1.32165327760315753913e+00, -2.48063824591302174150e-17,
174	1.32523664315974132322e+00, -2.85873121003886075697e-17,
175	1.32882972420595435459e+00,  4.08908622391016005195e-17,
176	1.33243254708316150037e+00, -5.10158663091674334319e-17,
177	1.33604513820414583236e+00, -5.89186635638880135250e-17,
178	1.33966752405330291609e+00,  8.92728259483173198426e-17,
179	1.34329973118683532185e+00, -5.80258089020143775130e-17,
180	1.34694178623294580355e+00,  3.22406510125467916913e-17,
181	1.35059371589203447428e+00, -8.28711038146241653260e-17,
182	1.35425554693689265129e+00,  7.70094837980298946162e-17,
183	1.35792730621290114179e+00, -9.52963574482518886709e-17,
184	1.36160902063822475405e+00,  1.53378766127066804593e-18,
185	1.36530071720401191548e+00, -1.00053631259747639350e-16,
186	1.36900242297459051599e+00,  9.59379791911884877256e-17,
187	1.37271416508766841424e+00, -4.49596059523484126201e-17,
188	1.37643597075453016920e+00, -6.89858893587180104162e-17,
189	1.38016786726023799048e+00,  1.05103145799699839462e-16,
190	1.38390988196383202258e+00, -6.77051165879478628716e-17,
191	1.38766204229852907481e+00,  8.42298427487541531762e-17,
192	1.39142437577192623621e+00, -4.90617486528898870821e-17,
193	1.39519690996620027157e+00, -9.32933622422549531960e-17,
194	1.39897967253831123635e+00, -9.61421320905132307233e-17,
195	1.40277269122020475933e+00, -5.29578324940798922316e-17,
196	1.40657599381901543545e+00,  7.03491481213642218800e-18,
197	1.41038960821727066275e+00,  4.16654872843506164270e-17,
198	1.41421356237309514547e+00, -9.66729331345291345105e-17,
199	1.41804788432041517510e+00,  2.27443854218552945230e-17,
200	1.42189260216916557589e+00, -1.60778289158902441338e-17,
201	1.42574774410549420800e+00,  9.88069075850060728430e-17,
202	1.42961333839197002327e+00, -1.20316424890536551792e-17,
203	1.43348941336778890054e+00, -5.80245424392682610310e-17,
204	1.43737599744898236764e+00, -4.20403401646755661225e-17,
205	1.44127311912862565713e+00,  5.60250365087898567501e-18,
206	1.44518080697704665027e+00, -3.02375813499398731940e-17,
207	1.44909908964203504311e+00, -6.25940500081930925441e-17,
208	1.45302799584905262265e+00, -5.77994860939610610226e-17,
209	1.45696755440144376514e+00,  5.64867945387699814049e-17,
210	1.46091779418064704466e+00, -5.60037718607521580013e-17,
211	1.46487874414640573129e+00,  9.53076754358715731900e-17,
212	1.46885043333698184220e+00,  8.46588275653362637570e-17,
213	1.47283289086936752810e+00,  6.69177408194058937165e-17,
214	1.47682614593949934623e+00, -3.48399455689279579579e-17,
215	1.48083022782247186733e+00, -9.68695210263061857841e-17,
216	1.48484516587275239274e+00,  1.07800867644074807559e-16,
217	1.48887098952439700383e+00,  6.15536715774287133031e-17,
218	1.49290772829126483501e+00,  1.41929201542840357707e-17,
219	1.49695541176723545540e+00, -2.86166325389915821109e-17,
220	1.50101406962642558440e+00, -6.41376727579023503859e-17,
221	1.50508373162340647333e+00,  7.07471061358284636429e-17,
222	1.50916442759342284141e+00, -1.01645532775429503911e-16,
223	1.51325618745260981335e+00,  8.88449785133871209093e-17,
224	1.51735904119821474190e+00, -4.30869947204334080070e-17,
225	1.52147301890881458952e+00, -5.99638767594568341985e-18,
226	1.52559815074453819506e+00,  1.11795187801605698722e-16,
227	1.52973446694728698603e+00,  3.78579211515721903683e-17,
228	1.53388199784095591305e+00,  8.87522684443844614135e-17,
229	1.53804077383165682669e+00,  1.01746723511613580618e-16,
230	1.54221082540794074411e+00,  7.94983480969762085616e-17,
231	1.54639218314102144802e+00,  1.06839600056572198028e-16,
232	1.55058487768499997372e+00, -1.46007065906893851791e-17,
233	1.55478893977708865215e+00, -8.00316135011603564104e-17,
234	1.55900440023783692922e+00,  3.78120705335752750188e-17,
235	1.56323128997135762930e+00,  7.48477764559073438896e-17,
236	1.56746963996555299659e+00, -1.03520617688497219883e-16,
237	1.57171948129234140268e+00, -3.34298400468720006928e-17,
238	1.57598084510788649659e+00, -1.01369164712783039808e-17,
239	1.58025376265282457844e+00, -5.16340292955446806159e-17,
240	1.58453826525249374946e+00, -1.93377170345857029304e-17,
241	1.58883438431716395023e+00, -5.99495011882447940052e-18,
242	1.59314215134226699888e+00, -1.00944065423119624890e-16,
243	1.59746159790862707339e+00,  2.48683927962209992069e-17,
244	1.60179275568269341434e+00, -6.05491745352778434252e-17,
245	1.60613565641677102924e+00, -1.03545452880599952591e-16,
246	1.61049033194925428347e+00,  2.47071925697978878522e-17,
247	1.61485681420486071325e+00, -7.31666339912512326264e-17,
248	1.61923513519486372836e+00,  2.09413341542290924068e-17,
249	1.62362532701732886764e+00, -3.58451285141447470996e-17,
250	1.62802742185734783398e+00, -6.71295508470708408630e-17,
251	1.63244145198727497181e+00,  9.85281923042999296414e-17,
252	1.63686744976696441078e+00,  7.69832507131987557450e-17,
253	1.64130544764400632118e+00, -9.24756873764070550805e-17,
254	1.64575547815396494578e+00, -1.01256799136747726038e-16,
255	1.65021757392061774183e+00,  9.13327958872990419009e-18,
256	1.65469176765619430114e+00,  9.64329430319602742879e-17,
257	1.65917809216161615815e+00, -7.27554555082304942180e-17,
258	1.66367658032673637614e+00,  5.89099269671309967045e-17,
259	1.66818726513058246397e+00,  4.26917801957061447430e-17,
260	1.67271017964159662839e+00, -5.47671596459956307616e-17,
261	1.67724535701787846875e+00,  8.30394950995073155275e-17,
262	1.68179283050742900407e+00,  8.19901002058149652013e-17,
263	1.68635263344839336774e+00, -7.18146327835800944212e-17,
264	1.69092479926930527867e+00, -9.66967147439488016590e-17,
265	1.69550936148933262260e+00,  7.23841687284516664081e-17,
266	1.70010635371852347753e+00, -8.02371937039770024589e-18,
267	1.70471580965805125096e+00, -2.72888328479728156257e-17,
268	1.70933776310046292579e+00, -9.86877945663293107628e-17,
269	1.71397224792992597386e+00,  6.47397510775336706412e-17,
270	1.71861929812247793414e+00, -1.85138041826311098821e-17,
271	1.72327894774627399244e+00, -9.52212380039379996275e-17,
272	1.72795123096183766975e+00, -1.07509818612046424459e-16,
273	1.73263618202231106658e+00, -1.69805107431541549407e-18,
274	1.73733383527370621735e+00,  3.16438929929295694659e-17,
275	1.74204422515515644498e+00, -1.52595911895078879236e-18,
276	1.74676738619916904760e+00, -1.07522904835075145042e-16,
277	1.75150335303187820735e+00, -5.12445042059672465939e-17,
278	1.75625216037329945351e+00,  2.96014069544887330703e-17,
279	1.76101384303758390359e+00, -7.94325312503922771057e-17,
280	1.76578843593327272643e+00,  9.46131501808326786660e-17,
281	1.77057597406355471392e+00,  5.96179451004055584767e-17,
282	1.77537649252652118825e+00,  6.42973179655657203396e-17,
283	1.78019002651542446181e+00, -5.28462728909161736517e-17,
284	1.78501661131893496481e+00,  1.53304001210313138184e-17,
285	1.78985628232140103755e+00, -4.15435466068334977098e-17,
286	1.79470907500310716820e+00,  1.82274584279120867698e-17,
287	1.79957502494053511732e+00, -2.52688923335889795224e-17,
288	1.80445416780662393208e+00, -5.17722240879331788328e-17,
289	1.80934653937103195886e+00, -9.03264140245002968190e-17,
290	1.81425217550039885595e+00, -9.96953153892034881983e-17,
291	1.81917111215860849427e+00,  7.40267690114583888997e-17,
292	1.82410338540705341259e+00, -1.01596278622770830650e-16,
293	1.82904903140489727420e+00,  6.88919290883569563697e-17,
294	1.83400808640934243066e+00,  3.28310722424562658722e-17,
295	1.83898058677589371079e+00,  6.91896974027251194233e-18,
296	1.84396656895862598446e+00, -5.93974202694996455028e-17,
297	1.84896606951045083811e+00,  9.02758044626108928816e-17,
298	1.85397912508338547077e+00,  9.76188749072759353840e-17,
299	1.85900577242882047990e+00, -9.52870546198994068663e-17,
300	1.86404604839778897940e+00,  6.54091268062057047791e-17,
301	1.86909998994123860427e+00, -9.93850521425506708290e-17,
302	1.87416763411029996256e+00, -6.12276341300414256164e-17,
303	1.87924901805656019427e+00, -1.62263155578358447799e-17,
304	1.88434417903233453195e+00, -8.22659312553371090551e-17,
305	1.88945315439093919352e+00, -9.00516828505912548531e-17,
306	1.89457598158696560731e+00,  3.40340353521652967060e-17,
307	1.89971269817655530332e+00, -3.85973976937851370678e-17,
308	1.90486334181767413831e+00,  6.53385751471827862895e-17,
309	1.91002795027038985154e+00, -5.90968800674406023686e-17,
310	1.91520656139714740007e+00, -1.06199460561959626376e-16,
311	1.92039921316304740273e+00,  7.11668154063031418621e-17,
312	1.92560594363612502811e+00, -9.91496376969374092749e-17,
313	1.93082679098762710623e+00,  6.16714970616910955284e-17,
314	1.93606179349229434727e+00,  1.03323859606763257448e-16,
315	1.94131098952864045160e+00, -6.63802989162148798984e-17,
316	1.94657441757923321823e+00,  6.81102234953387718436e-17,
317	1.95185211623097831790e+00, -2.19901696997935108603e-17,
318	1.95714412417540017941e+00,  8.96076779103666776760e-17,
319	1.96245048020892731699e+00,  1.09768440009135469493e-16,
320	1.96777122323317588126e+00, -1.03149280115311315109e-16,
321	1.97310639225523432039e+00, -7.45161786395603748608e-18,
322	1.97845602638795092787e+00,  4.03887531092781665750e-17,
323	1.98382016485021939189e+00, -2.20345441239106265716e-17,
324	1.98919884696726634310e+00,  8.20513263836919941553e-18,
325	1.99459211217094023461e+00,  1.79097103520026450854e-17
326};
327
328static const union {
329	unsigned	i[2];
330	double		d;
331} C[] = {
332	{ DBLWORD(0x43380000, 0x00000000) },
333	{ DBLWORD(0x40771547, 0x652b82fe) },
334	{ DBLWORD(0x3f662e42, 0xfee00000) },
335	{ DBLWORD(0x3d6a39ef, 0x35793c76) },
336	{ DBLWORD(0x3ff00000, 0x00000000) },
337	{ DBLWORD(0x3fdfffff, 0xfffffff6) },
338	{ DBLWORD(0x3fc55555, 0x721a1d14) },
339	{ DBLWORD(0x3fa55555, 0x6e0896af) },
340	{ DBLWORD(0x01000000, 0x00000000) },
341	{ DBLWORD(0x7f000000, 0x00000000) },
342	{ DBLWORD(0x40862e42, 0xfefa39ef) },
343	{ DBLWORD(0xc0874910, 0xd52d3051) },
344	{ DBLWORD(0xfff00000, 0x00000000) },
345	{ DBLWORD(0x00000000, 0x00000000) }
346};
347
348#define	round		C[0].d
349#define	invln2_256	C[1].d
350#define	ln2_256h	C[2].d
351#define	ln2_256l	C[3].d
352#define	one		C[4].d
353#define	B1		C[5].d
354#define	B2		C[6].d
355#define	B3		C[7].d
356#define	tiny		C[8].d
357#define	huge		C[9].d
358#define	othresh		C[10].d
359#define	uthresh		C[11].d
360#define	neginf		C[12].d
361#define	zero		C[13].d
362
363#define	PROCESS(N)						\
364	y##N = (x##N * invln2_256) + round;			\
365	j##N = LO(y##N);					\
366	y##N -= round;						\
367	k##N = j##N >> 8;					\
368	j##N = (j##N & 0xff) << 1;				\
369	x##N = (x##N - y##N * ln2_256h) - y##N * ln2_256l;	\
370	y##N = x##N * (one + x##N * (B1 + x##N * (B2 + x##N * B3)));	\
371	t##N = TBL[j##N];					\
372	y##N = t##N + (TBL[j##N + 1] + t##N * y##N);		\
373	if (k##N < -1021) {					\
374		HI(y##N) += (k##N + 0x3ef) << 20;		\
375		y##N *= tiny;					\
376	} else {						\
377		HI(y##N) += k##N << 20;				\
378	}							\
379	*y = y##N;						\
380	y += stridey
381
382#define	PREPROCESS(N, index, label)				\
383	hx = HI(x[0]);						\
384	ix = hx & ~0x80000000;					\
385	x##N = *x;						\
386	x += stridex;						\
387	if (ix >= 0x40862e42) {					\
388		if (ix >= 0x7ff00000) { /* x is inf or nan */	\
389			y[index] = (x##N == neginf)? zero :	\
390			    x##N * x##N;			\
391			goto label;				\
392		}						\
393		if (x##N > othresh) {				\
394			y[index] = huge * huge;			\
395			goto label;				\
396		}						\
397		if (x##N < uthresh) {				\
398			y[index] = tiny * tiny;			\
399			goto label;				\
400		}						\
401	} else if (ix < 0x3e300000) { /* |x| < 2^-28 */		\
402		y[index] = one + x##N;				\
403		goto label;					\
404	}
405
406void
407__vexp(int n, double *restrict x, int stridex, double *restrict y,
408    int stridey)
409{
410	double		x0, x1, x2, x3, x4, x5;
411	double		y0, y1, y2, y3, y4, y5;
412	double		t0, t1, t2, t3, t4, t5;
413	int		k0, k1, k2, k3, k4, k5;
414	int		j0, j1, j2, j3, j4, j5;
415	int		hx, ix;
416
417	y -= stridey;
418
419	for (;;) {
420begin:
421		if (--n < 0)
422			break;
423		y += stridey;
424
425		PREPROCESS(0, 0, begin);
426
427		if (--n < 0)
428			goto process1;
429
430		PREPROCESS(1, stridey, process1);
431
432		if (--n < 0)
433			goto process2;
434
435		PREPROCESS(2, stridey << 1, process2);
436
437		if (--n < 0)
438			goto process3;
439
440		PREPROCESS(3, (stridey << 1) + stridey, process3);
441
442		if (--n < 0)
443			goto process4;
444
445		PREPROCESS(4, stridey << 2, process4);
446
447		if (--n < 0)
448			goto process5;
449
450		PREPROCESS(5, (stridey << 2) + stridey, process5);
451
452		y0 = (x0 * invln2_256) + round;
453		y1 = (x1 * invln2_256) + round;
454		y2 = (x2 * invln2_256) + round;
455		y3 = (x3 * invln2_256) + round;
456		y4 = (x4 * invln2_256) + round;
457		y5 = (x5 * invln2_256) + round;
458
459		j0 = LO(y0);
460		j1 = LO(y1);
461		j2 = LO(y2);
462		j3 = LO(y3);
463		j4 = LO(y4);
464		j5 = LO(y5);
465
466		y0 -= round;
467		y1 -= round;
468		y2 -= round;
469		y3 -= round;
470		y4 -= round;
471		y5 -= round;
472
473		k0 = j0 >> 8;
474		k1 = j1 >> 8;
475		k2 = j2 >> 8;
476		k3 = j3 >> 8;
477		k4 = j4 >> 8;
478		k5 = j5 >> 8;
479
480		j0 = (j0 & 0xff) << 1;
481		j1 = (j1 & 0xff) << 1;
482		j2 = (j2 & 0xff) << 1;
483		j3 = (j3 & 0xff) << 1;
484		j4 = (j4 & 0xff) << 1;
485		j5 = (j5 & 0xff) << 1;
486
487		x0 = (x0 - y0 * ln2_256h) - y0 * ln2_256l;
488		x1 = (x1 - y1 * ln2_256h) - y1 * ln2_256l;
489		x2 = (x2 - y2 * ln2_256h) - y2 * ln2_256l;
490		x3 = (x3 - y3 * ln2_256h) - y3 * ln2_256l;
491		x4 = (x4 - y4 * ln2_256h) - y4 * ln2_256l;
492		x5 = (x5 - y5 * ln2_256h) - y5 * ln2_256l;
493
494		y0 = x0 * (one + x0 * (B1 + x0 * (B2 + x0 * B3)));
495		y1 = x1 * (one + x1 * (B1 + x1 * (B2 + x1 * B3)));
496		y2 = x2 * (one + x2 * (B1 + x2 * (B2 + x2 * B3)));
497		y3 = x3 * (one + x3 * (B1 + x3 * (B2 + x3 * B3)));
498		y4 = x4 * (one + x4 * (B1 + x4 * (B2 + x4 * B3)));
499		y5 = x5 * (one + x5 * (B1 + x5 * (B2 + x5 * B3)));
500
501		t0 = TBL[j0];
502		t1 = TBL[j1];
503		t2 = TBL[j2];
504		t3 = TBL[j3];
505		t4 = TBL[j4];
506		t5 = TBL[j5];
507
508		y0 = t0 + (TBL[j0 + 1] + t0 * y0);
509		y1 = t1 + (TBL[j1 + 1] + t1 * y1);
510		y2 = t2 + (TBL[j2 + 1] + t2 * y2);
511		y3 = t3 + (TBL[j3 + 1] + t3 * y3);
512		y4 = t4 + (TBL[j4 + 1] + t4 * y4);
513		y5 = t5 + (TBL[j5 + 1] + t5 * y5);
514
515		if (k0 < -1021) {
516			HI(y0) += (k0 + 0x3ef) << 20;
517			y0 *= tiny;
518		} else {
519			HI(y0) += k0 << 20;
520		}
521		if (k1 < -1021) {
522			HI(y1) += (k1 + 0x3ef) << 20;
523			y1 *= tiny;
524		} else {
525			HI(y1) += k1 << 20;
526		}
527		if (k2 < -1021) {
528			HI(y2) += (k2 + 0x3ef) << 20;
529			y2 *= tiny;
530		} else {
531			HI(y2) += k2 << 20;
532		}
533		if (k3 < -1021) {
534			HI(y3) += (k3 + 0x3ef) << 20;
535			y3 *= tiny;
536		} else {
537			HI(y3) += k3 << 20;
538		}
539		if (k4 < -1021) {
540			HI(y4) += (k4 + 0x3ef) << 20;
541			y4 *= tiny;
542		} else {
543			HI(y4) += k4 << 20;
544		}
545		if (k5 < -1021) {
546			HI(y5) += (k5 + 0x3ef) << 20;
547			y5 *= tiny;
548		} else {
549			HI(y5) += k5 << 20;
550		}
551
552		y[0] = y0;
553		y[stridey] = y1;
554		y[stridey << 1] = y2;
555		y[(stridey << 1) + stridey] = y3;
556		y[stridey << 2] = y4;
557		y[(stridey << 2) + stridey] = y5;
558		y += (stridey << 2) + stridey;
559		continue;
560
561process1:
562		PROCESS(0);
563		continue;
564
565process2:
566		PROCESS(0);
567		PROCESS(1);
568		continue;
569
570process3:
571		PROCESS(0);
572		PROCESS(1);
573		PROCESS(2);
574		continue;
575
576process4:
577		PROCESS(0);
578		PROCESS(1);
579		PROCESS(2);
580		PROCESS(3);
581		continue;
582
583process5:
584		PROCESS(0);
585		PROCESS(1);
586		PROCESS(2);
587		PROCESS(3);
588		PROCESS(4);
589	}
590}
591