00001
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029
00030
00031
00032
00033
00034
00035
00036
00037
00038 #ifndef MBEDTLS_BN_MUL_H
00039 #define MBEDTLS_BN_MUL_H
00040
00041 #include "bignum.h"
00042
00043 #if defined(MBEDTLS_HAVE_ASM)
00044
00045 #ifndef asm
00046 #define asm __asm
00047 #endif
00048
00049
00050 #if defined(__GNUC__) && \
00051 ( !defined(__ARMCC_VERSION) || __ARMCC_VERSION >= 6000000 )
00052
00053
00054
00055
00056
00057
00058
00059
00060
00061
00062
00063
00064
00065 #if defined(__GNUC__) && __GNUC__ < 5 && defined(__PIC__)
00066 #define MULADDC_CANNOT_USE_EBX
00067 #endif
00068
00069
00070
00071
00072
00073
00074
00075 #if defined(__i386__) && defined(__OPTIMIZE__) && !defined(MULADDC_CANNOT_USE_EBX)
00076
00077 #define MULADDC_INIT \
00078 asm( \
00079 "movl %%ebx, %0 \n\t" \
00080 "movl %5, %%esi \n\t" \
00081 "movl %6, %%edi \n\t" \
00082 "movl %7, %%ecx \n\t" \
00083 "movl %8, %%ebx \n\t"
00084
00085 #define MULADDC_CORE \
00086 "lodsl \n\t" \
00087 "mull %%ebx \n\t" \
00088 "addl %%ecx, %%eax \n\t" \
00089 "adcl $0, %%edx \n\t" \
00090 "addl (%%edi), %%eax \n\t" \
00091 "adcl $0, %%edx \n\t" \
00092 "movl %%edx, %%ecx \n\t" \
00093 "stosl \n\t"
00094
00095 #if defined(MBEDTLS_HAVE_SSE2)
00096
00097 #define MULADDC_HUIT \
00098 "movd %%ecx, %%mm1 \n\t" \
00099 "movd %%ebx, %%mm0 \n\t" \
00100 "movd (%%edi), %%mm3 \n\t" \
00101 "paddq %%mm3, %%mm1 \n\t" \
00102 "movd (%%esi), %%mm2 \n\t" \
00103 "pmuludq %%mm0, %%mm2 \n\t" \
00104 "movd 4(%%esi), %%mm4 \n\t" \
00105 "pmuludq %%mm0, %%mm4 \n\t" \
00106 "movd 8(%%esi), %%mm6 \n\t" \
00107 "pmuludq %%mm0, %%mm6 \n\t" \
00108 "movd 12(%%esi), %%mm7 \n\t" \
00109 "pmuludq %%mm0, %%mm7 \n\t" \
00110 "paddq %%mm2, %%mm1 \n\t" \
00111 "movd 4(%%edi), %%mm3 \n\t" \
00112 "paddq %%mm4, %%mm3 \n\t" \
00113 "movd 8(%%edi), %%mm5 \n\t" \
00114 "paddq %%mm6, %%mm5 \n\t" \
00115 "movd 12(%%edi), %%mm4 \n\t" \
00116 "paddq %%mm4, %%mm7 \n\t" \
00117 "movd %%mm1, (%%edi) \n\t" \
00118 "movd 16(%%esi), %%mm2 \n\t" \
00119 "pmuludq %%mm0, %%mm2 \n\t" \
00120 "psrlq $32, %%mm1 \n\t" \
00121 "movd 20(%%esi), %%mm4 \n\t" \
00122 "pmuludq %%mm0, %%mm4 \n\t" \
00123 "paddq %%mm3, %%mm1 \n\t" \
00124 "movd 24(%%esi), %%mm6 \n\t" \
00125 "pmuludq %%mm0, %%mm6 \n\t" \
00126 "movd %%mm1, 4(%%edi) \n\t" \
00127 "psrlq $32, %%mm1 \n\t" \
00128 "movd 28(%%esi), %%mm3 \n\t" \
00129 "pmuludq %%mm0, %%mm3 \n\t" \
00130 "paddq %%mm5, %%mm1 \n\t" \
00131 "movd 16(%%edi), %%mm5 \n\t" \
00132 "paddq %%mm5, %%mm2 \n\t" \
00133 "movd %%mm1, 8(%%edi) \n\t" \
00134 "psrlq $32, %%mm1 \n\t" \
00135 "paddq %%mm7, %%mm1 \n\t" \
00136 "movd 20(%%edi), %%mm5 \n\t" \
00137 "paddq %%mm5, %%mm4 \n\t" \
00138 "movd %%mm1, 12(%%edi) \n\t" \
00139 "psrlq $32, %%mm1 \n\t" \
00140 "paddq %%mm2, %%mm1 \n\t" \
00141 "movd 24(%%edi), %%mm5 \n\t" \
00142 "paddq %%mm5, %%mm6 \n\t" \
00143 "movd %%mm1, 16(%%edi) \n\t" \
00144 "psrlq $32, %%mm1 \n\t" \
00145 "paddq %%mm4, %%mm1 \n\t" \
00146 "movd 28(%%edi), %%mm5 \n\t" \
00147 "paddq %%mm5, %%mm3 \n\t" \
00148 "movd %%mm1, 20(%%edi) \n\t" \
00149 "psrlq $32, %%mm1 \n\t" \
00150 "paddq %%mm6, %%mm1 \n\t" \
00151 "movd %%mm1, 24(%%edi) \n\t" \
00152 "psrlq $32, %%mm1 \n\t" \
00153 "paddq %%mm3, %%mm1 \n\t" \
00154 "movd %%mm1, 28(%%edi) \n\t" \
00155 "addl $32, %%edi \n\t" \
00156 "addl $32, %%esi \n\t" \
00157 "psrlq $32, %%mm1 \n\t" \
00158 "movd %%mm1, %%ecx \n\t"
00159
00160 #define MULADDC_STOP \
00161 "emms \n\t" \
00162 "movl %4, %%ebx \n\t" \
00163 "movl %%ecx, %1 \n\t" \
00164 "movl %%edi, %2 \n\t" \
00165 "movl %%esi, %3 \n\t" \
00166 : "=m" (t), "=m" (c), "=m" (d), "=m" (s) \
00167 : "m" (t), "m" (s), "m" (d), "m" (c), "m" (b) \
00168 : "eax", "ebx", "ecx", "edx", "esi", "edi" \
00169 );
00170
00171 #else
00172
00173 #define MULADDC_STOP \
00174 "movl %4, %%ebx \n\t" \
00175 "movl %%ecx, %1 \n\t" \
00176 "movl %%edi, %2 \n\t" \
00177 "movl %%esi, %3 \n\t" \
00178 : "=m" (t), "=m" (c), "=m" (d), "=m" (s) \
00179 : "m" (t), "m" (s), "m" (d), "m" (c), "m" (b) \
00180 : "eax", "ebx", "ecx", "edx", "esi", "edi" \
00181 );
00182 #endif
00183 #endif
00184
00185 #if defined(__amd64__) || defined (__x86_64__)
00186
00187 #define MULADDC_INIT \
00188 asm( \
00189 "xorq %%r8, %%r8\n"
00190
00191 #define MULADDC_CORE \
00192 "movq (%%rsi), %%rax\n" \
00193 "mulq %%rbx\n" \
00194 "addq $8, %%rsi\n" \
00195 "addq %%rcx, %%rax\n" \
00196 "movq %%r8, %%rcx\n" \
00197 "adcq $0, %%rdx\n" \
00198 "nop \n" \
00199 "addq %%rax, (%%rdi)\n" \
00200 "adcq %%rdx, %%rcx\n" \
00201 "addq $8, %%rdi\n"
00202
00203 #define MULADDC_STOP \
00204 : "+c" (c), "+D" (d), "+S" (s) \
00205 : "b" (b) \
00206 : "rax", "rdx", "r8" \
00207 );
00208
00209 #endif
00210
00211 #if defined(__mc68020__) || defined(__mcpu32__)
00212
00213 #define MULADDC_INIT \
00214 asm( \
00215 "movl %3, %%a2 \n\t" \
00216 "movl %4, %%a3 \n\t" \
00217 "movl %5, %%d3 \n\t" \
00218 "movl %6, %%d2 \n\t" \
00219 "moveq #0, %%d0 \n\t"
00220
00221 #define MULADDC_CORE \
00222 "movel %%a2@+, %%d1 \n\t" \
00223 "mulul %%d2, %%d4:%%d1 \n\t" \
00224 "addl %%d3, %%d1 \n\t" \
00225 "addxl %%d0, %%d4 \n\t" \
00226 "moveq #0, %%d3 \n\t" \
00227 "addl %%d1, %%a3@+ \n\t" \
00228 "addxl %%d4, %%d3 \n\t"
00229
00230 #define MULADDC_STOP \
00231 "movl %%d3, %0 \n\t" \
00232 "movl %%a3, %1 \n\t" \
00233 "movl %%a2, %2 \n\t" \
00234 : "=m" (c), "=m" (d), "=m" (s) \
00235 : "m" (s), "m" (d), "m" (c), "m" (b) \
00236 : "d0", "d1", "d2", "d3", "d4", "a2", "a3" \
00237 );
00238
00239 #define MULADDC_HUIT \
00240 "movel %%a2@+, %%d1 \n\t" \
00241 "mulul %%d2, %%d4:%%d1 \n\t" \
00242 "addxl %%d3, %%d1 \n\t" \
00243 "addxl %%d0, %%d4 \n\t" \
00244 "addl %%d1, %%a3@+ \n\t" \
00245 "movel %%a2@+, %%d1 \n\t" \
00246 "mulul %%d2, %%d3:%%d1 \n\t" \
00247 "addxl %%d4, %%d1 \n\t" \
00248 "addxl %%d0, %%d3 \n\t" \
00249 "addl %%d1, %%a3@+ \n\t" \
00250 "movel %%a2@+, %%d1 \n\t" \
00251 "mulul %%d2, %%d4:%%d1 \n\t" \
00252 "addxl %%d3, %%d1 \n\t" \
00253 "addxl %%d0, %%d4 \n\t" \
00254 "addl %%d1, %%a3@+ \n\t" \
00255 "movel %%a2@+, %%d1 \n\t" \
00256 "mulul %%d2, %%d3:%%d1 \n\t" \
00257 "addxl %%d4, %%d1 \n\t" \
00258 "addxl %%d0, %%d3 \n\t" \
00259 "addl %%d1, %%a3@+ \n\t" \
00260 "movel %%a2@+, %%d1 \n\t" \
00261 "mulul %%d2, %%d4:%%d1 \n\t" \
00262 "addxl %%d3, %%d1 \n\t" \
00263 "addxl %%d0, %%d4 \n\t" \
00264 "addl %%d1, %%a3@+ \n\t" \
00265 "movel %%a2@+, %%d1 \n\t" \
00266 "mulul %%d2, %%d3:%%d1 \n\t" \
00267 "addxl %%d4, %%d1 \n\t" \
00268 "addxl %%d0, %%d3 \n\t" \
00269 "addl %%d1, %%a3@+ \n\t" \
00270 "movel %%a2@+, %%d1 \n\t" \
00271 "mulul %%d2, %%d4:%%d1 \n\t" \
00272 "addxl %%d3, %%d1 \n\t" \
00273 "addxl %%d0, %%d4 \n\t" \
00274 "addl %%d1, %%a3@+ \n\t" \
00275 "movel %%a2@+, %%d1 \n\t" \
00276 "mulul %%d2, %%d3:%%d1 \n\t" \
00277 "addxl %%d4, %%d1 \n\t" \
00278 "addxl %%d0, %%d3 \n\t" \
00279 "addl %%d1, %%a3@+ \n\t" \
00280 "addxl %%d0, %%d3 \n\t"
00281
00282 #endif
00283
00284 #if defined(__powerpc64__) || defined(__ppc64__)
00285
00286 #if defined(__MACH__) && defined(__APPLE__)
00287
00288 #define MULADDC_INIT \
00289 asm( \
00290 "ld r3, %3 \n\t" \
00291 "ld r4, %4 \n\t" \
00292 "ld r5, %5 \n\t" \
00293 "ld r6, %6 \n\t" \
00294 "addi r3, r3, -8 \n\t" \
00295 "addi r4, r4, -8 \n\t" \
00296 "addic r5, r5, 0 \n\t"
00297
00298 #define MULADDC_CORE \
00299 "ldu r7, 8(r3) \n\t" \
00300 "mulld r8, r7, r6 \n\t" \
00301 "mulhdu r9, r7, r6 \n\t" \
00302 "adde r8, r8, r5 \n\t" \
00303 "ld r7, 8(r4) \n\t" \
00304 "addze r5, r9 \n\t" \
00305 "addc r8, r8, r7 \n\t" \
00306 "stdu r8, 8(r4) \n\t"
00307
00308 #define MULADDC_STOP \
00309 "addze r5, r5 \n\t" \
00310 "addi r4, r4, 8 \n\t" \
00311 "addi r3, r3, 8 \n\t" \
00312 "std r5, %0 \n\t" \
00313 "std r4, %1 \n\t" \
00314 "std r3, %2 \n\t" \
00315 : "=m" (c), "=m" (d), "=m" (s) \
00316 : "m" (s), "m" (d), "m" (c), "m" (b) \
00317 : "r3", "r4", "r5", "r6", "r7", "r8", "r9" \
00318 );
00319
00320
00321 #else
00322
00323 #define MULADDC_INIT \
00324 asm( \
00325 "ld %%r3, %3 \n\t" \
00326 "ld %%r4, %4 \n\t" \
00327 "ld %%r5, %5 \n\t" \
00328 "ld %%r6, %6 \n\t" \
00329 "addi %%r3, %%r3, -8 \n\t" \
00330 "addi %%r4, %%r4, -8 \n\t" \
00331 "addic %%r5, %%r5, 0 \n\t"
00332
00333 #define MULADDC_CORE \
00334 "ldu %%r7, 8(%%r3) \n\t" \
00335 "mulld %%r8, %%r7, %%r6 \n\t" \
00336 "mulhdu %%r9, %%r7, %%r6 \n\t" \
00337 "adde %%r8, %%r8, %%r5 \n\t" \
00338 "ld %%r7, 8(%%r4) \n\t" \
00339 "addze %%r5, %%r9 \n\t" \
00340 "addc %%r8, %%r8, %%r7 \n\t" \
00341 "stdu %%r8, 8(%%r4) \n\t"
00342
00343 #define MULADDC_STOP \
00344 "addze %%r5, %%r5 \n\t" \
00345 "addi %%r4, %%r4, 8 \n\t" \
00346 "addi %%r3, %%r3, 8 \n\t" \
00347 "std %%r5, %0 \n\t" \
00348 "std %%r4, %1 \n\t" \
00349 "std %%r3, %2 \n\t" \
00350 : "=m" (c), "=m" (d), "=m" (s) \
00351 : "m" (s), "m" (d), "m" (c), "m" (b) \
00352 : "r3", "r4", "r5", "r6", "r7", "r8", "r9" \
00353 );
00354
00355 #endif
00356
00357 #elif defined(__powerpc__) || defined(__ppc__)
00358
00359 #if defined(__MACH__) && defined(__APPLE__)
00360
00361 #define MULADDC_INIT \
00362 asm( \
00363 "lwz r3, %3 \n\t" \
00364 "lwz r4, %4 \n\t" \
00365 "lwz r5, %5 \n\t" \
00366 "lwz r6, %6 \n\t" \
00367 "addi r3, r3, -4 \n\t" \
00368 "addi r4, r4, -4 \n\t" \
00369 "addic r5, r5, 0 \n\t"
00370
00371 #define MULADDC_CORE \
00372 "lwzu r7, 4(r3) \n\t" \
00373 "mullw r8, r7, r6 \n\t" \
00374 "mulhwu r9, r7, r6 \n\t" \
00375 "adde r8, r8, r5 \n\t" \
00376 "lwz r7, 4(r4) \n\t" \
00377 "addze r5, r9 \n\t" \
00378 "addc r8, r8, r7 \n\t" \
00379 "stwu r8, 4(r4) \n\t"
00380
00381 #define MULADDC_STOP \
00382 "addze r5, r5 \n\t" \
00383 "addi r4, r4, 4 \n\t" \
00384 "addi r3, r3, 4 \n\t" \
00385 "stw r5, %0 \n\t" \
00386 "stw r4, %1 \n\t" \
00387 "stw r3, %2 \n\t" \
00388 : "=m" (c), "=m" (d), "=m" (s) \
00389 : "m" (s), "m" (d), "m" (c), "m" (b) \
00390 : "r3", "r4", "r5", "r6", "r7", "r8", "r9" \
00391 );
00392
00393 #else
00394
00395 #define MULADDC_INIT \
00396 asm( \
00397 "lwz %%r3, %3 \n\t" \
00398 "lwz %%r4, %4 \n\t" \
00399 "lwz %%r5, %5 \n\t" \
00400 "lwz %%r6, %6 \n\t" \
00401 "addi %%r3, %%r3, -4 \n\t" \
00402 "addi %%r4, %%r4, -4 \n\t" \
00403 "addic %%r5, %%r5, 0 \n\t"
00404
00405 #define MULADDC_CORE \
00406 "lwzu %%r7, 4(%%r3) \n\t" \
00407 "mullw %%r8, %%r7, %%r6 \n\t" \
00408 "mulhwu %%r9, %%r7, %%r6 \n\t" \
00409 "adde %%r8, %%r8, %%r5 \n\t" \
00410 "lwz %%r7, 4(%%r4) \n\t" \
00411 "addze %%r5, %%r9 \n\t" \
00412 "addc %%r8, %%r8, %%r7 \n\t" \
00413 "stwu %%r8, 4(%%r4) \n\t"
00414
00415 #define MULADDC_STOP \
00416 "addze %%r5, %%r5 \n\t" \
00417 "addi %%r4, %%r4, 4 \n\t" \
00418 "addi %%r3, %%r3, 4 \n\t" \
00419 "stw %%r5, %0 \n\t" \
00420 "stw %%r4, %1 \n\t" \
00421 "stw %%r3, %2 \n\t" \
00422 : "=m" (c), "=m" (d), "=m" (s) \
00423 : "m" (s), "m" (d), "m" (c), "m" (b) \
00424 : "r3", "r4", "r5", "r6", "r7", "r8", "r9" \
00425 );
00426
00427 #endif
00428
00429 #endif
00430
00431
00432
00433
00434
00435 #if 0 && defined(__sparc__)
00436 #if defined(__sparc64__)
00437
00438 #define MULADDC_INIT \
00439 asm( \
00440 "ldx %3, %%o0 \n\t" \
00441 "ldx %4, %%o1 \n\t" \
00442 "ld %5, %%o2 \n\t" \
00443 "ld %6, %%o3 \n\t"
00444
00445 #define MULADDC_CORE \
00446 "ld [%%o0], %%o4 \n\t" \
00447 "inc 4, %%o0 \n\t" \
00448 "ld [%%o1], %%o5 \n\t" \
00449 "umul %%o3, %%o4, %%o4 \n\t" \
00450 "addcc %%o4, %%o2, %%o4 \n\t" \
00451 "rd %%y, %%g1 \n\t" \
00452 "addx %%g1, 0, %%g1 \n\t" \
00453 "addcc %%o4, %%o5, %%o4 \n\t" \
00454 "st %%o4, [%%o1] \n\t" \
00455 "addx %%g1, 0, %%o2 \n\t" \
00456 "inc 4, %%o1 \n\t"
00457
00458 #define MULADDC_STOP \
00459 "st %%o2, %0 \n\t" \
00460 "stx %%o1, %1 \n\t" \
00461 "stx %%o0, %2 \n\t" \
00462 : "=m" (c), "=m" (d), "=m" (s) \
00463 : "m" (s), "m" (d), "m" (c), "m" (b) \
00464 : "g1", "o0", "o1", "o2", "o3", "o4", \
00465 "o5" \
00466 );
00467
00468 #else
00469
00470 #define MULADDC_INIT \
00471 asm( \
00472 "ld %3, %%o0 \n\t" \
00473 "ld %4, %%o1 \n\t" \
00474 "ld %5, %%o2 \n\t" \
00475 "ld %6, %%o3 \n\t"
00476
00477 #define MULADDC_CORE \
00478 "ld [%%o0], %%o4 \n\t" \
00479 "inc 4, %%o0 \n\t" \
00480 "ld [%%o1], %%o5 \n\t" \
00481 "umul %%o3, %%o4, %%o4 \n\t" \
00482 "addcc %%o4, %%o2, %%o4 \n\t" \
00483 "rd %%y, %%g1 \n\t" \
00484 "addx %%g1, 0, %%g1 \n\t" \
00485 "addcc %%o4, %%o5, %%o4 \n\t" \
00486 "st %%o4, [%%o1] \n\t" \
00487 "addx %%g1, 0, %%o2 \n\t" \
00488 "inc 4, %%o1 \n\t"
00489
00490 #define MULADDC_STOP \
00491 "st %%o2, %0 \n\t" \
00492 "st %%o1, %1 \n\t" \
00493 "st %%o0, %2 \n\t" \
00494 : "=m" (c), "=m" (d), "=m" (s) \
00495 : "m" (s), "m" (d), "m" (c), "m" (b) \
00496 : "g1", "o0", "o1", "o2", "o3", "o4", \
00497 "o5" \
00498 );
00499
00500 #endif
00501 #endif
00502
00503 #if defined(__microblaze__) || defined(microblaze)
00504
00505 #define MULADDC_INIT \
00506 asm( \
00507 "lwi r3, %3 \n\t" \
00508 "lwi r4, %4 \n\t" \
00509 "lwi r5, %5 \n\t" \
00510 "lwi r6, %6 \n\t" \
00511 "andi r7, r6, 0xffff \n\t" \
00512 "bsrli r6, r6, 16 \n\t"
00513
00514 #define MULADDC_CORE \
00515 "lhui r8, r3, 0 \n\t" \
00516 "addi r3, r3, 2 \n\t" \
00517 "lhui r9, r3, 0 \n\t" \
00518 "addi r3, r3, 2 \n\t" \
00519 "mul r10, r9, r6 \n\t" \
00520 "mul r11, r8, r7 \n\t" \
00521 "mul r12, r9, r7 \n\t" \
00522 "mul r13, r8, r6 \n\t" \
00523 "bsrli r8, r10, 16 \n\t" \
00524 "bsrli r9, r11, 16 \n\t" \
00525 "add r13, r13, r8 \n\t" \
00526 "add r13, r13, r9 \n\t" \
00527 "bslli r10, r10, 16 \n\t" \
00528 "bslli r11, r11, 16 \n\t" \
00529 "add r12, r12, r10 \n\t" \
00530 "addc r13, r13, r0 \n\t" \
00531 "add r12, r12, r11 \n\t" \
00532 "addc r13, r13, r0 \n\t" \
00533 "lwi r10, r4, 0 \n\t" \
00534 "add r12, r12, r10 \n\t" \
00535 "addc r13, r13, r0 \n\t" \
00536 "add r12, r12, r5 \n\t" \
00537 "addc r5, r13, r0 \n\t" \
00538 "swi r12, r4, 0 \n\t" \
00539 "addi r4, r4, 4 \n\t"
00540
00541 #define MULADDC_STOP \
00542 "swi r5, %0 \n\t" \
00543 "swi r4, %1 \n\t" \
00544 "swi r3, %2 \n\t" \
00545 : "=m" (c), "=m" (d), "=m" (s) \
00546 : "m" (s), "m" (d), "m" (c), "m" (b) \
00547 : "r3", "r4", "r5", "r6", "r7", "r8", \
00548 "r9", "r10", "r11", "r12", "r13" \
00549 );
00550
00551 #endif
00552
00553 #if defined(__tricore__)
00554
00555 #define MULADDC_INIT \
00556 asm( \
00557 "ld.a %%a2, %3 \n\t" \
00558 "ld.a %%a3, %4 \n\t" \
00559 "ld.w %%d4, %5 \n\t" \
00560 "ld.w %%d1, %6 \n\t" \
00561 "xor %%d5, %%d5 \n\t"
00562
00563 #define MULADDC_CORE \
00564 "ld.w %%d0, [%%a2+] \n\t" \
00565 "madd.u %%e2, %%e4, %%d0, %%d1 \n\t" \
00566 "ld.w %%d0, [%%a3] \n\t" \
00567 "addx %%d2, %%d2, %%d0 \n\t" \
00568 "addc %%d3, %%d3, 0 \n\t" \
00569 "mov %%d4, %%d3 \n\t" \
00570 "st.w [%%a3+], %%d2 \n\t"
00571
00572 #define MULADDC_STOP \
00573 "st.w %0, %%d4 \n\t" \
00574 "st.a %1, %%a3 \n\t" \
00575 "st.a %2, %%a2 \n\t" \
00576 : "=m" (c), "=m" (d), "=m" (s) \
00577 : "m" (s), "m" (d), "m" (c), "m" (b) \
00578 : "d0", "d1", "e2", "d4", "a2", "a3" \
00579 );
00580
00581 #endif
00582
00583
00584
00585
00586
00587
00588
00589
00590
00591
00592
00593
00594
00595 #if defined(__GNUC__) && !defined(__OPTIMIZE__)
00596 #define MULADDC_CANNOT_USE_R7
00597 #endif
00598
00599 #if defined(__arm__) && !defined(MULADDC_CANNOT_USE_R7)
00600
00601 #if defined(__thumb__) && !defined(__thumb2__)
00602
00603 #define MULADDC_INIT \
00604 asm( \
00605 "ldr r0, %3 \n\t" \
00606 "ldr r1, %4 \n\t" \
00607 "ldr r2, %5 \n\t" \
00608 "ldr r3, %6 \n\t" \
00609 "lsr r7, r3, #16 \n\t" \
00610 "mov r9, r7 \n\t" \
00611 "lsl r7, r3, #16 \n\t" \
00612 "lsr r7, r7, #16 \n\t" \
00613 "mov r8, r7 \n\t"
00614
00615 #define MULADDC_CORE \
00616 "ldmia r0!, {r6} \n\t" \
00617 "lsr r7, r6, #16 \n\t" \
00618 "lsl r6, r6, #16 \n\t" \
00619 "lsr r6, r6, #16 \n\t" \
00620 "mov r4, r8 \n\t" \
00621 "mul r4, r6 \n\t" \
00622 "mov r3, r9 \n\t" \
00623 "mul r6, r3 \n\t" \
00624 "mov r5, r9 \n\t" \
00625 "mul r5, r7 \n\t" \
00626 "mov r3, r8 \n\t" \
00627 "mul r7, r3 \n\t" \
00628 "lsr r3, r6, #16 \n\t" \
00629 "add r5, r5, r3 \n\t" \
00630 "lsr r3, r7, #16 \n\t" \
00631 "add r5, r5, r3 \n\t" \
00632 "add r4, r4, r2 \n\t" \
00633 "mov r2, #0 \n\t" \
00634 "adc r5, r2 \n\t" \
00635 "lsl r3, r6, #16 \n\t" \
00636 "add r4, r4, r3 \n\t" \
00637 "adc r5, r2 \n\t" \
00638 "lsl r3, r7, #16 \n\t" \
00639 "add r4, r4, r3 \n\t" \
00640 "adc r5, r2 \n\t" \
00641 "ldr r3, [r1] \n\t" \
00642 "add r4, r4, r3 \n\t" \
00643 "adc r2, r5 \n\t" \
00644 "stmia r1!, {r4} \n\t"
00645
00646 #define MULADDC_STOP \
00647 "str r2, %0 \n\t" \
00648 "str r1, %1 \n\t" \
00649 "str r0, %2 \n\t" \
00650 : "=m" (c), "=m" (d), "=m" (s) \
00651 : "m" (s), "m" (d), "m" (c), "m" (b) \
00652 : "r0", "r1", "r2", "r3", "r4", "r5", \
00653 "r6", "r7", "r8", "r9", "cc" \
00654 );
00655
00656 #else
00657
00658 #define MULADDC_INIT \
00659 asm( \
00660 "ldr r0, %3 \n\t" \
00661 "ldr r1, %4 \n\t" \
00662 "ldr r2, %5 \n\t" \
00663 "ldr r3, %6 \n\t"
00664
00665 #define MULADDC_CORE \
00666 "ldr r4, [r0], #4 \n\t" \
00667 "mov r5, #0 \n\t" \
00668 "ldr r6, [r1] \n\t" \
00669 "umlal r2, r5, r3, r4 \n\t" \
00670 "adds r7, r6, r2 \n\t" \
00671 "adc r2, r5, #0 \n\t" \
00672 "str r7, [r1], #4 \n\t"
00673
00674 #define MULADDC_STOP \
00675 "str r2, %0 \n\t" \
00676 "str r1, %1 \n\t" \
00677 "str r0, %2 \n\t" \
00678 : "=m" (c), "=m" (d), "=m" (s) \
00679 : "m" (s), "m" (d), "m" (c), "m" (b) \
00680 : "r0", "r1", "r2", "r3", "r4", "r5", \
00681 "r6", "r7", "cc" \
00682 );
00683
00684 #endif
00685
00686 #endif
00687
00688 #if defined(__alpha__)
00689
00690 #define MULADDC_INIT \
00691 asm( \
00692 "ldq $1, %3 \n\t" \
00693 "ldq $2, %4 \n\t" \
00694 "ldq $3, %5 \n\t" \
00695 "ldq $4, %6 \n\t"
00696
00697 #define MULADDC_CORE \
00698 "ldq $6, 0($1) \n\t" \
00699 "addq $1, 8, $1 \n\t" \
00700 "mulq $6, $4, $7 \n\t" \
00701 "umulh $6, $4, $6 \n\t" \
00702 "addq $7, $3, $7 \n\t" \
00703 "cmpult $7, $3, $3 \n\t" \
00704 "ldq $5, 0($2) \n\t" \
00705 "addq $7, $5, $7 \n\t" \
00706 "cmpult $7, $5, $5 \n\t" \
00707 "stq $7, 0($2) \n\t" \
00708 "addq $2, 8, $2 \n\t" \
00709 "addq $6, $3, $3 \n\t" \
00710 "addq $5, $3, $3 \n\t"
00711
00712 #define MULADDC_STOP \
00713 "stq $3, %0 \n\t" \
00714 "stq $2, %1 \n\t" \
00715 "stq $1, %2 \n\t" \
00716 : "=m" (c), "=m" (d), "=m" (s) \
00717 : "m" (s), "m" (d), "m" (c), "m" (b) \
00718 : "$1", "$2", "$3", "$4", "$5", "$6", "$7" \
00719 );
00720 #endif
00721
00722 #if defined(__mips__) && !defined(__mips64)
00723
00724 #define MULADDC_INIT \
00725 asm( \
00726 "lw $10, %3 \n\t" \
00727 "lw $11, %4 \n\t" \
00728 "lw $12, %5 \n\t" \
00729 "lw $13, %6 \n\t"
00730
00731 #define MULADDC_CORE \
00732 "lw $14, 0($10) \n\t" \
00733 "multu $13, $14 \n\t" \
00734 "addi $10, $10, 4 \n\t" \
00735 "mflo $14 \n\t" \
00736 "mfhi $9 \n\t" \
00737 "addu $14, $12, $14 \n\t" \
00738 "lw $15, 0($11) \n\t" \
00739 "sltu $12, $14, $12 \n\t" \
00740 "addu $15, $14, $15 \n\t" \
00741 "sltu $14, $15, $14 \n\t" \
00742 "addu $12, $12, $9 \n\t" \
00743 "sw $15, 0($11) \n\t" \
00744 "addu $12, $12, $14 \n\t" \
00745 "addi $11, $11, 4 \n\t"
00746
00747 #define MULADDC_STOP \
00748 "sw $12, %0 \n\t" \
00749 "sw $11, %1 \n\t" \
00750 "sw $10, %2 \n\t" \
00751 : "=m" (c), "=m" (d), "=m" (s) \
00752 : "m" (s), "m" (d), "m" (c), "m" (b) \
00753 : "$9", "$10", "$11", "$12", "$13", "$14", "$15" \
00754 );
00755
00756 #endif
00757 #endif
00758
00759 #if (defined(_MSC_VER) && defined(_M_IX86)) || defined(__WATCOMC__)
00760
00761 #define MULADDC_INIT \
00762 __asm mov esi, s \
00763 __asm mov edi, d \
00764 __asm mov ecx, c \
00765 __asm mov ebx, b
00766
00767 #define MULADDC_CORE \
00768 __asm lodsd \
00769 __asm mul ebx \
00770 __asm add eax, ecx \
00771 __asm adc edx, 0 \
00772 __asm add eax, [edi] \
00773 __asm adc edx, 0 \
00774 __asm mov ecx, edx \
00775 __asm stosd
00776
00777 #if defined(MBEDTLS_HAVE_SSE2)
00778
00779 #define EMIT __asm _emit
00780
00781 #define MULADDC_HUIT \
00782 EMIT 0x0F EMIT 0x6E EMIT 0xC9 \
00783 EMIT 0x0F EMIT 0x6E EMIT 0xC3 \
00784 EMIT 0x0F EMIT 0x6E EMIT 0x1F \
00785 EMIT 0x0F EMIT 0xD4 EMIT 0xCB \
00786 EMIT 0x0F EMIT 0x6E EMIT 0x16 \
00787 EMIT 0x0F EMIT 0xF4 EMIT 0xD0 \
00788 EMIT 0x0F EMIT 0x6E EMIT 0x66 EMIT 0x04 \
00789 EMIT 0x0F EMIT 0xF4 EMIT 0xE0 \
00790 EMIT 0x0F EMIT 0x6E EMIT 0x76 EMIT 0x08 \
00791 EMIT 0x0F EMIT 0xF4 EMIT 0xF0 \
00792 EMIT 0x0F EMIT 0x6E EMIT 0x7E EMIT 0x0C \
00793 EMIT 0x0F EMIT 0xF4 EMIT 0xF8 \
00794 EMIT 0x0F EMIT 0xD4 EMIT 0xCA \
00795 EMIT 0x0F EMIT 0x6E EMIT 0x5F EMIT 0x04 \
00796 EMIT 0x0F EMIT 0xD4 EMIT 0xDC \
00797 EMIT 0x0F EMIT 0x6E EMIT 0x6F EMIT 0x08 \
00798 EMIT 0x0F EMIT 0xD4 EMIT 0xEE \
00799 EMIT 0x0F EMIT 0x6E EMIT 0x67 EMIT 0x0C \
00800 EMIT 0x0F EMIT 0xD4 EMIT 0xFC \
00801 EMIT 0x0F EMIT 0x7E EMIT 0x0F \
00802 EMIT 0x0F EMIT 0x6E EMIT 0x56 EMIT 0x10 \
00803 EMIT 0x0F EMIT 0xF4 EMIT 0xD0 \
00804 EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
00805 EMIT 0x0F EMIT 0x6E EMIT 0x66 EMIT 0x14 \
00806 EMIT 0x0F EMIT 0xF4 EMIT 0xE0 \
00807 EMIT 0x0F EMIT 0xD4 EMIT 0xCB \
00808 EMIT 0x0F EMIT 0x6E EMIT 0x76 EMIT 0x18 \
00809 EMIT 0x0F EMIT 0xF4 EMIT 0xF0 \
00810 EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x04 \
00811 EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
00812 EMIT 0x0F EMIT 0x6E EMIT 0x5E EMIT 0x1C \
00813 EMIT 0x0F EMIT 0xF4 EMIT 0xD8 \
00814 EMIT 0x0F EMIT 0xD4 EMIT 0xCD \
00815 EMIT 0x0F EMIT 0x6E EMIT 0x6F EMIT 0x10 \
00816 EMIT 0x0F EMIT 0xD4 EMIT 0xD5 \
00817 EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x08 \
00818 EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
00819 EMIT 0x0F EMIT 0xD4 EMIT 0xCF \
00820 EMIT 0x0F EMIT 0x6E EMIT 0x6F EMIT 0x14 \
00821 EMIT 0x0F EMIT 0xD4 EMIT 0xE5 \
00822 EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x0C \
00823 EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
00824 EMIT 0x0F EMIT 0xD4 EMIT 0xCA \
00825 EMIT 0x0F EMIT 0x6E EMIT 0x6F EMIT 0x18 \
00826 EMIT 0x0F EMIT 0xD4 EMIT 0xF5 \
00827 EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x10 \
00828 EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
00829 EMIT 0x0F EMIT 0xD4 EMIT 0xCC \
00830 EMIT 0x0F EMIT 0x6E EMIT 0x6F EMIT 0x1C \
00831 EMIT 0x0F EMIT 0xD4 EMIT 0xDD \
00832 EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x14 \
00833 EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
00834 EMIT 0x0F EMIT 0xD4 EMIT 0xCE \
00835 EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x18 \
00836 EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
00837 EMIT 0x0F EMIT 0xD4 EMIT 0xCB \
00838 EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x1C \
00839 EMIT 0x83 EMIT 0xC7 EMIT 0x20 \
00840 EMIT 0x83 EMIT 0xC6 EMIT 0x20 \
00841 EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
00842 EMIT 0x0F EMIT 0x7E EMIT 0xC9
00843
00844 #define MULADDC_STOP \
00845 EMIT 0x0F EMIT 0x77 \
00846 __asm mov c, ecx \
00847 __asm mov d, edi \
00848 __asm mov s, esi \
00849
00850 #else
00851
00852 #define MULADDC_STOP \
00853 __asm mov c, ecx \
00854 __asm mov d, edi \
00855 __asm mov s, esi \
00856
00857 #endif
00858 #endif
00859
00860 #endif
00861
00862 #if !defined(MULADDC_CORE)
00863 #if defined(MBEDTLS_HAVE_UDBL)
00864
00865 #define MULADDC_INIT \
00866 { \
00867 mbedtls_t_udbl r; \
00868 mbedtls_mpi_uint r0, r1;
00869
00870 #define MULADDC_CORE \
00871 r = *(s++) * (mbedtls_t_udbl) b; \
00872 r0 = (mbedtls_mpi_uint) r; \
00873 r1 = (mbedtls_mpi_uint)( r >> biL ); \
00874 r0 += c; r1 += (r0 < c); \
00875 r0 += *d; r1 += (r0 < *d); \
00876 c = r1; *(d++) = r0;
00877
00878 #define MULADDC_STOP \
00879 }
00880
00881 #else
00882 #define MULADDC_INIT \
00883 { \
00884 mbedtls_mpi_uint s0, s1, b0, b1; \
00885 mbedtls_mpi_uint r0, r1, rx, ry; \
00886 b0 = ( b << biH ) >> biH; \
00887 b1 = ( b >> biH );
00888
00889 #define MULADDC_CORE \
00890 s0 = ( *s << biH ) >> biH; \
00891 s1 = ( *s >> biH ); s++; \
00892 rx = s0 * b1; r0 = s0 * b0; \
00893 ry = s1 * b0; r1 = s1 * b1; \
00894 r1 += ( rx >> biH ); \
00895 r1 += ( ry >> biH ); \
00896 rx <<= biH; ry <<= biH; \
00897 r0 += rx; r1 += (r0 < rx); \
00898 r0 += ry; r1 += (r0 < ry); \
00899 r0 += c; r1 += (r0 < c); \
00900 r0 += *d; r1 += (r0 < *d); \
00901 c = r1; *(d++) = r0;
00902
00903 #define MULADDC_STOP \
00904 }
00905
00906 #endif
00907 #endif
00908
00909 #endif