multipathd: Suppress uninteresting data race reports
[multipath-tools/.git] / third-party / valgrind / valgrind.h
1 /* -*- c -*-
2    ----------------------------------------------------------------
3
4    Notice that the following BSD-style license applies to this one
5    file (valgrind.h) only.  The rest of Valgrind is licensed under the
6    terms of the GNU General Public License, version 2, unless
7    otherwise indicated.  See the COPYING file in the source
8    distribution for details.
9
10    ----------------------------------------------------------------
11
12    This file is part of Valgrind, a dynamic binary instrumentation
13    framework.
14
15    Copyright (C) 2000-2015 Julian Seward.  All rights reserved.
16
17    Redistribution and use in source and binary forms, with or without
18    modification, are permitted provided that the following conditions
19    are met:
20
21    1. Redistributions of source code must retain the above copyright
22       notice, this list of conditions and the following disclaimer.
23
24    2. The origin of this software must not be misrepresented; you must 
25       not claim that you wrote the original software.  If you use this 
26       software in a product, an acknowledgment in the product 
27       documentation would be appreciated but is not required.
28
29    3. Altered source versions must be plainly marked as such, and must
30       not be misrepresented as being the original software.
31
32    4. The name of the author may not be used to endorse or promote 
33       products derived from this software without specific prior written 
34       permission.
35
36    THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
37    OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
38    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
39    ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
40    DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
41    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
42    GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
43    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
44    WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
45    NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
46    SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
47
48    ----------------------------------------------------------------
49
50    Notice that the above BSD-style license applies to this one file
51    (valgrind.h) only.  The entire rest of Valgrind is licensed under
52    the terms of the GNU General Public License, version 2.  See the
53    COPYING file in the source distribution for details.
54
55    ---------------------------------------------------------------- 
56 */
57
58
59 /* This file is for inclusion into client (your!) code.
60
61    You can use these macros to manipulate and query Valgrind's 
62    execution inside your own programs.
63
64    The resulting executables will still run without Valgrind, just a
65    little bit more slowly than they otherwise would, but otherwise
66    unchanged.  When not running on valgrind, each client request
67    consumes very few (eg. 7) instructions, so the resulting performance
68    loss is negligible unless you plan to execute client requests
69    millions of times per second.  Nevertheless, if that is still a
70    problem, you can compile with the NVALGRIND symbol defined (gcc
71    -DNVALGRIND) so that client requests are not even compiled in.  */
72
73 #ifndef __VALGRIND_H
74 #define __VALGRIND_H
75
76
77 /* ------------------------------------------------------------------ */
78 /* VERSION NUMBER OF VALGRIND                                         */
79 /* ------------------------------------------------------------------ */
80
81 /* Specify Valgrind's version number, so that user code can
82    conditionally compile based on our version number.  Note that these
83    were introduced at version 3.6 and so do not exist in version 3.5
84    or earlier.  The recommended way to use them to check for "version
85    X.Y or later" is (eg)
86
87 #if defined(__VALGRIND_MAJOR__) && defined(__VALGRIND_MINOR__)   \
88     && (__VALGRIND_MAJOR__ > 3                                   \
89         || (__VALGRIND_MAJOR__ == 3 && __VALGRIND_MINOR__ >= 6))
90 */
91 #define __VALGRIND_MAJOR__    3
92 #define __VALGRIND_MINOR__    11
93
94
95 #include <stdarg.h>
96
97 /* Nb: this file might be included in a file compiled with -ansi.  So
98    we can't use C++ style "//" comments nor the "asm" keyword (instead
99    use "__asm__"). */
100
101 /* Derive some tags indicating what the target platform is.  Note
102    that in this file we're using the compiler's CPP symbols for
103    identifying architectures, which are different to the ones we use
104    within the rest of Valgrind.  Note, __powerpc__ is active for both
105    32 and 64-bit PPC, whereas __powerpc64__ is only active for the
106    latter (on Linux, that is).
107
108    Misc note: how to find out what's predefined in gcc by default:
109    gcc -Wp,-dM somefile.c
110 */
111 #undef PLAT_x86_darwin
112 #undef PLAT_amd64_darwin
113 #undef PLAT_x86_win32
114 #undef PLAT_amd64_win64
115 #undef PLAT_x86_linux
116 #undef PLAT_amd64_linux
117 #undef PLAT_ppc32_linux
118 #undef PLAT_ppc64be_linux
119 #undef PLAT_ppc64le_linux
120 #undef PLAT_arm_linux
121 #undef PLAT_arm64_linux
122 #undef PLAT_s390x_linux
123 #undef PLAT_mips32_linux
124 #undef PLAT_mips64_linux
125 #undef PLAT_tilegx_linux
126 #undef PLAT_x86_solaris
127 #undef PLAT_amd64_solaris
128
129
130 #if defined(__APPLE__) && defined(__i386__)
131 #  define PLAT_x86_darwin 1
132 #elif defined(__APPLE__) && defined(__x86_64__)
133 #  define PLAT_amd64_darwin 1
134 #elif (defined(__MINGW32__) && !defined(__MINGW64__)) \
135       || defined(__CYGWIN32__) \
136       || (defined(_WIN32) && defined(_M_IX86))
137 #  define PLAT_x86_win32 1
138 #elif defined(__MINGW64__) \
139       || (defined(_WIN64) && defined(_M_X64))
140 #  define PLAT_amd64_win64 1
141 #elif defined(__linux__) && defined(__i386__)
142 #  define PLAT_x86_linux 1
143 #elif defined(__linux__) && defined(__x86_64__) && !defined(__ILP32__)
144 #  define PLAT_amd64_linux 1
145 #elif defined(__linux__) && defined(__powerpc__) && !defined(__powerpc64__)
146 #  define PLAT_ppc32_linux 1
147 #elif defined(__linux__) && defined(__powerpc__) && defined(__powerpc64__) && _CALL_ELF != 2
148 /* Big Endian uses ELF version 1 */
149 #  define PLAT_ppc64be_linux 1
150 #elif defined(__linux__) && defined(__powerpc__) && defined(__powerpc64__) && _CALL_ELF == 2
151 /* Little Endian uses ELF version 2 */
152 #  define PLAT_ppc64le_linux 1
153 #elif defined(__linux__) && defined(__arm__) && !defined(__aarch64__)
154 #  define PLAT_arm_linux 1
155 #elif defined(__linux__) && defined(__aarch64__) && !defined(__arm__)
156 #  define PLAT_arm64_linux 1
157 #elif defined(__linux__) && defined(__s390__) && defined(__s390x__)
158 #  define PLAT_s390x_linux 1
159 #elif defined(__linux__) && defined(__mips__) && (__mips==64)
160 #  define PLAT_mips64_linux 1
161 #elif defined(__linux__) && defined(__mips__) && (__mips!=64)
162 #  define PLAT_mips32_linux 1
163 #elif defined(__linux__) && defined(__tilegx__)
164 #  define PLAT_tilegx_linux 1
165 #elif defined(__sun) && defined(__i386__)
166 #  define PLAT_x86_solaris 1
167 #elif defined(__sun) && defined(__x86_64__)
168 #  define PLAT_amd64_solaris 1
169 #else
170 /* If we're not compiling for our target platform, don't generate
171    any inline asms.  */
172 #  if !defined(NVALGRIND)
173 #    define NVALGRIND 1
174 #  endif
175 #endif
176
177
178 /* ------------------------------------------------------------------ */
179 /* ARCHITECTURE SPECIFICS for SPECIAL INSTRUCTIONS.  There is nothing */
180 /* in here of use to end-users -- skip to the next section.           */
181 /* ------------------------------------------------------------------ */
182
183 /*
184  * VALGRIND_DO_CLIENT_REQUEST(): a statement that invokes a Valgrind client
185  * request. Accepts both pointers and integers as arguments.
186  *
187  * VALGRIND_DO_CLIENT_REQUEST_STMT(): a statement that invokes a Valgrind
188  * client request that does not return a value.
189
190  * VALGRIND_DO_CLIENT_REQUEST_EXPR(): a C expression that invokes a Valgrind
191  * client request and whose value equals the client request result.  Accepts
192  * both pointers and integers as arguments.  Note that such calls are not
193  * necessarily pure functions -- they may have side effects.
194  */
195
196 #define VALGRIND_DO_CLIENT_REQUEST(_zzq_rlval, _zzq_default,            \
197                                    _zzq_request, _zzq_arg1, _zzq_arg2,  \
198                                    _zzq_arg3, _zzq_arg4, _zzq_arg5)     \
199   do { (_zzq_rlval) = VALGRIND_DO_CLIENT_REQUEST_EXPR((_zzq_default),   \
200                         (_zzq_request), (_zzq_arg1), (_zzq_arg2),       \
201                         (_zzq_arg3), (_zzq_arg4), (_zzq_arg5)); } while (0)
202
203 #define VALGRIND_DO_CLIENT_REQUEST_STMT(_zzq_request, _zzq_arg1,        \
204                            _zzq_arg2,  _zzq_arg3, _zzq_arg4, _zzq_arg5) \
205   do { (void) VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                        \
206                     (_zzq_request), (_zzq_arg1), (_zzq_arg2),           \
207                     (_zzq_arg3), (_zzq_arg4), (_zzq_arg5)); } while (0)
208
209 #if defined(NVALGRIND)
210
211 /* Define NVALGRIND to completely remove the Valgrind magic sequence
212    from the compiled code (analogous to NDEBUG's effects on
213    assert()) */
214 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
215         _zzq_default, _zzq_request,                               \
216         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
217       (_zzq_default)
218
219 #else  /* ! NVALGRIND */
220
221 /* The following defines the magic code sequences which the JITter
222    spots and handles magically.  Don't look too closely at them as
223    they will rot your brain.
224
225    The assembly code sequences for all architectures is in this one
226    file.  This is because this file must be stand-alone, and we don't
227    want to have multiple files.
228
229    For VALGRIND_DO_CLIENT_REQUEST, we must ensure that the default
230    value gets put in the return slot, so that everything works when
231    this is executed not under Valgrind.  Args are passed in a memory
232    block, and so there's no intrinsic limit to the number that could
233    be passed, but it's currently five.
234    
235    The macro args are: 
236       _zzq_rlval    result lvalue
237       _zzq_default  default value (result returned when running on real CPU)
238       _zzq_request  request code
239       _zzq_arg1..5  request params
240
241    The other two macros are used to support function wrapping, and are
242    a lot simpler.  VALGRIND_GET_NR_CONTEXT returns the value of the
243    guest's NRADDR pseudo-register and whatever other information is
244    needed to safely run the call original from the wrapper: on
245    ppc64-linux, the R2 value at the divert point is also needed.  This
246    information is abstracted into a user-visible type, OrigFn.
247
248    VALGRIND_CALL_NOREDIR_* behaves the same as the following on the
249    guest, but guarantees that the branch instruction will not be
250    redirected: x86: call *%eax, amd64: call *%rax, ppc32/ppc64:
251    branch-and-link-to-r11.  VALGRIND_CALL_NOREDIR is just text, not a
252    complete inline asm, since it needs to be combined with more magic
253    inline asm stuff to be useful.
254 */
255
256 /* ----------------- x86-{linux,darwin,solaris} ---------------- */
257
258 #if defined(PLAT_x86_linux)  ||  defined(PLAT_x86_darwin)  \
259     ||  (defined(PLAT_x86_win32) && defined(__GNUC__)) \
260     ||  defined(PLAT_x86_solaris)
261
262 typedef
263    struct { 
264       unsigned int nraddr; /* where's the code? */
265    }
266    OrigFn;
267
268 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
269                      "roll $3,  %%edi ; roll $13, %%edi\n\t"      \
270                      "roll $29, %%edi ; roll $19, %%edi\n\t"
271
272 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
273         _zzq_default, _zzq_request,                               \
274         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
275   __extension__                                                   \
276   ({volatile unsigned int _zzq_args[6];                           \
277     volatile unsigned int _zzq_result;                            \
278     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
279     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
280     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
281     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
282     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
283     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
284     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
285                      /* %EDX = client_request ( %EAX ) */         \
286                      "xchgl %%ebx,%%ebx"                          \
287                      : "=d" (_zzq_result)                         \
288                      : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
289                      : "cc", "memory"                             \
290                     );                                            \
291     _zzq_result;                                                  \
292   })
293
294 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
295   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
296     volatile unsigned int __addr;                                 \
297     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
298                      /* %EAX = guest_NRADDR */                    \
299                      "xchgl %%ecx,%%ecx"                          \
300                      : "=a" (__addr)                              \
301                      :                                            \
302                      : "cc", "memory"                             \
303                     );                                            \
304     _zzq_orig->nraddr = __addr;                                   \
305   }
306
307 #define VALGRIND_CALL_NOREDIR_EAX                                 \
308                      __SPECIAL_INSTRUCTION_PREAMBLE               \
309                      /* call-noredir *%EAX */                     \
310                      "xchgl %%edx,%%edx\n\t"
311
312 #define VALGRIND_VEX_INJECT_IR()                                 \
313  do {                                                            \
314     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
315                      "xchgl %%edi,%%edi\n\t"                     \
316                      : : : "cc", "memory"                        \
317                     );                                           \
318  } while (0)
319
320 #endif /* PLAT_x86_linux || PLAT_x86_darwin || (PLAT_x86_win32 && __GNUC__)
321           || PLAT_x86_solaris */
322
323 /* ------------------------- x86-Win32 ------------------------- */
324
325 #if defined(PLAT_x86_win32) && !defined(__GNUC__)
326
327 typedef
328    struct { 
329       unsigned int nraddr; /* where's the code? */
330    }
331    OrigFn;
332
333 #if defined(_MSC_VER)
334
335 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
336                      __asm rol edi, 3  __asm rol edi, 13          \
337                      __asm rol edi, 29 __asm rol edi, 19
338
339 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
340         _zzq_default, _zzq_request,                               \
341         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
342     valgrind_do_client_request_expr((uintptr_t)(_zzq_default),    \
343         (uintptr_t)(_zzq_request), (uintptr_t)(_zzq_arg1),        \
344         (uintptr_t)(_zzq_arg2), (uintptr_t)(_zzq_arg3),           \
345         (uintptr_t)(_zzq_arg4), (uintptr_t)(_zzq_arg5))
346
347 static __inline uintptr_t
348 valgrind_do_client_request_expr(uintptr_t _zzq_default, uintptr_t _zzq_request,
349                                 uintptr_t _zzq_arg1, uintptr_t _zzq_arg2,
350                                 uintptr_t _zzq_arg3, uintptr_t _zzq_arg4,
351                                 uintptr_t _zzq_arg5)
352 {
353     volatile uintptr_t _zzq_args[6];
354     volatile unsigned int _zzq_result;
355     _zzq_args[0] = (uintptr_t)(_zzq_request);
356     _zzq_args[1] = (uintptr_t)(_zzq_arg1);
357     _zzq_args[2] = (uintptr_t)(_zzq_arg2);
358     _zzq_args[3] = (uintptr_t)(_zzq_arg3);
359     _zzq_args[4] = (uintptr_t)(_zzq_arg4);
360     _zzq_args[5] = (uintptr_t)(_zzq_arg5);
361     __asm { __asm lea eax, _zzq_args __asm mov edx, _zzq_default
362             __SPECIAL_INSTRUCTION_PREAMBLE
363             /* %EDX = client_request ( %EAX ) */
364             __asm xchg ebx,ebx
365             __asm mov _zzq_result, edx
366     }
367     return _zzq_result;
368 }
369
370 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
371   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
372     volatile unsigned int __addr;                                 \
373     __asm { __SPECIAL_INSTRUCTION_PREAMBLE                        \
374             /* %EAX = guest_NRADDR */                             \
375             __asm xchg ecx,ecx                                    \
376             __asm mov __addr, eax                                 \
377     }                                                             \
378     _zzq_orig->nraddr = __addr;                                   \
379   }
380
381 #define VALGRIND_CALL_NOREDIR_EAX ERROR
382
383 #define VALGRIND_VEX_INJECT_IR()                                 \
384  do {                                                            \
385     __asm { __SPECIAL_INSTRUCTION_PREAMBLE                       \
386             __asm xchg edi,edi                                   \
387     }                                                            \
388  } while (0)
389
390 #else
391 #error Unsupported compiler.
392 #endif
393
394 #endif /* PLAT_x86_win32 */
395
396 /* ----------------- amd64-{linux,darwin,solaris} --------------- */
397
398 #if defined(PLAT_amd64_linux)  ||  defined(PLAT_amd64_darwin) \
399     ||  defined(PLAT_amd64_solaris) \
400     ||  (defined(PLAT_amd64_win64) && defined(__GNUC__))
401
402 typedef
403    struct { 
404       unsigned long int nraddr; /* where's the code? */
405    }
406    OrigFn;
407
408 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
409                      "rolq $3,  %%rdi ; rolq $13, %%rdi\n\t"      \
410                      "rolq $61, %%rdi ; rolq $51, %%rdi\n\t"
411
412 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
413         _zzq_default, _zzq_request,                               \
414         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
415     __extension__                                                 \
416     ({ volatile unsigned long int _zzq_args[6];                   \
417     volatile unsigned long int _zzq_result;                       \
418     _zzq_args[0] = (unsigned long int)(_zzq_request);             \
419     _zzq_args[1] = (unsigned long int)(_zzq_arg1);                \
420     _zzq_args[2] = (unsigned long int)(_zzq_arg2);                \
421     _zzq_args[3] = (unsigned long int)(_zzq_arg3);                \
422     _zzq_args[4] = (unsigned long int)(_zzq_arg4);                \
423     _zzq_args[5] = (unsigned long int)(_zzq_arg5);                \
424     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
425                      /* %RDX = client_request ( %RAX ) */         \
426                      "xchgq %%rbx,%%rbx"                          \
427                      : "=d" (_zzq_result)                         \
428                      : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
429                      : "cc", "memory"                             \
430                     );                                            \
431     _zzq_result;                                                  \
432     })
433
434 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
435   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
436     volatile unsigned long int __addr;                            \
437     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
438                      /* %RAX = guest_NRADDR */                    \
439                      "xchgq %%rcx,%%rcx"                          \
440                      : "=a" (__addr)                              \
441                      :                                            \
442                      : "cc", "memory"                             \
443                     );                                            \
444     _zzq_orig->nraddr = __addr;                                   \
445   }
446
447 #define VALGRIND_CALL_NOREDIR_RAX                                 \
448                      __SPECIAL_INSTRUCTION_PREAMBLE               \
449                      /* call-noredir *%RAX */                     \
450                      "xchgq %%rdx,%%rdx\n\t"
451
452 #define VALGRIND_VEX_INJECT_IR()                                 \
453  do {                                                            \
454     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
455                      "xchgq %%rdi,%%rdi\n\t"                     \
456                      : : : "cc", "memory"                        \
457                     );                                           \
458  } while (0)
459
460 #endif /* PLAT_amd64_linux || PLAT_amd64_darwin || PLAT_amd64_solaris */
461
462 /* ------------------------- amd64-Win64 ------------------------- */
463
464 #if defined(PLAT_amd64_win64) && !defined(__GNUC__)
465
466 #error Unsupported compiler.
467
468 #endif /* PLAT_amd64_win64 */
469
470 /* ------------------------ ppc32-linux ------------------------ */
471
472 #if defined(PLAT_ppc32_linux)
473
474 typedef
475    struct { 
476       unsigned int nraddr; /* where's the code? */
477    }
478    OrigFn;
479
480 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
481                     "rlwinm 0,0,3,0,31  ; rlwinm 0,0,13,0,31\n\t" \
482                     "rlwinm 0,0,29,0,31 ; rlwinm 0,0,19,0,31\n\t"
483
484 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
485         _zzq_default, _zzq_request,                               \
486         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
487                                                                   \
488     __extension__                                                 \
489   ({         unsigned int  _zzq_args[6];                          \
490              unsigned int  _zzq_result;                           \
491              unsigned int* _zzq_ptr;                              \
492     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
493     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
494     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
495     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
496     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
497     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
498     _zzq_ptr = _zzq_args;                                         \
499     __asm__ volatile("mr 3,%1\n\t" /*default*/                    \
500                      "mr 4,%2\n\t" /*ptr*/                        \
501                      __SPECIAL_INSTRUCTION_PREAMBLE               \
502                      /* %R3 = client_request ( %R4 ) */           \
503                      "or 1,1,1\n\t"                               \
504                      "mr %0,3"     /*result*/                     \
505                      : "=b" (_zzq_result)                         \
506                      : "b" (_zzq_default), "b" (_zzq_ptr)         \
507                      : "cc", "memory", "r3", "r4");               \
508     _zzq_result;                                                  \
509     })
510
511 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
512   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
513     unsigned int __addr;                                          \
514     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
515                      /* %R3 = guest_NRADDR */                     \
516                      "or 2,2,2\n\t"                               \
517                      "mr %0,3"                                    \
518                      : "=b" (__addr)                              \
519                      :                                            \
520                      : "cc", "memory", "r3"                       \
521                     );                                            \
522     _zzq_orig->nraddr = __addr;                                   \
523   }
524
525 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
526                      __SPECIAL_INSTRUCTION_PREAMBLE               \
527                      /* branch-and-link-to-noredir *%R11 */       \
528                      "or 3,3,3\n\t"
529
530 #define VALGRIND_VEX_INJECT_IR()                                 \
531  do {                                                            \
532     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
533                      "or 5,5,5\n\t"                              \
534                     );                                           \
535  } while (0)
536
537 #endif /* PLAT_ppc32_linux */
538
539 /* ------------------------ ppc64-linux ------------------------ */
540
541 #if defined(PLAT_ppc64be_linux)
542
543 typedef
544    struct { 
545       unsigned long int nraddr; /* where's the code? */
546       unsigned long int r2;  /* what tocptr do we need? */
547    }
548    OrigFn;
549
550 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
551                      "rotldi 0,0,3  ; rotldi 0,0,13\n\t"          \
552                      "rotldi 0,0,61 ; rotldi 0,0,51\n\t"
553
554 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
555         _zzq_default, _zzq_request,                               \
556         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
557                                                                   \
558   __extension__                                                   \
559   ({         unsigned long int  _zzq_args[6];                     \
560              unsigned long int  _zzq_result;                      \
561              unsigned long int* _zzq_ptr;                         \
562     _zzq_args[0] = (unsigned long int)(_zzq_request);             \
563     _zzq_args[1] = (unsigned long int)(_zzq_arg1);                \
564     _zzq_args[2] = (unsigned long int)(_zzq_arg2);                \
565     _zzq_args[3] = (unsigned long int)(_zzq_arg3);                \
566     _zzq_args[4] = (unsigned long int)(_zzq_arg4);                \
567     _zzq_args[5] = (unsigned long int)(_zzq_arg5);                \
568     _zzq_ptr = _zzq_args;                                         \
569     __asm__ volatile("mr 3,%1\n\t" /*default*/                    \
570                      "mr 4,%2\n\t" /*ptr*/                        \
571                      __SPECIAL_INSTRUCTION_PREAMBLE               \
572                      /* %R3 = client_request ( %R4 ) */           \
573                      "or 1,1,1\n\t"                               \
574                      "mr %0,3"     /*result*/                     \
575                      : "=b" (_zzq_result)                         \
576                      : "b" (_zzq_default), "b" (_zzq_ptr)         \
577                      : "cc", "memory", "r3", "r4");               \
578     _zzq_result;                                                  \
579   })
580
581 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
582   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
583     unsigned long int __addr;                                     \
584     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
585                      /* %R3 = guest_NRADDR */                     \
586                      "or 2,2,2\n\t"                               \
587                      "mr %0,3"                                    \
588                      : "=b" (__addr)                              \
589                      :                                            \
590                      : "cc", "memory", "r3"                       \
591                     );                                            \
592     _zzq_orig->nraddr = __addr;                                   \
593     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
594                      /* %R3 = guest_NRADDR_GPR2 */                \
595                      "or 4,4,4\n\t"                               \
596                      "mr %0,3"                                    \
597                      : "=b" (__addr)                              \
598                      :                                            \
599                      : "cc", "memory", "r3"                       \
600                     );                                            \
601     _zzq_orig->r2 = __addr;                                       \
602   }
603
604 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
605                      __SPECIAL_INSTRUCTION_PREAMBLE               \
606                      /* branch-and-link-to-noredir *%R11 */       \
607                      "or 3,3,3\n\t"
608
609 #define VALGRIND_VEX_INJECT_IR()                                 \
610  do {                                                            \
611     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
612                      "or 5,5,5\n\t"                              \
613                     );                                           \
614  } while (0)
615
616 #endif /* PLAT_ppc64be_linux */
617
618 #if defined(PLAT_ppc64le_linux)
619
620 typedef
621    struct {
622       unsigned long int nraddr; /* where's the code? */
623       unsigned long int r2;     /* what tocptr do we need? */
624    }
625    OrigFn;
626
627 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
628                      "rotldi 0,0,3  ; rotldi 0,0,13\n\t"          \
629                      "rotldi 0,0,61 ; rotldi 0,0,51\n\t"
630
631 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
632         _zzq_default, _zzq_request,                               \
633         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
634                                                                   \
635   __extension__                                                   \
636   ({         unsigned long int  _zzq_args[6];                     \
637              unsigned long int  _zzq_result;                      \
638              unsigned long int* _zzq_ptr;                         \
639     _zzq_args[0] = (unsigned long int)(_zzq_request);             \
640     _zzq_args[1] = (unsigned long int)(_zzq_arg1);                \
641     _zzq_args[2] = (unsigned long int)(_zzq_arg2);                \
642     _zzq_args[3] = (unsigned long int)(_zzq_arg3);                \
643     _zzq_args[4] = (unsigned long int)(_zzq_arg4);                \
644     _zzq_args[5] = (unsigned long int)(_zzq_arg5);                \
645     _zzq_ptr = _zzq_args;                                         \
646     __asm__ volatile("mr 3,%1\n\t" /*default*/                    \
647                      "mr 4,%2\n\t" /*ptr*/                        \
648                      __SPECIAL_INSTRUCTION_PREAMBLE               \
649                      /* %R3 = client_request ( %R4 ) */           \
650                      "or 1,1,1\n\t"                               \
651                      "mr %0,3"     /*result*/                     \
652                      : "=b" (_zzq_result)                         \
653                      : "b" (_zzq_default), "b" (_zzq_ptr)         \
654                      : "cc", "memory", "r3", "r4");               \
655     _zzq_result;                                                  \
656   })
657
658 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
659   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
660     unsigned long int __addr;                                     \
661     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
662                      /* %R3 = guest_NRADDR */                     \
663                      "or 2,2,2\n\t"                               \
664                      "mr %0,3"                                    \
665                      : "=b" (__addr)                              \
666                      :                                            \
667                      : "cc", "memory", "r3"                       \
668                     );                                            \
669     _zzq_orig->nraddr = __addr;                                   \
670     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
671                      /* %R3 = guest_NRADDR_GPR2 */                \
672                      "or 4,4,4\n\t"                               \
673                      "mr %0,3"                                    \
674                      : "=b" (__addr)                              \
675                      :                                            \
676                      : "cc", "memory", "r3"                       \
677                     );                                            \
678     _zzq_orig->r2 = __addr;                                       \
679   }
680
681 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                   \
682                      __SPECIAL_INSTRUCTION_PREAMBLE               \
683                      /* branch-and-link-to-noredir *%R12 */       \
684                      "or 3,3,3\n\t"
685
686 #define VALGRIND_VEX_INJECT_IR()                                 \
687  do {                                                            \
688     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
689                      "or 5,5,5\n\t"                              \
690                     );                                           \
691  } while (0)
692
693 #endif /* PLAT_ppc64le_linux */
694
695 /* ------------------------- arm-linux ------------------------- */
696
697 #if defined(PLAT_arm_linux)
698
699 typedef
700    struct { 
701       unsigned int nraddr; /* where's the code? */
702    }
703    OrigFn;
704
705 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
706             "mov r12, r12, ror #3  ; mov r12, r12, ror #13 \n\t"  \
707             "mov r12, r12, ror #29 ; mov r12, r12, ror #19 \n\t"
708
709 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
710         _zzq_default, _zzq_request,                               \
711         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
712                                                                   \
713   __extension__                                                   \
714   ({volatile unsigned int  _zzq_args[6];                          \
715     volatile unsigned int  _zzq_result;                           \
716     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
717     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
718     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
719     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
720     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
721     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
722     __asm__ volatile("mov r3, %1\n\t" /*default*/                 \
723                      "mov r4, %2\n\t" /*ptr*/                     \
724                      __SPECIAL_INSTRUCTION_PREAMBLE               \
725                      /* R3 = client_request ( R4 ) */             \
726                      "orr r10, r10, r10\n\t"                      \
727                      "mov %0, r3"     /*result*/                  \
728                      : "=r" (_zzq_result)                         \
729                      : "r" (_zzq_default), "r" (&_zzq_args[0])    \
730                      : "cc","memory", "r3", "r4");                \
731     _zzq_result;                                                  \
732   })
733
734 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
735   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
736     unsigned int __addr;                                          \
737     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
738                      /* R3 = guest_NRADDR */                      \
739                      "orr r11, r11, r11\n\t"                      \
740                      "mov %0, r3"                                 \
741                      : "=r" (__addr)                              \
742                      :                                            \
743                      : "cc", "memory", "r3"                       \
744                     );                                            \
745     _zzq_orig->nraddr = __addr;                                   \
746   }
747
748 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                    \
749                      __SPECIAL_INSTRUCTION_PREAMBLE               \
750                      /* branch-and-link-to-noredir *%R4 */        \
751                      "orr r12, r12, r12\n\t"
752
753 #define VALGRIND_VEX_INJECT_IR()                                 \
754  do {                                                            \
755     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
756                      "orr r9, r9, r9\n\t"                        \
757                      : : : "cc", "memory"                        \
758                     );                                           \
759  } while (0)
760
761 #endif /* PLAT_arm_linux */
762
763 /* ------------------------ arm64-linux ------------------------- */
764
765 #if defined(PLAT_arm64_linux)
766
767 typedef
768    struct { 
769       unsigned long int nraddr; /* where's the code? */
770    }
771    OrigFn;
772
773 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
774             "ror x12, x12, #3  ;  ror x12, x12, #13 \n\t"         \
775             "ror x12, x12, #51 ;  ror x12, x12, #61 \n\t"
776
777 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
778         _zzq_default, _zzq_request,                               \
779         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
780                                                                   \
781   __extension__                                                   \
782   ({volatile unsigned long int  _zzq_args[6];                     \
783     volatile unsigned long int  _zzq_result;                      \
784     _zzq_args[0] = (unsigned long int)(_zzq_request);             \
785     _zzq_args[1] = (unsigned long int)(_zzq_arg1);                \
786     _zzq_args[2] = (unsigned long int)(_zzq_arg2);                \
787     _zzq_args[3] = (unsigned long int)(_zzq_arg3);                \
788     _zzq_args[4] = (unsigned long int)(_zzq_arg4);                \
789     _zzq_args[5] = (unsigned long int)(_zzq_arg5);                \
790     __asm__ volatile("mov x3, %1\n\t" /*default*/                 \
791                      "mov x4, %2\n\t" /*ptr*/                     \
792                      __SPECIAL_INSTRUCTION_PREAMBLE               \
793                      /* X3 = client_request ( X4 ) */             \
794                      "orr x10, x10, x10\n\t"                      \
795                      "mov %0, x3"     /*result*/                  \
796                      : "=r" (_zzq_result)                         \
797                      : "r" ((unsigned long int)(_zzq_default)),   \
798                        "r" (&_zzq_args[0])                        \
799                      : "cc","memory", "x3", "x4");                \
800     _zzq_result;                                                  \
801   })
802
803 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
804   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
805     unsigned long int __addr;                                     \
806     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
807                      /* X3 = guest_NRADDR */                      \
808                      "orr x11, x11, x11\n\t"                      \
809                      "mov %0, x3"                                 \
810                      : "=r" (__addr)                              \
811                      :                                            \
812                      : "cc", "memory", "x3"                       \
813                     );                                            \
814     _zzq_orig->nraddr = __addr;                                   \
815   }
816
817 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                    \
818                      __SPECIAL_INSTRUCTION_PREAMBLE               \
819                      /* branch-and-link-to-noredir X8 */          \
820                      "orr x12, x12, x12\n\t"
821
822 #define VALGRIND_VEX_INJECT_IR()                                 \
823  do {                                                            \
824     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
825                      "orr x9, x9, x9\n\t"                        \
826                      : : : "cc", "memory"                        \
827                     );                                           \
828  } while (0)
829
830 #endif /* PLAT_arm64_linux */
831
832 /* ------------------------ s390x-linux ------------------------ */
833
834 #if defined(PLAT_s390x_linux)
835
836 typedef
837   struct {
838      unsigned long int nraddr; /* where's the code? */
839   }
840   OrigFn;
841
842 /* __SPECIAL_INSTRUCTION_PREAMBLE will be used to identify Valgrind specific
843  * code. This detection is implemented in platform specific toIR.c
844  * (e.g. VEX/priv/guest_s390_decoder.c).
845  */
846 #define __SPECIAL_INSTRUCTION_PREAMBLE                           \
847                      "lr 15,15\n\t"                              \
848                      "lr 1,1\n\t"                                \
849                      "lr 2,2\n\t"                                \
850                      "lr 3,3\n\t"
851
852 #define __CLIENT_REQUEST_CODE "lr 2,2\n\t"
853 #define __GET_NR_CONTEXT_CODE "lr 3,3\n\t"
854 #define __CALL_NO_REDIR_CODE  "lr 4,4\n\t"
855 #define __VEX_INJECT_IR_CODE  "lr 5,5\n\t"
856
857 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                         \
858        _zzq_default, _zzq_request,                               \
859        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
860   __extension__                                                  \
861  ({volatile unsigned long int _zzq_args[6];                      \
862    volatile unsigned long int _zzq_result;                       \
863    _zzq_args[0] = (unsigned long int)(_zzq_request);             \
864    _zzq_args[1] = (unsigned long int)(_zzq_arg1);                \
865    _zzq_args[2] = (unsigned long int)(_zzq_arg2);                \
866    _zzq_args[3] = (unsigned long int)(_zzq_arg3);                \
867    _zzq_args[4] = (unsigned long int)(_zzq_arg4);                \
868    _zzq_args[5] = (unsigned long int)(_zzq_arg5);                \
869    __asm__ volatile(/* r2 = args */                              \
870                     "lgr 2,%1\n\t"                               \
871                     /* r3 = default */                           \
872                     "lgr 3,%2\n\t"                               \
873                     __SPECIAL_INSTRUCTION_PREAMBLE               \
874                     __CLIENT_REQUEST_CODE                        \
875                     /* results = r3 */                           \
876                     "lgr %0, 3\n\t"                              \
877                     : "=d" (_zzq_result)                         \
878                     : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
879                     : "cc", "2", "3", "memory"                   \
880                    );                                            \
881    _zzq_result;                                                  \
882  })
883
884 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                      \
885  { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
886    volatile unsigned long int __addr;                            \
887    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
888                     __GET_NR_CONTEXT_CODE                        \
889                     "lgr %0, 3\n\t"                              \
890                     : "=a" (__addr)                              \
891                     :                                            \
892                     : "cc", "3", "memory"                        \
893                    );                                            \
894    _zzq_orig->nraddr = __addr;                                   \
895  }
896
897 #define VALGRIND_CALL_NOREDIR_R1                                 \
898                     __SPECIAL_INSTRUCTION_PREAMBLE               \
899                     __CALL_NO_REDIR_CODE
900
901 #define VALGRIND_VEX_INJECT_IR()                                 \
902  do {                                                            \
903     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
904                      __VEX_INJECT_IR_CODE);                      \
905  } while (0)
906
907 #endif /* PLAT_s390x_linux */
908
909 /* ------------------------- mips32-linux ---------------- */
910
911 #if defined(PLAT_mips32_linux)
912
913 typedef
914    struct { 
915       unsigned int nraddr; /* where's the code? */
916    }
917    OrigFn;
918
919 /* .word  0x342
920  * .word  0x742
921  * .word  0xC2
922  * .word  0x4C2*/
923 #define __SPECIAL_INSTRUCTION_PREAMBLE          \
924                      "srl $0, $0, 13\n\t"       \
925                      "srl $0, $0, 29\n\t"       \
926                      "srl $0, $0, 3\n\t"        \
927                      "srl $0, $0, 19\n\t"
928                     
929 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
930        _zzq_default, _zzq_request,                                \
931        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)     \
932   __extension__                                                   \
933   ({ volatile unsigned int _zzq_args[6];                          \
934     volatile unsigned int _zzq_result;                            \
935     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
936     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
937     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
938     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
939     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
940     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
941         __asm__ volatile("move $11, %1\n\t" /*default*/           \
942                      "move $12, %2\n\t" /*ptr*/                   \
943                      __SPECIAL_INSTRUCTION_PREAMBLE               \
944                      /* T3 = client_request ( T4 ) */             \
945                      "or $13, $13, $13\n\t"                       \
946                      "move %0, $11\n\t"     /*result*/            \
947                      : "=r" (_zzq_result)                         \
948                      : "r" (_zzq_default), "r" (&_zzq_args[0])    \
949                      : "$11", "$12");                             \
950     _zzq_result;                                                  \
951   })
952
953 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
954   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
955     volatile unsigned int __addr;                                 \
956     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
957                      /* %t9 = guest_NRADDR */                     \
958                      "or $14, $14, $14\n\t"                       \
959                      "move %0, $11"     /*result*/                \
960                      : "=r" (__addr)                              \
961                      :                                            \
962                      : "$11"                                      \
963                     );                                            \
964     _zzq_orig->nraddr = __addr;                                   \
965   }
966
967 #define VALGRIND_CALL_NOREDIR_T9                                 \
968                      __SPECIAL_INSTRUCTION_PREAMBLE              \
969                      /* call-noredir *%t9 */                     \
970                      "or $15, $15, $15\n\t"
971
972 #define VALGRIND_VEX_INJECT_IR()                                 \
973  do {                                                            \
974     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
975                      "or $11, $11, $11\n\t"                      \
976                     );                                           \
977  } while (0)
978
979
980 #endif /* PLAT_mips32_linux */
981
982 /* ------------------------- mips64-linux ---------------- */
983
984 #if defined(PLAT_mips64_linux)
985
986 typedef
987    struct {
988       unsigned long nraddr; /* where's the code? */
989    }
990    OrigFn;
991
992 /* dsll $0,$0, 3
993  * dsll $0,$0, 13
994  * dsll $0,$0, 29
995  * dsll $0,$0, 19*/
996 #define __SPECIAL_INSTRUCTION_PREAMBLE                              \
997                      "dsll $0,$0, 3 ; dsll $0,$0,13\n\t"            \
998                      "dsll $0,$0,29 ; dsll $0,$0,19\n\t"
999
1000 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                            \
1001        _zzq_default, _zzq_request,                                  \
1002        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)       \
1003   __extension__                                                     \
1004   ({ volatile unsigned long int _zzq_args[6];                       \
1005     volatile unsigned long int _zzq_result;                         \
1006     _zzq_args[0] = (unsigned long int)(_zzq_request);               \
1007     _zzq_args[1] = (unsigned long int)(_zzq_arg1);                  \
1008     _zzq_args[2] = (unsigned long int)(_zzq_arg2);                  \
1009     _zzq_args[3] = (unsigned long int)(_zzq_arg3);                  \
1010     _zzq_args[4] = (unsigned long int)(_zzq_arg4);                  \
1011     _zzq_args[5] = (unsigned long int)(_zzq_arg5);                  \
1012         __asm__ volatile("move $11, %1\n\t" /*default*/             \
1013                          "move $12, %2\n\t" /*ptr*/                 \
1014                          __SPECIAL_INSTRUCTION_PREAMBLE             \
1015                          /* $11 = client_request ( $12 ) */         \
1016                          "or $13, $13, $13\n\t"                     \
1017                          "move %0, $11\n\t"     /*result*/          \
1018                          : "=r" (_zzq_result)                       \
1019                          : "r" (_zzq_default), "r" (&_zzq_args[0])  \
1020                          : "$11", "$12");                           \
1021     _zzq_result;                                                    \
1022   })
1023
1024 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                         \
1025   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                     \
1026     volatile unsigned long int __addr;                              \
1027     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE                 \
1028                      /* $11 = guest_NRADDR */                       \
1029                      "or $14, $14, $14\n\t"                         \
1030                      "move %0, $11"     /*result*/                  \
1031                      : "=r" (__addr)                                \
1032                      :                                              \
1033                      : "$11");                                      \
1034     _zzq_orig->nraddr = __addr;                                     \
1035   }
1036
1037 #define VALGRIND_CALL_NOREDIR_T9                                    \
1038                      __SPECIAL_INSTRUCTION_PREAMBLE                 \
1039                      /* call-noredir $25 */                         \
1040                      "or $15, $15, $15\n\t"
1041
1042 #define VALGRIND_VEX_INJECT_IR()                                    \
1043  do {                                                               \
1044     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE                 \
1045                      "or $11, $11, $11\n\t"                         \
1046                     );                                              \
1047  } while (0)
1048
1049 #endif /* PLAT_mips64_linux */
1050
1051 /* ------------------------ tilegx-linux --------------- */
1052 #if defined(PLAT_tilegx_linux)
1053
1054 typedef
1055    struct {
1056       unsigned long long int nraddr; /* where's the code? */
1057    }
1058    OrigFn;
1059 /*** special instruction sequence.
1060      0:02b3c7ff91234fff { moveli zero, 4660 ; moveli zero, 22136 }
1061      8:0091a7ff95678fff { moveli zero, 22136 ; moveli zero, 4660 }
1062 ****/
1063
1064 #define __SPECIAL_INSTRUCTION_PREAMBLE                             \
1065    ".quad  0x02b3c7ff91234fff\n"                                   \
1066    ".quad  0x0091a7ff95678fff\n"
1067
1068 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                           \
1069    _zzq_default, _zzq_request,                                      \
1070    _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)          \
1071    ({ volatile unsigned long long int _zzq_args[6];                \
1072       volatile unsigned long long int _zzq_result;                 \
1073       _zzq_args[0] = (unsigned long long int)(_zzq_request);       \
1074       _zzq_args[1] = (unsigned long long int)(_zzq_arg1);          \
1075       _zzq_args[2] = (unsigned long long int)(_zzq_arg2);          \
1076       _zzq_args[3] = (unsigned long long int)(_zzq_arg3);          \
1077       _zzq_args[4] = (unsigned long long int)(_zzq_arg4);          \
1078       _zzq_args[5] = (unsigned long long int)(_zzq_arg5);          \
1079       __asm__ volatile("move r11, %1\n\t" /*default*/              \
1080                        "move r12, %2\n\t" /*ptr*/                  \
1081                        __SPECIAL_INSTRUCTION_PREAMBLE              \
1082                        /* r11 = client_request */                  \
1083                        "or r13, r13, r13\n\t"                      \
1084                        "move %0, r11\n\t"     /*result*/           \
1085                        : "=r" (_zzq_result)                        \
1086                        : "r" (_zzq_default), "r" (&_zzq_args[0])   \
1087                        : "memory", "r11", "r12");                  \
1088       _zzq_result;                                                 \
1089    })
1090
1091 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                        \
1092    {  volatile OrigFn* _zzq_orig = &(_zzq_rlval);                  \
1093       volatile unsigned long long int __addr;                      \
1094       __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
1095                        /* r11 = guest_NRADDR */                    \
1096                        "or r14, r14, r14\n"                        \
1097                        "move %0, r11\n"                            \
1098                        : "=r" (__addr)                             \
1099                        :                                           \
1100                        : "memory", "r11"                           \
1101                        );                                          \
1102       _zzq_orig->nraddr = __addr;                                  \
1103    }
1104
1105 #define VALGRIND_CALL_NOREDIR_R12                                  \
1106    __SPECIAL_INSTRUCTION_PREAMBLE                                  \
1107    "or r15, r15, r15\n\t"
1108
1109 #define VALGRIND_VEX_INJECT_IR()                                   \
1110    do {                                                            \
1111       __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
1112                        "or r11, r11, r11\n\t"                      \
1113                        );                                          \
1114    } while (0)
1115
1116 #endif /* PLAT_tilegx_linux */
1117
1118 /* Insert assembly code for other platforms here... */
1119
1120 #endif /* NVALGRIND */
1121
1122
1123 /* ------------------------------------------------------------------ */
1124 /* PLATFORM SPECIFICS for FUNCTION WRAPPING.  This is all very        */
1125 /* ugly.  It's the least-worst tradeoff I can think of.               */
1126 /* ------------------------------------------------------------------ */
1127
1128 /* This section defines magic (a.k.a appalling-hack) macros for doing
1129    guaranteed-no-redirection macros, so as to get from function
1130    wrappers to the functions they are wrapping.  The whole point is to
1131    construct standard call sequences, but to do the call itself with a
1132    special no-redirect call pseudo-instruction that the JIT
1133    understands and handles specially.  This section is long and
1134    repetitious, and I can't see a way to make it shorter.
1135
1136    The naming scheme is as follows:
1137
1138       CALL_FN_{W,v}_{v,W,WW,WWW,WWWW,5W,6W,7W,etc}
1139
1140    'W' stands for "word" and 'v' for "void".  Hence there are
1141    different macros for calling arity 0, 1, 2, 3, 4, etc, functions,
1142    and for each, the possibility of returning a word-typed result, or
1143    no result.
1144 */
1145
1146 /* Use these to write the name of your wrapper.  NOTE: duplicates
1147    VG_WRAP_FUNCTION_Z{U,Z} in pub_tool_redir.h.  NOTE also: inserts
1148    the default behaviour equivalance class tag "0000" into the name.
1149    See pub_tool_redir.h for details -- normally you don't need to
1150    think about this, though. */
1151
1152 /* Use an extra level of macroisation so as to ensure the soname/fnname
1153    args are fully macro-expanded before pasting them together. */
1154 #define VG_CONCAT4(_aa,_bb,_cc,_dd) _aa##_bb##_cc##_dd
1155
1156 #define I_WRAP_SONAME_FNNAME_ZU(soname,fnname)                    \
1157    VG_CONCAT4(_vgw00000ZU_,soname,_,fnname)
1158
1159 #define I_WRAP_SONAME_FNNAME_ZZ(soname,fnname)                    \
1160    VG_CONCAT4(_vgw00000ZZ_,soname,_,fnname)
1161
1162 /* Use this macro from within a wrapper function to collect the
1163    context (address and possibly other info) of the original function.
1164    Once you have that you can then use it in one of the CALL_FN_
1165    macros.  The type of the argument _lval is OrigFn. */
1166 #define VALGRIND_GET_ORIG_FN(_lval)  VALGRIND_GET_NR_CONTEXT(_lval)
1167
1168 /* Also provide end-user facilities for function replacement, rather
1169    than wrapping.  A replacement function differs from a wrapper in
1170    that it has no way to get hold of the original function being
1171    called, and hence no way to call onwards to it.  In a replacement
1172    function, VALGRIND_GET_ORIG_FN always returns zero. */
1173
1174 #define I_REPLACE_SONAME_FNNAME_ZU(soname,fnname)                 \
1175    VG_CONCAT4(_vgr00000ZU_,soname,_,fnname)
1176
1177 #define I_REPLACE_SONAME_FNNAME_ZZ(soname,fnname)                 \
1178    VG_CONCAT4(_vgr00000ZZ_,soname,_,fnname)
1179
1180 /* Derivatives of the main macros below, for calling functions
1181    returning void. */
1182
1183 #define CALL_FN_v_v(fnptr)                                        \
1184    do { volatile unsigned long _junk;                             \
1185         CALL_FN_W_v(_junk,fnptr); } while (0)
1186
1187 #define CALL_FN_v_W(fnptr, arg1)                                  \
1188    do { volatile unsigned long _junk;                             \
1189         CALL_FN_W_W(_junk,fnptr,arg1); } while (0)
1190
1191 #define CALL_FN_v_WW(fnptr, arg1,arg2)                            \
1192    do { volatile unsigned long _junk;                             \
1193         CALL_FN_W_WW(_junk,fnptr,arg1,arg2); } while (0)
1194
1195 #define CALL_FN_v_WWW(fnptr, arg1,arg2,arg3)                      \
1196    do { volatile unsigned long _junk;                             \
1197         CALL_FN_W_WWW(_junk,fnptr,arg1,arg2,arg3); } while (0)
1198
1199 #define CALL_FN_v_WWWW(fnptr, arg1,arg2,arg3,arg4)                \
1200    do { volatile unsigned long _junk;                             \
1201         CALL_FN_W_WWWW(_junk,fnptr,arg1,arg2,arg3,arg4); } while (0)
1202
1203 #define CALL_FN_v_5W(fnptr, arg1,arg2,arg3,arg4,arg5)             \
1204    do { volatile unsigned long _junk;                             \
1205         CALL_FN_W_5W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5); } while (0)
1206
1207 #define CALL_FN_v_6W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6)        \
1208    do { volatile unsigned long _junk;                             \
1209         CALL_FN_W_6W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6); } while (0)
1210
1211 #define CALL_FN_v_7W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6,arg7)   \
1212    do { volatile unsigned long _junk;                             \
1213         CALL_FN_W_7W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6,arg7); } while (0)
1214
1215 /* ----------------- x86-{linux,darwin,solaris} ---------------- */
1216
1217 #if defined(PLAT_x86_linux)  ||  defined(PLAT_x86_darwin) \
1218     ||  defined(PLAT_x86_solaris)
1219
1220 /* These regs are trashed by the hidden call.  No need to mention eax
1221    as gcc can already see that, plus causes gcc to bomb. */
1222 #define __CALLER_SAVED_REGS /*"eax"*/ "ecx", "edx"
1223
1224 /* Macros to save and align the stack before making a function
1225    call and restore it afterwards as gcc may not keep the stack
1226    pointer aligned if it doesn't realise calls are being made
1227    to other functions. */
1228
1229 #define VALGRIND_ALIGN_STACK               \
1230       "movl %%esp,%%edi\n\t"               \
1231       "andl $0xfffffff0,%%esp\n\t"
1232 #define VALGRIND_RESTORE_STACK             \
1233       "movl %%edi,%%esp\n\t"
1234
1235 /* These CALL_FN_ macros assume that on x86-linux, sizeof(unsigned
1236    long) == 4. */
1237
1238 #define CALL_FN_W_v(lval, orig)                                   \
1239    do {                                                           \
1240       volatile OrigFn        _orig = (orig);                      \
1241       volatile unsigned long _argvec[1];                          \
1242       volatile unsigned long _res;                                \
1243       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1244       __asm__ volatile(                                           \
1245          VALGRIND_ALIGN_STACK                                     \
1246          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1247          VALGRIND_CALL_NOREDIR_EAX                                \
1248          VALGRIND_RESTORE_STACK                                   \
1249          : /*out*/   "=a" (_res)                                  \
1250          : /*in*/    "a" (&_argvec[0])                            \
1251          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1252       );                                                          \
1253       lval = (__typeof__(lval)) _res;                             \
1254    } while (0)
1255
1256 #define CALL_FN_W_W(lval, orig, arg1)                             \
1257    do {                                                           \
1258       volatile OrigFn        _orig = (orig);                      \
1259       volatile unsigned long _argvec[2];                          \
1260       volatile unsigned long _res;                                \
1261       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1262       _argvec[1] = (unsigned long)(arg1);                         \
1263       __asm__ volatile(                                           \
1264          VALGRIND_ALIGN_STACK                                     \
1265          "subl $12, %%esp\n\t"                                    \
1266          "pushl 4(%%eax)\n\t"                                     \
1267          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1268          VALGRIND_CALL_NOREDIR_EAX                                \
1269          VALGRIND_RESTORE_STACK                                   \
1270          : /*out*/   "=a" (_res)                                  \
1271          : /*in*/    "a" (&_argvec[0])                            \
1272          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1273       );                                                          \
1274       lval = (__typeof__(lval)) _res;                             \
1275    } while (0)
1276
1277 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
1278    do {                                                           \
1279       volatile OrigFn        _orig = (orig);                      \
1280       volatile unsigned long _argvec[3];                          \
1281       volatile unsigned long _res;                                \
1282       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1283       _argvec[1] = (unsigned long)(arg1);                         \
1284       _argvec[2] = (unsigned long)(arg2);                         \
1285       __asm__ volatile(                                           \
1286          VALGRIND_ALIGN_STACK                                     \
1287          "subl $8, %%esp\n\t"                                     \
1288          "pushl 8(%%eax)\n\t"                                     \
1289          "pushl 4(%%eax)\n\t"                                     \
1290          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1291          VALGRIND_CALL_NOREDIR_EAX                                \
1292          VALGRIND_RESTORE_STACK                                   \
1293          : /*out*/   "=a" (_res)                                  \
1294          : /*in*/    "a" (&_argvec[0])                            \
1295          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1296       );                                                          \
1297       lval = (__typeof__(lval)) _res;                             \
1298    } while (0)
1299
1300 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
1301    do {                                                           \
1302       volatile OrigFn        _orig = (orig);                      \
1303       volatile unsigned long _argvec[4];                          \
1304       volatile unsigned long _res;                                \
1305       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1306       _argvec[1] = (unsigned long)(arg1);                         \
1307       _argvec[2] = (unsigned long)(arg2);                         \
1308       _argvec[3] = (unsigned long)(arg3);                         \
1309       __asm__ volatile(                                           \
1310          VALGRIND_ALIGN_STACK                                     \
1311          "subl $4, %%esp\n\t"                                     \
1312          "pushl 12(%%eax)\n\t"                                    \
1313          "pushl 8(%%eax)\n\t"                                     \
1314          "pushl 4(%%eax)\n\t"                                     \
1315          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1316          VALGRIND_CALL_NOREDIR_EAX                                \
1317          VALGRIND_RESTORE_STACK                                   \
1318          : /*out*/   "=a" (_res)                                  \
1319          : /*in*/    "a" (&_argvec[0])                            \
1320          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1321       );                                                          \
1322       lval = (__typeof__(lval)) _res;                             \
1323    } while (0)
1324
1325 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
1326    do {                                                           \
1327       volatile OrigFn        _orig = (orig);                      \
1328       volatile unsigned long _argvec[5];                          \
1329       volatile unsigned long _res;                                \
1330       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1331       _argvec[1] = (unsigned long)(arg1);                         \
1332       _argvec[2] = (unsigned long)(arg2);                         \
1333       _argvec[3] = (unsigned long)(arg3);                         \
1334       _argvec[4] = (unsigned long)(arg4);                         \
1335       __asm__ volatile(                                           \
1336          VALGRIND_ALIGN_STACK                                     \
1337          "pushl 16(%%eax)\n\t"                                    \
1338          "pushl 12(%%eax)\n\t"                                    \
1339          "pushl 8(%%eax)\n\t"                                     \
1340          "pushl 4(%%eax)\n\t"                                     \
1341          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1342          VALGRIND_CALL_NOREDIR_EAX                                \
1343          VALGRIND_RESTORE_STACK                                   \
1344          : /*out*/   "=a" (_res)                                  \
1345          : /*in*/    "a" (&_argvec[0])                            \
1346          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1347       );                                                          \
1348       lval = (__typeof__(lval)) _res;                             \
1349    } while (0)
1350
1351 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
1352    do {                                                           \
1353       volatile OrigFn        _orig = (orig);                      \
1354       volatile unsigned long _argvec[6];                          \
1355       volatile unsigned long _res;                                \
1356       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1357       _argvec[1] = (unsigned long)(arg1);                         \
1358       _argvec[2] = (unsigned long)(arg2);                         \
1359       _argvec[3] = (unsigned long)(arg3);                         \
1360       _argvec[4] = (unsigned long)(arg4);                         \
1361       _argvec[5] = (unsigned long)(arg5);                         \
1362       __asm__ volatile(                                           \
1363          VALGRIND_ALIGN_STACK                                     \
1364          "subl $12, %%esp\n\t"                                    \
1365          "pushl 20(%%eax)\n\t"                                    \
1366          "pushl 16(%%eax)\n\t"                                    \
1367          "pushl 12(%%eax)\n\t"                                    \
1368          "pushl 8(%%eax)\n\t"                                     \
1369          "pushl 4(%%eax)\n\t"                                     \
1370          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1371          VALGRIND_CALL_NOREDIR_EAX                                \
1372          VALGRIND_RESTORE_STACK                                   \
1373          : /*out*/   "=a" (_res)                                  \
1374          : /*in*/    "a" (&_argvec[0])                            \
1375          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1376       );                                                          \
1377       lval = (__typeof__(lval)) _res;                             \
1378    } while (0)
1379
1380 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
1381    do {                                                           \
1382       volatile OrigFn        _orig = (orig);                      \
1383       volatile unsigned long _argvec[7];                          \
1384       volatile unsigned long _res;                                \
1385       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1386       _argvec[1] = (unsigned long)(arg1);                         \
1387       _argvec[2] = (unsigned long)(arg2);                         \
1388       _argvec[3] = (unsigned long)(arg3);                         \
1389       _argvec[4] = (unsigned long)(arg4);                         \
1390       _argvec[5] = (unsigned long)(arg5);                         \
1391       _argvec[6] = (unsigned long)(arg6);                         \
1392       __asm__ volatile(                                           \
1393          VALGRIND_ALIGN_STACK                                     \
1394          "subl $8, %%esp\n\t"                                     \
1395          "pushl 24(%%eax)\n\t"                                    \
1396          "pushl 20(%%eax)\n\t"                                    \
1397          "pushl 16(%%eax)\n\t"                                    \
1398          "pushl 12(%%eax)\n\t"                                    \
1399          "pushl 8(%%eax)\n\t"                                     \
1400          "pushl 4(%%eax)\n\t"                                     \
1401          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1402          VALGRIND_CALL_NOREDIR_EAX                                \
1403          VALGRIND_RESTORE_STACK                                   \
1404          : /*out*/   "=a" (_res)                                  \
1405          : /*in*/    "a" (&_argvec[0])                            \
1406          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1407       );                                                          \
1408       lval = (__typeof__(lval)) _res;                             \
1409    } while (0)
1410
1411 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1412                                  arg7)                            \
1413    do {                                                           \
1414       volatile OrigFn        _orig = (orig);                      \
1415       volatile unsigned long _argvec[8];                          \
1416       volatile unsigned long _res;                                \
1417       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1418       _argvec[1] = (unsigned long)(arg1);                         \
1419       _argvec[2] = (unsigned long)(arg2);                         \
1420       _argvec[3] = (unsigned long)(arg3);                         \
1421       _argvec[4] = (unsigned long)(arg4);                         \
1422       _argvec[5] = (unsigned long)(arg5);                         \
1423       _argvec[6] = (unsigned long)(arg6);                         \
1424       _argvec[7] = (unsigned long)(arg7);                         \
1425       __asm__ volatile(                                           \
1426          VALGRIND_ALIGN_STACK                                     \
1427          "subl $4, %%esp\n\t"                                     \
1428          "pushl 28(%%eax)\n\t"                                    \
1429          "pushl 24(%%eax)\n\t"                                    \
1430          "pushl 20(%%eax)\n\t"                                    \
1431          "pushl 16(%%eax)\n\t"                                    \
1432          "pushl 12(%%eax)\n\t"                                    \
1433          "pushl 8(%%eax)\n\t"                                     \
1434          "pushl 4(%%eax)\n\t"                                     \
1435          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1436          VALGRIND_CALL_NOREDIR_EAX                                \
1437          VALGRIND_RESTORE_STACK                                   \
1438          : /*out*/   "=a" (_res)                                  \
1439          : /*in*/    "a" (&_argvec[0])                            \
1440          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1441       );                                                          \
1442       lval = (__typeof__(lval)) _res;                             \
1443    } while (0)
1444
1445 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1446                                  arg7,arg8)                       \
1447    do {                                                           \
1448       volatile OrigFn        _orig = (orig);                      \
1449       volatile unsigned long _argvec[9];                          \
1450       volatile unsigned long _res;                                \
1451       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1452       _argvec[1] = (unsigned long)(arg1);                         \
1453       _argvec[2] = (unsigned long)(arg2);                         \
1454       _argvec[3] = (unsigned long)(arg3);                         \
1455       _argvec[4] = (unsigned long)(arg4);                         \
1456       _argvec[5] = (unsigned long)(arg5);                         \
1457       _argvec[6] = (unsigned long)(arg6);                         \
1458       _argvec[7] = (unsigned long)(arg7);                         \
1459       _argvec[8] = (unsigned long)(arg8);                         \
1460       __asm__ volatile(                                           \
1461          VALGRIND_ALIGN_STACK                                     \
1462          "pushl 32(%%eax)\n\t"                                    \
1463          "pushl 28(%%eax)\n\t"                                    \
1464          "pushl 24(%%eax)\n\t"                                    \
1465          "pushl 20(%%eax)\n\t"                                    \
1466          "pushl 16(%%eax)\n\t"                                    \
1467          "pushl 12(%%eax)\n\t"                                    \
1468          "pushl 8(%%eax)\n\t"                                     \
1469          "pushl 4(%%eax)\n\t"                                     \
1470          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1471          VALGRIND_CALL_NOREDIR_EAX                                \
1472          VALGRIND_RESTORE_STACK                                   \
1473          : /*out*/   "=a" (_res)                                  \
1474          : /*in*/    "a" (&_argvec[0])                            \
1475          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1476       );                                                          \
1477       lval = (__typeof__(lval)) _res;                             \
1478    } while (0)
1479
1480 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1481                                  arg7,arg8,arg9)                  \
1482    do {                                                           \
1483       volatile OrigFn        _orig = (orig);                      \
1484       volatile unsigned long _argvec[10];                         \
1485       volatile unsigned long _res;                                \
1486       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1487       _argvec[1] = (unsigned long)(arg1);                         \
1488       _argvec[2] = (unsigned long)(arg2);                         \
1489       _argvec[3] = (unsigned long)(arg3);                         \
1490       _argvec[4] = (unsigned long)(arg4);                         \
1491       _argvec[5] = (unsigned long)(arg5);                         \
1492       _argvec[6] = (unsigned long)(arg6);                         \
1493       _argvec[7] = (unsigned long)(arg7);                         \
1494       _argvec[8] = (unsigned long)(arg8);                         \
1495       _argvec[9] = (unsigned long)(arg9);                         \
1496       __asm__ volatile(                                           \
1497          VALGRIND_ALIGN_STACK                                     \
1498          "subl $12, %%esp\n\t"                                    \
1499          "pushl 36(%%eax)\n\t"                                    \
1500          "pushl 32(%%eax)\n\t"                                    \
1501          "pushl 28(%%eax)\n\t"                                    \
1502          "pushl 24(%%eax)\n\t"                                    \
1503          "pushl 20(%%eax)\n\t"                                    \
1504          "pushl 16(%%eax)\n\t"                                    \
1505          "pushl 12(%%eax)\n\t"                                    \
1506          "pushl 8(%%eax)\n\t"                                     \
1507          "pushl 4(%%eax)\n\t"                                     \
1508          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1509          VALGRIND_CALL_NOREDIR_EAX                                \
1510          VALGRIND_RESTORE_STACK                                   \
1511          : /*out*/   "=a" (_res)                                  \
1512          : /*in*/    "a" (&_argvec[0])                            \
1513          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1514       );                                                          \
1515       lval = (__typeof__(lval)) _res;                             \
1516    } while (0)
1517
1518 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1519                                   arg7,arg8,arg9,arg10)           \
1520    do {                                                           \
1521       volatile OrigFn        _orig = (orig);                      \
1522       volatile unsigned long _argvec[11];                         \
1523       volatile unsigned long _res;                                \
1524       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1525       _argvec[1] = (unsigned long)(arg1);                         \
1526       _argvec[2] = (unsigned long)(arg2);                         \
1527       _argvec[3] = (unsigned long)(arg3);                         \
1528       _argvec[4] = (unsigned long)(arg4);                         \
1529       _argvec[5] = (unsigned long)(arg5);                         \
1530       _argvec[6] = (unsigned long)(arg6);                         \
1531       _argvec[7] = (unsigned long)(arg7);                         \
1532       _argvec[8] = (unsigned long)(arg8);                         \
1533       _argvec[9] = (unsigned long)(arg9);                         \
1534       _argvec[10] = (unsigned long)(arg10);                       \
1535       __asm__ volatile(                                           \
1536          VALGRIND_ALIGN_STACK                                     \
1537          "subl $8, %%esp\n\t"                                     \
1538          "pushl 40(%%eax)\n\t"                                    \
1539          "pushl 36(%%eax)\n\t"                                    \
1540          "pushl 32(%%eax)\n\t"                                    \
1541          "pushl 28(%%eax)\n\t"                                    \
1542          "pushl 24(%%eax)\n\t"                                    \
1543          "pushl 20(%%eax)\n\t"                                    \
1544          "pushl 16(%%eax)\n\t"                                    \
1545          "pushl 12(%%eax)\n\t"                                    \
1546          "pushl 8(%%eax)\n\t"                                     \
1547          "pushl 4(%%eax)\n\t"                                     \
1548          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1549          VALGRIND_CALL_NOREDIR_EAX                                \
1550          VALGRIND_RESTORE_STACK                                   \
1551          : /*out*/   "=a" (_res)                                  \
1552          : /*in*/    "a" (&_argvec[0])                            \
1553          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1554       );                                                          \
1555       lval = (__typeof__(lval)) _res;                             \
1556    } while (0)
1557
1558 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
1559                                   arg6,arg7,arg8,arg9,arg10,      \
1560                                   arg11)                          \
1561    do {                                                           \
1562       volatile OrigFn        _orig = (orig);                      \
1563       volatile unsigned long _argvec[12];                         \
1564       volatile unsigned long _res;                                \
1565       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1566       _argvec[1] = (unsigned long)(arg1);                         \
1567       _argvec[2] = (unsigned long)(arg2);                         \
1568       _argvec[3] = (unsigned long)(arg3);                         \
1569       _argvec[4] = (unsigned long)(arg4);                         \
1570       _argvec[5] = (unsigned long)(arg5);                         \
1571       _argvec[6] = (unsigned long)(arg6);                         \
1572       _argvec[7] = (unsigned long)(arg7);                         \
1573       _argvec[8] = (unsigned long)(arg8);                         \
1574       _argvec[9] = (unsigned long)(arg9);                         \
1575       _argvec[10] = (unsigned long)(arg10);                       \
1576       _argvec[11] = (unsigned long)(arg11);                       \
1577       __asm__ volatile(                                           \
1578          VALGRIND_ALIGN_STACK                                     \
1579          "subl $4, %%esp\n\t"                                     \
1580          "pushl 44(%%eax)\n\t"                                    \
1581          "pushl 40(%%eax)\n\t"                                    \
1582          "pushl 36(%%eax)\n\t"                                    \
1583          "pushl 32(%%eax)\n\t"                                    \
1584          "pushl 28(%%eax)\n\t"                                    \
1585          "pushl 24(%%eax)\n\t"                                    \
1586          "pushl 20(%%eax)\n\t"                                    \
1587          "pushl 16(%%eax)\n\t"                                    \
1588          "pushl 12(%%eax)\n\t"                                    \
1589          "pushl 8(%%eax)\n\t"                                     \
1590          "pushl 4(%%eax)\n\t"                                     \
1591          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1592          VALGRIND_CALL_NOREDIR_EAX                                \
1593          VALGRIND_RESTORE_STACK                                   \
1594          : /*out*/   "=a" (_res)                                  \
1595          : /*in*/    "a" (&_argvec[0])                            \
1596          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1597       );                                                          \
1598       lval = (__typeof__(lval)) _res;                             \
1599    } while (0)
1600
1601 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
1602                                   arg6,arg7,arg8,arg9,arg10,      \
1603                                   arg11,arg12)                    \
1604    do {                                                           \
1605       volatile OrigFn        _orig = (orig);                      \
1606       volatile unsigned long _argvec[13];                         \
1607       volatile unsigned long _res;                                \
1608       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1609       _argvec[1] = (unsigned long)(arg1);                         \
1610       _argvec[2] = (unsigned long)(arg2);                         \
1611       _argvec[3] = (unsigned long)(arg3);                         \
1612       _argvec[4] = (unsigned long)(arg4);                         \
1613       _argvec[5] = (unsigned long)(arg5);                         \
1614       _argvec[6] = (unsigned long)(arg6);                         \
1615       _argvec[7] = (unsigned long)(arg7);                         \
1616       _argvec[8] = (unsigned long)(arg8);                         \
1617       _argvec[9] = (unsigned long)(arg9);                         \
1618       _argvec[10] = (unsigned long)(arg10);                       \
1619       _argvec[11] = (unsigned long)(arg11);                       \
1620       _argvec[12] = (unsigned long)(arg12);                       \
1621       __asm__ volatile(                                           \
1622          VALGRIND_ALIGN_STACK                                     \
1623          "pushl 48(%%eax)\n\t"                                    \
1624          "pushl 44(%%eax)\n\t"                                    \
1625          "pushl 40(%%eax)\n\t"                                    \
1626          "pushl 36(%%eax)\n\t"                                    \
1627          "pushl 32(%%eax)\n\t"                                    \
1628          "pushl 28(%%eax)\n\t"                                    \
1629          "pushl 24(%%eax)\n\t"                                    \
1630          "pushl 20(%%eax)\n\t"                                    \
1631          "pushl 16(%%eax)\n\t"                                    \
1632          "pushl 12(%%eax)\n\t"                                    \
1633          "pushl 8(%%eax)\n\t"                                     \
1634          "pushl 4(%%eax)\n\t"                                     \
1635          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1636          VALGRIND_CALL_NOREDIR_EAX                                \
1637          VALGRIND_RESTORE_STACK                                   \
1638          : /*out*/   "=a" (_res)                                  \
1639          : /*in*/    "a" (&_argvec[0])                            \
1640          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1641       );                                                          \
1642       lval = (__typeof__(lval)) _res;                             \
1643    } while (0)
1644
1645 #endif /* PLAT_x86_linux || PLAT_x86_darwin || PLAT_x86_solaris */
1646
1647 /* ---------------- amd64-{linux,darwin,solaris} --------------- */
1648
1649 #if defined(PLAT_amd64_linux)  ||  defined(PLAT_amd64_darwin) \
1650     ||  defined(PLAT_amd64_solaris)
1651
1652 /* ARGREGS: rdi rsi rdx rcx r8 r9 (the rest on stack in R-to-L order) */
1653
1654 /* These regs are trashed by the hidden call. */
1655 #define __CALLER_SAVED_REGS /*"rax",*/ "rcx", "rdx", "rsi",       \
1656                             "rdi", "r8", "r9", "r10", "r11"
1657
1658 /* This is all pretty complex.  It's so as to make stack unwinding
1659    work reliably.  See bug 243270.  The basic problem is the sub and
1660    add of 128 of %rsp in all of the following macros.  If gcc believes
1661    the CFA is in %rsp, then unwinding may fail, because what's at the
1662    CFA is not what gcc "expected" when it constructs the CFIs for the
1663    places where the macros are instantiated.
1664
1665    But we can't just add a CFI annotation to increase the CFA offset
1666    by 128, to match the sub of 128 from %rsp, because we don't know
1667    whether gcc has chosen %rsp as the CFA at that point, or whether it
1668    has chosen some other register (eg, %rbp).  In the latter case,
1669    adding a CFI annotation to change the CFA offset is simply wrong.
1670
1671    So the solution is to get hold of the CFA using
1672    __builtin_dwarf_cfa(), put it in a known register, and add a
1673    CFI annotation to say what the register is.  We choose %rbp for
1674    this (perhaps perversely), because:
1675
1676    (1) %rbp is already subject to unwinding.  If a new register was
1677        chosen then the unwinder would have to unwind it in all stack
1678        traces, which is expensive, and
1679
1680    (2) %rbp is already subject to precise exception updates in the
1681        JIT.  If a new register was chosen, we'd have to have precise
1682        exceptions for it too, which reduces performance of the
1683        generated code.
1684
1685    However .. one extra complication.  We can't just whack the result
1686    of __builtin_dwarf_cfa() into %rbp and then add %rbp to the
1687    list of trashed registers at the end of the inline assembly
1688    fragments; gcc won't allow %rbp to appear in that list.  Hence
1689    instead we need to stash %rbp in %r15 for the duration of the asm,
1690    and say that %r15 is trashed instead.  gcc seems happy to go with
1691    that.
1692
1693    Oh .. and this all needs to be conditionalised so that it is
1694    unchanged from before this commit, when compiled with older gccs
1695    that don't support __builtin_dwarf_cfa.  Furthermore, since
1696    this header file is freestanding, it has to be independent of
1697    config.h, and so the following conditionalisation cannot depend on
1698    configure time checks.
1699
1700    Although it's not clear from
1701    'defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM)',
1702    this expression excludes Darwin.
1703    .cfi directives in Darwin assembly appear to be completely
1704    different and I haven't investigated how they work.
1705
1706    For even more entertainment value, note we have to use the
1707    completely undocumented __builtin_dwarf_cfa(), which appears to
1708    really compute the CFA, whereas __builtin_frame_address(0) claims
1709    to but actually doesn't.  See
1710    https://bugs.kde.org/show_bug.cgi?id=243270#c47
1711 */
1712 #if defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM)
1713 #  define __FRAME_POINTER                                         \
1714       ,"r"(__builtin_dwarf_cfa())
1715 #  define VALGRIND_CFI_PROLOGUE                                   \
1716       "movq %%rbp, %%r15\n\t"                                     \
1717       "movq %2, %%rbp\n\t"                                        \
1718       ".cfi_remember_state\n\t"                                   \
1719       ".cfi_def_cfa rbp, 0\n\t"
1720 #  define VALGRIND_CFI_EPILOGUE                                   \
1721       "movq %%r15, %%rbp\n\t"                                     \
1722       ".cfi_restore_state\n\t"
1723 #else
1724 #  define __FRAME_POINTER
1725 #  define VALGRIND_CFI_PROLOGUE
1726 #  define VALGRIND_CFI_EPILOGUE
1727 #endif
1728
1729 /* Macros to save and align the stack before making a function
1730    call and restore it afterwards as gcc may not keep the stack
1731    pointer aligned if it doesn't realise calls are being made
1732    to other functions. */
1733
1734 #define VALGRIND_ALIGN_STACK               \
1735       "movq %%rsp,%%r14\n\t"               \
1736       "andq $0xfffffffffffffff0,%%rsp\n\t"
1737 #define VALGRIND_RESTORE_STACK             \
1738       "movq %%r14,%%rsp\n\t"
1739
1740 /* These CALL_FN_ macros assume that on amd64-linux, sizeof(unsigned
1741    long) == 8. */
1742
1743 /* NB 9 Sept 07.  There is a nasty kludge here in all these CALL_FN_
1744    macros.  In order not to trash the stack redzone, we need to drop
1745    %rsp by 128 before the hidden call, and restore afterwards.  The
1746    nastyness is that it is only by luck that the stack still appears
1747    to be unwindable during the hidden call - since then the behaviour
1748    of any routine using this macro does not match what the CFI data
1749    says.  Sigh.
1750
1751    Why is this important?  Imagine that a wrapper has a stack
1752    allocated local, and passes to the hidden call, a pointer to it.
1753    Because gcc does not know about the hidden call, it may allocate
1754    that local in the redzone.  Unfortunately the hidden call may then
1755    trash it before it comes to use it.  So we must step clear of the
1756    redzone, for the duration of the hidden call, to make it safe.
1757
1758    Probably the same problem afflicts the other redzone-style ABIs too
1759    (ppc64-linux); but for those, the stack is
1760    self describing (none of this CFI nonsense) so at least messing
1761    with the stack pointer doesn't give a danger of non-unwindable
1762    stack. */
1763
1764 #define CALL_FN_W_v(lval, orig)                                        \
1765    do {                                                                \
1766       volatile OrigFn        _orig = (orig);                           \
1767       volatile unsigned long _argvec[1];                               \
1768       volatile unsigned long _res;                                     \
1769       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1770       __asm__ volatile(                                                \
1771          VALGRIND_CFI_PROLOGUE                                         \
1772          VALGRIND_ALIGN_STACK                                          \
1773          "subq $128,%%rsp\n\t"                                         \
1774          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1775          VALGRIND_CALL_NOREDIR_RAX                                     \
1776          VALGRIND_RESTORE_STACK                                        \
1777          VALGRIND_CFI_EPILOGUE                                         \
1778          : /*out*/   "=a" (_res)                                       \
1779          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1780          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1781       );                                                               \
1782       lval = (__typeof__(lval)) _res;                                  \
1783    } while (0)
1784
1785 #define CALL_FN_W_W(lval, orig, arg1)                                  \
1786    do {                                                                \
1787       volatile OrigFn        _orig = (orig);                           \
1788       volatile unsigned long _argvec[2];                               \
1789       volatile unsigned long _res;                                     \
1790       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1791       _argvec[1] = (unsigned long)(arg1);                              \
1792       __asm__ volatile(                                                \
1793          VALGRIND_CFI_PROLOGUE                                         \
1794          VALGRIND_ALIGN_STACK                                          \
1795          "subq $128,%%rsp\n\t"                                         \
1796          "movq 8(%%rax), %%rdi\n\t"                                    \
1797          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1798          VALGRIND_CALL_NOREDIR_RAX                                     \
1799          VALGRIND_RESTORE_STACK                                        \
1800          VALGRIND_CFI_EPILOGUE                                         \
1801          : /*out*/   "=a" (_res)                                       \
1802          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1803          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1804       );                                                               \
1805       lval = (__typeof__(lval)) _res;                                  \
1806    } while (0)
1807
1808 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                            \
1809    do {                                                                \
1810       volatile OrigFn        _orig = (orig);                           \
1811       volatile unsigned long _argvec[3];                               \
1812       volatile unsigned long _res;                                     \
1813       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1814       _argvec[1] = (unsigned long)(arg1);                              \
1815       _argvec[2] = (unsigned long)(arg2);                              \
1816       __asm__ volatile(                                                \
1817          VALGRIND_CFI_PROLOGUE                                         \
1818          VALGRIND_ALIGN_STACK                                          \
1819          "subq $128,%%rsp\n\t"                                         \
1820          "movq 16(%%rax), %%rsi\n\t"                                   \
1821          "movq 8(%%rax), %%rdi\n\t"                                    \
1822          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1823          VALGRIND_CALL_NOREDIR_RAX                                     \
1824          VALGRIND_RESTORE_STACK                                        \
1825          VALGRIND_CFI_EPILOGUE                                         \
1826          : /*out*/   "=a" (_res)                                       \
1827          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1828          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1829       );                                                               \
1830       lval = (__typeof__(lval)) _res;                                  \
1831    } while (0)
1832
1833 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                      \
1834    do {                                                                \
1835       volatile OrigFn        _orig = (orig);                           \
1836       volatile unsigned long _argvec[4];                               \
1837       volatile unsigned long _res;                                     \
1838       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1839       _argvec[1] = (unsigned long)(arg1);                              \
1840       _argvec[2] = (unsigned long)(arg2);                              \
1841       _argvec[3] = (unsigned long)(arg3);                              \
1842       __asm__ volatile(                                                \
1843          VALGRIND_CFI_PROLOGUE                                         \
1844          VALGRIND_ALIGN_STACK                                          \
1845          "subq $128,%%rsp\n\t"                                         \
1846          "movq 24(%%rax), %%rdx\n\t"                                   \
1847          "movq 16(%%rax), %%rsi\n\t"                                   \
1848          "movq 8(%%rax), %%rdi\n\t"                                    \
1849          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1850          VALGRIND_CALL_NOREDIR_RAX                                     \
1851          VALGRIND_RESTORE_STACK                                        \
1852          VALGRIND_CFI_EPILOGUE                                         \
1853          : /*out*/   "=a" (_res)                                       \
1854          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1855          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1856       );                                                               \
1857       lval = (__typeof__(lval)) _res;                                  \
1858    } while (0)
1859
1860 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)                \
1861    do {                                                                \
1862       volatile OrigFn        _orig = (orig);                           \
1863       volatile unsigned long _argvec[5];                               \
1864       volatile unsigned long _res;                                     \
1865       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1866       _argvec[1] = (unsigned long)(arg1);                              \
1867       _argvec[2] = (unsigned long)(arg2);                              \
1868       _argvec[3] = (unsigned long)(arg3);                              \
1869       _argvec[4] = (unsigned long)(arg4);                              \
1870       __asm__ volatile(                                                \
1871          VALGRIND_CFI_PROLOGUE                                         \
1872          VALGRIND_ALIGN_STACK                                          \
1873          "subq $128,%%rsp\n\t"                                         \
1874          "movq 32(%%rax), %%rcx\n\t"                                   \
1875          "movq 24(%%rax), %%rdx\n\t"                                   \
1876          "movq 16(%%rax), %%rsi\n\t"                                   \
1877          "movq 8(%%rax), %%rdi\n\t"                                    \
1878          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1879          VALGRIND_CALL_NOREDIR_RAX                                     \
1880          VALGRIND_RESTORE_STACK                                        \
1881          VALGRIND_CFI_EPILOGUE                                         \
1882          : /*out*/   "=a" (_res)                                       \
1883          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1884          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1885       );                                                               \
1886       lval = (__typeof__(lval)) _res;                                  \
1887    } while (0)
1888
1889 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)             \
1890    do {                                                                \
1891       volatile OrigFn        _orig = (orig);                           \
1892       volatile unsigned long _argvec[6];                               \
1893       volatile unsigned long _res;                                     \
1894       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1895       _argvec[1] = (unsigned long)(arg1);                              \
1896       _argvec[2] = (unsigned long)(arg2);                              \
1897       _argvec[3] = (unsigned long)(arg3);                              \
1898       _argvec[4] = (unsigned long)(arg4);                              \
1899       _argvec[5] = (unsigned long)(arg5);                              \
1900       __asm__ volatile(                                                \
1901          VALGRIND_CFI_PROLOGUE                                         \
1902          VALGRIND_ALIGN_STACK                                          \
1903          "subq $128,%%rsp\n\t"                                         \
1904          "movq 40(%%rax), %%r8\n\t"                                    \
1905          "movq 32(%%rax), %%rcx\n\t"                                   \
1906          "movq 24(%%rax), %%rdx\n\t"                                   \
1907          "movq 16(%%rax), %%rsi\n\t"                                   \
1908          "movq 8(%%rax), %%rdi\n\t"                                    \
1909          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1910          VALGRIND_CALL_NOREDIR_RAX                                     \
1911          VALGRIND_RESTORE_STACK                                        \
1912          VALGRIND_CFI_EPILOGUE                                         \
1913          : /*out*/   "=a" (_res)                                       \
1914          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1915          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1916       );                                                               \
1917       lval = (__typeof__(lval)) _res;                                  \
1918    } while (0)
1919
1920 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)        \
1921    do {                                                                \
1922       volatile OrigFn        _orig = (orig);                           \
1923       volatile unsigned long _argvec[7];                               \
1924       volatile unsigned long _res;                                     \
1925       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1926       _argvec[1] = (unsigned long)(arg1);                              \
1927       _argvec[2] = (unsigned long)(arg2);                              \
1928       _argvec[3] = (unsigned long)(arg3);                              \
1929       _argvec[4] = (unsigned long)(arg4);                              \
1930       _argvec[5] = (unsigned long)(arg5);                              \
1931       _argvec[6] = (unsigned long)(arg6);                              \
1932       __asm__ volatile(                                                \
1933          VALGRIND_CFI_PROLOGUE                                         \
1934          VALGRIND_ALIGN_STACK                                          \
1935          "subq $128,%%rsp\n\t"                                         \
1936          "movq 48(%%rax), %%r9\n\t"                                    \
1937          "movq 40(%%rax), %%r8\n\t"                                    \
1938          "movq 32(%%rax), %%rcx\n\t"                                   \
1939          "movq 24(%%rax), %%rdx\n\t"                                   \
1940          "movq 16(%%rax), %%rsi\n\t"                                   \
1941          "movq 8(%%rax), %%rdi\n\t"                                    \
1942          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1943          VALGRIND_CALL_NOREDIR_RAX                                     \
1944          VALGRIND_RESTORE_STACK                                        \
1945          VALGRIND_CFI_EPILOGUE                                         \
1946          : /*out*/   "=a" (_res)                                       \
1947          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1948          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1949       );                                                               \
1950       lval = (__typeof__(lval)) _res;                                  \
1951    } while (0)
1952
1953 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,        \
1954                                  arg7)                                 \
1955    do {                                                                \
1956       volatile OrigFn        _orig = (orig);                           \
1957       volatile unsigned long _argvec[8];                               \
1958       volatile unsigned long _res;                                     \
1959       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1960       _argvec[1] = (unsigned long)(arg1);                              \
1961       _argvec[2] = (unsigned long)(arg2);                              \
1962       _argvec[3] = (unsigned long)(arg3);                              \
1963       _argvec[4] = (unsigned long)(arg4);                              \
1964       _argvec[5] = (unsigned long)(arg5);                              \
1965       _argvec[6] = (unsigned long)(arg6);                              \
1966       _argvec[7] = (unsigned long)(arg7);                              \
1967       __asm__ volatile(                                                \
1968          VALGRIND_CFI_PROLOGUE                                         \
1969          VALGRIND_ALIGN_STACK                                          \
1970          "subq $136,%%rsp\n\t"                                         \
1971          "pushq 56(%%rax)\n\t"                                         \
1972          "movq 48(%%rax), %%r9\n\t"                                    \
1973          "movq 40(%%rax), %%r8\n\t"                                    \
1974          "movq 32(%%rax), %%rcx\n\t"                                   \
1975          "movq 24(%%rax), %%rdx\n\t"                                   \
1976          "movq 16(%%rax), %%rsi\n\t"                                   \
1977          "movq 8(%%rax), %%rdi\n\t"                                    \
1978          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1979          VALGRIND_CALL_NOREDIR_RAX                                     \
1980          VALGRIND_RESTORE_STACK                                        \
1981          VALGRIND_CFI_EPILOGUE                                         \
1982          : /*out*/   "=a" (_res)                                       \
1983          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1984          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1985       );                                                               \
1986       lval = (__typeof__(lval)) _res;                                  \
1987    } while (0)
1988
1989 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,        \
1990                                  arg7,arg8)                            \
1991    do {                                                                \
1992       volatile OrigFn        _orig = (orig);                           \
1993       volatile unsigned long _argvec[9];                               \
1994       volatile unsigned long _res;                                     \
1995       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1996       _argvec[1] = (unsigned long)(arg1);                              \
1997       _argvec[2] = (unsigned long)(arg2);                              \
1998       _argvec[3] = (unsigned long)(arg3);                              \
1999       _argvec[4] = (unsigned long)(arg4);                              \
2000       _argvec[5] = (unsigned long)(arg5);                              \
2001       _argvec[6] = (unsigned long)(arg6);                              \
2002       _argvec[7] = (unsigned long)(arg7);                              \
2003       _argvec[8] = (unsigned long)(arg8);                              \
2004       __asm__ volatile(                                                \
2005          VALGRIND_CFI_PROLOGUE                                         \
2006          VALGRIND_ALIGN_STACK                                          \
2007          "subq $128,%%rsp\n\t"                                         \
2008          "pushq 64(%%rax)\n\t"                                         \
2009          "pushq 56(%%rax)\n\t"                                         \
2010          "movq 48(%%rax), %%r9\n\t"                                    \
2011          "movq 40(%%rax), %%r8\n\t"                                    \
2012          "movq 32(%%rax), %%rcx\n\t"                                   \
2013          "movq 24(%%rax), %%rdx\n\t"                                   \
2014          "movq 16(%%rax), %%rsi\n\t"                                   \
2015          "movq 8(%%rax), %%rdi\n\t"                                    \
2016          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
2017          VALGRIND_CALL_NOREDIR_RAX                                     \
2018          VALGRIND_RESTORE_STACK                                        \
2019          VALGRIND_CFI_EPILOGUE                                         \
2020          : /*out*/   "=a" (_res)                                       \
2021          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
2022          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
2023       );                                                               \
2024       lval = (__typeof__(lval)) _res;                                  \
2025    } while (0)
2026
2027 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,        \
2028                                  arg7,arg8,arg9)                       \
2029    do {                                                                \
2030       volatile OrigFn        _orig = (orig);                           \
2031       volatile unsigned long _argvec[10];                              \
2032       volatile unsigned long _res;                                     \
2033       _argvec[0] = (unsigned long)_orig.nraddr;                        \
2034       _argvec[1] = (unsigned long)(arg1);                              \
2035       _argvec[2] = (unsigned long)(arg2);                              \
2036       _argvec[3] = (unsigned long)(arg3);                              \
2037       _argvec[4] = (unsigned long)(arg4);                              \
2038       _argvec[5] = (unsigned long)(arg5);                              \
2039       _argvec[6] = (unsigned long)(arg6);                              \
2040       _argvec[7] = (unsigned long)(arg7);                              \
2041       _argvec[8] = (unsigned long)(arg8);                              \
2042       _argvec[9] = (unsigned long)(arg9);                              \
2043       __asm__ volatile(                                                \
2044          VALGRIND_CFI_PROLOGUE                                         \
2045          VALGRIND_ALIGN_STACK                                          \
2046          "subq $136,%%rsp\n\t"                                         \
2047          "pushq 72(%%rax)\n\t"                                         \
2048          "pushq 64(%%rax)\n\t"                                         \
2049          "pushq 56(%%rax)\n\t"                                         \
2050          "movq 48(%%rax), %%r9\n\t"                                    \
2051          "movq 40(%%rax), %%r8\n\t"                                    \
2052          "movq 32(%%rax), %%rcx\n\t"                                   \
2053          "movq 24(%%rax), %%rdx\n\t"                                   \
2054          "movq 16(%%rax), %%rsi\n\t"                                   \
2055          "movq 8(%%rax), %%rdi\n\t"                                    \
2056          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
2057          VALGRIND_CALL_NOREDIR_RAX                                     \
2058          VALGRIND_RESTORE_STACK                                        \
2059          VALGRIND_CFI_EPILOGUE                                         \
2060          : /*out*/   "=a" (_res)                                       \
2061          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
2062          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
2063       );                                                               \
2064       lval = (__typeof__(lval)) _res;                                  \
2065    } while (0)
2066
2067 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,       \
2068                                   arg7,arg8,arg9,arg10)                \
2069    do {                                                                \
2070       volatile OrigFn        _orig = (orig);                           \
2071       volatile unsigned long _argvec[11];                              \
2072       volatile unsigned long _res;                                     \
2073       _argvec[0] = (unsigned long)_orig.nraddr;                        \
2074       _argvec[1] = (unsigned long)(arg1);                              \
2075       _argvec[2] = (unsigned long)(arg2);                              \
2076       _argvec[3] = (unsigned long)(arg3);                              \
2077       _argvec[4] = (unsigned long)(arg4);                              \
2078       _argvec[5] = (unsigned long)(arg5);                              \
2079       _argvec[6] = (unsigned long)(arg6);                              \
2080       _argvec[7] = (unsigned long)(arg7);                              \
2081       _argvec[8] = (unsigned long)(arg8);                              \
2082       _argvec[9] = (unsigned long)(arg9);                              \
2083       _argvec[10] = (unsigned long)(arg10);                            \
2084       __asm__ volatile(                                                \
2085          VALGRIND_CFI_PROLOGUE                                         \
2086          VALGRIND_ALIGN_STACK                                          \
2087          "subq $128,%%rsp\n\t"                                         \
2088          "pushq 80(%%rax)\n\t"                                         \
2089          "pushq 72(%%rax)\n\t"                                         \
2090          "pushq 64(%%rax)\n\t"                                         \
2091          "pushq 56(%%rax)\n\t"                                         \
2092          "movq 48(%%rax), %%r9\n\t"                                    \
2093          "movq 40(%%rax), %%r8\n\t"                                    \
2094          "movq 32(%%rax), %%rcx\n\t"                                   \
2095          "movq 24(%%rax), %%rdx\n\t"                                   \
2096          "movq 16(%%rax), %%rsi\n\t"                                   \
2097          "movq 8(%%rax), %%rdi\n\t"                                    \
2098          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
2099          VALGRIND_CALL_NOREDIR_RAX                                     \
2100          VALGRIND_RESTORE_STACK                                        \
2101          VALGRIND_CFI_EPILOGUE                                         \
2102          : /*out*/   "=a" (_res)                                       \
2103          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
2104          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
2105       );                                                               \
2106       lval = (__typeof__(lval)) _res;                                  \
2107    } while (0)
2108
2109 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,       \
2110                                   arg7,arg8,arg9,arg10,arg11)          \
2111    do {                                                                \
2112       volatile OrigFn        _orig = (orig);                           \
2113       volatile unsigned long _argvec[12];                              \
2114       volatile unsigned long _res;                                     \
2115       _argvec[0] = (unsigned long)_orig.nraddr;                        \
2116       _argvec[1] = (unsigned long)(arg1);                              \
2117       _argvec[2] = (unsigned long)(arg2);                              \
2118       _argvec[3] = (unsigned long)(arg3);                              \
2119       _argvec[4] = (unsigned long)(arg4);                              \
2120       _argvec[5] = (unsigned long)(arg5);                              \
2121       _argvec[6] = (unsigned long)(arg6);                              \
2122       _argvec[7] = (unsigned long)(arg7);                              \
2123       _argvec[8] = (unsigned long)(arg8);                              \
2124       _argvec[9] = (unsigned long)(arg9);                              \
2125       _argvec[10] = (unsigned long)(arg10);                            \
2126       _argvec[11] = (unsigned long)(arg11);                            \
2127       __asm__ volatile(                                                \
2128          VALGRIND_CFI_PROLOGUE                                         \
2129          VALGRIND_ALIGN_STACK                                          \
2130          "subq $136,%%rsp\n\t"                                         \
2131          "pushq 88(%%rax)\n\t"                                         \
2132          "pushq 80(%%rax)\n\t"                                         \
2133          "pushq 72(%%rax)\n\t"                                         \
2134          "pushq 64(%%rax)\n\t"                                         \
2135          "pushq 56(%%rax)\n\t"                                         \
2136          "movq 48(%%rax), %%r9\n\t"                                    \
2137          "movq 40(%%rax), %%r8\n\t"                                    \
2138          "movq 32(%%rax), %%rcx\n\t"                                   \
2139          "movq 24(%%rax), %%rdx\n\t"                                   \
2140          "movq 16(%%rax), %%rsi\n\t"                                   \
2141          "movq 8(%%rax), %%rdi\n\t"                                    \
2142          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
2143          VALGRIND_CALL_NOREDIR_RAX                                     \
2144          VALGRIND_RESTORE_STACK                                        \
2145          VALGRIND_CFI_EPILOGUE                                         \
2146          : /*out*/   "=a" (_res)                                       \
2147          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
2148          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
2149       );                                                               \
2150       lval = (__typeof__(lval)) _res;                                  \
2151    } while (0)
2152
2153 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,       \
2154                                 arg7,arg8,arg9,arg10,arg11,arg12)      \
2155    do {                                                                \
2156       volatile OrigFn        _orig = (orig);                           \
2157       volatile unsigned long _argvec[13];                              \
2158       volatile unsigned long _res;                                     \
2159       _argvec[0] = (unsigned long)_orig.nraddr;                        \
2160       _argvec[1] = (unsigned long)(arg1);                              \
2161       _argvec[2] = (unsigned long)(arg2);                              \
2162       _argvec[3] = (unsigned long)(arg3);                              \
2163       _argvec[4] = (unsigned long)(arg4);                              \
2164       _argvec[5] = (unsigned long)(arg5);                              \
2165       _argvec[6] = (unsigned long)(arg6);                              \
2166       _argvec[7] = (unsigned long)(arg7);                              \
2167       _argvec[8] = (unsigned long)(arg8);                              \
2168       _argvec[9] = (unsigned long)(arg9);                              \
2169       _argvec[10] = (unsigned long)(arg10);                            \
2170       _argvec[11] = (unsigned long)(arg11);                            \
2171       _argvec[12] = (unsigned long)(arg12);                            \
2172       __asm__ volatile(                                                \
2173          VALGRIND_CFI_PROLOGUE                                         \
2174          VALGRIND_ALIGN_STACK                                          \
2175          "subq $128,%%rsp\n\t"                                         \
2176          "pushq 96(%%rax)\n\t"                                         \
2177          "pushq 88(%%rax)\n\t"                                         \
2178          "pushq 80(%%rax)\n\t"                                         \
2179          "pushq 72(%%rax)\n\t"                                         \
2180          "pushq 64(%%rax)\n\t"                                         \
2181          "pushq 56(%%rax)\n\t"                                         \
2182          "movq 48(%%rax), %%r9\n\t"                                    \
2183          "movq 40(%%rax), %%r8\n\t"                                    \
2184          "movq 32(%%rax), %%rcx\n\t"                                   \
2185          "movq 24(%%rax), %%rdx\n\t"                                   \
2186          "movq 16(%%rax), %%rsi\n\t"                                   \
2187          "movq 8(%%rax), %%rdi\n\t"                                    \
2188          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
2189          VALGRIND_CALL_NOREDIR_RAX                                     \
2190          VALGRIND_RESTORE_STACK                                        \
2191          VALGRIND_CFI_EPILOGUE                                         \
2192          : /*out*/   "=a" (_res)                                       \
2193          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
2194          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
2195       );                                                               \
2196       lval = (__typeof__(lval)) _res;                                  \
2197    } while (0)
2198
2199 #endif /* PLAT_amd64_linux || PLAT_amd64_darwin || PLAT_amd64_solaris */
2200
2201 /* ------------------------ ppc32-linux ------------------------ */
2202
2203 #if defined(PLAT_ppc32_linux)
2204
2205 /* This is useful for finding out about the on-stack stuff:
2206
2207    extern int f9  ( int,int,int,int,int,int,int,int,int );
2208    extern int f10 ( int,int,int,int,int,int,int,int,int,int );
2209    extern int f11 ( int,int,int,int,int,int,int,int,int,int,int );
2210    extern int f12 ( int,int,int,int,int,int,int,int,int,int,int,int );
2211
2212    int g9 ( void ) {
2213       return f9(11,22,33,44,55,66,77,88,99);
2214    }
2215    int g10 ( void ) {
2216       return f10(11,22,33,44,55,66,77,88,99,110);
2217    }
2218    int g11 ( void ) {
2219       return f11(11,22,33,44,55,66,77,88,99,110,121);
2220    }
2221    int g12 ( void ) {
2222       return f12(11,22,33,44,55,66,77,88,99,110,121,132);
2223    }
2224 */
2225
2226 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
2227
2228 /* These regs are trashed by the hidden call. */
2229 #define __CALLER_SAVED_REGS                                       \
2230    "lr", "ctr", "xer",                                            \
2231    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
2232    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
2233    "r11", "r12", "r13"
2234
2235 /* Macros to save and align the stack before making a function
2236    call and restore it afterwards as gcc may not keep the stack
2237    pointer aligned if it doesn't realise calls are being made
2238    to other functions. */
2239
2240 #define VALGRIND_ALIGN_STACK               \
2241       "mr 28,1\n\t"                        \
2242       "rlwinm 1,1,0,0,27\n\t"
2243 #define VALGRIND_RESTORE_STACK             \
2244       "mr 1,28\n\t"
2245
2246 /* These CALL_FN_ macros assume that on ppc32-linux, 
2247    sizeof(unsigned long) == 4. */
2248
2249 #define CALL_FN_W_v(lval, orig)                                   \
2250    do {                                                           \
2251       volatile OrigFn        _orig = (orig);                      \
2252       volatile unsigned long _argvec[1];                          \
2253       volatile unsigned long _res;                                \
2254       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2255       __asm__ volatile(                                           \
2256          VALGRIND_ALIGN_STACK                                     \
2257          "mr 11,%1\n\t"                                           \
2258          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2259          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2260          VALGRIND_RESTORE_STACK                                   \
2261          "mr %0,3"                                                \
2262          : /*out*/   "=r" (_res)                                  \
2263          : /*in*/    "r" (&_argvec[0])                            \
2264          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2265       );                                                          \
2266       lval = (__typeof__(lval)) _res;                             \
2267    } while (0)
2268
2269 #define CALL_FN_W_W(lval, orig, arg1)                             \
2270    do {                                                           \
2271       volatile OrigFn        _orig = (orig);                      \
2272       volatile unsigned long _argvec[2];                          \
2273       volatile unsigned long _res;                                \
2274       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2275       _argvec[1] = (unsigned long)arg1;                           \
2276       __asm__ volatile(                                           \
2277          VALGRIND_ALIGN_STACK                                     \
2278          "mr 11,%1\n\t"                                           \
2279          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2280          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2281          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2282          VALGRIND_RESTORE_STACK                                   \
2283          "mr %0,3"                                                \
2284          : /*out*/   "=r" (_res)                                  \
2285          : /*in*/    "r" (&_argvec[0])                            \
2286          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2287       );                                                          \
2288       lval = (__typeof__(lval)) _res;                             \
2289    } while (0)
2290
2291 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
2292    do {                                                           \
2293       volatile OrigFn        _orig = (orig);                      \
2294       volatile unsigned long _argvec[3];                          \
2295       volatile unsigned long _res;                                \
2296       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2297       _argvec[1] = (unsigned long)arg1;                           \
2298       _argvec[2] = (unsigned long)arg2;                           \
2299       __asm__ volatile(                                           \
2300          VALGRIND_ALIGN_STACK                                     \
2301          "mr 11,%1\n\t"                                           \
2302          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2303          "lwz 4,8(11)\n\t"                                        \
2304          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2305          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2306          VALGRIND_RESTORE_STACK                                   \
2307          "mr %0,3"                                                \
2308          : /*out*/   "=r" (_res)                                  \
2309          : /*in*/    "r" (&_argvec[0])                            \
2310          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2311       );                                                          \
2312       lval = (__typeof__(lval)) _res;                             \
2313    } while (0)
2314
2315 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
2316    do {                                                           \
2317       volatile OrigFn        _orig = (orig);                      \
2318       volatile unsigned long _argvec[4];                          \
2319       volatile unsigned long _res;                                \
2320       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2321       _argvec[1] = (unsigned long)arg1;                           \
2322       _argvec[2] = (unsigned long)arg2;                           \
2323       _argvec[3] = (unsigned long)arg3;                           \
2324       __asm__ volatile(                                           \
2325          VALGRIND_ALIGN_STACK                                     \
2326          "mr 11,%1\n\t"                                           \
2327          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2328          "lwz 4,8(11)\n\t"                                        \
2329          "lwz 5,12(11)\n\t"                                       \
2330          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2331          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2332          VALGRIND_RESTORE_STACK                                   \
2333          "mr %0,3"                                                \
2334          : /*out*/   "=r" (_res)                                  \
2335          : /*in*/    "r" (&_argvec[0])                            \
2336          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2337       );                                                          \
2338       lval = (__typeof__(lval)) _res;                             \
2339    } while (0)
2340
2341 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
2342    do {                                                           \
2343       volatile OrigFn        _orig = (orig);                      \
2344       volatile unsigned long _argvec[5];                          \
2345       volatile unsigned long _res;                                \
2346       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2347       _argvec[1] = (unsigned long)arg1;                           \
2348       _argvec[2] = (unsigned long)arg2;                           \
2349       _argvec[3] = (unsigned long)arg3;                           \
2350       _argvec[4] = (unsigned long)arg4;                           \
2351       __asm__ volatile(                                           \
2352          VALGRIND_ALIGN_STACK                                     \
2353          "mr 11,%1\n\t"                                           \
2354          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2355          "lwz 4,8(11)\n\t"                                        \
2356          "lwz 5,12(11)\n\t"                                       \
2357          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2358          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2359          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2360          VALGRIND_RESTORE_STACK                                   \
2361          "mr %0,3"                                                \
2362          : /*out*/   "=r" (_res)                                  \
2363          : /*in*/    "r" (&_argvec[0])                            \
2364          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2365       );                                                          \
2366       lval = (__typeof__(lval)) _res;                             \
2367    } while (0)
2368
2369 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
2370    do {                                                           \
2371       volatile OrigFn        _orig = (orig);                      \
2372       volatile unsigned long _argvec[6];                          \
2373       volatile unsigned long _res;                                \
2374       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2375       _argvec[1] = (unsigned long)arg1;                           \
2376       _argvec[2] = (unsigned long)arg2;                           \
2377       _argvec[3] = (unsigned long)arg3;                           \
2378       _argvec[4] = (unsigned long)arg4;                           \
2379       _argvec[5] = (unsigned long)arg5;                           \
2380       __asm__ volatile(                                           \
2381          VALGRIND_ALIGN_STACK                                     \
2382          "mr 11,%1\n\t"                                           \
2383          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2384          "lwz 4,8(11)\n\t"                                        \
2385          "lwz 5,12(11)\n\t"                                       \
2386          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2387          "lwz 7,20(11)\n\t"                                       \
2388          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2389          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2390          VALGRIND_RESTORE_STACK                                   \
2391          "mr %0,3"                                                \
2392          : /*out*/   "=r" (_res)                                  \
2393          : /*in*/    "r" (&_argvec[0])                            \
2394          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2395       );                                                          \
2396       lval = (__typeof__(lval)) _res;                             \
2397    } while (0)
2398
2399 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
2400    do {                                                           \
2401       volatile OrigFn        _orig = (orig);                      \
2402       volatile unsigned long _argvec[7];                          \
2403       volatile unsigned long _res;                                \
2404       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2405       _argvec[1] = (unsigned long)arg1;                           \
2406       _argvec[2] = (unsigned long)arg2;                           \
2407     &n