30#if defined (__APPLE__)
31#define AARCH64_STACK_ALIGN 1
33#define AARCH64_STACK_ALIGN 16
39#define AARCH64_FFI_WITH_V (1 << AARCH64_FFI_WITH_V_BIT)
56#if defined (__clang__) && defined (__APPLE__)
58sys_icache_invalidate (
void *start,
size_t len);
62ffi_clear_cache (
void *start,
void *end)
64#if defined (__clang__) && defined (__APPLE__)
65 sys_icache_invalidate (start, (
char *)end - (
char *)start);
66#elif defined (__GNUC__)
67 __builtin___clear_cache (start, end);
69#error "Missing builtin to flush instruction cache"
76 return &context->
x[
n];
82#if defined __AARCH64EB__
83 return &context->
v[
n].
d[1].
s[1];
85 return &context->
v[
n].
d[0].
s[0];
92#if defined __AARCH64EB__
93 return &context->
v[
n].
d[1];
95 return &context->
v[
n].
d[0];
102 return &context->
v[
n];
115 return get_s_addr (context,
n);
116 case FFI_TYPE_DOUBLE:
117 return get_d_addr (context,
n);
118#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
120 return get_v_addr (context,
n);
124 case FFI_TYPE_UINT16:
125 case FFI_TYPE_SINT16:
126 case FFI_TYPE_UINT32:
127 case FFI_TYPE_SINT32:
129 case FFI_TYPE_POINTER:
130 case FFI_TYPE_UINT64:
131 case FFI_TYPE_SINT64:
132 return get_x_addr (context,
n);
144get_basic_type_alignment (
unsigned short type)
149#if defined (__APPLE__)
150 return sizeof (UINT32);
152 case FFI_TYPE_DOUBLE:
153 return sizeof (UINT64);
154#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
156 return sizeof (
long double);
160#if defined (__APPLE__)
161 return sizeof (UINT8);
163 case FFI_TYPE_UINT16:
164 case FFI_TYPE_SINT16:
165#if defined (__APPLE__)
166 return sizeof (UINT16);
168 case FFI_TYPE_UINT32:
170 case FFI_TYPE_SINT32:
171#if defined (__APPLE__)
172 return sizeof (UINT32);
174 case FFI_TYPE_POINTER:
175 case FFI_TYPE_UINT64:
176 case FFI_TYPE_SINT64:
177 return sizeof (UINT64);
188get_basic_type_size (
unsigned short type)
193 return sizeof (UINT32);
194 case FFI_TYPE_DOUBLE:
195 return sizeof (UINT64);
196#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
198 return sizeof (
long double);
201 return sizeof (UINT8);
203 return sizeof (SINT8);
204 case FFI_TYPE_UINT16:
205 return sizeof (UINT16);
206 case FFI_TYPE_SINT16:
207 return sizeof (SINT16);
208 case FFI_TYPE_UINT32:
209 return sizeof (UINT32);
211 case FFI_TYPE_SINT32:
212 return sizeof (SINT32);
213 case FFI_TYPE_POINTER:
214 case FFI_TYPE_UINT64:
215 return sizeof (UINT64);
216 case FFI_TYPE_SINT64:
217 return sizeof (SINT64);
239is_floating_type (
unsigned short type)
241 return (
type == FFI_TYPE_FLOAT ||
type == FFI_TYPE_DOUBLE
248get_homogeneous_type (ffi_type *ty)
250 if (ty->type == FFI_TYPE_STRUCT && ty->elements)
253 unsigned short candidate_type
254 = get_homogeneous_type (ty->elements[0]);
255 for (
i =1; ty->elements[
i];
i++)
257 unsigned short iteration_type = 0;
261 if (ty->elements[
i]->type == FFI_TYPE_STRUCT
262 && ty->elements[
i]->elements)
264 iteration_type = get_homogeneous_type (ty->elements[
i]);
268 iteration_type = ty->elements[
i]->type;
272 if (candidate_type != iteration_type)
273 return FFI_TYPE_STRUCT;
275 return candidate_type;
290element_count (ffi_type *ty)
292 if (ty->type == FFI_TYPE_STRUCT && ty->elements)
296 for (
n = 0; ty->elements[
n];
n++)
298 if (ty->elements[
n]->type == FFI_TYPE_STRUCT
299 && ty->elements[
n]->elements)
300 elems += element_count (ty->elements[
n]);
319 if (ty->type == FFI_TYPE_STRUCT
321 && is_floating_type (get_homogeneous_type (ty)))
323 unsigned n = element_count (ty);
324 return n >= 1 &&
n <= 4;
342is_register_candidate (ffi_type *ty)
348 case FFI_TYPE_DOUBLE:
349#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
353 case FFI_TYPE_UINT16:
354 case FFI_TYPE_UINT32:
355 case FFI_TYPE_UINT64:
356 case FFI_TYPE_POINTER:
358 case FFI_TYPE_SINT16:
359 case FFI_TYPE_SINT32:
361 case FFI_TYPE_SINT64:
364 case FFI_TYPE_STRUCT:
369 else if (ty->size > 16)
397is_v_register_candidate (ffi_type *ty)
399 return is_floating_type (ty->type)
400 || (ty->type == FFI_TYPE_STRUCT && is_hfa (ty));
415#if defined (__APPLE__)
416 unsigned allocating_variadic;
422arg_init (
struct arg_state *state,
size_t call_frame_size)
428#if defined (__APPLE__)
429 state->allocating_variadic = 0;
455 return get_x_addr (context, (state->
ngrn)++);
462 return get_s_addr (context, (state->
nsrn)++);
469 return get_d_addr (context, (state->
nsrn)++);
476 return get_v_addr (context, (state->
nsrn)++);
481allocate_to_stack (
struct arg_state *state,
void *stack,
size_t alignment,
490#if defined (__APPLE__)
491 if (state->allocating_variadic)
497 allocation = stack + state->
nsaa;
504copy_basic_type (
void *dest,
void *source,
unsigned short type)
511 *(
float *) dest = *(
float *) source;
513 case FFI_TYPE_DOUBLE:
514 *(
double *) dest = *(
double *) source;
516#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
518 *(
long double *) dest = *(
long double *) source;
522 *(
ffi_arg *) dest = *(UINT8 *) source;
525 *(
ffi_sarg *) dest = *(SINT8 *) source;
527 case FFI_TYPE_UINT16:
528 *(
ffi_arg *) dest = *(UINT16 *) source;
530 case FFI_TYPE_SINT16:
531 *(
ffi_sarg *) dest = *(SINT16 *) source;
533 case FFI_TYPE_UINT32:
534 *(
ffi_arg *) dest = *(UINT32 *) source;
537 case FFI_TYPE_SINT32:
538 *(
ffi_sarg *) dest = *(SINT32 *) source;
540 case FFI_TYPE_POINTER:
541 case FFI_TYPE_UINT64:
542 *(
ffi_arg *) dest = *(UINT64 *) source;
544 case FFI_TYPE_SINT64:
545 *(
ffi_sarg *) dest = *(SINT64 *) source;
556copy_hfa_to_reg_or_stack (
void *memory,
559 unsigned char *stack,
562 unsigned elems = element_count (ty);
563 if (available_v (state) < elems)
569 memcpy (allocate_to_stack (state, stack, ty->alignment, ty->size),
576 unsigned short type = get_homogeneous_type (ty);
577 for (
i = 0;
i < elems;
i++)
579 void *reg = allocate_to_v (context, state);
580 copy_basic_type (reg, memory,
type);
581 memory += get_basic_type_size (
type);
591allocate_to_register_or_stack (
struct call_context *context,
592 unsigned char *stack,
596 size_t alignment = get_basic_type_alignment (
type);
597 size_t size = alignment;
603 size =
sizeof (UINT32);
605 case FFI_TYPE_DOUBLE:
607 return allocate_to_d (context, state);
610#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
613 return allocate_to_v (context, state);
619 case FFI_TYPE_UINT16:
620 case FFI_TYPE_SINT16:
621 case FFI_TYPE_UINT32:
622 case FFI_TYPE_SINT32:
624 case FFI_TYPE_POINTER:
625 case FFI_TYPE_UINT64:
626 case FFI_TYPE_SINT64:
628 return allocate_to_x (context, state);
635 return allocate_to_stack (state, stack, alignment,
size);
643 unsigned char *stack,
649 allocate_to_register_or_stack (context, stack, state,
type),
658aarch64_prep_args (
struct call_context *context,
unsigned char *stack,
664 arg_init (&state,
ALIGN(ecif->
cif->bytes, 16));
666 for (
i = 0;
i < ecif->
cif->nargs;
i++)
668 ffi_type *ty = ecif->
cif->arg_types[
i];
678 case FFI_TYPE_DOUBLE:
679#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
684 case FFI_TYPE_UINT16:
685 case FFI_TYPE_SINT16:
686 case FFI_TYPE_UINT32:
688 case FFI_TYPE_SINT32:
689 case FFI_TYPE_POINTER:
690 case FFI_TYPE_UINT64:
691 case FFI_TYPE_SINT64:
692 copy_to_register_or_stack (context, stack, &state,
696 case FFI_TYPE_STRUCT:
699 copy_hfa_to_reg_or_stack (ecif->
avalue[
i], ty, context,
702 else if (ty->size > 16)
708 copy_to_register_or_stack (context, stack, &state,
709 &(ecif->
avalue[
i]), FFI_TYPE_POINTER);
711 else if (available_x (&state) >= (ty->size + 7) / 8)
718 for (j = 0; j < (ty->size + 7) / 8; j++)
720 memcpy (allocate_to_x (context, &state),
721 &(((UINT64 *) ecif->
avalue[
i])[j]),
733 memcpy (allocate_to_stack (&state, stack, ty->alignment,
734 ty->size), ecif->
avalue +
i, ty->size);
743#if defined (__APPLE__)
744 if (
i + 1 == ecif->
cif->aarch64_nfixedargs)
749 state.allocating_variadic = 1;
754 return ecif->
cif->aarch64_flags;
768 cif->aarch64_flags = 0;
770 if (is_v_register_candidate (cif->rtype))
777 for (
i = 0;
i < cif->nargs;
i++)
778 if (is_v_register_candidate (cif->arg_types[
i]))
785#if defined (__APPLE__)
786 cif->aarch64_nfixedargs = 0;
792#if defined (__APPLE__)
796 unsigned int nfixedargs,
797 unsigned int ntotalargs)
803 cif->aarch64_nfixedargs = nfixedargs;
813ffi_call (ffi_cif *cif,
void (*fn)(
void),
void *rvalue,
void **avalue)
832 stack_bytes =
ALIGN(cif->bytes, 16);
834 memset (&context, 0,
sizeof (context));
835 if (is_register_candidate (cif->rtype))
837 ffi_call_SYSV (aarch64_prep_args, &context, &ecif, stack_bytes, fn);
838 switch (cif->rtype->type)
842 case FFI_TYPE_DOUBLE:
843#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
848 case FFI_TYPE_UINT16:
849 case FFI_TYPE_SINT16:
850 case FFI_TYPE_UINT32:
851 case FFI_TYPE_SINT32:
852 case FFI_TYPE_POINTER:
853 case FFI_TYPE_UINT64:
855 case FFI_TYPE_SINT64:
857 void *addr = get_basic_type_addr (cif->rtype->type,
859 copy_basic_type (rvalue, addr, cif->rtype->type);
863 case FFI_TYPE_STRUCT:
864 if (is_hfa (cif->rtype))
867 unsigned short type = get_homogeneous_type (cif->rtype);
868 unsigned elems = element_count (cif->rtype);
869 for (j = 0; j < elems; j++)
871 void *reg = get_basic_type_addr (
type, &context, j);
872 copy_basic_type (rvalue, reg,
type);
873 rvalue += get_basic_type_size (
type);
878 size_t size =
ALIGN (cif->rtype->size, sizeof (UINT64));
879 memcpy (rvalue, get_x_addr (&context, 0),
size);
894 memcpy (get_x_addr (&context, 8), &rvalue,
sizeof (UINT64));
907static unsigned char trampoline [] =
908{ 0x70, 0x00, 0x00, 0x58,
909 0x91, 0x00, 0x00, 0x10,
910 0x00, 0x02, 0x1f, 0xd6
915#define FFI_INIT_TRAMPOLINE(TRAMP,FUN,CTX,FLAGS) \
916 ({unsigned char *__tramp = (unsigned char*)(TRAMP); \
917 UINT64 __fun = (UINT64)(FUN); \
918 UINT64 __ctx = (UINT64)(CTX); \
919 UINT64 __flags = (UINT64)(FLAGS); \
920 memcpy (__tramp, trampoline, sizeof (trampoline)); \
921 memcpy (__tramp + 12, &__fun, sizeof (__fun)); \
922 memcpy (__tramp + 20, &__ctx, sizeof (__ctx)); \
923 memcpy (__tramp + 28, &__flags, sizeof (__flags)); \
924 ffi_clear_cache(__tramp, __tramp + FFI_TRAMPOLINE_SIZE); \
930 void (*fun)(ffi_cif*,
void*,
void**,
void*),
941 closure->user_data = user_data;
967 ffi_cif *cif = closure->cif;
968 void **avalue = (
void**)
alloca (cif->nargs * sizeof (
void*));
973 arg_init (&state,
ALIGN(cif->bytes, 16));
975 for (
i = 0;
i < cif->nargs;
i++)
977 ffi_type *ty = cif->arg_types[
i];
987 case FFI_TYPE_UINT16:
988 case FFI_TYPE_SINT16:
989 case FFI_TYPE_UINT32:
990 case FFI_TYPE_SINT32:
992 case FFI_TYPE_POINTER:
993 case FFI_TYPE_UINT64:
994 case FFI_TYPE_SINT64:
996 case FFI_TYPE_DOUBLE:
997#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
999 avalue[
i] = allocate_to_register_or_stack (context, stack,
1004 case FFI_TYPE_STRUCT:
1007 unsigned n = element_count (ty);
1008 if (available_v (&state) <
n)
1011 avalue[
i] = allocate_to_stack (&state, stack, ty->alignment,
1016 switch (get_homogeneous_type (ty))
1018 case FFI_TYPE_FLOAT:
1031 UINT32 *p = avalue[
i] =
alloca (ty->size);
1032 for (j = 0; j < element_count (ty); j++)
1034 allocate_to_s (context, &state),
1039 case FFI_TYPE_DOUBLE:
1052 UINT64 *p = avalue[
i] =
alloca (ty->size);
1053 for (j = 0; j < element_count (ty); j++)
1055 allocate_to_d (context, &state),
1060#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
1063 allocate_to_v (context, &state),
1074 else if (ty->size > 16)
1079 allocate_to_register_or_stack (context, stack,
1080 &state, FFI_TYPE_POINTER),
1081 sizeof (avalue[
i]));
1083 else if (available_x (&state) >= (ty->size + 7) / 8)
1085 avalue[
i] = get_x_addr (context, state.
ngrn);
1086 state.
ngrn += (ty->size + 7) / 8;
1092 avalue[
i] = allocate_to_stack (&state, stack, ty->alignment,
1107 if (is_register_candidate (cif->rtype))
1115 rvalue =
alloca (cif->rtype->size);
1116 (closure->fun) (cif, rvalue, avalue, closure->user_data);
1120 switch (cif->rtype->type)
1125 case FFI_TYPE_UINT8:
1126 case FFI_TYPE_UINT16:
1127 case FFI_TYPE_UINT32:
1128 case FFI_TYPE_POINTER:
1129 case FFI_TYPE_UINT64:
1130 case FFI_TYPE_SINT8:
1131 case FFI_TYPE_SINT16:
1133 case FFI_TYPE_SINT32:
1134 case FFI_TYPE_SINT64:
1135 case FFI_TYPE_FLOAT:
1136 case FFI_TYPE_DOUBLE:
1137#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
1141 void *addr = get_basic_type_addr (cif->rtype->type, context, 0);
1142 copy_basic_type (addr, rvalue, cif->rtype->type);
1145 case FFI_TYPE_STRUCT:
1146 if (is_hfa (cif->rtype))
1149 unsigned short type = get_homogeneous_type (cif->rtype);
1150 unsigned elems = element_count (cif->rtype);
1151 for (j = 0; j < elems; j++)
1153 void *reg = get_basic_type_addr (
type, context, j);
1154 copy_basic_type (reg, rvalue,
type);
1155 rvalue += get_basic_type_size (
type);
1158 else if ((cif->rtype->size + 7) / 8 <
N_X_ARG_REG)
1160 size_t size =
ALIGN (cif->rtype->size, sizeof (UINT64)) ;
1161 memcpy (get_x_addr (context, 0), rvalue,
size);
1175 memcpy (&rvalue, get_x_addr (context, 8),
sizeof (UINT64));
1176 (closure->fun) (cif, rvalue, avalue, closure->user_data);
void ffi_closure_SYSV(ffi_closure *)
void FFI_HIDDEN ffi_closure_SYSV_inner(ffi_closure *closure, struct call_context *context, void *stack)
ffi_status ffi_prep_closure_loc(ffi_closure *closure, ffi_cif *cif, void(*fun)(ffi_cif *, void *, void **, void *), void *user_data, void *codeloc)
ffi_status ffi_prep_cif_machdep(ffi_cif *cif)
void ffi_call(ffi_cif *cif, void(*fn)(void), void *rvalue, void **avalue)
#define AARCH64_STACK_ALIGN
#define FFI_INIT_TRAMPOLINE(TRAMP, FUN, CTX, FLAGS)
#define AARCH64_FFI_WITH_V
void ffi_call_SYSV(unsigned(*)(struct call_context *context, unsigned char *, extended_cif *), struct call_context *context, extended_cif *, size_t, void(*fn)(void))
#define FFI_TYPE_LONGDOUBLE
ffi_status ffi_prep_cif_machdep_var(ffi_cif *cif, unsigned int nfixedargs, unsigned int ntotalargs)
VALUE type(ANYARGS)
ANYARGS-ed function type.
struct call_context::@83 v[AARCH64_N_VREG]
const UINT64 ffi_template_tramp_tile[] FFI_HIDDEN