13 #include "ruby/internal/config.h"
18 #include "internal/compile.h"
19 #include "internal/hash.h"
20 #include "internal/object.h"
21 #include "internal/variable.h"
24 #include "vm_callinfo.h"
26 #include "vm_insnhelper.h"
30 #include "insns_info.inc"
33 #define NOT_COMPILED_STACK_SIZE -1
34 #define ALREADY_COMPILED_P(status, pos) (status->stack_size_for_pos[pos] != NOT_COMPILED_STACK_SIZE)
37 struct inlined_call_context {
47 struct compile_status {
49 int *stack_size_for_pos;
61 struct rb_mjit_compile_info *compile_info;
62 bool merge_ivar_guards_p;
63 rb_serial_t ivar_serial;
64 size_t max_ivar_index;
67 struct inlined_call_context inline_context;
73 struct compile_branch {
74 unsigned int stack_size;
78 struct case_dispatch_var {
80 unsigned int base_pos;
87 return cd - body->call_data;
96 captured_cc_entries(
const struct compile_status *status)
98 VM_ASSERT(status->cc_entries_index != -1);
99 return mjit_iseq_cc_entries(status->compiled_iseq) + status->cc_entries_index;
106 return vm_cc_cme(cc) != NULL;
113 extern bool rb_vm_opt_cfunc_p(
CALL_CACHE cc,
int insn);
114 return has_valid_method_type(cc) &&
115 !(vm_cc_cme(cc)->def->type == VM_METHOD_TYPE_CFUNC && rb_vm_opt_cfunc_p(cc, insn));
123 extern bool rb_simple_iseq_p(
const rb_iseq_t *iseq);
125 && !(vm_ci_flag(ci) & VM_CALL_KW_SPLAT) && rb_simple_iseq_p(iseq)
126 && vm_ci_argc(ci) == (
unsigned int)iseq->body->
param.lead_num
127 && vm_call_iseq_optimizable_p(ci, cc);
132 maybe_special_const_class_p(
const VALUE klass)
145 struct case_dispatch_var *var = (
struct case_dispatch_var *)arg;
148 if (var->last_value != value) {
150 var->last_value = value;
151 fprintf(var->f,
" case %d:\n", offset);
152 fprintf(var->f,
" goto label_%d;\n", var->base_pos + offset);
153 fprintf(var->f,
" break;\n");
160 comment_id(
FILE *f,
ID id)
162 #ifdef MJIT_COMMENT_ID
173 case '*':
case '/':
if (prev != (c ^ (
'/' ^
'*')))
break;
174 case '\\':
case '"': fputc(
'\\', f);
184 unsigned int pos,
struct compile_status *status);
194 const unsigned int pos,
struct compile_status *status,
struct compile_branch *b)
196 unsigned int next_pos = pos + insn_len(insn);
199 #include "mjit_compile.inc"
204 if (!b->finish_p && next_pos < body->iseq_size && ALREADY_COMPILED_P(status, next_pos)) {
205 fprintf(f,
"goto label_%d;\n", next_pos);
208 if ((
unsigned int)status->stack_size_for_pos[next_pos] != b->stack_size) {
209 if (mjit_opts.warnings || mjit_opts.verbose)
210 fprintf(stderr,
"MJIT warning: JIT stack assumption is not the same between branches (%d != %u)\n",
211 status->stack_size_for_pos[next_pos], b->stack_size);
212 status->success =
false;
223 unsigned int pos,
struct compile_status *status)
225 struct compile_branch branch;
227 branch.stack_size = stack_size;
228 branch.finish_p =
false;
230 while (pos < body->iseq_size && !ALREADY_COMPILED_P(status, pos) && !branch.finish_p) {
231 int insn = rb_vm_insn_decode(body->iseq_encoded[pos]);
232 status->stack_size_for_pos[pos] = (int)branch.stack_size;
234 fprintf(f,
"\nlabel_%d: /* %s */\n", pos, insn_name(insn));
235 pos = compile_insn(f, body, insn, body->iseq_encoded + (pos+1), pos, status, &branch);
236 if (status->success && branch.stack_size > body->stack_max) {
237 if (mjit_opts.warnings || mjit_opts.verbose)
238 fprintf(stderr,
"MJIT warning: JIT stack size (%d) exceeded its max size (%d)\n", branch.stack_size, body->stack_max);
239 status->success =
false;
241 if (!status->success)
248 compile_inlined_cancel_handler(
FILE *f,
const struct rb_iseq_constant_body *body,
struct inlined_call_context *inline_context)
250 fprintf(f,
"\ncancel:\n");
251 fprintf(f,
" RB_DEBUG_COUNTER_INC(mjit_cancel);\n");
252 fprintf(f,
" rb_mjit_recompile_inlining(original_iseq);\n");
255 fprintf(f,
" const VALUE *current_pc = reg_cfp->pc;\n");
256 fprintf(f,
" VALUE *current_sp = reg_cfp->sp;\n");
257 fprintf(f,
" reg_cfp->pc = orig_pc;\n");
258 fprintf(f,
" reg_cfp->sp = orig_sp;\n\n");
261 fprintf(f,
" struct rb_calling_info calling;\n");
262 fprintf(f,
" calling.block_handler = VM_BLOCK_HANDLER_NONE;\n");
263 fprintf(f,
" calling.argc = %d;\n", inline_context->orig_argc);
264 fprintf(f,
" calling.recv = reg_cfp->self;\n");
265 fprintf(f,
" reg_cfp->self = orig_self;\n");
266 fprintf(f,
" vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE
", 0, %d, %d);\n\n",
267 inline_context->me, inline_context->param_size, inline_context->local_size);
270 fprintf(f,
" reg_cfp = ec->cfp;\n");
271 fprintf(f,
" reg_cfp->pc = current_pc;\n");
272 fprintf(f,
" reg_cfp->sp = current_sp;\n");
273 for (
unsigned int i = 0; i < body->stack_max; i++) {
274 fprintf(f,
" *(vm_base_ptr(reg_cfp) + %d) = stack[%d];\n", i, i);
278 fprintf(f,
" return vm_exec(ec, false);\n");
285 if (status->inlined_iseqs == NULL) {
286 compile_inlined_cancel_handler(f, body, &status->inline_context);
290 fprintf(f,
"\nsend_cancel:\n");
291 fprintf(f,
" RB_DEBUG_COUNTER_INC(mjit_cancel_send_inline);\n");
292 fprintf(f,
" rb_mjit_recompile_send(original_iseq);\n");
293 fprintf(f,
" goto cancel;\n");
295 fprintf(f,
"\nivar_cancel:\n");
296 fprintf(f,
" RB_DEBUG_COUNTER_INC(mjit_cancel_ivar_inline);\n");
297 fprintf(f,
" rb_mjit_recompile_ivar(original_iseq);\n");
298 fprintf(f,
" goto cancel;\n");
300 fprintf(f,
"\nexivar_cancel:\n");
301 fprintf(f,
" RB_DEBUG_COUNTER_INC(mjit_cancel_exivar_inline);\n");
302 fprintf(f,
" rb_mjit_recompile_exivar(original_iseq);\n");
303 fprintf(f,
" goto cancel;\n");
305 fprintf(f,
"\nconst_cancel:\n");
306 fprintf(f,
" rb_mjit_recompile_const(original_iseq);\n");
307 fprintf(f,
" goto cancel;\n");
309 fprintf(f,
"\ncancel:\n");
310 fprintf(f,
" RB_DEBUG_COUNTER_INC(mjit_cancel);\n");
311 if (status->local_stack_p) {
312 for (
unsigned int i = 0; i < body->stack_max; i++) {
313 fprintf(f,
" *(vm_base_ptr(reg_cfp) + %d) = stack[%d];\n", i, i);
316 fprintf(f,
" return Qundef;\n");
328 if (is_entries == NULL)
334 mjit_compile_body(
FILE *f,
const rb_iseq_t *iseq,
struct compile_status *status)
337 status->success =
true;
338 status->local_stack_p = !body->catch_except_p;
340 if (status->local_stack_p) {
341 fprintf(f,
" VALUE stack[%d];\n", body->stack_max);
344 fprintf(f,
" VALUE *stack = reg_cfp->sp;\n");
346 if (status->inlined_iseqs != NULL)
347 fprintf(f,
" static const rb_iseq_t *original_iseq = (const rb_iseq_t *)0x%"PRIxVALUE
";\n", (
VALUE)iseq);
348 fprintf(f,
" static const VALUE *const original_body_iseq = (VALUE *)0x%"PRIxVALUE
";\n",
349 (
VALUE)body->iseq_encoded);
350 fprintf(f,
" VALUE cfp_self = reg_cfp->self;\n");
351 fprintf(f,
"#undef GET_SELF\n");
352 fprintf(f,
"#define GET_SELF() cfp_self\n");
355 if (!status->compile_info->disable_ivar_cache && status->merge_ivar_guards_p) {
356 fprintf(f,
" if (UNLIKELY(!(RB_TYPE_P(GET_SELF(), T_OBJECT) && (rb_serial_t)%"PRI_SERIALT_PREFIX
"u == RCLASS_SERIAL(RBASIC(GET_SELF())->klass) &&", status->ivar_serial);
358 fprintf(f,
"%"PRIuSIZE
" < ROBJECT_NUMIV(GET_SELF())", status->max_ivar_index);
361 fprintf(f,
"ROBJECT_EMBED_LEN_MAX == ROBJECT_NUMIV(GET_SELF())");
363 fprintf(f,
"))) {\n");
364 fprintf(f,
" goto ivar_cancel;\n");
370 if (body->
param.flags.has_opt) {
373 fprintf(f,
" switch (reg_cfp->pc - reg_cfp->iseq->body->iseq_encoded) {\n");
374 for (i = 0; i <= body->
param.opt_num; i++) {
376 fprintf(f,
" case %"PRIdVALUE
":\n", pc_offset);
377 fprintf(f,
" goto label_%"PRIdVALUE
";\n", pc_offset);
382 compile_insns(f, body, 0, 0, status);
383 compile_cancel_handler(f, body, status);
384 fprintf(f,
"#undef GET_SELF");
385 return status->success;
400 if (body->catch_except_p)
403 unsigned int pos = 0;
404 while (pos < body->iseq_size) {
405 int insn = rb_vm_insn_decode(body->iseq_encoded[pos]);
411 if (insn == BIN(invokebuiltin) || insn == BIN(opt_invokebuiltin_delegate) || insn == BIN(opt_invokebuiltin_delegate_leave)) {
413 if (!body->builtin_inline_p)
416 else if (insn != BIN(leave) && insn_may_depend_on_sp_or_pc(insn, body->iseq_encoded + (pos + 1)))
421 case BIN(getlocal_WC_0):
422 case BIN(getlocal_WC_1):
424 case BIN(setlocal_WC_0):
425 case BIN(setlocal_WC_1):
426 case BIN(getblockparam):
427 case BIN(getblockparamproxy):
428 case BIN(setblockparam):
431 pos += insn_len(insn);
441 if (has_valid_method_type(cc) &&
442 !(vm_ci_flag(ci) & VM_CALL_TAILCALL) &&
443 vm_cc_cme(cc)->def->type == VM_METHOD_TYPE_ISEQ &&
444 fastpath_applied_iseq_p(ci, cc, iseq = def_iseq_ptr(vm_cc_cme(cc)->def)) &&
446 inlinable_iseq_p(iseq->body)) {
455 mjit_capture_is_entries(body, status->is_entries);
458 unsigned int pos = 0;
459 status->max_ivar_index = 0;
460 status->ivar_serial = 0;
462 while (pos < body->iseq_size) {
463 int insn = rb_vm_insn_decode(body->iseq_encoded[pos]);
464 if (insn == BIN(getinstancevariable) || insn == BIN(setinstancevariable)) {
465 IVC ic = (
IVC)body->iseq_encoded[pos+2];
467 if (ic_copy->entry) {
470 if (status->max_ivar_index < ic_copy->entry->index) {
471 status->max_ivar_index = ic_copy->entry->index;
474 if (status->ivar_serial == 0) {
475 status->ivar_serial = ic_copy->entry->class_serial;
477 else if (status->ivar_serial != ic_copy->entry->class_serial) {
479 status->merge_ivar_guards_p =
false;
484 pos += insn_len(insn);
486 status->merge_ivar_guards_p = status->ivar_serial > 0 && num_ivars >= 2;
490 #define INIT_COMPILE_STATUS(status, body, compile_root_p) do { \
491 status = (struct compile_status){ \
492 .stack_size_for_pos = (int *)alloca(sizeof(int) * body->iseq_size), \
493 .inlined_iseqs = compile_root_p ? \
494 alloca(sizeof(const struct rb_iseq_constant_body *) * body->iseq_size) : NULL, \
495 .is_entries = (body->is_size > 0) ? \
496 alloca(sizeof(union iseq_inline_storage_entry) * body->is_size) : NULL, \
497 .cc_entries_index = (body->ci_size > 0) ? \
498 mjit_capture_cc_entries(status.compiled_iseq, body) : -1, \
499 .compiled_id = status.compiled_id, \
500 .compiled_iseq = status.compiled_iseq, \
501 .compile_info = compile_root_p ? \
502 rb_mjit_iseq_compile_info(body) : alloca(sizeof(struct rb_mjit_compile_info)) \
504 memset(status.stack_size_for_pos, NOT_COMPILED_STACK_SIZE, sizeof(int) * body->iseq_size); \
505 if (compile_root_p) \
506 memset((void *)status.inlined_iseqs, 0, sizeof(const struct rb_iseq_constant_body *) * body->iseq_size); \
508 memset(status.compile_info, 0, sizeof(struct rb_mjit_compile_info)); \
512 precompile_inlinable_child_iseq(
FILE *f,
const rb_iseq_t *child_iseq,
struct compile_status *status,
515 struct compile_status child_status = { .compiled_iseq = status->compiled_iseq, .compiled_id = status->compiled_id };
516 INIT_COMPILE_STATUS(child_status, child_iseq->body,
false);
517 child_status.inline_context = (
struct inlined_call_context){
518 .orig_argc = vm_ci_argc(ci),
519 .me = (
VALUE)vm_cc_cme(cc),
520 .param_size = child_iseq->body->
param.size,
521 .local_size = child_iseq->body->local_table_size
523 if (child_iseq->body->ci_size > 0 && child_status.cc_entries_index == -1) {
526 init_ivar_compile_status(child_iseq->body, &child_status);
528 fprintf(f,
"ALWAYS_INLINE(static VALUE _mjit%d_inlined_%d(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const VALUE orig_self, const rb_iseq_t *original_iseq));\n", status->compiled_id, pos);
529 fprintf(f,
"static inline VALUE\n_mjit%d_inlined_%d(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const VALUE orig_self, const rb_iseq_t *original_iseq)\n{\n", status->compiled_id, pos);
530 fprintf(f,
" const VALUE *orig_pc = reg_cfp->pc;\n");
531 fprintf(f,
" VALUE *orig_sp = reg_cfp->sp;\n");
532 bool success = mjit_compile_body(f, child_iseq, &child_status);
533 fprintf(f,
"\n} /* end of _mjit%d_inlined_%d */\n\n", status->compiled_id, pos);
540 precompile_inlinable_iseqs(
FILE *f,
const rb_iseq_t *iseq,
struct compile_status *status)
543 unsigned int pos = 0;
544 while (pos < body->iseq_size) {
545 int insn = rb_vm_insn_decode(body->iseq_encoded[pos]);
546 if (insn == BIN(opt_send_without_block) || insn == BIN(opt_size)) {
549 const struct rb_callcache *cc = captured_cc_entries(status)[call_data_index(cd, body)];
551 extern bool rb_mjit_compiling_iseq_p(
const rb_iseq_t *iseq);
553 if ((child_iseq = rb_mjit_inlinable_iseq(ci, cc)) != NULL && rb_mjit_compiling_iseq_p(child_iseq)) {
554 status->inlined_iseqs[pos] = child_iseq->body;
556 if (mjit_opts.verbose >= 1)
557 fprintf(stderr,
"JIT inline: %s@%s:%d => %s@%s:%d\n",
559 RSTRING_PTR(rb_iseq_path(child_iseq)),
FIX2INT(child_iseq->body->location.first_lineno),
562 if (!precompile_inlinable_child_iseq(f, child_iseq, status, ci, cc, pos))
566 pos += insn_len(insn);
573 mjit_compile(
FILE *f,
const rb_iseq_t *iseq,
const char *funcname,
int id)
575 struct compile_status status = { .compiled_iseq = iseq->body, .compiled_id =
id };
576 INIT_COMPILE_STATUS(status, iseq->body,
true);
577 if (iseq->body->ci_size > 0 && status.cc_entries_index == -1) {
580 init_ivar_compile_status(iseq->body, &status);
582 if (!status.compile_info->disable_send_cache && !status.compile_info->disable_inlining) {
583 if (!precompile_inlinable_iseqs(f, iseq, &status))
588 fprintf(f,
"__declspec(dllexport)\n");
590 fprintf(f,
"VALUE\n%s(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp)\n{\n", funcname);
591 bool success = mjit_compile_body(f, iseq, &status);
592 fprintf(f,
"\n} // end of %s\n", funcname);
#define FIX2INT
Old name of RB_FIX2INT.
VALUE rb_cInteger
Module class.
VALUE rb_cNilClass
NilClass class.
VALUE rb_cFalseClass
FalseClass class.
VALUE rb_cSymbol
Sumbol class.
VALUE rb_cFloat
Float class.
VALUE rb_cTrueClass
TrueClass class.
VALUE rb_id2str(ID id)
Identical to rb_id2name(), except it returns a Ruby's String instead of C's.
@ ROBJECT_EMBED_LEN_MAX
Max possible number of instance variables that can be embedded.
static char * RSTRING_END(VALUE str)
Queries the end of the contents pointer of the string.
static char * RSTRING_PTR(VALUE str)
Queries the contents pointer of the string.
struct rb_iseq_constant_body::@152 param
parameter information
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
uintptr_t VALUE
Type that represents a Ruby object.