Ruby  3.1.4p223 (2023-03-30 revision HEAD)
mjit_compile.c
1 /**********************************************************************
2 
3  mjit_compile.c - MRI method JIT compiler
4 
5  Copyright (C) 2017 Takashi Kokubun <takashikkbn@gmail.com>.
6 
7 **********************************************************************/
8 
9 // NOTE: All functions in this file are executed on MJIT worker. So don't
10 // call Ruby methods (C functions that may call rb_funcall) or trigger
11 // GC (using ZALLOC, xmalloc, xfree, etc.) in this file.
12 
13 #include "ruby/internal/config.h" // defines USE_MJIT
14 
15 #if USE_MJIT
16 
17 #include "internal.h"
18 #include "internal/compile.h"
19 #include "internal/hash.h"
20 #include "internal/object.h"
21 #include "internal/variable.h"
22 #include "mjit.h"
23 #include "vm_core.h"
24 #include "vm_callinfo.h"
25 #include "vm_exec.h"
26 #include "vm_insnhelper.h"
27 
28 #include "builtin.h"
29 #include "insns.inc"
30 #include "insns_info.inc"
31 
32 // Macros to check if a position is already compiled using compile_status.stack_size_for_pos
33 #define NOT_COMPILED_STACK_SIZE -1
34 #define ALREADY_COMPILED_P(status, pos) (status->stack_size_for_pos[pos] != NOT_COMPILED_STACK_SIZE)
35 
36 // For propagating information needed for lazily pushing a frame.
37 struct inlined_call_context {
38  int orig_argc; // ci->orig_argc
39  VALUE me; // vm_cc_cme(cc)
40  int param_size; // def_iseq_ptr(vm_cc_cme(cc)->def)->body->param.size
41  int local_size; // def_iseq_ptr(vm_cc_cme(cc)->def)->body->local_table_size
42 };
43 
44 // Storage to keep compiler's status. This should have information
45 // which is global during one `mjit_compile` call. Ones conditional
46 // in each branch should be stored in `compile_branch`.
47 struct compile_status {
48  bool success; // has true if compilation has had no issue
49  int *stack_size_for_pos; // stack_size_for_pos[pos] has stack size for the position (otherwise -1)
50  // If true, JIT-ed code will use local variables to store pushed values instead of
51  // using VM's stack and moving stack pointer.
52  bool local_stack_p;
53  // Safely-accessible ivar cache entries copied from main thread.
54  union iseq_inline_storage_entry *is_entries;
55  // Index of call cache entries captured to compiled_iseq to be marked on GC
56  int cc_entries_index;
57  // A pointer to root (i.e. not inlined) iseq being compiled.
58  const struct rb_iseq_constant_body *compiled_iseq;
59  int compiled_id; // Just a copy of compiled_iseq->jit_unit->id
60  // Mutated optimization levels
61  struct rb_mjit_compile_info *compile_info;
62  bool merge_ivar_guards_p; // If true, merge guards of ivar accesses
63  rb_serial_t ivar_serial; // ic_serial of IVC in is_entries (used only when merge_ivar_guards_p)
64  size_t max_ivar_index; // Max IVC index in is_entries (used only when merge_ivar_guards_p)
65  // If `inlined_iseqs[pos]` is not NULL, `mjit_compile_body` tries to inline ISeq there.
66  const struct rb_iseq_constant_body **inlined_iseqs;
67  struct inlined_call_context inline_context;
68 };
69 
70 // Storage to keep data which is consistent in each conditional branch.
71 // This is created and used for one `compile_insns` call and its values
72 // should be copied for extra `compile_insns` call.
73 struct compile_branch {
74  unsigned int stack_size; // this simulates sp (stack pointer) of YARV
75  bool finish_p; // if true, compilation in this branch should stop and let another branch to be compiled
76 };
77 
78 struct case_dispatch_var {
79  FILE *f;
80  unsigned int base_pos;
81  VALUE last_value;
82 };
83 
84 static size_t
85 call_data_index(CALL_DATA cd, const struct rb_iseq_constant_body *body)
86 {
87  return cd - body->call_data;
88 }
89 
90 const struct rb_callcache ** mjit_iseq_cc_entries(const struct rb_iseq_constant_body *const body);
91 
92 // Using this function to refer to cc_entries allocated by `mjit_capture_cc_entries`
93 // instead of storing cc_entries in status directly so that we always refer to a new address
94 // returned by `realloc` inside it.
95 static const struct rb_callcache **
96 captured_cc_entries(const struct compile_status *status)
97 {
98  VM_ASSERT(status->cc_entries_index != -1);
99  return mjit_iseq_cc_entries(status->compiled_iseq) + status->cc_entries_index;
100 }
101 
102 // Returns true if call cache is still not obsoleted and vm_cc_cme(cc)->def->type is available.
103 static bool
104 has_valid_method_type(CALL_CACHE cc)
105 {
106  return vm_cc_cme(cc) != NULL;
107 }
108 
109 // Returns true if MJIT thinks this cc's opt_* insn may fallback to opt_send_without_block.
110 static bool
111 has_cache_for_send(CALL_CACHE cc, int insn)
112 {
113  extern bool rb_vm_opt_cfunc_p(CALL_CACHE cc, int insn);
114  return has_valid_method_type(cc) &&
115  !(vm_cc_cme(cc)->def->type == VM_METHOD_TYPE_CFUNC && rb_vm_opt_cfunc_p(cc, insn));
116 }
117 
118 // Returns true if iseq can use fastpath for setup, otherwise NULL. This becomes true in the same condition
119 // as CC_SET_FASTPATH (in vm_callee_setup_arg) is called from vm_call_iseq_setup.
120 static bool
121 fastpath_applied_iseq_p(const CALL_INFO ci, const CALL_CACHE cc, const rb_iseq_t *iseq)
122 {
123  extern bool rb_simple_iseq_p(const rb_iseq_t *iseq);
124  return iseq != NULL
125  && !(vm_ci_flag(ci) & VM_CALL_KW_SPLAT) && rb_simple_iseq_p(iseq) // Top of vm_callee_setup_arg. In this case, opt_pc is 0.
126  && vm_ci_argc(ci) == (unsigned int)iseq->body->param.lead_num // exclude argument_arity_error (assumption: `calling->argc == ci->orig_argc` in send insns)
127  && vm_call_iseq_optimizable_p(ci, cc); // CC_SET_FASTPATH condition
128 }
129 
130 // Return true if an object of the klass may be a special const. See: rb_class_of
131 static bool
132 maybe_special_const_class_p(const VALUE klass)
133 {
134  return klass == rb_cFalseClass
135  || klass == rb_cNilClass
136  || klass == rb_cTrueClass
137  || klass == rb_cInteger
138  || klass == rb_cSymbol
139  || klass == rb_cFloat;
140 }
141 
142 static int
143 compile_case_dispatch_each(VALUE key, VALUE value, VALUE arg)
144 {
145  struct case_dispatch_var *var = (struct case_dispatch_var *)arg;
146  unsigned int offset;
147 
148  if (var->last_value != value) {
149  offset = FIX2INT(value);
150  var->last_value = value;
151  fprintf(var->f, " case %d:\n", offset);
152  fprintf(var->f, " goto label_%d;\n", var->base_pos + offset);
153  fprintf(var->f, " break;\n");
154  }
155  return ST_CONTINUE;
156 }
157 
158 // Calling rb_id2str in MJIT worker causes random SEGV. So this is disabled by default.
159 static void
160 comment_id(FILE *f, ID id)
161 {
162 #ifdef MJIT_COMMENT_ID
163  VALUE name = rb_id2str(id);
164  const char *p, *e;
165  char c, prev = '\0';
166 
167  if (!name) return;
168  p = RSTRING_PTR(name);
169  e = RSTRING_END(name);
170  fputs("/* :\"", f);
171  for (; p < e; ++p) {
172  switch (c = *p) {
173  case '*': case '/': if (prev != (c ^ ('/' ^ '*'))) break;
174  case '\\': case '"': fputc('\\', f);
175  }
176  fputc(c, f);
177  prev = c;
178  }
179  fputs("\" */", f);
180 #endif
181 }
182 
183 static void compile_insns(FILE *f, const struct rb_iseq_constant_body *body, unsigned int stack_size,
184  unsigned int pos, struct compile_status *status);
185 
186 // Main function of JIT compilation, vm_exec_core counterpart for JIT. Compile one insn to `f`, may modify
187 // b->stack_size and return next position.
188 //
189 // When you add a new instruction to insns.def, it would be nice to have JIT compilation support here but
190 // it's optional. This JIT compiler just ignores ISeq which includes unknown instruction, and ISeq which
191 // does not have it can be compiled as usual.
192 static unsigned int
193 compile_insn(FILE *f, const struct rb_iseq_constant_body *body, const int insn, const VALUE *operands,
194  const unsigned int pos, struct compile_status *status, struct compile_branch *b)
195 {
196  unsigned int next_pos = pos + insn_len(insn);
197 
198 /*****************/
199  #include "mjit_compile.inc"
200 /*****************/
201 
202  // If next_pos is already compiled and this branch is not finished yet,
203  // next instruction won't be compiled in C code next and will need `goto`.
204  if (!b->finish_p && next_pos < body->iseq_size && ALREADY_COMPILED_P(status, next_pos)) {
205  fprintf(f, "goto label_%d;\n", next_pos);
206 
207  // Verify stack size assumption is the same among multiple branches
208  if ((unsigned int)status->stack_size_for_pos[next_pos] != b->stack_size) {
209  if (mjit_opts.warnings || mjit_opts.verbose)
210  fprintf(stderr, "MJIT warning: JIT stack assumption is not the same between branches (%d != %u)\n",
211  status->stack_size_for_pos[next_pos], b->stack_size);
212  status->success = false;
213  }
214  }
215 
216  return next_pos;
217 }
218 
219 // Compile one conditional branch. If it has branchXXX insn, this should be
220 // called multiple times for each branch.
221 static void
222 compile_insns(FILE *f, const struct rb_iseq_constant_body *body, unsigned int stack_size,
223  unsigned int pos, struct compile_status *status)
224 {
225  struct compile_branch branch;
226 
227  branch.stack_size = stack_size;
228  branch.finish_p = false;
229 
230  while (pos < body->iseq_size && !ALREADY_COMPILED_P(status, pos) && !branch.finish_p) {
231  int insn = rb_vm_insn_decode(body->iseq_encoded[pos]);
232  status->stack_size_for_pos[pos] = (int)branch.stack_size;
233 
234  fprintf(f, "\nlabel_%d: /* %s */\n", pos, insn_name(insn));
235  pos = compile_insn(f, body, insn, body->iseq_encoded + (pos+1), pos, status, &branch);
236  if (status->success && branch.stack_size > body->stack_max) {
237  if (mjit_opts.warnings || mjit_opts.verbose)
238  fprintf(stderr, "MJIT warning: JIT stack size (%d) exceeded its max size (%d)\n", branch.stack_size, body->stack_max);
239  status->success = false;
240  }
241  if (!status->success)
242  break;
243  }
244 }
245 
246 // Print the block to cancel inlined method call. It's supporting only `opt_send_without_block` for now.
247 static void
248 compile_inlined_cancel_handler(FILE *f, const struct rb_iseq_constant_body *body, struct inlined_call_context *inline_context)
249 {
250  fprintf(f, "\ncancel:\n");
251  fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel);\n");
252  fprintf(f, " rb_mjit_recompile_inlining(original_iseq);\n");
253 
254  // Swap pc/sp set on cancel with original pc/sp.
255  fprintf(f, " const VALUE *current_pc = reg_cfp->pc;\n");
256  fprintf(f, " VALUE *current_sp = reg_cfp->sp;\n");
257  fprintf(f, " reg_cfp->pc = orig_pc;\n");
258  fprintf(f, " reg_cfp->sp = orig_sp;\n\n");
259 
260  // Lazily push the current call frame.
261  fprintf(f, " struct rb_calling_info calling;\n");
262  fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); // assumes `opt_send_without_block`
263  fprintf(f, " calling.argc = %d;\n", inline_context->orig_argc);
264  fprintf(f, " calling.recv = reg_cfp->self;\n");
265  fprintf(f, " reg_cfp->self = orig_self;\n");
266  fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n\n",
267  inline_context->me, inline_context->param_size, inline_context->local_size); // fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE
268 
269  // Start usual cancel from here.
270  fprintf(f, " reg_cfp = ec->cfp;\n"); // work on the new frame
271  fprintf(f, " reg_cfp->pc = current_pc;\n");
272  fprintf(f, " reg_cfp->sp = current_sp;\n");
273  for (unsigned int i = 0; i < body->stack_max; i++) { // should be always `status->local_stack_p`
274  fprintf(f, " *(vm_base_ptr(reg_cfp) + %d) = stack[%d];\n", i, i);
275  }
276  // We're not just returning Qundef here so that caller's normal cancel handler can
277  // push back `stack` to `cfp->sp`.
278  fprintf(f, " return vm_exec(ec, false);\n");
279 }
280 
281 // Print the block to cancel JIT execution.
282 static void
283 compile_cancel_handler(FILE *f, const struct rb_iseq_constant_body *body, struct compile_status *status)
284 {
285  if (status->inlined_iseqs == NULL) { // the current ISeq is being inlined
286  compile_inlined_cancel_handler(f, body, &status->inline_context);
287  return;
288  }
289 
290  fprintf(f, "\nsend_cancel:\n");
291  fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_send_inline);\n");
292  fprintf(f, " rb_mjit_recompile_send(original_iseq);\n");
293  fprintf(f, " goto cancel;\n");
294 
295  fprintf(f, "\nivar_cancel:\n");
296  fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_ivar_inline);\n");
297  fprintf(f, " rb_mjit_recompile_ivar(original_iseq);\n");
298  fprintf(f, " goto cancel;\n");
299 
300  fprintf(f, "\nexivar_cancel:\n");
301  fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_exivar_inline);\n");
302  fprintf(f, " rb_mjit_recompile_exivar(original_iseq);\n");
303  fprintf(f, " goto cancel;\n");
304 
305  fprintf(f, "\nconst_cancel:\n");
306  fprintf(f, " rb_mjit_recompile_const(original_iseq);\n");
307  fprintf(f, " goto cancel;\n");
308 
309  fprintf(f, "\ncancel:\n");
310  fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel);\n");
311  if (status->local_stack_p) {
312  for (unsigned int i = 0; i < body->stack_max; i++) {
313  fprintf(f, " *(vm_base_ptr(reg_cfp) + %d) = stack[%d];\n", i, i);
314  }
315  }
316  fprintf(f, " return Qundef;\n");
317 }
318 
319 extern int
320 mjit_capture_cc_entries(const struct rb_iseq_constant_body *compiled_iseq, const struct rb_iseq_constant_body *captured_iseq);
321 
322 // Copy current is_entries and use it throughout the current compilation consistently.
323 // While ic->entry has been immutable since https://github.com/ruby/ruby/pull/3662,
324 // we still need this to avoid a race condition between entries and ivar_serial/max_ivar_index.
325 static void
326 mjit_capture_is_entries(const struct rb_iseq_constant_body *body, union iseq_inline_storage_entry *is_entries)
327 {
328  if (is_entries == NULL)
329  return;
330  memcpy(is_entries, body->is_entries, sizeof(union iseq_inline_storage_entry) * body->is_size);
331 }
332 
333 static bool
334 mjit_compile_body(FILE *f, const rb_iseq_t *iseq, struct compile_status *status)
335 {
336  const struct rb_iseq_constant_body *body = iseq->body;
337  status->success = true;
338  status->local_stack_p = !body->catch_except_p;
339 
340  if (status->local_stack_p) {
341  fprintf(f, " VALUE stack[%d];\n", body->stack_max);
342  }
343  else {
344  fprintf(f, " VALUE *stack = reg_cfp->sp;\n");
345  }
346  if (status->inlined_iseqs != NULL) // i.e. compile root
347  fprintf(f, " static const rb_iseq_t *original_iseq = (const rb_iseq_t *)0x%"PRIxVALUE";\n", (VALUE)iseq);
348  fprintf(f, " static const VALUE *const original_body_iseq = (VALUE *)0x%"PRIxVALUE";\n",
349  (VALUE)body->iseq_encoded);
350  fprintf(f, " VALUE cfp_self = reg_cfp->self;\n"); // cache self across the method
351  fprintf(f, "#undef GET_SELF\n");
352  fprintf(f, "#define GET_SELF() cfp_self\n");
353 
354  // Generate merged ivar guards first if needed
355  if (!status->compile_info->disable_ivar_cache && status->merge_ivar_guards_p) {
356  fprintf(f, " if (UNLIKELY(!(RB_TYPE_P(GET_SELF(), T_OBJECT) && (rb_serial_t)%"PRI_SERIALT_PREFIX"u == RCLASS_SERIAL(RBASIC(GET_SELF())->klass) &&", status->ivar_serial);
357  if (status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX) {
358  fprintf(f, "%"PRIuSIZE" < ROBJECT_NUMIV(GET_SELF())", status->max_ivar_index); // index < ROBJECT_NUMIV(obj) && !RB_FL_ANY_RAW(obj, ROBJECT_EMBED)
359  }
360  else {
361  fprintf(f, "ROBJECT_EMBED_LEN_MAX == ROBJECT_NUMIV(GET_SELF())"); // index < ROBJECT_NUMIV(obj) && RB_FL_ANY_RAW(obj, ROBJECT_EMBED)
362  }
363  fprintf(f, "))) {\n");
364  fprintf(f, " goto ivar_cancel;\n");
365  fprintf(f, " }\n");
366  }
367 
368  // Simulate `opt_pc` in setup_parameters_complex. Other PCs which may be passed by catch tables
369  // are not considered since vm_exec doesn't call mjit_exec for catch tables.
370  if (body->param.flags.has_opt) {
371  int i;
372  fprintf(f, "\n");
373  fprintf(f, " switch (reg_cfp->pc - reg_cfp->iseq->body->iseq_encoded) {\n");
374  for (i = 0; i <= body->param.opt_num; i++) {
375  VALUE pc_offset = body->param.opt_table[i];
376  fprintf(f, " case %"PRIdVALUE":\n", pc_offset);
377  fprintf(f, " goto label_%"PRIdVALUE";\n", pc_offset);
378  }
379  fprintf(f, " }\n");
380  }
381 
382  compile_insns(f, body, 0, 0, status);
383  compile_cancel_handler(f, body, status);
384  fprintf(f, "#undef GET_SELF");
385  return status->success;
386 }
387 
388 // Return true if the ISeq can be inlined without pushing a new control frame.
389 static bool
390 inlinable_iseq_p(const struct rb_iseq_constant_body *body)
391 {
392  // 1) If catch_except_p, caller frame should be preserved when callee catches an exception.
393  // Then we need to wrap `vm_exec()` but then we can't inline the call inside it.
394  //
395  // 2) If `body->catch_except_p` is false and `handles_sp?` of an insn is false,
396  // sp is not moved as we assume `status->local_stack_p = !body->catch_except_p`.
397  //
398  // 3) If `body->catch_except_p` is false and `always_leaf?` of an insn is true,
399  // pc is not moved.
400  if (body->catch_except_p)
401  return false;
402 
403  unsigned int pos = 0;
404  while (pos < body->iseq_size) {
405  int insn = rb_vm_insn_decode(body->iseq_encoded[pos]);
406  // All insns in the ISeq except `leave` (to be overridden in the inlined code)
407  // should meet following strong assumptions:
408  // * Do not require `cfp->sp` motion
409  // * Do not move `cfp->pc`
410  // * Do not read any `cfp->pc`
411  if (insn == BIN(invokebuiltin) || insn == BIN(opt_invokebuiltin_delegate) || insn == BIN(opt_invokebuiltin_delegate_leave)) {
412  // builtin insn's inlinability is handled by `Primitive.attr! 'inline'` per iseq
413  if (!body->builtin_inline_p)
414  return false;
415  }
416  else if (insn != BIN(leave) && insn_may_depend_on_sp_or_pc(insn, body->iseq_encoded + (pos + 1)))
417  return false;
418  // At this moment, `cfp->ep` in an inlined method is not working.
419  switch (insn) {
420  case BIN(getlocal):
421  case BIN(getlocal_WC_0):
422  case BIN(getlocal_WC_1):
423  case BIN(setlocal):
424  case BIN(setlocal_WC_0):
425  case BIN(setlocal_WC_1):
426  case BIN(getblockparam):
427  case BIN(getblockparamproxy):
428  case BIN(setblockparam):
429  return false;
430  }
431  pos += insn_len(insn);
432  }
433  return true;
434 }
435 
436 // Return an iseq pointer if cc has inlinable iseq.
437 const rb_iseq_t *
438 rb_mjit_inlinable_iseq(const struct rb_callinfo *ci, const struct rb_callcache *cc)
439 {
440  const rb_iseq_t *iseq;
441  if (has_valid_method_type(cc) &&
442  !(vm_ci_flag(ci) & VM_CALL_TAILCALL) && // inlining only non-tailcall path
443  vm_cc_cme(cc)->def->type == VM_METHOD_TYPE_ISEQ &&
444  fastpath_applied_iseq_p(ci, cc, iseq = def_iseq_ptr(vm_cc_cme(cc)->def)) &&
445  // CC_SET_FASTPATH in vm_callee_setup_arg
446  inlinable_iseq_p(iseq->body)) {
447  return iseq;
448  }
449  return NULL;
450 }
451 
452 static void
453 init_ivar_compile_status(const struct rb_iseq_constant_body *body, struct compile_status *status)
454 {
455  mjit_capture_is_entries(body, status->is_entries);
456 
457  int num_ivars = 0;
458  unsigned int pos = 0;
459  status->max_ivar_index = 0;
460  status->ivar_serial = 0;
461 
462  while (pos < body->iseq_size) {
463  int insn = rb_vm_insn_decode(body->iseq_encoded[pos]);
464  if (insn == BIN(getinstancevariable) || insn == BIN(setinstancevariable)) {
465  IVC ic = (IVC)body->iseq_encoded[pos+2];
466  IVC ic_copy = &(status->is_entries + ((union iseq_inline_storage_entry *)ic - body->is_entries))->iv_cache;
467  if (ic_copy->entry) { // Only initialized (ic_serial > 0) IVCs are optimized
468  num_ivars++;
469 
470  if (status->max_ivar_index < ic_copy->entry->index) {
471  status->max_ivar_index = ic_copy->entry->index;
472  }
473 
474  if (status->ivar_serial == 0) {
475  status->ivar_serial = ic_copy->entry->class_serial;
476  }
477  else if (status->ivar_serial != ic_copy->entry->class_serial) {
478  // Multiple classes have used this ISeq. Give up assuming one serial.
479  status->merge_ivar_guards_p = false;
480  return;
481  }
482  }
483  }
484  pos += insn_len(insn);
485  }
486  status->merge_ivar_guards_p = status->ivar_serial > 0 && num_ivars >= 2;
487 }
488 
489 // This needs to be macro instead of a function because it's using `alloca`.
490 #define INIT_COMPILE_STATUS(status, body, compile_root_p) do { \
491  status = (struct compile_status){ \
492  .stack_size_for_pos = (int *)alloca(sizeof(int) * body->iseq_size), \
493  .inlined_iseqs = compile_root_p ? \
494  alloca(sizeof(const struct rb_iseq_constant_body *) * body->iseq_size) : NULL, \
495  .is_entries = (body->is_size > 0) ? \
496  alloca(sizeof(union iseq_inline_storage_entry) * body->is_size) : NULL, \
497  .cc_entries_index = (body->ci_size > 0) ? \
498  mjit_capture_cc_entries(status.compiled_iseq, body) : -1, \
499  .compiled_id = status.compiled_id, \
500  .compiled_iseq = status.compiled_iseq, \
501  .compile_info = compile_root_p ? \
502  rb_mjit_iseq_compile_info(body) : alloca(sizeof(struct rb_mjit_compile_info)) \
503  }; \
504  memset(status.stack_size_for_pos, NOT_COMPILED_STACK_SIZE, sizeof(int) * body->iseq_size); \
505  if (compile_root_p) \
506  memset((void *)status.inlined_iseqs, 0, sizeof(const struct rb_iseq_constant_body *) * body->iseq_size); \
507  else \
508  memset(status.compile_info, 0, sizeof(struct rb_mjit_compile_info)); \
509 } while (0)
510 
511 static bool
512 precompile_inlinable_child_iseq(FILE *f, const rb_iseq_t *child_iseq, struct compile_status *status,
513  const struct rb_callinfo *ci, const struct rb_callcache *cc, unsigned int pos)
514 {
515  struct compile_status child_status = { .compiled_iseq = status->compiled_iseq, .compiled_id = status->compiled_id };
516  INIT_COMPILE_STATUS(child_status, child_iseq->body, false);
517  child_status.inline_context = (struct inlined_call_context){
518  .orig_argc = vm_ci_argc(ci),
519  .me = (VALUE)vm_cc_cme(cc),
520  .param_size = child_iseq->body->param.size,
521  .local_size = child_iseq->body->local_table_size
522  };
523  if (child_iseq->body->ci_size > 0 && child_status.cc_entries_index == -1) {
524  return false;
525  }
526  init_ivar_compile_status(child_iseq->body, &child_status);
527 
528  fprintf(f, "ALWAYS_INLINE(static VALUE _mjit%d_inlined_%d(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const VALUE orig_self, const rb_iseq_t *original_iseq));\n", status->compiled_id, pos);
529  fprintf(f, "static inline VALUE\n_mjit%d_inlined_%d(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const VALUE orig_self, const rb_iseq_t *original_iseq)\n{\n", status->compiled_id, pos);
530  fprintf(f, " const VALUE *orig_pc = reg_cfp->pc;\n");
531  fprintf(f, " VALUE *orig_sp = reg_cfp->sp;\n");
532  bool success = mjit_compile_body(f, child_iseq, &child_status);
533  fprintf(f, "\n} /* end of _mjit%d_inlined_%d */\n\n", status->compiled_id, pos);
534 
535  return success;
536 }
537 
538 // Compile inlinable ISeqs to C code in `f`. It returns true if it succeeds to compile them.
539 static bool
540 precompile_inlinable_iseqs(FILE *f, const rb_iseq_t *iseq, struct compile_status *status)
541 {
542  const struct rb_iseq_constant_body *body = iseq->body;
543  unsigned int pos = 0;
544  while (pos < body->iseq_size) {
545  int insn = rb_vm_insn_decode(body->iseq_encoded[pos]);
546  if (insn == BIN(opt_send_without_block) || insn == BIN(opt_size)) { // `compile_inlined_cancel_handler` supports only `opt_send_without_block`
547  CALL_DATA cd = (CALL_DATA)body->iseq_encoded[pos + 1];
548  const struct rb_callinfo *ci = cd->ci;
549  const struct rb_callcache *cc = captured_cc_entries(status)[call_data_index(cd, body)]; // use copy to avoid race condition
550 
551  extern bool rb_mjit_compiling_iseq_p(const rb_iseq_t *iseq);
552  const rb_iseq_t *child_iseq;
553  if ((child_iseq = rb_mjit_inlinable_iseq(ci, cc)) != NULL && rb_mjit_compiling_iseq_p(child_iseq)) {
554  status->inlined_iseqs[pos] = child_iseq->body;
555 
556  if (mjit_opts.verbose >= 1) // print beforehand because ISeq may be GCed during copy job.
557  fprintf(stderr, "JIT inline: %s@%s:%d => %s@%s:%d\n",
558  RSTRING_PTR(child_iseq->body->location.label),
559  RSTRING_PTR(rb_iseq_path(child_iseq)), FIX2INT(child_iseq->body->location.first_lineno),
560  RSTRING_PTR(iseq->body->location.label),
561  RSTRING_PTR(rb_iseq_path(iseq)), FIX2INT(iseq->body->location.first_lineno));
562  if (!precompile_inlinable_child_iseq(f, child_iseq, status, ci, cc, pos))
563  return false;
564  }
565  }
566  pos += insn_len(insn);
567  }
568  return true;
569 }
570 
571 // Compile ISeq to C code in `f`. It returns true if it succeeds to compile.
572 bool
573 mjit_compile(FILE *f, const rb_iseq_t *iseq, const char *funcname, int id)
574 {
575  struct compile_status status = { .compiled_iseq = iseq->body, .compiled_id = id };
576  INIT_COMPILE_STATUS(status, iseq->body, true);
577  if (iseq->body->ci_size > 0 && status.cc_entries_index == -1) {
578  return false;
579  }
580  init_ivar_compile_status(iseq->body, &status);
581 
582  if (!status.compile_info->disable_send_cache && !status.compile_info->disable_inlining) {
583  if (!precompile_inlinable_iseqs(f, iseq, &status))
584  return false;
585  }
586 
587 #ifdef _WIN32
588  fprintf(f, "__declspec(dllexport)\n");
589 #endif
590  fprintf(f, "VALUE\n%s(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp)\n{\n", funcname);
591  bool success = mjit_compile_body(f, iseq, &status);
592  fprintf(f, "\n} // end of %s\n", funcname);
593  return success;
594 }
595 
596 #endif // USE_MJIT
#define FIX2INT
Old name of RB_FIX2INT.
Definition: int.h:41
VALUE rb_cInteger
Module class.
Definition: numeric.c:192
VALUE rb_cNilClass
NilClass class.
Definition: object.c:55
VALUE rb_cFalseClass
FalseClass class.
Definition: object.c:57
VALUE rb_cSymbol
Sumbol class.
Definition: string.c:81
VALUE rb_cFloat
Float class.
Definition: numeric.c:191
VALUE rb_cTrueClass
TrueClass class.
Definition: object.c:56
VALUE rb_id2str(ID id)
Identical to rb_id2name(), except it returns a Ruby's String instead of C's.
Definition: symbol.c:935
@ ROBJECT_EMBED_LEN_MAX
Max possible number of instance variables that can be embedded.
Definition: robject.h:84
static char * RSTRING_END(VALUE str)
Queries the end of the contents pointer of the string.
Definition: rstring.h:527
static char * RSTRING_PTR(VALUE str)
Queries the contents pointer of the string.
Definition: rstring.h:497
Definition: vm_core.h:273
struct rb_iseq_constant_body::@152 param
parameter information
Definition: vm_core.h:281
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
Definition: value.h:52
uintptr_t VALUE
Type that represents a Ruby object.
Definition: value.h:40