Ruby  3.1.4p223 (2023-03-30 revision HEAD)
mjit.c
1 /**********************************************************************
2 
3  mjit.c - MRI method JIT compiler functions for Ruby's main thread
4 
5  Copyright (C) 2017 Vladimir Makarov <vmakarov@redhat.com>.
6 
7 **********************************************************************/
8 
9 // Functions in this file are never executed on MJIT worker thread.
10 // So you can safely use Ruby methods and GC in this file.
11 
12 // To share variables privately, include mjit_worker.c instead of linking.
13 
14 #include "ruby/internal/config.h" // defines USE_MJIT
15 
16 #if USE_MJIT
17 
18 #include "constant.h"
19 #include "id_table.h"
20 #include "internal.h"
21 #include "internal/class.h"
22 #include "internal/cont.h"
23 #include "internal/file.h"
24 #include "internal/hash.h"
25 #include "internal/warnings.h"
26 #include "vm_sync.h"
27 
28 #include "mjit_worker.c"
29 
30 extern int rb_thread_create_mjit_thread(void (*worker_func)(void));
31 
32 // Return an unique file name in /tmp with PREFIX and SUFFIX and
33 // number ID. Use getpid if ID == 0. The return file name exists
34 // until the next function call.
35 static char *
36 get_uniq_filename(unsigned long id, const char *prefix, const char *suffix)
37 {
38  char buff[70], *str = buff;
39  int size = sprint_uniq_filename(buff, sizeof(buff), id, prefix, suffix);
40  str = 0;
41  ++size;
42  str = xmalloc(size);
43  if (size <= (int)sizeof(buff)) {
44  memcpy(str, buff, size);
45  }
46  else {
47  sprint_uniq_filename(str, size, id, prefix, suffix);
48  }
49  return str;
50 }
51 
52 // Wait until workers don't compile any iseq. It is called at the
53 // start of GC.
54 void
55 mjit_gc_start_hook(void)
56 {
57  if (!mjit_enabled)
58  return;
59  CRITICAL_SECTION_START(4, "mjit_gc_start_hook");
60  while (in_jit) {
61  verbose(4, "Waiting wakeup from a worker for GC");
62  rb_native_cond_wait(&mjit_client_wakeup, &mjit_engine_mutex);
63  verbose(4, "Getting wakeup from a worker for GC");
64  }
65  in_gc++;
66  CRITICAL_SECTION_FINISH(4, "mjit_gc_start_hook");
67 }
68 
69 // Send a signal to workers to continue iseq compilations. It is
70 // called at the end of GC.
71 void
72 mjit_gc_exit_hook(void)
73 {
74  if (!mjit_enabled)
75  return;
76  CRITICAL_SECTION_START(4, "mjit_gc_exit_hook");
77  in_gc--;
78  RUBY_ASSERT_ALWAYS(in_gc >= 0);
79  if (!in_gc) {
80  verbose(4, "Sending wakeup signal to workers after GC");
81  rb_native_cond_broadcast(&mjit_gc_wakeup);
82  }
83  CRITICAL_SECTION_FINISH(4, "mjit_gc_exit_hook");
84 }
85 
86 // Prohibit calling JIT-ed code and let existing JIT-ed frames exit before the next insn.
87 void
88 mjit_cancel_all(const char *reason)
89 {
90  if (!mjit_enabled)
91  return;
92 
93  mjit_call_p = false;
94  if (mjit_opts.warnings || mjit_opts.verbose) {
95  fprintf(stderr, "JIT cancel: Disabled JIT-ed code because %s\n", reason);
96  }
97 }
98 
99 // Deal with ISeq movement from compactor
100 void
101 mjit_update_references(const rb_iseq_t *iseq)
102 {
103  if (!mjit_enabled)
104  return;
105 
106  CRITICAL_SECTION_START(4, "mjit_update_references");
107  if (iseq->body->jit_unit) {
108  iseq->body->jit_unit->iseq = (rb_iseq_t *)rb_gc_location((VALUE)iseq->body->jit_unit->iseq);
109  // We need to invalidate JIT-ed code for the ISeq because it embeds pointer addresses.
110  // To efficiently do that, we use the same thing as TracePoint and thus everything is cancelled for now.
111  // See mjit.h and tool/ruby_vm/views/_mjit_compile_insn.erb for how `mjit_call_p` is used.
112  mjit_cancel_all("GC.compact is used"); // TODO: instead of cancelling all, invalidate only this one and recompile it with some threshold.
113  }
114 
115  // Units in stale_units (list of over-speculated and invalidated code) are not referenced from
116  // `iseq->body->jit_unit` anymore (because new one replaces that). So we need to check them too.
117  // TODO: we should be able to reduce the number of units checked here.
118  struct rb_mjit_unit *unit = NULL;
119  list_for_each(&stale_units.head, unit, unode) {
120  if (unit->iseq == iseq) {
121  unit->iseq = (rb_iseq_t *)rb_gc_location((VALUE)unit->iseq);
122  }
123  }
124  CRITICAL_SECTION_FINISH(4, "mjit_update_references");
125 }
126 
127 // Iseqs can be garbage collected. This function should call when it
128 // happens. It removes iseq from the unit.
129 void
130 mjit_free_iseq(const rb_iseq_t *iseq)
131 {
132  if (!mjit_enabled)
133  return;
134 
135  CRITICAL_SECTION_START(4, "mjit_free_iseq");
136  RUBY_ASSERT_ALWAYS(in_gc);
137  RUBY_ASSERT_ALWAYS(!in_jit);
138  if (iseq->body->jit_unit) {
139  // jit_unit is not freed here because it may be referred by multiple
140  // lists of units. `get_from_list` and `mjit_finish` do the job.
141  iseq->body->jit_unit->iseq = NULL;
142  }
143  // Units in stale_units (list of over-speculated and invalidated code) are not referenced from
144  // `iseq->body->jit_unit` anymore (because new one replaces that). So we need to check them too.
145  // TODO: we should be able to reduce the number of units checked here.
146  struct rb_mjit_unit *unit = NULL;
147  list_for_each(&stale_units.head, unit, unode) {
148  if (unit->iseq == iseq) {
149  unit->iseq = NULL;
150  }
151  }
152  CRITICAL_SECTION_FINISH(4, "mjit_free_iseq");
153 }
154 
155 // Free unit list. This should be called only when worker is finished
156 // because node of unit_queue and one of active_units may have the same unit
157 // during proceeding unit.
158 static void
159 free_list(struct rb_mjit_unit_list *list, bool close_handle_p)
160 {
161  struct rb_mjit_unit *unit = 0, *next;
162 
163  list_for_each_safe(&list->head, unit, next, unode) {
164  list_del(&unit->unode);
165  if (!close_handle_p) unit->handle = NULL; /* Skip dlclose in free_unit() */
166 
167  if (list == &stale_units) { // `free_unit(unit)` crashes after GC.compact on `stale_units`
168  /*
169  * TODO: REVERT THIS BRANCH
170  * Debug the crash on stale_units w/ GC.compact and just use `free_unit(unit)`!!
171  */
172  if (unit->handle && dlclose(unit->handle)) {
173  mjit_warning("failed to close handle for u%d: %s", unit->id, dlerror());
174  }
175  clean_temp_files(unit);
176  free(unit);
177  }
178  else {
179  free_unit(unit);
180  }
181  }
182  list->length = 0;
183 }
184 
185 // Register a new continuation with execution context `ec`. Return MJIT info about
186 // the continuation.
187 struct mjit_cont *
188 mjit_cont_new(rb_execution_context_t *ec)
189 {
190  struct mjit_cont *cont;
191 
192  // We need to use calloc instead of something like ZALLOC to avoid triggering GC here.
193  // When this function is called from rb_thread_alloc through rb_threadptr_root_fiber_setup,
194  // the thread is still being prepared and marking it causes SEGV.
195  cont = calloc(1, sizeof(struct mjit_cont));
196  if (cont == NULL)
197  rb_memerror();
198  cont->ec = ec;
199 
200  CRITICAL_SECTION_START(3, "in mjit_cont_new");
201  if (first_cont == NULL) {
202  cont->next = cont->prev = NULL;
203  }
204  else {
205  cont->prev = NULL;
206  cont->next = first_cont;
207  first_cont->prev = cont;
208  }
209  first_cont = cont;
210  CRITICAL_SECTION_FINISH(3, "in mjit_cont_new");
211 
212  return cont;
213 }
214 
215 // Unregister continuation `cont`.
216 void
217 mjit_cont_free(struct mjit_cont *cont)
218 {
219  CRITICAL_SECTION_START(3, "in mjit_cont_new");
220  if (cont == first_cont) {
221  first_cont = cont->next;
222  if (first_cont != NULL)
223  first_cont->prev = NULL;
224  }
225  else {
226  cont->prev->next = cont->next;
227  if (cont->next != NULL)
228  cont->next->prev = cont->prev;
229  }
230  CRITICAL_SECTION_FINISH(3, "in mjit_cont_new");
231 
232  free(cont);
233 }
234 
235 // Finish work with continuation info.
236 static void
237 finish_conts(void)
238 {
239  struct mjit_cont *cont, *next;
240 
241  for (cont = first_cont; cont != NULL; cont = next) {
242  next = cont->next;
243  xfree(cont);
244  }
245 }
246 
247 // Create unit for `iseq`. This function may be called from an MJIT worker.
248 static void
249 create_unit(const rb_iseq_t *iseq)
250 {
251  struct rb_mjit_unit *unit;
252 
253  unit = calloc(1, sizeof(struct rb_mjit_unit));
254  if (unit == NULL)
255  return;
256 
257  unit->id = current_unit_num++;
258  unit->iseq = (rb_iseq_t *)iseq;
259  iseq->body->jit_unit = unit;
260 }
261 
262 // Return true if given ISeq body should be compiled by MJIT
263 static inline int
264 mjit_target_iseq_p(struct rb_iseq_constant_body *body)
265 {
266  return (body->type == ISEQ_TYPE_METHOD || body->type == ISEQ_TYPE_BLOCK)
267  && !body->builtin_inline_p;
268 }
269 
270 // If recompile_p is true, the call is initiated by mjit_recompile.
271 // This assumes the caller holds CRITICAL_SECTION when recompile_p is true.
272 static void
273 mjit_add_iseq_to_process(const rb_iseq_t *iseq, const struct rb_mjit_compile_info *compile_info, bool recompile_p)
274 {
275  if (!mjit_enabled || pch_status == PCH_FAILED)
276  return;
277  if (!mjit_target_iseq_p(iseq->body)) {
278  iseq->body->jit_func = (mjit_func_t)NOT_COMPILED_JIT_ISEQ_FUNC; // skip mjit_wait
279  return;
280  }
281 
282  if (!recompile_p) {
283  CRITICAL_SECTION_START(3, "in add_iseq_to_process");
284 
285  // This prevents multiple Ractors from enqueueing the same ISeq twice.
286  if (rb_multi_ractor_p() && (uintptr_t)iseq->body->jit_func != NOT_ADDED_JIT_ISEQ_FUNC) {
287  CRITICAL_SECTION_FINISH(3, "in add_iseq_to_process");
288  return;
289  }
290  }
291 
292  RB_DEBUG_COUNTER_INC(mjit_add_iseq_to_process);
293  iseq->body->jit_func = (mjit_func_t)NOT_READY_JIT_ISEQ_FUNC;
294  create_unit(iseq);
295  if (iseq->body->jit_unit == NULL)
296  // Failure in creating the unit.
297  return;
298  if (compile_info != NULL)
299  iseq->body->jit_unit->compile_info = *compile_info;
300  add_to_list(iseq->body->jit_unit, &unit_queue);
301  if (active_units.length >= mjit_opts.max_cache_size) {
302  unload_requests++;
303  }
304 
305  if (!recompile_p) {
306  verbose(3, "Sending wakeup signal to workers in mjit_add_iseq_to_process");
307  rb_native_cond_broadcast(&mjit_worker_wakeup);
308  CRITICAL_SECTION_FINISH(3, "in add_iseq_to_process");
309  }
310 }
311 
312 // Add ISEQ to be JITed in parallel with the current thread.
313 // Unload some JIT codes if there are too many of them.
314 void
315 rb_mjit_add_iseq_to_process(const rb_iseq_t *iseq)
316 {
317  mjit_add_iseq_to_process(iseq, NULL, false);
318 }
319 
320 // For this timeout seconds, --jit-wait will wait for JIT compilation finish.
321 #define MJIT_WAIT_TIMEOUT_SECONDS 60
322 
323 static void
324 mjit_wait(struct rb_iseq_constant_body *body)
325 {
326  struct timeval tv;
327  int tries = 0;
328  tv.tv_sec = 0;
329  tv.tv_usec = 1000;
330  while (body->jit_func == (mjit_func_t)NOT_READY_JIT_ISEQ_FUNC) {
331  tries++;
332  if (tries / 1000 > MJIT_WAIT_TIMEOUT_SECONDS || pch_status == PCH_FAILED) {
333  CRITICAL_SECTION_START(3, "in rb_mjit_wait_call to set jit_func");
334  body->jit_func = (mjit_func_t)NOT_COMPILED_JIT_ISEQ_FUNC; // JIT worker seems dead. Give up.
335  CRITICAL_SECTION_FINISH(3, "in rb_mjit_wait_call to set jit_func");
336  mjit_warning("timed out to wait for JIT finish");
337  break;
338  }
339 
340  CRITICAL_SECTION_START(3, "in rb_mjit_wait_call for a client wakeup");
341  rb_native_cond_broadcast(&mjit_worker_wakeup);
342  CRITICAL_SECTION_FINISH(3, "in rb_mjit_wait_call for a client wakeup");
343  rb_thread_wait_for(tv);
344  }
345 }
346 
347 // Wait for JIT compilation finish for --jit-wait, and call the function pointer
348 // if the compiled result is not NOT_COMPILED_JIT_ISEQ_FUNC.
349 VALUE
350 rb_mjit_wait_call(rb_execution_context_t *ec, struct rb_iseq_constant_body *body)
351 {
352  if (worker_stopped)
353  return Qundef;
354 
355  mjit_wait(body);
356  if ((uintptr_t)body->jit_func <= (uintptr_t)LAST_JIT_ISEQ_FUNC) {
357  return Qundef;
358  }
359  return body->jit_func(ec, ec->cfp);
360 }
361 
362 struct rb_mjit_compile_info*
363 rb_mjit_iseq_compile_info(const struct rb_iseq_constant_body *body)
364 {
365  assert(body->jit_unit != NULL);
366  return &body->jit_unit->compile_info;
367 }
368 
369 static void
370 mjit_recompile(const rb_iseq_t *iseq)
371 {
372  if ((uintptr_t)iseq->body->jit_func <= (uintptr_t)LAST_JIT_ISEQ_FUNC)
373  return;
374 
375  verbose(1, "JIT recompile: %s@%s:%d", RSTRING_PTR(iseq->body->location.label),
376  RSTRING_PTR(rb_iseq_path(iseq)), FIX2INT(iseq->body->location.first_lineno));
377  assert(iseq->body->jit_unit != NULL);
378 
379  if (UNLIKELY(mjit_opts.wait)) {
380  CRITICAL_SECTION_START(3, "in rb_mjit_recompile_iseq");
381  remove_from_list(iseq->body->jit_unit, &active_units);
382  add_to_list(iseq->body->jit_unit, &stale_units);
383  mjit_add_iseq_to_process(iseq, &iseq->body->jit_unit->compile_info, true);
384  CRITICAL_SECTION_FINISH(3, "in rb_mjit_recompile_iseq");
385  mjit_wait(iseq->body);
386  }
387  else {
388  // Lazily move active_units to stale_units to avoid race conditions around active_units with compaction.
389  // Also, it's lazily moved to unit_queue as well because otherwise it won't be added to stale_units properly.
390  // It's good to avoid a race condition between mjit_add_iseq_to_process and mjit_compile around jit_unit as well.
391  CRITICAL_SECTION_START(3, "in rb_mjit_recompile_iseq");
392  iseq->body->jit_unit->stale_p = true;
393  iseq->body->jit_func = (mjit_func_t)NOT_READY_JIT_ISEQ_FUNC;
394  pending_stale_p = true;
395  CRITICAL_SECTION_FINISH(3, "in rb_mjit_recompile_iseq");
396  }
397 }
398 
399 // Recompile iseq, disabling send optimization
400 void
401 rb_mjit_recompile_send(const rb_iseq_t *iseq)
402 {
403  rb_mjit_iseq_compile_info(iseq->body)->disable_send_cache = true;
404  mjit_recompile(iseq);
405 }
406 
407 // Recompile iseq, disabling ivar optimization
408 void
409 rb_mjit_recompile_ivar(const rb_iseq_t *iseq)
410 {
411  rb_mjit_iseq_compile_info(iseq->body)->disable_ivar_cache = true;
412  mjit_recompile(iseq);
413 }
414 
415 // Recompile iseq, disabling exivar optimization
416 void
417 rb_mjit_recompile_exivar(const rb_iseq_t *iseq)
418 {
419  rb_mjit_iseq_compile_info(iseq->body)->disable_exivar_cache = true;
420  mjit_recompile(iseq);
421 }
422 
423 // Recompile iseq, disabling method inlining
424 void
425 rb_mjit_recompile_inlining(const rb_iseq_t *iseq)
426 {
427  rb_mjit_iseq_compile_info(iseq->body)->disable_inlining = true;
428  mjit_recompile(iseq);
429 }
430 
431 // Recompile iseq, disabling getconstant inlining
432 void
433 rb_mjit_recompile_const(const rb_iseq_t *iseq)
434 {
435  rb_mjit_iseq_compile_info(iseq->body)->disable_const_cache = true;
436  mjit_recompile(iseq);
437 }
438 
439 extern VALUE ruby_archlibdir_path, ruby_prefix_path;
440 
441 // Initialize header_file, pch_file, libruby_pathflag. Return true on success.
442 static bool
443 init_header_filename(void)
444 {
445  int fd;
446 #ifdef LOAD_RELATIVE
447  // Root path of the running ruby process. Equal to RbConfig::TOPDIR.
448  VALUE basedir_val;
449 #endif
450  const char *basedir = "";
451  size_t baselen = 0;
452  char *p;
453 #ifdef _WIN32
454  static const char libpathflag[] =
455 # ifdef _MSC_VER
456  "-LIBPATH:"
457 # else
458  "-L"
459 # endif
460  ;
461  const size_t libpathflag_len = sizeof(libpathflag) - 1;
462 #endif
463 
464 #ifdef LOAD_RELATIVE
465  basedir_val = ruby_prefix_path;
466  basedir = StringValuePtr(basedir_val);
467  baselen = RSTRING_LEN(basedir_val);
468 #else
469  if (getenv("MJIT_SEARCH_BUILD_DIR")) {
470  // This path is not intended to be used on production, but using build directory's
471  // header file here because people want to run `make test-all` without running
472  // `make install`. Don't use $MJIT_SEARCH_BUILD_DIR except for test-all.
473 
474  struct stat st;
475  const char *hdr = dlsym(RTLD_DEFAULT, "MJIT_HEADER");
476  if (!hdr) {
477  verbose(1, "No MJIT_HEADER");
478  }
479  else if (hdr[0] != '/') {
480  verbose(1, "Non-absolute header file path: %s", hdr);
481  }
482  else if (stat(hdr, &st) || !S_ISREG(st.st_mode)) {
483  verbose(1, "Non-file header file path: %s", hdr);
484  }
485  else if ((st.st_uid != getuid()) || (st.st_mode & 022) ||
486  !rb_path_check(hdr)) {
487  verbose(1, "Unsafe header file: uid=%ld mode=%#o %s",
488  (long)st.st_uid, (unsigned)st.st_mode, hdr);
489  return FALSE;
490  }
491  else {
492  // Do not pass PRELOADENV to child processes, on
493  // multi-arch environment
494  verbose(3, "PRELOADENV("PRELOADENV")=%s", getenv(PRELOADENV));
495  // assume no other PRELOADENV in test-all
496  unsetenv(PRELOADENV);
497  verbose(3, "MJIT_HEADER: %s", hdr);
498  header_file = ruby_strdup(hdr);
499  if (!header_file) return false;
500  }
501  }
502  else
503 #endif
504 #ifndef _MSC_VER
505  {
506  // A name of the header file included in any C file generated by MJIT for iseqs.
507  static const char header_name[] = MJIT_HEADER_INSTALL_DIR "/" MJIT_MIN_HEADER_NAME;
508  const size_t header_name_len = sizeof(header_name) - 1;
509 
510  header_file = xmalloc(baselen + header_name_len + 1);
511  p = append_str2(header_file, basedir, baselen);
512  p = append_str2(p, header_name, header_name_len + 1);
513 
514  if ((fd = rb_cloexec_open(header_file, O_RDONLY, 0)) < 0) {
515  verbose(1, "Cannot access header file: %s", header_file);
516  xfree(header_file);
517  header_file = NULL;
518  return false;
519  }
520  (void)close(fd);
521  }
522 
523  pch_file = get_uniq_filename(0, MJIT_TMP_PREFIX "h", ".h.gch");
524 #else
525  {
526  static const char pch_name[] = MJIT_HEADER_INSTALL_DIR "/" MJIT_PRECOMPILED_HEADER_NAME;
527  const size_t pch_name_len = sizeof(pch_name) - 1;
528 
529  pch_file = xmalloc(baselen + pch_name_len + 1);
530  p = append_str2(pch_file, basedir, baselen);
531  p = append_str2(p, pch_name, pch_name_len + 1);
532  if ((fd = rb_cloexec_open(pch_file, O_RDONLY, 0)) < 0) {
533  verbose(1, "Cannot access precompiled header file: %s", pch_file);
534  xfree(pch_file);
535  pch_file = NULL;
536  return false;
537  }
538  (void)close(fd);
539  }
540 #endif
541 
542 #ifdef _WIN32
543  basedir_val = ruby_archlibdir_path;
544  basedir = StringValuePtr(basedir_val);
545  baselen = RSTRING_LEN(basedir_val);
546  libruby_pathflag = p = xmalloc(libpathflag_len + baselen + 1);
547  p = append_str(p, libpathflag);
548  p = append_str2(p, basedir, baselen);
549  *p = '\0';
550 #endif
551 
552  return true;
553 }
554 
555 #ifdef _WIN32
556 UINT rb_w32_system_tmpdir(WCHAR *path, UINT len);
557 #endif
558 
559 static char *
560 system_default_tmpdir(void)
561 {
562  // c.f. ext/etc/etc.c:etc_systmpdir()
563 #ifdef _WIN32
564  WCHAR tmppath[_MAX_PATH];
565  UINT len = rb_w32_system_tmpdir(tmppath, numberof(tmppath));
566  if (len) {
567  int blen = WideCharToMultiByte(CP_UTF8, 0, tmppath, len, NULL, 0, NULL, NULL);
568  char *tmpdir = xmalloc(blen + 1);
569  WideCharToMultiByte(CP_UTF8, 0, tmppath, len, tmpdir, blen, NULL, NULL);
570  tmpdir[blen] = '\0';
571  return tmpdir;
572  }
573 #elif defined _CS_DARWIN_USER_TEMP_DIR
574  char path[MAXPATHLEN];
575  size_t len = confstr(_CS_DARWIN_USER_TEMP_DIR, path, sizeof(path));
576  if (len > 0) {
577  char *tmpdir = xmalloc(len);
578  if (len > sizeof(path)) {
579  confstr(_CS_DARWIN_USER_TEMP_DIR, tmpdir, len);
580  }
581  else {
582  memcpy(tmpdir, path, len);
583  }
584  return tmpdir;
585  }
586 #endif
587  return 0;
588 }
589 
590 static int
591 check_tmpdir(const char *dir)
592 {
593  struct stat st;
594 
595  if (!dir) return FALSE;
596  if (stat(dir, &st)) return FALSE;
597 #ifndef S_ISDIR
598 # define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR)
599 #endif
600  if (!S_ISDIR(st.st_mode)) return FALSE;
601 #ifndef _WIN32
602 # ifndef S_IWOTH
603 # define S_IWOTH 002
604 # endif
605  if (st.st_mode & S_IWOTH) {
606 # ifdef S_ISVTX
607  if (!(st.st_mode & S_ISVTX)) return FALSE;
608 # else
609  return FALSE;
610 # endif
611  }
612  if (access(dir, W_OK)) return FALSE;
613 #endif
614  return TRUE;
615 }
616 
617 static char *
618 system_tmpdir(void)
619 {
620  char *tmpdir;
621 # define RETURN_ENV(name) \
622  if (check_tmpdir(tmpdir = getenv(name))) return ruby_strdup(tmpdir)
623  RETURN_ENV("TMPDIR");
624  RETURN_ENV("TMP");
625  tmpdir = system_default_tmpdir();
626  if (check_tmpdir(tmpdir)) return tmpdir;
627  return ruby_strdup("/tmp");
628 # undef RETURN_ENV
629 }
630 
631 // Minimum value for JIT cache size.
632 #define MIN_CACHE_SIZE 10
633 // Default permitted number of units with a JIT code kept in memory.
634 #define DEFAULT_MAX_CACHE_SIZE 10000
635 // A default threshold used to add iseq to JIT.
636 #define DEFAULT_MIN_CALLS_TO_ADD 10000
637 
638 // Start MJIT worker. Return TRUE if worker is successfully started.
639 static bool
640 start_worker(void)
641 {
642  stop_worker_p = false;
643  worker_stopped = false;
644 
645  if (!rb_thread_create_mjit_thread(mjit_worker)) {
646  mjit_enabled = false;
647  rb_native_mutex_destroy(&mjit_engine_mutex);
648  rb_native_cond_destroy(&mjit_pch_wakeup);
649  rb_native_cond_destroy(&mjit_client_wakeup);
650  rb_native_cond_destroy(&mjit_worker_wakeup);
651  rb_native_cond_destroy(&mjit_gc_wakeup);
652  verbose(1, "Failure in MJIT thread initialization\n");
653  return false;
654  }
655  return true;
656 }
657 
658 // There's no strndup on Windows
659 static char*
660 ruby_strndup(const char *str, size_t n)
661 {
662  char *ret = xmalloc(n + 1);
663  memcpy(ret, str, n);
664  ret[n] = '\0';
665  return ret;
666 }
667 
668 // Convert "foo bar" to {"foo", "bar", NULL} array. Caller is responsible for
669 // freeing a returned buffer and its elements.
670 static char **
671 split_flags(const char *flags)
672 {
673  char *buf[MAXPATHLEN];
674  int i = 0;
675  char *next;
676  for (; flags != NULL; flags = next) {
677  next = strchr(flags, ' ');
678  if (next == NULL) {
679  if (strlen(flags) > 0)
680  buf[i++] = strdup(flags);
681  }
682  else {
683  if (next > flags)
684  buf[i++] = ruby_strndup(flags, next - flags);
685  next++; // skip space
686  }
687  }
688 
689  char **ret = xmalloc(sizeof(char *) * (i + 1));
690  memcpy(ret, buf, sizeof(char *) * i);
691  ret[i] = NULL;
692  return ret;
693 }
694 
695 // Initialize MJIT. Start a thread creating the precompiled header and
696 // processing ISeqs. The function should be called first for using MJIT.
697 // If everything is successful, MJIT_INIT_P will be TRUE.
698 void
699 mjit_init(const struct mjit_options *opts)
700 {
701  mjit_opts = *opts;
702  mjit_enabled = true;
703  mjit_call_p = true;
704 
705  // Normalize options
706  if (mjit_opts.min_calls == 0)
707  mjit_opts.min_calls = DEFAULT_MIN_CALLS_TO_ADD;
708  if (mjit_opts.max_cache_size <= 0)
709  mjit_opts.max_cache_size = DEFAULT_MAX_CACHE_SIZE;
710  if (mjit_opts.max_cache_size < MIN_CACHE_SIZE)
711  mjit_opts.max_cache_size = MIN_CACHE_SIZE;
712 
713  // Initialize variables for compilation
714 #ifdef _MSC_VER
715  pch_status = PCH_SUCCESS; // has prebuilt precompiled header
716 #else
717  pch_status = PCH_NOT_READY;
718 #endif
719  cc_path = CC_COMMON_ARGS[0];
720  verbose(2, "MJIT: CC defaults to %s", cc_path);
721  cc_common_args = xmalloc(sizeof(CC_COMMON_ARGS));
722  memcpy((void *)cc_common_args, CC_COMMON_ARGS, sizeof(CC_COMMON_ARGS));
723  cc_added_args = split_flags(opts->debug_flags);
724  xfree(opts->debug_flags);
725 #if MJIT_CFLAGS_PIPE
726  // eliminate a flag incompatible with `-pipe`
727  for (size_t i = 0, j = 0; i < sizeof(CC_COMMON_ARGS) / sizeof(char *); i++) {
728  if (CC_COMMON_ARGS[i] && strncmp("-save-temps", CC_COMMON_ARGS[i], strlen("-save-temps")) == 0)
729  continue; // skip -save-temps flag
730  cc_common_args[j] = CC_COMMON_ARGS[i];
731  j++;
732  }
733 #endif
734 
735  tmp_dir = system_tmpdir();
736  verbose(2, "MJIT: tmp_dir is %s", tmp_dir);
737 
738  if (!init_header_filename()) {
739  mjit_enabled = false;
740  verbose(1, "Failure in MJIT header file name initialization\n");
741  return;
742  }
743  pch_owner_pid = getpid();
744 
745  // Initialize mutex
746  rb_native_mutex_initialize(&mjit_engine_mutex);
747  rb_native_cond_initialize(&mjit_pch_wakeup);
748  rb_native_cond_initialize(&mjit_client_wakeup);
749  rb_native_cond_initialize(&mjit_worker_wakeup);
750  rb_native_cond_initialize(&mjit_gc_wakeup);
751 
752  // Make sure the saved_ec of the initial thread's root_fiber is scanned by mark_ec_units.
753  //
754  // rb_threadptr_root_fiber_setup for the initial thread is called before mjit_init,
755  // meaning mjit_cont_new is skipped for the root_fiber. Therefore we need to call
756  // rb_fiber_init_mjit_cont again with mjit_enabled=true to set the root_fiber's mjit_cont.
757  rb_fiber_init_mjit_cont(GET_EC()->fiber_ptr);
758 
759  // Initialize worker thread
760  start_worker();
761 }
762 
763 static void
764 stop_worker(void)
765 {
766  rb_execution_context_t *ec = GET_EC();
767 
768  while (!worker_stopped) {
769  verbose(3, "Sending cancel signal to worker");
770  CRITICAL_SECTION_START(3, "in stop_worker");
771  stop_worker_p = true; // Setting this inside loop because RUBY_VM_CHECK_INTS may make this false.
772  rb_native_cond_broadcast(&mjit_worker_wakeup);
773  CRITICAL_SECTION_FINISH(3, "in stop_worker");
774  RUBY_VM_CHECK_INTS(ec);
775  }
776 }
777 
778 // Stop JIT-compiling methods but compiled code is kept available.
779 VALUE
780 mjit_pause(bool wait_p)
781 {
782  if (!mjit_enabled) {
783  rb_raise(rb_eRuntimeError, "MJIT is not enabled");
784  }
785  if (worker_stopped) {
786  return Qfalse;
787  }
788 
789  // Flush all queued units with no option or `wait: true`
790  if (wait_p) {
791  struct timeval tv;
792  tv.tv_sec = 0;
793  tv.tv_usec = 1000;
794 
795  while (unit_queue.length > 0 && active_units.length < mjit_opts.max_cache_size) { // inverse of condition that waits for mjit_worker_wakeup
796  CRITICAL_SECTION_START(3, "in mjit_pause for a worker wakeup");
797  rb_native_cond_broadcast(&mjit_worker_wakeup);
798  CRITICAL_SECTION_FINISH(3, "in mjit_pause for a worker wakeup");
799  rb_thread_wait_for(tv);
800  }
801  }
802 
803  stop_worker();
804  return Qtrue;
805 }
806 
807 // Restart JIT-compiling methods after mjit_pause.
808 VALUE
809 mjit_resume(void)
810 {
811  if (!mjit_enabled) {
812  rb_raise(rb_eRuntimeError, "MJIT is not enabled");
813  }
814  if (!worker_stopped) {
815  return Qfalse;
816  }
817 
818  if (!start_worker()) {
819  rb_raise(rb_eRuntimeError, "Failed to resume MJIT worker");
820  }
821  return Qtrue;
822 }
823 
824 // Skip calling `clean_temp_files` for units which currently exist in the list.
825 static void
826 skip_cleaning_object_files(struct rb_mjit_unit_list *list)
827 {
828  struct rb_mjit_unit *unit = NULL, *next;
829 
830  // No mutex for list, assuming MJIT worker does not exist yet since it's immediately after fork.
831  list_for_each_safe(&list->head, unit, next, unode) {
832 #if defined(_WIN32) // mswin doesn't reach here either. This is for MinGW.
833  if (unit->so_file) unit->so_file = NULL;
834 #endif
835  }
836 }
837 
838 // This is called after fork initiated by Ruby's method to launch MJIT worker thread
839 // for child Ruby process.
840 //
841 // In multi-process Ruby applications, child Ruby processes do most of the jobs.
842 // Thus we want child Ruby processes to enqueue ISeqs to MJIT worker's queue and
843 // call the JIT-ed code.
844 //
845 // But unfortunately current MJIT-generated code is process-specific. After the fork,
846 // JIT-ed code created by parent Ruby process cannot be used in child Ruby process
847 // because the code could rely on inline cache values (ivar's IC, send's CC) which
848 // may vary between processes after fork or embed some process-specific addresses.
849 //
850 // So child Ruby process can't request parent process to JIT an ISeq and use the code.
851 // Instead of that, MJIT worker thread is created for all child Ruby processes, even
852 // while child processes would end up with compiling the same ISeqs.
853 void
854 mjit_child_after_fork(void)
855 {
856  if (!mjit_enabled)
857  return;
858 
859  /* Let parent process delete the already-compiled object files.
860  This must be done before starting MJIT worker on child process. */
861  skip_cleaning_object_files(&active_units);
862 
863  /* MJIT worker thread is not inherited on fork. Start it for this child process. */
864  start_worker();
865 }
866 
867 // Edit 0 to 1 to enable this feature for investigating hot methods
868 #define MJIT_COUNTER 0
869 #if MJIT_COUNTER
870 static void
871 mjit_dump_total_calls(void)
872 {
873  struct rb_mjit_unit *unit;
874  fprintf(stderr, "[MJIT_COUNTER] total_calls of active_units:\n");
875  list_for_each(&active_units.head, unit, unode) {
876  const rb_iseq_t *iseq = unit->iseq;
877  fprintf(stderr, "%8ld: %s@%s:%d\n", iseq->body->total_calls, RSTRING_PTR(iseq->body->location.label),
878  RSTRING_PTR(rb_iseq_path(iseq)), FIX2INT(iseq->body->location.first_lineno));
879  }
880 }
881 #endif
882 
883 // Finish the threads processing units and creating PCH, finalize
884 // and free MJIT data. It should be called last during MJIT
885 // life.
886 //
887 // If close_handle_p is true, it calls dlclose() for JIT-ed code. So it should be false
888 // if the code can still be on stack. ...But it means to leak JIT-ed handle forever (FIXME).
889 void
890 mjit_finish(bool close_handle_p)
891 {
892  if (!mjit_enabled)
893  return;
894 
895  // Wait for pch finish
896  verbose(2, "Stopping worker thread");
897  CRITICAL_SECTION_START(3, "in mjit_finish to wakeup from pch");
898  // As our threads are detached, we could just cancel them. But it
899  // is a bad idea because OS processes (C compiler) started by
900  // threads can produce temp files. And even if the temp files are
901  // removed, the used C compiler still complaint about their
902  // absence. So wait for a clean finish of the threads.
903  while (pch_status == PCH_NOT_READY) {
904  verbose(3, "Waiting wakeup from make_pch");
905  rb_native_cond_wait(&mjit_pch_wakeup, &mjit_engine_mutex);
906  }
907  CRITICAL_SECTION_FINISH(3, "in mjit_finish to wakeup from pch");
908 
909  // Stop worker
910  stop_worker();
911 
912  rb_native_mutex_destroy(&mjit_engine_mutex);
913  rb_native_cond_destroy(&mjit_pch_wakeup);
914  rb_native_cond_destroy(&mjit_client_wakeup);
915  rb_native_cond_destroy(&mjit_worker_wakeup);
916  rb_native_cond_destroy(&mjit_gc_wakeup);
917 
918 #if MJIT_COUNTER
919  mjit_dump_total_calls();
920 #endif
921 
922 #ifndef _MSC_VER // mswin has prebuilt precompiled header
923  if (!mjit_opts.save_temps && getpid() == pch_owner_pid)
924  remove_file(pch_file);
925 
926  xfree(header_file); header_file = NULL;
927 #endif
928  xfree((void *)cc_common_args); cc_common_args = NULL;
929  for (char **flag = cc_added_args; *flag != NULL; flag++)
930  xfree(*flag);
931  xfree((void *)cc_added_args); cc_added_args = NULL;
932  xfree(tmp_dir); tmp_dir = NULL;
933  xfree(pch_file); pch_file = NULL;
934 
935  mjit_call_p = false;
936  free_list(&unit_queue, close_handle_p);
937  free_list(&active_units, close_handle_p);
938  free_list(&compact_units, close_handle_p);
939  free_list(&stale_units, close_handle_p);
940  finish_conts();
941 
942  mjit_enabled = false;
943  verbose(1, "Successful MJIT finish");
944 }
945 
946 // Called by rb_vm_mark().
947 //
948 // Mark an ISeq being compiled to prevent its CCs from being GC-ed, which
949 // an MJIT worker may concurrently see.
950 //
951 // Also mark active_units so that we do not GC ISeq which may still be
952 // referred to by mjit_recompile() or compact_all_jit_code().
953 void
954 mjit_mark(void)
955 {
956  if (!mjit_enabled)
957  return;
958  RUBY_MARK_ENTER("mjit");
959 
960  // We need to release a lock when calling rb_gc_mark to avoid doubly acquiring
961  // a lock by by mjit_gc_start_hook inside rb_gc_mark.
962  //
963  // Because an MJIT worker may modify active_units anytime, we need to convert
964  // the linked list to an array to safely loop its ISeqs without keeping a lock.
965  CRITICAL_SECTION_START(4, "mjit_mark");
966  int length = 0;
967  if (compiling_iseqs != NULL) {
968  while (compiling_iseqs[length]) length++;
969  }
970  length += active_units.length;
971  const rb_iseq_t **iseqs = ALLOCA_N(const rb_iseq_t *, length);
972 
973  struct rb_mjit_unit *unit = NULL;
974  int i = 0;
975  if (compiling_iseqs != NULL) {
976  while (compiling_iseqs[i]) {
977  iseqs[i] = compiling_iseqs[i];
978  i++;
979  }
980  }
981  list_for_each(&active_units.head, unit, unode) {
982  iseqs[i] = unit->iseq;
983  i++;
984  }
985  assert(i == length);
986  CRITICAL_SECTION_FINISH(4, "mjit_mark");
987 
988  for (i = 0; i < length; i++) {
989  if (iseqs[i] == NULL) // ISeq is GC-ed
990  continue;
991  rb_gc_mark((VALUE)iseqs[i]);
992  }
993 
994  RUBY_MARK_LEAVE("mjit");
995 }
996 
997 // Called by rb_iseq_mark() to mark cc_entries captured for MJIT
998 void
999 mjit_mark_cc_entries(const struct rb_iseq_constant_body *const body)
1000 {
1001  const struct rb_callcache **cc_entries;
1002  if (body->jit_unit && (cc_entries = body->jit_unit->cc_entries) != NULL) {
1003  // It must be `body->jit_unit->cc_entries_size` instead of `body->ci_size` to mark children's cc_entries
1004  for (unsigned int i = 0; i < body->jit_unit->cc_entries_size; i++) {
1005  const struct rb_callcache *cc = cc_entries[i];
1006  if (cc != NULL && vm_cc_markable(cc)) {
1007  // Pin `cc` and `cc->cme` against GC.compact as their addresses may be written in JIT-ed code.
1008  rb_gc_mark((VALUE)cc);
1009  rb_gc_mark((VALUE)vm_cc_cme(cc));
1010  }
1011  }
1012  }
1013 }
1014 
1015 #endif // USE_MJIT
#define RUBY_ASSERT_ALWAYS(expr)
A variant of RUBY_ASSERT that does not interface with RUBY_DEBUG.
Definition: assert.h:167
#define xfree
Old name of ruby_xfree.
Definition: xmalloc.h:58
#define Qundef
Old name of RUBY_Qundef.
#define xmalloc
Old name of ruby_xmalloc.
Definition: xmalloc.h:53
#define FIX2INT
Old name of RB_FIX2INT.
Definition: int.h:41
#define Qtrue
Old name of RUBY_Qtrue.
#define Qfalse
Old name of RUBY_Qfalse.
void rb_raise(VALUE exc, const char *fmt,...)
Exception entry point.
Definition: error.c:3025
VALUE rb_eRuntimeError
RuntimeError exception.
Definition: error.c:1097
void rb_gc_mark(VALUE obj)
Marks an object.
Definition: gc.c:6775
void rb_memerror(void)
Triggers out-of-memory error.
Definition: gc.c:11117
VALUE rb_gc_location(VALUE obj)
Finds a new "location" of an object.
Definition: gc.c:9754
int rb_path_check(const char *path)
This function is mysterious.
Definition: file.c:6347
int rb_cloexec_open(const char *pathname, int flags, mode_t mode)
Opens a file that closes on exec.
Definition: io.c:314
void rb_thread_wait_for(struct timeval time)
Identical to rb_thread_sleep(), except it takes struct timeval instead.
Definition: thread.c:1558
char * ruby_strdup(const char *str)
This is our own version of strdup(3) that uses ruby_xmalloc() instead of system malloc (benefits our ...
Definition: util.c:536
#define strdup(s)
Just another name of ruby_strdup.
Definition: util.h:176
#define ALLOCA_N(type, n)
Definition: memory.h:286
#define StringValuePtr(v)
Identical to StringValue, except it returns a char*.
Definition: rstring.h:82
static char * RSTRING_PTR(VALUE str)
Queries the contents pointer of the string.
Definition: rstring.h:497
static long RSTRING_LEN(VALUE str)
Queries the length of the string.
Definition: rstring.h:483
void rb_native_cond_initialize(rb_nativethread_cond_t *cond)
Fills the passed condition variable with an initial value.
void rb_native_cond_broadcast(rb_nativethread_cond_t *cond)
Signals a condition variable.
void rb_native_mutex_initialize(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_initialize.
void rb_native_mutex_destroy(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_destroy.
void rb_native_cond_destroy(rb_nativethread_cond_t *cond)
Destroys the passed condition variable.
void rb_native_cond_wait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex)
Waits for the passed condition variable to be signalled.
uintptr_t VALUE
Type that represents a Ruby object.
Definition: value.h:40