85 #define __EXTENSIONS__ 1
89 #include "vm_callinfo.h"
92 #include "ruby_assert.h"
98 #include "insns_info.inc"
99 #include "internal/compile.h"
102 #include <winsock2.h>
105 #include <sys/wait.h>
106 #include <sys/time.h>
113 #ifdef HAVE_SYS_PARAM_H
114 # include <sys/param.h>
122 # define MAXPATHLEN 1024
126 #define dlopen(name,flag) ((void*)LoadLibrary(name))
127 #define dlerror() strerror(rb_w32_map_errno(GetLastError()))
128 #define dlsym(handle,name) ((void*)GetProcAddress((handle),(name)))
129 #define dlclose(handle) (!FreeLibrary(handle))
132 #define waitpid(pid,stat_loc,options) (WaitForSingleObject((HANDLE)(pid), INFINITE), GetExitCodeProcess((HANDLE)(pid), (LPDWORD)(stat_loc)), CloseHandle((HANDLE)pid), (pid))
133 #define WIFEXITED(S) ((S) != STILL_ACTIVE)
134 #define WEXITSTATUS(S) (S)
135 #define WIFSIGNALED(S) (0)
136 typedef intptr_t pid_t;
140 #define MJIT_ATOMIC_SET(var, val) (void)ATOMIC_PTR_EXCHANGE(var, val)
142 #define MJIT_TMP_PREFIX "_ruby_mjit_"
148 # define USE_JIT_COMPACTION 0
150 # define USE_JIT_COMPACTION 1
155 struct list_node unode;
170 struct rb_mjit_compile_info compile_info;
173 unsigned int cc_entries_size;
178 struct list_head head;
191 extern void rb_native_cond_wait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex);
194 extern rb_pid_t ruby_waitpid_locked(
rb_vm_t *, rb_pid_t,
int *status,
int options, rb_nativethread_cond_t *cond);
199 struct mjit_options mjit_opts;
202 bool mjit_enabled =
false;
205 bool mjit_call_p =
false;
209 static struct rb_mjit_unit_list unit_queue = { LIST_HEAD_INIT(unit_queue.head) };
211 static struct rb_mjit_unit_list active_units = { LIST_HEAD_INIT(active_units.head) };
213 static struct rb_mjit_unit_list compact_units = { LIST_HEAD_INIT(compact_units.head) };
215 static struct rb_mjit_unit_list stale_units = { LIST_HEAD_INIT(stale_units.head) };
217 static int current_unit_num;
219 static rb_nativethread_lock_t mjit_engine_mutex;
221 static rb_nativethread_cond_t mjit_pch_wakeup;
224 static rb_nativethread_cond_t mjit_client_wakeup;
227 static rb_nativethread_cond_t mjit_worker_wakeup;
229 static rb_nativethread_cond_t mjit_gc_wakeup;
231 static int in_gc = 0;
233 static bool in_jit =
false;
235 static bool pending_stale_p =
false;
237 static int unload_requests = 0;
239 static int total_unloads = 0;
241 static bool stop_worker_p;
243 static bool worker_stopped =
true;
246 static char *tmp_dir;
249 static const char *cc_path;
251 static const char **cc_common_args;
253 static char **cc_added_args;
255 static char *pch_file;
257 static rb_pid_t pch_owner_pid;
260 static enum {PCH_NOT_READY, PCH_FAILED, PCH_SUCCESS} pch_status;
264 static char *header_file;
269 static char *libruby_pathflag;
272 #include "mjit_config.h"
274 #if defined(__GNUC__) && \
275 (!defined(__clang__) || \
276 (defined(__clang__) && (defined(__FreeBSD__) || defined(__GLIBC__))))
277 # define GCC_PIC_FLAGS "-Wfatal-errors", "-fPIC", "-shared", "-w", "-pipe",
278 # define MJIT_CFLAGS_PIPE 1
280 # define GCC_PIC_FLAGS
281 # define MJIT_CFLAGS_PIPE 0
286 #if defined __GNUC__ && !defined __clang__ && !defined(_WIN32) && !defined(__CYGWIN__) && !defined(_AIX) && !defined(__OpenBSD__)
287 # define GCC_NOSTDLIB_FLAGS "-nodefaultlibs", "-nostdlib",
289 # define GCC_NOSTDLIB_FLAGS
292 static const char *
const CC_COMMON_ARGS[] = {
293 MJIT_CC_COMMON MJIT_CFLAGS GCC_PIC_FLAGS
297 static const char *
const CC_DEBUG_ARGS[] = {MJIT_DEBUGFLAGS NULL};
298 static const char *
const CC_OPTIMIZE_ARGS[] = {MJIT_OPTFLAGS NULL};
300 static const char *
const CC_LDSHARED_ARGS[] = {MJIT_LDSHARED GCC_PIC_FLAGS NULL};
301 static const char *
const CC_DLDFLAGS_ARGS[] = {MJIT_DLDFLAGS NULL};
303 static const char *
const CC_LINKER_ARGS[] = {
304 #if defined __GNUC__ && !defined __clang__ && !defined(__OpenBSD__)
307 GCC_NOSTDLIB_FLAGS NULL
310 static const char *
const CC_LIBS[] = {
311 #if defined(_WIN32) || defined(__CYGWIN__)
314 #if defined __GNUC__ && !defined __clang__
320 #if defined __ANDROID__
326 #define CC_CODEFLAG_ARGS (mjit_opts.debug ? CC_DEBUG_ARGS : CC_OPTIMIZE_ARGS)
330 PRINTF_ARGS(
static void, 2, 3)
331 verbose(
int level, const
char *format, ...)
333 if (mjit_opts.verbose >= level) {
335 size_t len = strlen(format);
336 char *full_format = alloca(
sizeof(
char) * (len + 2));
339 memcpy(full_format, format, len);
340 full_format[len] =
'\n';
341 full_format[len+1] =
'\0';
343 va_start(args, format);
344 vfprintf(stderr, full_format, args);
349 PRINTF_ARGS(
static void, 1, 2)
350 mjit_warning(const
char *format, ...)
352 if (mjit_opts.warnings || mjit_opts.verbose) {
355 fprintf(stderr,
"MJIT warning: ");
356 va_start(args, format);
357 vfprintf(stderr, format, args);
359 fprintf(stderr,
"\n");
368 (void)RB_DEBUG_COUNTER_INC_IF(mjit_length_unit_queue, list == &unit_queue);
369 (void)RB_DEBUG_COUNTER_INC_IF(mjit_length_active_units, list == &active_units);
370 (void)RB_DEBUG_COUNTER_INC_IF(mjit_length_compact_units, list == &compact_units);
371 (void)RB_DEBUG_COUNTER_INC_IF(mjit_length_stale_units, list == &stale_units);
373 list_add_tail(&list->head, &unit->unode);
380 #if USE_DEBUG_COUNTER
381 rb_debug_counter_add(RB_DEBUG_COUNTER_mjit_length_unit_queue, -1, list == &unit_queue);
382 rb_debug_counter_add(RB_DEBUG_COUNTER_mjit_length_active_units, -1, list == &active_units);
383 rb_debug_counter_add(RB_DEBUG_COUNTER_mjit_length_compact_units, -1, list == &compact_units);
384 rb_debug_counter_add(RB_DEBUG_COUNTER_mjit_length_stale_units, -1, list == &stale_units);
387 list_del(&unit->unode);
392 remove_file(
const char *filename)
394 if (remove(filename)) {
395 mjit_warning(
"failed to remove \"%s\": %s", filename, strerror(errno));
405 char *so_file = unit->so_file;
407 unit->so_file = NULL;
409 remove_file(so_file);
427 unit->iseq->body->jit_func = (mjit_func_t)NOT_COMPILED_JIT_ISEQ_FUNC;
428 unit->iseq->body->jit_unit = NULL;
430 if (unit->cc_entries) {
431 void *entries = (
void *)unit->cc_entries;
434 if (unit->handle && dlclose(unit->handle)) {
435 mjit_warning(
"failed to close handle for u%d: %s", unit->id, dlerror());
437 clean_temp_files(unit);
443 CRITICAL_SECTION_START(
int level,
const char *msg)
445 verbose(level,
"Locking %s", msg);
447 verbose(level,
"Locked %s", msg);
453 CRITICAL_SECTION_FINISH(
int level,
const char *msg)
455 verbose(level,
"Unlocked %s", msg);
460 sprint_uniq_filename(
char *str,
size_t size,
unsigned long id,
const char *prefix,
const char *suffix)
462 return snprintf(str, size,
"%s/%sp%"PRI_PIDT_PREFIX"uu%lu%s", tmp_dir, prefix, getpid(),
id, suffix);
467 double ruby_real_ms_time(
void);
468 # define real_ms_time() ruby_real_ms_time()
473 # ifdef HAVE_CLOCK_GETTIME
475 # ifdef CLOCK_MONOTONIC
476 const clockid_t c = CLOCK_MONOTONIC;
478 const clockid_t c = CLOCK_REALTIME;
481 clock_gettime(c, &tv);
482 return tv.tv_nsec / 1000000.0 + tv.tv_sec * 1000.0;
486 gettimeofday(&tv, NULL);
487 return tv.tv_usec / 1000.0 + tv.tv_sec * 1000.0;
499 verbose(3,
"Waiting wakeup from GC");
506 list_for_each_safe(&list->head, unit, next, unode) {
507 if (unit->iseq == NULL) {
508 remove_from_list(unit, list);
513 if (best == NULL || best->iseq->body->total_calls < unit->iseq->body->total_calls) {
519 verbose(3,
"Sending wakeup signal to client in a mjit-worker for GC");
523 remove_from_list(best, list);
530 args_len(
char *
const *args)
534 for (i = 0; (args[i]) != NULL;i++)
542 form_args(
int num, ...)
547 char **args, **res, **tmp;
551 for (i = len = 0; i < num; i++) {
552 args = va_arg(argp,
char **);
554 if ((tmp = (
char **)realloc(res,
sizeof(
char *) * (len + n + 1))) == NULL) {
560 MEMCPY(res + len, args,
char *, n + 1);
567 COMPILER_WARNING_PUSH
568 #if __has_warning("-Wdeprecated-declarations") || RBIMPL_COMPILER_IS(GCC)
569 COMPILER_WARNING_IGNORED(-Wdeprecated-declarations)
574 start_process(
const char *abspath,
char *
const *argv)
580 verbose(1,
"MJIT: Failed to open a null device: %s", strerror(errno));
583 if (mjit_opts.verbose >= 2) {
585 fprintf(stderr,
"Starting process: %s", abspath);
586 for (
int i = 0; (arg = argv[i]) != NULL; i++)
587 fprintf(stderr,
" %s", arg);
588 fprintf(stderr,
"\n");
593 extern HANDLE rb_w32_start_process(
const char *abspath,
char *
const *argv,
int out_fd);
595 if (mjit_opts.verbose <= 1) {
602 pid = (pid_t)rb_w32_start_process(abspath, argv, out_fd);
604 verbose(1,
"MJIT: Failed to create process: %s", dlerror());
608 if ((pid = vfork()) == 0) {
610 if (mjit_opts.verbose == 0) {
614 dup2(dev_null, STDERR_FILENO);
615 dup2(dev_null, STDOUT_FILENO);
617 (void)close(dev_null);
618 pid = execv(abspath, argv);
622 verbose(1,
"MJIT: Error in execv: %s", abspath);
626 (void)close(dev_null);
635 exec_process(
const char *path,
char *
const argv[])
637 int stat, exit_code = -2;
638 rb_vm_t *vm = WAITPID_USE_SIGCHLD ? GET_VM() : 0;
639 rb_nativethread_cond_t cond;
646 pid_t pid = start_process(path, argv);
648 pid_t r = vm ? ruby_waitpid_locked(vm, pid, &stat, 0, &cond)
649 : waitpid(pid, &stat, 0);
651 if (errno == EINTR)
continue;
652 fprintf(stderr,
"[%"PRI_PIDT_PREFIX"d] waitpid(%lu): %s (SIGCHLD=%d,%u)\n",
653 getpid(), (
unsigned long)pid, strerror(errno),
654 RUBY_SIGCHLD, SIGCHLD_LOSSY);
658 if (WIFEXITED(stat)) {
659 exit_code = WEXITSTATUS(stat);
662 else if (WIFSIGNALED(stat)) {
677 remove_so_file(
const char *so_file,
struct rb_mjit_unit *unit)
681 unit->so_file =
strdup(so_file);
682 if (unit->so_file == NULL)
683 mjit_warning(
"failed to allocate memory to lazily remove '%s': %s", so_file, strerror(errno));
685 remove_file(so_file);
691 sprint_funcname(
char *funcname,
const struct rb_mjit_unit *unit)
694 if (iseq == NULL || (!mjit_opts.debug && !mjit_opts.debug_flags)) {
695 sprintf(funcname,
"_mjit%d", unit->id);
700 const char *path =
RSTRING_PTR(rb_iseq_path(iseq));
701 const char *lib =
"/lib/";
703 while (strstr(path, lib))
704 path = strstr(path, lib) + strlen(lib);
705 while (strstr(path, version))
706 path = strstr(path, version) + strlen(version);
709 const char *method =
RSTRING_PTR(iseq->body->location.label);
710 if (!strcmp(method,
"[]")) method =
"AREF";
711 if (!strcmp(method,
"[]=")) method =
"ASET";
714 sprintf(funcname,
"_mjit%d_%s_%s", unit->id, path, method);
715 for (
size_t i = 0; i < strlen(funcname); i++) {
716 char c = funcname[i];
717 if (!((
'a' <= c && c <=
'z') || (
'A' <= c && c <=
'Z') || (
'0' <= c && c <=
'9') || c ==
'_')) {
723 static const rb_iseq_t **compiling_iseqs = NULL;
726 set_compiling_iseqs(
const rb_iseq_t *iseq)
728 compiling_iseqs = calloc(iseq->body->iseq_size + 2,
sizeof(
rb_iseq_t *));
729 if (compiling_iseqs == NULL)
732 compiling_iseqs[0] = iseq;
735 unsigned int pos = 0;
736 while (pos < iseq->body->iseq_size) {
737 int insn = rb_vm_insn_decode(iseq->body->iseq_encoded[pos]);
738 if (insn == BIN(opt_send_without_block) || insn == BIN(opt_size)) {
741 const rb_iseq_t *iseq = rb_mjit_inlinable_iseq(cd->ci, cd->cc);
743 compiling_iseqs[i] = iseq;
747 pos += insn_len(insn);
753 free_compiling_iseqs(
void)
759 free(compiling_iseqs);
761 compiling_iseqs = NULL;
765 rb_mjit_compiling_iseq_p(
const rb_iseq_t *iseq)
767 assert(compiling_iseqs != NULL);
769 while (compiling_iseqs[i]) {
770 if (compiling_iseqs[i] == iseq)
return true;
776 static const int c_file_access_mode =
780 O_WRONLY|O_EXCL|O_CREAT;
782 #define append_str2(p, str, len) ((char *)memcpy((p), str, (len))+(len))
783 #define append_str(p, str) append_str2(p, str, sizeof(str)-1)
784 #define append_lit(p, str) append_str2(p, str, rb_strlen_lit(str))
789 compile_c_to_so(
const char *c_file,
const char *so_file)
791 const char *files[] = { NULL, NULL, NULL, NULL, NULL, NULL, NULL,
"-link", libruby_pathflag, NULL };
795 files[0] = p = alloca(
sizeof(
char) * (
rb_strlen_lit(
"-Fe") + strlen(so_file) + 1));
796 p = append_lit(p,
"-Fe");
797 p = append_str2(p, so_file, strlen(so_file));
803 char *obj_file = p = append_lit(p,
"-Fo");
804 p = append_str2(p, so_file, strlen(so_file) -
rb_strlen_lit(DLEXT));
805 p = append_lit(p,
".obj");
809 files[2] = p = alloca(
sizeof(
char) * (
rb_strlen_lit(
"-Yu") + strlen(pch_file) + 1));
810 p = append_lit(p,
"-Yu");
811 p = append_str2(p, pch_file, strlen(pch_file));
815 files[3] = p = alloca(
sizeof(
char) * (strlen(pch_file) + 1));
816 p = append_str2(p, pch_file, strlen(pch_file) - strlen(
".pch"));
817 p = append_lit(p,
".obj");
821 files[4] = p = alloca(
sizeof(
char) * (
rb_strlen_lit(
"-Tc") + strlen(c_file) + 1));
822 p = append_lit(p,
"-Tc");
823 p = append_str2(p, c_file, strlen(c_file));
829 p = append_lit(p,
"-Fd");
830 p = append_str2(p, so_file, strlen(so_file) -
rb_strlen_lit(DLEXT));
831 p = append_lit(p,
".pdb");
836 files[6] = p = alloca(
sizeof(
char) *
rb_strlen_lit(
"-Z7") + 1);
837 p = append_lit(p,
"-Z7");
840 char **args = form_args(5, CC_LDSHARED_ARGS, CC_CODEFLAG_ARGS,
841 files, CC_LIBS, CC_DLDFLAGS_ARGS);
845 int exit_code = exec_process(cc_path, args);
848 if (exit_code == 0) {
850 if (!mjit_opts.save_temps) {
852 remove_file(obj_file);
854 before_dot = obj_file + strlen(obj_file) -
rb_strlen_lit(
".obj");
855 append_lit(before_dot,
".lib"); remove_file(obj_file);
856 append_lit(before_dot,
".exp"); remove_file(obj_file);
857 append_lit(before_dot,
".pdb"); remove_file(obj_file);
861 verbose(2,
"compile_c_to_so: compile error: %d", exit_code);
863 return exit_code == 0;
871 const char *rest_args[] = {
879 "-o", pch_file, header_file,
883 verbose(2,
"Creating precompiled header");
884 char **args = form_args(4, cc_common_args, CC_CODEFLAG_ARGS, cc_added_args, rest_args);
886 mjit_warning(
"making precompiled header failed on forming args");
887 CRITICAL_SECTION_START(3,
"in make_pch");
888 pch_status = PCH_FAILED;
889 CRITICAL_SECTION_FINISH(3,
"in make_pch");
893 int exit_code = exec_process(cc_path, args);
896 CRITICAL_SECTION_START(3,
"in make_pch");
897 if (exit_code == 0) {
898 pch_status = PCH_SUCCESS;
901 mjit_warning(
"Making precompiled header failed on compilation. Stopping MJIT worker...");
902 pch_status = PCH_FAILED;
906 CRITICAL_SECTION_FINISH(3,
"in make_pch");
913 compile_c_to_so(
const char *c_file,
const char *so_file)
915 char* o_file = alloca(strlen(c_file) + 1);
916 strcpy(o_file, c_file);
917 o_file[strlen(c_file) - 1] =
'o';
919 const char *o_args[] = {
920 "-o", o_file, c_file,
922 "-include-pch", pch_file,
926 char **args = form_args(5, cc_common_args, CC_CODEFLAG_ARGS, cc_added_args, o_args, CC_LINKER_ARGS);
927 if (args == NULL)
return false;
928 int exit_code = exec_process(cc_path, args);
930 if (exit_code != 0) {
931 verbose(2,
"compile_c_to_so: failed to compile .c to .o: %d", exit_code);
935 const char *so_args[] = {
942 # if defined(__MACH__)
943 extern VALUE rb_libruby_selfpath;
944 const char *loader_args[] = {
"-bundle_loader",
StringValuePtr(rb_libruby_selfpath), NULL};
946 const char *loader_args[] = {NULL};
948 args = form_args(7, CC_LDSHARED_ARGS, CC_CODEFLAG_ARGS, so_args, loader_args, CC_LIBS, CC_DLDFLAGS_ARGS, CC_LINKER_ARGS);
949 if (args == NULL)
return false;
950 exit_code = exec_process(cc_path, args);
952 if (!mjit_opts.save_temps) remove_file(o_file);
953 if (exit_code != 0) {
954 verbose(2,
"compile_c_to_so: failed to link .o to .so: %d", exit_code);
956 return exit_code == 0;
960 #if USE_JIT_COMPACTION
961 static void compile_prelude(
FILE *f);
964 compile_compact_jit_code(
char* c_file)
968 if (fd < 0 || (f = fdopen(fd,
"w")) == NULL) {
970 if (fd >= 0) (void)close(fd);
971 verbose(1,
"Failed to fopen '%s', giving up JIT for it (%s)", c_file, strerror(e));
978 CRITICAL_SECTION_START(3,
"before mjit_compile to wait GC finish");
980 verbose(3,
"Waiting wakeup from GC");
984 bool iseq_gced =
false;
986 list_for_each_safe(&active_units.head, child_unit, next, unode) {
987 if (child_unit->iseq == NULL) {
989 verbose(1,
"JIT compaction: A method for JIT code u%d is obsoleted. Compaction will be skipped.", child_unit->id);
990 remove_from_list(child_unit, &active_units);
991 free_unit(child_unit);
995 CRITICAL_SECTION_FINISH(3,
"before mjit_compile to wait GC finish");
998 if (!mjit_opts.save_temps)
1010 bool success =
true;
1011 list_for_each(&active_units.head, child_unit, unode) {
1012 CRITICAL_SECTION_START(3,
"before set_compiling_iseqs");
1013 success &= set_compiling_iseqs(child_unit->iseq);
1014 CRITICAL_SECTION_FINISH(3,
"after set_compiling_iseqs");
1015 if (!success)
continue;
1017 char funcname[MAXPATHLEN];
1018 sprint_funcname(funcname, child_unit);
1020 long iseq_lineno = 0;
1021 if (
FIXNUM_P(child_unit->iseq->body->location.first_lineno))
1023 iseq_lineno =
FIX2LONG(child_unit->iseq->body->location.first_lineno);
1024 const char *sep =
"@";
1025 const char *iseq_label =
RSTRING_PTR(child_unit->iseq->body->location.label);
1026 const char *iseq_path =
RSTRING_PTR(rb_iseq_path(child_unit->iseq));
1027 if (!iseq_label) iseq_label = sep =
"";
1028 fprintf(f,
"\n/* %s%s%s:%ld */\n", iseq_label, sep, iseq_path, iseq_lineno);
1029 success &= mjit_compile(f, child_unit->iseq, funcname, child_unit->id);
1031 CRITICAL_SECTION_START(3,
"before compiling_iseqs free");
1032 free_compiling_iseqs();
1033 CRITICAL_SECTION_FINISH(3,
"after compiling_iseqs free");
1037 CRITICAL_SECTION_START(3,
"after mjit_compile to wakeup client for GC");
1039 verbose(3,
"Sending wakeup signal to client in a mjit-worker for GC");
1041 CRITICAL_SECTION_FINISH(3,
"in worker to wakeup client for GC");
1050 compact_all_jit_code(
void)
1053 static const char c_ext[] =
".c";
1054 static const char so_ext[] = DLEXT;
1055 char c_file[MAXPATHLEN], so_file[MAXPATHLEN];
1059 if (unit == NULL)
return;
1060 unit->id = current_unit_num++;
1061 sprint_uniq_filename(c_file, (
int)
sizeof(c_file), unit->id, MJIT_TMP_PREFIX, c_ext);
1062 sprint_uniq_filename(so_file, (
int)
sizeof(so_file), unit->id, MJIT_TMP_PREFIX, so_ext);
1064 bool success = compile_compact_jit_code(c_file);
1065 double start_time = real_ms_time();
1067 success = compile_c_to_so(c_file, so_file);
1068 if (!mjit_opts.save_temps)
1069 remove_file(c_file);
1071 double end_time = real_ms_time();
1074 void *handle = dlopen(so_file, RTLD_NOW);
1075 if (handle == NULL) {
1076 mjit_warning(
"failure in loading code from compacted '%s': %s", so_file, dlerror());
1080 unit->handle = handle;
1083 add_to_list(unit, &compact_units);
1085 if (!mjit_opts.save_temps)
1086 remove_so_file(so_file, unit);
1088 CRITICAL_SECTION_START(3,
"in compact_all_jit_code to read list");
1089 list_for_each(&active_units.head, cur, unode) {
1091 char funcname[MAXPATHLEN];
1092 sprint_funcname(funcname, cur);
1094 if ((func = dlsym(handle, funcname)) == NULL) {
1095 mjit_warning(
"skipping to reload '%s' from '%s': %s", funcname, so_file, dlerror());
1101 MJIT_ATOMIC_SET(cur->iseq->body->jit_func, (mjit_func_t)func);
1104 CRITICAL_SECTION_FINISH(3,
"in compact_all_jit_code to read list");
1105 verbose(1,
"JIT compaction (%.1fms): Compacted %d methods %s -> %s", end_time - start_time, active_units.length, c_file, so_file);
1109 verbose(1,
"JIT compaction failure (%.1fms): Failed to compact methods", end_time - start_time);
1115 load_func_from_so(
const char *so_file,
const char *funcname,
struct rb_mjit_unit *unit)
1117 void *handle, *func;
1119 handle = dlopen(so_file, RTLD_NOW);
1120 if (handle == NULL) {
1121 mjit_warning(
"failure in loading code from '%s': %s", so_file, dlerror());
1122 return (
void *)NOT_COMPILED_JIT_ISEQ_FUNC;
1125 func = dlsym(handle, funcname);
1126 unit->handle = handle;
1132 header_name_end(
const char *s)
1134 const char *e = s + strlen(s);
1136 static const char suffix[] =
".gch";
1139 if (e > s+
sizeof(suffix)-1 && strcmp(e-
sizeof(suffix)+1, suffix) == 0) {
1140 e -=
sizeof(suffix)-1;
1149 compile_prelude(
FILE *f)
1152 const char *s = pch_file;
1153 const char *e = header_name_end(s);
1155 fprintf(f,
"#include \"");
1157 for (; s < e; s++) {
1159 case '\\':
case '"':
1168 fprintf(f,
"void _pei386_runtime_relocator(void){}\n");
1169 fprintf(f,
"int __stdcall DllMainCRTStartup(void* hinstDLL, unsigned int fdwReason, void* lpvReserved) { return 1; }\n");
1178 static const char c_ext[] =
".c";
1179 static const char so_ext[] = DLEXT;
1180 char c_file[MAXPATHLEN], so_file[MAXPATHLEN], funcname[MAXPATHLEN];
1182 sprint_uniq_filename(c_file, (
int)
sizeof(c_file), unit->id, MJIT_TMP_PREFIX, c_ext);
1183 sprint_uniq_filename(so_file, (
int)
sizeof(so_file), unit->id, MJIT_TMP_PREFIX, so_ext);
1184 sprint_funcname(funcname, unit);
1188 if (fd < 0 || (f = fdopen(fd,
"w")) == NULL) {
1190 if (fd >= 0) (void)close(fd);
1191 verbose(1,
"Failed to fopen '%s', giving up JIT for it (%s)", c_file, strerror(e));
1192 return (mjit_func_t)NOT_COMPILED_JIT_ISEQ_FUNC;
1199 CRITICAL_SECTION_START(3,
"before mjit_compile to wait GC finish");
1201 verbose(3,
"Waiting wakeup from GC");
1205 in_jit = (unit->iseq != NULL);
1207 in_jit &= set_compiling_iseqs(unit->iseq);
1208 CRITICAL_SECTION_FINISH(3,
"before mjit_compile to wait GC finish");
1211 if (!mjit_opts.save_temps)
1212 remove_file(c_file);
1213 return (mjit_func_t)NOT_COMPILED_JIT_ISEQ_FUNC;
1217 long iseq_lineno = 0;
1218 if (
FIXNUM_P(unit->iseq->body->location.first_lineno))
1220 iseq_lineno =
FIX2LONG(unit->iseq->body->location.first_lineno);
1221 char *iseq_label = alloca(
RSTRING_LEN(unit->iseq->body->location.label) + 1);
1222 char *iseq_path = alloca(
RSTRING_LEN(rb_iseq_path(unit->iseq)) + 1);
1223 strcpy(iseq_label,
RSTRING_PTR(unit->iseq->body->location.label));
1224 strcpy(iseq_path,
RSTRING_PTR(rb_iseq_path(unit->iseq)));
1226 verbose(2,
"start compilation: %s@%s:%ld -> %s", iseq_label, iseq_path, iseq_lineno, c_file);
1227 fprintf(f,
"/* %s@%s:%ld */\n\n", iseq_label, iseq_path, iseq_lineno);
1228 bool success = mjit_compile(f, unit->iseq, funcname, unit->id);
1231 CRITICAL_SECTION_START(3,
"after mjit_compile to wakeup client for GC");
1232 free_compiling_iseqs();
1234 verbose(3,
"Sending wakeup signal to client in a mjit-worker for GC");
1236 CRITICAL_SECTION_FINISH(3,
"in worker to wakeup client for GC");
1240 if (!mjit_opts.save_temps)
1241 remove_file(c_file);
1242 verbose(1,
"JIT failure: %s@%s:%ld -> %s", iseq_label, iseq_path, iseq_lineno, c_file);
1243 return (mjit_func_t)NOT_COMPILED_JIT_ISEQ_FUNC;
1246 double start_time = real_ms_time();
1247 success = compile_c_to_so(c_file, so_file);
1248 if (!mjit_opts.save_temps)
1249 remove_file(c_file);
1250 double end_time = real_ms_time();
1253 verbose(2,
"Failed to generate so: %s", so_file);
1254 return (mjit_func_t)NOT_COMPILED_JIT_ISEQ_FUNC;
1257 void *func = load_func_from_so(so_file, funcname, unit);
1258 if (!mjit_opts.save_temps)
1259 remove_so_file(so_file, unit);
1261 if ((uintptr_t)func > (uintptr_t)LAST_JIT_ISEQ_FUNC) {
1262 verbose(1,
"JIT success (%.1fms): %s@%s:%ld -> %s",
1263 end_time - start_time, iseq_label, iseq_path, iseq_lineno, c_file);
1265 return (mjit_func_t)func;
1272 return body->jit_unit->cc_entries;
1286 unsigned int new_entries_size = unit->cc_entries_size + captured_iseq->ci_size;
1287 VM_ASSERT(captured_iseq->ci_size > 0);
1291 int cc_entries_index = unit->cc_entries_size;
1292 if (unit->cc_entries_size == 0) {
1293 VM_ASSERT(unit->cc_entries == NULL);
1294 unit->cc_entries = cc_entries = malloc(
sizeof(
struct rb_callcache *) * new_entries_size);
1295 if (cc_entries == NULL)
return -1;
1298 void *cc_ptr = (
void *)unit->cc_entries;
1299 cc_entries = realloc(cc_ptr,
sizeof(
struct rb_callcache *) * new_entries_size);
1300 if (cc_entries == NULL)
return -1;
1301 unit->cc_entries = cc_entries;
1302 cc_entries += cc_entries_index;
1304 unit->cc_entries_size = new_entries_size;
1307 for (
unsigned int i = 0; i < captured_iseq->ci_size; i++) {
1308 cc_entries[i] = captured_iseq->call_data[i].cc;
1311 return cc_entries_index;
1320 if (ec->vm_stack == NULL)
1322 for (cfp = RUBY_VM_END_CONTROL_FRAME(ec) - 1; ; cfp = RUBY_VM_NEXT_CONTROL_FRAME(cfp)) {
1324 if (cfp->pc && (iseq = cfp->iseq) != NULL
1325 && imemo_type((
VALUE) iseq) == imemo_iseq
1326 && (iseq->body->jit_unit) != NULL) {
1327 iseq->body->jit_unit->used_code_p =
true;
1352 int units_num = active_units.length;
1356 list_for_each_safe(&active_units.head, unit, next, unode) {
1357 if (unit->iseq == NULL) {
1358 remove_from_list(unit, &active_units);
1364 list_for_each(&active_units.head, unit, unode) {
1365 assert(unit->iseq != NULL && unit->handle != NULL);
1366 unit->used_code_p =
false;
1370 for (cont = first_cont; cont != NULL; cont = cont->next) {
1371 mark_ec_units(cont->ec);
1377 long unsigned prev_queue_calls = -1;
1380 long unsigned max_queue_calls = 0;
1381 list_for_each(&unit_queue.head, unit, unode) {
1382 if (unit->iseq != NULL && max_queue_calls < unit->iseq->body->total_calls
1383 && unit->iseq->body->total_calls < prev_queue_calls) {
1384 max_queue_calls = unit->iseq->body->total_calls;
1387 prev_queue_calls = max_queue_calls;
1389 bool unloaded_p =
false;
1390 list_for_each_safe(&active_units.head, unit, next, unode) {
1391 if (unit->used_code_p)
1394 if (max_queue_calls > unit->iseq->body->total_calls) {
1395 verbose(2,
"Unloading unit %d (calls=%lu, threshold=%lu)",
1396 unit->id, unit->iseq->body->total_calls, max_queue_calls);
1397 assert(unit->handle != NULL);
1398 remove_from_list(unit, &active_units);
1403 if (!unloaded_p)
break;
1406 if (units_num > active_units.length) {
1407 verbose(1,
"Too many JIT code -- %d units unloaded", units_num - active_units.length);
1408 total_unloads += units_num - active_units.length;
1412 static void mjit_add_iseq_to_process(
const rb_iseq_t *iseq,
const struct rb_mjit_compile_info *compile_info,
bool worker_p);
1422 int max_compact_size = mjit_opts.max_cache_size / 100;
1423 if (max_compact_size < 10) max_compact_size = 10;
1427 int throttle_threshold = mjit_opts.max_cache_size / 10;
1430 if (pch_status == PCH_NOT_READY) {
1434 if (pch_status == PCH_FAILED) {
1435 mjit_enabled =
false;
1436 CRITICAL_SECTION_START(3,
"in worker to update worker_stopped");
1437 worker_stopped =
true;
1438 verbose(3,
"Sending wakeup signal to client in a mjit-worker");
1440 CRITICAL_SECTION_FINISH(3,
"in worker to update worker_stopped");
1445 while (!stop_worker_p) {
1449 CRITICAL_SECTION_START(3,
"in worker dequeue");
1450 while ((list_empty(&unit_queue.head) || active_units.length >= mjit_opts.max_cache_size) && !stop_worker_p) {
1452 verbose(3,
"Getting wakeup from client");
1455 if (pending_stale_p) {
1456 pending_stale_p =
false;
1458 list_for_each_safe(&active_units.head, unit, next, unode) {
1459 if (unit->stale_p) {
1460 unit->stale_p =
false;
1461 remove_from_list(unit, &active_units);
1462 add_to_list(unit, &stale_units);
1464 mjit_add_iseq_to_process(unit->iseq, &unit->iseq->body->jit_unit->compile_info,
true);
1470 if (unload_requests >= throttle_threshold) {
1472 verbose(3,
"Waiting wakeup from GC");
1477 RB_DEBUG_COUNTER_INC(mjit_unload_units);
1479 unload_requests = 0;
1482 verbose(3,
"Sending wakeup signal to client in a mjit-worker for GC");
1485 if (active_units.length == mjit_opts.max_cache_size && mjit_opts.wait) {
1486 mjit_opts.max_cache_size++;
1487 verbose(1,
"No units can be unloaded -- incremented max-cache-size to %d for --jit-wait", mjit_opts.max_cache_size);
1490 unit = get_from_list(&unit_queue);
1491 CRITICAL_SECTION_FINISH(3,
"in worker dequeue");
1495 mjit_func_t func = convert_unit_to_func(unit);
1496 (void)RB_DEBUG_COUNTER_INC_IF(mjit_compile_failures, func == (mjit_func_t)NOT_COMPILED_JIT_ISEQ_FUNC);
1498 CRITICAL_SECTION_START(3,
"in jit func replace");
1500 verbose(3,
"Waiting wakeup from GC");
1504 if ((uintptr_t)func > (uintptr_t)LAST_JIT_ISEQ_FUNC) {
1505 add_to_list(unit, &active_units);
1508 MJIT_ATOMIC_SET(unit->iseq->body->jit_func, func);
1513 CRITICAL_SECTION_FINISH(3,
"in jit func replace");
1515 #if USE_JIT_COMPACTION
1517 if (compact_units.length < max_compact_size
1518 && ((!mjit_opts.wait && unit_queue.length == 0 && active_units.length > 1)
1519 || (active_units.length == mjit_opts.max_cache_size && compact_units.length * throttle_threshold <= total_unloads))) {
1520 compact_all_jit_code();
1527 worker_stopped =
true;
#define FIX2LONG
Old name of RB_FIX2LONG.
#define FIXNUM_P
Old name of RB_FIXNUM_P.
Defines RBIMPL_HAS_BUILTIN.
int rb_cloexec_open(const char *pathname, int flags, mode_t mode)
Opens a file that closes on exec.
#define rb_strlen_lit(str)
Length of a string literal.
#define strdup(s)
Just another name of ruby_strdup.
#define RUBY_API_VERSION_TEENY
Teeny version.
#define RUBY_API_VERSION_MAJOR
Major version.
#define RUBY_API_VERSION_MINOR
Minor version.
#define MEMCPY(p1, p2, type, n)
Handy macro to call memcpy.
#define PRI_PIDT_PREFIX
A rb_sprintf() format prefix to be used for a pid_t parameter.
#define StringValuePtr(v)
Identical to StringValue, except it returns a char*.
static char * RSTRING_PTR(VALUE str)
Queries the contents pointer of the string.
static long RSTRING_LEN(VALUE str)
Queries the length of the string.
void rb_native_mutex_lock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_lock.
void rb_native_cond_initialize(rb_nativethread_cond_t *cond)
Fills the passed condition variable with an initial value.
void rb_native_cond_broadcast(rb_nativethread_cond_t *cond)
Signals a condition variable.
void rb_native_mutex_initialize(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_initialize.
void rb_native_mutex_unlock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_unlock.
void rb_native_mutex_destroy(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_destroy.
void rb_native_cond_destroy(rb_nativethread_cond_t *cond)
Destroys the passed condition variable.
void rb_native_cond_signal(rb_nativethread_cond_t *cond)
Signals a condition variable.
void rb_native_cond_wait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex)
Waits for the passed condition variable to be signalled.
uintptr_t VALUE
Type that represents a Ruby object.
#define RBIMPL_WARNING_IGNORED(flag)
Suppresses a warning.
#define RBIMPL_WARNING_PUSH()
Pushes compiler warning state.
#define RBIMPL_WARNING_POP()
Pops compiler warning state.