mirrors/ruby
0
0 Fork 0
mirror of https://github.com/ruby/ruby.git synced 2026-10-28 16:36:20 +08:00
Code Activity
Files
cd3657cf06bb3a32756deea35ea2b6cdfd19eb41
ruby/vm.c
T
Matt Valentine-House 2ad6b99a98 GC: scope rb_gc_register_address to the calling Ractor 
The registered_globals list made every Ractor's GC scan every slot under
a shared lock. This commit splits it into per-Ractor lists.

We also maintain a global registry that references every Ractor that has
ever registered so that unregistering from a Ractor other than the one
that registered can look up the owning list.

When a Ractor is joined, the dead Ractor's slots are moved to the
joining Ractor's heap before the objspace merge happens.

This change tightens up the requirements for values stored in a
registered address.

Previously, Every Ractor's local GC scanned the shared list. So if a
slot held an unshareable object living in a different Ractor's heap,
that object was always kept alive regardless.

After this change, only the registering Ractor's GC scans its list. So if a
slot holds another Ractor's unshareable object, the owning Ractor's
local GC cannot see that reference, so it can collect the object while
the registered slot still points at it.

Storing such an object was already a Ractor containment violation. It
was just hidden by the shared scan.

[Feature #22277]
2026-09-25 19:24:10 +01:00

5413 lines
164 KiB
C
Raw Blame History

/**********************************************************************
Vm.c -
$Author$
Copyright (C) 2004-2007 Koichi Sasada
**********************************************************************/
#define vm_exec rb_vm_exec
#include "eval_intern.h"
#include "internal.h"
#include "internal/box.h"
#include "internal/class.h"
#include "internal/compile.h"
#include "internal/cont.h"
#include "internal/error.h"
#include "internal/encoding.h"
#include "internal/eval.h"
#include "internal/gc.h"
#include "internal/inits.h"
#include "internal/jit.h"
#include "internal/missing.h"
#include "internal/object.h"
#include "internal/proc.h"
#include "internal/ractor.h"
#include "internal/re.h"
#include "internal/ruby_parser.h"
#include "internal/st.h"
#include "internal/symbol.h"
#include "internal/thread.h"
#include "internal/transcode.h"
#include "internal/vm.h"
#include "internal/sanitizers.h"
#include "internal/variable.h"
#include "iseq.h"
#include "symbol.h" // This includes a macro for a more performant rb_id2sym.
#include "yjit.h"
#include "insns.inc"
#include "zjit.h"
#include "ruby/st.h"
#include "ruby/vm.h"
#include "vm_core.h"
#include "vm_callinfo.h"
#include "vm_debug.h"
#include "ruby/debug.h"
#include "vm_exec.h"
#include "vm_insnhelper.h"
#include "ractor_core.h"
#include "vm_sync.h"
#include "shape.h"
#include "builtin.h"
#include "probes.h"
#include "probes_helper.h"
static void *native_main_thread_stack_top;
bool ruby_vm_during_cleanup = false;
VALUE rb_str_concat_literals(size_t, const VALUE*);
VALUE vm_exec(rb_execution_context_t *);
extern const char *const rb_debug_counter_names[];
PUREFUNC(static inline const VALUE *VM_EP_LEP(const VALUE *));
static inline const VALUE *
VM_EP_LEP(const VALUE *ep)
{
while (!VM_ENV_LOCAL_P(ep)) {
ep = VM_ENV_PREV_EP(ep);
}
return ep;
}
static inline const rb_control_frame_t *
rb_vm_search_cf_from_ep(const rb_execution_context_t *ec, const rb_control_frame_t *cfp, const VALUE * const ep)
{
if (!ep) {
return NULL;
}
else {
const rb_control_frame_t * const eocfp = RUBY_VM_END_CONTROL_FRAME(ec); /* end of control frame pointer */
while (cfp < eocfp) {
if (cfp->ep == ep) {
return cfp;
}
cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
}
return NULL;
}
}
#if VM_CHECK_MODE > 0
// ruby_box_crashed defined in internal/box.h
#define VM_BOX_CRASHED() {ruby_box_crashed = true;}
#define VM_BOX_ASSERT(expr, msg) \
if (!(expr)) { ruby_box_crashed = true; rb_bug(msg); }
#else
#define VM_BOX_CRASHED() {}
#define VM_BOX_ASSERT(expr, msg) ((void)0)
#endif
static const VALUE *
VM_EP_RUBY_LEP(const rb_execution_context_t *ec, const rb_control_frame_t *current_cfp)
{
// rb_vmdebug_box_env_dump_raw() simulates this function
const VALUE *ep = current_cfp->ep;
const rb_control_frame_t * const eocfp = RUBY_VM_END_CONTROL_FRAME(ec); /* end of control frame pointer */
const rb_control_frame_t *cfp = current_cfp;
/**
* For IFUNC frames, returns the ep of the enclosing CFUNC frame.
*
* Usually CFUNC frame doesn't represent the current box and it should operate
* the caller box. See the example:
*
* # in the main box
* module Kernel
* def foo = "foo"
* module_function :foo
* end
*
* In the case above, `module_function` is defined in the root box.
* If `module_function` worked in the root box, `Kernel#foo` is invisible
* from it and it causes NameError: undefined method `foo` for module `Kernel`.
*
* But in cases of IFUNC (blocks written in C), IFUNC doesn't have its own box
* and its local env frame will be CFUNC frame.
* For example, `Enumerator#chunk` calls IFUNC blocks, written as `chunk_i` function.
*
* [1].chunk{ it.even? }.each{ ... }
*
* Before calling the Ruby block `{ it.even? }`, `#chunk` calls `chunk_i` as IFUNC
* to iterate the array's members (it's just like `#each`).
* We expect that `chunk_i` works as expected by the implementation of `#chunk`
* without any overwritten definitions from boxes.
* So the definitions on IFUNC frames should be equal to the caller CFUNC.
*
* NOTE: We traverse the cfp chain directly instead of using VM_EP_LEP.
* When an IFUNC env is escaped to the heap (e.g., due to a surrounding
* `binding` call), the env may acquire VM_ENV_FLAG_LOCAL, causing VM_EP_LEP
* to return the IFUNC ep itself rather than the enclosing CFUNC ep.
*
* NOTE: An IFUNC may also have no enclosing CFUNC frame at all, when an
* ifunc proc is invoked directly from Ruby code (e.g. Proc#call on a proc
* created by Method#to_proc). In that case the caller Ruby frame
* determines the box, so continue to the local ep walk below.
*/
while (VM_ENV_FRAME_TYPE_P(ep, VM_FRAME_MAGIC_IFUNC) ||
VM_ENV_FRAME_TYPE_P(ep, VM_FRAME_MAGIC_CFUNC)) {
bool from_ifunc = VM_ENV_FRAME_TYPE_P(ep, VM_FRAME_MAGIC_IFUNC);
cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
VM_BOX_ASSERT(RUBY_VM_VALID_CONTROL_FRAME_P(cfp, eocfp), "Valid caller control frame expected");
if (!RUBY_VM_VALID_CONTROL_FRAME_P(cfp, eocfp)) {
return NULL;
}
VM_BOX_ASSERT(cfp->ep, "Caller control frame should have a valid env");
ep = cfp->ep;
if (!ep) {
return NULL;
}
if (from_ifunc && VM_ENV_FRAME_TYPE_P(ep, VM_FRAME_MAGIC_CFUNC)) {
return ep;
}
}
while (!VM_ENV_LOCAL_P(ep)) {
ep = VM_ENV_PREV_EP(ep);
}
return ep;
}
const VALUE *
rb_vm_ep_local_ep(const VALUE *ep)
{
return VM_EP_LEP(ep);
}
PUREFUNC(static inline const VALUE *VM_CF_LEP(const rb_control_frame_t * const cfp));
static inline const VALUE *
VM_CF_LEP(const rb_control_frame_t * const cfp)
{
return VM_EP_LEP(cfp->ep);
}
static inline const VALUE *
VM_CF_PREV_EP(const rb_control_frame_t * const cfp)
{
return VM_ENV_PREV_EP(cfp->ep);
}
PUREFUNC(static inline VALUE VM_CF_BLOCK_HANDLER(const rb_control_frame_t * const cfp));
static inline VALUE
VM_CF_BLOCK_HANDLER(const rb_control_frame_t * const cfp)
{
const VALUE *ep;
if (VM_ENV_BOXED_P(cfp->ep)) {
VM_ASSERT(VM_ENV_LOCAL_P(cfp->ep));
/* Never set black_handler for VM_FRAME_MAGIC_TOP or VM_FRAME_MAGIC_CLASS
* and the specval is used for boxes (rb_box_t) in these case
*/
return VM_BLOCK_HANDLER_NONE;
}
ep = VM_CF_LEP(cfp);
return VM_ENV_BLOCK_HANDLER(ep);
}
int
rb_vm_cframe_keyword_p(const rb_control_frame_t *cfp)
{
return VM_FRAME_CFRAME_KW_P(cfp);
}
VALUE
rb_vm_frame_block_handler(const rb_control_frame_t *cfp)
{
return VM_CF_BLOCK_HANDLER(cfp);
}
#if VM_CHECK_MODE > 0
static int
VM_CFP_IN_HEAP_P(const rb_execution_context_t *ec, const rb_control_frame_t *cfp)
{
const VALUE *start = ec->vm_stack;
const VALUE *end = (VALUE *)ec->vm_stack + ec->vm_stack_size;
VM_ASSERT(start != NULL);
if (start <= (VALUE *)cfp && (VALUE *)cfp < end) {
return FALSE;
}
else {
return TRUE;
}
}
static int
VM_EP_IN_HEAP_P(const rb_execution_context_t *ec, const VALUE *ep)
{
const VALUE *start = ec->vm_stack;
const VALUE *end = (VALUE *)ec->cfp;
VM_ASSERT(start != NULL);
if (start <= ep && ep < end) {
return FALSE;
}
else {
return TRUE;
}
}
static int
vm_ep_in_heap_p_(const rb_execution_context_t *ec, const VALUE *ep)
{
if (VM_EP_IN_HEAP_P(ec, ep)) {
VALUE envval = ep[VM_ENV_DATA_INDEX_ENV]; /* VM_ENV_ENVVAL(ep); */
if (!UNDEF_P(envval)) {
const rb_env_t *env = (const rb_env_t *)envval;
VM_ASSERT(imemo_type_p(envval, imemo_env));
VM_ASSERT(VM_ENV_FLAGS(ep, VM_ENV_FLAG_ESCAPED));
VM_ASSERT(env->ep == ep);
}
return TRUE;
}
else {
return FALSE;
}
}
int
rb_vm_ep_in_heap_p(const VALUE *ep)
{
const rb_execution_context_t *ec = rb_current_execution_context(false);
if (ec == NULL || ec->vm_stack == NULL) return TRUE;
return vm_ep_in_heap_p_(ec, ep);
}
#endif
static struct rb_captured_block *
VM_CFP_TO_CAPTURED_BLOCK(const rb_control_frame_t *cfp)
{
VM_ASSERT(!VM_CFP_IN_HEAP_P(GET_EC(), cfp));
return (struct rb_captured_block *)&cfp->self;
}
static rb_control_frame_t *
VM_CAPTURED_BLOCK_TO_CFP(const struct rb_captured_block *captured)
{
rb_control_frame_t *cfp = ((rb_control_frame_t *)((VALUE *)(captured) - 3));
VM_ASSERT(!VM_CFP_IN_HEAP_P(GET_EC(), cfp));
VM_ASSERT(sizeof(rb_control_frame_t)/sizeof(VALUE) == 7 + VM_DEBUG_BP_CHECK ? 1 : 0);
return cfp;
}
static int
VM_BH_FROM_CFP_P(VALUE block_handler, const rb_control_frame_t *cfp)
{
const struct rb_captured_block *captured = VM_CFP_TO_CAPTURED_BLOCK(cfp);
return VM_TAGGED_PTR_REF(block_handler, 0x03) == captured;
}
static VALUE
vm_passed_block_handler(rb_execution_context_t *ec)
{
VALUE block_handler = ec->passed_block_handler;
ec->passed_block_handler = VM_BLOCK_HANDLER_NONE;
vm_block_handler_verify(block_handler);
return block_handler;
}
static rb_cref_t *
vm_cref_new0(VALUE klass, rb_method_visibility_t visi, int module_func, rb_cref_t *prev_cref, int pushed_by_eval, int use_prev_prev, int singleton)
{
VALUE refinements = Qnil;
int omod_shared = FALSE;
/* scope */
rb_scope_visibility_t scope_visi;
scope_visi.method_visi = visi;
scope_visi.module_func = module_func;
/* refinements */
if (prev_cref != NULL && prev_cref != (void *)1 /* TODO: why CREF_NEXT(cref) is 1? */) {
refinements = CREF_REFINEMENTS(prev_cref);
if (!NIL_P(refinements)) {
omod_shared = TRUE;
CREF_OMOD_SHARED_SET(prev_cref);
}
}
VM_ASSERT(singleton || klass);
rb_cref_t *cref = SHAREABLE_IMEMO_NEW(rb_cref_t, imemo_cref, refinements);
/* A cref is born shareable, so possibly-unshareable children (a singleton cref's
* self, `using`'s refinements hash) go through the write barrier to record a shref;
* a plain store would let the owner's local GC collect the child under the pinned
* cref. next is always a cref (shareable): plain store. */
if (!SPECIAL_CONST_P(refinements)) RB_OBJ_WRITTEN(cref, Qundef, refinements);
if (klass) {
RB_OBJ_WRITE(cref, &cref->klass_or_self, klass);
}
else {
cref->klass_or_self = 0;
}
cref->next = use_prev_prev ? CREF_NEXT(prev_cref) : prev_cref;
*((rb_scope_visibility_t *)&cref->scope_visi) = scope_visi;
if (pushed_by_eval) CREF_PUSHED_BY_EVAL_SET(cref);
if (omod_shared) CREF_OMOD_SHARED_SET(cref);
if (singleton) CREF_SINGLETON_SET(cref);
return cref;
}
static rb_cref_t *
vm_cref_new(VALUE klass, rb_method_visibility_t visi, int module_func, rb_cref_t *prev_cref, int pushed_by_eval, int singleton)
{
return vm_cref_new0(klass, visi, module_func, prev_cref, pushed_by_eval, FALSE, singleton);
}
static int
ref_delete_symkey(VALUE key, VALUE value, VALUE unused)
{
return SYMBOL_P(key) ? ST_DELETE : ST_CONTINUE;
}
rb_cref_t *
rb_vm_cref_dup(const rb_cref_t *cref)
{
const rb_scope_visibility_t *visi = CREF_SCOPE_VISI(cref);
rb_cref_t *next_cref = CREF_NEXT(cref), *new_cref;
int pushed_by_eval = CREF_PUSHED_BY_EVAL(cref);
int singleton = CREF_SINGLETON(cref);
new_cref = vm_cref_new(cref->klass_or_self, visi->method_visi, visi->module_func, next_cref, pushed_by_eval, singleton);
if (!NIL_P(CREF_REFINEMENTS(cref))) {
VALUE ref = rb_hash_dup(CREF_REFINEMENTS(cref));
rb_hash_foreach(ref, ref_delete_symkey, Qnil);
CREF_REFINEMENTS_SET(new_cref, ref);
CREF_OMOD_SHARED_UNSET(new_cref);
}
return new_cref;
}
rb_cref_t *
rb_vm_cref_dup_without_refinements(const rb_cref_t *cref)
{
const rb_scope_visibility_t *visi = CREF_SCOPE_VISI(cref);
rb_cref_t *next_cref = CREF_NEXT(cref), *new_cref;
int pushed_by_eval = CREF_PUSHED_BY_EVAL(cref);
int singleton = CREF_SINGLETON(cref);
new_cref = vm_cref_new(cref->klass_or_self, visi->method_visi, visi->module_func, next_cref, pushed_by_eval, singleton);
if (!NIL_P(CREF_REFINEMENTS(cref))) {
CREF_REFINEMENTS_SET(new_cref, Qnil);
CREF_OMOD_SHARED_UNSET(new_cref);
}
return new_cref;
}
static rb_cref_t *
vm_cref_new_toplevel(rb_execution_context_t *ec)
{
rb_cref_t *cref = vm_cref_new(rb_cObject, METHOD_VISI_PRIVATE /* toplevel visibility is private */, FALSE, NULL, FALSE, FALSE);
VALUE top_wrapper = rb_ec_thread_ptr(ec)->top_wrapper;
if (top_wrapper) {
cref = vm_cref_new(top_wrapper, METHOD_VISI_PRIVATE, FALSE, cref, FALSE, FALSE);
}
return cref;
}
rb_cref_t *
rb_vm_cref_new_toplevel(void)
{
return vm_cref_new_toplevel(GET_EC());
}
static void
vm_cref_dump(const char *mesg, const rb_cref_t *cref)
{
ruby_debug_printf("vm_cref_dump: %s (%p)\n", mesg, (void *)cref);
while (cref) {
VALUE path = rb_class_path(CREF_CLASS(cref));
ruby_debug_printf("= cref| klass: %.*s\n",
RSTRING_LENINT(path), RSTRING_PTR(path));
cref = CREF_NEXT(cref);
}
}
void
rb_vm_block_ep_update(VALUE obj, const struct rb_block *dst, const VALUE *ep)
{
*((const VALUE **)&dst->as.captured.ep) = ep;
RB_OBJ_WRITTEN(obj, Qundef, VM_ENV_ENVVAL(ep));
}
static void
vm_bind_update_env(VALUE bindval, rb_binding_t *bind, VALUE envval)
{
const rb_env_t *env = (rb_env_t *)envval;
RB_OBJ_WRITE(bindval, &bind->block.as.captured.code.iseq, env->iseq);
rb_vm_block_ep_update(bindval, &bind->block, env->ep);
}
#if VM_COLLECT_USAGE_DETAILS
static void vm_collect_usage_operand(int insn, int n, VALUE op);
static void vm_collect_usage_insn(int insn);
static void vm_collect_usage_register(int reg, int isset);
#endif
static VALUE vm_make_env_object(const rb_execution_context_t *ec, rb_control_frame_t *cfp);
static VALUE vm_invoke_bmethod(rb_execution_context_t *ec, rb_proc_t *proc, VALUE self,
int argc, const VALUE *argv, int kw_splat, VALUE block_handler,
const rb_callable_method_entry_t *me);
static VALUE vm_invoke_proc(rb_execution_context_t *ec, rb_proc_t *proc, VALUE self, int argc, const VALUE *argv, int kw_splat, VALUE block_handler, const rb_cref_t *cref);
#if USE_YJIT
// Counter to serve as a proxy for execution time, total number of calls
static unsigned int yjit_total_entry_hits = 0;
// Number of calls used to estimate how hot an ISEQ is
#define YJIT_CALL_COUNT_INTERV 20u
/// Test whether we are ready to compile an ISEQ or not
static inline bool
rb_yjit_threshold_hit(const rb_iseq_t *iseq, unsigned int entry_calls)
{
yjit_total_entry_hits += 1;
// Record the number of calls at the beginning of the interval
if (entry_calls + YJIT_CALL_COUNT_INTERV == rb_yjit_call_threshold) {
ISEQ_BODY(iseq)->yjit_calls_at_interv = yjit_total_entry_hits;
}
// Try to estimate the total time taken (total number of calls) to reach 20 calls to this ISEQ
// This give us a ratio of how hot/cold this ISEQ is
if (entry_calls == rb_yjit_call_threshold) {
// We expect threshold 1 to compile everything immediately
if (rb_yjit_call_threshold < YJIT_CALL_COUNT_INTERV) {
return true;
}
unsigned int num_calls = yjit_total_entry_hits - ISEQ_BODY(iseq)->yjit_calls_at_interv;
// Reject ISEQs that don't get called often enough
if (num_calls > rb_yjit_cold_threshold) {
rb_yjit_incr_counter("cold_iseq_entry");
return false;
}
return true;
}
return false;
}
#else
#define rb_yjit_threshold_hit(iseq, entry_calls) false
#endif
#if USE_YJIT
// Generate JIT code that supports the following kinds of ISEQ entries:
// * The first ISEQ on vm_exec (e.g. <main>, or Ruby methods/blocks
// called by a C method). The current frame has VM_FRAME_FLAG_FINISH.
// The current vm_exec stops if JIT code returns a non-Qundef value.
// * ISEQs called by the interpreter on vm_sendish (e.g. Ruby methods or
// blocks called by a Ruby frame that isn't compiled or side-exited).
// The current frame doesn't have VM_FRAME_FLAG_FINISH. The current
// vm_exec does NOT stop whether JIT code returns Qundef or not.
static inline rb_jit_func_t
yjit_compile(rb_execution_context_t *ec)
{
const rb_iseq_t *iseq = CFP_ISEQ(ec->cfp);
struct rb_iseq_constant_body *body = ISEQ_BODY(iseq);
// Increment the ISEQ's call counter and trigger JIT compilation if not compiled.
// Stop incrementing when not compiling (out of executable memory) so that
// ISEQs that failed to compile don't keep dirtying CoW pages after fork.
if (body->jit_entry == NULL && rb_yjit_compiling_p) {
body->jit_entry_calls++;
if (rb_yjit_threshold_hit(iseq, body->jit_entry_calls)) {
rb_yjit_compile_iseq(iseq, ec, false);
}
}
return body->jit_entry;
}
#else
# define yjit_compile(ec) ((rb_jit_func_t)0)
#endif
#if USE_ZJIT
static inline rb_jit_func_t
zjit_compile(rb_execution_context_t *ec)
{
const rb_iseq_t *iseq = CFP_ISEQ(ec->cfp);
struct rb_iseq_constant_body *body = ISEQ_BODY(iseq);
if (body->jit_entry == NULL && rb_zjit_compiling_p) {
body->jit_entry_calls++;
// At profile-threshold, rewrite some of the YARV instructions
// to zjit_* instructions to profile these instructions.
if (body->jit_entry_calls == rb_zjit_profile_threshold) {
rb_zjit_profile_enable(iseq);
}
// At call-threshold, compile the ISEQ with ZJIT.
if (body->jit_entry_calls == rb_zjit_call_threshold) {
rb_zjit_compile_iseq(iseq, ec, false);
}
}
return body->jit_entry;
}
#else
# define zjit_compile(ec) ((rb_jit_func_t)0)
#endif
#if USE_YJIT || USE_ZJIT
// Execute JIT code compiled by yjit_compile() or zjit_compile()
static inline VALUE
jit_exec(rb_execution_context_t *ec)
{
#if USE_YJIT
if (rb_yjit_enabled_p) {
rb_jit_func_t func = yjit_compile(ec);
if (func) {
return func(ec, ec->cfp);
}
return Qundef;
}
#endif
#if USE_ZJIT
void *zjit_entry = rb_zjit_entry;
if (zjit_entry) {
rb_jit_func_t func = zjit_compile(ec);
if (func) {
VALUE result = ((rb_zjit_func_t)zjit_entry)(ec, ec->cfp, func);
return result;
}
}
#endif
return Qundef;
}
// Generate JIT code that supports the following kind of ISEQ entry:
// * The first ISEQ pushed by vm_exec_handle_exception. The frame would
// point to a location specified by a catch table, and it doesn't have
// VM_FRAME_FLAG_FINISH. The current vm_exec stops if JIT code returns
// a non-Qundef value. So you should not return a non-Qundef value
// until ec->cfp is changed to a frame with VM_FRAME_FLAG_FINISH.
static inline rb_jit_func_t
jit_compile_exception(rb_execution_context_t *ec)
{
const rb_iseq_t *iseq = CFP_ISEQ(ec->cfp);
struct rb_iseq_constant_body *body = ISEQ_BODY(iseq);
#if USE_ZJIT
// rb_zjit_compiling_p is false until ZJIT is enabled, so no
// rb_zjit_enabled_p check is needed here.
if (body->jit_exception == NULL && rb_zjit_compiling_p) {
body->jit_exception_calls++;
// At profile-threshold, rewrite some of the YARV instructions
// to zjit_* instructions to profile these instructions.
if (body->jit_exception_calls == rb_zjit_profile_threshold) {
rb_zjit_profile_enable(iseq);
}
// At call-threshold, compile the ISEQ with ZJIT.
if (body->jit_exception_calls == rb_zjit_call_threshold) {
rb_zjit_compile_iseq(iseq, ec, true);
}
}
#endif
#if USE_YJIT
// Increment the ISEQ's call counter and trigger JIT compilation if not compiled.
// Like the ZJIT branch above, no rb_yjit_enabled_p check is needed here.
if (body->jit_exception == NULL && rb_yjit_compiling_p) {
body->jit_exception_calls++;
if (body->jit_exception_calls == rb_yjit_call_threshold) {
rb_yjit_compile_iseq(iseq, ec, true);
}
}
#endif
return body->jit_exception;
}
// Execute JIT code compiled by jit_compile_exception()
static inline VALUE
jit_exec_exception(rb_execution_context_t *ec)
{
rb_jit_func_t func = jit_compile_exception(ec);
if (func) {
// Call the JIT code
return func(ec, ec->cfp);
}
else {
return Qundef;
}
}
#else
# define jit_compile_exception(ec) ((rb_jit_func_t)0)
# define jit_exec(ec) Qundef
# define jit_exec_exception(ec) Qundef
#endif
static void add_opt_method_entry(const rb_method_entry_t *me);
#define RB_TYPE_2_P(obj, type1, type2) \
(RB_TYPE_P(obj, type1) || RB_TYPE_P(obj, type2))
#define RB_TYPE_3_P(obj, type1, type2, type3) \
(RB_TYPE_P(obj, type1) || RB_TYPE_P(obj, type2) || RB_TYPE_P(obj, type3))
#define VM_ASSERT_TYPE(obj, type) \
VM_ASSERT(RB_TYPE_P(obj, type), #obj ": %s", rb_obj_info(obj))
#define VM_ASSERT_TYPE2(obj, type1, type2) \
VM_ASSERT(RB_TYPE_2_P(obj, type1, type2), #obj ": %s", rb_obj_info(obj))
#define VM_ASSERT_TYPE3(obj, type1, type2, type3) \
VM_ASSERT(RB_TYPE_3_P(obj, type1, type2, type3), #obj ": %s", rb_obj_info(obj))
static const rb_box_t * current_box_on_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp);
#include "vm_insnhelper.c"
#include "vm_exec.c"
#include "vm_method.c"
#include "vm_eval.c"
#define PROCDEBUG 0
VALUE rb_cRubyVM;
VALUE rb_cThread;
VALUE rb_mRubyVMFrozenCore;
VALUE rb_block_param_proxy;
VALUE ruby_vm_const_missing_count = 0;
rb_vm_t *ruby_current_vm_ptr = NULL;
rb_ractor_t *ruby_single_main_ractor;
bool ruby_vm_keep_script_lines;
#ifdef RB_THREAD_LOCAL_SPECIFIER
RB_THREAD_LOCAL_SPECIFIER rb_execution_context_t *ruby_current_ec;
#ifdef RUBY_NT_SERIAL
RB_THREAD_LOCAL_SPECIFIER rb_atomic_t ruby_nt_serial;
#endif
// no-inline decl on vm_core.h
rb_execution_context_t *
rb_current_ec_noinline(void)
{
return ruby_current_ec;
}
void
rb_current_ec_set(rb_execution_context_t *ec)
{
ruby_current_ec = ec;
}
#ifdef RB_THREAD_CURRENT_EC_NOINLINE
rb_execution_context_t *
rb_current_ec(void)
{
return ruby_current_ec;
}
#endif
#else
native_tls_key_t ruby_current_ec_key;
// no-inline decl on vm_core.h
rb_execution_context_t *
rb_current_ec_noinline(void)
{
return native_tls_get(ruby_current_ec_key);
}
#endif
rb_event_flag_t ruby_vm_event_flags = 0;
rb_event_flag_t ruby_vm_event_enabled_global_flags = 0;
unsigned int ruby_vm_c_events_enabled = 0;
unsigned int ruby_vm_iseq_events_enabled = 0;
rb_serial_t ruby_vm_constant_cache_invalidations = 0;
rb_serial_t ruby_vm_constant_cache_misses = 0;
rb_serial_t ruby_vm_global_cvar_state = 1;
static const struct rb_callcache vm_empty_cc = {
.flags = T_IMEMO | (imemo_callcache << FL_USHIFT) | VM_CALLCACHE_UNMARKABLE,
.klass = Qundef,
.cme_ = NULL,
.call_ = vm_call_general,
.aux_ = {
.v = Qfalse,
}
};
static const struct rb_callcache vm_empty_cc_for_super = {
.flags = T_IMEMO | (imemo_callcache << FL_USHIFT) | VM_CALLCACHE_UNMARKABLE,
.klass = Qundef,
.cme_ = NULL,
.call_ = vm_call_super_method,
.aux_ = {
.v = Qfalse,
}
};
static void thread_free(void *ptr);
void
rb_vm_inc_const_missing_count(void)
{
ruby_vm_const_missing_count +=1;
}
int
rb_dtrace_setup(rb_execution_context_t *ec, VALUE klass, ID id,
struct ruby_dtrace_method_hook_args *args)
{
enum ruby_value_type type;
if (!klass) {
if (!ec) ec = GET_EC();
if (!rb_ec_frame_method_id_and_class(ec, &id, 0, &klass) || !klass)
return FALSE;
}
if (RB_TYPE_P(klass, T_ICLASS)) {
klass = RBASIC(klass)->klass;
}
else if (RCLASS_SINGLETON_P(klass)) {
klass = RCLASS_ATTACHED_OBJECT(klass);
if (NIL_P(klass)) return FALSE;
}
type = BUILTIN_TYPE(klass);
if (type == T_CLASS || type == T_ICLASS || type == T_MODULE) {
VALUE name = rb_class_path(klass);
const char *classname, *filename;
const char *methodname = rb_id2name(id);
if (methodname && (filename = rb_source_location_cstr(&args->line_no)) != 0) {
if (NIL_P(name) || !(classname = StringValuePtr(name)))
classname = "<unknown>";
args->classname = classname;
args->methodname = methodname;
args->filename = filename;
args->klass = klass;
args->name = name;
return TRUE;
}
}
return FALSE;
}
extern unsigned int redblack_buffer_size;
/*
* call-seq:
* RubyVM.stat -> Hash
* RubyVM.stat(hsh) -> hsh
* RubyVM.stat(Symbol) -> Numeric
*
* Returns a Hash containing implementation-dependent counters inside the VM.
*
* This hash includes information about method/constant caches:
*
* {
* :constant_cache_invalidations=>2,
* :constant_cache_misses=>14,
* :global_cvar_state=>27
* }
*
* If <tt>USE_DEBUG_COUNTER</tt> is enabled, debug counters will be included.
*
* The contents of the hash are implementation specific and may be changed in
* the future.
*
* This method is only expected to work on C Ruby.
*/
static VALUE
vm_stat(int argc, VALUE *argv, VALUE self)
{
static VALUE sym_constant_cache_invalidations, sym_constant_cache_misses, sym_global_cvar_state, sym_next_shape_id;
static VALUE sym_shape_cache_size;
VALUE arg = Qnil;
VALUE hash = Qnil, key = Qnil;
if (rb_check_arity(argc, 0, 1) == 1) {
arg = argv[0];
if (SYMBOL_P(arg))
key = arg;
else if (RB_TYPE_P(arg, T_HASH))
hash = arg;
else
rb_raise(rb_eTypeError, "non-hash or symbol given");
}
else {
hash = rb_hash_new();
}
#define S(s) sym_##s = ID2SYM(rb_intern_const(#s))
S(constant_cache_invalidations);
S(constant_cache_misses);
S(global_cvar_state);
S(next_shape_id);
S(shape_cache_size);
#undef S
#define SET(name, attr) \
if (key == sym_##name) \
return SERIALT2NUM(attr); \
else if (hash != Qnil) \
rb_hash_aset(hash, sym_##name, SERIALT2NUM(attr));
SET(constant_cache_invalidations, ruby_vm_constant_cache_invalidations);
SET(constant_cache_misses, ruby_vm_constant_cache_misses);
SET(global_cvar_state, ruby_vm_global_cvar_state);
SET(next_shape_id, (rb_serial_t)rb_shapes_count());
SET(shape_cache_size, (rb_serial_t)rb_shapes_cache_size());
#undef SET
#if USE_DEBUG_COUNTER
ruby_debug_counter_show_at_exit(FALSE);
for (size_t i = 0; i < RB_DEBUG_COUNTER_MAX; i++) {
const VALUE name = rb_sym_intern_ascii_cstr(rb_debug_counter_names[i]);
const VALUE boxed_value = SIZET2NUM(rb_debug_counter[i]);
if (key == name) {
return boxed_value;
}
else if (hash != Qnil) {
rb_hash_aset(hash, name, boxed_value);
}
}
#endif
if (!NIL_P(key)) { /* matched key should return above */
rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(key));
}
return hash;
}
/* control stack frame */
static void
vm_set_top_stack(rb_execution_context_t *ec, const rb_iseq_t *iseq, const rb_box_t *box)
{
if (ISEQ_BODY(iseq)->type != ISEQ_TYPE_TOP) {
rb_raise(rb_eTypeError, "Not a toplevel InstructionSequence");
}
/* for return */
vm_push_frame(ec, iseq, VM_FRAME_MAGIC_TOP | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH,
rb_ec_thread_ptr(ec)->top_self,
GC_GUARDED_PTR(box),
(VALUE)vm_cref_new_toplevel(ec), /* cref or me */
ISEQ_BODY(iseq)->iseq_encoded, ec->cfp->sp,
ISEQ_BODY(iseq)->local_table_size, ISEQ_BODY(iseq)->stack_max);
}
static void
vm_set_eval_stack(rb_execution_context_t *ec, const rb_iseq_t *iseq, const rb_cref_t *cref, const struct rb_block *base_block)
{
vm_push_frame(ec, iseq, VM_FRAME_MAGIC_EVAL | VM_FRAME_FLAG_FINISH,
vm_block_self(base_block), VM_GUARDED_PREV_EP(vm_block_ep(base_block)),
(VALUE)cref, /* cref or me */
ISEQ_BODY(iseq)->iseq_encoded,
ec->cfp->sp, ISEQ_BODY(iseq)->local_table_size,
ISEQ_BODY(iseq)->stack_max);
}
static void
vm_set_main_stack(rb_execution_context_t *ec, const rb_iseq_t *iseq)
{
VALUE toplevel_binding = rb_const_get(rb_cObject, rb_intern("TOPLEVEL_BINDING"));
rb_binding_t *bind;
GetBindingPtr(toplevel_binding, bind);
RUBY_ASSERT_MESG(bind, "TOPLEVEL_BINDING is not built");
vm_set_eval_stack(ec, iseq, 0, &bind->block);
/* save binding */
if (ISEQ_BODY(iseq)->local_table_size > 0) {
vm_bind_update_env(toplevel_binding, bind, vm_make_env_object(ec, ec->cfp));
}
}
rb_control_frame_t *
rb_vm_get_binding_creatable_next_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp)
{
while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) {
if (CFP_ISEQ(cfp)) {
return (rb_control_frame_t *)cfp;
}
cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
}
return 0;
}
rb_control_frame_t *
rb_vm_get_ruby_level_next_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp)
{
while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) {
if (VM_FRAME_RUBYFRAME_P(cfp)) {
return (rb_control_frame_t *)cfp;
}
cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
}
return 0;
}
static rb_control_frame_t *
vm_get_ruby_level_caller_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp)
{
if (VM_FRAME_RUBYFRAME_P(cfp)) {
return (rb_control_frame_t *)cfp;
}
cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) {
if (VM_FRAME_RUBYFRAME_P(cfp)) {
return (rb_control_frame_t *)cfp;
}
if (VM_ENV_FLAGS(cfp->ep, VM_FRAME_FLAG_PASSED) == FALSE) {
break;
}
cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
}
return 0;
}
void
rb_vm_pop_cfunc_frame(void)
{
rb_execution_context_t *ec = GET_EC();
rb_control_frame_t *cfp = ec->cfp;
const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(cfp);
EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_RETURN, cfp->self, me->def->original_id, me->called_id, me->owner, Qnil);
RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec, me->owner, me->def->original_id);
vm_pop_frame(ec, cfp, cfp->ep);
}
void
rb_vm_rewind_cfp(rb_execution_context_t *ec, rb_control_frame_t *cfp)
{
/* check skipped frame */
while (ec->cfp != cfp) {
#if VMDEBUG
printf("skipped frame: %s\n", vm_frametype_name(ec->cfp));
#endif
if (VM_FRAME_TYPE(ec->cfp) != VM_FRAME_MAGIC_CFUNC) {
rb_vm_pop_frame(ec);
}
else { /* unlikely path */
rb_vm_pop_cfunc_frame();
}
}
}
/* at exit */
void
ruby_vm_at_exit(void (*func)(rb_vm_t *))
{
rb_vm_t *vm = GET_VM();
rb_at_exit_list *nl = ALLOC(rb_at_exit_list);
nl->func = func;
nl->next = vm->at_exit;
vm->at_exit = nl;
}
static void
ruby_vm_run_at_exit_hooks(rb_vm_t *vm)
{
rb_at_exit_list *l = vm->at_exit;
while (l) {
rb_at_exit_list* t = l->next;
rb_vm_at_exit_func *func = l->func;
ruby_xfree(l);
l = t;
(*func)(vm);
}
}
/* Env */
static VALUE check_env_value(const rb_env_t *env);
static int
check_env(const rb_env_t *env)
{
fputs("---\n", stderr);
ruby_debug_printf("envptr: %p\n", (void *)&env->ep[0]);
ruby_debug_printf("envval: %10p ", (void *)env->ep[1]);
dp(env->ep[1]);
ruby_debug_printf("ep: %10p\n", (void *)env->ep);
if (rb_vm_env_prev_env(env)) {
fputs(">>\n", stderr);
check_env_value(rb_vm_env_prev_env(env));
fputs("<<\n", stderr);
}
return 1;
}
static VALUE
check_env_value(const rb_env_t *env)
{
if (check_env(env)) {
return (VALUE)env;
}
rb_bug("invalid env");
return Qnil; /* unreachable */
}
static VALUE
vm_block_handler_escape(const rb_execution_context_t *ec, VALUE block_handler)
{
switch (vm_block_handler_type(block_handler)) {
case block_handler_type_ifunc:
case block_handler_type_iseq:
return rb_vm_make_proc(ec, VM_BH_TO_CAPT_BLOCK(block_handler), rb_cProc);
case block_handler_type_symbol:
case block_handler_type_proc:
return block_handler;
}
VM_UNREACHABLE(vm_block_handler_escape);
return Qnil;
}
static VALUE
vm_make_env_each(const rb_execution_context_t * const ec, rb_control_frame_t *const cfp)
{
const VALUE * const ep = cfp->ep;
VALUE *env_body, *env_ep;
int local_size, env_size;
if (VM_ENV_ESCAPED_P(ep)) {
return VM_ENV_ENVVAL(ep);
}
if (!VM_ENV_LOCAL_P(ep)) {
const VALUE *prev_ep = VM_ENV_PREV_EP(ep);
if (!VM_ENV_ESCAPED_P(prev_ep)) {
rb_control_frame_t *prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
while (prev_cfp->ep != prev_ep) {
prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(prev_cfp);
VM_ASSERT(prev_cfp->ep != NULL);
}
vm_make_env_each(ec, prev_cfp);
VM_FORCE_WRITE_SPECIAL_CONST(&ep[VM_ENV_DATA_INDEX_SPECVAL], VM_GUARDED_PREV_EP(prev_cfp->ep));
}
}
else {
VM_ASSERT(VM_ENV_LOCAL_P(ep));
VALUE block_handler = VM_ENV_BLOCK_HANDLER(ep);
if (block_handler != VM_BLOCK_HANDLER_NONE) {
VALUE blockprocval = vm_block_handler_escape(ec, block_handler);
VM_STACK_ENV_WRITE(ep, VM_ENV_DATA_INDEX_SPECVAL, blockprocval);
}
}
const rb_iseq_t *iseq = CFP_ISEQ(cfp);
if (!VM_FRAME_RUBYFRAME_P(cfp)) {
local_size = VM_ENV_DATA_SIZE;
}
else {
local_size = ISEQ_BODY(iseq)->local_table_size;
if (ISEQ_BODY(iseq)->param.flags.forwardable && VM_ENV_LOCAL_P(cfp->ep)) {
int ci_offset = local_size - ISEQ_BODY(iseq)->param.size + VM_ENV_DATA_SIZE;
CALL_INFO ci = (CALL_INFO)VM_CF_LEP(cfp)[-ci_offset];
local_size += vm_ci_argc(ci);
}
local_size += VM_ENV_DATA_SIZE;
}
// Invalidate JIT code that assumes cfp->ep == vm_base_ptr(cfp).
// This is done before creating the imemo_env because VM_STACK_ENV_WRITE
// below leaves the on-stack ep in a state that is unsafe to GC.
// Once the enabled JIT has recorded this iseq's escape, the invalidations
// are no longer useful and can slow down Ractors.
if (VM_FRAME_RUBYFRAME_P(cfp) &&
!rbimpl_atomic_load(&ISEQ_BODY(iseq)->jit_ep_escape_recorded, RBIMPL_ATOMIC_RELAXED)) {
if (rb_yjit_enabled_p) rb_yjit_invalidate_ep_is_bp(iseq);
if (rb_zjit_enabled_p) rb_zjit_invalidate_no_ep_escape(iseq);
}
/*
* # local variables on a stack frame (N == local_size)
* [lvar1, lvar2, ..., lvarN, SPECVAL]
* ^
* ep[0]
*
* # moved local variables
* [lvar1, lvar2, ..., lvarN, SPECVAL, Envval, BlockProcval (if needed)]
* ^ ^
* env->env[0] ep[0]
*/
env_size = local_size +
1 /* envval */;
// Careful with order in the following sequence. Each allocation can move objects.
env_body = ALLOC_N(VALUE, env_size);
rb_env_t *env = IMEMO_NEW(rb_env_t, imemo_env, 0);
// Set up env without WB since it's brand new (similar to newobj_init(), newobj_fill())
MEMCPY(env_body, ep - (local_size - 1 /* specval */), VALUE, local_size);
env_ep = &env_body[local_size - 1 /* specval */];
env_ep[VM_ENV_DATA_INDEX_ENV] = (VALUE)env;
env->iseq = (rb_iseq_t *)(VM_FRAME_RUBYFRAME_P(cfp) ? iseq : NULL);
env->ep = env_ep;
env->env = env_body;
env->env_size = env_size;
cfp->ep = env_ep;
VM_ENV_FLAGS_SET(env_ep, VM_ENV_FLAG_ESCAPED | VM_ENV_FLAG_WB_REQUIRED);
VM_STACK_ENV_WRITE(ep, 0, (VALUE)env); /* GC mark */
#if 0
for (i = 0; i < local_size; i++) {
if (VM_FRAME_RUBYFRAME_P(cfp)) {
/* clear value stack for GC */
ep[-local_size + i] = 0;
}
}
#endif
return (VALUE)env;
}
static VALUE
vm_make_env_object(const rb_execution_context_t *ec, rb_control_frame_t *cfp)
{
VALUE envval = vm_make_env_each(ec, cfp);
if (PROCDEBUG) {
check_env_value((const rb_env_t *)envval);
}
return envval;
}
void
rb_vm_stack_to_heap(rb_execution_context_t *ec)
{
rb_control_frame_t *cfp = ec->cfp;
while ((cfp = rb_vm_get_binding_creatable_next_cfp(ec, cfp)) != 0) {
vm_make_env_object(ec, cfp);
cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
}
}
const rb_env_t *
rb_vm_env_prev_env(const rb_env_t *env)
{
const VALUE *ep = env->ep;
if (VM_ENV_LOCAL_P(ep)) {
return NULL;
}
else {
const VALUE *prev_ep = VM_ENV_PREV_EP(ep);
return VM_ENV_ENVVAL_PTR(prev_ep);
}
}
static int
collect_local_variables_in_iseq(const rb_iseq_t *iseq, const struct local_var_list *vars)
{
unsigned int i;
if (!iseq) return 0;
for (i = 0; i < ISEQ_BODY(iseq)->local_table_size; i++) {
local_var_list_add(vars, ISEQ_BODY(iseq)->local_table[i]);
}
return 1;
}
static void
collect_local_variables_in_env(const rb_env_t *env, const struct local_var_list *vars)
{
do {
if (VM_ENV_FLAGS(env->ep, VM_ENV_FLAG_ISOLATED)) break;
collect_local_variables_in_iseq(env->iseq, vars);
} while ((env = rb_vm_env_prev_env(env)) != NULL);
}
static int
vm_collect_local_variables_in_heap(const VALUE *ep, const struct local_var_list *vars)
{
if (VM_ENV_ESCAPED_P(ep)) {
collect_local_variables_in_env(VM_ENV_ENVVAL_PTR(ep), vars);
return 1;
}
else {
return 0;
}
}
VALUE
rb_vm_env_local_variables(const rb_env_t *env)
{
struct local_var_list vars;
local_var_list_init(&vars);
collect_local_variables_in_env(env, &vars);
return local_var_list_finish(&vars);
}
VALUE
rb_vm_env_numbered_parameters(const rb_env_t *env)
{
struct local_var_list vars;
local_var_list_init(&vars);
// if (VM_ENV_FLAGS(env->ep, VM_ENV_FLAG_ISOLATED)) break; // TODO: is this needed?
const rb_iseq_t *iseq = env->iseq;
unsigned int i;
if (!iseq) return 0;
for (i = 0; i < ISEQ_BODY(iseq)->local_table_size; i++) {
numparam_list_add(&vars, ISEQ_BODY(iseq)->local_table[i]);
}
return local_var_list_finish(&vars);
}
VALUE
rb_iseq_local_variables(const rb_iseq_t *iseq)
{
struct local_var_list vars;
local_var_list_init(&vars);
while (collect_local_variables_in_iseq(iseq, &vars)) {
iseq = ISEQ_BODY(iseq)->parent_iseq;
}
return local_var_list_finish(&vars);
}
/* Proc */
static VALUE
vm_proc_create_from_captured(VALUE klass,
const struct rb_captured_block *captured,
enum rb_block_type block_type,
int8_t is_from_method, int8_t is_lambda)
{
VALUE procval = rb_proc_alloc(klass, block_type);
rb_proc_t *proc = RTYPEDDATA_DATA(procval);
VM_ASSERT(VM_EP_IN_HEAP_P(GET_EC(), captured->ep));
/* copy block */
RB_OBJ_WRITE(procval, &proc->block.as.captured.code.val, captured->code.val);
RB_OBJ_WRITE(procval, &proc->block.as.captured.self, captured->self);
rb_vm_block_ep_update(procval, &proc->block, captured->ep);
vm_block_type_set(&proc->block, block_type);
proc->header.is_from_method = is_from_method;
proc->header.is_lambda = is_lambda;
return procval;
}
void
rb_vm_block_copy(VALUE obj, const struct rb_block *dst, const struct rb_block *src)
{
/* copy block */
switch (vm_block_type(src)) {
case block_type_iseq:
case block_type_ifunc:
RB_OBJ_WRITE(obj, &dst->as.captured.self, src->as.captured.self);
RB_OBJ_WRITE(obj, &dst->as.captured.code.val, src->as.captured.code.val);
rb_vm_block_ep_update(obj, dst, src->as.captured.ep);
break;
case block_type_symbol:
RB_OBJ_WRITE(obj, &dst->as.symbol, src->as.symbol);
break;
case block_type_proc:
RB_OBJ_WRITE(obj, &dst->as.proc, src->as.proc);
break;
}
}
static VALUE
proc_create(VALUE klass, const struct rb_block *block, int8_t is_from_method, int8_t is_lambda)
{
VALUE procval = rb_proc_alloc(klass, block->type);
rb_proc_t *proc = RTYPEDDATA_DATA(procval);
VM_ASSERT(VM_EP_IN_HEAP_P(GET_EC(), vm_block_ep(block)));
rb_vm_block_copy(procval, &proc->block, block);
vm_block_type_set(&proc->block, block->type);
proc->header.is_from_method = is_from_method;
proc->header.is_lambda = is_lambda;
return procval;
}
VALUE
rb_proc_dup_0(VALUE self)
{
VALUE procval;
rb_proc_t *src;
GetProcPtr(self, src);
switch (vm_block_type(&src->block)) {
case block_type_ifunc:
procval = rb_func_proc_dup(self);
break;
default:
procval = proc_create(rb_obj_class(self), &src->block, src->header.is_from_method, src->header.is_lambda);
break;
}
if (src->header.is_refined) {
rb_proc_t *dst;
GetProcPtr(procval, dst);
dst->header.is_refined = 1;
}
if (RB_OBJ_SHAREABLE_P(self)) RB_OBJ_SET_SHAREABLE(procval);
RB_GC_GUARD(self); /* for: body = rb_proc_dup(body) */
return procval;
}
VALUE
rb_proc_dup(VALUE self)
{
VALUE procval = rb_proc_dup_0(self);
VALUE recipe = rb_proc_refinements_recipe(self);
if (!NIL_P(recipe)) rb_proc_set_refinements_recipe(procval, recipe);
return procval;
}
/* Proc#refined: build a Proc that runs `iseq` with the refinements of
* `recipe`, sharing self's environment. `iseq` is normally self's own block
* iseq, which the copy replaces on the first call. */
VALUE
rb_proc_dup_with_iseq_and_recipe(VALUE self, const rb_iseq_t *iseq, VALUE recipe)
{
rb_proc_t *src;
GetProcPtr(self, src);
VM_ASSERT(vm_block_type(&src->block) == block_type_iseq);
struct rb_block block = src->block;
block.as.captured.code.iseq = iseq;
VALUE procval = proc_create(rb_obj_class(self), &block, src->header.is_from_method, src->header.is_lambda);
rb_proc_set_refinements_recipe(procval, recipe);
RB_GC_GUARD(self);
return procval;
}
struct collect_outer_variable_name_data {
VALUE ary;
VALUE read_only;
bool yield;
bool isolate;
};
static VALUE
ID2NUM(ID id)
{
if (SIZEOF_VOIDP > SIZEOF_LONG)
return ULL2NUM(id);
else
return ULONG2NUM(id);
}
static ID
NUM2ID(VALUE num)
{
if (SIZEOF_VOIDP > SIZEOF_LONG)
return (ID)NUM2ULL(num);
else
return (ID)NUM2ULONG(num);
}
static enum rb_id_table_iterator_result
collect_outer_variable_names(ID id, VALUE val, void *ptr)
{
struct collect_outer_variable_name_data *data = (struct collect_outer_variable_name_data *)ptr;
if (id == rb_intern("yield")) {
data->yield = true;
}
else {
VALUE *store;
if (data->isolate ||
val == Qtrue /* write */) {
store = &data->ary;
}
else {
store = &data->read_only;
}
if (*store == Qfalse) *store = rb_ary_new();
rb_ary_push(*store, ID2NUM(id));
}
return ID_TABLE_CONTINUE;
}
static const rb_env_t *
env_copy(const VALUE *src_ep, VALUE read_only_variables)
{
const rb_env_t *src_env = (rb_env_t *)VM_ENV_ENVVAL(src_ep);
VM_ASSERT(src_env->ep == src_ep);
VALUE *env_body = ZALLOC_N(VALUE, src_env->env_size); // fill with Qfalse
VALUE *ep = &env_body[src_env->env_size - 2];
const rb_env_t *copied_env = vm_env_new(ep, env_body, src_env->env_size, src_env->iseq);
// Copy after allocations above, since they can move objects in src_ep.
VALUE svar_val = src_ep[VM_ENV_DATA_INDEX_ME_CREF];
if (imemo_type_p(svar_val, imemo_svar)) {
const struct vm_svar *svar = (struct vm_svar *)svar_val;
if (svar->cref_or_me) {
svar_val = svar->cref_or_me;
}
else {
svar_val = Qfalse;
}
}
RB_OBJ_WRITE(copied_env, &ep[VM_ENV_DATA_INDEX_ME_CREF], svar_val);
ep[VM_ENV_DATA_INDEX_FLAGS] = src_ep[VM_ENV_DATA_INDEX_FLAGS] | VM_ENV_FLAG_ISOLATED;
if (!VM_ENV_LOCAL_P(src_ep)) {
VM_ENV_FLAGS_SET(ep, VM_ENV_FLAG_LOCAL);
}
if (read_only_variables) {
for (int i=RARRAY_LENINT(read_only_variables)-1; i>=0; i--) {
ID id = NUM2ID(RARRAY_AREF(read_only_variables, i));
const struct rb_iseq_constant_body *body = ISEQ_BODY(src_env->iseq);
for (unsigned int j=0; j<body->local_table_size; j++) {
if (id == body->local_table[j]) {
// check reassignment
if (iseq_lvar_state_get(iseq_lvar_states(body), j) == lvar_reassigned) {
VALUE name = rb_id2str(id);
VALUE msg = rb_sprintf("cannot make a shareable Proc because "
"the outer variable '%" PRIsVALUE "' may be reassigned.", name);
rb_exc_raise(rb_exc_new_str(rb_eRactorIsolationError, msg));
}
// check shareable
VALUE v = src_env->env[j];
if (!rb_ractor_shareable_p(v)) {
VALUE name = rb_id2str(id);
VALUE msg = rb_sprintf("cannot make a shareable Proc because it can refer"
" unshareable object %+" PRIsVALUE " from ", v);
if (name)
rb_str_catf(msg, "variable '%" PRIsVALUE "'", name);
else
rb_str_cat_cstr(msg, "a hidden variable");
rb_exc_raise(rb_exc_new_str(rb_eRactorIsolationError, msg));
}
RB_OBJ_WRITE((VALUE)copied_env, &env_body[j], v);
rb_ary_delete_at(read_only_variables, i);
break;
}
}
}
}
if (!VM_ENV_LOCAL_P(src_ep)) {
const VALUE *prev_ep = VM_ENV_PREV_EP(src_env->ep);
const rb_env_t *new_prev_env = env_copy(prev_ep, read_only_variables);
ep[VM_ENV_DATA_INDEX_SPECVAL] = VM_GUARDED_PREV_EP(new_prev_env->ep);
RB_OBJ_WRITTEN(copied_env, Qundef, new_prev_env);
VM_ENV_FLAGS_UNSET(ep, VM_ENV_FLAG_LOCAL);
}
else if (VM_ENV_BOXED_P(src_ep)) {
// A TOP/CLASS local env stores its box, not a block handler, in the
// SPECVAL slot (VM_ENV_BOX). Preserve it: method lookup inside the
// isolated proc reads the box back via rb_current_box(), and a
// cleared slot dereferences a NULL box.
ep[VM_ENV_DATA_INDEX_SPECVAL] = src_ep[VM_ENV_DATA_INDEX_SPECVAL];
}
else {
ep[VM_ENV_DATA_INDEX_SPECVAL] = VM_BLOCK_HANDLER_NONE;
}
RB_OBJ_SET_SHAREABLE((VALUE)copied_env);
return copied_env;
}
static void
proc_isolate_env(VALUE self, rb_proc_t *proc, VALUE read_only_variables)
{
const struct rb_captured_block *captured = &proc->block.as.captured;
const rb_env_t *env = env_copy(captured->ep, read_only_variables);
*((const VALUE **)&proc->block.as.captured.ep) = env->ep;
RB_OBJ_WRITTEN(self, Qundef, env);
}
static int
proc_has_ivar_i(ID name, VALUE val, st_data_t arg)
{
if (rb_is_instance_id(name)) {
*(bool *)arg = true;
return ST_STOP;
}
return ST_CONTINUE;
}
static VALUE
proc_shared_outer_variables(struct rb_id_table *outer_variables, bool isolate, const char *message)
{
struct collect_outer_variable_name_data data = {
.isolate = isolate,
.ary = Qfalse,
.read_only = Qfalse,
.yield = false,
};
rb_id_table_foreach(outer_variables, collect_outer_variable_names, (void *)&data);
if (data.ary != Qfalse) {
VALUE str = rb_sprintf("can not %s because it accesses outer variables", message);
VALUE ary = data.ary;
const char *sep = " (";
for (long i = 0; i < RARRAY_LEN(ary); i++) {
VALUE name = rb_id2str(NUM2ID(RARRAY_AREF(ary, i)));
if (!name) continue;
rb_str_cat_cstr(str, sep);
sep = ", ";
rb_str_append(str, name);
}
if (*sep == ',') rb_str_cat_cstr(str, ")");
rb_str_cat_cstr(str, data.yield ? " and uses 'yield'." : ".");
rb_exc_raise(rb_exc_new_str(rb_eRactorIsolationError, str));
}
else if (data.yield) {
rb_raise(rb_eRactorIsolationError, "can not %s because it uses 'yield'.", message);
}
return data.read_only;
}
VALUE
rb_proc_isolate_bang(VALUE self, VALUE replace_self)
{
const rb_iseq_t *iseq = vm_proc_iseq(self);
if (iseq) {
rb_proc_t *proc = (rb_proc_t *)RTYPEDDATA_DATA(self);
if (proc->block.type != block_type_iseq) rb_raise(rb_eRuntimeError, "not supported yet");
if (!UNDEF_P(replace_self)) {
VM_ASSERT(rb_ractor_shareable_p(replace_self));
RB_OBJ_WRITE(self, &proc->block.as.captured.self, replace_self);
}
if (ISEQ_BODY(iseq)->outer_variables) {
proc_shared_outer_variables(ISEQ_BODY(iseq)->outer_variables, true, "isolate a Proc");
}
proc_isolate_env(self, proc, Qfalse);
proc->header.is_isolated = TRUE;
RB_OBJ_WRITE(self, &proc->block.as.captured.self, Qnil);
}
/* ivars are not traversed here, so their values may be unshareable */
if (UNLIKELY(rb_obj_shape_has_ivars(self))) {
bool has_ivar = false;
rb_ivar_foreach(self, proc_has_ivar_i, (st_data_t)&has_ivar);
if (has_ivar) {
rb_raise(rb_eRactorIsolationError, "can not isolate a Proc because it has instance variables");
}
}
RB_OBJ_SET_SHAREABLE(self);
return self;
}
VALUE
rb_proc_isolate(VALUE self)
{
VALUE dst = rb_proc_dup(self);
rb_proc_isolate_bang(dst, Qundef);
return dst;
}
VALUE
rb_proc_ractor_make_shareable(VALUE self, VALUE replace_self)
{
const rb_iseq_t *iseq = vm_proc_iseq(self);
if (iseq) {
rb_proc_t *proc = (rb_proc_t *)RTYPEDDATA_DATA(self);
if (proc->block.type != block_type_iseq) rb_raise(rb_eRuntimeError, "not supported yet");
if (!UNDEF_P(replace_self)) {
RB_OBJ_WRITE(self, &proc->block.as.captured.self, replace_self);
}
if (!rb_ractor_shareable_p(vm_block_self(&proc->block))) {
rb_raise(rb_eRactorIsolationError,
"Proc's self is not shareable: %" PRIsVALUE,
self);
}
VALUE read_only_variables = Qfalse;
if (ISEQ_BODY(iseq)->outer_variables) {
read_only_variables =
proc_shared_outer_variables(ISEQ_BODY(iseq)->outer_variables, false, "make a Proc shareable");
}
proc_isolate_env(self, proc, read_only_variables);
proc->header.is_isolated = TRUE;
}
else {
const struct rb_block *block = vm_proc_block(self);
if (block->type != block_type_symbol) rb_raise(rb_eRuntimeError, "not supported yet");
VALUE proc_self = vm_block_self(block);
if (!rb_ractor_shareable_p(proc_self)) {
rb_raise(rb_eRactorIsolationError,
"Proc's self is not shareable: %" PRIsVALUE,
self);
}
}
RB_OBJ_SET_FROZEN_SHAREABLE(self);
return self;
}
VALUE
rb_vm_make_proc_lambda(const rb_execution_context_t *ec, const struct rb_captured_block *captured, VALUE klass, int8_t is_lambda)
{
VALUE procval;
enum imemo_type code_type = imemo_type(captured->code.val);
if (!VM_ENV_ESCAPED_P(captured->ep)) {
rb_control_frame_t *cfp = VM_CAPTURED_BLOCK_TO_CFP(captured);
vm_make_env_object(ec, cfp);
}
VM_ASSERT(VM_EP_IN_HEAP_P(ec, captured->ep));
VM_ASSERT(code_type == imemo_iseq || code_type == imemo_ifunc);
procval = vm_proc_create_from_captured(klass, captured,
code_type == imemo_iseq ? block_type_iseq : block_type_ifunc,
FALSE, is_lambda);
if (code_type == imemo_ifunc) {
struct vm_ifunc *ifunc = (struct vm_ifunc *)captured->code.val;
if (ifunc->svar_lep) {
VALUE ep0 = ifunc->svar_lep[0];
if (RB_TYPE_P(ep0, T_IMEMO) && imemo_type_p(ep0, imemo_env)) {
// `ep0 == imemo_env` means this ep is escaped to heap (in env object).
const rb_env_t *env = (const rb_env_t *)ep0;
ifunc->svar_lep = (VALUE *)env->ep;
}
else {
VM_ASSERT(FIXNUM_P(ep0));
if (ep0 & VM_ENV_FLAG_ESCAPED) {
// ok. do nothing
}
else {
ifunc->svar_lep = NULL;
}
}
}
}
return procval;
}
/* Binding */
VALUE
rb_vm_make_binding(const rb_execution_context_t *ec, const rb_control_frame_t *src_cfp)
{
rb_control_frame_t *cfp = rb_vm_get_binding_creatable_next_cfp(ec, src_cfp);
rb_control_frame_t *ruby_level_cfp = rb_vm_get_ruby_level_next_cfp(ec, src_cfp);
VALUE bindval, envval;
rb_binding_t *bind;
if (cfp == 0 || ruby_level_cfp == 0) {
rb_raise(rb_eRuntimeError, "Can't create Binding Object on top of Fiber.");
}
if (!VM_FRAME_RUBYFRAME_P(src_cfp) &&
!VM_FRAME_RUBYFRAME_P(RUBY_VM_PREVIOUS_CONTROL_FRAME(src_cfp))) {
rb_raise(rb_eRuntimeError, "Cannot create Binding object for non-Ruby caller");
}
envval = vm_make_env_object(ec, cfp);
bindval = rb_binding_alloc(rb_cBinding);
GetBindingPtr(bindval, bind);
vm_bind_update_env(bindval, bind, envval);
RB_OBJ_WRITE(bindval, &bind->block.as.captured.self, cfp->self);
RB_OBJ_WRITE(bindval, &bind->block.as.captured.code.iseq, CFP_ISEQ(cfp));
RB_OBJ_WRITE(bindval, &bind->pathobj, ISEQ_BODY(CFP_ISEQ(ruby_level_cfp))->location.pathobj);
bind->first_lineno = rb_vm_get_sourceline(ruby_level_cfp);
return bindval;
}
const VALUE *
rb_binding_add_dynavars(VALUE bindval, rb_binding_t *bind, int dyncount, const ID *dynvars)
{
VALUE envval, pathobj = bind->pathobj;
VALUE path = pathobj_path(pathobj);
VALUE realpath = pathobj_realpath(pathobj);
const struct rb_block *base_block;
const rb_env_t *env;
rb_execution_context_t *ec = GET_EC();
const rb_iseq_t *base_iseq, *iseq;
rb_node_scope_t tmp_node;
if (dyncount < 0) return 0;
base_block = &bind->block;
base_iseq = vm_block_iseq(base_block);
VALUE idtmp = 0;
rb_ast_id_table_t *dyns = ALLOCV(idtmp, sizeof(rb_ast_id_table_t) + dyncount * sizeof(ID));
dyns->size = dyncount;
MEMCPY(dyns->ids, dynvars, ID, dyncount);
rb_node_init(RNODE(&tmp_node), NODE_SCOPE);
tmp_node.nd_tbl = dyns;
tmp_node.nd_body = 0;
tmp_node.nd_parent = NULL;
tmp_node.nd_args = 0;
VALUE ast_value = rb_ruby_ast_new(RNODE(&tmp_node));
if (base_iseq) {
iseq = rb_iseq_new(ast_value, ISEQ_BODY(base_iseq)->location.label, path, realpath, base_iseq, ISEQ_TYPE_EVAL);
}
else {
VALUE tempstr = rb_fstring_lit("<temp>");
iseq = rb_iseq_new_top(ast_value, tempstr, tempstr, tempstr, NULL);
}
tmp_node.nd_tbl = 0; /* reset table */
ALLOCV_END(idtmp);
vm_set_eval_stack(ec, iseq, 0, base_block);
vm_bind_update_env(bindval, bind, envval = vm_make_env_object(ec, ec->cfp));
rb_vm_pop_frame(ec);
env = (const rb_env_t *)envval;
return env->env;
}
/* C -> Ruby: block */
static inline void
invoke_block(rb_execution_context_t *ec, const rb_iseq_t *iseq, VALUE self, const struct rb_captured_block *captured, const rb_cref_t *cref, VALUE type, int opt_pc)
{
int arg_size = ISEQ_BODY(iseq)->param.size;
vm_push_frame(ec, iseq, type | VM_FRAME_FLAG_FINISH, self,
VM_GUARDED_PREV_EP(captured->ep),
(VALUE)cref, /* cref or method */
ISEQ_BODY(iseq)->iseq_encoded + opt_pc,
ec->cfp->sp + arg_size,
ISEQ_BODY(iseq)->local_table_size - arg_size,
ISEQ_BODY(iseq)->stack_max);
}
static inline void
invoke_bmethod(rb_execution_context_t *ec, const rb_iseq_t *iseq, VALUE self, const struct rb_captured_block *captured, const rb_callable_method_entry_t *me, VALUE type, int opt_pc)
{
/* bmethod call from outside the VM */
int arg_size = ISEQ_BODY(iseq)->param.size;
VM_ASSERT(me->def->type == VM_METHOD_TYPE_BMETHOD);
vm_push_frame(ec, iseq, type | VM_FRAME_FLAG_BMETHOD, self,
VM_GUARDED_PREV_EP(captured->ep),
(VALUE)me,
ISEQ_BODY(iseq)->iseq_encoded + opt_pc,
ec->cfp->sp + 1 /* self */ + arg_size,
ISEQ_BODY(iseq)->local_table_size - arg_size,
ISEQ_BODY(iseq)->stack_max);
VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);
}
ALWAYS_INLINE(static VALUE
invoke_iseq_block_from_c(rb_execution_context_t *ec, const struct rb_captured_block *captured,
VALUE self, int argc, const VALUE *argv, int kw_splat, VALUE passed_block_handler,
const rb_cref_t *cref, int is_lambda, const rb_callable_method_entry_t *me));
static inline VALUE
invoke_iseq_block_from_c(rb_execution_context_t *ec, const struct rb_captured_block *captured,
VALUE self, int argc, const VALUE *argv, int kw_splat, VALUE passed_block_handler,
const rb_cref_t *cref, int is_lambda, const rb_callable_method_entry_t *me)
{
const rb_iseq_t *iseq = rb_iseq_check(captured->code.iseq);
int opt_pc;
VALUE type = VM_FRAME_MAGIC_BLOCK | (is_lambda ? VM_FRAME_FLAG_LAMBDA : 0);
rb_control_frame_t *cfp = ec->cfp;
VALUE *sp = cfp->sp;
int flags = (kw_splat ? VM_CALL_KW_SPLAT : 0);
VALUE *use_argv = (VALUE *)argv;
VALUE av[2];
stack_check(ec);
if (UNLIKELY(argc > VM_ARGC_STACK_MAX) &&
(VM_ARGC_STACK_MAX >= 1 ||
/* Skip ruby array for potential autosplat case */
(argc != 1 || is_lambda))) {
use_argv = vm_argv_ruby_array(av, argv, &flags, &argc, kw_splat);
}
CHECK_VM_STACK_OVERFLOW(cfp, argc + 1);
vm_check_canary(ec, sp);
VALUE *stack_argv = sp;
if (me) {
*sp = self; // bemthods need `self` on the VM stack
stack_argv++;
}
cfp->sp = stack_argv + argc;
MEMCPY(stack_argv, use_argv, VALUE, argc); // restrict: new stack space
opt_pc = vm_yield_setup_args(ec, iseq, argc, stack_argv, flags, passed_block_handler,
(is_lambda ? arg_setup_method : arg_setup_block));
cfp->sp = sp;
if (me == NULL) {
invoke_block(ec, iseq, self, captured, cref, type, opt_pc);
}
else {
invoke_bmethod(ec, iseq, self, captured, me, type, opt_pc);
}
return vm_exec(ec);
}
static VALUE
invoke_block_from_c_bh(rb_execution_context_t *ec, VALUE block_handler,
int argc, const VALUE *argv,
int kw_splat, VALUE passed_block_handler, const rb_cref_t *cref,
int is_lambda, int force_blockarg)
{
again:
switch (vm_block_handler_type(block_handler)) {
case block_handler_type_iseq:
{
const struct rb_captured_block *captured = VM_BH_TO_ISEQ_BLOCK(block_handler);
return invoke_iseq_block_from_c(ec, captured, captured->self,
argc, argv, kw_splat, passed_block_handler,
cref, is_lambda, NULL);
}
case block_handler_type_ifunc:
return vm_yield_with_cfunc(ec, VM_BH_TO_IFUNC_BLOCK(block_handler),
VM_BH_TO_IFUNC_BLOCK(block_handler)->self,
argc, argv, kw_splat, passed_block_handler, NULL);
case block_handler_type_symbol:
return vm_yield_with_symbol(ec, VM_BH_TO_SYMBOL(block_handler),
argc, argv, kw_splat, passed_block_handler);
case block_handler_type_proc:
{
VALUE procval = VM_BH_TO_PROC(block_handler);
rb_proc_t *po;
GetProcPtr(procval, po);
if (po->header.is_refined) cref = rb_proc_refinements_cref_for_call(procval);
if (force_blockarg == FALSE) {
is_lambda = po->header.is_lambda;
}
block_handler = vm_block_to_block_handler(&po->block);
goto again;
}
}
VM_UNREACHABLE(invoke_block_from_c_splattable);
return Qundef;
}
static inline VALUE
check_block_handler(rb_execution_context_t *ec)
{
VALUE block_handler = VM_CF_BLOCK_HANDLER(ec->cfp);
vm_block_handler_verify(block_handler);
if (UNLIKELY(block_handler == VM_BLOCK_HANDLER_NONE)) {
rb_vm_localjump_error("no block given", Qnil, 0);
}
return block_handler;
}
static VALUE
vm_yield_with_cref(rb_execution_context_t *ec, int argc, const VALUE *argv, int kw_splat, const rb_cref_t *cref, int is_lambda)
{
return invoke_block_from_c_bh(ec, check_block_handler(ec),
argc, argv, kw_splat, VM_BLOCK_HANDLER_NONE,
cref, is_lambda, FALSE);
}
static VALUE
vm_yield(rb_execution_context_t *ec, int argc, const VALUE *argv, int kw_splat)
{
return vm_yield_with_cref(ec, argc, argv, kw_splat, NULL, FALSE);
}
static VALUE
vm_yield_with_block(rb_execution_context_t *ec, int argc, const VALUE *argv, VALUE block_handler, int kw_splat)
{
return invoke_block_from_c_bh(ec, check_block_handler(ec),
argc, argv, kw_splat, block_handler,
NULL, FALSE, FALSE);
}
static VALUE
vm_yield_force_blockarg(rb_execution_context_t *ec, VALUE args)
{
return invoke_block_from_c_bh(ec, check_block_handler(ec), 1, &args,
RB_NO_KEYWORDS, VM_BLOCK_HANDLER_NONE, NULL, FALSE, TRUE);
}
ALWAYS_INLINE(static VALUE
invoke_block_from_c_proc(rb_execution_context_t *ec, const rb_proc_t *proc,
VALUE self, int argc, const VALUE *argv,
int kw_splat, VALUE passed_block_handler, int is_lambda,
const rb_cref_t *cref,
const rb_callable_method_entry_t *me));
static inline VALUE
invoke_block_from_c_proc(rb_execution_context_t *ec, const rb_proc_t *proc,
VALUE self, int argc, const VALUE *argv,
int kw_splat, VALUE passed_block_handler, int is_lambda,
const rb_cref_t *cref,
const rb_callable_method_entry_t *me)
{
const struct rb_block *block = &proc->block;
again:
switch (vm_block_type(block)) {
case block_type_iseq:
return invoke_iseq_block_from_c(ec, &block->as.captured, self, argc, argv, kw_splat, passed_block_handler, cref, is_lambda, me);
case block_type_ifunc:
if (kw_splat == 1) {
VALUE keyword_hash = argv[argc-1];
if (!RB_TYPE_P(keyword_hash, T_HASH)) {
keyword_hash = rb_to_hash_type(keyword_hash);
}
if (RHASH_EMPTY_P(keyword_hash)) {
argc--;
}
else {
((VALUE *)argv)[argc-1] = rb_hash_dup(keyword_hash);
}
}
return vm_yield_with_cfunc(ec, &block->as.captured, self, argc, argv, kw_splat, passed_block_handler, me);
case block_type_symbol:
return vm_yield_with_symbol(ec, block->as.symbol, argc, argv, kw_splat, passed_block_handler);
case block_type_proc:
is_lambda = block_proc_is_lambda(block->as.proc);
block = vm_proc_block(block->as.proc);
goto again;
}
VM_UNREACHABLE(invoke_block_from_c_proc);
return Qundef;
}
static VALUE
vm_invoke_proc(rb_execution_context_t *ec, rb_proc_t *proc, VALUE self,
int argc, const VALUE *argv, int kw_splat, VALUE passed_block_handler,
const rb_cref_t *cref)
{
return invoke_block_from_c_proc(ec, proc, self, argc, argv, kw_splat, passed_block_handler, proc->header.is_lambda, cref, NULL);
}
static VALUE
vm_invoke_bmethod(rb_execution_context_t *ec, rb_proc_t *proc, VALUE self,
int argc, const VALUE *argv, int kw_splat, VALUE block_handler, const rb_callable_method_entry_t *me)
{
/* bmethod procs never carry a refinement cref (Proc#refined rejects them) */
return invoke_block_from_c_proc(ec, proc, self, argc, argv, kw_splat, block_handler, TRUE, NULL, me);
}
VALUE
rb_vm_invoke_proc(rb_execution_context_t *ec, rb_proc_t *proc,
int argc, const VALUE *argv, int kw_splat, VALUE passed_block_handler,
const rb_cref_t *cref)
{
VALUE self = vm_block_self(&proc->block);
vm_block_handler_verify(passed_block_handler);
if (proc->header.is_from_method) {
return vm_invoke_bmethod(ec, proc, self, argc, argv, kw_splat, passed_block_handler, NULL);
}
else {
return vm_invoke_proc(ec, proc, self, argc, argv, kw_splat, passed_block_handler, cref);
}
}
VALUE
rb_vm_invoke_proc_with_self(rb_execution_context_t *ec, rb_proc_t *proc, VALUE self,
int argc, const VALUE *argv, int kw_splat, VALUE passed_block_handler,
const rb_cref_t *cref)
{
vm_block_handler_verify(passed_block_handler);
if (proc->header.is_from_method) {
return vm_invoke_bmethod(ec, proc, self, argc, argv, kw_splat, passed_block_handler, NULL);
}
else {
return vm_invoke_proc(ec, proc, self, argc, argv, kw_splat, passed_block_handler, cref);
}
}
/* special variable */
VALUE *
rb_vm_svar_lep(const rb_execution_context_t *ec, const rb_control_frame_t *cfp)
{
while (!CFP_PC(cfp) || !CFP_ISEQ(cfp)) {
if (VM_FRAME_TYPE(cfp) == VM_FRAME_MAGIC_IFUNC) {
struct vm_ifunc *ifunc = (struct vm_ifunc *)CFP_ISEQ(cfp);
return ifunc->svar_lep;
}
else {
cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
}
if (RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) {
return NULL;
}
}
return (VALUE *)VM_CF_LEP(cfp);
}
static VALUE
vm_cfp_svar_get(const rb_execution_context_t *ec, rb_control_frame_t *cfp, VALUE key)
{
return lep_svar_get(ec, rb_vm_svar_lep(ec, cfp), key);
}
static void
vm_cfp_svar_set(const rb_execution_context_t *ec, rb_control_frame_t *cfp, VALUE key, const VALUE val)
{
lep_svar_set(ec, rb_vm_svar_lep(ec, cfp), key, val);
}
static VALUE
vm_svar_get(const rb_execution_context_t *ec, VALUE key)
{
return vm_cfp_svar_get(ec, ec->cfp, key);
}
static void
vm_svar_set(const rb_execution_context_t *ec, VALUE key, VALUE val)
{
vm_cfp_svar_set(ec, ec->cfp, key, val);
}
VALUE
rb_backref_get(void)
{
return vm_svar_get(GET_EC(), VM_SVAR_BACKREF);
}
void
rb_backref_set(VALUE val)
{
vm_svar_set(GET_EC(), VM_SVAR_BACKREF, val);
}
VALUE
rb_lastline_get(void)
{
return vm_svar_get(GET_EC(), VM_SVAR_LASTLINE);
}
void
rb_lastline_set(VALUE val)
{
vm_svar_set(GET_EC(), VM_SVAR_LASTLINE, val);
}
void
rb_lastline_set_up(VALUE val, unsigned int up)
{
rb_control_frame_t * cfp = GET_EC()->cfp;
for(unsigned int i = 0; i < up; i++) {
cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
}
vm_cfp_svar_set(GET_EC(), cfp, VM_SVAR_LASTLINE, val);
}
/* misc */
const char *
rb_sourcefile(void)
{
const rb_execution_context_t *ec = GET_EC();
const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
if (cfp) {
return RSTRING_PTR(rb_iseq_path(CFP_ISEQ(cfp)));
}
else {
return 0;
}
}
int
rb_sourceline(void)
{
const rb_execution_context_t *ec = GET_EC();
const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
if (cfp) {
return rb_vm_get_sourceline(cfp);
}
else {
return 0;
}
}
VALUE
rb_source_location(int *pline)
{
const rb_execution_context_t *ec = GET_EC();
const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
if (cfp && VM_FRAME_RUBYFRAME_P(cfp)) {
if (pline) *pline = rb_vm_get_sourceline(cfp);
return rb_iseq_path(CFP_ISEQ(cfp));
}
else {
if (pline) *pline = 0;
return Qnil;
}
}
const char *
rb_source_location_cstr(int *pline)
{
VALUE path = rb_source_location(pline);
if (NIL_P(path)) return NULL;
return RSTRING_PTR(path);
}
rb_cref_t *
rb_vm_cref(void)
{
const rb_execution_context_t *ec = GET_EC();
return vm_ec_cref(ec);
}
rb_cref_t *
rb_vm_cref_replace_with_duplicated_cref(void)
{
const rb_execution_context_t *ec = GET_EC();
const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
rb_cref_t *cref = vm_cref_replace_with_duplicated_cref(cfp->ep);
ASSUME(cref);
return cref;
}
const rb_cref_t *
rb_vm_cref_in_context(VALUE self, VALUE cbase)
{
const rb_execution_context_t *ec = GET_EC();
const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
const rb_cref_t *cref;
if (!cfp || cfp->self != self) return NULL;
if (!vm_env_cref_by_cref(cfp->ep)) return NULL;
cref = vm_get_cref(cfp->ep);
if (CREF_CLASS(cref) != cbase) return NULL;
return cref;
}
#if 0
void
debug_cref(rb_cref_t *cref)
{
while (cref) {
dp(CREF_CLASS(cref));
printf("%ld\n", CREF_VISI(cref));
cref = CREF_NEXT(cref);
}
}
#endif
VALUE
rb_vm_cbase(void)
{
const rb_execution_context_t *ec = GET_EC();
const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
if (cfp == 0) {
rb_raise(rb_eRuntimeError, "Can't call on top of Fiber or Thread");
}
return vm_get_cbase(cfp->ep);
}
/* jump */
static VALUE
make_localjump_error(const char *mesg, VALUE value, int reason)
{
extern VALUE rb_eLocalJumpError;
VALUE exc = rb_exc_new2(rb_eLocalJumpError, mesg);
ID id;
switch (reason) {
case TAG_BREAK:
CONST_ID(id, "break");
break;
case TAG_REDO:
CONST_ID(id, "redo");
break;
case TAG_RETRY:
CONST_ID(id, "retry");
break;
case TAG_NEXT:
CONST_ID(id, "next");
break;
case TAG_RETURN:
CONST_ID(id, "return");
break;
default:
CONST_ID(id, "noreason");
break;
}
rb_iv_set(exc, "@exit_value", value);
rb_iv_set(exc, "@reason", ID2SYM(id));
return exc;
}
void
rb_vm_localjump_error(const char *mesg, VALUE value, int reason)
{
VALUE exc = make_localjump_error(mesg, value, reason);
rb_exc_raise(exc);
}
VALUE
rb_vm_make_jump_tag_but_local_jump(enum ruby_tag_type state, VALUE val)
{
const char *mesg;
switch (state) {
case TAG_RETURN:
mesg = "unexpected return";
break;
case TAG_BREAK:
mesg = "unexpected break";
break;
case TAG_NEXT:
mesg = "unexpected next";
break;
case TAG_REDO:
mesg = "unexpected redo";
val = Qnil;
break;
case TAG_RETRY:
mesg = "retry outside of rescue clause";
val = Qnil;
break;
default:
return Qnil;
}
if (UNDEF_P(val)) {
val = GET_EC()->tag->retval;
}
return make_localjump_error(mesg, val, state);
}
void
rb_vm_jump_tag_but_local_jump(enum ruby_tag_type state)
{
VALUE exc = rb_vm_make_jump_tag_but_local_jump(state, Qundef);
if (!NIL_P(exc)) rb_exc_raise(exc);
EC_JUMP_TAG(GET_EC(), state);
}
static rb_control_frame_t *
next_not_local_frame(rb_control_frame_t *cfp)
{
while (VM_ENV_LOCAL_P(cfp->ep)) {
cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
}
return cfp;
}
NORETURN(static void vm_iter_break(rb_execution_context_t *ec, VALUE val));
static void
vm_iter_break(rb_execution_context_t *ec, VALUE val)
{
rb_control_frame_t *cfp = next_not_local_frame(ec->cfp);
const VALUE *ep = VM_CF_PREV_EP(cfp);
const rb_control_frame_t *target_cfp = rb_vm_search_cf_from_ep(ec, cfp, ep);
if (!target_cfp) {
rb_vm_localjump_error("unexpected break", val, TAG_BREAK);
}
ec->errinfo = (VALUE)THROW_DATA_NEW(val, target_cfp, TAG_BREAK);
EC_JUMP_TAG(ec, TAG_BREAK);
}
void
rb_iter_break(void)
{
vm_iter_break(GET_EC(), Qnil);
}
void
rb_iter_break_value(VALUE val)
{
vm_iter_break(GET_EC(), val);
}
/* optimization: redefine management */
short ruby_vm_redefined_flag[BOP_LAST_];
static st_table *vm_opt_method_def_table = 0;
static st_table *vm_opt_mid_table = 0;
static void
rb_free_vm_opt_tables(void)
{
st_free_table(vm_opt_method_def_table);
st_free_table(vm_opt_mid_table);
}
static int
vm_redefinition_check_flag(VALUE klass)
{
if (klass == rb_cInteger) return INTEGER_REDEFINED_OP_FLAG;
if (klass == rb_cFloat) return FLOAT_REDEFINED_OP_FLAG;
if (klass == rb_cString) return STRING_REDEFINED_OP_FLAG;
if (klass == rb_cArray) return ARRAY_REDEFINED_OP_FLAG;
if (klass == rb_cHash) return HASH_REDEFINED_OP_FLAG;
if (klass == rb_cSymbol) return SYMBOL_REDEFINED_OP_FLAG;
#if 0
if (klass == rb_cTime) return TIME_REDEFINED_OP_FLAG;
#endif
if (klass == rb_cRegexp) return REGEXP_REDEFINED_OP_FLAG;
if (klass == rb_cNilClass) return NIL_REDEFINED_OP_FLAG;
if (klass == rb_cTrueClass) return TRUE_REDEFINED_OP_FLAG;
if (klass == rb_cFalseClass) return FALSE_REDEFINED_OP_FLAG;
if (klass == rb_cProc) return PROC_REDEFINED_OP_FLAG;
return 0;
}
int
rb_vm_check_optimizable_mid(VALUE mid)
{
if (!vm_opt_mid_table) {
return FALSE;
}
return st_lookup(vm_opt_mid_table, mid, NULL);
}
static int
vm_redefinition_check_method_type(const rb_method_entry_t *me)
{
if (me->called_id != me->def->original_id) {
return FALSE;
}
if (METHOD_ENTRY_BASIC(me)) return TRUE;
const rb_method_definition_t *def = me->def;
switch (def->type) {
case VM_METHOD_TYPE_CFUNC:
case VM_METHOD_TYPE_OPTIMIZED:
return TRUE;
default:
return FALSE;
}
}
static void
rb_vm_check_redefinition_opt_method(const rb_method_entry_t *me, VALUE klass)
{
st_data_t bop;
if (RB_TYPE_P(klass, T_ICLASS) && RICLASS_IS_ORIGIN_P(klass) &&
RB_TYPE_P(RBASIC_CLASS(klass), T_CLASS)) {
klass = RBASIC_CLASS(klass);
}
if (vm_redefinition_check_method_type(me)) {
if (st_lookup(vm_opt_method_def_table, (st_data_t)me->def, &bop)) {
int flag = vm_redefinition_check_flag(klass);
if (flag != 0) {
rb_category_warn(
RB_WARN_CATEGORY_PERFORMANCE,
"Redefining '%s#%s' disables interpreter and JIT optimizations",
rb_class2name(me->owner),
rb_id2name(me->called_id)
);
rb_yjit_bop_redefined(flag, (enum ruby_basic_operators)bop);
rb_zjit_bop_redefined(flag, (enum ruby_basic_operators)bop);
ruby_vm_redefined_flag[bop] |= flag;
}
}
}
}
static enum rb_id_table_iterator_result
check_redefined_method(ID mid, VALUE value, void *data)
{
VALUE klass = (VALUE)data;
const rb_method_entry_t *me = (rb_method_entry_t *)value;
const rb_method_entry_t *newme = rb_method_entry(klass, mid);
if (newme != me) rb_vm_check_redefinition_opt_method(me, me->owner);
return ID_TABLE_CONTINUE;
}
void
rb_vm_check_redefinition_by_prepend(VALUE klass)
{
if (!vm_redefinition_check_flag(klass)) return;
rb_id_table_foreach(RCLASS_M_TBL(RCLASS_ORIGIN(klass)), check_redefined_method, (void *)klass);
}
static void
add_opt_method_entry_bop(const rb_method_entry_t *me, ID mid, enum ruby_basic_operators bop)
{
st_insert(vm_opt_method_def_table, (st_data_t)me->def, (st_data_t)bop);
st_insert(vm_opt_mid_table, (st_data_t)mid, (st_data_t)Qtrue);
}
static void
add_opt_method(VALUE klass, ID mid, enum ruby_basic_operators bop)
{
const rb_method_entry_t *me = rb_method_entry_at(klass, mid);
if (me && vm_redefinition_check_method_type(me)) {
add_opt_method_entry_bop(me, mid, bop);
}
else {
rb_bug("undefined optimized method: %s", rb_id2name(mid));
}
}
static enum ruby_basic_operators vm_redefinition_bop_for_id(ID mid);
static void
add_opt_method_entry(const rb_method_entry_t *me)
{
if (me && vm_redefinition_check_method_type(me)) {
ID mid = me->called_id;
enum ruby_basic_operators bop = vm_redefinition_bop_for_id(mid);
if ((int)bop >= 0) {
add_opt_method_entry_bop(me, mid, bop);
}
}
}
static void
vm_init_redefined_flag(void)
{
ID mid;
enum ruby_basic_operators bop;
#define OP(mid_, bop_) (mid = id##mid_, bop = BOP_##bop_, ruby_vm_redefined_flag[bop] = 0)
#define C(k) add_opt_method(rb_c##k, mid, bop)
OP(PLUS, PLUS), (C(Integer), C(Float), C(String), C(Array));
OP(MINUS, MINUS), (C(Integer), C(Float));
OP(MULT, MULT), (C(Integer), C(Float));
OP(DIV, DIV), (C(Integer), C(Float));
OP(MOD, MOD), (C(Integer), C(Float));
OP(Eq, EQ), (C(Integer), C(Float), C(String), C(Symbol));
OP(Eqq, EQQ), (C(Integer), C(Float), C(Symbol), C(String),
C(NilClass), C(TrueClass), C(FalseClass));
OP(LT, LT), (C(Integer), C(Float));
OP(LE, LE), (C(Integer), C(Float));
OP(GT, GT), (C(Integer), C(Float));
OP(GE, GE), (C(Integer), C(Float));
OP(LTLT, LTLT), (C(String), C(Array));
OP(GTGT, GTGT), (C(Integer));
OP(AREF, AREF), (C(Array), C(Hash), C(Integer));
OP(ASET, ASET), (C(Array), C(Hash));
OP(Length, LENGTH), (C(Array), C(String), C(Hash));
OP(Size, SIZE), (C(Array), C(String), C(Hash));
OP(EmptyP, EMPTY_P), (C(Array), C(String), C(Hash));
OP(Succ, SUCC), (C(Integer), C(String));
OP(EqTilde, MATCH), (C(Regexp), C(String));
OP(Freeze, FREEZE), (C(String), C(Array), C(Hash));
OP(UMinus, UMINUS), (C(String));
OP(Max, MAX), (C(Array));
OP(Min, MIN), (C(Array));
OP(Hash, HASH), (C(Array));
OP(Call, CALL), (C(Proc));
OP(Yield, YIELD), (C(Proc));
OP(And, AND), (C(Integer));
OP(Or, OR), (C(Integer));
OP(NilP, NIL_P), (C(NilClass));
OP(Cmp, CMP), (C(Integer), C(Float), C(String));
OP(Default, DEFAULT), (C(Hash));
OP(IncludeP, INCLUDE_P), (C(Array));
#undef C
#undef OP
}
static enum ruby_basic_operators
vm_redefinition_bop_for_id(ID mid)
{
switch (mid) {
#define OP(mid_, bop_) case id##mid_: return BOP_##bop_
OP(PLUS, PLUS);
OP(MINUS, MINUS);
OP(MULT, MULT);
OP(DIV, DIV);
OP(MOD, MOD);
OP(Eq, EQ);
OP(Eqq, EQQ);
OP(LT, LT);
OP(LE, LE);
OP(GT, GT);
OP(GE, GE);
OP(LTLT, LTLT);
OP(AREF, AREF);
OP(ASET, ASET);
OP(Length, LENGTH);
OP(Size, SIZE);
OP(EmptyP, EMPTY_P);
OP(Succ, SUCC);
OP(EqTilde, MATCH);
OP(Freeze, FREEZE);
OP(UMinus, UMINUS);
OP(Max, MAX);
OP(Min, MIN);
OP(Hash, HASH);
OP(Call, CALL);
OP(Yield, YIELD);
OP(And, AND);
OP(Or, OR);
OP(NilP, NIL_P);
OP(Cmp, CMP);
OP(Default, DEFAULT);
OP(Pack, PACK);
#undef OP
}
return -1;
}
/* for vm development */
#if VMDEBUG
static const char *
vm_frametype_name(const rb_control_frame_t *cfp)
{
switch (VM_FRAME_TYPE(cfp)) {
case VM_FRAME_MAGIC_METHOD: return "method";
case VM_FRAME_MAGIC_BLOCK: return "block";
case VM_FRAME_MAGIC_CLASS: return "class";
case VM_FRAME_MAGIC_TOP: return "top";
case VM_FRAME_MAGIC_CFUNC: return "cfunc";
case VM_FRAME_MAGIC_IFUNC: return "ifunc";
case VM_FRAME_MAGIC_EVAL: return "eval";
case VM_FRAME_MAGIC_RESCUE: return "rescue";
default:
rb_bug("unknown frame");
}
}
#endif
static VALUE
frame_return_value(const struct vm_throw_data *err)
{
if (THROW_DATA_P(err) &&
THROW_DATA_STATE(err) == TAG_BREAK &&
THROW_DATA_CONSUMED_P(err) == FALSE) {
return THROW_DATA_VAL(err);
}
else {
return Qnil;
}
}
#if 0
/* for debug */
static const char *
frame_name(const rb_control_frame_t *cfp)
{
unsigned long type = VM_FRAME_TYPE(cfp);
#define C(t) if (type == VM_FRAME_MAGIC_##t) return #t
C(METHOD);
C(BLOCK);
C(CLASS);
C(TOP);
C(CFUNC);
C(PROC);
C(IFUNC);
C(EVAL);
C(LAMBDA);
C(RESCUE);
C(DUMMY);
#undef C
return "unknown";
}
#endif
// cfp_returning_with_value:
// Whether cfp is the last frame in the unwinding process for a non-local return.
static void
hook_before_rewind(rb_execution_context_t *ec, bool cfp_returning_with_value, int state, struct vm_throw_data *err)
{
if (state == TAG_RAISE && RBASIC(err)->klass == rb_eSysStackError) {
return;
}
else {
const rb_iseq_t *iseq = CFP_ISEQ(ec->cfp);
rb_hook_list_t *local_hooks = NULL;
unsigned int local_hooks_cnt = iseq->aux.exec.local_hooks_cnt;
if (RB_UNLIKELY(local_hooks_cnt > 0)) {
local_hooks = rb_iseq_local_hooks(iseq, rb_ec_ractor_ptr(ec), false);
}
switch (VM_FRAME_TYPE(ec->cfp)) {
case VM_FRAME_MAGIC_METHOD:
RUBY_DTRACE_METHOD_RETURN_HOOK(ec, 0, 0);
EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_RETURN, ec->cfp->self, 0, 0, 0, frame_return_value(err));
if (UNLIKELY(local_hooks && local_hooks->events & RUBY_EVENT_RETURN)) {
rb_exec_event_hook_orig(ec, local_hooks, RUBY_EVENT_RETURN,
ec->cfp->self, 0, 0, 0, frame_return_value(err), TRUE);
}
THROW_DATA_CONSUMED_SET(err);
break;
case VM_FRAME_MAGIC_BLOCK:
if (VM_FRAME_BMETHOD_P(ec->cfp)) {
VALUE bmethod_return_value = frame_return_value(err);
if (cfp_returning_with_value) {
// Non-local return terminating at a BMETHOD control frame.
bmethod_return_value = THROW_DATA_VAL(err);
}
EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_B_RETURN, ec->cfp->self, 0, 0, 0, bmethod_return_value);
if (UNLIKELY(local_hooks && local_hooks->events & RUBY_EVENT_B_RETURN)) {
rb_exec_event_hook_orig(ec, local_hooks, RUBY_EVENT_B_RETURN,
ec->cfp->self, 0, 0, 0, bmethod_return_value, TRUE);
}
const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(ec->cfp);
EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_RETURN, ec->cfp->self,
rb_vm_frame_method_entry(ec->cfp)->def->original_id,
rb_vm_frame_method_entry(ec->cfp)->called_id,
rb_vm_frame_method_entry(ec->cfp)->owner,
bmethod_return_value);
VM_ASSERT(me->def->type == VM_METHOD_TYPE_BMETHOD);
unsigned int local_hooks_cnt = me->def->body.bmethod.local_hooks_cnt;
if (UNLIKELY(local_hooks_cnt > 0)) {
local_hooks = rb_method_def_local_hooks(me->def, rb_ec_ractor_ptr(ec), false);
if (local_hooks && local_hooks->events & RUBY_EVENT_RETURN) {
rb_exec_event_hook_orig(ec, local_hooks, RUBY_EVENT_RETURN, ec->cfp->self,
rb_vm_frame_method_entry(ec->cfp)->def->original_id,
rb_vm_frame_method_entry(ec->cfp)->called_id,
rb_vm_frame_method_entry(ec->cfp)->owner,
bmethod_return_value, TRUE);
}
}
THROW_DATA_CONSUMED_SET(err);
}
else {
EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_B_RETURN, ec->cfp->self, 0, 0, 0, frame_return_value(err));
if (UNLIKELY(local_hooks && local_hooks->events & RUBY_EVENT_B_RETURN)) {
rb_exec_event_hook_orig(ec, local_hooks, RUBY_EVENT_B_RETURN,
ec->cfp->self, 0, 0, 0, frame_return_value(err), TRUE);
}
THROW_DATA_CONSUMED_SET(err);
}
break;
case VM_FRAME_MAGIC_CLASS:
EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_END, ec->cfp->self, 0, 0, 0, Qnil);
break;
}
}
}
/* evaluator body */
/* finish
VMe (h1) finish
VM finish F1 F2
cfunc finish F1 F2 C1
rb_funcall finish F1 F2 C1
VMe finish F1 F2 C1
VM finish F1 F2 C1 F3
F1 - F3 : pushed by VM
C1 : pushed by send insn (CFUNC)
struct CONTROL_FRAME {
VALUE *pc; // cfp[0], program counter
VALUE *sp; // cfp[1], stack pointer
rb_iseq_t *iseq; // cfp[2], iseq
VALUE self; // cfp[3], self
const VALUE *ep; // cfp[4], env pointer
const void *block_code; // cfp[5], block code
};
struct rb_captured_block {
VALUE self;
VALUE *ep;
union code;
};
struct METHOD_ENV {
VALUE param0;
...
VALUE paramN;
VALUE lvar1;
...
VALUE lvarM;
VALUE cref; // ep[-2]
VALUE special; // ep[-1]
VALUE flags; // ep[ 0] == lep[0]
};
struct BLOCK_ENV {
VALUE block_param0;
...
VALUE block_paramN;
VALUE block_lvar1;
...
VALUE block_lvarM;
VALUE cref; // ep[-2]
VALUE special; // ep[-1]
VALUE flags; // ep[ 0]
};
struct CLASS_ENV {
VALUE class_lvar0;
...
VALUE class_lvarN;
VALUE cref;
VALUE prev_ep; // for frame jump
VALUE flags;
};
struct C_METHOD_CONTROL_FRAME {
VALUE *pc; // 0
VALUE *sp; // stack pointer
rb_iseq_t *iseq; // cmi
VALUE self; // ?
VALUE *ep; // ep == lep
void *code; //
};
struct C_BLOCK_CONTROL_FRAME {
VALUE *pc; // point only "finish" insn
VALUE *sp; // sp
rb_iseq_t *iseq; // ?
VALUE self; //
VALUE *ep; // ep
void *code; //
};
*/
static inline VALUE
vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, VALUE errinfo);
static inline VALUE
vm_exec_loop(rb_execution_context_t *ec, enum ruby_tag_type state, struct rb_vm_tag *tag, VALUE result);
// for non-Emscripten Wasm build, use vm_exec with optimized setjmp for runtime performance
#if defined(__wasm__) && !defined(__EMSCRIPTEN__)
struct rb_vm_exec_context {
rb_execution_context_t *const ec;
struct rb_vm_tag *const tag;
VALUE result;
};
static void
vm_exec_bottom_main(void *context)
{
struct rb_vm_exec_context *ctx = context;
rb_execution_context_t *ec = ctx->ec;
ctx->result = vm_exec_loop(ec, TAG_NONE, ctx->tag, vm_exec_core(ec));
}
static void
vm_exec_bottom_rescue(void *context)
{
struct rb_vm_exec_context *ctx = context;
rb_execution_context_t *ec = ctx->ec;
ctx->result = vm_exec_loop(ec, rb_ec_tag_state(ec), ctx->tag, ec->errinfo);
}
#endif
VALUE
vm_exec(rb_execution_context_t *ec)
{
VALUE result = Qundef;
EC_PUSH_TAG(ec);
_tag.retval = Qnil;
#if defined(__wasm__) && !defined(__EMSCRIPTEN__)
struct rb_vm_exec_context ctx = {
.ec = ec,
.tag = &_tag,
};
struct rb_wasm_try_catch try_catch;
EC_REPUSH_TAG();
rb_wasm_try_catch_init(&try_catch, vm_exec_bottom_main, vm_exec_bottom_rescue, &ctx);
rb_wasm_try_catch_loop_run(&try_catch, &RB_VM_TAG_JMPBUF_GET(_tag.buf));
result = ctx.result;
#else
enum ruby_tag_type state;
if ((state = EC_EXEC_TAG()) == TAG_NONE) {
if (UNDEF_P(result = jit_exec(ec))) {
result = vm_exec_core(ec);
}
/* fallback to the VM */
result = vm_exec_loop(ec, TAG_NONE, &_tag, result);
}
else {
result = vm_exec_loop(ec, state, &_tag, ec->errinfo);
}
#endif
EC_POP_TAG();
return result;
}
static inline VALUE
vm_exec_loop(rb_execution_context_t *ec, enum ruby_tag_type state,
struct rb_vm_tag *tag, VALUE result)
{
if (state == TAG_NONE) { /* no jumps, result is discarded */
goto vm_loop_start;
}
rb_ec_raised_reset(ec, RAISED_STACKOVERFLOW | RAISED_NOMEMORY);
while (UNDEF_P(result = vm_exec_handle_exception(ec, state, result))) {
// caught a jump, exec the handler. JIT code in jit_exec_exception()
// may return Qundef to run remaining frames with vm_exec_core().
if (UNDEF_P(result = jit_exec_exception(ec))) {
result = vm_exec_core(ec);
}
vm_loop_start:
VM_ASSERT(ec->tag == tag);
/* when caught `throw`, `tag.state` is set. */
if ((state = tag->state) == TAG_NONE) break;
tag->state = TAG_NONE;
}
return result;
}
#if USE_ZJIT
// Materialize JITFrame-enabled CFP into interpreter-compatible CFP
static void
zjit_materialize_frames(const rb_execution_context_t *ec, rb_control_frame_t *cfp, bool materialize_target)
{
if (!rb_zjit_enabled_p) return;
const rb_control_frame_t *end_cfp = ec->tag->cfp;
VM_ASSERT(cfp <= end_cfp);
while (true) {
// If materialize_target is false, we skip materializing ec->tag->cfp.
//
// When JIT code calls a C function that does the same number of setjmps and
// longjmps, e.g. rb_hash_aref, it calls zjit_materialize_frames but goes
// back to the JIT code. In that case, we don't want to materialize the frame
// and clear cfp->jit_return, which will still be used by the JIT code.
//
// When JIT code calls a C function that does more longjmps than setjmps,
// it would not go back to the JIT code. So ec->tag->cfp should be materialized
// in that case.
if (cfp == end_cfp && !materialize_target) break;
if (CFP_ZJIT_FRAME_P(cfp)) {
const zjit_jit_frame_t *jit_frame = CFP_ZJIT_FRAME(cfp);
cfp->pc = jit_frame->pc;
cfp->_iseq = (rb_iseq_t *)jit_frame->iseq;
if (jit_frame->materialize_block_code) {
cfp->block_code = NULL;
}
// Materialize Ruby stack slots kept off the VM stack. On side exit,
// the exiting frame is already written by compile_exit_save_state()
// and skipped here after materialize_exit_trampoline clears its
// jit_return, so this restores older ZJIT frames from stack maps.
int32_t stack_size = (int32_t)jit_frame->stack_size;
if (stack_size > 0) {
VALUE *stack = cfp->sp;
for (int32_t i = 0; i < stack_size; i++) {
VALUE entry = jit_frame->stack[i];
if (ZJIT_STACK_MAP_VREG_P(entry)) {
// Decode a native stack slot offset generated by ZJIT's backend.
// It's an offset from NATIVE_BASE_PTR, which is copied into
// cfp->jit_return, to the encoded stack slot.
stack--;
*stack = ((VALUE *)cfp->jit_return)[-(ssize_t)ZJIT_STACK_MAP_VREG_INDEX(entry)];
}
else if (ZJIT_STACK_MAP_SKIP_P(entry)) {
stack -= ZJIT_STACK_MAP_SKIP_SIZE(entry);
}
else if (ZJIT_STACK_MAP_BASE_PTR_P(entry)) {
// This has to be the first code to align the write cursor for other entries
RUBY_ASSERT_ALWAYS(0 == i, "base_ptr stack map code only makes sense at 0");
VALUE *base_ptr = (VALUE *)((VALUE *)cfp->jit_return)[-(ssize_t)ZJIT_STACK_MAP_BASE_PTR_SLOT_INDEX(entry)];
stack = base_ptr + ZJIT_STACK_MAP_BASE_PTR_STACK_SIZE(entry);
}
else {
stack--;
*stack = entry;
}
}
}
cfp->jit_return = 0;
}
if (end_cfp == cfp) break;
cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
}
}
void
rb_zjit_materialize_frames(const rb_execution_context_t *ec, rb_control_frame_t *cfp)
{
zjit_materialize_frames(ec, cfp, true);
}
void
rb_zjit_materialize_frames_for_longjmp(const rb_execution_context_t *ec, rb_control_frame_t *cfp)
{
// A ZJIT frame active before the tag's setjmp is below it on the native
// stack and survives longjmp. Materialize only the frames unwound above it.
zjit_materialize_frames(ec, cfp, !ec->tag->zjit_frame_active);
}
#endif
static inline VALUE
vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, VALUE errinfo)
{
struct vm_throw_data *err = (struct vm_throw_data *)errinfo;
for (;;) {
unsigned int i;
const struct iseq_catch_table_entry *entry;
const struct iseq_catch_table *ct;
unsigned long epc, cont_pc, cont_sp;
const rb_iseq_t *catch_iseq;
VALUE type;
const rb_control_frame_t *escape_cfp;
cont_pc = cont_sp = 0;
catch_iseq = NULL;
while (CFP_PC(ec->cfp) == 0 || CFP_ISEQ(ec->cfp) == 0) {
if (UNLIKELY(VM_FRAME_TYPE(ec->cfp) == VM_FRAME_MAGIC_CFUNC)) {
EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_C_RETURN, ec->cfp->self,
rb_vm_frame_method_entry(ec->cfp)->def->original_id,
rb_vm_frame_method_entry(ec->cfp)->called_id,
rb_vm_frame_method_entry(ec->cfp)->owner, Qnil);
RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec,
rb_vm_frame_method_entry(ec->cfp)->owner,
rb_vm_frame_method_entry(ec->cfp)->def->original_id);
}
rb_vm_pop_frame(ec);
}
rb_control_frame_t *const cfp = ec->cfp;
epc = CFP_PC(cfp) - ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded;
escape_cfp = NULL;
if (state == TAG_BREAK || state == TAG_RETURN) {
escape_cfp = THROW_DATA_CATCH_FRAME(err);
if (cfp == escape_cfp) {
if (state == TAG_RETURN) {
if (!VM_FRAME_FINISHED_P(cfp)) {
THROW_DATA_CATCH_FRAME_SET(err, cfp + 1);
THROW_DATA_STATE_SET(err, state = TAG_BREAK);
}
else {
ct = ISEQ_BODY(CFP_ISEQ(cfp))->catch_table;
if (ct) for (i = 0; i < ct->size; i++) {
entry = UNALIGNED_MEMBER_PTR(ct, entries[i]);
if (entry->start < epc && entry->end >= epc) {
if (entry->type == CATCH_TYPE_ENSURE) {
catch_iseq = entry->iseq;
cont_pc = entry->cont;
cont_sp = entry->sp;
break;
}
}
}
if (catch_iseq == NULL) {
ec->errinfo = Qnil;
THROW_DATA_CATCH_FRAME_SET(err, cfp + 1);
// cfp == escape_cfp here so calling with cfp_returning_with_value = true
hook_before_rewind(ec, true, state, err);
rb_vm_pop_frame(ec);
return THROW_DATA_VAL(err);
}
}
/* through */
}
else {
/* TAG_BREAK */
*cfp->sp++ = THROW_DATA_VAL(err);
ec->errinfo = Qnil;
rb_zjit_materialize_frames(ec, cfp);
return Qundef;
}
}
}
if (state == TAG_RAISE) {
ct = ISEQ_BODY(CFP_ISEQ(cfp))->catch_table;
if (ct) for (i = 0; i < ct->size; i++) {
entry = UNALIGNED_MEMBER_PTR(ct, entries[i]);
if (entry->start < epc && entry->end >= epc) {
if (entry->type == CATCH_TYPE_RESCUE ||
entry->type == CATCH_TYPE_ENSURE) {
catch_iseq = entry->iseq;
cont_pc = entry->cont;
cont_sp = entry->sp;
break;
}
}
}
}
else if (state == TAG_RETRY) {
ct = ISEQ_BODY(CFP_ISEQ(cfp))->catch_table;
if (ct) for (i = 0; i < ct->size; i++) {
entry = UNALIGNED_MEMBER_PTR(ct, entries[i]);
if (entry->start < epc && entry->end >= epc) {
if (entry->type == CATCH_TYPE_ENSURE) {
catch_iseq = entry->iseq;
cont_pc = entry->cont;
cont_sp = entry->sp;
break;
}
else if (entry->type == CATCH_TYPE_RETRY) {
const rb_control_frame_t *escape_cfp;
escape_cfp = THROW_DATA_CATCH_FRAME(err);
if (cfp == escape_cfp) {
rb_zjit_materialize_frames(ec, cfp);
cfp->pc = ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded + entry->cont;
ec->errinfo = Qnil;
return Qundef;
}
}
}
}
}
else if ((state == TAG_BREAK && !escape_cfp) ||
(state == TAG_REDO) ||
(state == TAG_NEXT)) {
type = (const enum rb_catch_type[TAG_MASK]) {
[TAG_BREAK] = CATCH_TYPE_BREAK,
[TAG_NEXT] = CATCH_TYPE_NEXT,
[TAG_REDO] = CATCH_TYPE_REDO,
/* otherwise = dontcare */
}[state];
ct = ISEQ_BODY(CFP_ISEQ(cfp))->catch_table;
if (ct) for (i = 0; i < ct->size; i++) {
entry = UNALIGNED_MEMBER_PTR(ct, entries[i]);
if (entry->start < epc && entry->end >= epc) {
if (entry->type == CATCH_TYPE_ENSURE) {
catch_iseq = entry->iseq;
cont_pc = entry->cont;
cont_sp = entry->sp;
break;
}
else if (entry->type == type) {
rb_zjit_materialize_frames(ec, cfp);
cfp->pc = ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded + entry->cont;
cfp->sp = vm_base_ptr(cfp) + entry->sp;
if (state != TAG_REDO) {
*cfp->sp++ = THROW_DATA_VAL(err);
}
ec->errinfo = Qnil;
VM_ASSERT(ec->tag->state == TAG_NONE);
return Qundef;
}
}
}
}
else {
ct = ISEQ_BODY(CFP_ISEQ(cfp))->catch_table;
if (ct) for (i = 0; i < ct->size; i++) {
entry = UNALIGNED_MEMBER_PTR(ct, entries[i]);
if (entry->start < epc && entry->end >= epc) {
if (entry->type == CATCH_TYPE_ENSURE) {
catch_iseq = entry->iseq;
cont_pc = entry->cont;
cont_sp = entry->sp;
break;
}
}
}
}
if (catch_iseq != NULL) { /* found catch table */
/* enter catch scope */
const int arg_size = 1;
rb_iseq_check(catch_iseq);
rb_zjit_materialize_frames(ec, cfp); // vm_base_ptr looks at cfp->_iseq
cfp->sp = vm_base_ptr(cfp) + cont_sp;
cfp->pc = ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded + cont_pc;
/* push block frame */
cfp->sp[0] = (VALUE)err;
vm_push_frame(ec, catch_iseq, VM_FRAME_MAGIC_RESCUE,
cfp->self,
VM_GUARDED_PREV_EP(cfp->ep),
0, /* cref or me */
ISEQ_BODY(catch_iseq)->iseq_encoded,
cfp->sp + arg_size /* push value */,
ISEQ_BODY(catch_iseq)->local_table_size - arg_size,
ISEQ_BODY(catch_iseq)->stack_max);
state = 0;
ec->tag->state = TAG_NONE;
ec->errinfo = Qnil;
return Qundef;
}
else {
hook_before_rewind(ec, (cfp == escape_cfp), state, err);
if (VM_FRAME_FINISHED_P(ec->cfp)) {
rb_vm_pop_frame(ec);
ec->errinfo = (VALUE)err;
rb_vm_tag_jmpbuf_deinit(&ec->tag->buf);
ec->tag = ec->tag->prev;
EC_JUMP_TAG(ec, state);
}
else {
rb_vm_pop_frame(ec);
}
}
}
}
/* misc */
VALUE
rb_iseq_eval(const rb_iseq_t *iseq, const rb_box_t *box)
{
rb_execution_context_t *ec = GET_EC();
VALUE val;
vm_set_top_stack(ec, iseq, box);
val = vm_exec(ec);
return val;
}
VALUE
rb_iseq_eval_main(const rb_iseq_t *iseq)
{
rb_execution_context_t *ec = GET_EC();
VALUE val;
vm_set_main_stack(ec, iseq);
val = vm_exec(ec);
return val;
}
int
rb_vm_control_frame_id_and_class(const rb_control_frame_t *cfp, ID *idp, ID *called_idp, VALUE *klassp)
{
const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(cfp);
if (me) {
if (idp) *idp = me->def->original_id;
if (called_idp) *called_idp = me->called_id;
if (klassp) *klassp = me->owner;
return TRUE;
}
else {
return FALSE;
}
}
int
rb_ec_frame_method_id_and_class(const rb_execution_context_t *ec, ID *idp, ID *called_idp, VALUE *klassp)
{
return rb_vm_control_frame_id_and_class(ec->cfp, idp, called_idp, klassp);
}
int
rb_frame_method_id_and_class(ID *idp, VALUE *klassp)
{
return rb_ec_frame_method_id_and_class(GET_EC(), idp, 0, klassp);
}
VALUE
rb_vm_call_cfunc(VALUE recv, VALUE (*func)(VALUE), VALUE arg,
VALUE block_handler, VALUE filename)
{
rb_execution_context_t *ec = GET_EC();
const rb_control_frame_t *reg_cfp = ec->cfp;
const rb_iseq_t *iseq = rb_iseq_new(Qnil, filename, filename, Qnil, 0, ISEQ_TYPE_TOP);
const rb_box_t *box = rb_current_box();
VALUE val;
vm_push_frame(ec, iseq, VM_FRAME_MAGIC_TOP | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH,
recv, GC_GUARDED_PTR(box),
(VALUE)vm_cref_new_toplevel(ec), /* cref or me */
0, reg_cfp->sp, 0, 0);
val = (*func)(arg);
rb_vm_pop_frame(ec);
return val;
}
/* Ruby::Box */
VALUE
rb_vm_call_cfunc_in_box(VALUE recv, VALUE (*func)(VALUE, VALUE), VALUE arg1, VALUE arg2,
VALUE filename, const rb_box_t *box)
{
rb_execution_context_t *ec = GET_EC();
const rb_control_frame_t *reg_cfp = ec->cfp;
const rb_iseq_t *iseq = rb_iseq_new(Qnil, filename, filename, Qnil, 0, ISEQ_TYPE_TOP);
VALUE val;
vm_push_frame(ec, iseq, VM_FRAME_MAGIC_TOP | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH,
recv, GC_GUARDED_PTR(box),
(VALUE)vm_cref_new_toplevel(ec), /* cref or me */
0, reg_cfp->sp, 0, 0);
val = (*func)(arg1, arg2);
rb_vm_pop_frame(ec);
return val;
}
void
rb_vm_frame_flag_set_box_require(const rb_execution_context_t *ec)
{
VM_ASSERT(rb_box_available());
VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_BOX_REQUIRE);
}
static const rb_box_t *current_box_on_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp);
/**
* Returns the nearest user box in the caller frames, or NULL if there is none.
*
* Builtin methods written in Ruby are defined in the master box, so their own
* frame tells nothing about the caller. Those marked with
* `Primitive.attr! :caller_user_box` need the box owning the caller code, and the
* frames in between may belong to the master or the root box, e.g. when another
* builtin method or a proc made in the root box calls them.
*/
static const rb_box_t *
caller_user_box_on_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp)
{
const rb_control_frame_t * const eocfp = RUBY_VM_END_CONTROL_FRAME(ec);
while (RUBY_VM_VALID_CONTROL_FRAME_P(cfp, eocfp)) {
const rb_box_t *box = current_box_on_cfp(ec, cfp);
if (BOX_USER_P(box))
return box;
cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
}
return NULL;
}
static const rb_box_t *
current_box_on_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp)
{
rb_callable_method_entry_t *cme;
const rb_box_t *box;
const VALUE *lep = VM_EP_RUBY_LEP(ec, cfp);
VM_BOX_ASSERT(lep, "lep should be valid");
VM_BOX_ASSERT(rb_box_available(), "box should be available here");
if (VM_ENV_FRAME_TYPE_P(lep, VM_FRAME_MAGIC_METHOD) || VM_ENV_FRAME_TYPE_P(lep, VM_FRAME_MAGIC_CFUNC)) {
cme = check_method_entry(lep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
VM_BOX_ASSERT(cme, "cme should be valid");
VM_BOX_ASSERT(cme->def, "cme->def shold be valid");
if (cme->def->type == VM_METHOD_TYPE_ISEQ &&
(ISEQ_BODY(cme->def->body.iseq.iseqptr)->builtin_attrs & BUILTIN_ATTR_CALLER_USER_BOX)) {
const rb_control_frame_t *owner_cfp = rb_vm_search_cf_from_ep(ec, cfp, lep);
if (owner_cfp) {
box = caller_user_box_on_cfp(ec, RUBY_VM_PREVIOUS_CONTROL_FRAME(owner_cfp));
if (box)
return box;
}
}
return cme->def->box;
}
else if (VM_ENV_FRAME_TYPE_P(lep, VM_FRAME_MAGIC_TOP) || VM_ENV_FRAME_TYPE_P(lep, VM_FRAME_MAGIC_CLASS)) {
VM_BOX_ASSERT(VM_ENV_LOCAL_P(lep), "lep should be local on MAGIC_TOP or MAGIC_CLASS frames");
return VM_ENV_BOX(lep);
}
else if (VM_ENV_FRAME_TYPE_P(lep, VM_FRAME_MAGIC_DUMMY)) {
// No valid local ep found (just after process boot?)
// return the root box (the only valid box) until the main is initialized
box = rb_main_box();
if (box)
return box;
return rb_root_box();
}
else {
VM_BOX_CRASHED();
rb_bug("BUG: Local ep without cme/box, flags: %08lX", (unsigned long)lep[VM_ENV_DATA_INDEX_FLAGS]);
}
UNREACHABLE_RETURN(0);
}
const rb_box_t *
rb_vm_current_box(const rb_execution_context_t *ec)
{
return current_box_on_cfp(ec, ec->cfp);
}
static const rb_control_frame_t *
find_loader_control_frame(const rb_execution_context_t *ec, const rb_control_frame_t *cfp, const rb_control_frame_t *end_cfp)
{
while (RUBY_VM_VALID_CONTROL_FRAME_P(cfp, end_cfp)) {
if (!VM_ENV_FRAME_TYPE_P(cfp->ep, VM_FRAME_MAGIC_CFUNC))
break;
if (!BOX_MASTER_P(current_box_on_cfp(ec, cfp)))
break;
cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
}
VM_ASSERT(RUBY_VM_VALID_CONTROL_FRAME_P(cfp, end_cfp));
return cfp;
}
const rb_box_t *
rb_vm_loading_box(const rb_execution_context_t *ec)
{
const rb_control_frame_t *cfp, *current_cfp, *end_cfp;
if (!rb_box_available() || !ec)
return rb_root_box();
cfp = ec->cfp;
current_cfp = cfp;
end_cfp = RUBY_VM_END_CONTROL_FRAME(ec);
while (RUBY_VM_VALID_CONTROL_FRAME_P(cfp, end_cfp)) {
if (VM_ENV_FLAGS(cfp->ep, VM_FRAME_FLAG_BOX_REQUIRE)) {
if (RTEST(cfp->self) && BOX_OBJ_P(cfp->self)) {
// Box#require, #require_relative, #load
return rb_get_box_t(cfp->self);
}
// Kernel#require, #require_relative, #load
cfp = find_loader_control_frame(ec, cfp, end_cfp);
return current_box_on_cfp(ec, cfp);
}
cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
}
// no require/load with explicit boxes.
return current_box_on_cfp(ec, current_cfp);
}
/* vm */
void
rb_vm_update_references(void *ptr)
{
if (ptr) {
rb_vm_t *vm = ptr;
rb_gc_update_moved(&vm->self);
rb_gc_update_moved(&vm->orig_progname);
rb_gc_update_moved(&vm->cc_refinement_set);
if (vm->root_box)
rb_box_gc_update_references(vm->root_box);
if (vm->main_box)
rb_box_gc_update_references(vm->main_box);
rb_gc_update_values(RUBY_NSIG, vm->trap_list.cmd);
if (vm->coverages) {
rb_gc_update_moved(&vm->coverages);
rb_gc_update_moved(&vm->cme2counter);
rb_gc_update_moved(&vm->me_set);
}
}
}
void
rb_vm_each_stack_value(void *ptr, void (*cb)(VALUE, void*), void *ctx)
{
if (ptr) {
rb_vm_t *vm = ptr;
rb_ractor_t *r = 0;
ccan_list_for_each(&vm->ractor.set, r, vmlr_node) {
VM_ASSERT(rb_ractor_status_p(r, ractor_blocking) ||
rb_ractor_status_p(r, ractor_running));
if (r->threads.cnt > 0) {
rb_thread_t *th = 0;
ccan_list_for_each(&r->threads.set, th, lt_node) {
VM_ASSERT(th != NULL);
rb_execution_context_t * ec = th->ec;
if (ec->vm_stack) {
VALUE *p = ec->vm_stack;
VALUE *sp = ec->cfp->sp;
while (p < sp) {
if (!RB_SPECIAL_CONST_P(*p)) {
cb(*p, ctx);
}
p++;
}
}
}
}
}
}
}
static enum rb_id_table_iterator_result
vm_mark_negative_cme(VALUE val, void *dmy)
{
rb_gc_mark(val);
return ID_TABLE_CONTINUE;
}
void
rb_vm_mark(void *ptr)
{
RUBY_MARK_ENTER("vm");
RUBY_GC_INFO("-------------------------------------------------\n");
if (ptr) {
rb_vm_t *vm = ptr;
rb_ractor_t *r = 0;
long i;
ccan_list_for_each(&vm->ractor.set, r, vmlr_node) {
// ractor.set only contains blocking or running ractors
VM_ASSERT(rb_ractor_status_p(r, ractor_blocking) ||
rb_ractor_status_p(r, ractor_running));
rb_gc_mark(rb_ractor_self(r));
}
rb_gc_mark_movable(vm->self);
if (vm->root_box) {
rb_box_entry_mark(vm->root_box);
}
if (vm->main_box) {
rb_box_entry_mark(vm->main_box);
}
rb_gc_mark_movable(vm->orig_progname);
rb_gc_mark_movable(vm->coverages);
rb_gc_mark_movable(vm->cme2counter);
rb_gc_mark_movable(vm->me_set);
rb_gc_mark_movable(vm->cc_refinement_set);
rb_hook_list_mark(&vm->global_hooks);
rb_id_table_foreach_values(&vm->negative_cme_table, vm_mark_negative_cme, NULL);
rb_mark_tbl_no_pin(&vm->overloaded_cme_table);
for (i=0; i<VM_GLOBAL_CC_CACHE_TABLE_SIZE; i++) {
const struct rb_callcache *cc = vm->global_cc_cache_table[i];
if (cc != NULL) {
if (!vm_cc_invalidated_p(cc)) {
rb_gc_mark((VALUE)cc);
}
else {
vm->global_cc_cache_table[i] = NULL;
}
}
}
}
RUBY_MARK_LEAVE("vm");
}
#undef rb_vm_register_special_exception
void
rb_vm_register_special_exception_str(enum ruby_special_exceptions sp, VALUE cls, VALUE mesg)
{
rb_vm_t *vm = GET_VM();
VALUE exc = rb_exc_new3(cls, rb_obj_freeze(mesg));
OBJ_FREEZE(exc);
((VALUE *)vm->special_exceptions)[sp] = exc;
rb_vm_register_global_object(exc);
}
void rb_objspace_free_objects(void *objspace);
int
ruby_vm_destruct(rb_vm_t *vm)
{
{
// wait for native threads still winding down a dead coroutine: their
// reclaim frees through the objspace this function is about to
// destroy (see coroutine_thread_terminated)
void rb_thread_sched_wait_winding(rb_vm_t *vm);
rb_thread_sched_wait_winding(vm);
}
RUBY_FREE_ENTER("vm");
ruby_vm_during_cleanup = true;
rb_gc_stash_cleanup_objspace();
if (vm) {
rb_thread_t *th = vm->ractor.main_thread;
if (rb_free_at_exit) {
rb_free_encoded_insn_data();
rb_free_global_enc_table();
rb_free_loaded_builtin_table();
rb_free_global_symbol_table();
rb_free_shared_fiber_pool();
rb_free_transcoder_table();
rb_free_vm_opt_tables();
rb_free_warning();
rb_free_rb_global_tbl();
rb_id_table_free_items(&vm->negative_cme_table);
st_free_embedded_table(&vm->overloaded_cme_table);
// TODO: Is this ignorable for classext->m_tbl ?
// rb_id_table_free(RCLASS(rb_mRubyVMFrozenCore)->m_tbl);
st_free_embedded_table(&vm->static_ext_inits);
rb_id_table_free_items(&vm->constant_cache);
set_free_embedded_table(&vm->unused_block_warning_table);
rb_thread_free_native_thread(th);
#ifndef HAVE_SETPROCTITLE
ruby_free_proctitle();
#endif
}
else {
rb_fiber_reset_root_local_storage(th);
thread_free(th);
}
void *objspace = vm->ractor.main_ractor ? vm->ractor.main_ractor->objspace : NULL;
rb_vm_living_threads_init(vm);
ruby_vm_run_at_exit_hooks(vm);
st_free_embedded_table(&vm->ci_table);
RB_ALTSTACK_FREE(vm->main_altstack);
if (objspace) {
if (rb_free_at_exit) {
rb_objspace_free_objects(objspace);
rb_free_generic_fields_tbl_();
rb_free_default_rand_key();
}
rb_objspace_free(objspace);
}
if (rb_free_at_exit) {
free(vm->gc.registered_addrs.registry);
}
rb_native_mutex_destroy(&vm->once_lock);
rb_native_cond_destroy(&vm->once_cond);
/* after freeing objspace, you *can't* use ruby_xfree() */
ruby_current_vm_ptr = NULL;
if (rb_free_at_exit) {
#if USE_YJIT
rb_yjit_free_at_exit();
#endif
}
rb_box_unload_local_extensions();
}
RUBY_FREE_LEAVE("vm");
return 0;
}
// Used for VM memsize reporting. Returns the size of the at_exit list by
// looping through the linked list and adding up the size of the structs.
static enum rb_id_table_iterator_result
vm_memsize_constant_cache_i(ID id, VALUE ics, void *size)
{
*((size_t *) size) += rb_set_memsize((set_table *) ics);
return ID_TABLE_CONTINUE;
}
// Returns a size_t representing the memory footprint of the VM's constant
// cache, which is the memsize of the table as well as the memsize of all of the
// nested tables.
static size_t
vm_memsize_constant_cache(void)
{
rb_vm_t *vm = GET_VM();
size_t size = rb_id_table_memsize(&vm->constant_cache) - sizeof(struct rb_id_table);
rb_id_table_foreach(&vm->constant_cache, vm_memsize_constant_cache_i, &size);
return size;
}
static size_t
vm_memsize_at_exit_list(rb_at_exit_list *at_exit)
{
size_t size = 0;
while (at_exit) {
size += sizeof(rb_at_exit_list);
at_exit = at_exit->next;
}
return size;
}
// Used for VM memsize reporting. Returns the size of the builtin function
// table if it has been defined.
static size_t
vm_memsize_builtin_function_table(const struct rb_builtin_function *builtin_function_table)
{
return builtin_function_table == NULL ? 0 : sizeof(struct rb_builtin_function);
}
// Reports the memsize of the VM struct object and the structs that are
// associated with it.
static size_t
vm_memsize(const void *ptr)
{
rb_vm_t *vm = GET_VM();
return (
sizeof(rb_vm_t) +
rb_vm_memsize_postponed_job_queue() +
vm_memsize_at_exit_list(vm->at_exit) +
(rb_st_memsize(&vm->ci_table) - sizeof(struct st_table)) +
vm_memsize_builtin_function_table(vm->builtin_function_table) +
(rb_id_table_memsize(&vm->negative_cme_table) - sizeof(struct rb_id_table)) +
(rb_st_memsize(&vm->overloaded_cme_table) - sizeof(struct st_table)) +
vm_memsize_constant_cache()
);
// TODO
// struct { struct ccan_list_head set; } ractor;
// void *main_altstack; #ifdef USE_SIGALTSTACK
// struct rb_objspace *objspace;
}
const rb_data_type_t ruby_vm_data_type = {
"VM",
{0, 0, vm_memsize,},
0, 0, RUBY_TYPED_THREAD_SAFE_FREE
};
#define vm_data_type ruby_vm_data_type
static VALUE
vm_default_params(void)
{
rb_vm_t *vm = GET_VM();
VALUE result = rb_hash_new_capa(4);
#define SET(name) rb_hash_aset(result, ID2SYM(rb_intern(#name)), SIZET2NUM(vm->default_params.name));
SET(thread_vm_stack_size);
SET(thread_machine_stack_size);
SET(fiber_vm_stack_size);
SET(fiber_machine_stack_size);
#undef SET
rb_obj_freeze(result);
return result;
}
static size_t
get_param(const char *name, size_t default_value, size_t min_value)
{
const char *envval;
size_t result = default_value;
if ((envval = getenv(name)) != 0) {
long val = atol(envval);
if (val < (long)min_value) {
val = (long)min_value;
}
result = (size_t)(((val -1 + RUBY_VM_SIZE_ALIGN) / RUBY_VM_SIZE_ALIGN) * RUBY_VM_SIZE_ALIGN);
}
if (0) ruby_debug_printf("%s: %"PRIuSIZE"\n", name, result); /* debug print */
return result;
}
static void
check_machine_stack_size(size_t *sizep)
{
#ifdef PTHREAD_STACK_MIN
size_t size = *sizep;
#endif
#ifdef PTHREAD_STACK_MIN
if (size < (size_t)PTHREAD_STACK_MIN) {
*sizep = (size_t)PTHREAD_STACK_MIN * 2;
}
#endif
}
static void
vm_default_params_setup(rb_vm_t *vm)
{
vm->default_params.thread_vm_stack_size =
get_param("RUBY_THREAD_VM_STACK_SIZE",
RUBY_VM_THREAD_VM_STACK_SIZE,
RUBY_VM_THREAD_VM_STACK_SIZE_MIN);
vm->default_params.thread_machine_stack_size =
get_param("RUBY_THREAD_MACHINE_STACK_SIZE",
RUBY_VM_THREAD_MACHINE_STACK_SIZE,
RUBY_VM_THREAD_MACHINE_STACK_SIZE_MIN);
vm->default_params.fiber_vm_stack_size =
get_param("RUBY_FIBER_VM_STACK_SIZE",
RUBY_VM_FIBER_VM_STACK_SIZE,
RUBY_VM_FIBER_VM_STACK_SIZE_MIN);
vm->default_params.fiber_machine_stack_size =
get_param("RUBY_FIBER_MACHINE_STACK_SIZE",
RUBY_VM_FIBER_MACHINE_STACK_SIZE,
RUBY_VM_FIBER_MACHINE_STACK_SIZE_MIN);
/* environment dependent check */
check_machine_stack_size(&vm->default_params.thread_machine_stack_size);
check_machine_stack_size(&vm->default_params.fiber_machine_stack_size);
}
static void
vm_init2(rb_vm_t *vm)
{
rb_vm_living_threads_init(vm);
vm->thread_report_on_exception = 1;
vm->src_encoding_index = -1;
vm_default_params_setup(vm);
}
void
rb_execution_context_update(rb_execution_context_t *ec)
{
/* update VM stack */
if (ec->vm_stack) {
long i;
VM_ASSERT(ec->cfp);
VALUE *p = ec->vm_stack;
VALUE *sp = ec->cfp->sp;
rb_control_frame_t *cfp = ec->cfp;
rb_control_frame_t *limit_cfp = (void *)(ec->vm_stack + ec->vm_stack_size);
// ZJIT leaves uninitialized slots on the VM stack, so we cannot
// safely use rb_gc_location on such slots.
if (!rb_zjit_enabled_p) {
for (i = 0; i < (long)(sp - p); i++) {
rb_gc_update_moved(&p[i]);
}
}
while (cfp != limit_cfp) {
const VALUE *ep = cfp->ep;
rb_gc_update_moved(&cfp->self);
if (CFP_ZJIT_FRAME_P(cfp)) {
const zjit_jit_frame_t *jit_frame = CFP_ZJIT_FRAME(cfp);
rb_zjit_jit_frame_update_references((zjit_jit_frame_t *)jit_frame);
// materialize_block_code means cfp->block_code is lazy and may
// still contain stale data from a previous frame. Otherwise it
// was initialized by ZJIT and may have been written later by
// vm_caller_setup_arg_block (ISEQ frames) or rb_iterate0 (C frames).
if (!jit_frame->materialize_block_code) {
rb_gc_update_moved_ptr(&cfp->block_code);
}
}
else {
rb_gc_update_moved_ptr(&cfp->_iseq);
rb_gc_update_moved_ptr(&cfp->block_code);
}
if (!VM_ENV_LOCAL_P(ep)) {
const VALUE *prev_ep = VM_ENV_PREV_EP(ep);
if (VM_ENV_FLAGS(prev_ep, VM_ENV_FLAG_ESCAPED)) {
rb_gc_update_moved((VALUE *)&prev_ep[VM_ENV_DATA_INDEX_ENV]);
}
if (VM_ENV_FLAGS(ep, VM_ENV_FLAG_ESCAPED)) {
rb_gc_update_moved((VALUE *)&ep[VM_ENV_DATA_INDEX_ENV]);
rb_gc_update_moved((VALUE *)&ep[VM_ENV_DATA_INDEX_ME_CREF]);
}
}
cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
}
}
rb_gc_update_moved(&ec->storage);
rb_gc_update_moved(&ec->gen_fields_cache.obj);
rb_gc_update_moved(&ec->gen_fields_cache.fields_obj);
}
static enum rb_id_table_iterator_result
mark_local_storage_i(VALUE local, void *data)
{
rb_gc_mark(local);
return ID_TABLE_CONTINUE;
}
void
rb_execution_context_mark(const rb_execution_context_t *ec)
{
/* mark VM stack */
if (ec->vm_stack) {
VM_ASSERT(ec->cfp);
VALUE *p = ec->vm_stack;
VALUE *sp = ec->cfp->sp;
rb_control_frame_t *cfp = ec->cfp;
rb_control_frame_t *limit_cfp = (void *)(ec->vm_stack + ec->vm_stack_size);
for (long i = 0; i < (long)(sp - p); i++) {
if (rb_zjit_enabled_p) {
// ZJIT leaves uninitialized slots on the VM stack, so we need
// to mark such slots conservatively.
rb_gc_mark_maybe(p[i]);
}
else {
rb_gc_mark_movable(p[i]);
}
}
while (cfp != limit_cfp) {
const VALUE *ep = cfp->ep;
VM_ASSERT(!!VM_ENV_FLAGS(ep, VM_ENV_FLAG_ESCAPED) == vm_ep_in_heap_p_(ec, ep));
rb_gc_mark_movable(cfp->self);
if (CFP_ZJIT_FRAME_P(cfp)) {
const zjit_jit_frame_t *jit_frame = CFP_ZJIT_FRAME(cfp);
rb_gc_mark_movable((VALUE)jit_frame->iseq);
// materialize_block_code means cfp->block_code is lazy and may
// still contain stale data from a previous frame. Otherwise it
// was initialized by ZJIT and may have been written later by
// vm_caller_setup_arg_block (ISEQ frames) or rb_iterate0 (C frames).
if (!jit_frame->materialize_block_code) {
rb_gc_mark_movable((VALUE)cfp->block_code);
}
}
else {
rb_gc_mark_movable((VALUE)cfp->_iseq);
rb_gc_mark_movable((VALUE)cfp->block_code);
}
if (VM_ENV_LOCAL_P(ep) && VM_ENV_BOXED_P(ep)) {
const rb_box_t *box = VM_ENV_BOX(ep);
if (BOX_USER_P(box)) {
rb_gc_mark_movable(box->box_object);
}
}
if (!VM_ENV_LOCAL_P(ep)) {
const VALUE *prev_ep = VM_ENV_PREV_EP(ep);
if (VM_ENV_FLAGS(prev_ep, VM_ENV_FLAG_ESCAPED)) {
rb_gc_mark_movable(prev_ep[VM_ENV_DATA_INDEX_ENV]);
}
if (VM_ENV_FLAGS(ep, VM_ENV_FLAG_ESCAPED)) {
rb_gc_mark_movable(ep[VM_ENV_DATA_INDEX_ENV]);
rb_gc_mark(ep[VM_ENV_DATA_INDEX_ME_CREF]);
}
}
cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
}
}
/* mark machine stack */
if (ec->machine.stack_start && ec->machine.stack_end &&
/* marked for current ec at the first stage of marking */
ec != rb_gc_get_ec()) {
rb_gc_mark_machine_context(ec);
}
rb_gc_mark(ec->errinfo);
rb_gc_mark(ec->root_svar);
if (ec->local_storage) {
rb_id_table_foreach_values(ec->local_storage, mark_local_storage_i, NULL);
}
rb_gc_mark(ec->local_storage_recursive_hash);
rb_gc_mark(ec->local_storage_recursive_hash_for_trace);
rb_gc_mark(ec->private_const_reference);
rb_gc_mark_movable(ec->storage);
}
void rb_fiber_mark_self(rb_fiber_t *fib);
void rb_fiber_update_self(rb_fiber_t *fib);
void rb_threadptr_root_fiber_setup(rb_thread_t *th);
void rb_root_fiber_obj_setup(rb_thread_t *th, void *objspace);
void rb_threadptr_root_fiber_release(rb_thread_t *th);
static void
thread_compact(void *ptr)
{
rb_thread_t *th = ptr;
rb_gc_update_moved(&th->self);
}
/* Mark the heap objects a thread owns (the caller handles ec and fiber). Split
* out of thread_mark so that a local GC can root them straight from the Ractor's
* local roots (rb_ractor_mark_local_roots). */
void
rb_thread_mark_owned_roots(rb_thread_t *th)
{
/* mark ruby objects */
switch (th->invoke_type) {
case thread_invoke_type_proc:
case thread_invoke_type_ractor_proc:
rb_gc_mark(th->invoke_arg.proc.proc);
rb_gc_mark(th->invoke_arg.proc.args);
break;
case thread_invoke_type_func:
rb_gc_mark_maybe((VALUE)th->invoke_arg.func.arg);
break;
default:
break;
}
rb_gc_mark(th->thgroup);
rb_gc_mark(th->value);
rb_gc_mark(th->pending_interrupt_queue);
rb_gc_mark(th->pending_interrupt_mask_stack);
rb_gc_mark(th->top_self);
rb_gc_mark(th->top_wrapper);
rb_gc_mark(th->last_status);
rb_gc_mark(th->locking_mutex);
rb_gc_mark(th->name);
rb_gc_mark(th->scheduler);
rb_threadptr_interrupt_exec_task_mark(th);
}
static void
thread_mark(void *ptr)
{
rb_thread_t *th = ptr;
RUBY_MARK_ENTER("thread");
// ec is null when setting up the thread in rb_threadptr_root_fiber_setup
if (th->ec) {
rb_fiber_mark_self(th->ec->fiber_ptr);
}
/* A live thread wrapper keeps its Ractor object alive (and through its dfree the
* rb_ractor_t), so an inherited Thread keeps a dead Ractor alive just as it does
* upstream. */
if (th->ractor) rb_gc_mark(rb_ractor_self(th->ractor));
if (th->root_fiber) rb_fiber_mark_self(th->root_fiber);
RUBY_ASSERT(th->ec == NULL || th->ec == rb_fiberptr_get_ec(th->ec->fiber_ptr));
rb_thread_mark_owned_roots(th);
RUBY_MARK_LEAVE("thread");
}
void rb_threadptr_sched_free(rb_thread_t *th); // thread_*.c
static void
thread_free(void *ptr)
{
rb_thread_t *th = ptr;
/* The final self collection sweeps the wrapper of the very thread running it; the
* struct is still under that thread's feet (GET_EC resolves through it), so the
* thread frees the struct itself at its last step (rb_ractor_postmortem_free). */
if (th->ec != NULL && th->ec == rb_current_execution_context(false)) {
th->self = 0;
return;
}
rb_thread_free_body(ptr);
}
void
rb_thread_free_body(void *ptr)
{
rb_thread_t *th = ptr;
RUBY_FREE_ENTER("thread");
rb_threadptr_sched_free(th);
// destroyed here rather than during teardown: nothing can interrupt a
// thread that is unreachable and off its Ractor's living set
rb_native_mutex_destroy(&th->interrupt_lock);
if (th->locking_mutex != Qfalse) {
rb_bug("thread_free: locking_mutex must be NULL (%p:%p)", (void *)th, (void *)th->locking_mutex);
}
if (th->keeping_mutexes != NULL) {
rb_bug("thread_free: keeping_mutexes must be NULL (%p:%p)", (void *)th, (void *)th->keeping_mutexes);
}
ruby_xfree(th->specific_storage);
if (th->vm && th->vm->ractor.main_thread == th) {
RUBY_GC_INFO("MRI main thread\n");
}
else {
// ruby_xfree(th->nt);
// TODO: MN system collect nt, but without MN system it should be freed here.
if (!th->main_thread) {
ruby_xfree(th);
}
}
RUBY_FREE_LEAVE("thread");
}
static size_t
thread_memsize(const void *ptr)
{
const rb_thread_t *th = ptr;
size_t size = sizeof(rb_thread_t);
if (!th->root_fiber) {
size += th->ec->vm_stack_size * sizeof(VALUE);
}
if (th->ec->local_storage) {
size += rb_id_table_memsize(th->ec->local_storage);
}
return size;
}
#define thread_data_type ruby_threadptr_data_type
const rb_data_type_t ruby_threadptr_data_type = {
"VM/thread",
{
thread_mark,
thread_free,
thread_memsize,
thread_compact,
},
0, 0, RUBY_TYPED_THREAD_SAFE_FREE
};
VALUE
rb_obj_is_thread(VALUE obj)
{
return RBOOL(rb_typeddata_is_kind_of(obj, &thread_data_type));
}
static VALUE
thread_alloc(VALUE klass, void *objspace)
{
return rb_data_typed_object_zalloc_in_objspace(objspace, klass, sizeof(rb_thread_t), &thread_data_type);
}
void
rb_ec_set_vm_stack(rb_execution_context_t *ec, VALUE *stack, size_t size)
{
ec->vm_stack = stack;
ec->vm_stack_size = size;
}
void
rb_ec_initialize_vm_stack(rb_execution_context_t *ec, VALUE *stack, size_t size)
{
rb_ec_set_vm_stack(ec, stack, size);
#if VM_CHECK_MODE > 0
MEMZERO(stack, VALUE, size); // malloc memory could have the VM canary in it
#endif
ec->cfp = (void *)(ec->vm_stack + ec->vm_stack_size);
vm_push_frame(ec,
NULL /* dummy iseq */,
VM_FRAME_MAGIC_DUMMY | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH | VM_FRAME_FLAG_CFRAME /* dummy frame */,
Qnil /* dummy self */, VM_BLOCK_HANDLER_NONE /* dummy block ptr */,
0 /* dummy cref/me */,
0 /* dummy pc */, ec->vm_stack, 0, 0
);
}
void
rb_ec_clear_vm_stack(rb_execution_context_t *ec)
{
// set cfp to NULL before clearing the stack in case `thread_profile_frames`
// gets called in this middle of `rb_ec_set_vm_stack` via signal handler.
ec->cfp = NULL;
rb_ec_set_vm_stack(ec, NULL, 0);
}
void
rb_ec_close(rb_execution_context_t *ec)
{
// Fiber storage is not accessible from outside the running fiber, so it is safe to clear it here.
ec->storage = Qnil;
}
static void
th_init(rb_thread_t *th, VALUE self, rb_vm_t *vm)
{
const rb_box_t *box = rb_current_box();
th->self = self;
ccan_list_head_init(&th->interrupt_exec_tasks);
// initialized here (not at thread creation) so that every Thread object
// -- including allocated-but-never-started ones -- owns a valid mutex:
// thread_free destroys it unconditionally
rb_native_mutex_initialize(&th->interrupt_lock);
rb_threadptr_root_fiber_setup(th);
/* All threads are blocking until a non-blocking fiber is scheduled */
th->blocking = 1;
th->scheduler = Qnil;
if (self == 0) {
size_t size = vm->default_params.thread_vm_stack_size / sizeof(VALUE);
VALUE *stack = ALLOC_N(VALUE, size);
rb_ec_initialize_vm_stack(th->ec, stack, size);
rb_thread_malloc_stack_set(th, stack, size);
}
else {
VM_ASSERT(th->ec->cfp == NULL);
VM_ASSERT(th->ec->vm_stack == NULL);
VM_ASSERT(th->ec->vm_stack_size == 0);
}
th->status = THREAD_RUNNABLE;
th->last_status = Qnil;
th->top_wrapper = 0;
if (box->top_self) {
th->top_self = box->top_self;
}
else {
th->top_self = 0;
}
th->value = Qundef;
th->ec->errinfo = Qnil;
th->ec->root_svar = Qfalse;
th->ec->local_storage_recursive_hash = Qnil;
th->ec->local_storage_recursive_hash_for_trace = Qnil;
th->ec->storage = Qnil;
th->ec->ractor_id = rb_ractor_id(th->ractor);
#if OPT_CALL_THREADED_CODE
th->retval = Qundef;
#endif
th->name = Qnil;
th->report_on_exception = vm->thread_report_on_exception;
th->ext_config.ractor_safe = true;
#if USE_RUBY_DEBUG_LOG
static rb_atomic_t thread_serial = 1;
th->serial = RUBY_ATOMIC_FETCH_ADD(thread_serial, 1);
RUBY_DEBUG_LOG("th:%u", th->serial);
#endif
}
VALUE
rb_thread_alloc_in_objspace(VALUE klass, void *objspace)
{
VALUE self = thread_alloc(klass, objspace);
rb_thread_t *target_th = rb_thread_ptr(self);
target_th->ractor = GET_RACTOR();
th_init(target_th, self, target_th->vm = GET_VM());
rb_root_fiber_obj_setup(target_th, objspace);
return self;
}
VALUE
rb_thread_alloc(VALUE klass)
{
return rb_thread_alloc_in_objspace(klass, GET_RACTOR()->objspace);
}
#define REWIND_CFP(expr) do { \
rb_execution_context_t *ec__ = GET_EC(); \
VALUE *const curr_sp = (ec__->cfp++)->sp; \
VALUE *const saved_sp = ec__->cfp->sp; \
ec__->cfp->sp = curr_sp; \
expr; \
(ec__->cfp--)->sp = saved_sp; \
} while (0)
static VALUE
m_core_set_method_alias(VALUE self, VALUE cbase, VALUE sym1, VALUE sym2)
{
REWIND_CFP({
rb_alias(cbase, SYM2ID(sym1), SYM2ID(sym2));
});
return Qnil;
}
static VALUE
m_core_set_variable_alias(VALUE self, VALUE sym1, VALUE sym2)
{
REWIND_CFP({
rb_alias_variable(SYM2ID(sym1), SYM2ID(sym2));
});
return Qnil;
}
static VALUE
m_core_undef_method(VALUE self, VALUE cbase, VALUE sym)
{
REWIND_CFP({
ID mid = SYM2ID(sym);
rb_undef(cbase, mid);
rb_clear_method_cache(self, mid);
});
return Qnil;
}
static VALUE
m_core_set_postexe(VALUE self)
{
rb_ractor_ensure_main_ractor("can not use END{} in non-main Ractors");
rb_set_end_proc(rb_call_end_proc, rb_block_proc());
return Qnil;
}
static VALUE
core_hash_merge(VALUE hash, long argc, const VALUE *argv, bool dup)
{
if (NIL_P(hash)) {
hash = rb_cHash_empty_frozen;
}
else {
hash = rb_to_hash_type(hash);
Check_Type(hash, T_HASH);
}
return rb_hash_merge2_bulk(hash, argc, argv, dup);
}
static VALUE
m_core_hash_merge_ptr(int argc, VALUE *argv, VALUE recv)
{
VALUE hash = argv[0];
VM_ASSERT(argc % 2 == 1);
REWIND_CFP(hash = core_hash_merge(hash, argc - 1, argv + 1, true));
return hash;
}
static VALUE
m_core_hash_merge_bang_ptr(int argc, VALUE *argv, VALUE recv)
{
VALUE hash = argv[0];
VM_ASSERT(argc % 2 == 1);
REWIND_CFP(hash = core_hash_merge(hash, argc - 1, argv + 1, false));
return hash;
}
static VALUE
core_hash_merge_kwd(VALUE hash, VALUE kw, bool dup)
{
kw = rb_to_hash_type(kw);
if (NIL_P(hash)) {
return dup ? rb_hash_resurrect(kw) : kw;
}
else {
hash = rb_to_hash_type(hash);
Check_Type(hash, T_HASH);
return rb_hash_merge2(hash, kw, dup);
}
}
static VALUE
m_core_hash_merge_kwd(VALUE recv, VALUE hash, VALUE kw)
{
// We don't own `hash` so we can't mutate it, nor just return it.
if (NIL_P(kw)) {
if (NIL_P(hash)) {
// If we knew that we're dealing with keyword arguments, and not a hash literal,
// we could return nil here.
return rb_hash_new();
}
hash = rb_hash_resurrect(hash);
}
else {
REWIND_CFP(hash = core_hash_merge_kwd(hash, kw, true));
}
VM_ASSERT(CLASS_OF(hash));
return hash;
}
static VALUE
m_core_hash_merge_bang_kwd(VALUE recv, VALUE hash, VALUE kw)
{
// We own `hash` we should mutate it in place if possible.
if (NIL_P(kw)) {
if (NIL_P(hash)) {
hash = rb_hash_new();
}
else {
hash = rb_hash_resurrect(hash);
}
}
else {
REWIND_CFP(hash = core_hash_merge_kwd(hash, kw, false));
}
VM_ASSERT(CLASS_OF(hash));
return hash;
}
static VALUE
core_hash_coerce(VALUE hash)
{
if (NIL_P(hash)) {
return rb_hash_new();
}
VALUE new_hash = rb_to_hash_type(hash);
if (new_hash == hash) {
new_hash = rb_hash_dup(new_hash);
}
return new_hash;
}
static VALUE
m_core_hash_coerce(VALUE recv, VALUE hash)
{
REWIND_CFP(hash = core_hash_coerce(hash));
VM_ASSERT(CLASS_OF(hash));
return hash;
}
static VALUE
m_core_make_shareable(VALUE recv, VALUE obj)
{
return rb_ractor_make_shareable(obj);
}
static VALUE
m_core_make_shareable_copy(VALUE recv, VALUE obj)
{
return rb_ractor_make_shareable_copy(obj);
}
static VALUE
m_core_ensure_shareable(VALUE recv, VALUE obj, VALUE name)
{
return rb_ractor_ensure_shareable(obj, name);
}
extern VALUE *rb_gc_stack_start;
extern size_t rb_gc_stack_maxsize;
/* debug functions */
/* :nodoc: */
static VALUE
sdr(VALUE self)
{
rb_vm_bugreport(NULL, stderr);
return Qnil;
}
/* :nodoc: */
static VALUE
nsdr(VALUE self)
{
VALUE ary = rb_ary_new();
#ifdef HAVE_BACKTRACE
#include <execinfo.h>
#define MAX_NATIVE_TRACE 1024
static void *trace[MAX_NATIVE_TRACE];
int n = (int)backtrace(trace, MAX_NATIVE_TRACE);
char **syms = backtrace_symbols(trace, n);
int i;
if (syms == 0) {
rb_memerror();
}
for (i=0; i<n; i++) {
rb_ary_push(ary, rb_str_new2(syms[i]));
}
free(syms); /* OK */
#endif
return ary;
}
#if VM_COLLECT_USAGE_DETAILS
static VALUE usage_analysis_insn_start(VALUE self);
static VALUE usage_analysis_operand_start(VALUE self);
static VALUE usage_analysis_register_start(VALUE self);
static VALUE usage_analysis_insn_stop(VALUE self);
static VALUE usage_analysis_operand_stop(VALUE self);
static VALUE usage_analysis_register_stop(VALUE self);
static VALUE usage_analysis_insn_running(VALUE self);
static VALUE usage_analysis_operand_running(VALUE self);
static VALUE usage_analysis_register_running(VALUE self);
static VALUE usage_analysis_insn_clear(VALUE self);
static VALUE usage_analysis_operand_clear(VALUE self);
static VALUE usage_analysis_register_clear(VALUE self);
#endif
static VALUE
f_raise(int c, VALUE *v, VALUE _)
{
return rb_f_raise(c, v);
}
static VALUE
f_proc(VALUE _)
{
return rb_block_proc();
}
static VALUE
f_lambda(VALUE _)
{
return rb_block_lambda();
}
static VALUE
f_sprintf(int c, const VALUE *v, VALUE _)
{
return rb_f_sprintf(c, v);
}
/* :nodoc: */
static VALUE
vm_mtbl(VALUE self, VALUE obj, VALUE sym)
{
vm_mtbl_dump(CLASS_OF(obj), RTEST(sym) ? SYM2ID(sym) : 0);
return Qnil;
}
/* :nodoc: */
static VALUE
vm_mtbl2(VALUE self, VALUE obj, VALUE sym)
{
vm_mtbl_dump(obj, RTEST(sym) ? SYM2ID(sym) : 0);
return Qnil;
}
/*
* call-seq:
* RubyVM.keep_script_lines -> true or false
*
* Return current +keep_script_lines+ status. Now it only returns
* +true+ of +false+, but it can return other objects in future.
*
* Note that this is an API for ruby internal use, debugging,
* and research. Do not use this for any other purpose.
* The compatibility is not guaranteed.
*/
static VALUE
vm_keep_script_lines(VALUE self)
{
return RBOOL(ruby_vm_keep_script_lines);
}
/*
* call-seq:
* RubyVM.keep_script_lines = true / false
*
* It set +keep_script_lines+ flag. If the flag is set, all
* loaded scripts are recorded in a interpreter process.
*
* Note that this is an API for ruby internal use, debugging,
* and research. Do not use this for any other purpose.
* The compatibility is not guaranteed.
*/
static VALUE
vm_keep_script_lines_set(VALUE self, VALUE flags)
{
ruby_vm_keep_script_lines = RTEST(flags);
return flags;
}
void
Init_VM(void)
{
VALUE opts;
VALUE klass;
VALUE fcore;
/*
* Document-class: RubyVM
*
* The RubyVM module only exists on MRI. +RubyVM+ is not defined in
* other Ruby implementations such as JRuby and TruffleRuby.
*
* The RubyVM module provides some access to MRI internals.
* This module is for very limited purposes, such as debugging,
* prototyping, and research. Normal users must not use it.
* This module is not portable between Ruby implementations.
*/
rb_cRubyVM = rb_define_class("RubyVM", rb_cObject);
rb_undef_alloc_func(rb_cRubyVM);
rb_undef_method(CLASS_OF(rb_cRubyVM), "new");
rb_define_singleton_method(rb_cRubyVM, "stat", vm_stat, -1);
rb_define_singleton_method(rb_cRubyVM, "keep_script_lines", vm_keep_script_lines, 0);
rb_define_singleton_method(rb_cRubyVM, "keep_script_lines=", vm_keep_script_lines_set, 1);
#if USE_DEBUG_COUNTER
rb_define_singleton_method(rb_cRubyVM, "reset_debug_counters", rb_debug_counter_reset, 0);
rb_define_singleton_method(rb_cRubyVM, "show_debug_counters", rb_debug_counter_show, 0);
#endif
/* FrozenCore (hidden) */
fcore = rb_class_new(rb_cBasicObject);
rb_set_class_path(fcore, rb_cRubyVM, "FrozenCore");
rb_vm_register_global_object(rb_mod_name(fcore));
klass = rb_singleton_class(fcore);
rb_define_method_id(klass, id_core_set_method_alias, m_core_set_method_alias, 3);
rb_define_method_id(klass, id_core_set_variable_alias, m_core_set_variable_alias, 2);
rb_define_method_id(klass, id_core_undef_method, m_core_undef_method, 2);
rb_define_method_id(klass, id_core_set_postexe, m_core_set_postexe, 0);
rb_define_method_id(klass, id_core_hash_merge_ptr, m_core_hash_merge_ptr, -1);
rb_define_method_id(klass, id_core_hash_merge_bang_ptr, m_core_hash_merge_bang_ptr, -1);
rb_define_method_id(klass, id_core_hash_merge_kwd, m_core_hash_merge_kwd, 2);
rb_define_method_id(klass, id_core_hash_merge_bang_kwd, m_core_hash_merge_bang_kwd, 2);
rb_define_method_id(klass, id_core_hash_coerce, m_core_hash_coerce, 1);
rb_define_method_id(klass, id_core_raise, f_raise, -1);
rb_define_method_id(klass, id_core_sprintf, f_sprintf, -1);
rb_define_method_id(klass, idProc, f_proc, 0);
rb_define_method_id(klass, idLambda, f_lambda, 0);
rb_define_method(klass, "make_shareable", m_core_make_shareable, 1);
rb_define_method(klass, "make_shareable_copy", m_core_make_shareable_copy, 1);
rb_define_method(klass, "ensure_shareable", m_core_ensure_shareable, 2);
rb_obj_freeze(fcore);
RBASIC_CLEAR_CLASS(klass);
rb_obj_freeze(klass);
rb_vm_register_global_object(fcore);
rb_mRubyVMFrozenCore = fcore;
/*
* Document-class: Thread
*
* Threads are the Ruby implementation for a concurrent programming model.
*
* Programs that require multiple threads of execution are a perfect
* candidate for Ruby's Thread class.
*
* For example, we can create a new thread separate from the main thread's
* execution using ::new.
*
* thr = Thread.new { puts "What's the big deal" }
*
* Then we are able to pause the execution of the main thread and allow
* our new thread to finish, using #join:
*
* thr.join #=> "What's the big deal"
*
* If we don't call +thr.join+ before the main thread terminates, then all
* other threads including +thr+ will be killed.
*
* Alternatively, you can use an array for handling multiple threads at
* once, like in the following example:
*
* threads = []
* threads << Thread.new { puts "What's the big deal" }
* threads << Thread.new { 3.times { puts "Threads are fun!" } }
*
* After creating a few threads we wait for them all to finish
* consecutively.
*
* threads.each { |thr| thr.join }
*
* To retrieve the last value of a thread, use #value
*
* thr = Thread.new { sleep 1; "Useful value" }
* thr.value #=> "Useful value"
*
* === Thread initialization
*
* In order to create new threads, Ruby provides ::new, ::start, and
* ::fork. A block must be provided with each of these methods, otherwise
* a ThreadError will be raised.
*
* When subclassing the Thread class, the +initialize+ method of your
* subclass will be ignored by ::start and ::fork. Otherwise, be sure to
* call super in your +initialize+ method.
*
* === Thread termination
*
* For terminating threads, Ruby provides a variety of ways to do this.
*
* The class method ::kill, is meant to exit a given thread:
*
* thr = Thread.new { sleep }
* Thread.kill(thr) # sends exit() to thr
*
* Alternatively, you can use the instance method #exit, or any of its
* aliases #kill or #terminate.
*
* thr.exit
*
* === Thread status
*
* Ruby provides a few instance methods for querying the state of a given
* thread. To get a string with the current thread's state use #status
*
* thr = Thread.new { sleep }
* thr.status # => "sleep"
* thr.exit
* thr.status # => false
*
* You can also use #alive? to tell if the thread is running or sleeping,
* and #stop? if the thread is dead or sleeping.
*
* === Thread variables and scope
*
* Since threads are created with blocks, the same rules apply to other
* Ruby blocks for variable scope. Any local variables created within this
* block are accessible to only this thread.
*
* ==== Fiber-local vs. Thread-local
*
* Each fiber has its own bucket for Thread#[] storage. When you set a
* new fiber-local it is only accessible within this Fiber. To illustrate:
*
* Thread.new {
* Thread.current[:foo] = "bar"
* Fiber.new {
* p Thread.current[:foo] # => nil
* }.resume
* }.join
*
* This example uses #[] for getting and #[]= for setting fiber-locals,
* you can also use #keys to list the fiber-locals for a given
* thread and #key? to check if a fiber-local exists.
*
* When it comes to thread-locals, they are accessible within the entire
* scope of the thread. Given the following example:
*
* Thread.new{
* Thread.current.thread_variable_set(:foo, 1)
* p Thread.current.thread_variable_get(:foo) # => 1
* Fiber.new{
* Thread.current.thread_variable_set(:foo, 2)
* p Thread.current.thread_variable_get(:foo) # => 2
* }.resume
* p Thread.current.thread_variable_get(:foo) # => 2
* }.join
*
* You can see that the thread-local +:foo+ carried over into the fiber
* and was changed to +2+ by the end of the thread.
*
* This example makes use of #thread_variable_set to create new
* thread-locals, and #thread_variable_get to reference them.
*
* There is also #thread_variables to list all thread-locals, and
* #thread_variable? to check if a given thread-local exists.
*
* === Exception handling
*
* When an unhandled exception is raised inside a thread, it will
* terminate. By default, this exception will not propagate to other
* threads. The exception is stored and when another thread calls #value
* or #join, the exception will be re-raised in that thread.
*
* t = Thread.new{ raise 'something went wrong' }
* t.value #=> RuntimeError: something went wrong
*
* An exception can be raised from outside the thread using the
* Thread#raise instance method, which takes the same parameters as
* Kernel#raise.
*
* Setting Thread.abort_on_exception = true, Thread#abort_on_exception =
* true, or $DEBUG = true will cause a subsequent unhandled exception
* raised in a thread to be automatically re-raised in the main thread.
*
* With the addition of the class method ::handle_interrupt, you can now
* handle exceptions asynchronously with threads.
*
* === Scheduling
*
* Ruby provides a few ways to support scheduling threads in your program.
*
* The first way is by using the class method ::stop, to put the current
* running thread to sleep and schedule the execution of another thread.
*
* Once a thread is asleep, you can use the instance method #wakeup to
* mark your thread as eligible for scheduling.
*
* You can also try ::pass, which attempts to pass execution to another
* thread but is dependent on the OS whether a running thread will switch
* or not. The same goes for #priority, which lets you hint to the thread
* scheduler which threads you want to take precedence when passing
* execution. This method is also dependent on the OS and may be ignored
* on some platforms.
*
*/
rb_cThread = rb_define_class("Thread", rb_cObject);
rb_undef_alloc_func(rb_cThread);
#if VM_COLLECT_USAGE_DETAILS
/* ::RubyVM::USAGE_ANALYSIS_* */
#define define_usage_analysis_hash(name) /* shut up rdoc -C */ \
rb_define_const(rb_cRubyVM, "USAGE_ANALYSIS_" #name, rb_hash_new())
define_usage_analysis_hash(INSN);
define_usage_analysis_hash(REGS);
define_usage_analysis_hash(INSN_BIGRAM);
rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_INSN_START", usage_analysis_insn_start, 0);
rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_OPERAND_START", usage_analysis_operand_start, 0);
rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_REGISTER_START", usage_analysis_register_start, 0);
rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_INSN_STOP", usage_analysis_insn_stop, 0);
rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_OPERAND_STOP", usage_analysis_operand_stop, 0);
rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_REGISTER_STOP", usage_analysis_register_stop, 0);
rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_INSN_RUNNING", usage_analysis_insn_running, 0);
rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_OPERAND_RUNNING", usage_analysis_operand_running, 0);
rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_REGISTER_RUNNING", usage_analysis_register_running, 0);
rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_INSN_CLEAR", usage_analysis_insn_clear, 0);
rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_OPERAND_CLEAR", usage_analysis_operand_clear, 0);
rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_REGISTER_CLEAR", usage_analysis_register_clear, 0);
#endif
/* ::RubyVM::OPTS
* An Array of VM build options.
* This constant is MRI specific.
*/
rb_define_const(rb_cRubyVM, "OPTS", opts = rb_ary_new());
#if OPT_DIRECT_THREADED_CODE
rb_ary_push(opts, rb_str_new2("direct threaded code"));
#elif OPT_TOKEN_THREADED_CODE
rb_ary_push(opts, rb_str_new2("token threaded code"));
#elif OPT_CALL_THREADED_CODE
rb_ary_push(opts, rb_str_new2("call threaded code"));
#endif
#if OPT_OPERANDS_UNIFICATION
rb_ary_push(opts, rb_str_new2("operands unification"));
#endif
#if OPT_INSTRUCTIONS_UNIFICATION
rb_ary_push(opts, rb_str_new2("instructions unification"));
#endif
#if OPT_INLINE_METHOD_CACHE
rb_ary_push(opts, rb_str_new2("inline method cache"));
#endif
/* ::RubyVM::INSTRUCTION_NAMES
* A list of bytecode instruction names in MRI.
* This constant is MRI specific.
*/
rb_define_const(rb_cRubyVM, "INSTRUCTION_NAMES", rb_insns_name_array());
/* ::RubyVM::DEFAULT_PARAMS
* This constant exposes the VM's default parameters.
* Note that changing these values does not affect VM execution.
* Specification is not stable and you should not depend on this value.
* Of course, this constant is MRI specific.
*/
rb_define_const(rb_cRubyVM, "DEFAULT_PARAMS", vm_default_params());
/* debug functions ::RubyVM::SDR(), ::RubyVM::NSDR() */
#if VMDEBUG
rb_define_singleton_method(rb_cRubyVM, "SDR", sdr, 0);
rb_define_singleton_method(rb_cRubyVM, "NSDR", nsdr, 0);
rb_define_singleton_method(rb_cRubyVM, "mtbl", vm_mtbl, 2);
rb_define_singleton_method(rb_cRubyVM, "mtbl2", vm_mtbl2, 2);
#else
(void)sdr;
(void)nsdr;
(void)vm_mtbl;
(void)vm_mtbl2;
#endif
/* VM bootstrap: phase 2 */
{
rb_vm_t *vm = ruby_current_vm_ptr;
rb_thread_t *th = GET_THREAD();
VALUE filename = rb_fstring_lit("<main>");
const rb_iseq_t *iseq = rb_iseq_new(Qnil, filename, filename, Qnil, 0, ISEQ_TYPE_TOP);
// Ractor setup
rb_ractor_main_setup(vm, th->ractor, th);
/* create vm object */
vm->self = TypedData_Wrap_Struct(rb_cRubyVM, &vm_data_type, vm);
/* create main thread */
th->self = TypedData_Wrap_Struct(rb_cThread, &thread_data_type, th);
vm->ractor.main_thread = th;
vm->ractor.main_ractor = th->ractor;
th->vm = vm;
th->top_wrapper = 0;
th->top_self = rb_vm_top_self();
rb_root_fiber_obj_setup(th, th->ractor->objspace);
rb_vm_register_global_object((VALUE)iseq);
th->ec->cfp->_iseq = iseq;
th->ec->cfp->pc = ISEQ_BODY(iseq)->iseq_encoded;
th->ec->cfp->self = th->top_self;
VM_ENV_FLAGS_UNSET(th->ec->cfp->ep, VM_FRAME_FLAG_CFRAME);
VM_STACK_ENV_WRITE(th->ec->cfp->ep, VM_ENV_DATA_INDEX_ME_CREF, (VALUE)vm_cref_new(rb_cObject, METHOD_VISI_PRIVATE, FALSE, NULL, FALSE, FALSE));
/*
* The Binding of the top level scope
*/
rb_define_global_const("TOPLEVEL_BINDING", rb_binding_new());
#ifdef _WIN32
rb_objspace_gc_enable(vm->ractor.main_ractor->objspace);
#endif
}
vm_init_redefined_flag();
rb_block_param_proxy = rb_obj_alloc(rb_cObject);
VALUE proxy_singleton = rb_singleton_class(rb_block_param_proxy);
rb_add_method_optimized(proxy_singleton, idCall, OPTIMIZED_METHOD_TYPE_BLOCK_CALL, 0, METHOD_VISI_PUBLIC);
rb_add_method_optimized(proxy_singleton, idAREF, OPTIMIZED_METHOD_TYPE_BLOCK_CALL, 0, METHOD_VISI_PUBLIC);
rb_add_method_optimized(proxy_singleton, idYield, OPTIMIZED_METHOD_TYPE_BLOCK_CALL, 0, METHOD_VISI_PUBLIC);
rb_add_method_optimized(proxy_singleton, idEqq, OPTIMIZED_METHOD_TYPE_BLOCK_CALL, 0, METHOD_VISI_PUBLIC);
rb_obj_freeze(rb_block_param_proxy);
rb_vm_register_global_object(rb_block_param_proxy);
/* vm_backtrace.c */
Init_vm_backtrace();
}
void
rb_vm_set_progname(VALUE filename)
{
rb_thread_t *th = GET_VM()->ractor.main_thread;
rb_control_frame_t *cfp = (void *)(th->ec->vm_stack + th->ec->vm_stack_size);
--cfp;
filename = rb_str_new_frozen(filename);
rb_iseq_pathobj_set(CFP_ISEQ(cfp), filename, rb_iseq_realpath(CFP_ISEQ(cfp)));
}
extern const struct st_hash_type rb_fstring_hash_type;
static rb_vm_t _vm;
static rb_thread_t _main_thread = {
.vm = &_vm,
.main_thread = 1,
};
void
Init_BareVM(void)
{
/* VM bootstrap: phase 1 */
rb_vm_t *vm = &_vm;
rb_thread_t *th = &_main_thread;
// setup the VM
vm_init2(vm);
ruby_current_vm_ptr = vm;
/* The boot objspace belongs to the main Ractor, so the main Ractor has to exist
* before rb_gc_init_objspaces allocates it. */
vm->ractor.main_ractor = rb_ractor_main_alloc();
rb_native_mutex_initialize(&vm->gc.registered_addrs.lock);
rb_gc_init_objspaces();
vm->ractor.main_ractor->newobj_cache = rb_gc_ractor_cache_alloc(vm->ractor.main_ractor);
rb_id_table_init(&vm->negative_cme_table, 16);
st_init_existing_numtable_with_size(&vm->overloaded_cme_table, 0);
st_init_existing_strtable_with_size(&vm->static_ext_inits, 0);
set_init_embedded_numtable_with_size(&vm->unused_block_warning_table, 0);
vm->global_hooks.type = hook_list_type_global;
// setup main thread
th->nt = ZALLOC(struct rb_native_thread);
th->ractor = vm->ractor.main_ractor;
Init_native_thread(th);
rb_jit_cont_init();
th_init(th, 0, vm);
rb_ractor_set_current_ec(th->ractor, th->ec);
/* n.b. native_main_thread_stack_top is set by the INIT_STACK macro */
ruby_thread_init_stack(th, native_main_thread_stack_top);
// setup ractor system
rb_native_mutex_initialize(&vm->ractor.sync.lock);
rb_native_cond_initialize(&vm->ractor.sync.terminate_cond);
rb_native_mutex_initialize(&vm->ractor.generic_fields_lock);
vm->gc.orphan_merge_pjob = POSTPONED_JOB_HANDLE_INVALID;
vm_opt_method_def_table = st_init_numtable();
vm_opt_mid_table = st_init_numtable();
}
void
ruby_init_stack(void *addr)
{
native_main_thread_stack_top = addr;
}
#ifndef _WIN32
#include <unistd.h>
#include <sys/mman.h>
#endif
void
rb_vm_register_global_object(VALUE obj)
{
RUBY_ASSERT(!RB_SPECIAL_CONST_P(obj));
if (RB_SPECIAL_CONST_P(obj)) {
return;
}
switch (RB_BUILTIN_TYPE(obj)) {
case T_CLASS:
case T_MODULE:
if (FL_TEST(obj, RCLASS_IS_ROOT)) {
return;
}
FL_SET(obj, RCLASS_IS_ROOT);
break;
default:
break;
}
/* Register in the current Ractor's own pin list (a raw array). No lock: only the
* owner appends and only the owner's GC marks it; the merge that inherits a list
* runs stop-the-world. */
rb_ractor_t *cr = GET_RACTOR();
if (cr->registered_marks_cnt == cr->registered_marks_capa) {
size_t nc = cr->registered_marks_capa ? cr->registered_marks_capa * 2 : 64;
VALUE *p = realloc(cr->registered_marks, nc * sizeof(VALUE));
if (!p) rb_bug("rb_vm_register_global_object: out of memory");
cr->registered_marks = p;
cr->registered_marks_capa = nc;
}
cr->registered_marks[cr->registered_marks_cnt++] = obj;
RB_GC_GUARD(obj);
}
VALUE rb_cc_refinement_set_create(void);
void
Init_vm_objects(void)
{
rb_vm_t *vm = GET_VM();
st_init_existing_table_with_size(&vm->ci_table, &vm_ci_hashtype, 0);
vm->cc_refinement_set = rb_cc_refinement_set_create();
}
// Whether JIT is enabled or not, we need to load/undef `#with_jit` for other builtins.
#include "jit_hook.rbinc"
#include "jit_undef.rbinc"
// Stub for builtin function when not building YJIT units
#if !USE_YJIT
void Init_builtin_yjit(void) {}
#endif
// Stub for builtin function when not building ZJIT units
#if !USE_ZJIT
void Init_builtin_zjit(void) {}
#endif
/* top self */
static VALUE
main_to_s(VALUE obj)
{
return rb_str_new2("main");
}
VALUE
rb_vm_top_self(void)
{
const rb_box_t *box = rb_current_box();
VM_ASSERT(box);
VM_ASSERT(box->top_self);
return box->top_self;
}
void
Init_top_self(void)
{
rb_vm_t *vm = GET_VM();
vm->root_box = (rb_box_t *)rb_root_box();
vm->root_box->top_self = rb_obj_alloc(rb_cObject);
rb_define_singleton_method(vm->root_box->top_self, "to_s", main_to_s, 0);
rb_define_alias(rb_singleton_class(vm->root_box->top_self), "inspect", "to_s");
}
VALUE *
rb_ruby_verbose_ptr(void)
{
rb_ractor_t *cr = GET_RACTOR();
return &cr->verbose;
}
VALUE *
rb_ruby_debug_ptr(void)
{
rb_ractor_t *cr = GET_RACTOR();
return &cr->debug;
}
bool rb_free_at_exit = false;
bool
ruby_free_at_exit_p(void)
{
return rb_free_at_exit;
}
/* iseq.c */
VALUE rb_insn_operand_intern(const rb_iseq_t *iseq,
VALUE insn, int op_no, VALUE op,
int len, size_t pos, VALUE *pnop, VALUE child);
#if VM_COLLECT_USAGE_DETAILS
#define HASH_ASET(h, k, v) rb_hash_aset((h), (st_data_t)(k), (st_data_t)(v))
/* uh = {
* insn(Fixnum) => ihash(Hash)
* }
* ihash = {
* -1(Fixnum) => count, # insn usage
* 0(Fixnum) => ophash, # operand usage
* }
* ophash = {
* val(interned string) => count(Fixnum)
* }
*/
static void
vm_analysis_insn(int insn)
{
ID usage_hash;
ID bigram_hash;
static int prev_insn = -1;
VALUE uh;
VALUE ihash;
VALUE cv;
CONST_ID(usage_hash, "USAGE_ANALYSIS_INSN");
CONST_ID(bigram_hash, "USAGE_ANALYSIS_INSN_BIGRAM");
uh = rb_const_get(rb_cRubyVM, usage_hash);
if (NIL_P(ihash = rb_hash_aref(uh, INT2FIX(insn)))) {
ihash = rb_hash_new();
HASH_ASET(uh, INT2FIX(insn), ihash);
}
if (NIL_P(cv = rb_hash_aref(ihash, INT2FIX(-1)))) {
cv = INT2FIX(0);
}
HASH_ASET(ihash, INT2FIX(-1), INT2FIX(FIX2INT(cv) + 1));
/* calc bigram */
if (prev_insn != -1) {
VALUE bi;
VALUE ary[2];
VALUE cv;
ary[0] = INT2FIX(prev_insn);
ary[1] = INT2FIX(insn);
bi = rb_ary_new4(2, &ary[0]);
uh = rb_const_get(rb_cRubyVM, bigram_hash);
if (NIL_P(cv = rb_hash_aref(uh, bi))) {
cv = INT2FIX(0);
}
HASH_ASET(uh, bi, INT2FIX(FIX2INT(cv) + 1));
}
prev_insn = insn;
}
static void
vm_analysis_operand(int insn, int n, VALUE op)
{
ID usage_hash;
VALUE uh;
VALUE ihash;
VALUE ophash;
VALUE valstr;
VALUE cv;
CONST_ID(usage_hash, "USAGE_ANALYSIS_INSN");
uh = rb_const_get(rb_cRubyVM, usage_hash);
if (NIL_P(ihash = rb_hash_aref(uh, INT2FIX(insn)))) {
ihash = rb_hash_new();
HASH_ASET(uh, INT2FIX(insn), ihash);
}
if (NIL_P(ophash = rb_hash_aref(ihash, INT2FIX(n)))) {
ophash = rb_hash_new();
HASH_ASET(ihash, INT2FIX(n), ophash);
}
/* intern */
valstr = rb_insn_operand_intern(CFP_ISEQ(GET_EC()->cfp), insn, n, op, 0, 0, 0, 0);
/* set count */
if (NIL_P(cv = rb_hash_aref(ophash, valstr))) {
cv = INT2FIX(0);
}
HASH_ASET(ophash, valstr, INT2FIX(FIX2INT(cv) + 1));
}
static void
vm_analysis_register(int reg, int isset)
{
ID usage_hash;
VALUE uh;
VALUE valstr;
static const char regstrs[][5] = {
"pc", /* 0 */
"sp", /* 1 */
"ep", /* 2 */
"cfp", /* 3 */
"self", /* 4 */
"iseq", /* 5 */
};
static const char getsetstr[][4] = {
"get",
"set",
};
static VALUE syms[sizeof(regstrs) / sizeof(regstrs[0])][2];
VALUE cv;
CONST_ID(usage_hash, "USAGE_ANALYSIS_REGS");
if (syms[0] == 0) {
char buff[0x10];
int i;
for (i = 0; i < (int)(sizeof(regstrs) / sizeof(regstrs[0])); i++) {
int j;
for (j = 0; j < 2; j++) {
snprintf(buff, 0x10, "%d %s %-4s", i, getsetstr[j], regstrs[i]);
syms[i][j] = ID2SYM(rb_intern(buff));
}
}
}
valstr = syms[reg][isset];
uh = rb_const_get(rb_cRubyVM, usage_hash);
if (NIL_P(cv = rb_hash_aref(uh, valstr))) {
cv = INT2FIX(0);
}
HASH_ASET(uh, valstr, INT2FIX(FIX2INT(cv) + 1));
}
#undef HASH_ASET
static void (*ruby_vm_collect_usage_func_insn)(int insn) = NULL;
static void (*ruby_vm_collect_usage_func_operand)(int insn, int n, VALUE op) = NULL;
static void (*ruby_vm_collect_usage_func_register)(int reg, int isset) = NULL;
/* :nodoc: */
static VALUE
usage_analysis_insn_start(VALUE self)
{
ruby_vm_collect_usage_func_insn = vm_analysis_insn;
return Qnil;
}
/* :nodoc: */
static VALUE
usage_analysis_operand_start(VALUE self)
{
ruby_vm_collect_usage_func_operand = vm_analysis_operand;
return Qnil;
}
/* :nodoc: */
static VALUE
usage_analysis_register_start(VALUE self)
{
ruby_vm_collect_usage_func_register = vm_analysis_register;
return Qnil;
}
/* :nodoc: */
static VALUE
usage_analysis_insn_stop(VALUE self)
{
ruby_vm_collect_usage_func_insn = 0;
return Qnil;
}
/* :nodoc: */
static VALUE
usage_analysis_operand_stop(VALUE self)
{
ruby_vm_collect_usage_func_operand = 0;
return Qnil;
}
/* :nodoc: */
static VALUE
usage_analysis_register_stop(VALUE self)
{
ruby_vm_collect_usage_func_register = 0;
return Qnil;
}
/* :nodoc: */
static VALUE
usage_analysis_insn_running(VALUE self)
{
return RBOOL(ruby_vm_collect_usage_func_insn != 0);
}
/* :nodoc: */
static VALUE
usage_analysis_operand_running(VALUE self)
{
return RBOOL(ruby_vm_collect_usage_func_operand != 0);
}
/* :nodoc: */
static VALUE
usage_analysis_register_running(VALUE self)
{
return RBOOL(ruby_vm_collect_usage_func_register != 0);
}
static VALUE
usage_analysis_clear(VALUE self, ID usage_hash)
{
VALUE uh;
uh = rb_const_get(self, usage_hash);
rb_hash_clear(uh);
return Qtrue;
}
/* :nodoc: */
static VALUE
usage_analysis_insn_clear(VALUE self)
{
ID usage_hash;
ID bigram_hash;
CONST_ID(usage_hash, "USAGE_ANALYSIS_INSN");
CONST_ID(bigram_hash, "USAGE_ANALYSIS_INSN_BIGRAM");
usage_analysis_clear(rb_cRubyVM, usage_hash);
return usage_analysis_clear(rb_cRubyVM, bigram_hash);
}
/* :nodoc: */
static VALUE
usage_analysis_operand_clear(VALUE self)
{
ID usage_hash;
CONST_ID(usage_hash, "USAGE_ANALYSIS_INSN");
return usage_analysis_clear(self, usage_hash);
}
/* :nodoc: */
static VALUE
usage_analysis_register_clear(VALUE self)
{
ID usage_hash;
CONST_ID(usage_hash, "USAGE_ANALYSIS_REGS");
return usage_analysis_clear(self, usage_hash);
}
/* @param insn instruction number */
static void
vm_collect_usage_insn(int insn)
{
if (RUBY_DTRACE_INSN_ENABLED()) {
RUBY_DTRACE_INSN(rb_insns_name(insn));
}
if (ruby_vm_collect_usage_func_insn)
(*ruby_vm_collect_usage_func_insn)(insn);
}
/* @param insn instruction number
* @param n n-th operand
* @param op operand value
*/
static void
vm_collect_usage_operand(int insn, int n, VALUE op)
{
if (RUBY_DTRACE_INSN_OPERAND_ENABLED()) {
VALUE valstr;
valstr = rb_insn_operand_intern(CFP_ISEQ(GET_EC()->cfp), insn, n, op, 0, 0, 0, 0);
RUBY_DTRACE_INSN_OPERAND(RSTRING_PTR(valstr), rb_insns_name(insn));
RB_GC_GUARD(valstr);
}
if (ruby_vm_collect_usage_func_operand)
(*ruby_vm_collect_usage_func_operand)(insn, n, op);
}
/* @param reg register id. see code of vm_analysis_register() */
/* @param isset 0: read, 1: write */
static void
vm_collect_usage_register(int reg, int isset)
{
if (ruby_vm_collect_usage_func_register)
(*ruby_vm_collect_usage_func_register)(reg, isset);
}
#endif
const struct rb_callcache *
rb_vm_empty_cc(void)
{
return &vm_empty_cc;
}
const struct rb_callcache *
rb_vm_empty_cc_for_super(void)
{
return &vm_empty_cc_for_super;
}
#include "vm_call_iseq_optimized.inc" /* required from vm_insnhelper.c */
View Git Blame Copy Permalink