ruby/ractor_core.h

#ifndef RUBY_RACTOR_CORE_H
#define RUBY_RACTOR_CORE_H
#include "internal/gc.h"
#include "internal/sanitizers.h"
#include "ruby/ruby.h"
#include "ruby/ractor.h"
#include "vm_core.h"
#include "id_table.h"
#include "vm_debug.h"
#include "hrtime.h"

#ifndef RACTOR_CHECK_MODE
#define RACTOR_CHECK_MODE (VM_CHECK_MODE || RUBY_DEBUG) && (SIZEOF_UINT64_T == SIZEOF_VALUE)
#endif

/* Record the registration-time VALUE in each rb_gc_register_address entry so the
 * verifier can distinguish a transitioned cross-Ractor store from conservative
 * garbage.  Debug and ASAN builds only; production keeps one-word entries. */
#ifndef RB_GC_REGISTERED_ADDR_CHECK
# if RUBY_DEBUG || defined(RUBY_ASAN_ENABLED)
#  define RB_GC_REGISTERED_ADDR_CHECK 1
# else
#  define RB_GC_REGISTERED_ADDR_CHECK 0
# endif
#endif

// experimental flag because it is not sure it is the common pattern
#define RUBY_TYPED_FROZEN_SHAREABLE_NO_REC RUBY_FL_FINALIZE

/* An in-flight move payload, serialized off-heap (defined in ractor.c). */
struct rb_ractor_courier;

struct rb_ractor_sync {
    // ractor lock
    rb_nativethread_lock_t lock;

#if RACTOR_CHECK_MODE > 0
    VALUE locked_by;
#endif

    // incoming messages
    struct ractor_queue *recv_queue;

    // waiting threads for receiving
    struct ccan_list_head waiters;

    // ports
    VALUE default_port_value;
    struct st_table *ports;
    size_t next_port_id;

    /* The baskets this Ractor holds that are on no queue: one it is building to send,
     * and one it has taken off a queue and is materializing.  A queued basket is rooted
     * by its queue instead.  Only the owner touches this list. */
    struct ccan_list_head off_queue_baskets;

    // monitors
    struct ccan_list_head monitors;

    // value
    rb_ractor_t *successor;
    VALUE legacy;
    bool legacy_exc;
    bool legacy_taken; /* Ractor#value already returned the value */
};

struct ractor_basket;

// created
//   | ready to run
// ====================== inserted to vm->ractor
//   v
// blocking <---+ all threads are blocking
//   |          |
//   v          |
// running -----+
//   | all threads are terminated.
// ====================== removed from vm->ractor
//   v
// terminated
//
// status is protected by VM lock (global state)
enum ractor_status {
    ractor_created,
    ractor_running,
    ractor_blocking,
    ractor_terminated,
};

struct rb_ractor_registered_addr {
    VALUE *addr;
#if RB_GC_REGISTERED_ADDR_CHECK
    /* A raw provenance snapshot for verification only. Never mark or dereference it. */
    VALUE initial_value;
#endif
};

struct rb_ractor_struct {
    struct rb_ractor_pub pub;
    struct rb_ractor_sync sync;

    /* rb_gc_register_mark_object pins, per Ractor: the owner marks them (live via
     * rb_ractor_mark_local_roots, unmerged zombie via the zombie scan) and a merge moves
     * them to the survivor.  Raw malloc, so a merge during sweep cannot re-enter GC. */
    VALUE *registered_marks;
    size_t registered_marks_cnt, registered_marks_capa;

    struct rb_ractor_registered_addr *registered_addrs;
    size_t registered_addrs_cnt, registered_addrs_capa;
    bool registered_addrs_listed;

    /* traversal-API mark redirect (NULL outside a traversal).  Per Ractor so a
     * concurrent traversal on another Ractor is never observed.  A modular GC's
     * Ractor-less marking worker threads read vm->gc.mark_func_data instead. */
    struct gc_mark_func_data_struct *mark_func_data;

    // thread management
    struct {
        struct ccan_list_head set;
        unsigned int cnt;
        unsigned int blocking_cnt;
        unsigned int sleeper;
        struct rb_thread_sched sched;
        rb_execution_context_t *running_ec;
        rb_thread_t *main;
        // MN termination epilogue: keeps the dying thread marked (like a set
        // member) between leaving the living set and its last use
        rb_thread_t *dying_th;

        // `main` is in rb_thread_terminate_all(), waiting for the others to go
        bool terminating;
    } threads;

    /* Postponed jobs targeted at this Ractor
     * (rb_postponed_job_trigger_for_ractor): bits index the VM-wide
     * preregistration table; any of this Ractor's threads drains them
     * in rb_postponed_job_flush. */
    rb_atomic_t postponed_job_triggered_bits;

    VALUE thgroup_default;

    VALUE name;
    VALUE loc;

    enum ractor_status status_;

    struct ccan_list_node vmlr_node;
    bool in_terminated_set;  /* vmlr_node is on vm->ractor.terminated_set */
    /* The final self collection ran (ractor_postmortem_collect): from then on
     * rb_ractor_mark_local_roots roots only the join value and the registered_marks
     * pins, so the dying thread's own scaffolding can be collected. */
    bool postmortem;

    // ractor local data

    rb_serial_t next_ec_serial;

    st_table *local_storage;
    struct rb_id_table *idkey_local_storage;
    VALUE local_storage_store_lock;

    /* 0 until first use: rb_ractor_stdin and friends build them lazily, with plain
     * stores (rooted via ractor_mark_unshareable_parts; a write barrier on the
     * shareable wrapper would shref-pin them). */
    VALUE r_stdin;
    VALUE r_stdout;
    VALUE r_stderr;
    VALUE verbose;
    VALUE debug;

    bool malloc_gc_disabled;
    bool main_ractor;
    void *newobj_cache;

    /* This Ractor's objspace.  The main Ractor receives the boot objspace from
     * rb_gc_init_objspaces; a non-main Ractor shares the main one until it gets its
     * own (while this is NULL). */
    void *objspace;

    /* A child Ractor's objspace is populated (Thread/Fiber wrappers) before it joins
     * vm->ractor.set, so a whole-VM walk would miss it.  Park it here from wrapper
     * allocation until vm_insert_ractor clears it (under the VM lock) so the global GC
     * still enumerates it. */
    void *creating_child_objspace;

}; // rb_ractor_t is defined in vm_core.h

/* Mark the GC roots held in Ractor r's C structs (from the root scan in gc.c). */
void rb_ractor_mark_local_roots(rb_ractor_t *r);
void rb_ractor_mark_terminated_join_value(rb_ractor_t *r);
void rb_ractor_reap_dead_ports(rb_ractor_t *r);

/* Move src's registered_marks to dst and leave src empty (on join or when an orphan
 * is absorbed).  An absorb can run during a GC sweep, so the implementation uses raw
 * realloc (ractor.c). */
void rb_ractor_absorb_registered_marks(rb_ractor_t *dst, rb_ractor_t *src);

void rb_ractor_absorb_registered_addrs_without_gc(rb_ractor_t *dst, rb_ractor_t *src);

enum ractor_wakeup_status {
    wakeup_none,
    wakeup_by_send,
    wakeup_by_interrupt,
    wakeup_by_close,
};

struct ractor_waiter {
    enum ractor_wakeup_status wakeup_status;
    rb_thread_t *th;
    struct ccan_list_node node;
    rb_atomic_t event_serial;

    // absolute deadline for this wait, NULL when there is no timeout
    const rb_hrtime_t *end;
};

static inline VALUE
rb_ractor_self(const rb_ractor_t *r)
{
    return r->pub.self;
}

rb_ractor_t *rb_ractor_main_alloc(void);
void rb_ractor_main_setup(rb_vm_t *vm, rb_ractor_t *main_ractor, rb_thread_t *main_thread);
void rb_ractor_atexit(rb_execution_context_t *ec, VALUE result);
void rb_ractor_atexit_exception(rb_execution_context_t *ec);
void rb_ractor_teardown(rb_execution_context_t *ec);
void rb_ractor_receive_parameters(rb_execution_context_t *ec, rb_ractor_t *g, int len, VALUE *ptr);
void rb_ractor_send_parameters(rb_execution_context_t *ec, rb_ractor_t *g, VALUE args);
void rb_ractor_setup_default_port(rb_ractor_t *r);

VALUE rb_thread_create_ractor(rb_ractor_t *g, VALUE args, VALUE proc); // defined in thread.c

int rb_ractor_living_thread_num(const rb_ractor_t *);
VALUE rb_ractor_thread_list(void);
bool rb_ractor_p(VALUE rv);

void rb_ractor_living_threads_init(rb_ractor_t *r);
void rb_ractor_living_threads_insert(rb_ractor_t *r, rb_thread_t *th);
void rb_ractor_living_threads_remove(rb_ractor_t *r, rb_thread_t *th);

/* The structs the final self collection swept while the dying thread still stood on
 * them; that thread frees them at its very last step (rb_ractor_postmortem_free). */
struct rb_ractor_postmortem_frees {
    struct rb_thread_struct *th;
    struct rb_fiber_struct *fiber;
};
void rb_ractor_postmortem(rb_thread_t *th, struct rb_ractor_postmortem_frees *pf);
void rb_ractor_postmortem_free(const struct rb_ractor_postmortem_frees *pf);
void rb_ractor_cancel_creation(rb_ractor_t *r, rb_thread_t *th);
void rb_ractor_blocking_threads_inc(rb_ractor_t *r, const char *file, int line); // TODO: file, line only for RUBY_DEBUG_LOG
void rb_ractor_blocking_threads_dec(rb_ractor_t *r, const char *file, int line); // TODO: file, line only for RUBY_DEBUG_LOG

void rb_ractor_vm_barrier_interrupt_running_thread(rb_ractor_t *r);
void rb_ractor_terminate_interrupt_main_thread(rb_ractor_t *r);
void rb_ractor_terminate_all(void);
bool rb_ractor_main_p_(void);
void rb_ractor_atfork(rb_vm_t *vm, rb_thread_t *th);
void rb_ractor_terminate_atfork(rb_vm_t *vm, rb_ractor_t *th);
VALUE rb_ractor_require(VALUE feature, bool silent);
VALUE rb_ractor_autoload_load(VALUE space, ID id);

VALUE rb_ractor_ensure_shareable(VALUE obj, VALUE name);
st_table *rb_ractor_targeted_hooks(rb_ractor_t *cr);

RUBY_SYMBOL_EXPORT_BEGIN
void rb_ractor_finish_marking(bool full_mark);

bool rb_ractor_shareable_p_continue(VALUE obj);

// THIS FUNCTION SHOULD NOT CALL WHILE INCREMENTAL MARKING!!
// This function is for T_DATA::free_func
void rb_ractor_local_storage_delkey(rb_ractor_local_key_t key);

RUBY_SYMBOL_EXPORT_END

static inline bool
rb_ractor_main_p(void)
{
    if (ruby_single_main_ractor) {
        return true;
    }
    else {
        return rb_ractor_main_p_();
    }
}

static inline bool
rb_ractor_status_p(rb_ractor_t *r, enum ractor_status status)
{
    return r->status_ == status;
}

static inline void
rb_ractor_sleeper_threads_inc(rb_ractor_t *r)
{
    r->threads.sleeper++;
}

static inline void
rb_ractor_sleeper_threads_dec(rb_ractor_t *r)
{
    r->threads.sleeper--;
}

static inline void
rb_ractor_sleeper_threads_clear(rb_ractor_t *r)
{
    r->threads.sleeper = 0;
}

static inline int
rb_ractor_sleeper_thread_num(rb_ractor_t *r)
{
    return r->threads.sleeper;
}

static inline void
rb_ractor_thread_switch(rb_ractor_t *cr, rb_thread_t *th, bool always_reset)
{
    RUBY_DEBUG_LOG("th:%d->%u%s",
                   cr->threads.running_ec ? (int)rb_th_serial(cr->threads.running_ec->thread_ptr) : -1,
                   rb_th_serial(th), cr->threads.running_ec == th->ec ? " (same)" : "");

    if (cr->threads.running_ec != th->ec || always_reset) {
        th->running_time_us = 0;
    }

    if (cr->threads.running_ec != th->ec) {
        if (0) {
            ruby_debug_printf("rb_ractor_thread_switch ec:%p->%p\n",
                              (void *)cr->threads.running_ec, (void *)th->ec);
        }
    }
    else {
        return;
    }

    cr->threads.running_ec = th->ec;

    VM_ASSERT(cr == GET_RACTOR());
}

#define rb_ractor_set_current_ec(cr, ec) rb_ractor_set_current_ec_(cr, ec, __FILE__, __LINE__)
#ifdef RB_THREAD_LOCAL_SPECIFIER
void rb_current_ec_set(rb_execution_context_t *ec);
#endif

static inline void
rb_ractor_set_current_ec_(rb_ractor_t *cr, rb_execution_context_t *ec, const char *file, int line)
{
#ifdef RB_THREAD_LOCAL_SPECIFIER
    rb_current_ec_set(ec);
#else
    native_tls_set(ruby_current_ec_key, ec);
#endif
    RUBY_DEBUG_LOG2(file, line, "ec:%p->%p", (void *)cr->threads.running_ec, (void *)ec);
    VM_ASSERT(ec == NULL || cr->threads.running_ec != ec);
    cr->threads.running_ec = ec;
}

void rb_vm_ractor_blocking_cnt_inc(rb_vm_t *vm, rb_ractor_t *cr, const char *file, int line);
void rb_vm_ractor_blocking_cnt_dec(rb_vm_t *vm, rb_ractor_t *cr, const char *file, int line);

static inline rb_serial_t
rb_ractor_id(const rb_ractor_t *r)
{
    return r->pub.id;
}

static inline void
rb_ractor_targeted_hooks_incr(rb_ractor_t *cr)
{
    cr->pub.targeted_hooks_cnt++;
}

static inline void
rb_ractor_targeted_hooks_decr(rb_ractor_t *cr)
{
    RUBY_ASSERT(cr->pub.targeted_hooks_cnt > 0);
    cr->pub.targeted_hooks_cnt--;
}

static inline unsigned int
rb_ractor_targeted_hooks_cnt(rb_ractor_t *cr)
{
    return cr->pub.targeted_hooks_cnt;
}

#if RACTOR_CHECK_MODE > 0

extern bool rb_ractor_ignore_belonging_flag;

/* An object's owning Ractor is decided by the objspace its page belongs to
 * (rb_gc_obj_foreign_p).  Putting an unshareable object on the VM stack of anyone
 * but its owner is a containment violation. */
static inline VALUE
rb_ractor_confirm_belonging(VALUE obj)
{
    if (rb_ractor_ignore_belonging_flag) return obj;
    if (SPECIAL_CONST_P(obj) || RB_OBJ_SHAREABLE_P(obj)) return obj;

    if (UNLIKELY(rb_gc_obj_foreign_p(obj))) {
        rp(obj);
        rb_bug("rb_ractor_confirm_belonging: unshareable object of another Ractor's objspace");
    }
    return obj;
}

static inline void
rb_ractor_ignore_belonging(bool flag)
{
    rb_ractor_ignore_belonging_flag = flag;
}

#else
#define rb_ractor_confirm_belonging(obj) obj
#define rb_ractor_ignore_belonging(flag) (0)
#endif

#endif /* RUBY_RACTOR_CORE_H */