Index: array.c =================================================================== --- array.c (revision 63705) +++ array.c (working copy) @@ -18,6 +18,9 @@ #include "probes.h" #include "id.h" #include "debug_counter.h" +#include "gc.h" + +// #define ARRAY_DEBUG #ifndef ARRAY_DEBUG # define NDEBUG @@ -53,6 +56,8 @@ VALUE rb_cArray; #define FL_SET_EMBED(a) do { \ assert(!ARY_SHARED_P(a)); \ FL_SET((a), RARRAY_EMBED_FLAG); \ + FL_UNSET_RAW((a), RARRAY_TRANSIENT_FLAG); \ + ary_verify(a); \ } while (0) #define FL_UNSET_EMBED(ary) FL_UNSET((ary), RARRAY_EMBED_FLAG|RARRAY_EMBED_LEN_MASK) #define FL_SET_SHARED(ary) do { \ @@ -130,11 +135,48 @@ VALUE rb_cArray; } while (0) #define FL_SET_SHARED_ROOT(ary) do { \ assert(!ARY_EMBED_P(ary)); \ + assert(!ARY_TRANSIENT_P(ary)); \ FL_SET((ary), RARRAY_SHARED_ROOT_FLAG); \ } while (0) #define ARY_SET(a, i, v) RARRAY_ASET((assert(!ARY_SHARED_P(a)), (a)), (i), (v)) + +#ifdef ARRAY_DEBUG +#define ary_verify(ary) ary_verify_(ary, __FILE__, __LINE__) + +static void +ary_verify_(VALUE ary, const char *file, int line) +{ + if (FL_TEST(ary, ELTS_SHARED)) { + VALUE root = RARRAY(ary)->as.heap.aux.shared; + const VALUE *ptr = RARRAY_CONST_PTR(ary); + const VALUE *root_ptr = RARRAY_CONST_PTR(root); + long len = RARRAY_LEN(ary), root_len = RARRAY_LEN(root); + assert(FL_TEST(root, RARRAY_SHARED_ROOT_FLAG)); + assert(root_ptr <= ptr && ptr + len <= root_ptr + root_len); + ary_verify(root); + } + else if (ARY_EMBED_P(ary)) { + assert(!ARY_TRANSIENT_P(ary)); + assert(!ARY_SHARED_P(ary)); + assert(RARRAY_LEN(ary) <= RARRAY_EMBED_LEN_MAX); + } + else { +#if 0 + const VALUE *ptr = RARRAY_CONST_PTR(ary); + long i, len = RARRAY_LEN(ary); + volatile VALUE v; + for (i=0; ias.heap.aux.capa; + + if (ARY_TRANSIENT_P(ary)) { + if (new_capa <= old_capa) { + /* do nothing */ + } + else { + VALUE *new_ptr = rb_transient_heap_alloc(ary, sizeof(VALUE) * new_capa); + + if (new_ptr == NULL) { + new_ptr = ALLOC_N(VALUE, new_capa); + FL_UNSET_RAW(ary, RARRAY_TRANSIENT_FLAG); + } + + MEMCPY(new_ptr, ARY_HEAP_PTR(ary), VALUE, old_capa); + ARY_SET_PTR(ary, new_ptr); + } + } + else { + SIZED_REALLOC_N(RARRAY(ary)->as.heap.ptr, VALUE, new_capa, old_capa); + } +} + +void +rb_ary_detransient(VALUE ary) +{ + // fprintf(stderr, "rb_ary_detransient:\n"); + // fprintf(stderr, "(1) %s\n", rb_obj_info(ary)); + + if (ARY_TRANSIENT_P(ary)) { + VALUE *new_ptr; + long capa = RARRAY(ary)->as.heap.aux.capa; + long len = RARRAY(ary)->as.heap.len; + + assert(ARY_OWNS_HEAP_P(ary)); + assert(ARY_TRANSIENT_P(ary)); + + if (ARY_SHARED_ROOT_P(ary)) { + capa = len; + } + + new_ptr = ALLOC_N(VALUE, capa); + MEMCPY(new_ptr, ARY_HEAP_PTR(ary), VALUE, capa); + RARRAY(ary)->as.heap.ptr = new_ptr; + /* do not use ARY_SET_PTR() because they assert !frozen */ + FL_UNSET_RAW(ary, RARRAY_TRANSIENT_FLAG); + + // fprintf(stderr, "(2) %s\n", rb_obj_info(ary)); + } +} + static void ary_resize_capa(VALUE ary, long capacity) { assert(RARRAY_LEN(ary) <= capacity); assert(!OBJ_FROZEN(ary)); assert(!ARY_SHARED_P(ary)); + + // fprintf(stderr, "ary_resize_capa (%ld): %s\n", capacity, rb_obj_info(ary)); + if (capacity > RARRAY_EMBED_LEN_MAX) { if (ARY_EMBED_P(ary)) { long len = ARY_EMBED_LEN(ary); - VALUE *ptr = ALLOC_N(VALUE, (capacity)); + VALUE *ptr = ary_heap_alloc(ary, capacity); + MEMCPY(ptr, ARY_EMBED_PTR(ary), VALUE, len); FL_UNSET_EMBED(ary); ARY_SET_PTR(ary, ptr); ARY_SET_HEAP_LEN(ary, len); } else { - SIZED_REALLOC_N(RARRAY(ary)->as.heap.ptr, VALUE, capacity, RARRAY(ary)->as.heap.aux.capa); + // fprintf(stderr, "ary_resize_capa %s\n", rb_obj_info(ary)); + ary_heap_realloc(ary, capacity); } - ARY_SET_CAPA(ary, (capacity)); + ARY_SET_CAPA(ary, capacity); + // fprintf(stderr, "-> ary_resize_capa: %s\n", rb_obj_info(ary)); + + // fprintf(stderr, "ary_resize_capa %p len:%ld capa:%ld - %s\n", (void *)ary, RARRAY_LEN(ary), capacity, rb_obj_info(ary)); } else { if (!ARY_EMBED_P(ary)) { long len = RARRAY_LEN(ary); + long old_capa = RARRAY(ary)->as.heap.aux.capa; const VALUE *ptr = RARRAY_CONST_PTR(ary); - if (len > capacity) len = capacity; MEMCPY((VALUE *)RARRAY(ary)->as.ary, ptr, VALUE, len); + ary_heap_free_ptr(ary, ptr, old_capa); + FL_SET_EMBED(ary); ARY_SET_LEN(ary, len); - ruby_sized_xfree((VALUE *)ptr, RARRAY(ary)->as.heap.aux.capa); + + // fprintf(stderr, "ary_resize_capa: heap->embed %p len:%ld\n", (void *)ary, len); } } + + ary_verify(ary); } static inline void @@ -242,8 +396,9 @@ ary_shrink_capa(VALUE ary) long old_capa = RARRAY(ary)->as.heap.aux.capa; assert(!ARY_SHARED_P(ary)); assert(old_capa >= capacity); - if (old_capa > capacity) - SIZED_REALLOC_N(RARRAY(ary)->as.heap.ptr, VALUE, capacity, old_capa); + if (old_capa > capacity) ary_heap_realloc(ary, capacity); + + ary_verify(ary); } static void @@ -258,7 +413,10 @@ ary_double_capa(VALUE ary, long min) new_capa = (ARY_MAX_SIZE - min) / 2; } new_capa += min; + // fprintf(stderr, "ary_double_capa: %p %d\n", (void *)ary, FL_TEST(ary, RARRAY_TRANSIENT_FLAG) ? 1 : 0); ary_resize_capa(ary, new_capa); + + ary_verify(ary); } static void @@ -282,6 +440,8 @@ rb_ary_unshare(VALUE ary) VALUE shared = RARRAY(ary)->as.heap.aux.shared; rb_ary_decrement_share(shared); FL_UNSET_SHARED(ary); + + ary_verify(ary); } static inline void @@ -314,6 +474,7 @@ static inline void rb_ary_modify_check(VALUE ary) { rb_check_frozen(ary); + ary_verify(ary); } void @@ -343,7 +504,7 @@ rb_ary_modify(VALUE ary) rb_ary_decrement_share(shared); } else { - VALUE *ptr = ALLOC_N(VALUE, len); + VALUE *ptr = ary_heap_alloc(ary, len); MEMCPY(ptr, RARRAY_CONST_PTR(ary), VALUE, len); rb_ary_unshare(ary); ARY_SET_CAPA(ary, len); @@ -352,6 +513,7 @@ rb_ary_modify(VALUE ary) rb_gc_writebarrier_remember(ary); } + ary_verify(ary); } static VALUE @@ -370,6 +532,8 @@ ary_ensure_room_for_push(VALUE ary, long if (ARY_SHARED_OCCUPIED(shared)) { if (RARRAY_CONST_PTR(ary) - RARRAY_CONST_PTR(shared) + new_len <= RARRAY_LEN(shared)) { rb_ary_modify_check(ary); + + ary_verify(shared); return shared; } else { @@ -379,6 +543,8 @@ ary_ensure_room_for_push(VALUE ary, long if (new_len > capa - (capa >> 6)) { ary_double_capa(ary, new_len); } + + ary_verify(ary); return ary; } } @@ -393,6 +559,7 @@ ary_ensure_room_for_push(VALUE ary, long ary_double_capa(ary, new_len); } + ary_verify(ary); return ary; } @@ -449,6 +616,8 @@ empty_ary_alloc(VALUE klass) return ary_alloc(klass); } +void rb_transient_heap_dump(void); + static VALUE ary_new(VALUE klass, long capa) { @@ -465,7 +634,7 @@ ary_new(VALUE klass, long capa) ary = ary_alloc(klass); if (capa > RARRAY_EMBED_LEN_MAX) { - ptr = ALLOC_N(VALUE, capa); + ptr = ary_heap_alloc(ary, capa); FL_UNSET_EMBED(ary); ARY_SET_PTR(ary, ptr); ARY_SET_CAPA(ary, capa); @@ -529,7 +698,9 @@ rb_ary_new_from_values(long n, const VAL VALUE rb_ary_tmp_new(long capa) { - return ary_new(0, capa); + VALUE ary = ary_new(0, capa); + rb_ary_detransient(ary); + return ary; } VALUE @@ -546,7 +717,7 @@ rb_ary_free(VALUE ary) { if (ARY_OWNS_HEAP_P(ary)) { RB_DEBUG_COUNTER_INC(obj_ary_ptr); - ruby_sized_xfree((void *)ARY_HEAP_PTR(ary), ARY_HEAP_SIZE(ary)); + ary_heap_free(ary); } else { RB_DEBUG_COUNTER_INC(obj_ary_embed); @@ -569,13 +740,15 @@ ary_discard(VALUE ary) { rb_ary_free(ary); RBASIC(ary)->flags |= RARRAY_EMBED_FLAG; - RBASIC(ary)->flags &= ~RARRAY_EMBED_LEN_MASK; + RBASIC(ary)->flags &= ~(RARRAY_EMBED_LEN_MASK | RARRAY_TRANSIENT_FLAG); } static VALUE ary_make_shared(VALUE ary) { assert(!ARY_EMBED_P(ary)); + ary_verify(ary); + if (ARY_SHARED_P(ary)) { return ARY_SHARED(ary); } @@ -583,6 +756,7 @@ ary_make_shared(VALUE ary) return ary; } else if (OBJ_FROZEN(ary)) { + rb_ary_detransient(ary); ary_shrink_capa(ary); FL_SET_SHARED_ROOT(ary); ARY_SET_SHARED_NUM(ary, 1); @@ -590,17 +764,23 @@ ary_make_shared(VALUE ary) } else { long capa = ARY_CAPA(ary), len = RARRAY_LEN(ary); + const VALUE *ptr; NEWOBJ_OF(shared, struct RArray, 0, T_ARRAY | (RGENGC_WB_PROTECTED_ARRAY ? FL_WB_PROTECTED : 0)); - FL_UNSET_EMBED(shared); + rb_ary_detransient(ary); + ptr = ARY_HEAP_PTR(ary); + + FL_UNSET_EMBED(shared); ARY_SET_LEN((VALUE)shared, capa); - ARY_SET_PTR((VALUE)shared, RARRAY_CONST_PTR(ary)); + ARY_SET_PTR((VALUE)shared, ptr); ary_mem_clear((VALUE)shared, len, capa - len); FL_SET_SHARED_ROOT(shared); ARY_SET_SHARED_NUM((VALUE)shared, 1); FL_SET_SHARED(ary); ARY_SET_SHARED(ary, (VALUE)shared); OBJ_FREEZE(shared); + + ary_verify(ary); return (VALUE)shared; } } @@ -736,7 +916,7 @@ rb_ary_initialize(int argc, VALUE *argv, rb_ary_modify(ary); if (argc == 0) { if (ARY_OWNS_HEAP_P(ary) && RARRAY_CONST_PTR(ary) != 0) { - ruby_sized_xfree((void *)RARRAY_CONST_PTR(ary), ARY_HEAP_SIZE(ary)); + ary_heap_free(ary); } rb_ary_unshare_safe(ary); FL_SET_EMBED(ary); @@ -784,6 +964,17 @@ rb_ary_initialize(int argc, VALUE *argv, return ary; } +static VALUE +ary_initialize_copy(VALUE self, VALUE orig) +{ + FL_UNSET(self, + FL_USER1 | FL_USER2 | FL_USER3 | /* embed */ + FL_USER5 | /* shard */ + FL_USER13); + + return rb_ary_replace(self, orig); +} + /* * Returns a new array populated with the given objects. * @@ -858,6 +1049,8 @@ ary_make_partial(VALUE ary, VALUE klass, ARY_INCREASE_PTR(result, offset); ARY_SET_LEN(result, len); + + ary_verify(result); return result; } } @@ -973,6 +1166,7 @@ rb_ary_pop(VALUE ary) } --n; ARY_SET_LEN(ary, n); + ary_verify(ary); return RARRAY_AREF(ary, n); } @@ -1006,6 +1200,7 @@ rb_ary_pop_m(int argc, VALUE *argv, VALU rb_ary_modify_check(ary); result = ary_take_first_or_last(argc, argv, ary, ARY_TAKE_LAST); ARY_INCREASE_LEN(ary, -RARRAY_LEN(result)); + ary_verify(ary); return result; } @@ -1024,6 +1219,7 @@ rb_ary_shift(VALUE ary) MEMMOVE(ptr, ptr+1, VALUE, len-1); }); /* WB: no new reference */ ARY_INCREASE_LEN(ary, -1); + ary_verify(ary); return top; } assert(!ARY_EMBED_P(ary)); /* ARY_EMBED_LEN_MAX < ARY_DEFAULT_SIZE */ @@ -1037,6 +1233,8 @@ rb_ary_shift(VALUE ary) ARY_INCREASE_PTR(ary, 1); /* shift ptr */ ARY_INCREASE_LEN(ary, -1); + ary_verify(ary); + return top; } @@ -1096,6 +1294,7 @@ rb_ary_shift_m(int argc, VALUE *argv, VA } ARY_INCREASE_LEN(ary, -n); + ary_verify(ary); return result; } @@ -1129,6 +1328,8 @@ ary_ensure_room_for_unshift(VALUE ary, i /* use shared array for big "queues" */ if (new_len > ARY_DEFAULT_SIZE * 4) { + ary_verify(ary); + /* make a room for unshifted items */ capa = ARY_CAPA(ary); ary_make_shared(ary); @@ -1146,6 +1347,8 @@ ary_ensure_room_for_unshift(VALUE ary, i } ARY_SET_PTR(ary, head - argc); assert(ARY_SHARED_OCCUPIED(ARY_SHARED(ary))); + + ary_verify(ary); return ARY_SHARED(ary); } else { @@ -1154,6 +1357,7 @@ ary_ensure_room_for_unshift(VALUE ary, i MEMMOVE(ptr + argc, ptr, VALUE, len); }); + ary_verify(ary); return ary; } } @@ -1667,11 +1871,12 @@ rb_ary_resize(VALUE ary, long len) } else { if (olen > len + ARY_DEFAULT_SIZE) { - SIZED_REALLOC_N(RARRAY(ary)->as.heap.ptr, VALUE, len, RARRAY(ary)->as.heap.aux.capa); + ary_heap_realloc(ary, len); ARY_SET_CAPA(ary, len); } ARY_SET_HEAP_LEN(ary, len); } + ary_verify(ary); return ary; } @@ -2496,7 +2701,7 @@ rb_ary_sort_bang(VALUE ary) rb_ary_unshare(ary); } else { - ruby_sized_xfree((void *)ARY_HEAP_PTR(ary), ARY_HEAP_SIZE(ary)); + ary_heap_free(ary); } ARY_SET_PTR(ary, RARRAY_CONST_PTR(tmp)); ARY_SET_HEAP_LEN(ary, len); @@ -3100,7 +3305,7 @@ rb_ary_delete_at(VALUE ary, long pos) MEMMOVE(ptr+pos, ptr+pos+1, VALUE, len-pos-1); }); ARY_INCREASE_LEN(ary, -1); - + ary_verify(ary); return del; } @@ -3204,6 +3409,7 @@ ary_reject(VALUE orig, VALUE result) for (i = 0; i < RARRAY_LEN(orig); i++) { VALUE v = RARRAY_AREF(orig, i); + if (!RTEST(rb_yield(v))) { rb_ary_push(result, v); } @@ -3485,7 +3691,7 @@ rb_ary_replace(VALUE copy, VALUE orig) VALUE shared = 0; if (ARY_OWNS_HEAP_P(copy)) { - RARRAY_PTR_USE(copy, ptr, ruby_sized_xfree(ptr, ARY_HEAP_SIZE(copy))); + ary_heap_free(copy); } else if (ARY_SHARED_P(copy)) { shared = ARY_SHARED(copy); @@ -3501,7 +3707,7 @@ rb_ary_replace(VALUE copy, VALUE orig) else { VALUE shared = ary_make_shared(orig); if (ARY_OWNS_HEAP_P(copy)) { - RARRAY_PTR_USE(copy, ptr, ruby_sized_xfree(ptr, ARY_HEAP_SIZE(copy))); + ary_heap_free(copy); } else { rb_ary_unshare_safe(copy); @@ -3528,16 +3734,19 @@ VALUE rb_ary_clear(VALUE ary) { rb_ary_modify_check(ary); - ARY_SET_LEN(ary, 0); if (ARY_SHARED_P(ary)) { if (!ARY_EMBED_P(ary)) { rb_ary_unshare(ary); FL_SET_EMBED(ary); + ARY_SET_EMBED_LEN(ary, 0); } } - else if (ARY_DEFAULT_SIZE * 2 < ARY_CAPA(ary)) { + else { + ARY_SET_LEN(ary, 0); + if (ARY_DEFAULT_SIZE * 2 < ARY_CAPA(ary)) { ary_resize_capa(ary, ARY_DEFAULT_SIZE * 2); } + } return ary; } @@ -6248,7 +6457,7 @@ Init_Array(void) rb_define_singleton_method(rb_cArray, "[]", rb_ary_s_create, -1); rb_define_singleton_method(rb_cArray, "try_convert", rb_ary_s_try_convert, 1); rb_define_method(rb_cArray, "initialize", rb_ary_initialize, -1); - rb_define_method(rb_cArray, "initialize_copy", rb_ary_replace, 1); + rb_define_method(rb_cArray, "initialize_copy", ary_initialize_copy, 1); rb_define_method(rb_cArray, "inspect", rb_ary_inspect, 0); rb_define_alias(rb_cArray, "to_s", "inspect"); Index: common.mk =================================================================== --- common.mk (revision 63705) +++ common.mk (working copy) @@ -131,6 +131,7 @@ COMMONOBJS = array.$(OBJEXT) \ thread.$(OBJEXT) \ time.$(OBJEXT) \ transcode.$(OBJEXT) \ + transient_heap.$(OBJEXT) \ util.$(OBJEXT) \ variable.$(OBJEXT) \ version.$(OBJEXT) \ @@ -2827,6 +2828,7 @@ transcode.$(OBJEXT): {$(VPATH)}st.h transcode.$(OBJEXT): {$(VPATH)}subst.h transcode.$(OBJEXT): {$(VPATH)}transcode.c transcode.$(OBJEXT): {$(VPATH)}transcode_data.h +transient_heap.$(OBJEXT): {$(VPATH)}transient_heap.c util.$(OBJEXT): $(hdrdir)/ruby/ruby.h util.$(OBJEXT): $(top_srcdir)/include/ruby.h util.$(OBJEXT): {$(VPATH)}config.h Index: gc.c =================================================================== --- gc.c (revision 63705) +++ gc.c (working copy) @@ -837,6 +837,11 @@ int ruby_disable_gc = 0; void rb_iseq_mark(const rb_iseq_t *iseq); void rb_iseq_free(const rb_iseq_t *iseq); +void rb_transient_heap_start_marking(int full_marking); +void rb_transient_heap_finish_marking(void); +void rb_transient_heap_mark(VALUE obj, const void *ptr); +void rb_transient_heap_promote(VALUE obj); + void rb_gcdebug_print_obj_condition(VALUE obj); static void rb_objspace_call_finalizer(rb_objspace_t *objspace); @@ -1188,6 +1193,7 @@ RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(rb_obj { MARK_IN_BITMAP(&page->uncollectible_bits[0], obj); objspace->rgengc.old_objects++; + rb_transient_heap_promote(obj); #if RGENGC_PROFILE >= 2 objspace->profile.total_promoted_count++; @@ -4606,13 +4612,21 @@ gc_mark_children(rb_objspace_t *objspace case T_ARRAY: if (FL_TEST(obj, ELTS_SHARED)) { - gc_mark(objspace, any->as.array.as.heap.aux.shared); + VALUE root = any->as.array.as.heap.aux.shared; + gc_mark(objspace, root); } else { long i, len = RARRAY_LEN(obj); const VALUE *ptr = RARRAY_CONST_PTR(obj); for (i=0; i < len; i++) { - gc_mark(objspace, *ptr++); + gc_mark(objspace, ptr[i]); + } + + if (objspace->mark_func_data == NULL) { + if (!FL_TEST_RAW(obj, RARRAY_EMBED_FLAG) && + ARY_TRANSIENT_P(obj)) { + rb_transient_heap_mark(obj, ptr); + } } } break; @@ -5605,6 +5619,8 @@ gc_marks_finish(rb_objspace_t *objspace) #endif } + rb_transient_heap_finish_marking(); + gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_END_MARK, 0); return TRUE; @@ -6474,6 +6490,7 @@ gc_start(rb_objspace_t *objspace, int re objspace->profile.heap_used_at_gc_start = heap_allocated_pages; gc_prof_setup_new_record(objspace, reason); gc_reset_malloc_info(objspace); + rb_transient_heap_start_marking(do_full_mark); gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_START, 0 /* TODO: pass minor/immediate flag? */); GC_ASSERT(during_gc); @@ -9450,6 +9467,7 @@ rb_raw_obj_info(char *buff, const int bu #if USE_RGENGC const int age = RVALUE_FLAGS_AGE(RBASIC(obj)->flags); + if (is_pointer_to_heap(&rb_objspace, (void *)obj)) { snprintf(buff, buff_size, "%p [%d%s%s%s%s] %s", (void *)obj, age, C(RVALUE_UNCOLLECTIBLE_BITMAP(obj), "L"), @@ -9457,6 +9475,13 @@ rb_raw_obj_info(char *buff, const int bu C(RVALUE_MARKING_BITMAP(obj), "R"), C(RVALUE_WB_UNPROTECTED_BITMAP(obj), "U"), obj_type_name(obj)); + } + else { + /* fake */ + snprintf(buff, buff_size, "%p [%dXXXX] %s", + (void *)obj, age, + obj_type_name(obj)); + } #else snprintf(buff, buff_size, "%p [%s] %s", (void *)obj, @@ -9486,10 +9511,25 @@ rb_raw_obj_info(char *buff, const int bu UNEXPECTED_NODE(rb_raw_obj_info); break; case T_ARRAY: - snprintf(buff, buff_size, "%s [%s%s] len: %d", buff, + if (FL_TEST(obj, ELTS_SHARED)) { + snprintf(buff, buff_size, "%s shared -> %s", buff, + rb_obj_info(RARRAY(obj)->as.heap.aux.shared)); + } + else if (FL_TEST(obj, RARRAY_EMBED_FLAG)) { + snprintf(buff, buff_size, "%s [%s%s] len: %d (embed)", buff, C(ARY_EMBED_P(obj), "E"), C(ARY_SHARED_P(obj), "S"), (int)RARRAY_LEN(obj)); + } + else { + snprintf(buff, buff_size, "%s [%s%s%s] len: %d, capa:%d ptr:%p", buff, + C(ARY_EMBED_P(obj), "E"), + C(ARY_SHARED_P(obj), "S"), + C(ARY_TRANSIENT_P(obj), "T"), + (int)RARRAY_LEN(obj), + ARY_EMBED_P(obj) ? -1 : (int)RARRAY(obj)->as.heap.aux.capa, + RARRAY_CONST_PTR(obj)); + } break; case T_STRING: { snprintf(buff, buff_size, "%s %s", buff, RSTRING_PTR(obj)); Index: inits.c =================================================================== --- inits.c (revision 63705) +++ inits.c (working copy) @@ -16,6 +16,7 @@ void rb_call_inits(void) { + CALL(TransientHeap); CALL(Method); CALL(RandomSeedCore); CALL(sym); Index: internal.h =================================================================== --- internal.h (revision 63705) +++ internal.h (working copy) @@ -1073,6 +1073,9 @@ VALUE rb_gvar_defined(struct rb_global_e struct vtm; /* defined by timev.h */ /* array.c */ +#define RARRAY_TRANSIENT_FLAG FL_USER13 +#define ARY_TRANSIENT_P(ary) FL_TEST_RAW((ary), RARRAY_TRANSIENT_FLAG) + VALUE rb_ary_last(int, const VALUE *, VALUE); void rb_ary_set_len(VALUE, long); void rb_ary_delete_same(VALUE, VALUE); Index: transient_heap.c =================================================================== --- transient_heap.c (revision 0) +++ transient_heap.c (working copy) @@ -0,0 +1,702 @@ +#include "ruby/ruby.h" +#include "ruby/debug.h" +#include "vm_debug.h" +#include "gc.h" +#include "internal.h" +#include +#include +#include "ruby_assert.h" + +/* + * 1: enable assertions + * 2: enable verify + */ +#ifndef TRANSIENT_HEAP_CHECK_MODE +#define TRANSIENT_HEAP_CHECK_MODE 0 +#endif +#define TH_ASSERT(expr) RUBY_ASSERT_MESG_WHEN(TRANSIENT_HEAP_CHECK_MODE > 0, expr, #expr) + +/* + * 1: show events + * 2: show dump at events + * 3: show all operations + */ +#define TRANSIENT_HEAP_DEBUG 0 + +/* Provide blocks infinitely for debug. + * This mode generates blocks unlimitedly + * and prohibit access free'ed blocks to check invalid access. + */ +#define TRANSIENT_HEAP_INFINITE_BLOCK_MODE 0 + +enum transient_heap_status { + transient_heap_none, + transient_heap_marking, + transient_heap_escaping +}; + +struct transient_heap_block { + struct transient_heap_block_header { + int16_t size; /* sizeof(block) = TRANSIENT_HEAP_BLOCK_SIZE - sizeof(struct transient_heap_block_header) */ + int16_t index; + int16_t last_marked_index; + int16_t objects; + struct transient_heap_block *next_block; + } info; + char buff[]; +}; + +struct transient_heap { + struct transient_heap_block *using_blocks; + struct transient_heap_block *marked_blocks; + struct transient_heap_block *free_blocks; + int total_objects; + int total_marked_objects; + int total_blocks; + enum transient_heap_status status; + + VALUE *promoted_objects; + int promoted_objects_size; + int promoted_objects_index; +}; + +struct transient_alloc_header { + uint16_t magic; + uint16_t size; + int16_t next_marked_index; + int16_t dummy; + VALUE obj; +}; + +static struct transient_heap global_transient_heap; + + +#define TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE 1024 + + /* K M */ +#define TRANSIENT_HEAP_TOTAL_SIZE (1024 * 1024 * 16) +#define TRANSIENT_HEAP_BLOCK_SIZE (1024 * 32 ) /* int16_t */ +#define TRANSIENT_HEAP_ALLOC_MAX (1024 * 2 ) +#define TRANSIENT_HEAP_BLOCK_NUM (TRANSIENT_HEAP_TOTAL_SIZE / TRANSIENT_HEAP_BLOCK_SIZE) + +#define TRANSIENT_HEAP_ALLOC_MAGIC 0xfeab +#define TRANSIENT_HEAP_ALLOC_ALIGN RUBY_ALIGNOF(void *) + +#define TRANSIENT_HEAP_ALLOC_MARKING_LAST -1 +#define TRANSIENT_HEAP_ALLOC_MARKING_FREE -2 + +#define ROUND_UP(v, a) (((size_t)(v) + (a) - 1) & ~((a) - 1)) + +static void +transient_heap_block_dump(struct transient_heap* theap, struct transient_heap_block *block) +{ + int i=0, n=0; + struct transient_alloc_header *header = NULL; + + while (iinfo.index) { + header = (void *)&block->buff[i]; + fprintf(stderr, "%4d %8d %p size:%4d next:%4d %s\n", n, i, header, header->size, header->next_marked_index, rb_obj_info(header->obj)); + i += header->size; + n++; + } +} + +static void +transient_heap_blocks_dump(struct transient_heap* theap, struct transient_heap_block *block, const char *type_str) +{ + while (block) { + fprintf(stderr, "- transient_heap_dump: %s:%p index:%d objects:%d last_marked_index:%d next:%p\n", + type_str, block, block->info.index, block->info.objects, block->info.last_marked_index, block->info.next_block); + + transient_heap_block_dump(theap, block); + block = block->info.next_block; + } +} + +static void +transient_heap_dump(struct transient_heap* theap) +{ + fprintf(stderr, "transient_heap_dump objects:%d marked_objects:%d blocks:%d\n", theap->total_objects, theap->total_marked_objects, theap->total_blocks); + transient_heap_blocks_dump(theap, theap->using_blocks, "using_blocks"); + transient_heap_blocks_dump(theap, theap->marked_blocks, "marked_blocks"); + transient_heap_blocks_dump(theap, theap->free_blocks, "free_blocks"); +} + +void +rb_transient_heap_dump(void) +{ + transient_heap_dump(&global_transient_heap); +} + +#if TRANSIENT_HEAP_CHECK_MODE >= 2 +static int +transient_heap_block_verify(struct transient_heap *theap, struct transient_heap_block *block) +{ + int i=0, n=0; + struct transient_alloc_header *header; + + while (iinfo.index) { + header = (void *)&block->buff[i]; + TH_ASSERT(header->magic == TRANSIENT_HEAP_ALLOC_MAGIC); + n ++; + i += header->size; + } + TH_ASSERT(block->info.objects == n); + + return n; +} +#endif + +static void +transient_heap_verify(struct transient_heap *theap) +{ +#if TRANSIENT_HEAP_CHECK_MODE >= 2 + struct transient_heap_block *block; + int n=0; + + // using_blocks + block = theap->using_blocks; + while (block) { + n += transient_heap_block_verify(theap, block); + block = block->info.next_block; + } + + // marked_blocks + block = theap->marked_blocks; + while (block) { + n += transient_heap_block_verify(theap, block); + TH_ASSERT(block->info.index > 0); + block = block->info.next_block; + } + + TH_ASSERT(n == theap->total_objects); + TH_ASSERT(n >= theap->total_marked_objects); +#endif +} + +static struct transient_heap* +transient_heap_get(void) +{ + struct transient_heap* theap = &global_transient_heap; + transient_heap_verify(theap); + return theap; +} + +static void +reset_block(struct transient_heap_block *block) +{ + block->info.size = TRANSIENT_HEAP_BLOCK_SIZE - sizeof(struct transient_heap_block_header); + block->info.index = 0; + block->info.objects = 0; + block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST; + block->info.next_block = NULL; +} + +static void +connect_to_free_blocks(struct transient_heap *theap, struct transient_heap_block *block) +{ + block->info.next_block = theap->free_blocks; + theap->free_blocks = block; +} + +static void +connect_to_using_blocks(struct transient_heap *theap, struct transient_heap_block *block) +{ + block->info.next_block = theap->using_blocks; + theap->using_blocks = block; +} + +#if 0 +static void +connect_to_marked_blocks(struct transient_heap *theap, struct transient_heap_block *block) +{ + block->info.next_block = theap->marked_blocks; + theap->marked_blocks = block; +} +#endif + +static void +append_to_marked_blocks(struct transient_heap *theap, struct transient_heap_block *append_blocks) +{ + if (theap->marked_blocks) { + struct transient_heap_block *block = theap->marked_blocks, *last_block = NULL; + while (block) { + last_block = block; + block = block->info.next_block; + } + + TH_ASSERT(last_block->info.next_block == NULL); + last_block->info.next_block = append_blocks; + } + else { + theap->marked_blocks = append_blocks; + } +} + +static struct transient_heap_block * +transient_heap_block_alloc(struct transient_heap* theap) +{ + struct transient_heap_block *block; + block = mmap(NULL, TRANSIENT_HEAP_BLOCK_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0); + if (block == MAP_FAILED) rb_bug("transient_heap_block_alloc: err:%d\n", errno); + + reset_block(block); + + TH_ASSERT(((intptr_t)block->buff & (TRANSIENT_HEAP_ALLOC_ALIGN-1)) == 0); + + theap->total_blocks++; + // fprintf(stderr, "transient_heap_block_alloc: %d\n", theap->total_blocks); + return block; +} + +static struct transient_heap_block * +transient_heap_allocatable_block(struct transient_heap* theap) +{ + struct transient_heap_block *block; + +#if TRANSIENT_HEAP_INFINITE_BLOCK_MODE + block = transient_heap_block_alloc(theap); +#else + // get one block from free_blocks + block = theap->free_blocks; + if (block) { + theap->free_blocks = block->info.next_block; + block->info.next_block = NULL; + } +#endif + + return block; +} + +static struct transient_alloc_header * +transient_heap_allocatable_header(struct transient_heap* theap, size_t size) +{ + struct transient_heap_block *block = theap->using_blocks; + + while (block) { + TH_ASSERT(block->info.size >= block->info.index); + + if (block->info.size - block->info.index >= (int32_t)size) { + struct transient_alloc_header *header = (void *)&block->buff[block->info.index]; + block->info.index += size; + block->info.objects++; + return header; + } + else { + block = transient_heap_allocatable_block(theap); + if (block) connect_to_using_blocks(theap, block); + } + } + + return NULL; +} + +void * +rb_transient_heap_alloc(VALUE obj, size_t req_size) +{ + struct transient_heap* theap = transient_heap_get(); + size_t size = ROUND_UP(req_size + sizeof(struct transient_alloc_header), TRANSIENT_HEAP_ALLOC_ALIGN); + + if (size > TRANSIENT_HEAP_ALLOC_MAX) { + // fprintf(stderr, "rb_transient_heap_alloc: NULL (too big: %ld)\n", (long)size); + return NULL; + } + else if (RB_OBJ_PROMOTED_RAW(obj)) { + // fprintf(stderr, "rb_transient_heap_alloc: NULL (not for promoted objects)\n"); + return NULL; + } + else { + struct transient_alloc_header *header = transient_heap_allocatable_header(theap, size); + if (header) { + void *ptr; + + header->size = size; + header->magic = TRANSIENT_HEAP_ALLOC_MAGIC; + header->next_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_FREE; + header->obj = obj; // TODO: for verify + + // stat info + theap->total_objects++; + ptr = header + 1; + // fprintf(stderr, "rb_transient_heap_alloc: header:%p ptr:%p size:%d obj:%s\n", header, ptr, (int)size, rb_obj_info(obj)); + return ptr; + } + else { + // fprintf(stderr, "rb_transient_heap_alloc: NULL (no enough space: %ld)\n", (long)size); + return NULL; + } + } +} + +void +Init_TransientHeap(void) +{ + int i, block_num; + struct transient_heap* theap = transient_heap_get(); + +#if TRANSIENT_HEAP_INFINITE_BLOCK_MODE + block_num = 1; +#else + TH_ASSERT(TRANSIENT_HEAP_BLOCK_SIZE * TRANSIENT_HEAP_BLOCK_NUM == TRANSIENT_HEAP_TOTAL_SIZE); + block_num = TRANSIENT_HEAP_BLOCK_NUM; +#endif + for (i=0; iusing_blocks = transient_heap_block_alloc(theap); + + theap->promoted_objects_size = TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE; + theap->promoted_objects_index = 0; + /* should not use ALLOC_N to be free from GC */ + theap->promoted_objects = malloc(sizeof(VALUE) * theap->promoted_objects_size); + if (theap->promoted_objects == NULL) rb_bug("Init_TransientHeap: malloc failed."); +} + +static struct transient_heap_block * +blocks_alloc_header_to_block(struct transient_heap *theap, struct transient_heap_block *blocks, struct transient_alloc_header *header) +{ + struct transient_heap_block *block = blocks; + + TH_ASSERT(theap->status == transient_heap_marking); + + while (block) { + if (block->buff <= (char *)header && (char *)header < block->buff + block->info.size) { + return block; + } + block = block->info.next_block; + } + + return NULL; +} + +static struct transient_heap_block * +alloc_header_to_block(struct transient_heap *theap, struct transient_alloc_header *header) +{ + struct transient_heap_block *block; + + if ((block = blocks_alloc_header_to_block(theap, theap->marked_blocks, header)) != NULL) { + if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "alloc_header_to_block: found in marked_blocks\n"); + return block; + } + else if ((block = blocks_alloc_header_to_block(theap, theap->using_blocks, header)) != NULL) { + if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "alloc_header_to_block: found in using_blocks\n"); + return block; + } + else { + transient_heap_dump(theap); + rb_bug("alloc_header_to_block: not found in mark_blocks (%p)\n", header); + } +} + +void +rb_transient_heap_mark(VALUE obj, const void *ptr) +{ + struct transient_alloc_header *header = (void *)ptr; + header = header - 1; + + if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_mark: %s (%p)\n", rb_obj_info(obj), ptr); + + if (header->next_marked_index != TRANSIENT_HEAP_ALLOC_MARKING_FREE) { + // already marked + return; + } + + if (header->magic != TRANSIENT_HEAP_ALLOC_MAGIC) { + struct transient_heap* theap = transient_heap_get(); + transient_heap_dump(theap); + rb_bug("rb_transient_heap_mark: magic is broken"); + } + else if (header->obj != obj) { + struct transient_heap* theap = transient_heap_get(); + transient_heap_dump(theap); + rb_bug("rb_transient_heap_mark: unmatch\n"); + } + else { + struct transient_heap* theap = transient_heap_get(); + struct transient_heap_block *block = alloc_header_to_block(theap, header); + header->next_marked_index = block->info.last_marked_index; + block->info.last_marked_index = (int)((char *)header - block->buff); + theap->total_marked_objects++; + } +} + +static void * +transient_heap_ptr(VALUE obj, int error) +{ + void *ptr; + + switch (BUILTIN_TYPE(obj)) { + case T_ARRAY: + if (ARY_TRANSIENT_P(obj)) { + ptr = (VALUE *)RARRAY_CONST_PTR(obj); + } + else { + ptr = NULL; + } + break; + default: + if (error) { + rb_bug("transient_heap_ptr: unknown obj %s\n", rb_obj_info(obj)); + } + else { + ptr = NULL; + } + } + + return ptr; +} + +void +rb_transient_heap_promote(VALUE obj) +{ + + if (transient_heap_ptr(obj, FALSE)) { + struct transient_heap* theap = transient_heap_get(); + + if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_promote: %s\n", rb_obj_info(obj)); + + if (theap->promoted_objects_size <= theap->promoted_objects_index) { + theap->promoted_objects_size *= 2; + if (TRANSIENT_HEAP_DEBUG >= 0) fprintf(stderr, "rb_transient_heap_promote: expand table to %d\n", theap->promoted_objects_size); + theap->promoted_objects = realloc(theap->promoted_objects, theap->promoted_objects_size * sizeof(VALUE)); + if (theap->promoted_objects == NULL) rb_bug("rb_transient_heap_promote: realloc failed"); + } + theap->promoted_objects[theap->promoted_objects_index++] = obj; + } + else { + /* ignore */ + } +} + +void +rb_transient_heap_promoted(VALUE obj, const void *ptr) +{ + struct transient_alloc_header *header = (void *)ptr; + header = header - 1; + + +} + +static struct transient_alloc_header * +alloc_header(struct transient_heap_block* block, int index) +{ + return (void *)&block->buff[index]; +} + +void rb_ary_detransient(VALUE ary); + +static void +transient_heap_reset(void) +{ + struct transient_heap* theap = transient_heap_get(); + struct transient_heap_block* block; + + if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_reset\n"); + + block = theap->marked_blocks; + while (block) { + struct transient_heap_block *next_block = block->info.next_block; + theap->total_objects -= block->info.objects; +#if TRANSIENT_HEAP_INFINITE_BLOCK_MODE + // debug mode + if (madvise(block, TRANSIENT_HEAP_BLOCK_SIZE, MADV_DONTNEED) != 0) { + rb_bug("madvise err:%d", errno); + } + if (mprotect(block, TRANSIENT_HEAP_BLOCK_SIZE, PROT_NONE) != 0) { + rb_bug("mprotect err:%d", errno); + } + theap->total_blocks--; +#else + reset_block(block); + connect_to_free_blocks(theap, block); +#endif + block = next_block; + } + + theap->marked_blocks = NULL; + theap->total_marked_objects = 0; +} + +static void +transient_heap_block_escape(struct transient_heap* theap, struct transient_heap_block* block) +{ + int marked_index = block->info.last_marked_index; + block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST; + + while (marked_index >= 0) { + struct transient_alloc_header *header = alloc_header(block, marked_index); + VALUE obj = header->obj; + + if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, " * transient_heap_block_escape %p %s\n", header, rb_obj_info(obj)); + + if (obj != Qnil) { + switch (BUILTIN_TYPE(obj)) { + case T_ARRAY: + rb_ary_detransient(obj); + break; + default: + rb_bug("unsupporeted"); + } + header->obj = Qundef; // to verify + } + marked_index = header->next_marked_index; + } +} + +static void +transient_heap_update_status(struct transient_heap* theap, enum transient_heap_status status) +{ + TH_ASSERT(theap->status != status); + theap->status = status; +} + +static void +transient_heap_escape(void *dmy) +{ + struct transient_heap* theap = transient_heap_get(); + + if (theap->status == transient_heap_marking) { + if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_escape: skip while transient_heap_marking\n"); + } + else { + VALUE gc_disabled = rb_gc_disable(); + struct transient_heap_block* block; + + if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_escape start total_blocks:%d\n", theap->total_blocks); + if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap); + + TH_ASSERT(theap->status == transient_heap_none); + transient_heap_update_status(theap, transient_heap_escaping); + + // escape marked blocks + block = theap->marked_blocks; + while (block) { + transient_heap_block_escape(theap, block); + block = block->info.next_block; + } + + // escape using blocks + // only affect incremental marking + block = theap->using_blocks; + while (block) { + transient_heap_block_escape(theap, block); + block = block->info.next_block; + } + + // all objects in marked_objects are escaped. + transient_heap_reset(); + + if (TRANSIENT_HEAP_DEBUG > 0) { + fprintf(stderr, "!! transient_heap_escape end total_blocks:%d\n", theap->total_blocks); + // transient_heap_dump(theap); + } + + transient_heap_verify(theap); + transient_heap_update_status(theap, transient_heap_none); + if (gc_disabled != Qtrue) rb_gc_enable(); + } +} + +static void +clear_marked_index(struct transient_heap_block* block) +{ + int marked_index = block->info.last_marked_index; + + while (marked_index != TRANSIENT_HEAP_ALLOC_MARKING_LAST) { + struct transient_alloc_header *header = alloc_header(block, marked_index); + TH_ASSERT(marked_index != TRANSIENT_HEAP_ALLOC_MARKING_FREE); + // fprintf(stderr, "clear_marked_index - block:%p mark_index:%d\n", block, marked_index); + + marked_index = header->next_marked_index; + header->next_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_FREE; + } + + block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST; +} + +void +rb_transient_heap_start_marking(int full_marking) +{ + struct transient_heap* theap = transient_heap_get(); + struct transient_heap_block* block; + + if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! rb_transient_heap_start_marking objects:%d blocks:%d full_marking:%d\n", + theap->total_objects, theap->total_blocks, full_marking); + if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap); + + // clear marking info + block = theap->marked_blocks; + while (block) { + clear_marked_index(block); + block = block->info.next_block; + } + + block = theap->using_blocks; + while (block) { + clear_marked_index(block); + block = block->info.next_block; + } + + if (theap->using_blocks) { + if (theap->using_blocks->info.objects > 0) { + append_to_marked_blocks(theap, theap->using_blocks); + theap->using_blocks = NULL; + } + else { + append_to_marked_blocks(theap, theap->using_blocks->info.next_block); + theap->using_blocks->info.next_block = NULL; + } + } + + if (theap->using_blocks == NULL) { + theap->using_blocks = transient_heap_allocatable_block(theap); + } + + TH_ASSERT(theap->status == transient_heap_none); + transient_heap_update_status(theap, transient_heap_marking); + theap->total_marked_objects = 0; + + if (full_marking) { + theap->promoted_objects_index = 0; + } + else { /* mark promoted objects */ + int i; + for (i=0; ipromoted_objects_index; i++) { + VALUE obj = theap->promoted_objects[i]; + void *ptr = transient_heap_ptr(obj, TRUE); + if (ptr) { + rb_transient_heap_mark(obj, ptr); + } + } + } + + transient_heap_verify(theap); +} + +void +rb_transient_heap_finish_marking(void) +{ + struct transient_heap* theap = transient_heap_get(); + + if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! rb_transient_heap_finish_marking objects:%d, marked:%d\n", + theap->total_objects, + theap->total_marked_objects); + if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap); + + TH_ASSERT(theap->total_objects >= theap->total_marked_objects); + + TH_ASSERT(theap->status == transient_heap_marking); + transient_heap_update_status(theap, transient_heap_none); + + if (theap->total_marked_objects > 0) { + if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "-> rb_transient_heap_finish_marking register escape func.\n"); + rb_postponed_job_register_one(0, transient_heap_escape, NULL); + } + else { + transient_heap_reset(); + } + + transient_heap_verify(theap); +} Property changes on: transient_heap.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +LF \ No newline at end of property Added: svn:executable ## -0,0 +1 ## +* \ No newline at end of property