From a6bbe1548ae8fc8be5a82f112aae1afd86f0104c Mon Sep 17 00:00:00 2001 From: Jemma Issroff Date: Tue, 16 Aug 2022 16:13:21 -0400 Subject: [PATCH] This is a squashed commit of all of our work on object shapes. Please see Ruby [Feature #18776] --- common.mk | 282 +++++++++ compile.c | 25 +- debug_counter.h | 10 +- ext/coverage/depend | 3 + ext/objspace/depend | 3 + ext/objspace/objspace.c | 1 + gc.c | 233 ++++--- id_table.c | 2 +- id_table.h | 2 - include/ruby/internal/core/robject.h | 26 +- include/ruby/internal/fl_type.h | 19 +- inits.c | 1 + internal.h | 3 - internal/class.h | 15 +- internal/imemo.h | 2 + internal/object.h | 22 +- internal/variable.h | 6 + iseq.c | 16 +- marshal.c | 10 +- misc/lldb_cruby.py | 6 +- mjit_compile.c | 12 +- object.c | 48 +- shape.c | 579 ++++++++++++++++++ shape.h | 54 ++ .../reachable_objects_from_spec.rb | 2 +- test/-ext-/marshal/test_internal_ivar.rb | 1 + test/objspace/test_objspace.rb | 19 +- test/ruby/test_mjit.rb | 4 +- test/ruby/test_shapes.rb | 139 +++++ tool/ruby_vm/views/_mjit_compile_ivar.erb | 39 +- variable.c | 550 ++++++++++------- variable.h | 1 + vm.c | 37 ++ vm_callinfo.h | 130 +++- vm_core.h | 19 +- vm_insnhelper.c | 517 +++++++++++----- yjit/bindgen/src/main.rs | 1 + yjit/src/asm/x86_64/mod.rs | 2 +- yjit/src/codegen.rs | 80 ++- yjit/src/cruby.rs | 6 +- yjit/src/cruby_bindings.inc.rs | 34 +- 41 files changed, 2352 insertions(+), 609 deletions(-) create mode 100644 shape.c create mode 100644 shape.h create mode 100644 test/ruby/test_shapes.rb diff --git a/common.mk b/common.mk index cf08764bc9..1412c7f8d6 100644 --- a/common.mk +++ b/common.mk @@ -134,6 +134,7 @@ COMMONOBJS = array.$(OBJEXT) \ regsyntax.$(OBJEXT) \ ruby.$(OBJEXT) \ scheduler.$(OBJEXT) \ + shape.$(OBJEXT) \ signal.$(OBJEXT) \ sprintf.$(OBJEXT) \ st.$(OBJEXT) \ @@ -1773,6 +1774,7 @@ array.$(OBJEXT): $(top_srcdir)/internal/proc.h array.$(OBJEXT): $(top_srcdir)/internal/rational.h array.$(OBJEXT): $(top_srcdir)/internal/serial.h array.$(OBJEXT): $(top_srcdir)/internal/static_assert.h +array.$(OBJEXT): $(top_srcdir)/internal/variable.h array.$(OBJEXT): $(top_srcdir)/internal/vm.h array.$(OBJEXT): $(top_srcdir)/internal/warnings.h array.$(OBJEXT): {$(VPATH)}array.c @@ -1789,6 +1791,7 @@ array.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h array.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h array.$(OBJEXT): {$(VPATH)}builtin.h array.$(OBJEXT): {$(VPATH)}config.h +array.$(OBJEXT): {$(VPATH)}constant.h array.$(OBJEXT): {$(VPATH)}debug_counter.h array.$(OBJEXT): {$(VPATH)}defines.h array.$(OBJEXT): {$(VPATH)}encoding.h @@ -1969,6 +1972,7 @@ ast.$(OBJEXT): $(top_srcdir)/internal/parse.h ast.$(OBJEXT): $(top_srcdir)/internal/serial.h ast.$(OBJEXT): $(top_srcdir)/internal/static_assert.h ast.$(OBJEXT): $(top_srcdir)/internal/symbol.h +ast.$(OBJEXT): $(top_srcdir)/internal/variable.h ast.$(OBJEXT): $(top_srcdir)/internal/vm.h ast.$(OBJEXT): $(top_srcdir)/internal/warnings.h ast.$(OBJEXT): {$(VPATH)}assert.h @@ -1986,9 +1990,11 @@ ast.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h ast.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h ast.$(OBJEXT): {$(VPATH)}builtin.h ast.$(OBJEXT): {$(VPATH)}config.h +ast.$(OBJEXT): {$(VPATH)}constant.h ast.$(OBJEXT): {$(VPATH)}defines.h ast.$(OBJEXT): {$(VPATH)}encoding.h ast.$(OBJEXT): {$(VPATH)}id.h +ast.$(OBJEXT): {$(VPATH)}id_table.h ast.$(OBJEXT): {$(VPATH)}intern.h ast.$(OBJEXT): {$(VPATH)}internal.h ast.$(OBJEXT): {$(VPATH)}internal/abi.h @@ -2346,6 +2352,7 @@ builtin.$(OBJEXT): $(top_srcdir)/internal/gc.h builtin.$(OBJEXT): $(top_srcdir)/internal/imemo.h builtin.$(OBJEXT): $(top_srcdir)/internal/serial.h builtin.$(OBJEXT): $(top_srcdir)/internal/static_assert.h +builtin.$(OBJEXT): $(top_srcdir)/internal/variable.h builtin.$(OBJEXT): $(top_srcdir)/internal/vm.h builtin.$(OBJEXT): $(top_srcdir)/internal/warnings.h builtin.$(OBJEXT): {$(VPATH)}assert.h @@ -2363,8 +2370,10 @@ builtin.$(OBJEXT): {$(VPATH)}builtin.c builtin.$(OBJEXT): {$(VPATH)}builtin.h builtin.$(OBJEXT): {$(VPATH)}builtin_binary.inc builtin.$(OBJEXT): {$(VPATH)}config.h +builtin.$(OBJEXT): {$(VPATH)}constant.h builtin.$(OBJEXT): {$(VPATH)}defines.h builtin.$(OBJEXT): {$(VPATH)}id.h +builtin.$(OBJEXT): {$(VPATH)}id_table.h builtin.$(OBJEXT): {$(VPATH)}intern.h builtin.$(OBJEXT): {$(VPATH)}internal.h builtin.$(OBJEXT): {$(VPATH)}internal/abi.h @@ -3118,6 +3127,7 @@ compile.$(OBJEXT): {$(VPATH)}re.h compile.$(OBJEXT): {$(VPATH)}regex.h compile.$(OBJEXT): {$(VPATH)}ruby_assert.h compile.$(OBJEXT): {$(VPATH)}ruby_atomic.h +compile.$(OBJEXT): {$(VPATH)}shape.h compile.$(OBJEXT): {$(VPATH)}st.h compile.$(OBJEXT): {$(VPATH)}subst.h compile.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h @@ -3142,6 +3152,7 @@ complex.$(OBJEXT): $(top_srcdir)/internal/object.h complex.$(OBJEXT): $(top_srcdir)/internal/rational.h complex.$(OBJEXT): $(top_srcdir)/internal/serial.h complex.$(OBJEXT): $(top_srcdir)/internal/static_assert.h +complex.$(OBJEXT): $(top_srcdir)/internal/variable.h complex.$(OBJEXT): $(top_srcdir)/internal/vm.h complex.$(OBJEXT): $(top_srcdir)/internal/warnings.h complex.$(OBJEXT): {$(VPATH)}assert.h @@ -3156,6 +3167,7 @@ complex.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h complex.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h complex.$(OBJEXT): {$(VPATH)}complex.c complex.$(OBJEXT): {$(VPATH)}config.h +complex.$(OBJEXT): {$(VPATH)}constant.h complex.$(OBJEXT): {$(VPATH)}defines.h complex.$(OBJEXT): {$(VPATH)}id.h complex.$(OBJEXT): {$(VPATH)}id_table.h @@ -3319,6 +3331,7 @@ cont.$(OBJEXT): $(top_srcdir)/internal/imemo.h cont.$(OBJEXT): $(top_srcdir)/internal/proc.h cont.$(OBJEXT): $(top_srcdir)/internal/serial.h cont.$(OBJEXT): $(top_srcdir)/internal/static_assert.h +cont.$(OBJEXT): $(top_srcdir)/internal/variable.h cont.$(OBJEXT): $(top_srcdir)/internal/vm.h cont.$(OBJEXT): $(top_srcdir)/internal/warnings.h cont.$(OBJEXT): {$(VPATH)}$(COROUTINE_H) @@ -3334,6 +3347,7 @@ cont.$(OBJEXT): {$(VPATH)}backward/2/long_long.h cont.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h cont.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h cont.$(OBJEXT): {$(VPATH)}config.h +cont.$(OBJEXT): {$(VPATH)}constant.h cont.$(OBJEXT): {$(VPATH)}cont.c cont.$(OBJEXT): {$(VPATH)}debug_counter.h cont.$(OBJEXT): {$(VPATH)}defines.h @@ -3513,6 +3527,7 @@ debug.$(OBJEXT): $(top_srcdir)/internal/imemo.h debug.$(OBJEXT): $(top_srcdir)/internal/serial.h debug.$(OBJEXT): $(top_srcdir)/internal/signal.h debug.$(OBJEXT): $(top_srcdir)/internal/static_assert.h +debug.$(OBJEXT): $(top_srcdir)/internal/variable.h debug.$(OBJEXT): $(top_srcdir)/internal/vm.h debug.$(OBJEXT): $(top_srcdir)/internal/warnings.h debug.$(OBJEXT): {$(VPATH)}assert.h @@ -3527,6 +3542,7 @@ debug.$(OBJEXT): {$(VPATH)}backward/2/long_long.h debug.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h debug.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h debug.$(OBJEXT): {$(VPATH)}config.h +debug.$(OBJEXT): {$(VPATH)}constant.h debug.$(OBJEXT): {$(VPATH)}debug.c debug.$(OBJEXT): {$(VPATH)}debug_counter.h debug.$(OBJEXT): {$(VPATH)}defines.h @@ -3697,6 +3713,7 @@ debug.$(OBJEXT): {$(VPATH)}ractor.h debug.$(OBJEXT): {$(VPATH)}ractor_core.h debug.$(OBJEXT): {$(VPATH)}ruby_assert.h debug.$(OBJEXT): {$(VPATH)}ruby_atomic.h +debug.$(OBJEXT): {$(VPATH)}shape.h debug.$(OBJEXT): {$(VPATH)}st.h debug.$(OBJEXT): {$(VPATH)}subst.h debug.$(OBJEXT): {$(VPATH)}symbol.h @@ -3881,6 +3898,7 @@ dir.$(OBJEXT): $(top_srcdir)/internal/object.h dir.$(OBJEXT): $(top_srcdir)/internal/serial.h dir.$(OBJEXT): $(top_srcdir)/internal/static_assert.h dir.$(OBJEXT): $(top_srcdir)/internal/string.h +dir.$(OBJEXT): $(top_srcdir)/internal/variable.h dir.$(OBJEXT): $(top_srcdir)/internal/vm.h dir.$(OBJEXT): $(top_srcdir)/internal/warnings.h dir.$(OBJEXT): {$(VPATH)}assert.h @@ -3895,6 +3913,7 @@ dir.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h dir.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h dir.$(OBJEXT): {$(VPATH)}builtin.h dir.$(OBJEXT): {$(VPATH)}config.h +dir.$(OBJEXT): {$(VPATH)}constant.h dir.$(OBJEXT): {$(VPATH)}defines.h dir.$(OBJEXT): {$(VPATH)}dir.c dir.$(OBJEXT): {$(VPATH)}dir.rbinc @@ -5381,6 +5400,7 @@ encoding.$(OBJEXT): $(top_srcdir)/internal/object.h encoding.$(OBJEXT): $(top_srcdir)/internal/serial.h encoding.$(OBJEXT): $(top_srcdir)/internal/static_assert.h encoding.$(OBJEXT): $(top_srcdir)/internal/string.h +encoding.$(OBJEXT): $(top_srcdir)/internal/variable.h encoding.$(OBJEXT): $(top_srcdir)/internal/vm.h encoding.$(OBJEXT): $(top_srcdir)/internal/warnings.h encoding.$(OBJEXT): {$(VPATH)}assert.h @@ -5394,6 +5414,7 @@ encoding.$(OBJEXT): {$(VPATH)}backward/2/long_long.h encoding.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h encoding.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h encoding.$(OBJEXT): {$(VPATH)}config.h +encoding.$(OBJEXT): {$(VPATH)}constant.h encoding.$(OBJEXT): {$(VPATH)}debug_counter.h encoding.$(OBJEXT): {$(VPATH)}defines.h encoding.$(OBJEXT): {$(VPATH)}encindex.h @@ -5580,6 +5601,7 @@ enum.$(OBJEXT): $(top_srcdir)/internal/rational.h enum.$(OBJEXT): $(top_srcdir)/internal/re.h enum.$(OBJEXT): $(top_srcdir)/internal/serial.h enum.$(OBJEXT): $(top_srcdir)/internal/static_assert.h +enum.$(OBJEXT): $(top_srcdir)/internal/variable.h enum.$(OBJEXT): $(top_srcdir)/internal/vm.h enum.$(OBJEXT): $(top_srcdir)/internal/warnings.h enum.$(OBJEXT): {$(VPATH)}assert.h @@ -5593,6 +5615,7 @@ enum.$(OBJEXT): {$(VPATH)}backward/2/long_long.h enum.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h enum.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h enum.$(OBJEXT): {$(VPATH)}config.h +enum.$(OBJEXT): {$(VPATH)}constant.h enum.$(OBJEXT): {$(VPATH)}defines.h enum.$(OBJEXT): {$(VPATH)}encoding.h enum.$(OBJEXT): {$(VPATH)}enum.c @@ -6414,6 +6437,7 @@ file.$(OBJEXT): $(top_srcdir)/internal/serial.h file.$(OBJEXT): $(top_srcdir)/internal/static_assert.h file.$(OBJEXT): $(top_srcdir)/internal/string.h file.$(OBJEXT): $(top_srcdir)/internal/thread.h +file.$(OBJEXT): $(top_srcdir)/internal/variable.h file.$(OBJEXT): $(top_srcdir)/internal/vm.h file.$(OBJEXT): $(top_srcdir)/internal/warnings.h file.$(OBJEXT): {$(VPATH)}assert.h @@ -6427,6 +6451,7 @@ file.$(OBJEXT): {$(VPATH)}backward/2/long_long.h file.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h file.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h file.$(OBJEXT): {$(VPATH)}config.h +file.$(OBJEXT): {$(VPATH)}constant.h file.$(OBJEXT): {$(VPATH)}defines.h file.$(OBJEXT): {$(VPATH)}dln.h file.$(OBJEXT): {$(VPATH)}encindex.h @@ -6821,6 +6846,7 @@ gc.$(OBJEXT): {$(VPATH)}regex.h gc.$(OBJEXT): {$(VPATH)}regint.h gc.$(OBJEXT): {$(VPATH)}ruby_assert.h gc.$(OBJEXT): {$(VPATH)}ruby_atomic.h +gc.$(OBJEXT): {$(VPATH)}shape.h gc.$(OBJEXT): {$(VPATH)}st.h gc.$(OBJEXT): {$(VPATH)}subst.h gc.$(OBJEXT): {$(VPATH)}symbol.h @@ -6847,6 +6873,7 @@ goruby.$(OBJEXT): $(top_srcdir)/internal/gc.h goruby.$(OBJEXT): $(top_srcdir)/internal/imemo.h goruby.$(OBJEXT): $(top_srcdir)/internal/serial.h goruby.$(OBJEXT): $(top_srcdir)/internal/static_assert.h +goruby.$(OBJEXT): $(top_srcdir)/internal/variable.h goruby.$(OBJEXT): $(top_srcdir)/internal/vm.h goruby.$(OBJEXT): $(top_srcdir)/internal/warnings.h goruby.$(OBJEXT): {$(VPATH)}assert.h @@ -6862,11 +6889,13 @@ goruby.$(OBJEXT): {$(VPATH)}backward/2/long_long.h goruby.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h goruby.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h goruby.$(OBJEXT): {$(VPATH)}config.h +goruby.$(OBJEXT): {$(VPATH)}constant.h goruby.$(OBJEXT): {$(VPATH)}defines.h goruby.$(OBJEXT): {$(VPATH)}golf_prelude.c goruby.$(OBJEXT): {$(VPATH)}golf_prelude.rb goruby.$(OBJEXT): {$(VPATH)}goruby.c goruby.$(OBJEXT): {$(VPATH)}id.h +goruby.$(OBJEXT): {$(VPATH)}id_table.h goruby.$(OBJEXT): {$(VPATH)}intern.h goruby.$(OBJEXT): {$(VPATH)}internal.h goruby.$(OBJEXT): {$(VPATH)}internal/abi.h @@ -7041,6 +7070,7 @@ hash.$(OBJEXT): $(top_srcdir)/internal/string.h hash.$(OBJEXT): $(top_srcdir)/internal/symbol.h hash.$(OBJEXT): $(top_srcdir)/internal/thread.h hash.$(OBJEXT): $(top_srcdir)/internal/time.h +hash.$(OBJEXT): $(top_srcdir)/internal/variable.h hash.$(OBJEXT): $(top_srcdir)/internal/vm.h hash.$(OBJEXT): $(top_srcdir)/internal/warnings.h hash.$(OBJEXT): {$(VPATH)}assert.h @@ -7054,6 +7084,7 @@ hash.$(OBJEXT): {$(VPATH)}backward/2/long_long.h hash.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h hash.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h hash.$(OBJEXT): {$(VPATH)}config.h +hash.$(OBJEXT): {$(VPATH)}constant.h hash.$(OBJEXT): {$(VPATH)}debug_counter.h hash.$(OBJEXT): {$(VPATH)}defines.h hash.$(OBJEXT): {$(VPATH)}encoding.h @@ -8002,6 +8033,7 @@ iseq.$(OBJEXT): {$(VPATH)}oniguruma.h iseq.$(OBJEXT): {$(VPATH)}ractor.h iseq.$(OBJEXT): {$(VPATH)}ruby_assert.h iseq.$(OBJEXT): {$(VPATH)}ruby_atomic.h +iseq.$(OBJEXT): {$(VPATH)}shape.h iseq.$(OBJEXT): {$(VPATH)}st.h iseq.$(OBJEXT): {$(VPATH)}subst.h iseq.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h @@ -8731,6 +8763,7 @@ marshal.$(OBJEXT): $(top_srcdir)/internal/string.h marshal.$(OBJEXT): $(top_srcdir)/internal/struct.h marshal.$(OBJEXT): $(top_srcdir)/internal/symbol.h marshal.$(OBJEXT): $(top_srcdir)/internal/util.h +marshal.$(OBJEXT): $(top_srcdir)/internal/variable.h marshal.$(OBJEXT): $(top_srcdir)/internal/vm.h marshal.$(OBJEXT): $(top_srcdir)/internal/warnings.h marshal.$(OBJEXT): {$(VPATH)}assert.h @@ -8745,6 +8778,7 @@ marshal.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h marshal.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h marshal.$(OBJEXT): {$(VPATH)}builtin.h marshal.$(OBJEXT): {$(VPATH)}config.h +marshal.$(OBJEXT): {$(VPATH)}constant.h marshal.$(OBJEXT): {$(VPATH)}defines.h marshal.$(OBJEXT): {$(VPATH)}encindex.h marshal.$(OBJEXT): {$(VPATH)}encoding.h @@ -8906,6 +8940,7 @@ marshal.$(OBJEXT): {$(VPATH)}marshal.rbinc marshal.$(OBJEXT): {$(VPATH)}missing.h marshal.$(OBJEXT): {$(VPATH)}onigmo.h marshal.$(OBJEXT): {$(VPATH)}oniguruma.h +marshal.$(OBJEXT): {$(VPATH)}shape.h marshal.$(OBJEXT): {$(VPATH)}st.h marshal.$(OBJEXT): {$(VPATH)}subst.h marshal.$(OBJEXT): {$(VPATH)}util.h @@ -8919,6 +8954,7 @@ math.$(OBJEXT): $(top_srcdir)/internal/math.h math.$(OBJEXT): $(top_srcdir)/internal/object.h math.$(OBJEXT): $(top_srcdir)/internal/serial.h math.$(OBJEXT): $(top_srcdir)/internal/static_assert.h +math.$(OBJEXT): $(top_srcdir)/internal/variable.h math.$(OBJEXT): $(top_srcdir)/internal/vm.h math.$(OBJEXT): $(top_srcdir)/internal/warnings.h math.$(OBJEXT): {$(VPATH)}assert.h @@ -8932,6 +8968,7 @@ math.$(OBJEXT): {$(VPATH)}backward/2/long_long.h math.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h math.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h math.$(OBJEXT): {$(VPATH)}config.h +math.$(OBJEXT): {$(VPATH)}constant.h math.$(OBJEXT): {$(VPATH)}defines.h math.$(OBJEXT): {$(VPATH)}id_table.h math.$(OBJEXT): {$(VPATH)}intern.h @@ -9258,6 +9295,7 @@ miniinit.$(OBJEXT): $(top_srcdir)/internal/gc.h miniinit.$(OBJEXT): $(top_srcdir)/internal/imemo.h miniinit.$(OBJEXT): $(top_srcdir)/internal/serial.h miniinit.$(OBJEXT): $(top_srcdir)/internal/static_assert.h +miniinit.$(OBJEXT): $(top_srcdir)/internal/variable.h miniinit.$(OBJEXT): $(top_srcdir)/internal/vm.h miniinit.$(OBJEXT): $(top_srcdir)/internal/warnings.h miniinit.$(OBJEXT): {$(VPATH)}array.rb @@ -9275,12 +9313,14 @@ miniinit.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h miniinit.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h miniinit.$(OBJEXT): {$(VPATH)}builtin.h miniinit.$(OBJEXT): {$(VPATH)}config.h +miniinit.$(OBJEXT): {$(VPATH)}constant.h miniinit.$(OBJEXT): {$(VPATH)}defines.h miniinit.$(OBJEXT): {$(VPATH)}dir.rb miniinit.$(OBJEXT): {$(VPATH)}encoding.h miniinit.$(OBJEXT): {$(VPATH)}gc.rb miniinit.$(OBJEXT): {$(VPATH)}gem_prelude.rb miniinit.$(OBJEXT): {$(VPATH)}id.h +miniinit.$(OBJEXT): {$(VPATH)}id_table.h miniinit.$(OBJEXT): {$(VPATH)}intern.h miniinit.$(OBJEXT): {$(VPATH)}internal.h miniinit.$(OBJEXT): {$(VPATH)}internal/abi.h @@ -9482,6 +9522,7 @@ mjit.$(OBJEXT): $(top_srcdir)/internal/hash.h mjit.$(OBJEXT): $(top_srcdir)/internal/imemo.h mjit.$(OBJEXT): $(top_srcdir)/internal/serial.h mjit.$(OBJEXT): $(top_srcdir)/internal/static_assert.h +mjit.$(OBJEXT): $(top_srcdir)/internal/variable.h mjit.$(OBJEXT): $(top_srcdir)/internal/vm.h mjit.$(OBJEXT): $(top_srcdir)/internal/warnings.h mjit.$(OBJEXT): {$(VPATH)}assert.h @@ -9676,6 +9717,7 @@ mjit.$(OBJEXT): {$(VPATH)}ractor.h mjit.$(OBJEXT): {$(VPATH)}ractor_core.h mjit.$(OBJEXT): {$(VPATH)}ruby_assert.h mjit.$(OBJEXT): {$(VPATH)}ruby_atomic.h +mjit.$(OBJEXT): {$(VPATH)}shape.h mjit.$(OBJEXT): {$(VPATH)}st.h mjit.$(OBJEXT): {$(VPATH)}subst.h mjit.$(OBJEXT): {$(VPATH)}thread.h @@ -9880,6 +9922,7 @@ mjit_compile.$(OBJEXT): {$(VPATH)}mjit_unit.h mjit_compile.$(OBJEXT): {$(VPATH)}node.h mjit_compile.$(OBJEXT): {$(VPATH)}ruby_assert.h mjit_compile.$(OBJEXT): {$(VPATH)}ruby_atomic.h +mjit_compile.$(OBJEXT): {$(VPATH)}shape.h mjit_compile.$(OBJEXT): {$(VPATH)}st.h mjit_compile.$(OBJEXT): {$(VPATH)}subst.h mjit_compile.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h @@ -10472,6 +10515,7 @@ object.$(OBJEXT): {$(VPATH)}onigmo.h object.$(OBJEXT): {$(VPATH)}oniguruma.h object.$(OBJEXT): {$(VPATH)}probes.dmyh object.$(OBJEXT): {$(VPATH)}probes.h +object.$(OBJEXT): {$(VPATH)}shape.h object.$(OBJEXT): {$(VPATH)}st.h object.$(OBJEXT): {$(VPATH)}subst.h object.$(OBJEXT): {$(VPATH)}util.h @@ -10890,6 +10934,7 @@ proc.$(OBJEXT): $(top_srcdir)/internal/serial.h proc.$(OBJEXT): $(top_srcdir)/internal/static_assert.h proc.$(OBJEXT): $(top_srcdir)/internal/string.h proc.$(OBJEXT): $(top_srcdir)/internal/symbol.h +proc.$(OBJEXT): $(top_srcdir)/internal/variable.h proc.$(OBJEXT): $(top_srcdir)/internal/vm.h proc.$(OBJEXT): $(top_srcdir)/internal/warnings.h proc.$(OBJEXT): {$(VPATH)}assert.h @@ -10904,6 +10949,7 @@ proc.$(OBJEXT): {$(VPATH)}backward/2/long_long.h proc.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h proc.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h proc.$(OBJEXT): {$(VPATH)}config.h +proc.$(OBJEXT): {$(VPATH)}constant.h proc.$(OBJEXT): {$(VPATH)}defines.h proc.$(OBJEXT): {$(VPATH)}encoding.h proc.$(OBJEXT): {$(VPATH)}eval_intern.h @@ -11320,6 +11366,7 @@ ractor.$(OBJEXT): $(top_srcdir)/internal/static_assert.h ractor.$(OBJEXT): $(top_srcdir)/internal/string.h ractor.$(OBJEXT): $(top_srcdir)/internal/struct.h ractor.$(OBJEXT): $(top_srcdir)/internal/thread.h +ractor.$(OBJEXT): $(top_srcdir)/internal/variable.h ractor.$(OBJEXT): $(top_srcdir)/internal/vm.h ractor.$(OBJEXT): $(top_srcdir)/internal/warnings.h ractor.$(OBJEXT): {$(VPATH)}assert.h @@ -11335,6 +11382,7 @@ ractor.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h ractor.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h ractor.$(OBJEXT): {$(VPATH)}builtin.h ractor.$(OBJEXT): {$(VPATH)}config.h +ractor.$(OBJEXT): {$(VPATH)}constant.h ractor.$(OBJEXT): {$(VPATH)}debug_counter.h ractor.$(OBJEXT): {$(VPATH)}defines.h ractor.$(OBJEXT): {$(VPATH)}encoding.h @@ -11901,6 +11949,7 @@ rational.$(OBJEXT): $(top_srcdir)/internal/object.h rational.$(OBJEXT): $(top_srcdir)/internal/rational.h rational.$(OBJEXT): $(top_srcdir)/internal/serial.h rational.$(OBJEXT): $(top_srcdir)/internal/static_assert.h +rational.$(OBJEXT): $(top_srcdir)/internal/variable.h rational.$(OBJEXT): $(top_srcdir)/internal/vm.h rational.$(OBJEXT): $(top_srcdir)/internal/warnings.h rational.$(OBJEXT): {$(VPATH)}assert.h @@ -11914,6 +11963,7 @@ rational.$(OBJEXT): {$(VPATH)}backward/2/long_long.h rational.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h rational.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h rational.$(OBJEXT): {$(VPATH)}config.h +rational.$(OBJEXT): {$(VPATH)}constant.h rational.$(OBJEXT): {$(VPATH)}defines.h rational.$(OBJEXT): {$(VPATH)}id.h rational.$(OBJEXT): {$(VPATH)}id_table.h @@ -13477,6 +13527,7 @@ scheduler.$(OBJEXT): $(top_srcdir)/internal/imemo.h scheduler.$(OBJEXT): $(top_srcdir)/internal/serial.h scheduler.$(OBJEXT): $(top_srcdir)/internal/static_assert.h scheduler.$(OBJEXT): $(top_srcdir)/internal/thread.h +scheduler.$(OBJEXT): $(top_srcdir)/internal/variable.h scheduler.$(OBJEXT): $(top_srcdir)/internal/vm.h scheduler.$(OBJEXT): $(top_srcdir)/internal/warnings.h scheduler.$(OBJEXT): {$(VPATH)}assert.h @@ -13491,10 +13542,12 @@ scheduler.$(OBJEXT): {$(VPATH)}backward/2/long_long.h scheduler.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h scheduler.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h scheduler.$(OBJEXT): {$(VPATH)}config.h +scheduler.$(OBJEXT): {$(VPATH)}constant.h scheduler.$(OBJEXT): {$(VPATH)}defines.h scheduler.$(OBJEXT): {$(VPATH)}encoding.h scheduler.$(OBJEXT): {$(VPATH)}fiber/scheduler.h scheduler.$(OBJEXT): {$(VPATH)}id.h +scheduler.$(OBJEXT): {$(VPATH)}id_table.h scheduler.$(OBJEXT): {$(VPATH)}intern.h scheduler.$(OBJEXT): {$(VPATH)}internal.h scheduler.$(OBJEXT): {$(VPATH)}internal/abi.h @@ -13821,6 +13874,208 @@ setproctitle.$(OBJEXT): {$(VPATH)}setproctitle.c setproctitle.$(OBJEXT): {$(VPATH)}st.h setproctitle.$(OBJEXT): {$(VPATH)}subst.h setproctitle.$(OBJEXT): {$(VPATH)}util.h +shape.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h +shape.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h +shape.$(OBJEXT): $(CCAN_DIR)/list/list.h +shape.$(OBJEXT): $(CCAN_DIR)/str/str.h +shape.$(OBJEXT): $(hdrdir)/ruby/ruby.h +shape.$(OBJEXT): $(top_srcdir)/internal/array.h +shape.$(OBJEXT): $(top_srcdir)/internal/class.h +shape.$(OBJEXT): $(top_srcdir)/internal/compilers.h +shape.$(OBJEXT): $(top_srcdir)/internal/gc.h +shape.$(OBJEXT): $(top_srcdir)/internal/imemo.h +shape.$(OBJEXT): $(top_srcdir)/internal/serial.h +shape.$(OBJEXT): $(top_srcdir)/internal/static_assert.h +shape.$(OBJEXT): $(top_srcdir)/internal/symbol.h +shape.$(OBJEXT): $(top_srcdir)/internal/variable.h +shape.$(OBJEXT): $(top_srcdir)/internal/vm.h +shape.$(OBJEXT): $(top_srcdir)/internal/warnings.h +shape.$(OBJEXT): {$(VPATH)}assert.h +shape.$(OBJEXT): {$(VPATH)}atomic.h +shape.$(OBJEXT): {$(VPATH)}backward/2/assume.h +shape.$(OBJEXT): {$(VPATH)}backward/2/attributes.h +shape.$(OBJEXT): {$(VPATH)}backward/2/bool.h +shape.$(OBJEXT): {$(VPATH)}backward/2/gcc_version_since.h +shape.$(OBJEXT): {$(VPATH)}backward/2/inttypes.h +shape.$(OBJEXT): {$(VPATH)}backward/2/limits.h +shape.$(OBJEXT): {$(VPATH)}backward/2/long_long.h +shape.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h +shape.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h +shape.$(OBJEXT): {$(VPATH)}config.h +shape.$(OBJEXT): {$(VPATH)}constant.h +shape.$(OBJEXT): {$(VPATH)}debug_counter.h +shape.$(OBJEXT): {$(VPATH)}defines.h +shape.$(OBJEXT): {$(VPATH)}encoding.h +shape.$(OBJEXT): {$(VPATH)}id.h +shape.$(OBJEXT): {$(VPATH)}id_table.h +shape.$(OBJEXT): {$(VPATH)}intern.h +shape.$(OBJEXT): {$(VPATH)}internal.h +shape.$(OBJEXT): {$(VPATH)}internal/abi.h +shape.$(OBJEXT): {$(VPATH)}internal/anyargs.h +shape.$(OBJEXT): {$(VPATH)}internal/arithmetic.h +shape.$(OBJEXT): {$(VPATH)}internal/arithmetic/char.h +shape.$(OBJEXT): {$(VPATH)}internal/arithmetic/double.h +shape.$(OBJEXT): {$(VPATH)}internal/arithmetic/fixnum.h +shape.$(OBJEXT): {$(VPATH)}internal/arithmetic/gid_t.h +shape.$(OBJEXT): {$(VPATH)}internal/arithmetic/int.h +shape.$(OBJEXT): {$(VPATH)}internal/arithmetic/intptr_t.h +shape.$(OBJEXT): {$(VPATH)}internal/arithmetic/long.h +shape.$(OBJEXT): {$(VPATH)}internal/arithmetic/long_long.h +shape.$(OBJEXT): {$(VPATH)}internal/arithmetic/mode_t.h +shape.$(OBJEXT): {$(VPATH)}internal/arithmetic/off_t.h +shape.$(OBJEXT): {$(VPATH)}internal/arithmetic/pid_t.h +shape.$(OBJEXT): {$(VPATH)}internal/arithmetic/short.h +shape.$(OBJEXT): {$(VPATH)}internal/arithmetic/size_t.h +shape.$(OBJEXT): {$(VPATH)}internal/arithmetic/st_data_t.h +shape.$(OBJEXT): {$(VPATH)}internal/arithmetic/uid_t.h +shape.$(OBJEXT): {$(VPATH)}internal/assume.h +shape.$(OBJEXT): {$(VPATH)}internal/attr/alloc_size.h +shape.$(OBJEXT): {$(VPATH)}internal/attr/artificial.h +shape.$(OBJEXT): {$(VPATH)}internal/attr/cold.h +shape.$(OBJEXT): {$(VPATH)}internal/attr/const.h +shape.$(OBJEXT): {$(VPATH)}internal/attr/constexpr.h +shape.$(OBJEXT): {$(VPATH)}internal/attr/deprecated.h +shape.$(OBJEXT): {$(VPATH)}internal/attr/diagnose_if.h +shape.$(OBJEXT): {$(VPATH)}internal/attr/enum_extensibility.h +shape.$(OBJEXT): {$(VPATH)}internal/attr/error.h +shape.$(OBJEXT): {$(VPATH)}internal/attr/flag_enum.h +shape.$(OBJEXT): {$(VPATH)}internal/attr/forceinline.h +shape.$(OBJEXT): {$(VPATH)}internal/attr/format.h +shape.$(OBJEXT): {$(VPATH)}internal/attr/maybe_unused.h +shape.$(OBJEXT): {$(VPATH)}internal/attr/noalias.h +shape.$(OBJEXT): {$(VPATH)}internal/attr/nodiscard.h +shape.$(OBJEXT): {$(VPATH)}internal/attr/noexcept.h +shape.$(OBJEXT): {$(VPATH)}internal/attr/noinline.h +shape.$(OBJEXT): {$(VPATH)}internal/attr/nonnull.h +shape.$(OBJEXT): {$(VPATH)}internal/attr/noreturn.h +shape.$(OBJEXT): {$(VPATH)}internal/attr/pure.h +shape.$(OBJEXT): {$(VPATH)}internal/attr/restrict.h +shape.$(OBJEXT): {$(VPATH)}internal/attr/returns_nonnull.h +shape.$(OBJEXT): {$(VPATH)}internal/attr/warning.h +shape.$(OBJEXT): {$(VPATH)}internal/attr/weakref.h +shape.$(OBJEXT): {$(VPATH)}internal/cast.h +shape.$(OBJEXT): {$(VPATH)}internal/compiler_is.h +shape.$(OBJEXT): {$(VPATH)}internal/compiler_is/apple.h +shape.$(OBJEXT): {$(VPATH)}internal/compiler_is/clang.h +shape.$(OBJEXT): {$(VPATH)}internal/compiler_is/gcc.h +shape.$(OBJEXT): {$(VPATH)}internal/compiler_is/intel.h +shape.$(OBJEXT): {$(VPATH)}internal/compiler_is/msvc.h +shape.$(OBJEXT): {$(VPATH)}internal/compiler_is/sunpro.h +shape.$(OBJEXT): {$(VPATH)}internal/compiler_since.h +shape.$(OBJEXT): {$(VPATH)}internal/config.h +shape.$(OBJEXT): {$(VPATH)}internal/constant_p.h +shape.$(OBJEXT): {$(VPATH)}internal/core.h +shape.$(OBJEXT): {$(VPATH)}internal/core/rarray.h +shape.$(OBJEXT): {$(VPATH)}internal/core/rbasic.h +shape.$(OBJEXT): {$(VPATH)}internal/core/rbignum.h +shape.$(OBJEXT): {$(VPATH)}internal/core/rclass.h +shape.$(OBJEXT): {$(VPATH)}internal/core/rdata.h +shape.$(OBJEXT): {$(VPATH)}internal/core/rfile.h +shape.$(OBJEXT): {$(VPATH)}internal/core/rhash.h +shape.$(OBJEXT): {$(VPATH)}internal/core/robject.h +shape.$(OBJEXT): {$(VPATH)}internal/core/rregexp.h +shape.$(OBJEXT): {$(VPATH)}internal/core/rstring.h +shape.$(OBJEXT): {$(VPATH)}internal/core/rstruct.h +shape.$(OBJEXT): {$(VPATH)}internal/core/rtypeddata.h +shape.$(OBJEXT): {$(VPATH)}internal/ctype.h +shape.$(OBJEXT): {$(VPATH)}internal/dllexport.h +shape.$(OBJEXT): {$(VPATH)}internal/dosish.h +shape.$(OBJEXT): {$(VPATH)}internal/encoding/coderange.h +shape.$(OBJEXT): {$(VPATH)}internal/encoding/ctype.h +shape.$(OBJEXT): {$(VPATH)}internal/encoding/encoding.h +shape.$(OBJEXT): {$(VPATH)}internal/encoding/pathname.h +shape.$(OBJEXT): {$(VPATH)}internal/encoding/re.h +shape.$(OBJEXT): {$(VPATH)}internal/encoding/sprintf.h +shape.$(OBJEXT): {$(VPATH)}internal/encoding/string.h +shape.$(OBJEXT): {$(VPATH)}internal/encoding/symbol.h +shape.$(OBJEXT): {$(VPATH)}internal/encoding/transcode.h +shape.$(OBJEXT): {$(VPATH)}internal/error.h +shape.$(OBJEXT): {$(VPATH)}internal/eval.h +shape.$(OBJEXT): {$(VPATH)}internal/event.h +shape.$(OBJEXT): {$(VPATH)}internal/fl_type.h +shape.$(OBJEXT): {$(VPATH)}internal/gc.h +shape.$(OBJEXT): {$(VPATH)}internal/glob.h +shape.$(OBJEXT): {$(VPATH)}internal/globals.h +shape.$(OBJEXT): {$(VPATH)}internal/has/attribute.h +shape.$(OBJEXT): {$(VPATH)}internal/has/builtin.h +shape.$(OBJEXT): {$(VPATH)}internal/has/c_attribute.h +shape.$(OBJEXT): {$(VPATH)}internal/has/cpp_attribute.h +shape.$(OBJEXT): {$(VPATH)}internal/has/declspec_attribute.h +shape.$(OBJEXT): {$(VPATH)}internal/has/extension.h +shape.$(OBJEXT): {$(VPATH)}internal/has/feature.h +shape.$(OBJEXT): {$(VPATH)}internal/has/warning.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/array.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/bignum.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/class.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/compar.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/complex.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/cont.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/dir.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/enum.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/enumerator.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/error.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/eval.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/file.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/gc.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/hash.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/io.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/load.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/marshal.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/numeric.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/object.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/parse.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/proc.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/process.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/random.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/range.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/rational.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/re.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/ruby.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/select.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/select/largesize.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/signal.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/sprintf.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/string.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/struct.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/thread.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/time.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/variable.h +shape.$(OBJEXT): {$(VPATH)}internal/intern/vm.h +shape.$(OBJEXT): {$(VPATH)}internal/interpreter.h +shape.$(OBJEXT): {$(VPATH)}internal/iterator.h +shape.$(OBJEXT): {$(VPATH)}internal/memory.h +shape.$(OBJEXT): {$(VPATH)}internal/method.h +shape.$(OBJEXT): {$(VPATH)}internal/module.h +shape.$(OBJEXT): {$(VPATH)}internal/newobj.h +shape.$(OBJEXT): {$(VPATH)}internal/rgengc.h +shape.$(OBJEXT): {$(VPATH)}internal/scan_args.h +shape.$(OBJEXT): {$(VPATH)}internal/special_consts.h +shape.$(OBJEXT): {$(VPATH)}internal/static_assert.h +shape.$(OBJEXT): {$(VPATH)}internal/stdalign.h +shape.$(OBJEXT): {$(VPATH)}internal/stdbool.h +shape.$(OBJEXT): {$(VPATH)}internal/symbol.h +shape.$(OBJEXT): {$(VPATH)}internal/value.h +shape.$(OBJEXT): {$(VPATH)}internal/value_type.h +shape.$(OBJEXT): {$(VPATH)}internal/variable.h +shape.$(OBJEXT): {$(VPATH)}internal/warning_push.h +shape.$(OBJEXT): {$(VPATH)}internal/xmalloc.h +shape.$(OBJEXT): {$(VPATH)}method.h +shape.$(OBJEXT): {$(VPATH)}missing.h +shape.$(OBJEXT): {$(VPATH)}node.h +shape.$(OBJEXT): {$(VPATH)}onigmo.h +shape.$(OBJEXT): {$(VPATH)}oniguruma.h +shape.$(OBJEXT): {$(VPATH)}ruby_assert.h +shape.$(OBJEXT): {$(VPATH)}ruby_atomic.h +shape.$(OBJEXT): {$(VPATH)}shape.c +shape.$(OBJEXT): {$(VPATH)}shape.h +shape.$(OBJEXT): {$(VPATH)}st.h +shape.$(OBJEXT): {$(VPATH)}subst.h +shape.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h +shape.$(OBJEXT): {$(VPATH)}thread_native.h +shape.$(OBJEXT): {$(VPATH)}vm_core.h +shape.$(OBJEXT): {$(VPATH)}vm_debug.h +shape.$(OBJEXT): {$(VPATH)}vm_opts.h +shape.$(OBJEXT): {$(VPATH)}vm_sync.h signal.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h signal.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h signal.$(OBJEXT): $(CCAN_DIR)/list/list.h @@ -13837,6 +14092,7 @@ signal.$(OBJEXT): $(top_srcdir)/internal/signal.h signal.$(OBJEXT): $(top_srcdir)/internal/static_assert.h signal.$(OBJEXT): $(top_srcdir)/internal/string.h signal.$(OBJEXT): $(top_srcdir)/internal/thread.h +signal.$(OBJEXT): $(top_srcdir)/internal/variable.h signal.$(OBJEXT): $(top_srcdir)/internal/vm.h signal.$(OBJEXT): $(top_srcdir)/internal/warnings.h signal.$(OBJEXT): {$(VPATH)}assert.h @@ -13851,6 +14107,7 @@ signal.$(OBJEXT): {$(VPATH)}backward/2/long_long.h signal.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h signal.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h signal.$(OBJEXT): {$(VPATH)}config.h +signal.$(OBJEXT): {$(VPATH)}constant.h signal.$(OBJEXT): {$(VPATH)}debug_counter.h signal.$(OBJEXT): {$(VPATH)}defines.h signal.$(OBJEXT): {$(VPATH)}encoding.h @@ -14041,6 +14298,7 @@ sprintf.$(OBJEXT): $(top_srcdir)/internal/serial.h sprintf.$(OBJEXT): $(top_srcdir)/internal/static_assert.h sprintf.$(OBJEXT): $(top_srcdir)/internal/string.h sprintf.$(OBJEXT): $(top_srcdir)/internal/symbol.h +sprintf.$(OBJEXT): $(top_srcdir)/internal/variable.h sprintf.$(OBJEXT): $(top_srcdir)/internal/vm.h sprintf.$(OBJEXT): $(top_srcdir)/internal/warnings.h sprintf.$(OBJEXT): {$(VPATH)}assert.h @@ -14054,6 +14312,7 @@ sprintf.$(OBJEXT): {$(VPATH)}backward/2/long_long.h sprintf.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h sprintf.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h sprintf.$(OBJEXT): {$(VPATH)}config.h +sprintf.$(OBJEXT): {$(VPATH)}constant.h sprintf.$(OBJEXT): {$(VPATH)}defines.h sprintf.$(OBJEXT): {$(VPATH)}encoding.h sprintf.$(OBJEXT): {$(VPATH)}id.h @@ -14583,6 +14842,7 @@ string.$(OBJEXT): $(top_srcdir)/internal/serial.h string.$(OBJEXT): $(top_srcdir)/internal/static_assert.h string.$(OBJEXT): $(top_srcdir)/internal/string.h string.$(OBJEXT): $(top_srcdir)/internal/transcode.h +string.$(OBJEXT): $(top_srcdir)/internal/variable.h string.$(OBJEXT): $(top_srcdir)/internal/vm.h string.$(OBJEXT): $(top_srcdir)/internal/warnings.h string.$(OBJEXT): {$(VPATH)}assert.h @@ -14597,6 +14857,7 @@ string.$(OBJEXT): {$(VPATH)}backward/2/long_long.h string.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h string.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h string.$(OBJEXT): {$(VPATH)}config.h +string.$(OBJEXT): {$(VPATH)}constant.h string.$(OBJEXT): {$(VPATH)}debug_counter.h string.$(OBJEXT): {$(VPATH)}defines.h string.$(OBJEXT): {$(VPATH)}encindex.h @@ -14820,6 +15081,7 @@ struct.$(OBJEXT): $(top_srcdir)/internal/static_assert.h struct.$(OBJEXT): $(top_srcdir)/internal/string.h struct.$(OBJEXT): $(top_srcdir)/internal/struct.h struct.$(OBJEXT): $(top_srcdir)/internal/symbol.h +struct.$(OBJEXT): $(top_srcdir)/internal/variable.h struct.$(OBJEXT): $(top_srcdir)/internal/vm.h struct.$(OBJEXT): $(top_srcdir)/internal/warnings.h struct.$(OBJEXT): {$(VPATH)}assert.h @@ -14835,6 +15097,7 @@ struct.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h struct.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h struct.$(OBJEXT): {$(VPATH)}builtin.h struct.$(OBJEXT): {$(VPATH)}config.h +struct.$(OBJEXT): {$(VPATH)}constant.h struct.$(OBJEXT): {$(VPATH)}defines.h struct.$(OBJEXT): {$(VPATH)}encoding.h struct.$(OBJEXT): {$(VPATH)}id.h @@ -15016,6 +15279,7 @@ symbol.$(OBJEXT): $(top_srcdir)/internal/serial.h symbol.$(OBJEXT): $(top_srcdir)/internal/static_assert.h symbol.$(OBJEXT): $(top_srcdir)/internal/string.h symbol.$(OBJEXT): $(top_srcdir)/internal/symbol.h +symbol.$(OBJEXT): $(top_srcdir)/internal/variable.h symbol.$(OBJEXT): $(top_srcdir)/internal/vm.h symbol.$(OBJEXT): $(top_srcdir)/internal/warnings.h symbol.$(OBJEXT): {$(VPATH)}assert.h @@ -15029,6 +15293,7 @@ symbol.$(OBJEXT): {$(VPATH)}backward/2/long_long.h symbol.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h symbol.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h symbol.$(OBJEXT): {$(VPATH)}config.h +symbol.$(OBJEXT): {$(VPATH)}constant.h symbol.$(OBJEXT): {$(VPATH)}debug_counter.h symbol.$(OBJEXT): {$(VPATH)}defines.h symbol.$(OBJEXT): {$(VPATH)}encoding.h @@ -15224,6 +15489,7 @@ thread.$(OBJEXT): $(top_srcdir)/internal/static_assert.h thread.$(OBJEXT): $(top_srcdir)/internal/string.h thread.$(OBJEXT): $(top_srcdir)/internal/thread.h thread.$(OBJEXT): $(top_srcdir)/internal/time.h +thread.$(OBJEXT): $(top_srcdir)/internal/variable.h thread.$(OBJEXT): $(top_srcdir)/internal/vm.h thread.$(OBJEXT): $(top_srcdir)/internal/warnings.h thread.$(OBJEXT): {$(VPATH)}assert.h @@ -15239,6 +15505,7 @@ thread.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h thread.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h thread.$(OBJEXT): {$(VPATH)}builtin.h thread.$(OBJEXT): {$(VPATH)}config.h +thread.$(OBJEXT): {$(VPATH)}constant.h thread.$(OBJEXT): {$(VPATH)}debug.h thread.$(OBJEXT): {$(VPATH)}debug_counter.h thread.$(OBJEXT): {$(VPATH)}defines.h @@ -15633,6 +15900,7 @@ transcode.$(OBJEXT): $(top_srcdir)/internal/serial.h transcode.$(OBJEXT): $(top_srcdir)/internal/static_assert.h transcode.$(OBJEXT): $(top_srcdir)/internal/string.h transcode.$(OBJEXT): $(top_srcdir)/internal/transcode.h +transcode.$(OBJEXT): $(top_srcdir)/internal/variable.h transcode.$(OBJEXT): $(top_srcdir)/internal/warnings.h transcode.$(OBJEXT): {$(VPATH)}assert.h transcode.$(OBJEXT): {$(VPATH)}backward/2/assume.h @@ -15645,6 +15913,7 @@ transcode.$(OBJEXT): {$(VPATH)}backward/2/long_long.h transcode.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h transcode.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h transcode.$(OBJEXT): {$(VPATH)}config.h +transcode.$(OBJEXT): {$(VPATH)}constant.h transcode.$(OBJEXT): {$(VPATH)}defines.h transcode.$(OBJEXT): {$(VPATH)}encoding.h transcode.$(OBJEXT): {$(VPATH)}id.h @@ -16352,6 +16621,7 @@ variable.$(OBJEXT): {$(VPATH)}ractor.h variable.$(OBJEXT): {$(VPATH)}ractor_core.h variable.$(OBJEXT): {$(VPATH)}ruby_assert.h variable.$(OBJEXT): {$(VPATH)}ruby_atomic.h +variable.$(OBJEXT): {$(VPATH)}shape.h variable.$(OBJEXT): {$(VPATH)}st.h variable.$(OBJEXT): {$(VPATH)}subst.h variable.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h @@ -16377,6 +16647,7 @@ version.$(OBJEXT): $(top_srcdir)/internal/gc.h version.$(OBJEXT): $(top_srcdir)/internal/imemo.h version.$(OBJEXT): $(top_srcdir)/internal/serial.h version.$(OBJEXT): $(top_srcdir)/internal/static_assert.h +version.$(OBJEXT): $(top_srcdir)/internal/variable.h version.$(OBJEXT): $(top_srcdir)/internal/vm.h version.$(OBJEXT): $(top_srcdir)/internal/warnings.h version.$(OBJEXT): $(top_srcdir)/revision.h @@ -16393,9 +16664,11 @@ version.$(OBJEXT): {$(VPATH)}backward/2/long_long.h version.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h version.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h version.$(OBJEXT): {$(VPATH)}config.h +version.$(OBJEXT): {$(VPATH)}constant.h version.$(OBJEXT): {$(VPATH)}debug_counter.h version.$(OBJEXT): {$(VPATH)}defines.h version.$(OBJEXT): {$(VPATH)}id.h +version.$(OBJEXT): {$(VPATH)}id_table.h version.$(OBJEXT): {$(VPATH)}intern.h version.$(OBJEXT): {$(VPATH)}internal.h version.$(OBJEXT): {$(VPATH)}internal/abi.h @@ -16779,6 +17052,7 @@ vm.$(OBJEXT): {$(VPATH)}ractor.h vm.$(OBJEXT): {$(VPATH)}ractor_core.h vm.$(OBJEXT): {$(VPATH)}ruby_assert.h vm.$(OBJEXT): {$(VPATH)}ruby_atomic.h +vm.$(OBJEXT): {$(VPATH)}shape.h vm.$(OBJEXT): {$(VPATH)}st.h vm.$(OBJEXT): {$(VPATH)}subst.h vm.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h @@ -16815,6 +17089,7 @@ vm_backtrace.$(OBJEXT): $(top_srcdir)/internal/imemo.h vm_backtrace.$(OBJEXT): $(top_srcdir)/internal/serial.h vm_backtrace.$(OBJEXT): $(top_srcdir)/internal/static_assert.h vm_backtrace.$(OBJEXT): $(top_srcdir)/internal/string.h +vm_backtrace.$(OBJEXT): $(top_srcdir)/internal/variable.h vm_backtrace.$(OBJEXT): $(top_srcdir)/internal/vm.h vm_backtrace.$(OBJEXT): $(top_srcdir)/internal/warnings.h vm_backtrace.$(OBJEXT): {$(VPATH)}assert.h @@ -16829,11 +17104,13 @@ vm_backtrace.$(OBJEXT): {$(VPATH)}backward/2/long_long.h vm_backtrace.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h vm_backtrace.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h vm_backtrace.$(OBJEXT): {$(VPATH)}config.h +vm_backtrace.$(OBJEXT): {$(VPATH)}constant.h vm_backtrace.$(OBJEXT): {$(VPATH)}debug.h vm_backtrace.$(OBJEXT): {$(VPATH)}defines.h vm_backtrace.$(OBJEXT): {$(VPATH)}encoding.h vm_backtrace.$(OBJEXT): {$(VPATH)}eval_intern.h vm_backtrace.$(OBJEXT): {$(VPATH)}id.h +vm_backtrace.$(OBJEXT): {$(VPATH)}id_table.h vm_backtrace.$(OBJEXT): {$(VPATH)}intern.h vm_backtrace.$(OBJEXT): {$(VPATH)}internal.h vm_backtrace.$(OBJEXT): {$(VPATH)}internal/abi.h @@ -17202,6 +17479,7 @@ vm_sync.$(OBJEXT): $(top_srcdir)/internal/gc.h vm_sync.$(OBJEXT): $(top_srcdir)/internal/imemo.h vm_sync.$(OBJEXT): $(top_srcdir)/internal/serial.h vm_sync.$(OBJEXT): $(top_srcdir)/internal/static_assert.h +vm_sync.$(OBJEXT): $(top_srcdir)/internal/variable.h vm_sync.$(OBJEXT): $(top_srcdir)/internal/vm.h vm_sync.$(OBJEXT): $(top_srcdir)/internal/warnings.h vm_sync.$(OBJEXT): {$(VPATH)}assert.h @@ -17216,6 +17494,7 @@ vm_sync.$(OBJEXT): {$(VPATH)}backward/2/long_long.h vm_sync.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h vm_sync.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h vm_sync.$(OBJEXT): {$(VPATH)}config.h +vm_sync.$(OBJEXT): {$(VPATH)}constant.h vm_sync.$(OBJEXT): {$(VPATH)}debug_counter.h vm_sync.$(OBJEXT): {$(VPATH)}defines.h vm_sync.$(OBJEXT): {$(VPATH)}gc.h @@ -17393,6 +17672,7 @@ vm_trace.$(OBJEXT): $(top_srcdir)/internal/imemo.h vm_trace.$(OBJEXT): $(top_srcdir)/internal/serial.h vm_trace.$(OBJEXT): $(top_srcdir)/internal/static_assert.h vm_trace.$(OBJEXT): $(top_srcdir)/internal/symbol.h +vm_trace.$(OBJEXT): $(top_srcdir)/internal/variable.h vm_trace.$(OBJEXT): $(top_srcdir)/internal/vm.h vm_trace.$(OBJEXT): $(top_srcdir)/internal/warnings.h vm_trace.$(OBJEXT): {$(VPATH)}assert.h @@ -17408,12 +17688,14 @@ vm_trace.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h vm_trace.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h vm_trace.$(OBJEXT): {$(VPATH)}builtin.h vm_trace.$(OBJEXT): {$(VPATH)}config.h +vm_trace.$(OBJEXT): {$(VPATH)}constant.h vm_trace.$(OBJEXT): {$(VPATH)}debug.h vm_trace.$(OBJEXT): {$(VPATH)}debug_counter.h vm_trace.$(OBJEXT): {$(VPATH)}defines.h vm_trace.$(OBJEXT): {$(VPATH)}encoding.h vm_trace.$(OBJEXT): {$(VPATH)}eval_intern.h vm_trace.$(OBJEXT): {$(VPATH)}id.h +vm_trace.$(OBJEXT): {$(VPATH)}id_table.h vm_trace.$(OBJEXT): {$(VPATH)}intern.h vm_trace.$(OBJEXT): {$(VPATH)}internal.h vm_trace.$(OBJEXT): {$(VPATH)}internal/abi.h diff --git a/compile.c b/compile.c index 484399abc6..5bc10e631f 100644 --- a/compile.c +++ b/compile.c @@ -2058,20 +2058,7 @@ cdhash_set_label_i(VALUE key, VALUE val, VALUE ptr) static inline VALUE get_ivar_ic_value(rb_iseq_t *iseq,ID id) { - VALUE val; - struct rb_id_table *tbl = ISEQ_COMPILE_DATA(iseq)->ivar_cache_table; - if (tbl) { - if (rb_id_table_lookup(tbl,id,&val)) { - return val; - } - } - else { - tbl = rb_id_table_create(1); - ISEQ_COMPILE_DATA(iseq)->ivar_cache_table = tbl; - } - val = INT2FIX(ISEQ_BODY(iseq)->ivc_size++); - rb_id_table_insert(tbl,id,val); - return val; + return INT2FIX(ISEQ_BODY(iseq)->ivc_size++); } static inline VALUE @@ -2432,10 +2419,14 @@ iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *const anchor) break; } /* [ TS_IVC | TS_ICVARC | TS_ISE | TS_IC ] */ + case TS_IVC: /* inline ivar cache */ + { + unsigned int ic_index = FIX2UINT(operands[j]); + vm_ic_attr_index_initialize(((IVC)&body->is_entries[ic_index]), INVALID_SHAPE_ID); + } case TS_IC: /* inline cache: constants */ case TS_ISE: /* inline storage entry: `once` insn */ case TS_ICVARC: /* inline cvar cache */ - case TS_IVC: /* inline ivar cache */ { unsigned int ic_index = FIX2UINT(operands[j]); IC ic = &ISEQ_IS_ENTRY_START(body, type)[ic_index].ic_cache; @@ -11379,6 +11370,10 @@ ibf_load_code(const struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t bytecod ISE ic = ISEQ_IS_ENTRY_START(load_body, operand_type) + op; code[code_index] = (VALUE)ic; + if (operand_type == TS_IVC) { + vm_ic_attr_index_initialize(((IVC)code[code_index]), INVALID_SHAPE_ID); + } + if (insn == BIN(opt_getinlinecache) && operand_type == TS_IC) { // Store the instruction index for opt_getinlinecache on the IC for // YJIT to invalidate code when opt_setinlinecache runs. diff --git a/debug_counter.h b/debug_counter.h index 3f0dec948f..c8d74f7e28 100644 --- a/debug_counter.h +++ b/debug_counter.h @@ -130,7 +130,6 @@ RB_DEBUG_COUNTER(frame_C2R) /* instance variable counts * * * ivar_get_ic_hit/miss: ivar_get inline cache (ic) hit/miss counts (VM insn) - * * ivar_get_ic_miss_serial: ivar_get ic miss reason by serial (VM insn) * * ivar_get_ic_miss_unset: ... by unset (VM insn) * * ivar_get_ic_miss_noobject: ... by "not T_OBJECT" (VM insn) * * ivar_set_...: same counts with ivar_set (VM insn) @@ -140,17 +139,17 @@ RB_DEBUG_COUNTER(frame_C2R) */ RB_DEBUG_COUNTER(ivar_get_ic_hit) RB_DEBUG_COUNTER(ivar_get_ic_miss) -RB_DEBUG_COUNTER(ivar_get_ic_miss_serial) -RB_DEBUG_COUNTER(ivar_get_ic_miss_unset) RB_DEBUG_COUNTER(ivar_get_ic_miss_noobject) RB_DEBUG_COUNTER(ivar_set_ic_hit) RB_DEBUG_COUNTER(ivar_set_ic_miss) -RB_DEBUG_COUNTER(ivar_set_ic_miss_serial) -RB_DEBUG_COUNTER(ivar_set_ic_miss_unset) RB_DEBUG_COUNTER(ivar_set_ic_miss_iv_hit) RB_DEBUG_COUNTER(ivar_set_ic_miss_noobject) RB_DEBUG_COUNTER(ivar_get_base) RB_DEBUG_COUNTER(ivar_set_base) +RB_DEBUG_COUNTER(ivar_get_ic_miss_set) +RB_DEBUG_COUNTER(ivar_get_cc_miss_set) +RB_DEBUG_COUNTER(ivar_get_ic_miss_unset) +RB_DEBUG_COUNTER(ivar_get_cc_miss_unset) /* local variable counts * @@ -321,6 +320,7 @@ RB_DEBUG_COUNTER(obj_imemo_parser_strterm) RB_DEBUG_COUNTER(obj_imemo_callinfo) RB_DEBUG_COUNTER(obj_imemo_callcache) RB_DEBUG_COUNTER(obj_imemo_constcache) +RB_DEBUG_COUNTER(obj_imemo_shape) /* ar_table */ RB_DEBUG_COUNTER(artable_hint_hit) diff --git a/ext/coverage/depend b/ext/coverage/depend index 57d368d3f5..039dac0abd 100644 --- a/ext/coverage/depend +++ b/ext/coverage/depend @@ -165,7 +165,9 @@ coverage.o: $(top_srcdir)/ccan/check_type/check_type.h coverage.o: $(top_srcdir)/ccan/container_of/container_of.h coverage.o: $(top_srcdir)/ccan/list/list.h coverage.o: $(top_srcdir)/ccan/str/str.h +coverage.o: $(top_srcdir)/constant.h coverage.o: $(top_srcdir)/gc.h +coverage.o: $(top_srcdir)/id_table.h coverage.o: $(top_srcdir)/internal.h coverage.o: $(top_srcdir)/internal/array.h coverage.o: $(top_srcdir)/internal/compilers.h @@ -176,6 +178,7 @@ coverage.o: $(top_srcdir)/internal/sanitizers.h coverage.o: $(top_srcdir)/internal/serial.h coverage.o: $(top_srcdir)/internal/static_assert.h coverage.o: $(top_srcdir)/internal/thread.h +coverage.o: $(top_srcdir)/internal/variable.h coverage.o: $(top_srcdir)/internal/vm.h coverage.o: $(top_srcdir)/internal/warnings.h coverage.o: $(top_srcdir)/method.h diff --git a/ext/objspace/depend b/ext/objspace/depend index c4da8031cc..60c8617d19 100644 --- a/ext/objspace/depend +++ b/ext/objspace/depend @@ -533,7 +533,9 @@ objspace_dump.o: $(top_srcdir)/ccan/check_type/check_type.h objspace_dump.o: $(top_srcdir)/ccan/container_of/container_of.h objspace_dump.o: $(top_srcdir)/ccan/list/list.h objspace_dump.o: $(top_srcdir)/ccan/str/str.h +objspace_dump.o: $(top_srcdir)/constant.h objspace_dump.o: $(top_srcdir)/gc.h +objspace_dump.o: $(top_srcdir)/id_table.h objspace_dump.o: $(top_srcdir)/internal.h objspace_dump.o: $(top_srcdir)/internal/array.h objspace_dump.o: $(top_srcdir)/internal/compilers.h @@ -544,6 +546,7 @@ objspace_dump.o: $(top_srcdir)/internal/sanitizers.h objspace_dump.o: $(top_srcdir)/internal/serial.h objspace_dump.o: $(top_srcdir)/internal/static_assert.h objspace_dump.o: $(top_srcdir)/internal/string.h +objspace_dump.o: $(top_srcdir)/internal/variable.h objspace_dump.o: $(top_srcdir)/internal/vm.h objspace_dump.o: $(top_srcdir)/internal/warnings.h objspace_dump.o: $(top_srcdir)/method.h diff --git a/ext/objspace/objspace.c b/ext/objspace/objspace.c index 0b1b094325..364d36e696 100644 --- a/ext/objspace/objspace.c +++ b/ext/objspace/objspace.c @@ -644,6 +644,7 @@ count_imemo_objects(int argc, VALUE *argv, VALUE self) INIT_IMEMO_TYPE_ID(imemo_callinfo); INIT_IMEMO_TYPE_ID(imemo_callcache); INIT_IMEMO_TYPE_ID(imemo_constcache); + INIT_IMEMO_TYPE_ID(imemo_shape); #undef INIT_IMEMO_TYPE_ID } diff --git a/gc.c b/gc.c index ecb4aa7e20..33529c34a4 100644 --- a/gc.c +++ b/gc.c @@ -2895,8 +2895,7 @@ rb_class_instance_allocate_internal(VALUE klass, VALUE flags, bool wb_protected) GC_ASSERT((flags & RUBY_T_MASK) == T_OBJECT); GC_ASSERT(flags & ROBJECT_EMBED); - st_table *index_tbl = RCLASS_IV_INDEX_TBL(klass); - uint32_t index_tbl_num_entries = index_tbl == NULL ? 0 : (uint32_t)index_tbl->num_entries; + uint32_t index_tbl_num_entries = RCLASS_EXT(klass)->max_iv_count; size_t size; bool embed = true; @@ -2972,6 +2971,7 @@ rb_imemo_name(enum imemo_type type) IMEMO_NAME(callinfo); IMEMO_NAME(callcache); IMEMO_NAME(constcache); + IMEMO_NAME(shape); #undef IMEMO_NAME } return "unknown"; @@ -3018,6 +3018,14 @@ imemo_memsize(VALUE obj) case imemo_iseq: size += rb_iseq_memsize((rb_iseq_t *)obj); break; + case imemo_shape: + { + struct rb_id_table* edges = ((rb_shape_t *) obj)->edges; + if (edges) { + size += rb_id_table_memsize(edges); + } + break; + } case imemo_env: size += RANY(obj)->as.imemo.env.env_size * sizeof(VALUE); break; @@ -3206,20 +3214,6 @@ rb_free_const_table(struct rb_id_table *tbl) rb_id_table_free(tbl); } -static int -free_iv_index_tbl_free_i(st_data_t key, st_data_t value, st_data_t data) -{ - xfree((void *)value); - return ST_CONTINUE; -} - -static void -iv_index_tbl_free(struct st_table *tbl) -{ - st_foreach(tbl, free_iv_index_tbl_free_i, 0); - st_free_table(tbl); -} - // alive: if false, target pointers can be freed already. // To check it, we need objspace parameter. static void @@ -3387,6 +3381,22 @@ obj_free_object_id(rb_objspace_t *objspace, VALUE obj) } } +static enum rb_id_table_iterator_result +remove_child_shapes_parent(VALUE value, void *ref) +{ + rb_shape_t * shape = (rb_shape_t *) value; + GC_ASSERT(IMEMO_TYPE_P(shape, imemo_shape)); + + // If both objects live on the same page and we're currently + // sweeping that page, then we need to assert that neither are marked + if (GET_HEAP_PAGE(shape) == GET_HEAP_PAGE(shape->parent)) { + GC_ASSERT(!MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(shape), shape)); + } + + shape->parent = NULL; + return ID_TABLE_CONTINUE; +} + static int obj_free(rb_objspace_t *objspace, VALUE obj) { @@ -3435,6 +3445,24 @@ obj_free(rb_objspace_t *objspace, VALUE obj) RB_DEBUG_COUNTER_INC(obj_obj_transient); } else { + // A shape can be collected before an object is collected (if both + // happened to be garbage at the same time), so when we look up the shape, _do not_ + // assert that the shape is an IMEMO because it could be null + rb_shape_t *shape = rb_shape_get_shape_by_id_without_assertion(ROBJECT_SHAPE_ID(obj)); + if (shape) { + VALUE klass = RBASIC_CLASS(obj); + + // Increment max_iv_count if applicable, used to determine size pool allocation + uint32_t num_of_ivs = shape->iv_count; + if (RCLASS_EXT(klass)->max_iv_count < num_of_ivs) { + RCLASS_EXT(klass)->max_iv_count = num_of_ivs; + } + + // "No cache" objects have a singleton iv_index_tbl that we need to free + if (shape && rb_shape_no_cache_shape_p(shape)) { + rb_id_table_free(ROBJECT(obj)->as.heap.iv_index_tbl); + } + } xfree(RANY(obj)->as.object.as.heap.ivptr); RB_DEBUG_COUNTER_INC(obj_obj_ptr); } @@ -3449,9 +3477,6 @@ obj_free(rb_objspace_t *objspace, VALUE obj) if (RCLASS_CONST_TBL(obj)) { rb_free_const_table(RCLASS_CONST_TBL(obj)); } - if (RCLASS_IV_INDEX_TBL(obj)) { - iv_index_tbl_free(RCLASS_IV_INDEX_TBL(obj)); - } if (RCLASS_CVC_TBL(obj)) { rb_id_table_foreach_values(RCLASS_CVC_TBL(obj), cvar_table_free_i, NULL); rb_id_table_free(RCLASS_CVC_TBL(obj)); @@ -3728,8 +3753,39 @@ obj_free(rb_objspace_t *objspace, VALUE obj) case imemo_constcache: RB_DEBUG_COUNTER_INC(obj_imemo_constcache); break; - } - return TRUE; + case imemo_shape: + { + rb_shape_t *shape = (rb_shape_t *)obj; + rb_shape_t *parent = shape->parent; + + if (parent) { + RUBY_ASSERT(IMEMO_TYPE_P(parent, imemo_shape)); + RUBY_ASSERT(parent->edges); + VALUE res; // Only used to temporarily store lookup value + if (rb_id_table_lookup(parent->edges, shape->edge_name, &res)) { + if ((rb_shape_t *)res == shape) { + rb_id_table_delete(parent->edges, shape->edge_name); + } + } + else { + rb_bug("Edge %s should exist", rb_id2name(shape->edge_name)); + } + } + if (shape->edges) { + rb_id_table_foreach_values(shape->edges, remove_child_shapes_parent, NULL); + rb_id_table_free(shape->edges); + shape->edges = NULL; + } + + shape->parent = NULL; + + rb_shape_set_shape_by_id(SHAPE_ID(shape), NULL); + + RB_DEBUG_COUNTER_INC(obj_imemo_shape); + break; + } + } + return TRUE; default: rb_bug("gc_sweep(): unknown data type 0x%x(%p) 0x%"PRIxVALUE, @@ -4860,7 +4916,15 @@ obj_memsize_of(VALUE obj, int use_all_types) if (!(RBASIC(obj)->flags & ROBJECT_EMBED)) { size += ROBJECT_NUMIV(obj) * sizeof(VALUE); } - break; + else { + // We can't look up the shape here because `obj_memsize_of` is used during + // the sweep phase when RGENGC_CHECK_MODE is enabled. The shape may have been + // collected, so we just want to check the ID + if (NO_CACHE_SHAPE_ID == rb_shape_get_shape_id(obj)) { + size += rb_id_table_memsize(ROBJECT(obj)->as.heap.iv_index_tbl); + } + } + break; case T_MODULE: case T_CLASS: if (RCLASS_EXT(obj)) { @@ -4873,10 +4937,6 @@ obj_memsize_of(VALUE obj, int use_all_types) if (RCLASS_CVC_TBL(obj)) { size += rb_id_table_memsize(RCLASS_CVC_TBL(obj)); } - if (RCLASS_IV_INDEX_TBL(obj)) { - // TODO: more correct value - size += st_memsize(RCLASS_IV_INDEX_TBL(obj)); - } if (RCLASS_EXT(obj)->iv_tbl) { size += st_memsize(RCLASS_EXT(obj)->iv_tbl); } @@ -7154,6 +7214,21 @@ gc_mark_imemo(rb_objspace_t *objspace, VALUE obj) const struct rb_callcache *cc = (const struct rb_callcache *)obj; // should not mark klass here gc_mark(objspace, (VALUE)vm_cc_cme(cc)); + + // Check it's an attr_(reader|writer) + if (cc->cme_ && (cc->cme_->def->type == VM_METHOD_TYPE_ATTRSET || + cc->cme_->def->type == VM_METHOD_TYPE_IVAR)) { + shape_id_t source_shape_id = vm_cc_attr_index_source_shape_id(cc); + shape_id_t dest_shape_id = vm_cc_attr_index_dest_shape_id(cc); + if (source_shape_id != INVALID_SHAPE_ID) { + rb_shape_t *shape = rb_shape_get_shape_by_id(source_shape_id); + rb_gc_mark((VALUE)shape); + } + if (dest_shape_id != INVALID_SHAPE_ID) { + rb_shape_t *shape = rb_shape_get_shape_by_id(dest_shape_id); + rb_gc_mark((VALUE)shape); + } + } } return; case imemo_constcache: @@ -7162,6 +7237,14 @@ gc_mark_imemo(rb_objspace_t *objspace, VALUE obj) gc_mark(objspace, ice->value); } return; + case imemo_shape: + { + rb_shape_t *shape = (rb_shape_t *)obj; + if (!rb_shape_root_shape_p(shape)) { + rb_gc_mark((VALUE)shape->parent); + } + } + return; #if VM_CHECK_MODE > 0 default: VM_UNREACHABLE(gc_mark_imemo); @@ -7205,6 +7288,7 @@ gc_mark_children(rb_objspace_t *objspace, VALUE obj) } gc_mark(objspace, any->as.basic.klass); + rb_gc_mark((VALUE)rb_shape_get_shape(obj)); switch (BUILTIN_TYPE(obj)) { case T_CLASS: @@ -9765,6 +9849,10 @@ gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj) GC_ASSERT(!SPECIAL_CONST_P(obj)); switch (BUILTIN_TYPE(obj)) { + case T_IMEMO: + if (IMEMO_TYPE_P(obj, imemo_shape)) { + return FALSE; + } case T_NONE: case T_NIL: case T_MOVED: @@ -9778,7 +9866,6 @@ gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj) case T_STRING: case T_OBJECT: case T_FLOAT: - case T_IMEMO: case T_ARRAY: case T_BIGNUM: case T_ICLASS: @@ -10178,6 +10265,38 @@ gc_update_values(rb_objspace_t *objspace, long n, VALUE *values) } } +static enum rb_id_table_iterator_result +check_id_table_move(VALUE value, void *data) +{ + rb_objspace_t *objspace = (rb_objspace_t *)data; + + if (gc_object_moved_p(objspace, (VALUE)value)) { + return ID_TABLE_REPLACE; + } + + return ID_TABLE_CONTINUE; +} + +static enum rb_id_table_iterator_result +update_id_table(VALUE *value, void *data, int existing) +{ + rb_objspace_t *objspace = (rb_objspace_t *)data; + + if (gc_object_moved_p(objspace, (VALUE)*value)) { + *value = rb_gc_location((VALUE)*value); + } + + return ID_TABLE_CONTINUE; +} + +static void +update_m_tbl(rb_objspace_t *objspace, struct rb_id_table *tbl) +{ + if (tbl) { + rb_id_table_foreach_values_with_replace(tbl, check_id_table_move, update_id_table, objspace); + } +} + static void gc_ref_update_imemo(rb_objspace_t *objspace, VALUE obj) { @@ -10250,24 +10369,23 @@ gc_ref_update_imemo(rb_objspace_t *objspace, VALUE obj) case imemo_tmpbuf: case imemo_callinfo: break; + case imemo_shape: + { + rb_shape_t * shape = (rb_shape_t *)obj; + if(shape->edges) { + update_m_tbl(objspace, shape->edges); + } + if (shape->parent) { + shape->parent = (rb_shape_t *)rb_gc_location((VALUE)shape->parent); + } + } + break; default: rb_bug("not reachable %d", imemo_type(obj)); break; } } -static enum rb_id_table_iterator_result -check_id_table_move(VALUE value, void *data) -{ - rb_objspace_t *objspace = (rb_objspace_t *)data; - - if (gc_object_moved_p(objspace, (VALUE)value)) { - return ID_TABLE_REPLACE; - } - - return ID_TABLE_CONTINUE; -} - /* Returns the new location of an object, if it moved. Otherwise returns * the existing location. */ VALUE @@ -10300,26 +10418,6 @@ rb_gc_location(VALUE value) return destination; } -static enum rb_id_table_iterator_result -update_id_table(VALUE *value, void *data, int existing) -{ - rb_objspace_t *objspace = (rb_objspace_t *)data; - - if (gc_object_moved_p(objspace, (VALUE)*value)) { - *value = rb_gc_location((VALUE)*value); - } - - return ID_TABLE_CONTINUE; -} - -static void -update_m_tbl(rb_objspace_t *objspace, struct rb_id_table *tbl) -{ - if (tbl) { - rb_id_table_foreach_values_with_replace(tbl, check_id_table_move, update_id_table, objspace); - } -} - static enum rb_id_table_iterator_result update_cc_tbl_i(VALUE ccs_ptr, void *data) { @@ -10407,15 +10505,6 @@ update_subclass_entries(rb_objspace_t *objspace, rb_subclass_entry_t *entry) } } -static int -update_iv_index_tbl_i(st_data_t key, st_data_t value, st_data_t arg) -{ - rb_objspace_t *objspace = (rb_objspace_t *)arg; - struct rb_iv_index_tbl_entry *ent = (struct rb_iv_index_tbl_entry *)value; - UPDATE_IF_MOVED(objspace, ent->class_value); - return ST_CONTINUE; -} - static void update_class_ext(rb_objspace_t *objspace, rb_classext_t *ext) { @@ -10423,11 +10512,6 @@ update_class_ext(rb_objspace_t *objspace, rb_classext_t *ext) UPDATE_IF_MOVED(objspace, ext->includer); UPDATE_IF_MOVED(objspace, ext->refined_class); update_subclass_entries(objspace, ext->subclasses); - - // ext->iv_index_tbl - if (ext->iv_index_tbl) { - st_foreach(ext->iv_index_tbl, update_iv_index_tbl_i, (st_data_t)objspace); - } } static void @@ -10669,6 +10753,8 @@ gc_update_references(rb_objspace_t *objspace) struct heap_page *page = NULL; + rb_vm_update_references(vm); + for (int i = 0; i < SIZE_POOL_COUNT; i++) { bool should_set_mark_bits = TRUE; rb_size_pool_t *size_pool = &size_pools[i]; @@ -10687,7 +10773,6 @@ gc_update_references(rb_objspace_t *objspace) } } } - rb_vm_update_references(vm); rb_transient_heap_update_references(); rb_gc_update_global_tbl(); global_symbols.ids = rb_gc_location(global_symbols.ids); diff --git a/id_table.c b/id_table.c index a9a041b955..11c9e7637f 100644 --- a/id_table.c +++ b/id_table.c @@ -114,7 +114,7 @@ rb_id_table_clear(struct rb_id_table *tbl) MEMZERO(tbl->items, item_t, tbl->capa); } -size_t +MJIT_FUNC_EXPORTED size_t rb_id_table_size(const struct rb_id_table *tbl) { return (size_t)tbl->num; diff --git a/id_table.h b/id_table.h index 9d9eb5648e..f3f46489a1 100644 --- a/id_table.h +++ b/id_table.h @@ -4,8 +4,6 @@ #include #include "ruby/ruby.h" -struct rb_id_table; - /* compatible with ST_* */ enum rb_id_table_iterator_result { ID_TABLE_CONTINUE = ST_CONTINUE, diff --git a/include/ruby/internal/core/robject.h b/include/ruby/internal/core/robject.h index 7823061d8f..a40c5b00d0 100644 --- a/include/ruby/internal/core/robject.h +++ b/include/ruby/internal/core/robject.h @@ -46,7 +46,6 @@ #define ROBJECT_EMBED ROBJECT_EMBED #define ROBJECT_NUMIV ROBJECT_NUMIV #define ROBJECT_IVPTR ROBJECT_IVPTR -#define ROBJECT_IV_INDEX_TBL ROBJECT_IV_INDEX_TBL /** @endcond */ /** @@ -132,7 +131,7 @@ struct RObject { * * This is a shortcut for `RCLASS_IV_INDEX_TBL(rb_obj_class(obj))`. */ - struct st_table *iv_index_tbl; + struct rb_id_table *iv_index_tbl; } heap; #if USE_RVARGC @@ -221,4 +220,27 @@ ROBJECT_IVPTR(VALUE obj) } } +#ifndef shape_id_t +typedef uint16_t shape_id_t; +#define shape_id_t shape_id_t +#endif + +static inline shape_id_t +ROBJECT_SHAPE_ID(VALUE obj) +{ + RBIMPL_ASSERT_TYPE(obj, RUBY_T_OBJECT); + return (shape_id_t)(0xffff & (RBASIC(obj)->flags >> 16)); +} + +static inline void +ROBJECT_SET_SHAPE_ID(VALUE obj, shape_id_t shape_id) +{ + // Ractors are occupying the upper 32 bits of flags + // Object shapes are occupying the next 16 bits + // 4 bits are unused + // 12 bits are occupied by RUBY_FL (see RUBY_FL_USHIFT) + // | XXXX ractor_id | shape_id | UUUU flags | + RBASIC(obj)->flags &= 0xffffffff0000ffff; + RBASIC(obj)->flags |= ((uint32_t)(shape_id) << 16); +} #endif /* RBIMPL_ROBJECT_H */ diff --git a/include/ruby/internal/fl_type.h b/include/ruby/internal/fl_type.h index c51bd2e9d9..7383426b23 100644 --- a/include/ruby/internal/fl_type.h +++ b/include/ruby/internal/fl_type.h @@ -941,21 +941,8 @@ RB_OBJ_FREEZE_RAW(VALUE obj) RB_FL_SET_RAW(obj, RUBY_FL_FREEZE); } -/** - * Prevents further modifications to the given object. ::rb_eFrozenError shall - * be raised if modification is attempted. - * - * @param[out] x Object in question. - */ -static inline void -rb_obj_freeze_inline(VALUE x) -{ - if (RB_FL_ABLE(x)) { - RB_OBJ_FREEZE_RAW(x); - if (RBASIC_CLASS(x) && !(RBASIC(x)->flags & RUBY_FL_SINGLETON)) { - rb_freeze_singleton_class(x); - } - } -} +RUBY_SYMBOL_EXPORT_BEGIN +void rb_obj_freeze_inline(VALUE obj); +RUBY_SYMBOL_EXPORT_END #endif /* RBIMPL_FL_TYPE_H */ diff --git a/inits.c b/inits.c index 22ba6d5a8c..789b22bc2f 100644 --- a/inits.c +++ b/inits.c @@ -77,6 +77,7 @@ rb_call_inits(void) CALL(vm_stack_canary); CALL(ast); CALL(gc_stress); + CALL(shape); // enable builtin loading CALL(builtin); diff --git a/internal.h b/internal.h index 0740ae99e5..695c9cfb7e 100644 --- a/internal.h +++ b/internal.h @@ -48,9 +48,6 @@ #undef RHASH_TBL #undef RHASH_EMPTY_P -/* internal/object.h */ -#undef ROBJECT_IV_INDEX_TBL - /* internal/struct.h */ #undef RSTRUCT_LEN #undef RSTRUCT_PTR diff --git a/internal/class.h b/internal/class.h index ae680564a6..7906c339ea 100644 --- a/internal/class.h +++ b/internal/class.h @@ -26,9 +26,9 @@ struct rb_subclass_entry { }; struct rb_iv_index_tbl_entry { - uint32_t index; - rb_serial_t class_serial; - VALUE class_value; + uint32_t idx; + uint16_t source_shape_id; + uint16_t dest_shape_id; }; struct rb_cvar_class_tbl_entry { @@ -38,7 +38,6 @@ struct rb_cvar_class_tbl_entry { }; struct rb_classext_struct { - struct st_table *iv_index_tbl; // ID -> struct rb_iv_index_tbl_entry struct st_table *iv_tbl; #if SIZEOF_SERIAL_T == SIZEOF_VALUE /* otherwise m_tbl is in struct RClass */ struct rb_id_table *m_tbl; @@ -64,6 +63,8 @@ struct rb_classext_struct { const VALUE refined_class; rb_alloc_func_t allocator; const VALUE includer; + uint32_t max_iv_count; + uint16_t shape_id; }; struct RClass { @@ -102,7 +103,6 @@ typedef struct rb_classext_struct rb_classext_t; #define RCLASS_CALLABLE_M_TBL(c) (RCLASS_EXT(c)->callable_m_tbl) #define RCLASS_CC_TBL(c) (RCLASS_EXT(c)->cc_tbl) #define RCLASS_CVC_TBL(c) (RCLASS_EXT(c)->cvc_tbl) -#define RCLASS_IV_INDEX_TBL(c) (RCLASS_EXT(c)->iv_index_tbl) #define RCLASS_ORIGIN(c) (RCLASS_EXT(c)->origin_) #define RCLASS_REFINED_CLASS(c) (RCLASS_EXT(c)->refined_class) #if SIZEOF_SERIAL_T == SIZEOF_VALUE @@ -127,6 +127,11 @@ typedef struct rb_classext_struct rb_classext_t; #define RCLASS_SUPERCLASSES_INCLUDE_SELF FL_USER2 #define RICLASS_ORIGIN_SHARED_MTBL FL_USER3 +#ifndef shape_id_t +typedef uint16_t shape_id_t; +#define shape_id_t shape_id_t +#endif + /* class.c */ void rb_class_subclass_add(VALUE super, VALUE klass); void rb_class_remove_from_super_subclasses(VALUE); diff --git a/internal/imemo.h b/internal/imemo.h index 91b524e0a6..976c14581a 100644 --- a/internal/imemo.h +++ b/internal/imemo.h @@ -14,6 +14,7 @@ #include "internal/gc.h" /* for RB_OBJ_WRITE */ #include "ruby/internal/stdbool.h" /* for bool */ #include "ruby/ruby.h" /* for rb_block_call_func_t */ +#include "ruby/internal/rgengc.h" #ifndef IMEMO_DEBUG # define IMEMO_DEBUG 0 @@ -45,6 +46,7 @@ enum imemo_type { imemo_callinfo = 11, imemo_callcache = 12, imemo_constcache = 13, + imemo_shape = 14, }; /* CREF (Class REFerence) is defined in method.h */ diff --git a/internal/object.h b/internal/object.h index 88f3a44bc6..a93282a8bc 100644 --- a/internal/object.h +++ b/internal/object.h @@ -10,10 +10,7 @@ */ #include "ruby/ruby.h" /* for VALUE */ #include "internal/class.h" /* for RCLASS_IV_INDEX_TBL */ - -#ifdef ROBJECT_IV_INDEX_TBL -# undef ROBJECT_IV_INDEX_TBL -#endif +#include "internal/variable.h" /* for shapes */ /* object.c */ VALUE rb_class_search_ancestor(VALUE klass, VALUE super); @@ -26,7 +23,6 @@ int rb_bool_expected(VALUE, const char *, int raise); static inline void RBASIC_CLEAR_CLASS(VALUE obj); static inline void RBASIC_SET_CLASS_RAW(VALUE obj, VALUE klass); static inline void RBASIC_SET_CLASS(VALUE obj, VALUE klass); -static inline struct st_table *ROBJECT_IV_INDEX_TBL_inline(VALUE obj); RUBY_SYMBOL_EXPORT_BEGIN /* object.c (export) */ @@ -64,20 +60,4 @@ RBASIC_SET_CLASS(VALUE obj, VALUE klass) RBASIC_SET_CLASS_RAW(obj, klass); RB_OBJ_WRITTEN(obj, oldv, klass); } - -RBIMPL_ATTR_PURE() -static inline struct st_table * -ROBJECT_IV_INDEX_TBL_inline(VALUE obj) -{ - if (RB_FL_ANY_RAW(obj, ROBJECT_EMBED)) { - VALUE klass = rb_obj_class(obj); - return RCLASS_IV_INDEX_TBL(klass); - } - else { - const struct RObject *const ptr = ROBJECT(obj); - return ptr->as.heap.iv_index_tbl; - } -} -#define ROBJECT_IV_INDEX_TBL ROBJECT_IV_INDEX_TBL_inline - #endif /* INTERNAL_OBJECT_H */ diff --git a/internal/variable.h b/internal/variable.h index 1a19e8964b..1c6beadb5d 100644 --- a/internal/variable.h +++ b/internal/variable.h @@ -37,6 +37,10 @@ static inline void ROBJ_TRANSIENT_SET(VALUE obj); static inline void ROBJ_TRANSIENT_UNSET(VALUE obj); uint32_t rb_obj_ensure_iv_index_mapping(VALUE obj, ID id); +struct gen_ivtbl; +int rb_gen_ivtbl_get(VALUE obj, ID id, struct gen_ivtbl **ivtbl); + +shape_id_t rb_generic_shape_id(VALUE obj); RUBY_SYMBOL_EXPORT_BEGIN /* variable.c (export) */ void rb_mark_generic_ivar(VALUE); @@ -52,6 +56,8 @@ VALUE rb_gvar_set(ID, VALUE); VALUE rb_gvar_defined(ID); void rb_const_warn_if_deprecated(const rb_const_entry_t *, VALUE, ID); void rb_init_iv_list(VALUE obj); +void rb_ensure_iv_list_size(VALUE obj, uint32_t len, uint32_t newsize); +struct gen_ivtbl * rb_ensure_generic_iv_list_size(VALUE obj, uint32_t newsize); MJIT_SYMBOL_EXPORT_END static inline bool diff --git a/iseq.c b/iseq.c index f17a2d49b6..a19ddb5d73 100644 --- a/iseq.c +++ b/iseq.c @@ -277,13 +277,15 @@ rb_iseq_each_value(const rb_iseq_t *iseq, iseq_value_itr_t * func, void *data) // IVC entries for (unsigned int i = 0; i < body->ivc_size; i++, is_entries++) { IVC ivc = (IVC)is_entries; - if (ivc->entry) { - RUBY_ASSERT(!RB_TYPE_P(ivc->entry->class_value, T_NONE)); - - VALUE nv = func(data, ivc->entry->class_value); - if (ivc->entry->class_value != nv) { - ivc->entry->class_value = nv; - } + shape_id_t source_shape_id = vm_ic_attr_index_source_shape_id(ivc); + shape_id_t dest_shape_id = vm_ic_attr_index_dest_shape_id(ivc); + if (source_shape_id != INVALID_SHAPE_ID) { + rb_shape_t *shape = rb_shape_get_shape_by_id(source_shape_id); + func(data, (VALUE)shape); + } + if (dest_shape_id != INVALID_SHAPE_ID) { + rb_shape_t *shape = rb_shape_get_shape_by_id(dest_shape_id); + func(data, (VALUE)shape); } } diff --git a/marshal.c b/marshal.c index 1eeebf7729..650a6c853d 100644 --- a/marshal.c +++ b/marshal.c @@ -39,6 +39,7 @@ #include "ruby/st.h" #include "ruby/util.h" #include "builtin.h" +#include "shape.h" #define BITSPERSHORT (2*CHAR_BIT) #define SHORTMASK ((1<num_ivar) { - rb_raise(rb_eRuntimeError, "instance variable added to %"PRIsVALUE" instance", - CLASS_OF(arg->obj)); - } --ivarg->num_ivar; w_symbol(ID2SYM(id), arg->arg); w_object(value, arg->arg, arg->limit); @@ -720,9 +717,14 @@ has_ivars(VALUE obj, VALUE encname, VALUE *ivobj) static void w_ivar_each(VALUE obj, st_index_t num, struct dump_call_arg *arg) { + shape_id_t shape_id = rb_shape_get_shape_id(arg->obj); struct w_ivar_arg ivarg = {arg, num}; if (!num) return; rb_ivar_foreach(obj, w_obj_each, (st_data_t)&ivarg); + if (shape_id != rb_shape_get_shape_id(arg->obj)) { + rb_raise(rb_eRuntimeError, "instance variable added to %"PRIsVALUE" instance", + CLASS_OF(arg->obj)); + } if (ivarg.num_ivar) { rb_raise(rb_eRuntimeError, "instance variable removed from %"PRIsVALUE" instance", CLASS_OF(arg->obj)); diff --git a/misc/lldb_cruby.py b/misc/lldb_cruby.py index c38b9c62a0..997db16134 100755 --- a/misc/lldb_cruby.py +++ b/misc/lldb_cruby.py @@ -417,9 +417,13 @@ def lldb_inspect(debugger, target, result, val): elif flType == RUBY_T_IMEMO: # I'm not sure how to get IMEMO_MASK out of lldb. It's not in globals() imemo_type = (flags >> RUBY_FL_USHIFT) & 0x0F # IMEMO_MASK + print("T_IMEMO: ", file=result) append_command_output(debugger, "p (enum imemo_type) %d" % imemo_type, result) - append_command_output(debugger, "p *(struct MEMO *) %0#x" % val.GetValueAsUnsigned(), result) + if imemo_type == imemo_shape: + append_command_output(debugger, "p *(rb_shape_t *) %0#x" % val.GetValueAsUnsigned(), result) + else: + append_command_output(debugger, "p *(struct MEMO *) %0#x" % val.GetValueAsUnsigned(), result) elif flType == RUBY_T_STRUCT: tRTypedData = target.FindFirstType("struct RStruct").GetPointerType() val = val.Cast(tRTypedData) diff --git a/mjit_compile.c b/mjit_compile.c index 2c7996c258..8a955f778c 100644 --- a/mjit_compile.c +++ b/mjit_compile.c @@ -352,7 +352,7 @@ mjit_compile_body(FILE *f, const rb_iseq_t *iseq, struct compile_status *status) // Generate merged ivar guards first if needed if (!status->compile_info->disable_ivar_cache && status->merge_ivar_guards_p) { - fprintf(f, " if (UNLIKELY(!(RB_TYPE_P(GET_SELF(), T_OBJECT) && (rb_serial_t)%"PRI_SERIALT_PREFIX"u == RCLASS_SERIAL(RBASIC(GET_SELF())->klass) &&", status->ivar_serial); + fprintf(f, " if (UNLIKELY(!(RB_TYPE_P(GET_SELF(), T_OBJECT) && (rb_serial_t)%"PRI_SERIALT_PREFIX"u == rb_shape_get_shape_id(GET_SELF()) &&", status->ivar_serial); #if USE_RVARGC fprintf(f, "%"PRIuSIZE" < ROBJECT_NUMIV(GET_SELF())", status->max_ivar_index); // index < ROBJECT_NUMIV(obj) #else @@ -467,17 +467,17 @@ init_ivar_compile_status(const struct rb_iseq_constant_body *body, struct compil if (insn == BIN(getinstancevariable) || insn == BIN(setinstancevariable)) { IVC ic = (IVC)body->iseq_encoded[pos+2]; IVC ic_copy = &(status->is_entries + ((union iseq_inline_storage_entry *)ic - body->is_entries))->iv_cache; - if (ic_copy->entry) { // Only initialized (ic_serial > 0) IVCs are optimized + if (vm_ic_attr_index_p(ic_copy)) { // Only initialized (ic_serial > 0) IVCs are optimized num_ivars++; - if (status->max_ivar_index < ic_copy->entry->index) { - status->max_ivar_index = ic_copy->entry->index; + if (status->max_ivar_index < vm_ic_attr_index(ic_copy)) { + status->max_ivar_index = vm_ic_attr_index(ic_copy); } if (status->ivar_serial == 0) { - status->ivar_serial = ic_copy->entry->class_serial; + status->ivar_serial = vm_ic_attr_index_source_shape_id(ic_copy); } - else if (status->ivar_serial != ic_copy->entry->class_serial) { + else if (status->ivar_serial != vm_ic_attr_index_source_shape_id(ic_copy)) { // Multiple classes have used this ISeq. Give up assuming one serial. status->merge_ivar_guards_p = false; return; diff --git a/object.c b/object.c index d1743b554b..2a4512630c 100644 --- a/object.c +++ b/object.c @@ -39,6 +39,7 @@ #include "ruby/util.h" #include "ruby/assert.h" #include "builtin.h" +#include "shape.h" /*! * \addtogroup object @@ -271,22 +272,59 @@ rb_obj_copy_ivar(VALUE dest, VALUE obj) VALUE *src_buf = ROBJECT_IVPTR(obj); uint32_t dest_len = ROBJECT_NUMIV(dest); uint32_t src_len = ROBJECT_NUMIV(obj); - uint32_t len = dest_len < src_len ? dest_len : src_len; + uint32_t max_len = dest_len < src_len ? src_len : dest_len; - MEMCPY(dest_buf, src_buf, VALUE, len); + rb_ensure_iv_list_size(dest, dest_len, max_len); + + dest_len = ROBJECT_NUMIV(dest); + uint32_t min_len = dest_len > src_len ? src_len : dest_len; + + if (RBASIC(obj)->flags & ROBJECT_EMBED) { + src_buf = ROBJECT(obj)->as.ary; + + // embedded -> embedded + if (RBASIC(dest)->flags & ROBJECT_EMBED) { + dest_buf = ROBJECT(dest)->as.ary; + } + // embedded -> extended + else { + dest_buf = ROBJECT(dest)->as.heap.ivptr; + } + } + // extended -> extended + else { + RUBY_ASSERT(!(RBASIC(dest)->flags & ROBJECT_EMBED)); + dest_buf = ROBJECT(dest)->as.heap.ivptr; + src_buf = ROBJECT(obj)->as.heap.ivptr; + } + + MEMCPY(dest_buf, src_buf, VALUE, min_len); } static void -init_copy(VALUE dest, VALUE obj) +init_copy(VALUE dest, VALUE obj, bool preserve_frozen) { if (OBJ_FROZEN(dest)) { rb_raise(rb_eTypeError, "[bug] frozen object (%s) allocated", rb_obj_classname(dest)); } RBASIC(dest)->flags &= ~(T_MASK|FL_EXIVAR); + // Copies the shape id from obj to dest RBASIC(dest)->flags |= RBASIC(obj)->flags & (T_MASK|FL_EXIVAR); rb_copy_wb_protected_attribute(dest, obj); rb_copy_generic_ivar(dest, obj); rb_gc_copy_finalizer(dest, obj); + + rb_shape_t *shape_to_set = rb_shape_get_shape(obj); + + // If the object is frozen, the "dup"'d object will *not* be frozen, + // so we need to copy the frozen shape's parent to the new object. + if (!preserve_frozen && RB_OBJ_FROZEN((VALUE)shape_to_set)) { + shape_to_set = shape_to_set->parent; + } + + // shape ids are different + rb_shape_set_shape(dest, shape_to_set); + if (RB_TYPE_P(obj, T_OBJECT)) { rb_obj_copy_ivar(dest, obj); } @@ -386,7 +424,7 @@ mutable_obj_clone(VALUE obj, VALUE kwfreeze) rb_singleton_class_attached(singleton, clone); } - init_copy(clone, obj); + init_copy(clone, obj, true); switch (kwfreeze) { case Qnil: @@ -486,7 +524,7 @@ rb_obj_dup(VALUE obj) return obj; } dup = rb_obj_alloc(rb_obj_class(obj)); - init_copy(dup, obj); + init_copy(dup, obj, false); rb_funcall(dup, id_init_dup, 1, obj); return dup; diff --git a/shape.c b/shape.c new file mode 100644 index 0000000000..ec90ca179f --- /dev/null +++ b/shape.c @@ -0,0 +1,579 @@ +#include "vm_core.h" +#include "vm_sync.h" +#include "shape.h" +#include "internal/class.h" +#include "internal/symbol.h" + +/* + * Getters for root_shape, frozen_root_shape and no_cache_shape + */ +static rb_shape_t* +rb_shape_get_root_shape(void) { + rb_vm_t *vm = GET_VM(); + return vm->root_shape; +} + +static rb_shape_t* +rb_shape_get_frozen_root_shape(void) { + rb_vm_t *vm = GET_VM(); + return vm->frozen_root_shape; +} + +static rb_shape_t* +rb_shape_get_no_cache_shape(void) { + rb_vm_t *vm = GET_VM(); + return vm->no_cache_shape; +} + +/* + * Predicate methods for root_shape and no_cache_shape + */ +bool +rb_shape_root_shape_p(rb_shape_t* shape) { + return shape == rb_shape_get_root_shape(); +} + +bool +rb_shape_no_cache_shape_p(rb_shape_t * shape) +{ + return shape == rb_shape_get_no_cache_shape(); +} + +/* + * Shape getters + */ +rb_shape_t* +rb_shape_get_shape_by_id(shape_id_t shape_id) +{ + RUBY_ASSERT(shape_id != INVALID_SHAPE_ID); + + rb_vm_t *vm = GET_VM(); + rb_shape_t *shape = vm->shape_list[shape_id]; + RUBY_ASSERT(IMEMO_TYPE_P(shape, imemo_shape)); + return shape; +} + +rb_shape_t* +rb_shape_get_shape_by_id_without_assertion(shape_id_t shape_id) +{ + RUBY_ASSERT(shape_id != INVALID_SHAPE_ID); + + rb_vm_t *vm = GET_VM(); + rb_shape_t *shape = vm->shape_list[shape_id]; + return shape; +} + +static inline shape_id_t +shape_get_shape_id(rb_shape_t *shape) { + return (shape_id_t)(0xffff & (shape->flags >> 16)); +} + +static inline shape_id_t +RCLASS_SHAPE_ID(VALUE obj) +{ + return RCLASS_EXT(obj)->shape_id; +} + +shape_id_t +rb_shape_get_shape_id(VALUE obj) +{ + shape_id_t shape_id = ROOT_SHAPE_ID; + + if (RB_SPECIAL_CONST_P(obj)) { + return SHAPE_ID(rb_shape_get_frozen_root_shape()); + } + + switch (BUILTIN_TYPE(obj)) { + case T_OBJECT: + return ROBJECT_SHAPE_ID(obj); + break; + case T_CLASS: + case T_MODULE: + return RCLASS_SHAPE_ID(obj); + case T_IMEMO: + if (imemo_type(obj) == imemo_shape) { + return shape_get_shape_id((rb_shape_t *)obj); + break; + } + default: + return rb_generic_shape_id(obj); + } + + RUBY_ASSERT(shape_id < MAX_SHAPE_ID); + return shape_id; +} + +rb_shape_t* +rb_shape_get_shape(VALUE obj) +{ + return rb_shape_get_shape_by_id(rb_shape_get_shape_id(obj)); +} + +enum transition_type { + SHAPE_IVAR, + SHAPE_FROZEN, +}; + +static shape_id_t +get_next_shape_id(void) +{ + rb_vm_t *vm = GET_VM(); + int next_shape_id = 0; + + /* + * Speedup for getting next shape_id by using bitmaps + * TODO: Can further optimize here by nesting more bitmaps + */ + for (int i = 0; i < 2048; i++) { + uint32_t cur_bitmap = vm->shape_bitmaps[i]; + if (~cur_bitmap) { + uint32_t copied_curbitmap = ~cur_bitmap; + + uint32_t count = 0; + while (!(copied_curbitmap & 0x1)) { + copied_curbitmap >>= 1; + count++; + } + next_shape_id = i * 32 + count; + break; + } + } + + if (next_shape_id > vm->max_shape_count) { + vm->max_shape_count = next_shape_id; + } + + return next_shape_id; +} + +static bool +rb_shape_lookup_id(rb_shape_t* shape, ID id) { + while (shape->parent) { + if (shape->edge_name == id) { + return true; + } + shape = shape->parent; + } + return false; +} + +static rb_shape_t* +get_next_shape_internal(rb_shape_t* shape, ID id, VALUE obj, enum transition_type tt) +{ + rb_shape_t *res = NULL; + RB_VM_LOCK_ENTER(); + { + // no_cache_shape should only transition to other no_cache_shapes + if(shape == rb_shape_get_no_cache_shape()) return shape; + + if (rb_shape_lookup_id(shape, id)) { + // If shape already contains the ivar that is being set, we'll return shape + res = shape; + } + else { + if (!shape->edges) { + shape->edges = rb_id_table_create(0); + } + + // Lookup the shape in edges - if there's already an edge and a corresponding shape for it, + // we can return that. Otherwise, we'll need to get a new shape + if (!rb_id_table_lookup(shape->edges, id, (VALUE *)&res) || rb_objspace_garbage_object_p((VALUE)res)) { + // In this case, the shape exists, but the shape is garbage, so we need to recreate it + if (res) { + rb_id_table_delete(shape->edges, id); + res->parent = NULL; + } + + shape_id_t next_shape_id = get_next_shape_id(); + + if (next_shape_id == MAX_SHAPE_ID) { + res = rb_shape_get_no_cache_shape(); + } + else { + RUBY_ASSERT(next_shape_id < MAX_SHAPE_ID); + rb_shape_t * new_shape = rb_shape_alloc(next_shape_id, + id, + shape); + + // Check if we should update max_iv_count on the object's class + if (BUILTIN_TYPE(obj) == T_OBJECT) { + VALUE klass = rb_obj_class(obj); + uint32_t cur_iv_count = RCLASS_EXT(klass)->max_iv_count; + uint32_t new_iv_count = new_shape->iv_count; + if (new_iv_count > cur_iv_count) { + RCLASS_EXT(klass)->max_iv_count = new_iv_count; + } + } + + rb_id_table_insert(shape->edges, id, (VALUE)new_shape); + RB_OBJ_WRITTEN((VALUE)new_shape, Qundef, (VALUE)shape); + + rb_shape_set_shape_by_id(next_shape_id, new_shape); + + if (tt == SHAPE_FROZEN) { + new_shape->iv_count--; + RB_OBJ_FREEZE_RAW((VALUE)new_shape); + } + + res = new_shape; + } + } + } + } + RB_VM_LOCK_LEAVE(); + return res; +} + +MJIT_FUNC_EXPORTED int +rb_shape_frozen_shape_p(rb_shape_t* shape) +{ + return RB_OBJ_FROZEN((VALUE)shape); +} + +void +rb_shape_transition_shape_frozen(VALUE obj) +{ + rb_shape_t* shape = rb_shape_get_shape(obj); + RUBY_ASSERT(shape); + + if (rb_shape_frozen_shape_p(shape)) { + return; + } + + rb_shape_t* next_shape; + + if (shape == rb_shape_get_root_shape()) { + switch(BUILTIN_TYPE(obj)) { + case T_OBJECT: + case T_CLASS: + case T_MODULE: + break; + default: + return; + } + next_shape = rb_shape_get_frozen_root_shape(); + } + else { + static ID id_frozen; + if (!id_frozen) { + id_frozen = rb_make_internal_id(); + } + + next_shape = get_next_shape_internal(shape, (ID)id_frozen, obj, SHAPE_FROZEN); + } + + RUBY_ASSERT(next_shape); + rb_shape_set_shape(obj, next_shape); +} + +void +rb_shape_transition_shape(VALUE obj, ID id, rb_shape_t *shape) +{ + rb_shape_t* next_shape = rb_shape_get_next(shape, obj, id); + if (shape == next_shape) { + return; + } + + if (BUILTIN_TYPE(obj) == T_OBJECT && next_shape == rb_shape_get_no_cache_shape()) { + // If the object is embedded, we need to make it extended so that + // the instance variable index table can be stored on the object. + // The "no cache shape" is a singleton and is allowed to be shared + // among objects, so it cannot store instance variable index information. + // Therefore we need to store the iv index table on the object itself. + // Embedded objects don't have the room for an iv index table, so + // we'll force it to be extended + uint32_t num_iv = ROBJECT_NUMIV(obj); + // Make the object extended + rb_ensure_iv_list_size(obj, num_iv, num_iv + 1); + RUBY_ASSERT(!(RBASIC(obj)->flags & ROBJECT_EMBED)); + ROBJECT(obj)->as.heap.iv_index_tbl = rb_shape_generate_iv_table(shape); + } + + RUBY_ASSERT(!rb_objspace_garbage_object_p((VALUE)next_shape)); + rb_shape_set_shape(obj, next_shape); +} + +rb_shape_t* +rb_shape_get_next(rb_shape_t* shape, VALUE obj, ID id) +{ + return get_next_shape_internal(shape, id, obj, SHAPE_IVAR); +} + +int +rb_shape_get_iv_index(rb_shape_t * shape, ID id, VALUE *value) { + size_t depth = 0; + int counting = FALSE; + while (shape->parent) { + if (counting) { + depth++; + } + else { + counting = (shape->edge_name == id); + } + shape = shape->parent; + } + if (counting) { + *value = depth; + } + return counting; +} + +static rb_shape_t * +shape_alloc(void) +{ + rb_shape_t *shape = (rb_shape_t *)rb_imemo_new(imemo_shape, 0, 0, 0, 0); + FL_SET_RAW((VALUE)shape, RUBY_FL_SHAREABLE); + return shape; +} + +rb_shape_t * +rb_shape_alloc(shape_id_t shape_id, ID edge_name, rb_shape_t * parent) +{ + rb_shape_t * shape = shape_alloc(); + rb_shape_set_shape_id((VALUE)shape, shape_id); + + shape->edge_name = edge_name; + shape->iv_count = parent ? parent->iv_count + 1 : 0; + + RB_OBJ_WRITE(shape, &shape->parent, parent); + + RUBY_ASSERT(!parent || IMEMO_TYPE_P(parent, imemo_shape)); + + return shape; +} + +MJIT_FUNC_EXPORTED struct rb_id_table * +rb_shape_generate_iv_table(rb_shape_t* shape) { + if (rb_shape_frozen_shape_p(shape)) { + return rb_shape_generate_iv_table(shape->parent); + } + + struct rb_id_table *iv_table = rb_id_table_create(0); + uint32_t index = 0; + while (shape->parent) { + rb_id_table_insert(iv_table, shape->edge_name, shape->iv_count - (VALUE)index - 1); + index++; + shape = shape->parent; + } + + return iv_table; +} + +MJIT_FUNC_EXPORTED void +rb_shape_set_shape(VALUE obj, rb_shape_t* shape) +{ + RUBY_ASSERT(IMEMO_TYPE_P(shape, imemo_shape)); + if(rb_shape_set_shape_id(obj, shape_get_shape_id(shape))) { + RB_OBJ_WRITTEN(obj, Qundef, (VALUE)shape); + } +} + +static void +rb_shape_set_shape_in_bitmap(shape_id_t shape_id) { + uint16_t bitmap_index = shape_id >> 5; + uint32_t mask = (1 << (shape_id & 31)); + GET_VM()->shape_bitmaps[bitmap_index] |= mask; +} + +static void +rb_shape_unset_shape_in_bitmap(shape_id_t shape_id) { + uint16_t bitmap_index = shape_id >> 5; + uint32_t mask = (1 << (shape_id & 31)); + GET_VM()->shape_bitmaps[bitmap_index] &= ~mask; +} + +void +rb_shape_set_shape_by_id(shape_id_t shape_id, rb_shape_t *shape) +{ + rb_vm_t *vm = GET_VM(); + + RUBY_ASSERT(shape == NULL || IMEMO_TYPE_P(shape, imemo_shape)); + + if (shape == NULL) { + rb_shape_unset_shape_in_bitmap(shape_id); + } + else { + rb_shape_set_shape_in_bitmap(shape_id); + } + + vm->shape_list[shape_id] = shape; +} + +VALUE rb_cShape; + +static void +shape_mark(void *ptr) +{ + rb_gc_mark((VALUE)ptr); +} + +/* + * Exposing Shape to Ruby via RubyVM.debug_shape + */ +static const rb_data_type_t shape_data_type = { + "Shape", + {shape_mark, NULL, NULL,}, + 0, 0, RUBY_TYPED_FREE_IMMEDIATELY|RUBY_TYPED_WB_PROTECTED +}; + +static VALUE +rb_shape_id(VALUE self) { + rb_shape_t * shape; + TypedData_Get_Struct(self, rb_shape_t, &shape_data_type, shape); + return INT2NUM(SHAPE_ID(shape)); +} + +static VALUE +rb_shape_parent_id(VALUE self) +{ + rb_shape_t * shape; + TypedData_Get_Struct(self, rb_shape_t, &shape_data_type, shape); + return INT2NUM(SHAPE_ID(shape->parent)); +} + +static VALUE parse_key(ID key) { + if ((key & RUBY_ID_INTERNAL) == RUBY_ID_INTERNAL) { + return LONG2NUM(key); + } else { + return ID2SYM(key); + } +} + +static VALUE +rb_shape_t_to_rb_cShape(rb_shape_t *shape) { + union { const rb_shape_t *in; void *out; } deconst; + VALUE res; + deconst.in = shape; + res = TypedData_Wrap_Struct(rb_cShape, &shape_data_type, deconst.out); + RB_OBJ_WRITTEN(res, Qundef, shape); + + return res; +} + +static enum rb_id_table_iterator_result rb_edges_to_hash(ID key, VALUE value, void *ref) +{ + rb_hash_aset(*(VALUE *)ref, parse_key(key), rb_shape_t_to_rb_cShape((rb_shape_t*)value)); + return ID_TABLE_CONTINUE; +} + +static VALUE +rb_shape_edges(VALUE self) +{ + rb_shape_t* shape; + TypedData_Get_Struct(self, rb_shape_t, &shape_data_type, shape); + + VALUE hash = rb_hash_new(); + + if (shape->edges) { + rb_id_table_foreach(shape->edges, rb_edges_to_hash, &hash); + } + + return hash; +} + +static VALUE +rb_shape_export_depth(VALUE self) +{ + rb_shape_t* shape; + TypedData_Get_Struct(self, rb_shape_t, &shape_data_type, shape); + + unsigned int depth = 0; + while (shape->parent) { + depth++; + shape = shape->parent; + } + return INT2NUM(depth); +} + +static VALUE +rb_shape_parent(VALUE self) +{ + rb_shape_t * shape; + TypedData_Get_Struct(self, rb_shape_t, &shape_data_type, shape); + return rb_shape_t_to_rb_cShape(shape->parent); +} + +VALUE rb_shape_debug_shape(VALUE self, VALUE obj) { + return rb_shape_t_to_rb_cShape(rb_shape_get_shape(obj)); +} + +VALUE rb_shape_debug_root_shape(VALUE self) { + return rb_shape_t_to_rb_cShape(rb_shape_get_root_shape()); +} + +VALUE rb_shape_debug_frozen_root_shape(VALUE self) { + return rb_shape_t_to_rb_cShape(rb_shape_get_frozen_root_shape()); +} + +VALUE rb_obj_shape(rb_shape_t* shape); + +static enum rb_id_table_iterator_result collect_keys_and_values(ID key, VALUE value, void *ref) +{ + rb_hash_aset(*(VALUE *)ref, parse_key(key), rb_obj_shape((rb_shape_t*)value)); + return ID_TABLE_CONTINUE; +} + +static VALUE edges(struct rb_id_table* edges) +{ + VALUE hash = rb_hash_new(); + if (edges) + rb_id_table_foreach(edges, collect_keys_and_values, &hash); + return hash; +} + +VALUE rb_obj_shape(rb_shape_t* shape) { + VALUE rb_shape = rb_hash_new(); + + rb_hash_aset(rb_shape, ID2SYM(rb_intern("id")), INT2NUM(SHAPE_ID(shape))); + rb_hash_aset(rb_shape, ID2SYM(rb_intern("edges")), edges(shape->edges)); + + if (shape == rb_shape_get_root_shape()) { + rb_hash_aset(rb_shape, ID2SYM(rb_intern("parent_id")), INT2NUM(ROOT_SHAPE_ID)); + } + else { + rb_hash_aset(rb_shape, ID2SYM(rb_intern("parent_id")), INT2NUM(SHAPE_ID(shape->parent))); + } + + rb_hash_aset(rb_shape, ID2SYM(rb_intern("edge_name")), rb_id2str(shape->edge_name)); + return rb_shape; +} + +static VALUE shape_transition_tree(VALUE self) { + return rb_obj_shape(rb_shape_get_root_shape()); +} + +static VALUE shape_count(VALUE self) { + int shape_count = 0; + for(int i=0; imax_shape_count); +} + +void +Init_shape(void) +{ + rb_cShape = rb_define_class_under(rb_cRubyVM, "Shape", rb_cObject); + rb_undef_alloc_func(rb_cShape); + + rb_define_method(rb_cShape, "parent_id", rb_shape_parent_id, 0); + rb_define_method(rb_cShape, "parent", rb_shape_parent, 0); + rb_define_method(rb_cShape, "edges", rb_shape_edges, 0); + rb_define_method(rb_cShape, "depth", rb_shape_export_depth, 0); + rb_define_method(rb_cShape, "id", rb_shape_id, 0); + + rb_define_module_function(rb_cRubyVM, "debug_shape_transition_tree", shape_transition_tree, 0); + rb_define_module_function(rb_cRubyVM, "debug_shape_count", shape_count, 0); + rb_define_singleton_method(rb_cRubyVM, "debug_shape", rb_shape_debug_shape, 1); + rb_define_singleton_method(rb_cRubyVM, "debug_max_shape_count", shape_max_shape_count, 0); + rb_define_singleton_method(rb_cRubyVM, "debug_root_shape", rb_shape_debug_root_shape, 0); + rb_define_singleton_method(rb_cRubyVM, "debug_frozen_root_shape", rb_shape_debug_frozen_root_shape, 0); +} diff --git a/shape.h b/shape.h new file mode 100644 index 0000000000..4ce6bbf513 --- /dev/null +++ b/shape.h @@ -0,0 +1,54 @@ +#define USE_SHAPE_CACHE_P (SIZEOF_UINT64_T == SIZEOF_VALUE) + +#ifndef shape_id_t +typedef uint16_t shape_id_t; +#define shape_id_t shape_id_t +#endif + +#ifndef rb_shape +struct rb_shape { + VALUE flags; // Shape ID and frozen status encoded within flags + struct rb_shape * parent; // Pointer to the parent + struct rb_id_table * edges; // id_table from ID (ivar) to next shape + ID edge_name; // ID (ivar) for transition from parent to rb_shape + uint32_t iv_count; +}; +#endif + +#ifndef rb_shape_t +typedef struct rb_shape rb_shape_t; +#define rb_shape_t rb_shape_t +#endif + +# define MAX_SHAPE_ID 0xFFFE +# define NO_CACHE_SHAPE_ID (0x2) +# define INVALID_SHAPE_ID (MAX_SHAPE_ID + 1) +# define ROOT_SHAPE_ID 0x0 +# define FROZEN_ROOT_SHAPE_ID 0x1 + +#define SHAPE_ID(shape) rb_shape_get_shape_id((VALUE)shape) + +bool rb_shape_root_shape_p(rb_shape_t* shape); + +rb_shape_t* rb_shape_get_shape_by_id_without_assertion(shape_id_t shape_id); + +MJIT_SYMBOL_EXPORT_BEGIN +bool rb_shape_no_cache_shape_p(rb_shape_t * shape); +rb_shape_t* rb_shape_get_shape_by_id(shape_id_t shape_id); +void rb_shape_set_shape(VALUE obj, rb_shape_t* shape); +shape_id_t rb_shape_get_shape_id(VALUE obj); +rb_shape_t* rb_shape_get_shape(VALUE obj); +int rb_shape_frozen_shape_p(rb_shape_t* shape); +void rb_shape_transition_shape_frozen(VALUE obj); +void rb_shape_transition_shape(VALUE obj, ID id, rb_shape_t *shape); +rb_shape_t* rb_shape_get_next(rb_shape_t* shape, VALUE obj, ID id); +int rb_shape_get_iv_index(rb_shape_t * shape, ID id, VALUE * value); +MJIT_SYMBOL_EXPORT_END + +rb_shape_t * rb_shape_alloc(shape_id_t shape_id, ID edge_name, rb_shape_t * parent); +struct rb_id_table * rb_shape_generate_iv_table(rb_shape_t* shape); + +bool rb_shape_set_shape_id(VALUE obj, shape_id_t shape_id); +void rb_shape_set_shape_by_id(shape_id_t, rb_shape_t *); + +VALUE rb_obj_debug_shape(VALUE self, VALUE obj); diff --git a/spec/ruby/library/objectspace/reachable_objects_from_spec.rb b/spec/ruby/library/objectspace/reachable_objects_from_spec.rb index 7e70bc8569..61ebc9f93d 100644 --- a/spec/ruby/library/objectspace/reachable_objects_from_spec.rb +++ b/spec/ruby/library/objectspace/reachable_objects_from_spec.rb @@ -17,7 +17,7 @@ it "enumerates objects directly reachable from a given object" do ObjectSpace.reachable_objects_from(['a', 'b', 'c']).should include(Array, 'a', 'b', 'c') - ObjectSpace.reachable_objects_from(Object.new).should == [Object] + ObjectSpace.reachable_objects_from(Object.new).should include(Object) end it "finds an object stored in an Array" do diff --git a/test/-ext-/marshal/test_internal_ivar.rb b/test/-ext-/marshal/test_internal_ivar.rb index a32138f6e8..9359c7f113 100644 --- a/test/-ext-/marshal/test_internal_ivar.rb +++ b/test/-ext-/marshal/test_internal_ivar.rb @@ -7,6 +7,7 @@ module Bug end module Bug::Marshal class TestInternalIVar < Test::Unit::TestCase def test_marshal + pend "We don't support IVs with ID of 0" v = InternalIVar.new("hello", "world", "bye") assert_equal("hello", v.normal) assert_equal("world", v.internal) diff --git a/test/objspace/test_objspace.rb b/test/objspace/test_objspace.rb index 3b90319858..4db05b5945 100644 --- a/test/objspace/test_objspace.rb +++ b/test/objspace/test_objspace.rb @@ -116,12 +116,19 @@ def test_reachable_objects_from opts = %w[--disable-gem --disable=frozen-string-literal -robjspace] assert_separately opts, "#{<<-"begin;"}\n#{<<-'end;'}" begin; + def assert_reachable_object_as_expected(expectation, reachable_objects_from_array) + reachable_objects = ObjectSpace.reachable_objects_from(reachable_objects_from_array) + imemo_reachable, other_reachable = reachable_objects.partition { _1.inspect =~ /IMEMO/ } + assert_equal(imemo_reachable.size, 1) + assert_match(/IMEMO/, imemo_reachable.first.inspect) + assert_equal(expectation, other_reachable) + end + assert_equal(nil, ObjectSpace.reachable_objects_from(nil)) - assert_equal([Array, 'a', 'b', 'c'], ObjectSpace.reachable_objects_from(['a', 'b', 'c'])) - - assert_equal([Array, 'a', 'a', 'a'], ObjectSpace.reachable_objects_from(['a', 'a', 'a'])) - assert_equal([Array, 'a', 'a'], ObjectSpace.reachable_objects_from(['a', v = 'a', v])) - assert_equal([Array, 'a'], ObjectSpace.reachable_objects_from([v = 'a', v, v])) + assert_reachable_object_as_expected([Array, 'a', 'b', 'c'], ['a', 'b', 'c']) + assert_reachable_object_as_expected([Array, 'a', 'a', 'a'], ['a', 'a', 'a']) + assert_reachable_object_as_expected([Array, 'a', 'a'], ['a', v = 'a', v]) + assert_reachable_object_as_expected([Array, 'a'], [v = 'a', v, v]) long_ary = Array.new(1_000){''} max = 0 @@ -650,7 +657,7 @@ def test_name_error_message begin bar rescue => err - _, m = ObjectSpace.reachable_objects_from(err) + _, _, m = ObjectSpace.reachable_objects_from(err) end assert_equal(m, m.clone) end diff --git a/test/ruby/test_mjit.rb b/test/ruby/test_mjit.rb index 02be88aa32..68121c8bda 100644 --- a/test/ruby/test_mjit.rb +++ b/test/ruby/test_mjit.rb @@ -831,7 +831,7 @@ def test(obj, recursive: nil) end def test_inlined_exivar - assert_eval_with_jit("#{<<~"begin;"}\n#{<<~"end;"}", stdout: "aaa", success_count: 3, recompile_count: 1, min_calls: 2) + assert_eval_with_jit("#{<<~"begin;"}\n#{<<~"end;"}", stdout: "aaa", success_count: 4, recompile_count: 2, min_calls: 2) begin; class Foo < Hash def initialize @@ -850,7 +850,7 @@ def bar end def test_inlined_undefined_ivar - assert_eval_with_jit("#{<<~"begin;"}\n#{<<~"end;"}", stdout: "bbb", success_count: 3, min_calls: 3) + assert_eval_with_jit("#{<<~"begin;"}\n#{<<~"end;"}", stdout: "bbb", success_count: 2, min_calls: 2) begin; class Foo def initialize diff --git a/test/ruby/test_shapes.rb b/test/ruby/test_shapes.rb new file mode 100644 index 0000000000..83976b792b --- /dev/null +++ b/test/ruby/test_shapes.rb @@ -0,0 +1,139 @@ +# frozen_string_literal: false +require 'test/unit' + +# These test the functionality of object shapes +class TestShapes < Test::Unit::TestCase + class Example + def initialize + @a = 1 + end + end + + # RubyVM.debug_shape returns new instances of shape objects for + # each call. This helper method allows us to define equality for + # shapes + def assert_shape_equal(shape1, shape2) + assert_equal(shape1.id, shape2.id) + assert_equal(shape1.parent_id, shape2.parent_id) + assert_equal(shape1.depth, shape2.depth) + end + + def refute_shape_equal(shape1, shape2) + refute_equal(shape1.id, shape2.id) + end + + def test_new_obj_has_root_shape + assert_shape_equal(RubyVM.debug_root_shape, RubyVM.debug_shape(Object.new)) + end + + def test_frozen_new_obj_has_frozen_root_shape + assert_shape_equal( + RubyVM.debug_frozen_root_shape, + RubyVM.debug_shape(Object.new.freeze) + ) + end + + def test_str_has_root_shape + assert_shape_equal(RubyVM.debug_root_shape, RubyVM.debug_shape("")) + end + + def test_array_has_root_shape + assert_shape_equal(RubyVM.debug_root_shape, RubyVM.debug_shape([])) + end + + def test_hash_has_root_shape + assert_shape_equal(RubyVM.debug_root_shape, RubyVM.debug_shape({})) + end + + def test_true_has_frozen_root_shape + assert_shape_equal(RubyVM.debug_frozen_root_shape, RubyVM.debug_shape(true)) + end + + def test_nil_has_frozen_root_shape + assert_shape_equal(RubyVM.debug_frozen_root_shape, RubyVM.debug_shape(nil)) + end + + def test_basic_shape_transition + obj = Example.new + refute_equal(RubyVM.debug_root_shape, RubyVM.debug_shape(obj)) + assert_shape_equal(RubyVM.debug_root_shape.edges[:@a], RubyVM.debug_shape(obj)) + assert_equal(obj.instance_variable_get(:@a), 1) + end + + def test_different_objects_make_same_transition + obj = Example.new + obj2 = "" + obj2.instance_variable_set(:@a, 1) + assert_shape_equal(RubyVM.debug_shape(obj), RubyVM.debug_shape(obj2)) + end + + def test_duplicating_objects + obj = Example.new + obj2 = obj.dup + assert_shape_equal(RubyVM.debug_shape(obj), RubyVM.debug_shape(obj2)) + end + + def test_freezing_and_duplicating_object + obj = Object.new.freeze + obj2 = obj.dup + refute_predicate(obj2, :frozen?) + refute_equal(RubyVM.debug_shape(obj).id, RubyVM.debug_shape(obj2).id) + end + + def test_freezing_and_duplicating_object_with_ivars + obj = Example.new.freeze + obj2 = obj.dup + refute_predicate(obj2, :frozen?) + refute_shape_equal(RubyVM.debug_shape(obj), RubyVM.debug_shape(obj2)) + assert_equal(obj2.instance_variable_get(:@a), 1) + end + + def test_freezing_and_duplicating_string + str = "str".freeze + str2 = str.dup + refute_predicate(str2, :frozen?) + refute_equal(RubyVM.debug_shape(str).id, RubyVM.debug_shape(str2).id) + end + + def test_freezing_and_duplicating_string_with_ivars + str = "str" + str.instance_variable_set(:@a, 1) + str.freeze + str2 = str.dup + refute_predicate(str2, :frozen?) + refute_equal(RubyVM.debug_shape(str).id, RubyVM.debug_shape(str2).id) + assert_equal(str2.instance_variable_get(:@a), 1) + end + + def test_freezing_and_cloning_objects + obj = Object.new.freeze + obj2 = obj.clone(freeze: true) + assert_predicate(obj2, :frozen?) + assert_shape_equal(RubyVM.debug_shape(obj), RubyVM.debug_shape(obj2)) + end + + def test_freezing_and_cloning_object_with_ivars + obj = Example.new.freeze + obj2 = obj.clone(freeze: true) + assert_predicate(obj2, :frozen?) + assert_shape_equal(RubyVM.debug_shape(obj), RubyVM.debug_shape(obj2)) + assert_equal(obj2.instance_variable_get(:@a), 1) + end + + def test_freezing_and_cloning_string + str = "str".freeze + str2 = str.clone(freeze: true) + assert_predicate(str2, :frozen?) + assert_shape_equal(RubyVM.debug_shape(str), RubyVM.debug_shape(str2)) + end + + def test_freezing_and_cloning_string_with_ivars + str = "str" + str.instance_variable_set(:@a, 1) + str.freeze + str2 = str.clone(freeze: true) + assert_predicate(str2, :frozen?) + assert_shape_equal(RubyVM.debug_shape(str), RubyVM.debug_shape(str2)) + assert_equal(str2.instance_variable_get(:@a), 1) + end +end diff --git a/tool/ruby_vm/views/_mjit_compile_ivar.erb b/tool/ruby_vm/views/_mjit_compile_ivar.erb index 1425b3b055..d1a8713407 100644 --- a/tool/ruby_vm/views/_mjit_compile_ivar.erb +++ b/tool/ruby_vm/views/_mjit_compile_ivar.erb @@ -16,18 +16,18 @@ % # compiler: Use copied IVC to avoid race condition IVC ic_copy = &(status->is_entries + ((union iseq_inline_storage_entry *)ic - body->is_entries))->iv_cache; % - if (!status->compile_info->disable_ivar_cache && ic_copy->entry) { // Only ic_copy is enabled. + if (!status->compile_info->disable_ivar_cache && vm_ic_attr_index_p(ic_copy)) { // Only ic_copy is enabled. % # JIT: optimize away motion of sp and pc. This path does not call rb_warning() and so it's always leaf and not `handles_sp`. % # <%= render 'mjit_compile_pc_and_sp', locals: { insn: insn } -%> % % # JIT: prepare vm_getivar/vm_setivar arguments and variables fprintf(f, "{\n"); fprintf(f, " VALUE obj = GET_SELF();\n"); - fprintf(f, " const uint32_t index = %u;\n", (ic_copy->entry->index)); + fprintf(f, " const uint32_t index = %u;\n", vm_ic_attr_index(ic_copy)); if (status->merge_ivar_guards_p) { % # JIT: Access ivar without checking these VM_ASSERTed prerequisites as we checked them in the beginning of `mjit_compile_body` fprintf(f, " VM_ASSERT(RB_TYPE_P(obj, T_OBJECT));\n"); - fprintf(f, " VM_ASSERT((rb_serial_t)%"PRI_SERIALT_PREFIX"u == RCLASS_SERIAL(RBASIC(obj)->klass));\n", ic_copy->entry->class_serial); + fprintf(f, " VM_ASSERT((shape_id_t)%hu == ROBJECT_SHAPE_ID(obj));\n", vm_ic_attr_shape_id(ic_copy)); fprintf(f, " VM_ASSERT(index < ROBJECT_NUMIV(obj));\n"); % if insn.name == 'setinstancevariable' #if USE_RVARGC @@ -53,16 +53,29 @@ %end } else { - fprintf(f, " const rb_serial_t ic_serial = (rb_serial_t)%"PRI_SERIALT_PREFIX"u;\n", ic_copy->entry->class_serial); + shape_id_t source_shape_id = vm_ic_attr_shape_id(ic_copy); % # JIT: cache hit path of vm_getivar/vm_setivar, or cancel JIT (recompile it with exivar) % if insn.name == 'setinstancevariable' - fprintf(f, " if (LIKELY(RB_TYPE_P(obj, T_OBJECT) && ic_serial == RCLASS_SERIAL(RBASIC(obj)->klass) && index < ROBJECT_NUMIV(obj) && !RB_OBJ_FROZEN_RAW(obj))) {\n"); - fprintf(f, " VALUE *ptr = ROBJECT_IVPTR(obj);\n"); - fprintf(f, " RB_OBJ_WRITE(obj, &ptr[index], stack[%d]);\n", b->stack_size - 1); - fprintf(f, " }\n"); + shape_id_t dest_shape_id = vm_ic_attr_index_dest_shape_id(ic_copy); + fprintf(f, " shape_id_t shape_id = (shape_id_t)%hu;\n", source_shape_id); + fprintf(f, " shape_id_t dest_shape_id = %hu;\n", dest_shape_id); + if (source_shape_id == dest_shape_id) { + fprintf(f, " if (LIKELY(RB_TYPE_P(obj, T_OBJECT) && dest_shape_id == ROBJECT_SHAPE_ID(obj) && index < ROBJECT_NUMIV(obj) && !RB_OBJ_FROZEN_RAW(obj))) {\n"); + fprintf(f, " VALUE *ptr = ROBJECT_IVPTR(obj);\n"); + fprintf(f, " RB_OBJ_WRITE(obj, &ptr[index], stack[%d]);\n", b->stack_size - 1); + fprintf(f, " }\n"); + } + else { + fprintf(f, " if (LIKELY(RB_TYPE_P(obj, T_OBJECT) && shape_id == ROBJECT_SHAPE_ID(obj) && index < ROBJECT_NUMIV(obj) && !RB_OBJ_FROZEN_RAW(obj))) {\n"); + fprintf(f, " ROBJECT_SET_SHAPE_ID(obj, dest_shape_id);\n"); + fprintf(f, " VALUE *ptr = ROBJECT_IVPTR(obj);\n"); + fprintf(f, " RB_OBJ_WRITE(obj, &ptr[index], stack[%d]);\n", b->stack_size - 1); + fprintf(f, " }\n"); + } % else fprintf(f, " VALUE val;\n"); - fprintf(f, " if (LIKELY(RB_TYPE_P(obj, T_OBJECT) && ic_serial == RCLASS_SERIAL(RBASIC(obj)->klass) && index < ROBJECT_NUMIV(obj) && (val = ROBJECT_IVPTR(obj)[index]) != Qundef)) {\n"); + fprintf(f, " shape_id_t shape_id = (shape_id_t)%hu;\n", source_shape_id); + fprintf(f, " if (LIKELY(RB_TYPE_P(obj, T_OBJECT) && shape_id == ROBJECT_SHAPE_ID(obj) && index < ROBJECT_NUMIV(obj) && (val = ROBJECT_IVPTR(obj)[index]) != Qundef)) {\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size); fprintf(f, " }\n"); % end @@ -79,19 +92,19 @@ break; } % if insn.name == 'getinstancevariable' - else if (!status->compile_info->disable_exivar_cache && ic_copy->entry) { + else if (!status->compile_info->disable_exivar_cache && vm_ic_attr_index_p(ic_copy)) { % # JIT: optimize away motion of sp and pc. This path does not call rb_warning() and so it's always leaf and not `handles_sp`. % # <%= render 'mjit_compile_pc_and_sp', locals: { insn: insn } -%> % % # JIT: prepare vm_getivar's arguments and variables fprintf(f, "{\n"); fprintf(f, " VALUE obj = GET_SELF();\n"); - fprintf(f, " const rb_serial_t ic_serial = (rb_serial_t)%"PRI_SERIALT_PREFIX"u;\n", ic_copy->entry->class_serial); - fprintf(f, " const uint32_t index = %u;\n", ic_copy->entry->index); + fprintf(f, " shape_id_t shape_id = (shape_id_t)%hu;\n", vm_ic_attr_shape_id(ic_copy)); + fprintf(f, " const uint32_t index = %u;\n", vm_ic_attr_index(ic_copy)); % # JIT: cache hit path of vm_getivar, or cancel JIT (recompile it without any ivar optimization) fprintf(f, " struct gen_ivtbl *ivtbl;\n"); fprintf(f, " VALUE val;\n"); - fprintf(f, " if (LIKELY(FL_TEST_RAW(obj, FL_EXIVAR) && ic_serial == RCLASS_SERIAL(RBASIC(obj)->klass) && rb_ivar_generic_ivtbl_lookup(obj, &ivtbl) && index < ivtbl->numiv && (val = ivtbl->ivptr[index]) != Qundef)) {\n"); + fprintf(f, " if (LIKELY(FL_TEST_RAW(obj, FL_EXIVAR) && shape_id == rb_shape_get_shape_id(obj) && rb_ivar_generic_ivtbl_lookup(obj, &ivtbl) && index < ivtbl->numiv && (val = ivtbl->ivptr[index]) != Qundef)) {\n"); fprintf(f, " stack[%d] = val;\n", b->stack_size); fprintf(f, " }\n"); fprintf(f, " else {\n"); diff --git a/variable.c b/variable.c index a3512adc99..c34ac75937 100644 --- a/variable.c +++ b/variable.c @@ -34,6 +34,7 @@ #include "ruby/st.h" #include "ruby/util.h" #include "transient_heap.h" +#include "shape.h" #include "variable.h" #include "vm_core.h" #include "ractor_core.h" @@ -64,10 +65,11 @@ static st_table *generic_iv_tbl_; struct ivar_update { union { - st_table *iv_index_tbl; + size_t iv_index_tbl_size; struct gen_ivtbl *ivtbl; } u; - st_data_t index; + uint32_t index; + rb_shape_t* shape; int iv_extended; }; @@ -897,27 +899,28 @@ rb_alias_variable(ID name1, ID name2) } static bool -iv_index_tbl_lookup(struct st_table *tbl, ID id, uint32_t *indexp) +iv_index_tbl_lookup(VALUE obj, ID id, uint32_t *indexp) { st_data_t ent_data; - int r; + int r = 0; - if (tbl == NULL) return false; + rb_shape_t* shape = rb_shape_get_shape(obj); - RB_VM_LOCK_ENTER(); - { - r = st_lookup(tbl, (st_data_t)id, &ent_data); - } - RB_VM_LOCK_LEAVE(); - - if (r) { - struct rb_iv_index_tbl_entry *ent = (void *)ent_data; - *indexp = ent->index; - return true; + if (rb_shape_no_cache_shape_p(shape)) { + struct rb_id_table *iv_table = ROBJECT(obj)->as.heap.iv_index_tbl; + if (iv_table) { + r = rb_id_table_lookup(iv_table, (st_data_t)id, &ent_data); + } } else { - return false; + r = rb_shape_get_iv_index(shape, id, &ent_data); } + + if (r) { + *indexp = (uint32_t)ent_data; + } + + return r; } static void @@ -957,7 +960,20 @@ generic_ivtbl_no_ractor_check(VALUE obj) } static int -gen_ivtbl_get(VALUE obj, ID id, struct gen_ivtbl **ivtbl) +gen_ivtbl_get_unlocked(VALUE obj, ID id, struct gen_ivtbl **ivtbl) +{ + st_data_t data; + + if (st_lookup(generic_ivtbl(obj, id, false), (st_data_t)obj, &data)) { + *ivtbl = (struct gen_ivtbl *)data; + return 1; + } + + return 0; +} + +MJIT_FUNC_EXPORTED int +rb_gen_ivtbl_get(VALUE obj, ID id, struct gen_ivtbl **ivtbl) { st_data_t data; int r = 0; @@ -977,7 +993,7 @@ gen_ivtbl_get(VALUE obj, ID id, struct gen_ivtbl **ivtbl) MJIT_FUNC_EXPORTED int rb_ivar_generic_ivtbl_lookup(VALUE obj, struct gen_ivtbl **ivtbl) { - return gen_ivtbl_get(obj, 0, ivtbl); + return rb_gen_ivtbl_get(obj, 0, ivtbl); } MJIT_FUNC_EXPORTED VALUE @@ -985,7 +1001,7 @@ rb_ivar_generic_lookup_with_index(VALUE obj, ID id, uint32_t index) { struct gen_ivtbl *ivtbl; - if (gen_ivtbl_get(obj, id, &ivtbl)) { + if (rb_gen_ivtbl_get(obj, id, &ivtbl)) { if (LIKELY(index < ivtbl->numiv)) { VALUE val = ivtbl->ivptr[index]; return val; @@ -1000,11 +1016,10 @@ generic_ivar_delete(VALUE obj, ID id, VALUE undef) { struct gen_ivtbl *ivtbl; - if (gen_ivtbl_get(obj, id, &ivtbl)) { - st_table *iv_index_tbl = RCLASS_IV_INDEX_TBL(rb_obj_class(obj)); + if (rb_gen_ivtbl_get(obj, id, &ivtbl)) { uint32_t index; - if (iv_index_tbl && iv_index_tbl_lookup(iv_index_tbl, id, &index)) { + if (iv_index_tbl_lookup(obj, id, &index)) { if (index < ivtbl->numiv) { VALUE ret = ivtbl->ivptr[index]; @@ -1021,13 +1036,12 @@ generic_ivar_get(VALUE obj, ID id, VALUE undef) { struct gen_ivtbl *ivtbl; - if (gen_ivtbl_get(obj, id, &ivtbl)) { - st_table *iv_index_tbl = RCLASS_IV_INDEX_TBL(rb_obj_class(obj)); + if (rb_gen_ivtbl_get(obj, id, &ivtbl)) { uint32_t index; - if (iv_index_tbl && iv_index_tbl_lookup(iv_index_tbl, id, &index)) { - if (index < ivtbl->numiv) { - VALUE ret = ivtbl->ivptr[index]; + if (iv_index_tbl_lookup(obj, id, &index)) { + if (index < ivtbl->numiv) { + VALUE ret = ivtbl->ivptr[index]; return ret == Qundef ? undef : ret; } @@ -1045,6 +1059,8 @@ gen_ivtbl_bytes(size_t n) static struct gen_ivtbl * gen_ivtbl_resize(struct gen_ivtbl *old, uint32_t n) { + RUBY_ASSERT(n > 0); + uint32_t len = old ? old->numiv : 0; struct gen_ivtbl *ivtbl = xrealloc(old, gen_ivtbl_bytes(n)); @@ -1073,7 +1089,7 @@ static uint32_t iv_index_tbl_newsize(struct ivar_update *ivup) { if (!ivup->iv_extended) { - return (uint32_t)ivup->u.iv_index_tbl->num_entries; + return (uint32_t)ivup->u.iv_index_tbl_size; } else { uint32_t index = (uint32_t)ivup->index; /* should not overflow */ @@ -1099,8 +1115,10 @@ generic_ivar_update(st_data_t *k, st_data_t *v, st_data_t u, int existing) FL_SET((VALUE)*k, FL_EXIVAR); uint32_t newsize = iv_index_tbl_newsize(ivup); ivtbl = gen_ivtbl_resize(ivtbl, newsize); + // Reinsert in to the hash table because ivtbl might be a newly resized chunk of memory *v = (st_data_t)ivtbl; ivup->u.ivtbl = ivtbl; + ivtbl->shape_id = SHAPE_ID(ivup->shape); return ST_CONTINUE; } @@ -1108,11 +1126,10 @@ static VALUE generic_ivar_defined(VALUE obj, ID id) { struct gen_ivtbl *ivtbl; - st_table *iv_index_tbl = RCLASS_IV_INDEX_TBL(rb_obj_class(obj)); uint32_t index; - if (!iv_index_tbl_lookup(iv_index_tbl, id, &index)) return Qfalse; - if (!gen_ivtbl_get(obj, id, &ivtbl)) return Qfalse; + if (!iv_index_tbl_lookup(obj, id, &index)) return Qfalse; + if (!rb_gen_ivtbl_get(obj, id, &ivtbl)) return Qfalse; return RBOOL((index < ivtbl->numiv) && (ivtbl->ivptr[index] != Qundef)); } @@ -1122,11 +1139,9 @@ generic_ivar_remove(VALUE obj, ID id, VALUE *valp) { struct gen_ivtbl *ivtbl; uint32_t index; - st_table *iv_index_tbl = RCLASS_IV_INDEX_TBL(rb_obj_class(obj)); - if (!iv_index_tbl) return 0; - if (!iv_index_tbl_lookup(iv_index_tbl, id, &index)) return 0; - if (!gen_ivtbl_get(obj, id, &ivtbl)) return 0; + if (!iv_index_tbl_lookup(obj, id, &index)) return 0; + if (!rb_gen_ivtbl_get(obj, id, &ivtbl)) return 0; if (index < ivtbl->numiv) { if (ivtbl->ivptr[index] != Qundef) { @@ -1153,8 +1168,9 @@ rb_mark_generic_ivar(VALUE obj) { struct gen_ivtbl *ivtbl; - if (gen_ivtbl_get(obj, 0, &ivtbl)) { - gen_ivtbl_mark(ivtbl); + if (rb_gen_ivtbl_get(obj, 0, &ivtbl)) { + rb_gc_mark((VALUE)rb_shape_get_shape_by_id(ivtbl->shape_id)); + gen_ivtbl_mark(ivtbl); } } @@ -1182,8 +1198,8 @@ rb_generic_ivar_memsize(VALUE obj) { struct gen_ivtbl *ivtbl; - if (gen_ivtbl_get(obj, 0, &ivtbl)) - return gen_ivtbl_bytes(ivtbl->numiv); + if (rb_gen_ivtbl_get(obj, 0, &ivtbl)) + return gen_ivtbl_bytes(ivtbl->numiv); return 0; } @@ -1262,7 +1278,7 @@ rb_ivar_lookup(VALUE obj, ID id, VALUE undef) VALUE *ptr = ROBJECT_IVPTR(obj); VALUE val; - if (iv_index_tbl_lookup(ROBJECT_IV_INDEX_TBL(obj), id, &index) && + if (iv_index_tbl_lookup(obj, id, &index) && index < len && (val = ptr[index]) != Qundef) { return val; @@ -1316,7 +1332,6 @@ static VALUE rb_ivar_delete(VALUE obj, ID id, VALUE undef) { VALUE *ptr; - struct st_table *iv_index_tbl; uint32_t len, index; rb_check_frozen(obj); @@ -1324,8 +1339,8 @@ rb_ivar_delete(VALUE obj, ID id, VALUE undef) case T_OBJECT: len = ROBJECT_NUMIV(obj); ptr = ROBJECT_IVPTR(obj); - iv_index_tbl = ROBJECT_IV_INDEX_TBL(obj); - if (iv_index_tbl_lookup(iv_index_tbl, id, &index) && + + if (iv_index_tbl_lookup(obj, id, &index) && index < len) { VALUE val = ptr[index]; ptr[index] = Qundef; @@ -1359,67 +1374,63 @@ rb_attr_delete(VALUE obj, ID id) return rb_ivar_delete(obj, id, Qnil); } -static st_table * -iv_index_tbl_make(VALUE obj, VALUE klass) -{ - st_table *iv_index_tbl; - - if (UNLIKELY(!klass)) { - rb_raise(rb_eTypeError, "hidden object cannot have instance variables"); - } - - if ((iv_index_tbl = RCLASS_IV_INDEX_TBL(klass)) == NULL) { - RB_VM_LOCK_ENTER(); - if ((iv_index_tbl = RCLASS_IV_INDEX_TBL(klass)) == NULL) { - iv_index_tbl = RCLASS_IV_INDEX_TBL(klass) = st_init_numtable(); - } - RB_VM_LOCK_LEAVE(); - } - - return iv_index_tbl; -} - static void -iv_index_tbl_extend(struct ivar_update *ivup, ID id, VALUE klass) +iv_index_tbl_extend(VALUE obj, struct ivar_update *ivup, ID id) { ASSERT_vm_locking(); - st_data_t ent_data; - struct rb_iv_index_tbl_entry *ent; + VALUE ent_data; + struct rb_id_table *iv_table; + int r = 0; - if (st_lookup(ivup->u.iv_index_tbl, (st_data_t)id, &ent_data)) { - ent = (void *)ent_data; - ivup->index = ent->index; - return; + if (rb_shape_no_cache_shape_p(ivup->shape)) { + iv_table = ROBJECT(obj)->as.heap.iv_index_tbl; + if ((VALUE)iv_table == Qundef) { + iv_table = rb_id_table_create(0); + ROBJECT(obj)->as.heap.iv_index_tbl = iv_table; + } + + uint32_t index = (uint32_t)rb_id_table_size(iv_table); + rb_id_table_insert(iv_table, id, (VALUE)index); + ivup->u.iv_index_tbl_size = rb_id_table_size(iv_table); + r = rb_id_table_lookup(iv_table, id, &ent_data); + } + else { + // This sets the iv table in the ivup struct + ivup->u.iv_index_tbl_size = ivup->shape->iv_count; + r = rb_shape_get_iv_index(ivup->shape, id, &ent_data); + } + + if (r) { + ivup->index = (uint32_t) ent_data; + ivup->iv_extended = 1; } - if (ivup->u.iv_index_tbl->num_entries >= INT_MAX) { - rb_raise(rb_eArgError, "too many instance variables"); + else { + rb_bug("unreachable. Shape was not found for id: %s", rb_id2name(id)); } - ent = ALLOC(struct rb_iv_index_tbl_entry); - ent->index = ivup->index = (uint32_t)ivup->u.iv_index_tbl->num_entries; - ent->class_value = klass; - ent->class_serial = RCLASS_SERIAL(klass); - st_add_direct(ivup->u.iv_index_tbl, (st_data_t)id, (st_data_t)ent); - ivup->iv_extended = 1; } static void generic_ivar_set(VALUE obj, ID id, VALUE val) { - VALUE klass = rb_obj_class(obj); struct ivar_update ivup; + // The returned shape will have `id` in its iv_table + rb_shape_t * shape = rb_shape_get_next(rb_shape_get_shape(obj), obj, id); + ivup.shape = shape; ivup.iv_extended = 0; - ivup.u.iv_index_tbl = iv_index_tbl_make(obj, klass); RB_VM_LOCK_ENTER(); { - iv_index_tbl_extend(&ivup, id, klass); - st_update(generic_ivtbl(obj, id, false), (st_data_t)obj, generic_ivar_update, - (st_data_t)&ivup); + iv_index_tbl_extend(obj, &ivup, id); + if (!st_update(generic_ivtbl(obj, id, false), (st_data_t)obj, generic_ivar_update, + (st_data_t)&ivup)) { + RB_OBJ_WRITTEN(obj, Qundef, shape); + } } RB_VM_LOCK_LEAVE(); ivup.u.ivtbl->ivptr[ivup.index] = val; + rb_shape_set_shape(obj, shape); RB_OBJ_WRITTEN(obj, Qundef, val); } @@ -1486,8 +1497,8 @@ rb_obj_transient_heap_evacuate(VALUE obj, int promote) } #endif -static void -init_iv_list(VALUE obj, uint32_t len, uint32_t newsize, st_table *index_tbl) +void +rb_ensure_iv_list_size(VALUE obj, uint32_t len, uint32_t newsize) { VALUE *ptr = ROBJECT_IVPTR(obj); VALUE *newptr; @@ -1510,16 +1521,34 @@ init_iv_list(VALUE obj, uint32_t len, uint32_t newsize, st_table *index_tbl) #else ROBJECT(obj)->as.heap.numiv = newsize; #endif - ROBJECT(obj)->as.heap.iv_index_tbl = index_tbl; +} + +struct gen_ivtbl * +rb_ensure_generic_iv_list_size(VALUE obj, uint32_t newsize) +{ + struct gen_ivtbl * ivtbl = 0; + + RB_VM_LOCK_ENTER(); + { + if (UNLIKELY(!gen_ivtbl_get_unlocked(obj, 0, &ivtbl) || newsize > ivtbl->numiv)) { + ivtbl = gen_ivtbl_resize(ivtbl, newsize); + st_insert(generic_ivtbl_no_ractor_check(obj), (st_data_t)obj, (st_data_t)ivtbl); + FL_SET_RAW(obj, FL_EXIVAR); + } + } + RB_VM_LOCK_LEAVE(); + + RUBY_ASSERT(ivtbl); + + return ivtbl; } void rb_init_iv_list(VALUE obj) { - st_table *index_tbl = ROBJECT_IV_INDEX_TBL(obj); - uint32_t newsize = (uint32_t)index_tbl->num_entries; + uint32_t newsize = rb_shape_get_shape(obj)->iv_count * 2.0; uint32_t len = ROBJECT_NUMIV(obj); - init_iv_list(obj, len, newsize, index_tbl); + rb_ensure_iv_list_size(obj, len, newsize < len ? len : newsize); } // Retrieve or create the id-to-index mapping for a given object and an @@ -1527,14 +1556,13 @@ rb_init_iv_list(VALUE obj) static struct ivar_update obj_ensure_iv_index_mapping(VALUE obj, ID id) { - VALUE klass = rb_obj_class(obj); struct ivar_update ivup; ivup.iv_extended = 0; - ivup.u.iv_index_tbl = iv_index_tbl_make(obj, klass); + ivup.shape = rb_shape_get_shape(obj); RB_VM_LOCK_ENTER(); { - iv_index_tbl_extend(&ivup, id, klass); + iv_index_tbl_extend(obj, &ivup, id); } RB_VM_LOCK_LEAVE(); @@ -1552,28 +1580,125 @@ uint32_t rb_obj_ensure_iv_index_mapping(VALUE obj, ID id) { RUBY_ASSERT(RB_TYPE_P(obj, T_OBJECT)); - // This uint32_t cast shouldn't lose information as it's checked in - // iv_index_tbl_extend(). The index is stored as an uint32_t in - // struct rb_iv_index_tbl_entry. - return (uint32_t)obj_ensure_iv_index_mapping(obj, id).index; -} + rb_shape_transition_shape(obj, id, rb_shape_get_shape_by_id(ROBJECT_SHAPE_ID(obj))); -static VALUE -obj_ivar_set(VALUE obj, ID id, VALUE val) -{ - uint32_t len; struct ivar_update ivup = obj_ensure_iv_index_mapping(obj, id); - - len = ROBJECT_NUMIV(obj); - if (len <= ivup.index) { + uint32_t len = ROBJECT_NUMIV(obj); + if (len <= (ivup.index)) { uint32_t newsize = iv_index_tbl_newsize(&ivup); - init_iv_list(obj, len, newsize, ivup.u.iv_index_tbl); + rb_ensure_iv_list_size(obj, len, newsize); } - RB_OBJ_WRITE(obj, &ROBJECT_IVPTR(obj)[ivup.index], val); + RUBY_ASSERT(ivup.index <= ROBJECT_NUMIV(obj)); + return ivup.index; +} +static VALUE +obj_ivar_set(VALUE obj, ID id, VALUE val) +{ + uint32_t index = rb_obj_ensure_iv_index_mapping(obj, id); + RB_OBJ_WRITE(obj, &ROBJECT_IVPTR(obj)[index], val); return val; } +/* Set the instance variable +val+ on object +obj+ at ivar name +id+. + * This function only works with T_OBJECT objects, so make sure + * +obj+ is of type T_OBJECT before using this function. + */ +VALUE +rb_vm_set_ivar_id(VALUE obj, ID id, VALUE val) +{ + rb_check_frozen_internal(obj); + obj_ivar_set(obj, id, val); + return val; +} + +MJIT_FUNC_EXPORTED shape_id_t +rb_generic_shape_id(VALUE obj) +{ + struct gen_ivtbl *ivtbl = 0; + shape_id_t shape_id = 0; + + RB_VM_LOCK_ENTER(); + { + st_table* global_iv_table = generic_ivtbl(obj, 0, false); + + if (global_iv_table && st_lookup(global_iv_table, obj, (st_data_t *)&ivtbl)) { + shape_id = ivtbl->shape_id; + } + else if (OBJ_FROZEN(obj)) { + shape_id = FROZEN_ROOT_SHAPE_ID; + } + } + RB_VM_LOCK_LEAVE(); + + return shape_id; +} + +bool +rb_shape_set_shape_id(VALUE obj, shape_id_t shape_id) +{ + if (rb_shape_get_shape_id(obj) == shape_id) { + return false; + } + + switch (BUILTIN_TYPE(obj)) { + case T_OBJECT: + ROBJECT_SET_SHAPE_ID(obj, shape_id); + break; + case T_CLASS: + case T_MODULE: + { + RCLASS_EXT(obj)->shape_id = shape_id; + break; + } + case T_IMEMO: + if (imemo_type(obj) == imemo_shape) { + RBASIC(obj)->flags &= 0xffffffff0000ffff; + RBASIC(obj)->flags |= ((uint32_t)(shape_id) << 16); + } + break; + default: + { + if (shape_id != FROZEN_ROOT_SHAPE_ID) { + struct gen_ivtbl *ivtbl = 0; + RB_VM_LOCK_ENTER(); + { + st_table* global_iv_table = generic_ivtbl(obj, 0, false); + + if (st_lookup(global_iv_table, obj, (st_data_t *)&ivtbl)) { + ivtbl->shape_id = shape_id; + } + else { + rb_bug("Expected shape_id entry in global iv table"); + } + } + RB_VM_LOCK_LEAVE(); + } + } + } + + return true; +} + +/** + * Prevents further modifications to the given object. ::rb_eFrozenError shall + * be raised if modification is attempted. + * + * @param[out] x Object in question. + */ +void rb_obj_freeze_inline(VALUE x) +{ + if (RB_FL_ABLE(x)) { + RB_OBJ_FREEZE_RAW(x); + + rb_shape_transition_shape_frozen(x); + + if (RBASIC_CLASS(x) && !(RBASIC(x)->flags & RUBY_FL_SINGLETON)) { + rb_freeze_singleton_class(x); + } + } +} + static void ivar_set(VALUE obj, ID id, VALUE val) { @@ -1581,10 +1706,18 @@ ivar_set(VALUE obj, ID id, VALUE val) switch (BUILTIN_TYPE(obj)) { case T_OBJECT: - obj_ivar_set(obj, id, val); - break; + { + /* + * Array of existing shapes which we can index into w a shape_id + * Hash (tree representation) of ivar transitions between shapes + */ + obj_ivar_set(obj, id, val); + break; + } case T_CLASS: case T_MODULE: + // TODO: Transition shapes on classes + //rb_shape_transition_shape(obj, id, rb_shape_get_shape_by_id(RCLASS_SHAPE_ID(obj))); IVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR(id); rb_class_ivar_set(obj, id, val); break; @@ -1615,14 +1748,12 @@ VALUE rb_ivar_defined(VALUE obj, ID id) { VALUE val; - struct st_table *iv_index_tbl; uint32_t index; if (SPECIAL_CONST_P(obj)) return Qfalse; switch (BUILTIN_TYPE(obj)) { case T_OBJECT: - iv_index_tbl = ROBJECT_IV_INDEX_TBL(obj); - if (iv_index_tbl_lookup(iv_index_tbl, id, &index) && + if (iv_index_tbl_lookup(obj, id, &index) && index < ROBJECT_NUMIV(obj) && (val = ROBJECT_IVPTR(obj)[index]) != Qundef) { return Qtrue; @@ -1644,48 +1775,71 @@ rb_ivar_defined(VALUE obj, ID id) typedef int rb_ivar_foreach_callback_func(ID key, VALUE val, st_data_t arg); st_data_t rb_st_nth_key(st_table *tab, st_index_t index); -static ID -iv_index_tbl_nth_id(st_table *iv_index_tbl, uint32_t index) -{ - st_data_t key; - RB_VM_LOCK_ENTER(); - { - key = rb_st_nth_key(iv_index_tbl, index); - } - RB_VM_LOCK_LEAVE(); - return (ID)key; +static void +iterate_over_shapes_with_callback(VALUE obj, rb_shape_t *shape, VALUE* iv_list, int numiv, rb_ivar_foreach_callback_func *callback, st_data_t arg) { + if (rb_shape_root_shape_p(shape)) { + return; + } + else if (rb_shape_frozen_shape_p(shape)) { + iterate_over_shapes_with_callback(obj, shape->parent, iv_list, numiv, callback, arg); + return; + } + else if (numiv <= 0) { + rb_bug("bad numiv iterating over shapes\n"); + } + else { + rb_shape_t *parent_shape; + if (rb_shape_no_cache_shape_p(shape)) { + parent_shape = shape; + } + else { + parent_shape = shape->parent; + } + iterate_over_shapes_with_callback(obj, parent_shape, iv_list, numiv - 1, callback, arg); + + if (iv_list[numiv - 1] != Qundef) { + ID id = shape->edge_name; + + callback(id, iv_list[numiv - 1], arg); + } + return; + } } -static inline bool -ivar_each_i(st_table *iv_index_tbl, VALUE val, uint32_t i, rb_ivar_foreach_callback_func *func, st_data_t arg) +static enum rb_id_table_iterator_result +add_to_array(ID id, VALUE val, void * data) { - if (val != Qundef) { - ID id = iv_index_tbl_nth_id(iv_index_tbl, i); - switch (func(id, val, arg)) { - case ST_CHECK: - case ST_CONTINUE: - break; - case ST_STOP: - return true; - default: - rb_bug("unreachable"); - } - } - return false; + uint32_t idx = (uint32_t)val; + VALUE * list = (VALUE *)data; + list[idx] = id; + return ID_TABLE_CONTINUE; } static void obj_ivar_each(VALUE obj, rb_ivar_foreach_callback_func *func, st_data_t arg) { - st_table *iv_index_tbl = ROBJECT_IV_INDEX_TBL(obj); - if (!iv_index_tbl) return; - uint32_t i=0; + rb_shape_t* shape = rb_shape_get_shape(obj); + struct rb_id_table *iv_index_tbl; - for (i=0; i < ROBJECT_NUMIV(obj); i++) { - VALUE val = ROBJECT_IVPTR(obj)[i]; - if (ivar_each_i(iv_index_tbl, val, i, func, arg)) { + if (rb_shape_no_cache_shape_p(shape)) { + iv_index_tbl = ROBJECT(obj)->as.heap.iv_index_tbl; + uint32_t table_size = (uint32_t) rb_id_table_size(iv_index_tbl); + VALUE * array = xmalloc(sizeof(VALUE) * table_size); + + rb_id_table_foreach(iv_index_tbl, add_to_array, array); + for (uint32_t i = 0; i < table_size; i++) { + func(array[i], ROBJECT_IVPTR(obj)[i], arg); + } + + xfree(array); + } + else { + iv_index_tbl = rb_shape_generate_iv_table(rb_shape_get_shape(obj)); + if (!iv_index_tbl) { return; } + rb_shape_t * shape = rb_shape_get_shape(obj); + iterate_over_shapes_with_callback(obj, shape, ROBJECT_IVPTR(obj), (int)shape->iv_count, func, arg); } } @@ -1693,82 +1847,38 @@ static void gen_ivar_each(VALUE obj, rb_ivar_foreach_callback_func *func, st_data_t arg) { struct gen_ivtbl *ivtbl; - st_table *iv_index_tbl = RCLASS_IV_INDEX_TBL(rb_obj_class(obj)); + rb_shape_t *shape = rb_shape_get_shape(obj); + struct rb_id_table *iv_index_tbl = rb_shape_generate_iv_table(shape); if (!iv_index_tbl) return; - if (!gen_ivtbl_get(obj, 0, &ivtbl)) return; + if (!rb_gen_ivtbl_get(obj, 0, &ivtbl)) return; - for (uint32_t i=0; inumiv; i++) { - VALUE val = ivtbl->ivptr[i]; - if (ivar_each_i(iv_index_tbl, val, i, func, arg)) { - return; - } - } -} - -struct givar_copy { - VALUE obj; - VALUE klass; - st_table *iv_index_tbl; - struct gen_ivtbl *ivtbl; -}; - -static int -gen_ivar_copy(ID id, VALUE val, st_data_t arg) -{ - struct givar_copy *c = (struct givar_copy *)arg; - struct ivar_update ivup; - - ivup.iv_extended = 0; - ivup.u.iv_index_tbl = c->iv_index_tbl; - - RB_VM_LOCK_ENTER(); - { - iv_index_tbl_extend(&ivup, id, c->klass); - } - RB_VM_LOCK_LEAVE(); - - if (ivup.index >= c->ivtbl->numiv) { - uint32_t newsize = iv_index_tbl_newsize(&ivup); - c->ivtbl = gen_ivtbl_resize(c->ivtbl, newsize); - } - c->ivtbl->ivptr[ivup.index] = val; - - RB_OBJ_WRITTEN(c->obj, Qundef, val); - - return ST_CONTINUE; + iterate_over_shapes_with_callback(obj, rb_shape_get_shape(obj), ivtbl->ivptr, (int)shape->iv_count, func, arg); } void rb_copy_generic_ivar(VALUE clone, VALUE obj) { - struct gen_ivtbl *ivtbl; + struct gen_ivtbl *obj_ivtbl; + struct gen_ivtbl *new_ivtbl; rb_check_frozen(clone); if (!FL_TEST(obj, FL_EXIVAR)) { goto clear; } - if (gen_ivtbl_get(obj, 0, &ivtbl)) { - struct givar_copy c; - uint32_t i; - if (gen_ivtbl_count(ivtbl) == 0) + if (rb_gen_ivtbl_get(obj, 0, &obj_ivtbl)) { + if (gen_ivtbl_count(obj_ivtbl) == 0) goto clear; - if (gen_ivtbl_get(clone, 0, &c.ivtbl)) { - for (i = 0; i < c.ivtbl->numiv; i++) - c.ivtbl->ivptr[i] = Qundef; - } - else { - c.ivtbl = gen_ivtbl_resize(0, ivtbl->numiv); - FL_SET(clone, FL_EXIVAR); + new_ivtbl = gen_ivtbl_resize(0, obj_ivtbl->numiv); + FL_SET(clone, FL_EXIVAR); + + for (uint32_t i=0; inumiv; i++) { + new_ivtbl->ivptr[i] = obj_ivtbl->ivptr[i]; + RB_OBJ_WRITTEN(clone, Qundef, &new_ivtbl[i]); } - VALUE klass = rb_obj_class(clone); - c.iv_index_tbl = iv_index_tbl_make(clone, klass); - c.obj = clone; - c.klass = klass; - gen_ivar_each(obj, gen_ivar_copy, (st_data_t)&c); /* * c.ivtbl may change in gen_ivar_copy due to realloc, * no need to free @@ -1776,9 +1886,11 @@ rb_copy_generic_ivar(VALUE clone, VALUE obj) RB_VM_LOCK_ENTER(); { generic_ivtbl_no_ractor_check(clone); - st_insert(generic_ivtbl_no_ractor_check(obj), (st_data_t)clone, (st_data_t)c.ivtbl); + st_insert(generic_ivtbl_no_ractor_check(obj), (st_data_t)clone, (st_data_t)new_ivtbl); } RB_VM_LOCK_LEAVE(); + + rb_shape_set_shape(clone, rb_shape_get_shape(obj)); } return; @@ -1846,17 +1958,17 @@ rb_ivar_count(VALUE obj) switch (BUILTIN_TYPE(obj)) { case T_OBJECT: - if (ROBJECT_IV_INDEX_TBL(obj) != 0) { - st_index_t i, count, num = ROBJECT_NUMIV(obj); - const VALUE *const ivptr = ROBJECT_IVPTR(obj); - for (i = count = 0; i < num; ++i) { - if (ivptr[i] != Qundef) { - count++; - } - } - return count; - } - break; + if (rb_shape_get_shape(obj)->iv_count > 0) { + st_index_t i, count, num = ROBJECT_NUMIV(obj); + const VALUE *const ivptr = ROBJECT_IVPTR(obj); + for (i = count = 0; i < num; ++i) { + if (ivptr[i] != Qundef) { + count++; + } + } + return count; + } + break; case T_CLASS: case T_MODULE: if ((tbl = RCLASS_IV_TBL(obj)) != 0) { @@ -1867,11 +1979,11 @@ rb_ivar_count(VALUE obj) if (FL_TEST(obj, FL_EXIVAR)) { struct gen_ivtbl *ivtbl; - if (gen_ivtbl_get(obj, 0, &ivtbl)) { - return gen_ivtbl_count(ivtbl); - } - } - break; + if (rb_gen_ivtbl_get(obj, 0, &ivtbl)) { + return gen_ivtbl_count(ivtbl); + } + } + break; } return 0; } @@ -1966,7 +2078,6 @@ rb_obj_remove_instance_variable(VALUE obj, VALUE name) VALUE val = Qnil; const ID id = id_for_var(obj, name, an, instance); st_data_t n, v; - struct st_table *iv_index_tbl; uint32_t index; rb_check_frozen(obj); @@ -1976,8 +2087,7 @@ rb_obj_remove_instance_variable(VALUE obj, VALUE name) switch (BUILTIN_TYPE(obj)) { case T_OBJECT: - iv_index_tbl = ROBJECT_IV_INDEX_TBL(obj); - if (iv_index_tbl_lookup(iv_index_tbl, id, &index) && + if (iv_index_tbl_lookup(obj, id, &index) && index < ROBJECT_NUMIV(obj) && (val = ROBJECT_IVPTR(obj)[index]) != Qundef) { ROBJECT_IVPTR(obj)[index] = Qundef; diff --git a/variable.h b/variable.h index 55596b00de..5e77d11fd6 100644 --- a/variable.h +++ b/variable.h @@ -11,6 +11,7 @@ /* per-object */ struct gen_ivtbl { + uint16_t shape_id; uint32_t numiv; VALUE ivptr[FLEX_ARY_LEN]; }; diff --git a/vm.c b/vm.c index 4b1a30e7ca..8d6bb37172 100644 --- a/vm.c +++ b/vm.c @@ -26,6 +26,7 @@ #include "internal/thread.h" #include "internal/vm.h" #include "internal/sanitizers.h" +#include "internal/variable.h" #include "iseq.h" #include "mjit.h" #include "yjit.h" @@ -2623,6 +2624,12 @@ rb_vm_update_references(void *ptr) vm->top_self = rb_gc_location(vm->top_self); vm->orig_progname = rb_gc_location(vm->orig_progname); + for (int i = 0; i < MAX_SHAPE_ID; i++) { + if (vm->shape_list[i]) { + vm->shape_list[i] = (rb_shape_t *)rb_gc_location((VALUE)vm->shape_list[i]); + } + } + rb_gc_update_tbl_refs(vm->overloaded_cme_table); if (vm->coverages) { @@ -2704,6 +2711,9 @@ rb_vm_mark(void *ptr) obj_ary++; } + rb_gc_mark((VALUE)vm->root_shape); + rb_gc_mark((VALUE)vm->frozen_root_shape); + rb_gc_mark((VALUE)vm->no_cache_shape); rb_gc_mark_movable(vm->load_path); rb_gc_mark_movable(vm->load_path_snapshot); RUBY_MARK_MOVABLE_UNLESS_NULL(vm->load_path_check_cache); @@ -3935,6 +3945,33 @@ Init_vm_objects(void) vm->mark_object_ary = rb_ary_hidden_new(128); vm->loading_table = st_init_strtable(); vm->frozen_strings = st_init_table_with_size(&rb_fstring_hash_type, 10000); + vm->shape_list = xcalloc(MAX_SHAPE_ID, sizeof(rb_shape_t *)); + for (int i = 0; i < 2048; i++) { + vm->shape_bitmaps[i] = 0; + } + vm->max_shape_count = 0; + + // Root shape + vm->root_shape = rb_shape_alloc(ROOT_SHAPE_ID, + 0, + 0); + rb_shape_set_shape_by_id(ROOT_SHAPE_ID, vm->root_shape); + RB_OBJ_WRITTEN(vm->root_shape, Qundef, (VALUE)vm); + + // Frozen root shape + vm->frozen_root_shape = rb_shape_alloc(FROZEN_ROOT_SHAPE_ID, + rb_make_internal_id(), + vm->root_shape); + RB_OBJ_FREEZE_RAW((VALUE)vm->frozen_root_shape); + rb_shape_set_shape_by_id(FROZEN_ROOT_SHAPE_ID, vm->frozen_root_shape); + RB_OBJ_WRITTEN(vm->frozen_root_shape, Qundef, (VALUE)vm); + + // No cache shape + vm->no_cache_shape = rb_shape_alloc(NO_CACHE_SHAPE_ID, + 0, + 0); + rb_shape_set_shape_by_id(NO_CACHE_SHAPE_ID, vm->no_cache_shape); + RB_OBJ_WRITTEN(vm->no_cache_shape, Qundef, (VALUE)vm); } /* Stub for builtin function when not building YJIT units*/ diff --git a/vm_callinfo.h b/vm_callinfo.h index fd2215be7d..4f9aaa24b9 100644 --- a/vm_callinfo.h +++ b/vm_callinfo.h @@ -9,7 +9,9 @@ */ #include "debug_counter.h" +#include "internal/variable.h" #include "internal/class.h" +#include "shape.h" enum vm_call_flag_bits { VM_CALL_ARGS_SPLAT_bit, /* m(*args) */ @@ -284,7 +286,18 @@ struct rb_callcache { const vm_call_handler call_; union { - const unsigned int attr_index; + /* + * attr_index is also storing source_shape_id and dest_shape_id in the + * following way: + * + * ---16 bits-------|---16 bits-----|-----32 bits----- + * source_shape_id | dest_shape_id | attr_index + */ +#if USE_SHAPE_CACHE_P + const uint64_t attr_index; +#else + const uint32_t attr_index; +#endif const enum method_missing_reason method_missing_reason; /* used by method_missing */ VALUE v; } aux_; @@ -293,12 +306,30 @@ struct rb_callcache { #define VM_CALLCACHE_UNMARKABLE IMEMO_FL_USER0 #define VM_CALLCACHE_ON_STACK IMEMO_FL_USER1 +extern const struct rb_callcache *rb_vm_empty_cc(void); +extern const struct rb_callcache *rb_vm_empty_cc_for_super(void); + +#define vm_cc_empty() rb_vm_empty_cc() + +static inline void +vm_cc_attr_index_initialize(const struct rb_callcache *cc, shape_id_t shape_id) +{ + VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache)); + VM_ASSERT(cc != vm_cc_empty()); +#if USE_SHAPE_CACHE_P + *(uint64_t *)&cc->aux_.attr_index = ((uint64_t)(shape_id) << 48) | ((uint64_t)(shape_id) << 32) | 0; +#else + *(uint32_t *)&cc->aux_.attr_index = 0; +#endif +} + static inline const struct rb_callcache * vm_cc_new(VALUE klass, const struct rb_callable_method_entry_struct *cme, vm_call_handler call) { const struct rb_callcache *cc = (const struct rb_callcache *)rb_imemo_new(imemo_callcache, (VALUE)cme, (VALUE)call, 0, klass); + vm_cc_attr_index_initialize(cc, INVALID_SHAPE_ID); RB_DEBUG_COUNTER_INC(cc_new); return cc; } @@ -354,26 +385,82 @@ static inline unsigned int vm_cc_attr_index(const struct rb_callcache *cc) { VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache)); - return cc->aux_.attr_index - 1; + return (int)(cc->aux_.attr_index - 1); } static inline bool vm_cc_attr_index_p(const struct rb_callcache *cc) { VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache)); - return cc->aux_.attr_index > 0; +#if USE_SHAPE_CACHE_P + return (cc->aux_.attr_index & 0xFFFFFFFF) != 0; +#else + return (cc->aux_.attr_index & 0xFFFF) != 0; +#endif } -static inline uint32_t -vm_ic_entry_index(const struct iseq_inline_iv_cache_entry *ic) +static inline uint16_t +vm_cc_attr_shape_id(const struct rb_callcache *cc) { - return ic->entry->index; + VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache)); +#if USE_SHAPE_CACHE_P + return (cc->aux_.attr_index >> 32) & 0xFFFF; +#else + return NO_CACHE_SHAPE_ID; +#endif +} + +static inline uint16_t +vm_cc_attr_index_source_shape_id(const struct rb_callcache *cc) +{ + VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache)); + +#if USE_SHAPE_CACHE_P + return cc->aux_.attr_index >> 48; +#else + return NO_CACHE_SHAPE_ID; +#endif +} + +static inline uint16_t +vm_cc_attr_index_dest_shape_id(const struct rb_callcache *cc) +{ + VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache)); +#if USE_SHAPE_CACHE_P + return (cc->aux_.attr_index >> 32) & 0xFFFF; +#else + return NO_CACHE_SHAPE_ID; +#endif +} + +static inline unsigned int +vm_ic_attr_index(const struct iseq_inline_iv_cache_entry *ic) +{ + return (int)ic->attr_index - 1; } static inline bool -vm_ic_entry_p(const struct iseq_inline_iv_cache_entry *ic) +vm_ic_attr_index_p(const struct iseq_inline_iv_cache_entry *ic) { - return ic->entry; + return ic->attr_index > 0; +} + +static inline uint16_t +vm_ic_attr_shape_id(const struct iseq_inline_iv_cache_entry *ic) +{ + return ic->source_shape_id; +} + +static inline uint16_t +vm_ic_attr_index_source_shape_id(const struct iseq_inline_iv_cache_entry *ic) +{ + return ic->source_shape_id; +} + +static inline uint16_t +vm_ic_attr_index_dest_shape_id(const struct iseq_inline_iv_cache_entry *ic) +{ + return ic->dest_shape_id; } static inline unsigned int @@ -407,10 +494,6 @@ vm_cc_valid_p(const struct rb_callcache *cc, const rb_callable_method_entry_t *c } } -extern const struct rb_callcache *rb_vm_empty_cc(void); -extern const struct rb_callcache *rb_vm_empty_cc_for_super(void); -#define vm_cc_empty() rb_vm_empty_cc() - /* callcache: mutate */ static inline void @@ -422,26 +505,31 @@ vm_cc_call_set(const struct rb_callcache *cc, vm_call_handler call) } static inline void -vm_cc_attr_index_set(const struct rb_callcache *cc, int index) +vm_cc_attr_index_set(const struct rb_callcache *cc, int index, shape_id_t source_shape_id, shape_id_t dest_shape_id) { VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache)); VM_ASSERT(cc != vm_cc_empty()); - *(int *)&cc->aux_.attr_index = index + 1; +#if USE_SHAPE_CACHE_P + *(uint64_t *)&cc->aux_.attr_index = ((uint64_t)source_shape_id << 48) | ((uint64_t)dest_shape_id << 32) | (index + 1); +#else + *(uint32_t *)&cc->aux_.attr_index = index + 1; +#endif } static inline void -vm_ic_entry_set(struct iseq_inline_iv_cache_entry *ic, struct rb_iv_index_tbl_entry *entry, const rb_iseq_t *iseq) +vm_ic_attr_index_set(const rb_iseq_t *iseq, const struct iseq_inline_iv_cache_entry *ic, int index, shape_id_t source_shape_id, shape_id_t dest_shape_id) { - ic->entry = entry; - RB_OBJ_WRITTEN(iseq, Qundef, entry->class_value); + *(uint16_t *)&ic->source_shape_id = source_shape_id; + *(uint16_t *)&ic->dest_shape_id = dest_shape_id; + *(uint32_t *)&ic->attr_index = index + 1; } static inline void -vm_cc_attr_index_initialize(const struct rb_callcache *cc) +vm_ic_attr_index_initialize(const struct iseq_inline_iv_cache_entry *ic, shape_id_t shape_id) { - VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache)); - VM_ASSERT(cc != vm_cc_empty()); - *(int *)&cc->aux_.attr_index = 0; + *(uint16_t *)&ic->source_shape_id = shape_id; + *(uint16_t *)&ic->dest_shape_id = shape_id; + *(uint32_t *)&ic->attr_index = 0; } static inline void diff --git a/vm_core.h b/vm_core.h index 717f116800..35a98513ca 100644 --- a/vm_core.h +++ b/vm_core.h @@ -92,6 +92,7 @@ extern int ruby_assert_critical_section_entered; #include "internal.h" #include "internal/array.h" #include "internal/serial.h" +#include "internal/variable.h" #include "internal/vm.h" #include "method.h" #include "node.h" @@ -262,7 +263,9 @@ struct iseq_inline_constant_cache { }; struct iseq_inline_iv_cache_entry { - struct rb_iv_index_tbl_entry *entry; + shape_id_t source_shape_id; + shape_id_t dest_shape_id; + uint32_t attr_index; }; struct iseq_inline_cvar_cache_entry { @@ -335,6 +338,12 @@ pathobj_realpath(VALUE pathobj) /* Forward declarations */ struct rb_mjit_unit; +struct rb_shape; + +#ifndef rb_shape_t +typedef struct rb_shape rb_shape_t; +#define rb_shape_t rb_shape_t +#endif typedef uintptr_t iseq_bits_t; #define ISEQ_IS_SIZE(body) (body->ic_size + body->ivc_size + body->ise_size + body->icvarc_size) @@ -674,6 +683,14 @@ typedef struct rb_vm_struct { VALUE mark_object_ary; const VALUE special_exceptions[ruby_special_error_count]; + /* object shapes */ + rb_shape_t **shape_list; + rb_shape_t *root_shape; + rb_shape_t *frozen_root_shape; + rb_shape_t *no_cache_shape; + shape_id_t max_shape_count; + uint32_t shape_bitmaps[2048]; + /* load */ VALUE top_self; VALUE load_path; diff --git a/vm_insnhelper.c b/vm_insnhelper.c index 1812f7ce71..3ff122c688 100644 --- a/vm_insnhelper.c +++ b/vm_insnhelper.c @@ -50,6 +50,7 @@ MJIT_STATIC VALUE ruby_vm_special_exception_copy(VALUE exc) { VALUE e = rb_obj_alloc(rb_class_real(RBASIC_CLASS(exc))); + rb_shape_set_shape(e, rb_shape_get_shape(exc)); rb_obj_copy_ivar(e, exc); return e; } @@ -1065,35 +1066,20 @@ vm_get_cvar_base(const rb_cref_t *cref, const rb_control_frame_t *cfp, int top_l return klass; } -static bool -iv_index_tbl_lookup(struct st_table *iv_index_tbl, ID id, struct rb_iv_index_tbl_entry **ent) -{ - int found; - st_data_t ent_data; - - if (iv_index_tbl == NULL) return false; - - RB_VM_LOCK_ENTER(); - { - found = st_lookup(iv_index_tbl, (st_data_t)id, &ent_data); - } - RB_VM_LOCK_LEAVE(); - if (found) *ent = (struct rb_iv_index_tbl_entry *)ent_data; - - return found ? true : false; -} - -ALWAYS_INLINE(static void fill_ivar_cache(const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr, struct rb_iv_index_tbl_entry *ent)); - +ALWAYS_INLINE(static void fill_ivar_cache(const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr, uint32_t index, shape_id_t shape_id)); static inline void -fill_ivar_cache(const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr, struct rb_iv_index_tbl_entry *ent) +fill_ivar_cache(const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr, uint32_t index, shape_id_t shape_id) { // fill cache - if (!is_attr) { - vm_ic_entry_set(ic, ent, iseq); + if (is_attr) { + if (vm_cc_markable(cc)) { + vm_cc_attr_index_set(cc, index, shape_id, shape_id); + RB_OBJ_WRITTEN(cc, Qundef, rb_shape_get_shape_by_id(shape_id)); + } } else { - vm_cc_attr_index_set(cc, ent->index); + vm_ic_attr_index_set(iseq, ic, index, shape_id, shape_id); + RB_OBJ_WRITTEN(iseq, Qundef, rb_shape_get_shape_by_id(shape_id)); } } @@ -1103,60 +1089,136 @@ vm_getivar(VALUE obj, ID id, const rb_iseq_t *iseq, IVC ic, const struct rb_call { #if OPT_IC_FOR_IVAR VALUE val = Qundef; + shape_id_t shape_id; + VALUE * ivar_list; if (SPECIAL_CONST_P(obj)) { - // frozen? + goto general_path; } - else if (LIKELY(is_attr ? - RB_DEBUG_COUNTER_INC_UNLESS(ivar_get_ic_miss_unset, vm_cc_attr_index_p(cc)) : - RB_DEBUG_COUNTER_INC_UNLESS(ivar_get_ic_miss_serial, vm_ic_entry_p(ic) && ic->entry->class_serial == RCLASS_SERIAL(RBASIC(obj)->klass)))) { - uint32_t index = !is_attr ? vm_ic_entry_index(ic): (vm_cc_attr_index(cc)); + switch (BUILTIN_TYPE(obj)) { + case T_OBJECT: + { + shape_id = ROBJECT_SHAPE_ID(obj); + ivar_list = ROBJECT_IVPTR(obj); + } + break; + case T_CLASS: + case T_MODULE: + { + goto general_path; + } + default: + if (FL_TEST_RAW(obj, FL_EXIVAR)) { + struct gen_ivtbl *ivtbl; + rb_ivar_generic_ivtbl_lookup(obj, &ivtbl); + shape_id = ivtbl->shape_id; + //shape_id = rb_shape_get_shape_id(obj); + ivar_list = ivtbl->ivptr; + } else { + return Qnil; + } + } + + if (shape_id == NO_CACHE_SHAPE_ID) { + goto general_path; + } + + shape_id_t cached_id; + + if (is_attr) { + cached_id = vm_cc_attr_shape_id(cc); + } + else { + cached_id = vm_ic_attr_shape_id(ic); + } + + if (LIKELY(cached_id == shape_id)) { RB_DEBUG_COUNTER_INC(ivar_get_ic_hit); - if (LIKELY(BUILTIN_TYPE(obj) == T_OBJECT) && - LIKELY(index < ROBJECT_NUMIV(obj))) { - val = ROBJECT_IVPTR(obj)[index]; + uint32_t index; - VM_ASSERT(rb_ractor_shareable_p(obj) ? rb_ractor_shareable_p(val) : true); + if (is_attr) { + if (vm_cc_attr_index_p(cc)) { + index = vm_cc_attr_index(cc); + } + else { + goto general_path; + } } - else if (FL_TEST_RAW(obj, FL_EXIVAR)) { - val = rb_ivar_generic_lookup_with_index(obj, id, index); + else { + if (vm_ic_attr_index_p(ic)) { + index = vm_ic_attr_index(ic); + } + else { + goto general_path; + } } + val = ivar_list[index]; + VM_ASSERT(rb_ractor_shareable_p(obj) ? rb_ractor_shareable_p(val) : true); + goto ret; } - else { - struct rb_iv_index_tbl_entry *ent; - - if (BUILTIN_TYPE(obj) == T_OBJECT) { - struct st_table *iv_index_tbl = ROBJECT_IV_INDEX_TBL(obj); - - if (iv_index_tbl && iv_index_tbl_lookup(iv_index_tbl, id, &ent)) { - fill_ivar_cache(iseq, ic, cc, is_attr, ent); - - // get value - if (ent->index < ROBJECT_NUMIV(obj)) { - val = ROBJECT_IVPTR(obj)[ent->index]; - - VM_ASSERT(rb_ractor_shareable_p(obj) ? rb_ractor_shareable_p(val) : true); - } - } - } - else if (FL_TEST_RAW(obj, FL_EXIVAR)) { - struct st_table *iv_index_tbl = RCLASS_IV_INDEX_TBL(rb_obj_class(obj)); - - if (iv_index_tbl && iv_index_tbl_lookup(iv_index_tbl, id, &ent)) { - fill_ivar_cache(iseq, ic, cc, is_attr, ent); - val = rb_ivar_generic_lookup_with_index(obj, id, ent->index); + else { // cache miss case + uint32_t iv_index; + +#if RUBY_DEBUG + if (is_attr) { + if (cached_id != INVALID_SHAPE_ID) { + RB_DEBUG_COUNTER_INC(ivar_get_cc_miss_set); + } else { + RB_DEBUG_COUNTER_INC(ivar_get_cc_miss_unset); } } else { - // T_CLASS / T_MODULE + if (cached_id != INVALID_SHAPE_ID) { + RB_DEBUG_COUNTER_INC(ivar_get_ic_miss_set); + } else { + RB_DEBUG_COUNTER_INC(ivar_get_ic_miss_unset); + } + } +#endif + + // We're only caching information for T_OBJECTS and things that store + // ivars in EXIVAR + if (!(LIKELY(BUILTIN_TYPE(obj) == T_OBJECT) || FL_TEST_RAW(obj, FL_EXIVAR))) { goto general_path; } - ret: + VALUE iv_index_value; + rb_shape_t *shape = rb_shape_get_shape(obj); + if (rb_shape_get_iv_index(shape, id, &iv_index_value)) { + // This fills in the cache with the shared cache object. + // "ent" is the shared cache object + iv_index = (uint32_t)iv_index_value; + fill_ivar_cache(iseq, ic, cc, is_attr, (uint32_t) iv_index_value, shape_id); + + // get value + if (LIKELY(BUILTIN_TYPE(obj) == T_OBJECT) && + LIKELY(iv_index < ROBJECT_NUMIV(obj))) { + val = ROBJECT_IVPTR(obj)[iv_index]; + + VM_ASSERT(rb_ractor_shareable_p(obj) ? rb_ractor_shareable_p(val) : true); + } + else if (FL_TEST_RAW(obj, FL_EXIVAR)) { + val = rb_ivar_generic_lookup_with_index(obj, id, iv_index); + } + } + else { + if (is_attr) { + if (vm_cc_markable(cc)) { + vm_cc_attr_index_initialize(cc, shape_id); + RB_OBJ_WRITTEN(cc, Qundef, shape); + } + } + else { + vm_ic_attr_index_initialize(ic, shape_id); + RB_OBJ_WRITTEN(iseq, Qundef, shape); + } + } + +ret: if (LIKELY(val != Qundef)) { return val; } @@ -1164,7 +1226,8 @@ vm_getivar(VALUE obj, ID id, const rb_iseq_t *iseq, IVC ic, const struct rb_call return Qnil; } } - general_path: + +general_path: #endif /* OPT_IC_FOR_IVAR */ RB_DEBUG_COUNTER_INC(ivar_get_ic_miss); @@ -1176,6 +1239,29 @@ vm_getivar(VALUE obj, ID id, const rb_iseq_t *iseq, IVC ic, const struct rb_call } } +struct check_shape { + rb_shape_t *shape; + bool found; +}; + +static void +populate_cache(uint32_t index, rb_shape_t *shape, rb_shape_t *next_shape, ID id, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, bool is_attr) +{ + // Cache population code + if (is_attr) { + if (vm_cc_markable(cc)) { + vm_cc_attr_index_set(cc, (int)(index), SHAPE_ID(shape), SHAPE_ID(next_shape)); + RB_OBJ_WRITTEN(cc, Qundef, (VALUE)shape); + RB_OBJ_WRITTEN(cc, Qundef, (VALUE)next_shape); + } + } + else { + vm_ic_attr_index_set(iseq, ic, (int)index, SHAPE_ID(shape), SHAPE_ID(next_shape)); + RB_OBJ_WRITTEN(iseq, Qundef, (VALUE)shape); + RB_OBJ_WRITTEN(iseq, Qundef, (VALUE)next_shape); + } +} + ALWAYS_INLINE(static VALUE vm_setivar_slowpath(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr)); NOINLINE(static VALUE vm_setivar_slowpath_ivar(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic)); NOINLINE(static VALUE vm_setivar_slowpath_attr(VALUE obj, ID id, VALUE val, const struct rb_callcache *cc)); @@ -1183,35 +1269,93 @@ NOINLINE(static VALUE vm_setivar_slowpath_attr(VALUE obj, ID id, VALUE val, cons static VALUE vm_setivar_slowpath(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr) { - rb_check_frozen_internal(obj); - #if OPT_IC_FOR_IVAR - if (RB_TYPE_P(obj, T_OBJECT)) { - struct st_table *iv_index_tbl = ROBJECT_IV_INDEX_TBL(obj); - struct rb_iv_index_tbl_entry *ent; + switch (BUILTIN_TYPE(obj)) { + case T_OBJECT: + { + rb_check_frozen_internal(obj); - if (iv_index_tbl_lookup(iv_index_tbl, id, &ent)) { - if (!is_attr) { - vm_ic_entry_set(ic, ent, iseq); - } - else if (ent->index >= INT_MAX) { - rb_raise(rb_eArgError, "too many instance variables"); - } - else { - vm_cc_attr_index_set(cc, (int)(ent->index)); - } + uint32_t index; + + uint32_t num_iv = ROBJECT_NUMIV(obj); + rb_shape_t* shape = rb_shape_get_shape(obj); + rb_shape_t* next_shape = rb_shape_get_next(shape, obj, id); + if (shape != next_shape) { + rb_shape_set_shape(obj, next_shape); + } + + // cache -> no cache + // ensure object isn't embedded + // copy the iv idnex table + // set the iv index table pointer on the object + // no cache -> no cache + // + // both caches + if (!rb_shape_no_cache_shape_p(shape) && rb_shape_no_cache_shape_p(next_shape)) { + // Ensure the object is *not* embedded + rb_ensure_iv_list_size(obj, num_iv, num_iv + 1); + RUBY_ASSERT(!(RBASIC(obj)->flags & ROBJECT_EMBED)); + + // Save the IV index table on the instance + ROBJECT(obj)->as.heap.iv_index_tbl = rb_shape_generate_iv_table(shape); + } - uint32_t index = ent->index; + if (rb_shape_no_cache_shape_p(shape) || rb_shape_no_cache_shape_p(next_shape)) { + index = (uint32_t)rb_id_table_size(ROBJECT(obj)->as.heap.iv_index_tbl); + rb_id_table_insert(ROBJECT(obj)->as.heap.iv_index_tbl, id, (VALUE)index); + } + else { + VALUE x; + if (rb_shape_get_iv_index(next_shape, id, &x)) { // based off the has stored in the transition tree + index = (uint32_t)x; + if (index >= INT_MAX) { + rb_raise(rb_eArgError, "too many instance variables"); + } - if (UNLIKELY(index >= ROBJECT_NUMIV(obj))) { - rb_init_iv_list(obj); + populate_cache(index, shape, next_shape, id, iseq, ic, cc, is_attr); + } + else { + rb_bug("didn't find the id\n"); + } + } + + // Ensure the IV buffer is wide enough to store the IV + if (UNLIKELY(index >= num_iv)) { + rb_init_iv_list(obj); + } + + // TODO: fix this index still + VALUE *ptr = ROBJECT_IVPTR(obj); + RB_OBJ_WRITE(obj, &ptr[index], val); + RB_DEBUG_COUNTER_INC(ivar_set_ic_miss_iv_hit); + + return val; } - VALUE *ptr = ROBJECT_IVPTR(obj); - RB_OBJ_WRITE(obj, &ptr[index], val); - RB_DEBUG_COUNTER_INC(ivar_set_ic_miss_iv_hit); + case T_CLASS: + case T_MODULE: + break; + default: + { + rb_shape_t * shape = rb_shape_get_shape(obj); + rb_ivar_set(obj, id, val); + rb_shape_t * next_shape = rb_shape_get_shape(obj); + VALUE x; + uint32_t index; - return val; - } + if (rb_shape_get_iv_index(next_shape, id, &x)) { // based off the has stored in the transition tree + index = (uint32_t)x; + if (index >= INT_MAX) { + rb_raise(rb_eArgError, "too many instance variables"); + } + + populate_cache(index, shape, next_shape, id, iseq, ic, cc, is_attr); + } + else { + rb_bug("didn't find the id\n"); + } + + return val; + } } #endif RB_DEBUG_COUNTER_INC(ivar_set_ic_miss); @@ -1230,39 +1374,97 @@ vm_setivar_slowpath_attr(VALUE obj, ID id, VALUE val, const struct rb_callcache return vm_setivar_slowpath(obj, id, val, NULL, NULL, cc, true); } -static inline VALUE -vm_setivar(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr) +NOINLINE(static VALUE vm_setivar_default(VALUE obj, ID id, VALUE val, shape_id_t source_shape_id, shape_id_t dest_shape_id, uint32_t index)); +static VALUE +vm_setivar_default(VALUE obj, ID id, VALUE val, shape_id_t source_shape_id, shape_id_t dest_shape_id, uint32_t index) { -#if OPT_IC_FOR_IVAR - if (LIKELY(RB_TYPE_P(obj, T_OBJECT)) && - LIKELY(!RB_OBJ_FROZEN_RAW(obj))) { + shape_id_t shape_id = rb_generic_shape_id(obj); - VM_ASSERT(!rb_ractor_shareable_p(obj)); + if (shape_id != NO_CACHE_SHAPE_ID) { + // Do we have a cache hit *and* is the CC intitialized + if (shape_id == source_shape_id) { + RUBY_ASSERT(dest_shape_id != INVALID_SHAPE_ID && shape_id != INVALID_SHAPE_ID); - if (LIKELY( - (!is_attr && RB_DEBUG_COUNTER_INC_UNLESS(ivar_set_ic_miss_serial, vm_ic_entry_p(ic) && ic->entry->class_serial == RCLASS_SERIAL(RBASIC(obj)->klass))) || - ( is_attr && RB_DEBUG_COUNTER_INC_UNLESS(ivar_set_ic_miss_unset, vm_cc_attr_index_p(cc))))) { - uint32_t index = !is_attr ? vm_ic_entry_index(ic) : vm_cc_attr_index(cc); - - if (UNLIKELY(index >= ROBJECT_NUMIV(obj))) { - rb_init_iv_list(obj); + struct gen_ivtbl *ivtbl = 0; + if (dest_shape_id != shape_id) { + ivtbl = rb_ensure_generic_iv_list_size(obj, index + 1); + ivtbl->shape_id = dest_shape_id; + RB_OBJ_WRITTEN(obj, Qundef, rb_shape_get_shape_by_id(dest_shape_id)); + } + else { + // Just get the IV table + RUBY_ASSERT(GET_VM()->shape_list[dest_shape_id]); + rb_gen_ivtbl_get(obj, 0, &ivtbl); } - VALUE *ptr = ROBJECT_IVPTR(obj); + + VALUE *ptr = ivtbl->ivptr; + + // Ensuring we have a place to store the IV value RB_OBJ_WRITE(obj, &ptr[index], val); + RB_DEBUG_COUNTER_INC(ivar_set_ic_hit); + return val; /* inline cache hit */ } } - else { + + return Qundef; +} + +static inline VALUE +vm_setivar(VALUE obj, ID id, VALUE val, shape_id_t source_shape_id, shape_id_t dest_shape_id, uint32_t index) +{ +#if OPT_IC_FOR_IVAR + switch (BUILTIN_TYPE(obj)) { + case T_OBJECT: + { + VM_ASSERT(!rb_ractor_shareable_p(obj) || rb_obj_frozen_p(obj)); + // If object's shape id is the same as the source + // then do the shape transition and write the ivar + // If object's shape id is the same as the dest + // then write the ivar + shape_id_t shape_id = ROBJECT_SHAPE_ID(obj); + + if (shape_id != NO_CACHE_SHAPE_ID) { + // Do we have a cache hit *and* is the CC intitialized + if (shape_id == source_shape_id) { + RUBY_ASSERT(dest_shape_id != INVALID_SHAPE_ID && shape_id != INVALID_SHAPE_ID); + + VM_ASSERT(!rb_ractor_shareable_p(obj)); + + if (dest_shape_id != shape_id) { + if (UNLIKELY(index >= ROBJECT_NUMIV(obj))) { + rb_init_iv_list(obj); + } + ROBJECT_SET_SHAPE_ID(obj, dest_shape_id); + RB_OBJ_WRITTEN(obj, Qundef, rb_shape_get_shape_by_id(dest_shape_id)); + } + else { + RUBY_ASSERT(GET_VM()->shape_list[dest_shape_id]); + } + + RUBY_ASSERT(index < ROBJECT_NUMIV(obj)); + + VALUE *ptr = ROBJECT_IVPTR(obj); + + RB_OBJ_WRITE(obj, &ptr[index], val); + + RB_DEBUG_COUNTER_INC(ivar_set_ic_hit); + + return val; + } + } + } + break; + case T_CLASS: + case T_MODULE: RB_DEBUG_COUNTER_INC(ivar_set_ic_miss_noobject); + default: + break; } + + return Qundef; #endif /* OPT_IC_FOR_IVAR */ - if (is_attr) { - return vm_setivar_slowpath_attr(obj, id, val, cc); - } - else { - return vm_setivar_slowpath_ivar(obj, id, val, iseq, ic); - } } static VALUE @@ -1357,7 +1559,22 @@ vm_getinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, IVC ic) static inline void vm_setinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, VALUE val, IVC ic) { - vm_setivar(obj, id, val, iseq, ic, 0, 0); + shape_id_t source_shape_id = vm_ic_attr_index_source_shape_id(ic); + uint32_t index = vm_ic_attr_index(ic); + shape_id_t dest_shape_id = vm_ic_attr_index_dest_shape_id(ic); + if (vm_setivar(obj, id, val, source_shape_id, dest_shape_id, index) == Qundef) { + switch (BUILTIN_TYPE(obj)) { + case T_OBJECT: + case T_CLASS: + case T_MODULE: + break; + default: + if (vm_setivar_default(obj, id, val, source_shape_id, dest_shape_id, index) != Qundef) { + return; + } + } + vm_setivar_slowpath_ivar(obj, id, val, iseq, ic); + } } void @@ -1366,28 +1583,6 @@ rb_vm_setinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, VALUE val, IV vm_setinstancevariable(iseq, obj, id, val, ic); } -/* Set the instance variable +val+ on object +obj+ at the +index+. - * This function only works with T_OBJECT objects, so make sure - * +obj+ is of type T_OBJECT before using this function. - */ -VALUE -rb_vm_set_ivar_idx(VALUE obj, uint32_t index, VALUE val) -{ - RUBY_ASSERT(RB_TYPE_P(obj, T_OBJECT)); - - rb_check_frozen_internal(obj); - - VM_ASSERT(!rb_ractor_shareable_p(obj)); - - if (UNLIKELY(index >= ROBJECT_NUMIV(obj))) { - rb_init_iv_list(obj); - } - VALUE *ptr = ROBJECT_IVPTR(obj); - RB_OBJ_WRITE(obj, &ptr[index], val); - - return val; -} - static VALUE vm_throw_continue(const rb_execution_context_t *ec, VALUE err) { @@ -3047,17 +3242,45 @@ vm_call_ivar(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_call const struct rb_callcache *cc = calling->cc; RB_DEBUG_COUNTER_INC(ccf_ivar); cfp->sp -= 1; - return vm_getivar(calling->recv, vm_cc_cme(cc)->def->body.attr.id, NULL, NULL, cc, TRUE); + VALUE ivar = vm_getivar(calling->recv, vm_cc_cme(cc)->def->body.attr.id, NULL, NULL, cc, TRUE); + return ivar; } static VALUE -vm_call_attrset(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling) +vm_call_attrset_direct(rb_execution_context_t *ec, rb_control_frame_t *cfp, const struct rb_callcache *cc, VALUE obj) { - const struct rb_callcache *cc = calling->cc; RB_DEBUG_COUNTER_INC(ccf_attrset); VALUE val = *(cfp->sp - 1); cfp->sp -= 2; - return vm_setivar(calling->recv, vm_cc_cme(cc)->def->body.attr.id, val, NULL, NULL, cc, 1); + shape_id_t source_shape_id = vm_cc_attr_index_source_shape_id(cc); + uint32_t index = vm_cc_attr_index(cc); + shape_id_t dest_shape_id = vm_cc_attr_index_dest_shape_id(cc); + ID id = vm_cc_cme(cc)->def->body.attr.id; + rb_check_frozen_internal(obj); + VALUE res = vm_setivar(obj, id, val, source_shape_id, dest_shape_id, index); + if (res == Qundef) { + switch (BUILTIN_TYPE(obj)) { + case T_OBJECT: + case T_CLASS: + case T_MODULE: + break; + default: + { + res = vm_setivar_default(obj, id, val, source_shape_id, dest_shape_id, index); + if (res != Qundef) { + return res; + } + } + } + res = vm_setivar_slowpath_attr(obj, id, val, cc); + } + return res; +} + +static VALUE +vm_call_attrset(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling) +{ + return vm_call_attrset_direct(ec, cfp, calling->cc, calling->recv); } bool @@ -3647,19 +3870,37 @@ vm_call_method_each_type(rb_execution_context_t *ec, rb_control_frame_t *cfp, st CALLER_SETUP_ARG(cfp, calling, ci); CALLER_REMOVE_EMPTY_KW_SPLAT(cfp, calling, ci); - rb_check_arity(calling->argc, 1, 1); - vm_cc_attr_index_initialize(cc); + rb_check_arity(calling->argc, 1, 1); + const unsigned int aset_mask = (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT | VM_CALL_KWARG); - VM_CALL_METHOD_ATTR(v, - vm_call_attrset(ec, cfp, calling), - CC_SET_FASTPATH(cc, vm_call_attrset, !(vm_ci_flag(ci) & aset_mask))); + + if (vm_cc_markable(cc)) { + vm_cc_attr_index_initialize(cc, INVALID_SHAPE_ID); + VM_CALL_METHOD_ATTR(v, + vm_call_attrset_direct(ec, cfp, cc, calling->recv), + CC_SET_FASTPATH(cc, vm_call_attrset, !(vm_ci_flag(ci) & aset_mask))); + } else { +#if USE_SHAPE_CACHE_P + cc = &VM_CC_ON_STACK(cc->klass, + cc->call_, + { .attr_index = ((uint64_t)INVALID_SHAPE_ID << 48 | (uint64_t)INVALID_SHAPE_ID << 32) }, + cc->cme_); +#else + cc = &VM_CC_ON_STACK(cc->klass, cc->call_, { .attr_index = 0 }, cc->cme_); +#endif + VM_CALL_METHOD_ATTR(v, + vm_call_attrset_direct(ec, cfp, cc, calling->recv), + CC_SET_FASTPATH(cc, vm_call_attrset, !(vm_ci_flag(ci) & aset_mask))); + } return v; case VM_METHOD_TYPE_IVAR: CALLER_SETUP_ARG(cfp, calling, ci); CALLER_REMOVE_EMPTY_KW_SPLAT(cfp, calling, ci); rb_check_arity(calling->argc, 0, 0); - vm_cc_attr_index_initialize(cc); + if (vm_cc_markable(cc)) { + vm_cc_attr_index_initialize(cc, INVALID_SHAPE_ID); + } const unsigned int ivar_mask = (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT); VM_CALL_METHOD_ATTR(v, vm_call_ivar(ec, cfp, calling), diff --git a/yjit/bindgen/src/main.rs b/yjit/bindgen/src/main.rs index f54addc795..dfe8a0c234 100644 --- a/yjit/bindgen/src/main.rs +++ b/yjit/bindgen/src/main.rs @@ -289,6 +289,7 @@ fn main() { // From internal/variable.h .allowlist_function("rb_gvar_(get|set)") .allowlist_function("rb_obj_ensure_iv_index_mapping") + .allowlist_function("rb_shape_get_shape_id") // From include/ruby/internal/intern/variable.h .allowlist_function("rb_attr_get") diff --git a/yjit/src/asm/x86_64/mod.rs b/yjit/src/asm/x86_64/mod.rs index 6eb7efaa0a..b0f992bff3 100644 --- a/yjit/src/asm/x86_64/mod.rs +++ b/yjit/src/asm/x86_64/mod.rs @@ -621,7 +621,7 @@ fn write_rm_multi(cb: &mut CodeBlock, op_mem_reg8: u8, op_mem_reg_pref: u8, op_r write_rm(cb, sz_pref, rex_w, X86Opnd::None, opnd0, op_ext_imm, &[op_mem_imm_lrg]); cb.write_int(uimm.value, if opnd_size > 32 { 32 } else { opnd_size.into() }); } else { - panic!("immediate value too large"); + panic!("{} immediate value too large", num_bits); } }, _ => unreachable!() diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index 119477f505..835a542585 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -1897,7 +1897,6 @@ fn gen_set_ivar( jit: &mut JITState, ctx: &mut Context, cb: &mut CodeBlock, - recv: VALUE, ivar_name: ID, ) -> CodegenStatus { // Save the PC and SP because the callee may allocate @@ -1908,13 +1907,11 @@ fn gen_set_ivar( let val_opnd = ctx.stack_pop(1); let recv_opnd = ctx.stack_pop(1); - let ivar_index: u32 = unsafe { rb_obj_ensure_iv_index_mapping(recv, ivar_name) }; - - // Call rb_vm_set_ivar_idx with the receiver, the index of the ivar, and the value + // Call rb_vm_set_ivar_id with the receiver, the ivar name, and the value mov(cb, C_ARG_REGS[0], recv_opnd); - mov(cb, C_ARG_REGS[1], imm_opnd(ivar_index.into())); + mov(cb, C_ARG_REGS[1], uimm_opnd(ivar_name.into())); mov(cb, C_ARG_REGS[2], val_opnd); - call_ptr(cb, REG0, rb_vm_set_ivar_idx as *const u8); + call_ptr(cb, REG0, rb_vm_set_ivar_id as *const u8); let out_opnd = ctx.stack_push(Type::Unknown); mov(cb, out_opnd, RAX); @@ -2119,26 +2116,26 @@ fn gen_getinstancevariable( let ivar_name = jit_get_arg(jit, 0).as_u64(); let comptime_val = jit_peek_at_self(jit); - let comptime_val_klass = comptime_val.class_of(); + let comptime_val_shape = comptime_val.shape_of(); // Generate a side exit let side_exit = get_side_exit(jit, ocb, ctx); + // Guard that the receiver has the same shape as the one from compile time. + jit_guard_known_shape( + jit, + ctx, + cb, + ocb, + comptime_val_shape, + GET_IVAR_MAX_DEPTH, + 0, + side_exit, + ); + // Guard that the receiver has the same class as the one from compile time. mov(cb, REG0, mem_opnd(64, REG_CFP, RUBY_OFFSET_CFP_SELF)); - jit_guard_known_klass( - jit, - ctx, - cb, - ocb, - comptime_val_klass, - SelfOpnd, - comptime_val, - GET_IVAR_MAX_DEPTH, - side_exit, - ); - gen_get_ivar( jit, ctx, @@ -3538,6 +3535,35 @@ fn jit_guard_known_klass( } } +/// Guard that self or a stack operand has the same shape as `known_shape`, using +/// `sample_instance` to speculate about the shape of the runtime value. +/// FIXNUM and on-heap integers are treated as if they have distinct classes, and +/// the guard generated for one will fail for the other. +/// +/// Recompile as contingency if possible, or take side exit a last resort. + +fn jit_guard_known_shape( + jit: &mut JITState, + ctx: &mut Context, + cb: &mut CodeBlock, + ocb: &mut OutlinedCb, + known_shape: u16, + max_chain_depth: i32, + stack_index: i32, + side_exit: CodePtr, +) { + mov(cb, C_ARG_REGS[0], ctx.stack_opnd(stack_index)); + call_ptr(cb, REG0, rb_shape_get_shape_id as *const u8); + // panic if too many bits + let shape_into: i64 = known_shape.into(); + if sig_imm_size(shape_into) > 32 { + panic!("{} shape is too big", sig_imm_size(shape_into)); + } + + cmp(cb, REG0, imm_opnd(shape_into)); + jit_chain_guard(JCC_JNE, jit, ctx, cb, ocb, max_chain_depth, side_exit); +} + // Generate ancestry guard for protected callee. // Calls to protected callees only go through when self.is_a?(klass_that_defines_the_callee). fn jit_protected_callee_ancestry_guard( @@ -4957,9 +4983,21 @@ fn gen_send_general( return CantCompile; } - mov(cb, REG0, recv); let ivar_name = unsafe { get_cme_def_body_attr_id(cme) }; + let comptime_recv_shape = comptime_recv.shape_of(); + jit_guard_known_shape( + jit, + ctx, + cb, + ocb, + comptime_recv_shape, + SEND_MAX_DEPTH, + argc, + side_exit + ); + + mov(cb, REG0, recv); return gen_get_ivar( jit, ctx, @@ -4986,7 +5024,7 @@ fn gen_send_general( return CantCompile; } else { let ivar_name = unsafe { get_cme_def_body_attr_id(cme) }; - return gen_set_ivar(jit, ctx, cb, comptime_recv, ivar_name); + return gen_set_ivar(jit, ctx, cb, ivar_name); } } // Block method, e.g. define_method(:foo) { :my_block } diff --git a/yjit/src/cruby.rs b/yjit/src/cruby.rs index 9195578172..93056b71a2 100644 --- a/yjit/src/cruby.rs +++ b/yjit/src/cruby.rs @@ -269,7 +269,7 @@ extern "C" { obj: VALUE, v: VALUE, ) -> bool; - pub fn rb_vm_set_ivar_idx(obj: VALUE, idx: u32, val: VALUE) -> VALUE; + pub fn rb_vm_set_ivar_id(obj: VALUE, idx: u32, val: VALUE) -> VALUE; pub fn rb_vm_setinstancevariable(iseq: IseqPtr, obj: VALUE, id: ID, val: VALUE, ic: IVC); pub fn rb_aliased_callable_method_entry( me: *const rb_callable_method_entry_t, @@ -477,6 +477,10 @@ impl VALUE { unsafe { CLASS_OF(self) } } + pub fn shape_of(self) -> u16 { + unsafe { rb_shape_get_shape_id(self) } + } + pub fn as_isize(self) -> isize { let VALUE(is) = self; is as isize diff --git a/yjit/src/cruby_bindings.inc.rs b/yjit/src/cruby_bindings.inc.rs index 31f09ef98d..ef2f3b80fe 100644 --- a/yjit/src/cruby_bindings.inc.rs +++ b/yjit/src/cruby_bindings.inc.rs @@ -115,6 +115,7 @@ pub const ROBJECT_OFFSET_NUMIV: i32 = 16; pub const ROBJECT_OFFSET_AS_HEAP_IVPTR: i32 = 24; pub const ROBJECT_OFFSET_AS_HEAP_IV_INDEX_TBL: i32 = 32; pub const ROBJECT_OFFSET_AS_ARY: i32 = 24; +pub type shape_id_t = u16; extern "C" { pub static mut rb_mKernel: VALUE; } @@ -476,6 +477,15 @@ extern "C" { ) -> VALUE; } pub type rb_serial_t = ::std::os::raw::c_ulonglong; +extern "C" { + pub fn rb_obj_ensure_iv_index_mapping(obj: VALUE, id: ID) -> u32; +} +extern "C" { + pub fn rb_gvar_get(arg1: ID) -> VALUE; +} +extern "C" { + pub fn rb_gvar_set(arg1: ID, arg2: VALUE) -> VALUE; +} extern "C" { pub fn rb_class_allocate_instance(klass: VALUE) -> VALUE; } @@ -496,6 +506,7 @@ pub const imemo_parser_strterm: imemo_type = 10; pub const imemo_callinfo: imemo_type = 11; pub const imemo_callcache: imemo_type = 12; pub const imemo_constcache: imemo_type = 13; +pub const imemo_shape: imemo_type = 14; pub type imemo_type = u32; pub const METHOD_VISI_UNDEF: rb_method_visibility_t = 0; pub const METHOD_VISI_PUBLIC: rb_method_visibility_t = 1; @@ -563,9 +574,10 @@ pub struct iseq_inline_constant_cache { pub get_insn_idx: ::std::os::raw::c_uint, } #[repr(C)] -#[derive(Debug, Copy, Clone)] pub struct iseq_inline_iv_cache_entry { - pub entry: *mut rb_iv_index_tbl_entry, + pub source_shape_id: shape_id_t, + pub dest_shape_id: shape_id_t, + pub attr_index: u32, } #[repr(C)] #[derive(Debug, Copy, Clone)] @@ -658,17 +670,14 @@ extern "C" { ) -> *const rb_callable_method_entry_t; } #[repr(C)] -pub struct rb_iv_index_tbl_entry { - pub index: u32, - pub class_serial: rb_serial_t, - pub class_value: VALUE, -} -#[repr(C)] pub struct rb_cvar_class_tbl_entry { pub index: u32, pub global_cvar_state: rb_serial_t, pub class_value: VALUE, } +extern "C" { + pub fn rb_shape_get_shape_id(obj: VALUE) -> shape_id_t; +} pub const VM_CALL_ARGS_SPLAT_bit: vm_call_flag_bits = 0; pub const VM_CALL_ARGS_BLOCKARG_bit: vm_call_flag_bits = 1; pub const VM_CALL_FCALL_bit: vm_call_flag_bits = 2; @@ -713,15 +722,6 @@ extern "C" { extern "C" { pub fn rb_hash_resurrect(hash: VALUE) -> VALUE; } -extern "C" { - pub fn rb_obj_ensure_iv_index_mapping(obj: VALUE, id: ID) -> u32; -} -extern "C" { - pub fn rb_gvar_get(arg1: ID) -> VALUE; -} -extern "C" { - pub fn rb_gvar_set(arg1: ID, arg2: VALUE) -> VALUE; -} extern "C" { pub fn rb_vm_insn_decode(encoded: VALUE) -> ::std::os::raw::c_int; } -- 2.35.1