Buffer_alloc: initialize the buffer before calling rb_ivar_set

ChangeLog

@@ -1,3 +1,5 @@
+* Fix a possible GC crash when GC trigger inside `MessagePack::Buffer.new` (#314).
+
 2022-09-30 1.6.0:
 
 * Fix a potential use-after-free bug in Buffer_free when accessing a packer or unpacker buffer. 

ext/msgpack/buffer_class.c

@@ -91,8 +91,8 @@ static VALUE Buffer_alloc(VALUE klass)
 {
     msgpack_buffer_t* b;
     VALUE buffer = TypedData_Make_Struct(klass, msgpack_buffer_t, &buffer_data_type, b);
-    rb_ivar_set(buffer, s_at_owner, Qnil);
     msgpack_buffer_init(b);
+    rb_ivar_set(buffer, s_at_owner, Qnil);
     return buffer;
 }
 

spec/cruby/buffer_spec.rb

@@ -589,4 +589,15 @@
     buffer.read_all
     expect(ObjectSpace.memsize_of(buffer)).to be == empty_size
   end
+
+  it "doesn't crash when marking an uninitialized buffer" do
+    stress = GC.stress
+    begin
+      GC.stress = true
+
+      MessagePack::Buffer.new
+    ensure
+      GC.stress = stress
+    end
+  end
 end

spec/packer_spec.rb

@@ -572,4 +572,16 @@ def to_msgpack_ext
         [0xc9, 65538, -1].pack('CNC') + "a"*65538
     end
   end
+
+  it "doesn't crash when marking an uninitialized buffer" do
+    stress = GC.stress
+    begin
+      GC.stress = true
+
+      MessagePack::Packer.new.buffer
+      Object.new
+    ensure
+      GC.stress = stress
+    end
+  end
 end

spec/unpacker_spec.rb

@@ -866,4 +866,20 @@ def flatten(struct, results = [])
       end
     end
   end
+
+  it "doesn't crash when marking an uninitialized buffer" do
+    if RUBY_PLATFORM == "java"
+      pending("THe java extension is missing Unpacker#buffer https://github.com/msgpack/msgpack-ruby/issues/315")
+    end
+
+    stress = GC.stress
+    begin
+      GC.stress = true
+
+      MessagePack::Unpacker.new.buffer
+      Object.new
+    ensure
+      GC.stress = stress
+    end
+  end
 end