Packer: store the parent buffer mapped strings on the stack

Since the parent buffer isn't marked while we're executing the
extension proc, we need to ensure all the chunks `mapped_strings`
won't be GCed nor moved.

To do that, we copy all of them onto the stack.

It's a bit disgusting, but until we refactor the buffer implementation
to support recursive extensions better, it's the simplest solution.

Author: byroot

ChangeLog

@@ -1,3 +1,4 @@
+* Fix a potential GC bug when packing data using recursive extensions and buffers containing over 512KkiB of data (See #341).
 * Fix a regression where feeding an empty string to an Unpacker would be considered like the end of the buffer.
 
 2023-05-19 1.7.1:

ext/msgpack/packer.c

@@ -114,6 +114,40 @@ bool msgpack_packer_try_write_with_ext_type_lookup(msgpack_packer_t* pk, VALUE v
     }
 
     if(ext_flags & MSGPACK_EXT_RECURSIVE) {
+        // HACK: While we call the proc, the current pk->buffer won't be reachable
+        // as it will be stored on the stack.
+        // To ensure all the `mapped_string` reference in that buffer are properly
+        // marked and pined, we copy them all on the stack.
+        VALUE* mapped_strings = NULL;
+        size_t mapped_strings_count = 0;
+        msgpack_buffer_chunk_t* c = pk->buffer.head;
+        while (c != &pk->buffer.tail) {
+            if (c->mapped_string != NO_MAPPED_STRING) {
+                mapped_strings_count++;
+            }
+            c = c->next;
+        }
+        if (c->mapped_string != NO_MAPPED_STRING) {
+            mapped_strings_count++;
+        }
+
+        if (mapped_strings_count > 0) {
+            mapped_strings = ALLOCA_N(VALUE, mapped_strings_count);
+            mapped_strings_count = 0;
+            c = pk->buffer.head;
+            while (c != &pk->buffer.tail) {
+                if (c->mapped_string != NO_MAPPED_STRING) {
+                    mapped_strings[mapped_strings_count] = c->mapped_string;
+                    mapped_strings_count++;
+                }
+                c = c->next;
+            }
+            if (c->mapped_string != NO_MAPPED_STRING) {
+                mapped_strings[mapped_strings_count] = c->mapped_string;
+                mapped_strings_count++;
+            }
+        }
+
         msgpack_buffer_t parent_buffer = pk->buffer;
         msgpack_buffer_init(PACKER_BUFFER_(pk));
 
@@ -132,6 +166,10 @@ bool msgpack_packer_try_write_with_ext_type_lookup(msgpack_packer_t* pk, VALUE v
             pk->buffer = parent_buffer;
             msgpack_packer_write_ext(pk, ext_type, payload);
         }
+
+        if (mapped_strings_count > 0) {
+            RB_GC_GUARD(mapped_strings[0]);
+        }
     } else {
         VALUE payload = rb_proc_call_with_block(proc, 1, &v, Qnil);
         StringValue(payload);