bytearray: Handle realloc() failure in byte_array_grow()

byte_array_grow() assigned the result of realloc() directly to
ba->data and increased the capacity even if realloc() failed, so the
following memcpy() in byte_array_append() wrote to NULL + len.

On failure, free the old buffer and leave the byte array in a failed
state (data == NULL), which byte_array_append() already ignores. The
callers check for that state:

- the writers (bin, xml, json, openstep and the text output formats)
  return PLIST_ERR_NO_MEM instead of handing out a NULL/truncated buffer
- node_to_xml() base64-encodes <data> directly into the grown buffer,
  so it must bail out there (only guarded by assert() before)
- the OpenStep parser returns PLIST_ERR_NO_MEM for <hex data> instead of
  silently returning truncated data

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

8 files changed, 72 insertions, 2 deletions

diff --git a/src/bplist.c b/src/bplist.c
index ea829c7..8168572 100644
--- a/src/bplist.c
+++ b/src/bplist.c
@@ -1612,6 +1612,13 @@ plist_err_t plist_to_bin(plist_t plist, char **plist_bin, uint32_t * length)
 
     byte_array_append(bplist_buff, &trailer, sizeof(bplist_trailer_t));
 
+    if (!bplist_buff->data) {
+        byte_array_free(bplist_buff);
+        *plist_bin = NULL;
+        *length = 0;
+        return PLIST_ERR_NO_MEM;
+    }
+
     //set output buffer and size
     *plist_bin = (char*)bplist_buff->data;
     *length = bplist_buff->len;
diff --git a/src/bytearray.c b/src/bytearray.c
index 39fad5f..05ea0bb 100644
--- a/src/bytearray.c
+++ b/src/bytearray.c
@@ -54,11 +54,21 @@ void byte_array_free(bytearray_t *ba)
 
 void byte_array_grow(bytearray_t *ba, size_t amount)
 {
-	if (ba->stream) {
+	if (ba->stream || !ba->data) {
 		return;
 	}
 	size_t increase = (amount > PAGE_SIZE) ? (amount+(PAGE_SIZE-1)) & (~(PAGE_SIZE-1)) : PAGE_SIZE;
-	ba->data = realloc(ba->data, ba->capacity + increase);
+	void *newdata = realloc(ba->data, ba->capacity + increase);
+	if (!newdata) {
+		/* out of memory: put the array into a failed state (data == NULL),
+		 * further appends are ignored and callers must check ba->data */
+		free(ba->data);
+		ba->data = NULL;
+		ba->len = 0;
+		ba->capacity = 0;
+		return;
+	}
+	ba->data = newdata;
 	ba->capacity += increase;
 }
 
@@ -76,6 +86,9 @@ void byte_array_append(bytearray_t *ba, void *buf, size_t len)
 		if (len > remaining) {
 			size_t needed = len - remaining;
 			byte_array_grow(ba, needed);
+			if (!ba->data) {
+				return;
+			}
 		}
 		memcpy(((char*)ba->data) + ba->len, buf, len);
 	}
diff --git a/src/jplist.c b/src/jplist.c
index c29f760..d065f4d 100644
--- a/src/jplist.c
+++ b/src/jplist.c
@@ -480,6 +480,13 @@ plist_err_t plist_to_json_with_options(plist_t plist, char **plist_json, uint32_
 
     str_buf_append(outbuf, "\0", 1);
 
+    if (!outbuf->data) {
+        str_buf_free(outbuf);
+        *plist_json = NULL;
+        *length = 0;
+        return PLIST_ERR_NO_MEM;
+    }
+
     *plist_json = (char*)outbuf->data;
     *length = outbuf->len - 1;
 
diff --git a/src/oplist.c b/src/oplist.c
index 77a99cd..a635302 100644
--- a/src/oplist.c
+++ b/src/oplist.c
@@ -531,6 +531,13 @@ plist_err_t plist_to_openstep_with_options(plist_t plist, char **openstep, uint3
 
     str_buf_append(outbuf, "\0", 1);
 
+    if (!outbuf->data) {
+        str_buf_free(outbuf);
+        *openstep = NULL;
+        *length = 0;
+        return PLIST_ERR_NO_MEM;
+    }
+
     *openstep = (char*)outbuf->data;
     *length = outbuf->len - 1;
 
@@ -780,6 +787,11 @@ static plist_err_t node_from_openstep(parse_ctx ctx, plist_t *plist)
                 }
                 b = (b << 4) + HEX_DIGIT(*ctx->pos);
                 byte_array_append(bytes, &b, 1);
+                if (!bytes->data) {
+                    PLIST_OSTEP_ERR("Out of memory while parsing data at offset %ld\n", (long int)(ctx->pos - ctx->start));
+                    ctx->err = PLIST_ERR_NO_MEM;
+                    break;
+                }
                 ctx->pos++;
             }
             if (ctx->err) {
diff --git a/src/out-default.c b/src/out-default.c
index 13b9d9c..845be3c 100644
--- a/src/out-default.c
+++ b/src/out-default.c
@@ -454,6 +454,13 @@ plist_err_t plist_write_to_string_default(plist_t plist, char **output, uint32_t
     }
     str_buf_append(outbuf, "\0", 1);
 
+    if (!outbuf->data) {
+        str_buf_free(outbuf);
+        *output = NULL;
+        *length = 0;
+        return PLIST_ERR_NO_MEM;
+    }
+
     *output = (char*)outbuf->data;
     *length = outbuf->len - 1;
 
diff --git a/src/out-limd.c b/src/out-limd.c
index 83a5e26..259d036 100644
--- a/src/out-limd.c
+++ b/src/out-limd.c
@@ -435,6 +435,13 @@ plist_err_t plist_write_to_string_limd(plist_t plist, char **output, uint32_t* l
     }
     str_buf_append(outbuf, "\0", 1);
 
+    if (!outbuf->data) {
+        str_buf_free(outbuf);
+        *output = NULL;
+        *length = 0;
+        return PLIST_ERR_NO_MEM;
+    }
+
     *output = (char*)outbuf->data;
     *length = outbuf->len - 1;
 
diff --git a/src/out-plutil.c b/src/out-plutil.c
index e603f31..ef17b7c 100644
--- a/src/out-plutil.c
+++ b/src/out-plutil.c
@@ -443,6 +443,13 @@ plist_err_t plist_write_to_string_plutil(plist_t plist, char **output, uint32_t*
     }
     str_buf_append(outbuf, "\0", 1);
 
+    if (!outbuf->data) {
+        str_buf_free(outbuf);
+        *output = NULL;
+        *length = 0;
+        return PLIST_ERR_NO_MEM;
+    }
+
     *output = (char*)outbuf->data;
     *length = outbuf->len - 1;
 
diff --git a/src/xplist.c b/src/xplist.c
index b2c134e..4e6e009 100644
--- a/src/xplist.c
+++ b/src/xplist.c
@@ -279,6 +279,9 @@ static plist_err_t node_to_xml(node_t node, bytearray_t **outbuf, uint32_t depth
             size_t amount = (node_data->length / 3 * 4) + 4 + (((node_data->length / maxread) + 1) * (indent+1));
             if ((*outbuf)->len + amount > (*outbuf)->capacity) {
                 str_buf_grow(*outbuf, amount);
+                if (!(*outbuf)->data) {
+                    return PLIST_ERR_NO_MEM;
+                }
             }
             while (j < node_data->length) {
                 for (i = 0; i < indent; i++) {
@@ -545,6 +548,13 @@ plist_err_t plist_to_xml(plist_t plist, char **plist_xml, uint32_t * length)
 
     str_buf_append(outbuf, XML_PLIST_EPILOG, sizeof(XML_PLIST_EPILOG));
 
+    if (!outbuf->data) {
+        str_buf_free(outbuf);
+        *plist_xml = NULL;
+        *length = 0;
+        return PLIST_ERR_NO_MEM;
+    }
+
     *plist_xml = (char*)outbuf->data;
     *length = outbuf->len - 1;