Harden allocation failure handling and size arithmetic

- check malloc/calloc/realloc results in constructors, the JSON and
  OpenStep writers, the XML text-part helpers and base64 decoding, and
  return PLIST_ERR_NO_MEM instead of asserting or dereferencing NULL
- guard size calculations against overflow in the UTF-8/UTF-16BE
  conversions, ptrarray, and bytearray
- make hash_table_insert() report failure instead of silently dropping
  the entry; writers propagate PLIST_ERR_NO_MEM, and dict lookup caches
  are discarded so lookups fall back to a linear search
- don't overwrite an earlier parse error code in the XML parser, and
  report PLIST_ERR_NO_MEM when reading text content fails for lack of memory
- OpenStep parser: check node/data/string allocations and dict/array
  insertion results

Based on #323 by bg0d-droid.

Co-authored-by: bg0d-droid <81825610+bg0d-droid@users.noreply.github.com>

1 files changed, 16 insertions, 3 deletions

diff --git a/src/bplist.c b/src/bplist.c
index e5ace3a..45caae4 100644
--- a/src/bplist.c
+++ b/src/bplist.c
@@ -397,7 +397,11 @@ static char *plist_utf16be_to_utf8(uint16_t *unistr, size_t len, size_t *items_r
 	uint32_t w;
 	int read_lead_surrogate = 0;
 
-	/* allocate with enough space */
+	/* allocate with enough space, guarding against integer overflow */
+	if (len > (SIZE_MAX / 4) - 1) {
+		PLIST_BIN_ERR("%s: Input length %" PRIu64 " is too large\n", __func__, (uint64_t)len);
+		return NULL;
+	}
 	outbuf = (char*)malloc(4*(len+1));
 	if (!outbuf) {
 		PLIST_BIN_ERR("%s: Could not allocate %" PRIu64 " bytes\n", __func__, (uint64_t)(4*(len+1)));
@@ -1112,13 +1116,18 @@ static plist_err_t serialize_plist(node_t node, void* data, uint32_t depth)
     }
 
     // mark as active
-    hash_table_insert(ser->in_stack, node, (void*)1);
+    if (hash_table_insert(ser->in_stack, node, (void*)1) < 0) {
+        return PLIST_ERR_NO_MEM;
+    }
 
     // insert new ref
     index_val = (uint64_t *) malloc(sizeof(uint64_t));
     if (!index_val) return PLIST_ERR_NO_MEM;
     *index_val = ser->objects->len;
-    hash_table_insert(ser->ref_table, node, index_val);
+    if (hash_table_insert(ser->ref_table, node, index_val) < 0) {
+        free(index_val);
+        return PLIST_ERR_NO_MEM;
+    }
 
     // now append current node to object array
     if (ptr_array_add(ser->objects, node) < 0) {
@@ -1228,6 +1237,10 @@ static uint16_t *plist_utf8_to_utf16be(const unsigned char *unistr, size_t size,
     unsigned char c2;
     unsigned char c3;
 
+    if (size > (SIZE_MAX / sizeof(uint16_t) - 1) / 2) {
+        PLIST_BIN_ERR("%s: Input size %" PRIu64 " is too large\n", __func__, (uint64_t)size);
+        return NULL;
+    }
     outbuf = (uint16_t*)malloc(((size*2)+1)*sizeof(uint16_t));
     if (!outbuf) {
         PLIST_BIN_ERR("%s: Could not allocate %" PRIu64 " bytes\n", __func__, (uint64_t)((size*2)+1)*sizeof(uint16_t));