Harden allocation failure handling and size arithmetic

- check malloc/calloc/realloc results in constructors, the JSON and
  OpenStep writers, the XML text-part helpers and base64 decoding, and
  return PLIST_ERR_NO_MEM instead of asserting or dereferencing NULL
- guard size calculations against overflow in the UTF-8/UTF-16BE
  conversions, ptrarray, and bytearray
- make hash_table_insert() report failure instead of silently dropping
  the entry; writers propagate PLIST_ERR_NO_MEM, and dict lookup caches
  are discarded so lookups fall back to a linear search
- don't overwrite an earlier parse error code in the XML parser, and
  report PLIST_ERR_NO_MEM when reading text content fails for lack of memory
- OpenStep parser: check node/data/string allocations and dict/array
  insertion results

Based on #323 by bg0d-droid.

Co-authored-by: bg0d-droid <81825610+bg0d-droid@users.noreply.github.com>

1 files changed, 21 insertions, 3 deletions

diff --git a/src/ptrarray.c b/src/ptrarray.c
index bdcd428..eb9ff0a 100644
--- a/src/ptrarray.c
+++ b/src/ptrarray.c
@@ -20,11 +20,24 @@
  */
 #include "ptrarray.h"
 #include <string.h>
+#include <stdint.h>
+#include <limits.h>
 
 ptrarray_t *ptr_array_new(int capacity)
 {
-	ptrarray_t *pa = (ptrarray_t*)malloc(sizeof(ptrarray_t));
-	pa->pdata = (void**)malloc(sizeof(void*) * capacity);
+	ptrarray_t *pa;
+	if (capacity < 0 || (size_t)capacity > SIZE_MAX / sizeof(void*)) {
+		return NULL;
+	}
+	pa = (ptrarray_t*)malloc(sizeof(ptrarray_t));
+	if (!pa) {
+		return NULL;
+	}
+	pa->pdata = (void**)malloc(sizeof(void*) * (size_t)capacity);
+	if (!pa->pdata) {
+		free(pa);
+		return NULL;
+	}
 	pa->capacity = capacity;
 	pa->capacity_step = (capacity > 4096) ? 4096 : capacity;
 	pa->len = 0;
@@ -45,7 +58,12 @@ int ptr_array_insert(ptrarray_t *pa, void *data, long array_index)
 	if (!pa || !pa->pdata) return -1;
 	long remaining = pa->capacity-pa->len;
 	if (remaining == 0) {
-		void **newdata = (void**)realloc(pa->pdata, sizeof(void*) * (pa->capacity + pa->capacity_step));
+		void **newdata;
+		if (pa->capacity > INT_MAX - pa->capacity_step ||
+		    (size_t)(pa->capacity + pa->capacity_step) > SIZE_MAX / sizeof(void*)) {
+			return -1;
+		}
+		newdata = (void**)realloc(pa->pdata, sizeof(void*) * (size_t)(pa->capacity + pa->capacity_step));
 		if (!newdata) {
 			return -1;
 		}