Harden allocation failure handling and size arithmetic

- check malloc/calloc/realloc results in constructors, the JSON and
  OpenStep writers, the XML text-part helpers and base64 decoding, and
  return PLIST_ERR_NO_MEM instead of asserting or dereferencing NULL
- guard size calculations against overflow in the UTF-8/UTF-16BE
  conversions, ptrarray, and bytearray
- make hash_table_insert() report failure instead of silently dropping
  the entry; writers propagate PLIST_ERR_NO_MEM, and dict lookup caches
  are discarded so lookups fall back to a linear search
- don't overwrite an earlier parse error code in the XML parser, and
  report PLIST_ERR_NO_MEM when reading text content fails for lack of memory
- OpenStep parser: check node/data/string allocations and dict/array
  insertion results

Based on #323 by bg0d-droid.

Co-authored-by: bg0d-droid <81825610+bg0d-droid@users.noreply.github.com>

1 files changed, 12 insertions, 1 deletions

diff --git a/src/jplist.c b/src/jplist.c
index d065f4d..fe0b5cb 100644
--- a/src/jplist.c
+++ b/src/jplist.c
@@ -126,6 +126,9 @@ static plist_err_t node_to_json(node_t node, bytearray_t **outbuf, uint32_t dept
 
     case PLIST_INT:
         val = (char*)malloc(64);
+        if (!val) {
+            return PLIST_ERR_NO_MEM;
+        }
         if (node_data->length == 16) {
             val_len = snprintf(val, 64, "%" PRIu64, node_data->intval);
         } else {
@@ -137,6 +140,9 @@ static plist_err_t node_to_json(node_t node, bytearray_t **outbuf, uint32_t dept
 
     case PLIST_REAL:
         val = (char*)malloc(64);
+        if (!val) {
+            return PLIST_ERR_NO_MEM;
+        }
         val_len = dtostr(val, 64, node_data->realval);
         str_buf_append(*outbuf, val, val_len);
         free(val);
@@ -285,6 +291,9 @@ static plist_err_t node_to_json(node_t node, bytearray_t **outbuf, uint32_t dept
     case PLIST_UID:
         if (coerce) {
             val = (char*)malloc(64);
+            if (!val) {
+                return PLIST_ERR_NO_MEM;
+            }
             if (node_data->length == 16) {
                 val_len = snprintf(val, 64, "%" PRIu64, node_data->intval);
             } else {
@@ -322,7 +331,9 @@ static plist_err_t _node_estimate_size(node_t node, uint64_t *size, uint32_t dep
     }
 
     // mark as visited
-    hash_table_insert(visited, node, (void*)1);
+    if (hash_table_insert(visited, node, (void*)1) < 0) {
+        return PLIST_ERR_NO_MEM;
+    }
 
     data = plist_get_data(node);
     if (node->children) {