diff --git a/libavcodec/aac/aacdec.c b/libavcodec/aac/aacdec.c
index 4920f5e77f..325e52834f 100644
--- a/libavcodec/aac/aacdec.c
+++ b/libavcodec/aac/aacdec.c
@@ -62,6 +62,7 @@
 #include "libavutil/opt.h"
 #include "libavutil/tx.h"
 #include "libavutil/version.h"
+#include "libavcodec/refstruct.h"
 
 /*
  * supported tools
@@ -421,6 +422,26 @@ static uint64_t sniff_channel_order(uint8_t (*layout_map)[3], int tags)
     return layout;
 }
 
+static void copy_oc(OutputConfiguration *dst, OutputConfiguration *src)
+{
+    int i;
+
+    for (i = 0; i < src->usac.nb_elems; i++) {
+        AACUsacElemConfig *src_e = &src->usac.elems[i];
+        AACUsacElemConfig *dst_e = &dst->usac.elems[i];
+        /* dst_e->ext.pl_buf is guaranteed to be set to src_e->ext.pl_buf
+         * upon this function's return */
+        ff_refstruct_replace(&dst_e->ext.pl_buf, src_e->ext.pl_buf);
+    }
+
+    /* Unref all additional buffers to close leaks */
+    for (; i < dst->usac.nb_elems; i++)
+        ff_refstruct_unref(&dst->usac.elems[i].ext.pl_buf);
+
+    /* Set all other properties */
+    *dst = *src;
+}
+
 /**
  * Save current output configuration if and only if it has been locked.
  */
@@ -429,7 +450,7 @@ static int push_output_configuration(AACDecContext *ac)
     int pushed = 0;
 
     if (ac->oc[1].status == OC_LOCKED || ac->oc[0].status == OC_NONE) {
-        ac->oc[0] = ac->oc[1];
+        copy_oc(&ac->oc[0], &ac->oc[1]);
         pushed = 1;
     }
     ac->oc[1].status = OC_NONE;
@@ -443,7 +464,8 @@ static int push_output_configuration(AACDecContext *ac)
 static void pop_output_configuration(AACDecContext *ac)
 {
     if (ac->oc[1].status != OC_LOCKED && ac->oc[0].status != OC_NONE) {
-        ac->oc[1] = ac->oc[0];
+        copy_oc(&ac->oc[1], &ac->oc[0]);
+
         ac->avctx->ch_layout = ac->oc[1].ch_layout;
         ff_aac_output_configure(ac, ac->oc[1].layout_map, ac->oc[1].layout_map_tags,
                                 ac->oc[1].status, 0);
@@ -1110,7 +1132,7 @@ static av_cold int decode_close(AVCodecContext *avctx)
         AACUSACConfig *usac = &oc->usac;
         for (int j = 0; j < usac->nb_elems; j++) {
             AACUsacElemConfig *ec = &usac->elems[j];
-            av_freep(&ec->ext.pl_data);
+            ff_refstruct_unref(&ec->ext.pl_buf);
         }
     }
 
diff --git a/libavcodec/aac/aacdec.h b/libavcodec/aac/aacdec.h
index e5a79a7139..b3763fdccc 100644
--- a/libavcodec/aac/aacdec.h
+++ b/libavcodec/aac/aacdec.h
@@ -344,7 +344,7 @@ typedef struct AACUsacElemConfig {
         uint8_t payload_frag;
         uint32_t default_len;
         uint32_t pl_data_offset;
-        uint8_t *pl_data;
+        uint8_t *pl_buf;
     } ext;
 } AACUsacElemConfig;
 
@@ -353,7 +353,7 @@ typedef struct AACUSACConfig {
     uint16_t core_frame_len;
     uint16_t stream_identifier;
 
-    AACUsacElemConfig elems[64];
+    AACUsacElemConfig elems[MAX_ELEM_ID];
     int nb_elems;
 
     struct {
diff --git a/libavcodec/aac/aacdec_usac.c b/libavcodec/aac/aacdec_usac.c
index 3e5d31676c..6612523043 100644
--- a/libavcodec/aac/aacdec_usac.c
+++ b/libavcodec/aac/aacdec_usac.c
@@ -24,12 +24,13 @@
 #include "aacdec_ac.h"
 
 #include "libavcodec/aacsbr.h"
-
 #include "libavcodec/aactab.h"
-#include "libavutil/mem.h"
 #include "libavcodec/mpeg4audio.h"
 #include "libavcodec/unary.h"
 
+#include "libavutil/mem.h"
+#include "libavcodec/refstruct.h"
+
 /* Number of scalefactor bands per complex prediction band, equal to 2. */
 #define SFB_PER_PRED_BAND 2
 
@@ -1574,7 +1575,6 @@ static int parse_audio_preroll(AACDecContext *ac, GetBitContext *gb)
 static int parse_ext_ele(AACDecContext *ac, AACUsacElemConfig *e,
                          GetBitContext *gb)
 {
-    uint8_t *tmp;
     uint8_t pl_frag_start = 1;
     uint8_t pl_frag_end = 1;
     uint32_t len;
@@ -1601,18 +1601,26 @@ static int parse_ext_ele(AACDecContext *ac, AACUsacElemConfig *e,
     if (pl_frag_start)
         e->ext.pl_data_offset = 0;
 
-    /* If an extension starts and ends this packet, we can directly use it */
+    /* If an extension starts and ends this packet, we can directly use it below.
+     * Otherwise, we have to copy it to a buffer and accumulate it. */
     if (!(pl_frag_start && pl_frag_end)) {
-        tmp = av_realloc(e->ext.pl_data, e->ext.pl_data_offset + len);
-        if (!tmp) {
-            av_free(e->ext.pl_data);
+        /* Reallocate the data */
+        uint8_t *tmp_buf = ff_refstruct_alloc_ext(e->ext.pl_data_offset + len,
+                                                  FF_REFSTRUCT_FLAG_NO_ZEROING,
+                                                  NULL, NULL);
+        if (!tmp_buf)
             return AVERROR(ENOMEM);
-        }
-        e->ext.pl_data = tmp;
+
+        /* Copy the data over only if we had saved data to begin with */
+        if (e->ext.pl_buf)
+            memcpy(tmp_buf, e->ext.pl_buf, e->ext.pl_data_offset);
+
+        ff_refstruct_unref(&e->ext.pl_buf);
+        e->ext.pl_buf = tmp_buf;
 
         /* Readout data to a buffer */
         for (int i = 0; i < len; i++)
-            e->ext.pl_data[e->ext.pl_data_offset + i] = get_bits(gb, 8);
+            e->ext.pl_buf[e->ext.pl_data_offset + i] = get_bits(gb, 8);
     }
 
     e->ext.pl_data_offset += len;
@@ -1624,7 +1632,7 @@ static int parse_ext_ele(AACDecContext *ac, AACUsacElemConfig *e,
         GetBitContext *gb2 = gb;
         GetBitContext gbc;
         if (!(pl_frag_start && pl_frag_end)) {
-            ret = init_get_bits8(&gbc, e->ext.pl_data, pl_len);
+            ret = init_get_bits8(&gbc, e->ext.pl_buf, pl_len);
             if (ret < 0)
                 return ret;
 
@@ -1642,7 +1650,7 @@ static int parse_ext_ele(AACDecContext *ac, AACUsacElemConfig *e,
             /* This should never happen */
             av_assert0(0);
         }
-        av_freep(&e->ext.pl_data);
+        ff_refstruct_unref(&e->ext.pl_buf);
         if (ret < 0)
             return ret;