diff --git a/src/alloc.c b/src/alloc.c
index af08336177..0bea10610f 100644
--- a/src/alloc.c
+++ b/src/alloc.c
@@ -3157,8 +3157,8 @@ cleanup_vector (struct Lisp_Vector *vector)
     {
       struct Lisp_Native_Comp_Unit *cu =
 	PSEUDOVEC_STRUCT (vector, Lisp_Native_Comp_Unit);
-      eassert (cu->handle);
-      dynlib_close (cu->handle);
+      if (cu->handle)
+	dynlib_close (cu->handle);
     }
   else if (NATIVE_COMP_FLAG
 	   && PSEUDOVECTOR_TYPEP (&vector->header, PVEC_SUBR))
diff --git a/src/comp.c b/src/comp.c
index e6f672de25..abc3535dc6 100644
--- a/src/comp.c
+++ b/src/comp.c
@@ -4832,6 +4832,10 @@ load_comp_unit (struct Lisp_Native_Comp_Unit *comp_u, bool loading_dump,
        We must *never* mess with static pointers in an already loaded
        eln.  */
     {
+      /* Invalidate the handle for the CU we are leaving for garbage
+	 collection.  */
+      comp_u->handle = NULL;
+      /* Swap CU for the old one.  */
       comp_u_lisp_obj = *saved_cu;
       comp_u = XNATIVE_COMP_UNIT (comp_u_lisp_obj);
       comp_u->loaded_once = true;