diff --git a/packages/rich-text/src/component/use-anchor.js b/packages/rich-text/src/component/use-anchor.js
index af9e0e4d87dc3e..7ea3281cb60d3a 100644
--- a/packages/rich-text/src/component/use-anchor.js
+++ b/packages/rich-text/src/component/use-anchor.js
@@ -168,7 +168,12 @@ export function useAnchor( { editableContentElement, settings = {} } ) {
 		editableContentElement.addEventListener( 'focusin', attach );
 		editableContentElement.addEventListener( 'focusout', detach );
 
-		return detach;
+		return () => {
+			detach();
+
+			editableContentElement.removeEventListener( 'focusin', attach );
+			editableContentElement.removeEventListener( 'focusout', detach );
+		};
 	}, [ editableContentElement, tagName, className ] );
 
 	return anchor;