diff --git a/lisp/auth-source.el b/lisp/auth-source.el
index 90b58f560c0..274578185d2 100644
--- a/lisp/auth-source.el
+++ b/lisp/auth-source.el
@@ -2467,14 +2467,11 @@ read-passwd--hide-password-icon
   :version "30.1"
   :help-echo "mouse-1: Toggle password visibility")
 
-(defvar read-passwd--mode-line-buffer nil
-  "Buffer to modify `mode-line-format' for showing/hiding passwords.")
-
 (defvar read-passwd--mode-line-icon nil
   "Propertized mode line icon for showing/hiding passwords.")
 
 (defvar read-passwd--hide-password t
-  "Toggle whether password should be hidden in minubuffer.")
+  "Toggle whether password should be hidden in minibuffer.")
 
 (defun read-passwd--hide-password ()
   "Make password in minibuffer hidden or visible."
@@ -2492,30 +2489,25 @@ read-passwd-toggle-visibility
   "Toggle minibuffer contents visibility.
 Adapt also mode line."
   (interactive)
-  (let ((win (active-minibuffer-window)))
-    (unless win (error "No active minibuffer"))
-    ;; FIXME: In case of a recursive minibuffer, this may select the wrong
-    ;; mini-buffer.
-    (with-current-buffer (window-buffer win)
-      (setq read-passwd--hide-password (not read-passwd--hide-password))
-      (with-current-buffer read-passwd--mode-line-buffer
-        (setq read-passwd--mode-line-icon
-              `(:propertize
-                ,(if icon-preference
-                     (icon-string
-                      (if read-passwd--hide-password
-                          'read-passwd--show-password-icon
-                        'read-passwd--hide-password-icon))
-                   "")
-                mouse-face mode-line-highlight
-                keymap
-                ,(eval-when-compile
-                   (let ((map (make-sparse-keymap)))
-                     (define-key map [mode-line mouse-1]
-                                 #'read-passwd-toggle-visibility)
-                     map))))
-        (force-mode-line-update))
-      (read-passwd--hide-password))))
+  (when (minibufferp)
+    (setq read-passwd--hide-password (not read-passwd--hide-password))
+    (setq read-passwd--mode-line-icon
+          `(:propertize
+            ,(if icon-preference
+                 (icon-string
+                  (if read-passwd--hide-password
+                      'read-passwd--show-password-icon
+                    'read-passwd--hide-password-icon))
+               "")
+            mouse-face mode-line-highlight
+            keymap
+            ,(eval-when-compile
+               (let ((map (make-sparse-keymap)))
+                 (define-key map [mode-line mouse-1]
+                             #'read-passwd-toggle-visibility)
+                 map))))
+    (force-mode-line-update 'all)
+    (read-passwd--hide-password)))
 
 (defvar read-passwd-map
   ;; BEWARE: `defconst' would purecopy it, breaking the sharing with
@@ -2534,26 +2526,25 @@ read-passwd-mode
   :keymap read-passwd-map
   :version "30.1"
 
-  (setq read-passwd--hide-password nil
-        ;; Stolen from `eldoc-minibuffer-message'.
-        read-passwd--mode-line-buffer
-        (window-buffer
-         (or (window-in-direction 'above (minibuffer-window))
-	     (minibuffer-selected-window)
-	     (get-largest-window))))
-
-  (if read-passwd-mode
-      (with-current-buffer read-passwd--mode-line-buffer
-        ;; Add `read-passwd--mode-line-icon'.
-        (when (listp mode-line-format)
-          (setq mode-line-format
-                (cons '(:eval read-passwd--mode-line-icon)
-	              mode-line-format))))
-    (with-current-buffer read-passwd--mode-line-buffer
-      ;; Remove `read-passwd--mode-line-icon'.
-      (when (listp mode-line-format)
-        (setq mode-line-format (cdr mode-line-format)))))
+  (setq read-passwd--hide-password nil)
+  (or mode-line-front-space
+      (setq mode-line-front-space
+	    '(:eval (if (display-graphic-p) " " "-"))))
 
+  (let ((mode-line-string '(:eval read-passwd--mode-line-icon)))
+    (cond
+     (read-passwd-mode
+      ;; Add `read-passwd--mode-line-icon'.
+      (setq mode-line-front-space
+	    (cons mode-line-string mode-line-front-space)))
+     ;; Remove `read-passwd--mode-line-icon'.
+     ((equal (car mode-line-front-space)
+	     mode-line-string)
+      (setq mode-line-front-space (cdr mode-line-front-space)))
+     (t
+      (delete mode-line-string mode-line-front-space))))
+
+  (force-mode-line-update 'all)
   (when read-passwd-mode
     (read-passwd-toggle-visibility)))