diff --git a/lisp/progmodes/xref.el b/lisp/progmodes/xref.el
index dbafa00c3ad..9b64ef4de10 100644
--- a/lisp/progmodes/xref.el
+++ b/lisp/progmodes/xref.el
@@ -1468,7 +1468,6 @@ xref-show-xrefs
   (xref--show-xrefs fetcher display-action))
 
 (defun xref--show-xrefs (fetcher display-action &optional _always-show-list)
-  (xref--push-markers)
   (unless (functionp fetcher)
     ;; Old convention.
     (let ((xrefs fetcher))
@@ -1479,21 +1478,30 @@ xref--show-xrefs
                 (prog1
                     xrefs
                   (setq xrefs 'called-already)))))))
-  (funcall xref-show-xrefs-function fetcher
-           `((window . ,(selected-window))
-             (display-action . ,display-action)
-             (auto-jump . ,xref-auto-jump-to-first-xref))))
+  (let ((cb (current-buffer))
+        (pt (point)))
+    (funcall xref-show-xrefs-function fetcher
+             `((window . ,(selected-window))
+               (display-action . ,display-action)
+               (auto-jump . ,xref-auto-jump-to-first-xref)))
+    (xref--push-markers cb pt)))
 
 (defun xref--show-defs (xrefs display-action)
-  (xref--push-markers)
-  (funcall xref-show-definitions-function xrefs
-           `((window . ,(selected-window))
-             (display-action . ,display-action)
-             (auto-jump . ,xref-auto-jump-to-first-definition))))
-
-(defun xref--push-markers ()
-  (unless (region-active-p) (push-mark nil t))
-  (xref-push-marker-stack))
+  (let ((cb (current-buffer))
+        (pt (point)))
+    (funcall xref-show-definitions-function xrefs
+             `((window . ,(selected-window))
+               (display-action . ,display-action)
+               (auto-jump . ,xref-auto-jump-to-first-definition)))
+    (xref--push-markers cb pt)))
+
+(defun xref--push-markers (buf pt)
+  (when (buffer-live-p buf)
+    (save-excursion
+      (with-no-warnings (set-buffer buf))
+      (goto-char pt)
+      (unless (region-active-p) (push-mark nil t))
+      (xref-push-marker-stack))))
 
 (defun xref--prompt-p (command)
   (or (eq xref-prompt-for-identifier t)