diff --git a/lisp/progmodes/xref.el b/lisp/progmodes/xref.el
index 0213ab3cc58..49ad3500d15 100644
--- a/lisp/progmodes/xref.el
+++ b/lisp/progmodes/xref.el
@@ -771,19 +771,22 @@ xref-query-replace-in-results
                       (format "Query-replace (regexp) %s with: " fr)
                     "Query-replace all matches with: ")))
      (list fr (read-regexp prompt))))
-  (let* (item xrefs iter)
+  (let* ((positions (make-hash-table :test 'eq))
+         item xrefs iter)
     (save-excursion
       (while (setq item (xref--search-property 'xref-item))
         (when (xref-match-length item)
+          (puthash item (point) positions)
           (push item xrefs))))
     (unwind-protect
         (progn
           (goto-char (point-min))
-          (setq iter (xref--buf-pairs-iterator (nreverse xrefs)))
-          (xref--query-replace-1 from to iter))
+          (setq iter (xref--buf-pairs-iterator (nreverse xrefs)
+                                               positions))
+          (xref--query-replace-1 from to iter (current-buffer)))
       (funcall iter :cleanup))))
 
-(defun xref--buf-pairs-iterator (xrefs)
+(defun xref--buf-pairs-iterator (xrefs positions)
   (let (chunk-done item next-pair file-buf pairs all-pairs)
     (lambda (action)
       (pcase action
@@ -801,9 +804,10 @@ xref--buf-pairs-iterator
                       (beg (xref-location-marker loc))
                       (end (move-marker (make-marker)
                                         (+ beg (xref-match-length item))
-                                        (marker-buffer beg))))
-                 (let ((pair (cons beg end)))
-                   (push pair all-pairs)
+                                        (marker-buffer beg)))
+                      (xref-buf-pos (gethash item positions)))
+                 (let ((positions (list beg end xref-buf-pos)))
+                   (push positions all-pairs)
                    ;; Perform sanity check first.
                    (xref--goto-location loc)
                    (if (xref--outdated-p item)
@@ -811,17 +815,17 @@ xref--buf-pairs-iterator
                      (cond
                       ((null file-buf)
                        (setq file-buf (marker-buffer beg))
-                       (push pair pairs))
+                       (push positions pairs))
                       ((equal file-buf (marker-buffer beg))
-                       (push pair pairs))
+                       (push positions pairs))
                       (t
                        (setq chunk-done t
-                             next-pair pair))))))))
+                             next-pair positions))))))))
            (cons file-buf (nreverse pairs))))
         (:cleanup
          (dolist (pair all-pairs)
-           (move-marker (car pair) nil)
-           (move-marker (cdr pair) nil)))))))
+           (move-marker (nth 0 pair) nil)
+           (move-marker (nth 1 pair) nil)))))))
 
 (defun xref--outdated-p (item)
   "Check that the match location at current position is up-to-date.
@@ -857,7 +861,7 @@ xref--outdated-p
              (funcall check)))))))
 
 ;; FIXME: Write a nicer UI.
-(defun xref--query-replace-1 (from to iter)
+(defun xref--query-replace-1 (from to iter xref-buf)
   (let* ((query-replace-lazy-highlight nil)
          (continue t)
          did-it-once buf-pairs pairs
@@ -872,11 +876,15 @@ xref--query-replace-1
                  (<= end current-end))))
          (replace-re-search-function
           (lambda (from &optional _bound noerror)
-            (let (found pair)
+            (let (found positions)
               (while (and (not found) pairs)
-                (setq pair (pop pairs)
-                      current-beg (car pair)
-                      current-end (cdr pair))
+                (setq positions (pop pairs)
+                      current-beg (nth 0 positions)
+                      current-end (nth 1 positions))
+                (when (buffer-live-p xref-buf)
+                  (with-current-buffer xref-buf
+                    (goto-char (nth 2 positions))
+                    (xref--set-arrow)))
                 (goto-char current-beg)
                 (when (re-search-forward from current-end noerror)
                   (setq found t)))