diff --git a/lisp/vc/diff-mode.el b/lisp/vc/diff-mode.el
index b91a2ba45a..175687f184 100644
--- a/lisp/vc/diff-mode.el
+++ b/lisp/vc/diff-mode.el
@@ -2077,7 +2088,9 @@ diff--font-lock-refined
                   (point) 'diff--font-lock-refined nil max)))
     (let* ((min (point))
            (beg (or (ignore-errors (diff-beginning-of-hunk))
-                    (ignore-errors (diff-hunk-next) (point))
+                    (ignore-errors (let ((diff-auto-refine-mode nil))
+                                     (diff-hunk-next))
+                                   (point))
                     max)))
       (while (< beg max)
         (let ((end
@@ -2092,7 +2105,10 @@ diff--font-lock-refined
               (overlay-put ol 'modification-hooks
                            '(diff--font-lock-refine--refresh))))
           (goto-char (max beg end))
-          (setq beg (or (ignore-errors (diff-hunk-next) (point)) max)))))))
+          (setq beg (or (ignore-errors (let ((diff-auto-refine-mode nil))
+                                         (diff-hunk-next))
+                                       (point))
+                        max)))))))
 
 (defun diff--font-lock-refine--refresh (ol _after _beg _end &optional _len)
   (delete-overlay ol))