diff --git a/src/xdisp.c b/src/xdisp.c
index 47b8141463..14f070d487 100644
--- a/src/xdisp.c
+++ b/src/xdisp.c
@@ -20872,7 +20872,12 @@ maybe_produce_line_number (struct it *it)
   /* Compute the required width if needed.  */
   if (!it->lnum_width)
     {
-      if (NATNUMP (Vdisplay_line_number_width))
+      Lisp_Object cache = buffer_local_value (Qdisplay_line_number_width_cache,
+                                              it->w->contents);
+
+      if (NATNUMP (cache))
+        it->lnum_width = XFASTINT (cache);
+      else if (NATNUMP (Vdisplay_line_number_width))
 	it->lnum_width = XFASTINT (Vdisplay_line_number_width);
 
       /* Max line number to be displayed cannot be more than the one
@@ -20891,6 +20896,9 @@ maybe_produce_line_number (struct it *it)
 	max_lnum = this_line + it->w->desired_matrix->nrows - 1 - it->vpos;
       max_lnum = max (1, max_lnum);
       it->lnum_width = max (it->lnum_width, log10 (max_lnum) + 1);
+      if (display_line_numbers_grow_only)
+        Fset (Fmake_local_variable (Qdisplay_line_number_width_cache),
+              make_number (it->lnum_width));
       eassert (it->lnum_width > 0);
     }
   if (EQ (Vdisplay_line_numbers, Qrelative))
@@ -32594,6 +32602,18 @@ Any other value is treated as nil.  */);
   DEFSYM (Qdisplay_line_number_width, "display-line-number-width");
   Fmake_variable_buffer_local (Qdisplay_line_number_width);
 
+  DEFVAR_BOOL ("display-line-numbers-grow-only", display_line_numbers_grow_only,
+    doc: /* Non-nil means only dynamically grow the display,
+and never shrink. */);
+  display_line_numbers_grow_only = false;
+
+  DEFVAR_LISP ("display-line-number-width-cache", Vdisplay_line_number_width_cache,
+               doc: /* Stores the maximum line number width seen. */);
+  Vdisplay_line_number_width_cache = Qnil;
+  DEFSYM (Qdisplay_line_number_width_cache, "display-line-number-width-cache");
+  Fmake_variable_buffer_local (Qdisplay_line_number_width_cache);
+  Funintern (Qdisplay_line_number_width_cache, Qnil);
+
   DEFVAR_LISP ("display-line-numbers-current-absolute",
 	       Vdisplay_line_numbers_current_absolute,
     doc: /* Non-nil means display absolute number of current line.