Memory leaks in font objects

Memory leaks in font objects

[Dmitry Antipov]

[-- Attachment #1: Type: text/plain, Size: 636 bytes --]

On my system, (length (x-list-fonts "*")) is 3007, and this function:

(defun bloat-font ()
   (interactive)
   (let ((fonts (x-list-fonts "*")))
     (while fonts
       (condition-case nil (set-frame-font (car fonts)) (error nil))
       (setq fonts (cdr fonts))
       (redisplay))))

causes Emacs (default GTK3 build) to grow up to ~360M RSS, mostly
because a lot of font resources aren't freed when font objects are
reclaimed.  Proper reclamation reduces RSS to ~150M.  This is the
most giant memory leak I've ever seen.  I'm trying to fix this by
adding simple finalization support for Lisp_Vectors; comments are
welcome.

Dmitry

[-- Attachment #2: font_finalize.patch --]
[-- Type: text/x-patch, Size: 4433 bytes --]

=== modified file 'src/alloc.c'
--- src/alloc.c	2013-10-23 16:07:30 +0000
+++ src/alloc.c	2013-10-24 05:48:47 +0000
@@ -2837,6 +2837,24 @@
   return vroundup (size);
 }
 
+/* Called when VECTOR is finally reclaimed by GC.
+   For now this is needed for font objects only.  */
+
+static void
+finalize_vector (struct Lisp_Vector *vector)
+{
+  if (PSEUDOVECTOR_TYPEP (&vector->header, PVEC_FONT)
+      && ((vector->header.size & PSEUDOVECTOR_SIZE_MASK)
+	  == FONT_OBJECT_MAX))
+    {
+      struct font *font = (struct font *) vector;
+		  
+      eassert (font->driver);
+      if (font->driver->finalize)
+	font->driver->finalize (font);
+    }
+}
+
 /* Reclaim space used by unmarked vectors.  */
 
 static void
@@ -2871,6 +2889,7 @@
 	    {
 	      ptrdiff_t total_bytes;
 
+	      finalize_vector (vector);
 	      nbytes = vector_nbytes (vector);
 	      total_bytes = nbytes;
 	      next = ADVANCE (vector, nbytes);
@@ -2882,6 +2901,7 @@
 		{
 		  if (VECTOR_MARKED_P (next))
 		    break;
+		  finalize_vector (next);
 		  nbytes = vector_nbytes (next);
 		  total_bytes += nbytes;
 		  next = ADVANCE (next, nbytes);

=== modified file 'src/font.h'
--- src/font.h	2013-09-15 17:58:46 +0000
+++ src/font.h	2013-10-24 05:47:08 +0000
@@ -548,6 +548,10 @@
   /* Close FONT on frame F.  */
   void (*close) (struct frame *f, struct font *font);
 
+  /* Finalize FONT when Lisp font-object is reclaimed.
+     NOTE: this is called by GC.  */
+  void (*finalize) (struct font *font);
+
   /* Optional (if FACE->extra is not used).
      Prepare FACE for displaying characters by FONT on frame F by
      storing some data in FACE->extra.  If successful, return 0.

=== modified file 'src/ftfont.c'
--- src/ftfont.c	2013-10-11 06:32:29 +0000
+++ src/ftfont.c	2013-10-24 05:39:56 +0000
@@ -530,6 +530,7 @@
     NULL,			/* free_entity */
     ftfont_open,
     ftfont_close,
+    NULL,			/* no finalizer yet */
     /* We can't draw a text without device dependent functions.  */
     NULL,			/* prepare_face */
     NULL,			/* done_face */

=== modified file 'src/xfont.c'
--- src/xfont.c	2013-09-24 06:43:20 +0000
+++ src/xfont.c	2013-10-24 05:44:40 +0000
@@ -120,6 +120,7 @@
 static Lisp_Object xfont_list_family (struct frame *);
 static Lisp_Object xfont_open (struct frame *, Lisp_Object, int);
 static void xfont_close (struct frame *, struct font *);
+static void xfont_finalize (struct font *);
 static int xfont_prepare_face (struct frame *, struct face *);
 static int xfont_has_char (Lisp_Object, int);
 static unsigned xfont_encode_char (struct font *, int);
@@ -139,6 +140,7 @@
     NULL,
     xfont_open,
     xfont_close,
+    xfont_finalize,
     xfont_prepare_face,
     NULL,
     xfont_has_char,
@@ -892,9 +894,24 @@
 static void
 xfont_close (struct frame *f, struct font *font)
 {
-  block_input ();
-  XFreeFont (FRAME_X_DISPLAY (f), ((struct xfont_info *) font)->xfont);
-  unblock_input ();
+  xfont_finalize (font);
+}
+
+/* Finalize X font FONT.  NOTE: this is called by GC.  */
+
+static void
+xfont_finalize (struct font *font)
+{
+  struct xfont_info *xfi = (struct xfont_info *) font;
+
+  if (xfi->xfont)
+    {
+      eassert (xfi->display);
+      block_input ();
+      XFreeFont (xfi->display, xfi->xfont);
+      unblock_input ();
+      xfi->xfont = NULL;
+    }
 }
 
 static int

=== modified file 'src/xftfont.c'
--- src/xftfont.c	2013-08-03 03:29:03 +0000
+++ src/xftfont.c	2013-10-24 05:43:12 +0000
@@ -494,10 +494,22 @@
   if (xftfont_info->otf)
     OTF_close (xftfont_info->otf);
 #endif
-  block_input ();
-  XftUnlockFace (xftfont_info->xftfont);
-  XftFontClose (xftfont_info->display, xftfont_info->xftfont);
-  unblock_input ();
+  if (xftfont_info->xftfont)
+    {
+      block_input ();
+      XftUnlockFace (xftfont_info->xftfont);
+      XftFontClose (xftfont_info->display, xftfont_info->xftfont);
+      unblock_input ();
+      xftfont_info->xftfont = NULL;
+    }
+}
+
+/* Finalize XFT font FONT.  NOTE: this is called by GC.  */
+
+static void
+xftfont_finalize (struct font *font)
+{
+  xftfont_close (NULL, font);
 }
 
 static int
@@ -763,6 +775,7 @@
   xftfont_driver.match = xftfont_match;
   xftfont_driver.open = xftfont_open;
   xftfont_driver.close = xftfont_close;
+  xftfont_driver.finalize = xftfont_finalize;
   xftfont_driver.prepare_face = xftfont_prepare_face;
   xftfont_driver.done_face = xftfont_done_face;
   xftfont_driver.has_char = xftfont_has_char;

Re: Memory leaks in font objects

[Stefan Monnier]

> On my system, (length (x-list-fonts "*")) is 3007, and this function:

> (defun bloat-font ()
>   (interactive)
>   (let ((fonts (x-list-fonts "*")))
>     (while fonts
>       (condition-case nil (set-frame-font (car fonts)) (error nil))
>       (setq fonts (cdr fonts))
>       (redisplay))))

> causes Emacs (default GTK3 build) to grow up to ~360M RSS, mostly
> because a lot of font resources aren't freed when font objects are
> reclaimed.

That's unfortunate, indeed.

> Proper reclamation reduces RSS to ~150M.  This is the
> most giant memory leak I've ever seen.

Oddly enough, I'm not sure it's a leak.  Here's the result of my test:

             VM   RSS
   initial  204    24
   after 1  522   290
   after 2  539   305
   after 3  542   308
   after 4  542   308

So it's more like "once malloced, the memory can be reused by Emacs, but
not by another process because we never return it to the OS".
   
> --- src/font.h	2013-09-15 17:58:46 +0000
> +++ src/font.h	2013-10-24 05:47:08 +0000
> @@ -548,6 +548,10 @@
>    /* Close FONT on frame F.  */
>    void (*close) (struct frame *f, struct font *font);
 
> +  /* Finalize FONT when Lisp font-object is reclaimed.
> +     NOTE: this is called by GC.  */
> +  void (*finalize) (struct font *font);
> +
>    /* Optional (if FACE->extra is not used).
>       Prepare FACE for displaying characters by FONT on frame F by
>       storing some data in FACE->extra.  If successful, return 0.

What the difference between "close" and "finalize"?

I mean, your code does add what I'd call "finalization" to vectors, but
what it does to fonts wouldn't qualify as "finalization" to me, rather
it's just "freeing".  IOW to me, the vector's finalization code doesn't
*finalize* the font but *frees* the font.


        Stefan

Re: Memory leaks in font objects

[Dmitry Antipov]

On 10/24/2013 06:19 PM, Stefan Monnier wrote:

> Oddly enough, I'm not sure it's a leak.  Here's the result of my test:
>
>               VM   RSS
>     initial  204    24
>     after 1  522   290
>     after 2  539   305
>     after 3  542   308
>     after 4  542   308
>
> So it's more like "once malloced, the memory can be reused by Emacs, but
> not by another process because we never return it to the OS".

IIUC no :-(. I tried the following:

initial: 32M RSS
then M-x bloat-font, M-x set-frame-font "fixed", M-x garbage-collect: 372M
then M-x byte-force-recompile [all lisp/ subdir], M-x garbage-collect: 437M

If 372M is (mostly) free but just too fragmented to release, byte-force-recompile
should reuse a lot of memory - so why +65M? But, 2nd run of bloat-font do not add
too much. Why? Consider XLoadQueryFont example. When you do not call XFreeFont but
just lost XFontStruct pointer, the font is still allocated by Xlib. Since you lost
the pointer, you can't use this font unless you call XLoadQueryFont for the same
font again and got the pointer to previously allocated resource (assuming that you
do not lost the connection to X display). So 2nd, 3rd, etc. calls to bloat-font
will just re-use font resources you lost before. But, if you need to allocate
something else, you will need more memory.

> What the difference between "close" and "finalize"?

Nothing except "close" is a part of existing interface which I don't want
to change (at this moment at least), and requires extra frame argument.

Dmitry

Re: Memory leaks in font objects

[Stefan Monnier]

>> Oddly enough, I'm not sure it's a leak.  Here's the result of my test:
>> 
>> VM   RSS
>> initial  204    24
>> after 1  522   290
>> after 2  539   305
>> after 3  542   308
>> after 4  542   308
>> 
>> So it's more like "once malloced, the memory can be reused by Emacs, but
>> not by another process because we never return it to the OS".

> IIUC no :-(. I tried the following:

You misunderstood: I did not say that those megabytes are available
to malloc.  I just pointed out that it's a similar phenomenon to the one
that takes place with malloc.

> But, if you need to allocate something else, you will need
> more memory.

But the objects are still usable: there is still a pointer to them
somewhere and some sequence of events can bring Emacs to a state where
those objects are used again.  So it is not strictly speaking a leak, in
my book.

Also, I'm curious how you bumped into this problem.  Was it by looking
at the code, or was it in "real life use", or in some test you happened
to be running?

The fact that the wasted memory doesn't grow indefinitely, and only
grows with the fonts actually used, makes me think that it shouldn't be
a problem in real life usage.

>> What the difference between "close" and "finalize"?
> Nothing except "close" is a part of existing interface which I don't want
> to change (at this moment at least), and requires extra frame argument.

So it'd be better to merge the two.


        Stefan