[PATCH] unexec() for Cygwin, try #3

[PATCH] unexec() for Cygwin, try #3

[Joe Buehler]

- code formatting fixed
- debugging printf's conditionalized and off by default
- added support for Xaw3d scrollbar widgets
- strcasecmp instead of strcmp for .exe suffix
- added comment about object ordering for link (comment from Stephan Monier)

Please install in CVS for me, I don't have write access (and given how
little I know about CVS, it would be somewhat dangerous to give it to me...)
-- 
Joe Buehler

Index: lisp/ChangeLog
===================================================================
RCS file: /cvsroot/emacs/emacs/lisp/ChangeLog,v
retrieving revision 1.5775
diff -u -r1.5775 ChangeLog
--- lisp/ChangeLog	18 Mar 2004 03:07:38 -0000	1.5775
+++ lisp/ChangeLog	30 Mar 2004 13:59:33 -0000
@@ -1,3 +1,7 @@
+2004-03-29  Joe Buehler  <jbuehler@hekiman.com>
+
+	* loadup.el: added cygwin to system-type list, for unexec() support.
+
  2004-03-17  Luc Teirlinck  <teirllm@auburn.edu>

  	* simple.el (clone-buffer): Doc fix.
Index: lisp/loadup.el
===================================================================
RCS file: /cvsroot/emacs/emacs/lisp/loadup.el,v
retrieving revision 1.133
diff -u -r1.133 loadup.el
--- lisp/loadup.el	8 Nov 2003 01:39:07 -0000	1.133
+++ lisp/loadup.el	30 Mar 2004 13:59:34 -0000
@@ -330,7 +330,7 @@
        (dump-emacs "emacs" "temacs")
        (message "%d pure bytes used" pure-bytes-used)
        ;; Recompute NAME now, so that it isn't set when we dump.
-      (if (not (memq system-type '(ms-dos windows-nt)))
+      (if (not (memq system-type '(ms-dos windows-nt cygwin)))
  	  (let ((name (concat "emacs-" emacs-version)))
  	    (while (string-match "[^-+_.a-zA-Z0-9]+" name)
  	      (setq name (concat (downcase (substring name 0 (match-beginning 0)))
Index: src/ChangeLog
===================================================================
RCS file: /cvsroot/emacs/emacs/src/ChangeLog,v
retrieving revision 1.3586
diff -u -r1.3586 ChangeLog
--- src/ChangeLog	18 Mar 2004 02:59:33 -0000	1.3586
+++ src/ChangeLog	30 Mar 2004 13:59:40 -0000
@@ -1,3 +1,17 @@
+2004-03-29  Joe Buehler  <jbuehler@hekimian.com>
+
+	* s/cygwin.h: changes for Cygwin unexec() support, changes in
+	Cygwin itself.  Added support for Xaw3d scrollbars.
+
+	* puresize.h: set up PURE_P() for Cygwin unexec() support
+
+	* lastfile.c: define my_endbss[] for Cygwin unexec() support.
+
+	* gmalloc.c (__default_morecore): use bss_sbrk(), not __sbrk(),
+	before Cygwin unexec.
+
+	* Makefile.in: link changes for Cygwin unexec() support.
+
  2004-03-17  Stefan Monnier  <monnier@iro.umontreal.ca>

  	* fileio.c (Fread_file_name): Set completion-ignore-case for
Index: src/Makefile.in
===================================================================
RCS file: /cvsroot/emacs/emacs/src/Makefile.in,v
retrieving revision 1.295
diff -u -r1.295 Makefile.in
--- src/Makefile.in	14 Mar 2004 18:43:56 -0000	1.295
+++ src/Makefile.in	30 Mar 2004 13:59:41 -0000
@@ -564,6 +564,10 @@
  #endif
  #endif

+#ifdef CYGWIN
+CYGWIN_OBJ = sheap.o
+#endif
+
  #ifdef HAVE_CARBON
  mac = $(dot)$(dot)/mac/
  MAC_OBJ = mac.o macterm.o macfns.o macmenu.o fontset.o fringe.o image.o
@@ -586,7 +590,7 @@
  	process.o callproc.o \
  	region-cache.o sound.o atimer.o \
  	doprnt.o strftime.o intervals.o textprop.o composite.o md5.o \
-	$(MSDOS_OBJ) $(MAC_OBJ)
+	$(MSDOS_OBJ) $(MAC_OBJ) $(CYGWIN_OBJ)

  /* Object files used on some machine or other.
     These go in the DOC file on all machines
@@ -623,8 +627,6 @@
  rallocobj = ralloc.o
  #endif

-mallocobj = $(gmallocobj) $(rallocobj) vm-limit.o
-
  #endif /* SYSTEM_MALLOC */


@@ -643,7 +645,12 @@

  /* define otherobj as list of object files that make-docfile
     should not be told about.  */
-otherobj= $(termcapobj) lastfile.o $(mallocobj) $(allocaobj) $(widgetobj) $(LIBOBJS)
+#ifdef CYGWIN
+/* Cygwin differs because of its unexec(). */
+otherobj= $(termcapobj) $(gmallocobj) $(rallocobj) lastfile.o vm-limit.o $(allocaobj) $(widgetobj) $(LIBOBJS)
+#else
+otherobj= $(termcapobj) lastfile.o $(gmallocobj) $(rallocobj) vm-limit.o $(allocaobj) $(widgetobj) $(LIBOBJS)
+#endif

  #ifdef HAVE_MOUSE
  #define MOUSE_SUPPORT ${lispsource}mouse.elc \
Index: src/gmalloc.c
===================================================================
RCS file: /cvsroot/emacs/emacs/src/gmalloc.c,v
retrieving revision 1.14
diff -u -r1.14 gmalloc.c
--- src/gmalloc.c	1 Sep 2003 15:45:52 -0000	1.14
+++ src/gmalloc.c	30 Mar 2004 14:09:41 -0000
@@ -352,6 +352,10 @@
  #include <errno.h>

  /* How to really get more memory.  */
+#if defined(CYGWIN)
+extern __ptr_t bss_sbrk PP ((ptrdiff_t __size));
+extern int bss_sbrk_did_unexec;
+#endif
  __ptr_t (*__morecore) PP ((ptrdiff_t __size)) = __default_morecore;

  /* Debugging hook for `malloc'.  */
@@ -1572,7 +1576,14 @@
  __default_morecore (increment)
       __malloc_ptrdiff_t increment;
  {
-  __ptr_t result = (__ptr_t) __sbrk (increment);
+  __ptr_t result;
+#if defined(CYGWIN)
+  if (!bss_sbrk_did_unexec)
+    {
+      return bss_sbrk (increment);
+    }
+#endif
+  result = (__ptr_t) __sbrk (increment);
    if (result == (__ptr_t) -1)
      return NULL;
    return result;
Index: src/lastfile.c
===================================================================
RCS file: /cvsroot/emacs/emacs/src/lastfile.c,v
retrieving revision 1.8
diff -u -r1.8 lastfile.c
--- src/lastfile.c	1 Sep 2003 15:45:56 -0000	1.8
+++ src/lastfile.c	30 Mar 2004 13:59:41 -0000
@@ -40,7 +40,7 @@

  char my_edata[] = "End of Emacs initialized data";

-#ifdef WINDOWSNT
+#if defined(WINDOWSNT) || defined(CYGWIN)
  /* Help unexec locate the end of the .bss area used by Emacs (which
     isn't always a separate section in NT executables).  */
  char my_endbss[1];
Index: src/puresize.h
===================================================================
RCS file: /cvsroot/emacs/emacs/src/puresize.h,v
retrieving revision 1.70
diff -u -r1.70 puresize.h
--- src/puresize.h	1 Sep 2003 15:45:56 -0000	1.70
+++ src/puresize.h	30 Mar 2004 13:59:41 -0000
@@ -68,7 +68,7 @@
  \f
  /* Define PURE_P.  */

-#ifdef VIRT_ADDR_VARIES
+#if defined(VIRT_ADDR_VARIES) || defined(CYGWIN)
  /* For machines like APOLLO where text and data can go anywhere
     in virtual memory.  */

Index: src/s/cygwin.h
===================================================================
RCS file: /cvsroot/emacs/emacs/src/s/cygwin.h,v
retrieving revision 1.3
diff -u -r1.3 cygwin.h
--- src/s/cygwin.h	1 Sep 2003 15:45:58 -0000	1.3
+++ src/s/cygwin.h	30 Mar 2004 13:59:41 -0000
@@ -115,10 +115,8 @@
  #define PENDING_OUTPUT_COUNT(FILE) ((FILE)->_p - (FILE)->_bf._base)
  #define GETPGRP_NO_ARG 1
  #define SYSV_SYSTEM_DIR 1
-/* -lutil comes from inetutils and has pty functions in it */
-#define LIBS_SYSTEM -lutil
-/* undumping is not implemented yet */
-#define CANNOT_DUMP 1
+#define LIB_STANDARD_LIBSRC
+#define UNEXEC unexcw.o
  #define POSIX_SIGNALS 1
  /* force the emacs image to start high in memory, so dll relocation
     can put things in low memory without causing all sorts of grief for
@@ -137,7 +135,7 @@
  /*#define HAVE_VFORK*/
  /* Xaw3d causes problems -- might have been fixed by NARROWPROTO
     above, but I haven't tried it */
-#undef HAVE_XAW3D
+/*#undef HAVE_XAW3D*/

  /* vfork() interacts badly with setsid(), causing ptys to fail to
     change their controlling terminal */
--- src/sheap.c	2004-03-30 09:00:57.193000000 -0500
+++ src/sheap.c	2004-03-30 08:55:34.910120000 -0500
@@ -0,0 +1,101 @@
+/* simulate sbrk() with an array in .bss, for unexec() support for Cygwin;
+   complete rewrite of xemacs Cygwin unexec() code
+
+   Copyright (C) 2004
+   Free Software Foundation, Inc.
+
+This file is part of GNU Emacs.
+
+GNU Emacs is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2, or (at your option)
+any later version.
+
+GNU Emacs is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with GNU Emacs; see the file COPYING.  If not, write to
+the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+Boston, MA 02111-1307, USA.  */
+
+#include <config.h>
+#include <stdio.h>
+#include "lisp.h"
+
+#include <unistd.h>
+
+#ifdef HAVE_X_WINDOWS
+#define STATIC_HEAP_SIZE	(7 * 1024 * 1024)
+#else
+#define STATIC_HEAP_SIZE	(7 * 1024 * 1024)
+#endif
+
+int debug_sheap = 0;
+
+#define BLOCKSIZE 4096
+
+char bss_sbrk_buffer[STATIC_HEAP_SIZE];
+char *bss_sbrk_ptr;
+int bss_sbrk_did_unexec;
+
+void *
+bss_sbrk (ptrdiff_t request_size)
+{
+  if (!bss_sbrk_ptr)
+    {
+      bss_sbrk_ptr = bss_sbrk_buffer;
+#ifdef CYGWIN
+      sbrk (BLOCKSIZE);		/* force space for fork to work */
+#endif
+    }
+
+  if (!(int) request_size)
+    {
+      return (bss_sbrk_ptr);
+    }
+  else if (bss_sbrk_ptr + (int) request_size < bss_sbrk_buffer)
+    {
+      printf
+	("attempt to free too much: avail %d used %d failed request %d\n",
+	 STATIC_HEAP_SIZE, bss_sbrk_ptr - bss_sbrk_buffer,
+	 (int) request_size);
+      exit (-1);
+      return 0;
+    }
+  else if (bss_sbrk_ptr + (int) request_size >
+	   bss_sbrk_buffer + STATIC_HEAP_SIZE)
+    {
+      printf ("static heap exhausted: avail %d used %d failed request %d\n",
+	      STATIC_HEAP_SIZE,
+	      bss_sbrk_ptr - bss_sbrk_buffer, (int) request_size);
+      exit (-1);
+      return 0;
+    }
+  else if ((int) request_size < 0)
+    {
+      bss_sbrk_ptr += (int) request_size;
+      if (debug_sheap)
+	printf ("freed size %d\n", request_size);
+      return bss_sbrk_ptr;
+    }
+  else
+    {
+      char *ret = bss_sbrk_ptr;
+      if (debug_sheap)
+	printf ("allocated 0x%08x size %d\n", ret, request_size);
+      bss_sbrk_ptr += (int) request_size;
+      return ret;
+    }
+}
+
+void
+report_sheap_usage (int die_if_pure_storage_exceeded)
+{
+  char buf[200];
+  sprintf (buf, "Static heap usage: %d of %d bytes",
+	   bss_sbrk_ptr - bss_sbrk_buffer, STATIC_HEAP_SIZE);
+  message ("%s", buf);
+}
--- src/unexcw.c	2004-03-30 09:00:57.503000000 -0500
+++ src/unexcw.c	2004-03-30 08:56:47.454433600 -0500
@@ -0,0 +1,304 @@
+/* unexec() support for Cygwin;
+   complete rewrite of xemacs Cygwin unexec() code
+
+   Copyright (C) 2004
+   Free Software Foundation, Inc.
+
+This file is part of GNU Emacs.
+
+GNU Emacs is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2, or (at your option)
+any later version.
+
+GNU Emacs is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with GNU Emacs; see the file COPYING.  If not, write to
+the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+Boston, MA 02111-1307, USA.  */
+
+#include <config.h>
+#include <lisp.h>
+#include <stdio.h>
+#include <fcntl.h>
+#include <a.out.h>
+#include <unistd.h>
+#include <assert.h>
+
+#define DOTEXE ".exe"
+
+extern int bss_sbrk_did_unexec;
+
+/* emacs symbols that indicate where bss and data end for emacs internals */
+extern char my_endbss[];
+extern char my_edata[];
+
+/*
+** header for Windows executable files
+*/
+typedef struct
+{
+  FILHDR file_header;
+  PEAOUTHDR file_optional_header;
+  SCNHDR section_header[32];
+} exe_header_t;
+
+int debug_unexcw = 0;
+
+/*
+** Read the header from the executable into memory so we can more easily access it.
+*/
+static exe_header_t *
+read_exe_header (int fd, exe_header_t * exe_header_buffer)
+{
+  int i;
+  int ret;
+
+  assert (fd >= 0);
+  assert (exe_header_buffer != 0);
+
+  ret = lseek (fd, 0L, SEEK_SET);
+  assert (ret != -1);
+
+  ret =
+    read (fd, &exe_header_buffer->file_header,
+	  sizeof (exe_header_buffer->file_header));
+  assert (ret == sizeof (exe_header_buffer->file_header));
+
+  assert (exe_header_buffer->file_header.e_magic == 0x5a4d);
+  assert (exe_header_buffer->file_header.nt_signature == 0x4550);
+  assert (exe_header_buffer->file_header.f_magic == 0x014c);
+  assert (exe_header_buffer->file_header.f_nscns > 0);
+  assert (exe_header_buffer->file_header.f_nscns <=
+	  sizeof (exe_header_buffer->section_header) /
+	  sizeof (exe_header_buffer->section_header[0]));
+  assert (exe_header_buffer->file_header.f_opthdr > 0);
+
+  ret =
+    read (fd, &exe_header_buffer->file_optional_header,
+	  sizeof (exe_header_buffer->file_optional_header));
+  assert (ret == sizeof (exe_header_buffer->file_optional_header));
+
+  assert (exe_header_buffer->file_optional_header.magic == 0x010b);
+
+  for (i = 0; i < exe_header_buffer->file_header.f_nscns; ++i)
+    {
+      ret =
+	read (fd, &exe_header_buffer->section_header[i],
+	      sizeof (exe_header_buffer->section_header[i]));
+      assert (ret == sizeof (exe_header_buffer->section_header[i]));
+    }
+
+  return (exe_header_buffer);
+}
+
+/*
+** Fix the dumped emacs executable:
+**
+** - copy .data section data of interest from running executable into
+**   output .exe file
+**
+** - convert .bss section into an initialized data section (like
+**   .data) and copy .bss section data of interest from running
+**   executable into output .exe file
+*/
+static void
+fixup_executable (int fd)
+{
+  exe_header_t exe_header_buffer;
+  exe_header_t *exe_header;
+  int i;
+  int ret;
+  int found_data = 0;
+  int found_bss = 0;
+
+  exe_header = read_exe_header (fd, &exe_header_buffer);
+  assert (exe_header != 0);
+
+  assert (exe_header->file_header.f_nscns > 0);
+  for (i = 0; i < exe_header->file_header.f_nscns; ++i)
+    {
+      unsigned long start_address =
+	exe_header->section_header[i].s_vaddr +
+	exe_header->file_optional_header.ImageBase;
+      unsigned long end_address =
+	exe_header->section_header[i].s_vaddr +
+	exe_header->file_optional_header.ImageBase +
+	exe_header->section_header[i].s_paddr;
+      if (debug_unexcw)
+	printf ("%8s start 0x%08x end 0x%08x\n",
+		exe_header->section_header[i].s_name,
+		start_address, end_address);
+      if (my_edata >= (char *) start_address
+	  && my_edata < (char *) end_address)
+	{
+	  /* data section */
+	  ret =
+	    lseek (fd, (long) (exe_header->section_header[i].s_scnptr),
+		   SEEK_SET);
+	  assert (ret != -1);
+	  ret =
+	    write (fd, (char *) start_address,
+		   my_edata - (char *) start_address);
+	  assert (ret == my_edata - (char *) start_address);
+	  ++found_data;
+	  if (debug_unexcw)
+	    printf ("         .data, mem start 0x%08x mem length %d\n",
+		    start_address, my_edata - (char *) start_address);
+	  if (debug_unexcw)
+	    printf ("         .data, file start %d file length %d\n",
+		    (int) exe_header->section_header[i].s_scnptr,
+		    (int) exe_header->section_header[i].s_paddr);
+	}
+      else if (my_endbss >= (char *) start_address
+	       && my_endbss < (char *) end_address)
+	{
+	  /* bss section */
+	  ++found_bss;
+	  if (exe_header->section_header[i].s_flags & 0x00000080)
+	    {
+	      /* convert uninitialized data section to initialized data section */
+	      struct stat statbuf;
+	      ret = fstat (fd, &statbuf);
+	      assert (ret != -1);
+
+	      exe_header->section_header[i].s_flags &= ~0x00000080;
+	      exe_header->section_header[i].s_flags |= 0x00000040;
+
+	      exe_header->section_header[i].s_scnptr =
+		(statbuf.st_size +
+		 exe_header->file_optional_header.FileAlignment) /
+		exe_header->file_optional_header.FileAlignment *
+		exe_header->file_optional_header.FileAlignment;
+
+	      exe_header->section_header[i].s_size =
+		(exe_header->section_header[i].s_paddr +
+		 exe_header->file_optional_header.FileAlignment) /
+		exe_header->file_optional_header.FileAlignment *
+		exe_header->file_optional_header.FileAlignment;
+
+	      ret =
+		lseek (fd,
+		       (long) (exe_header->section_header[i].s_scnptr +
+			       exe_header->section_header[i].s_size - 1),
+		       SEEK_SET);
+	      assert (ret != -1);
+	      ret = write (fd, "", 1);
+	      assert (ret == 1);
+
+	      ret =
+		lseek (fd,
+		       (long) ((char *) &exe_header->section_header[i] -
+			       (char *) exe_header), SEEK_SET);
+	      assert (ret != -1);
+	      ret =
+		write (fd, &exe_header->section_header[i],
+		       sizeof (exe_header->section_header[i]));
+	      assert (ret == sizeof (exe_header->section_header[i]));
+	      if (debug_unexcw)
+		printf ("         seek to %ld, write %d\n",
+			(long) ((char *) &exe_header->section_header[i] -
+				(char *) exe_header),
+			sizeof (exe_header->section_header[i]));
+	    }
+	  /* write initialized data section */
+	  ret =
+	    lseek (fd, (long) (exe_header->section_header[i].s_scnptr),
+		   SEEK_SET);
+	  assert (ret != -1);
+	  ret =
+	    write (fd, (char *) start_address,
+		   my_endbss - (char *) start_address);
+	  assert (ret == (my_endbss - (char *) start_address));
+	  if (debug_unexcw)
+	    printf ("         .bss, mem start 0x%08x mem length %d\n",
+		    start_address, my_endbss - (char *) start_address);
+	  if (debug_unexcw)
+	    printf ("         .bss, file start %d file length %d\n",
+		    (int) exe_header->section_header[i].s_scnptr,
+		    (int) exe_header->section_header[i].s_paddr);
+	}
+    }
+  assert (found_bss == 1);
+  assert (found_data == 1);
+}
+
+/*
+** Windows likes .exe suffixes on executables.
+*/
+static char *
+add_exe_suffix_if_necessary (const char *name, char *modified)
+{
+  int i = strlen (name);
+  if (i <= (sizeof (DOTEXE) - 1))
+    {
+      sprintf (modified, "%s%s", name, DOTEXE);
+    }
+  else if (!strcasecmp (name + i - (sizeof (DOTEXE) - 1), DOTEXE))
+    {
+      strcpy (modified, name);
+    }
+  else
+    {
+      sprintf (modified, "%s%s", name, DOTEXE);
+    }
+  return (modified);
+}
+
+int
+unexec (char *outfile, char *infile, unsigned start_data, unsigned d1,
+	unsigned d2)
+{
+  char infile_buffer[FILENAME_MAX];
+  char outfile_buffer[FILENAME_MAX];
+  int fd_in;
+  int fd_out;
+  int ret;
+  int ret2;
+
+  if (bss_sbrk_did_unexec)
+    {
+      /* can only dump once */
+      printf ("You can only dump emacs once on this platform.\n");
+      return (1);
+    }
+
+  report_sheap_usage (1);
+
+  infile = add_exe_suffix_if_necessary (infile, infile_buffer);
+  outfile = add_exe_suffix_if_necessary (outfile, outfile_buffer);
+
+  fd_in = open (infile, O_RDONLY | O_BINARY);
+  assert (fd_in >= 0);
+  fd_out = open (outfile, O_RDWR | O_TRUNC | O_CREAT | O_BINARY, 0755);
+  assert (fd_out >= 0);
+  for (;;)
+    {
+      char buffer[4096];
+      ret = read (fd_in, buffer, sizeof (buffer));
+      if (ret == 0)
+	{
+	  /* eof */
+	  break;
+	}
+      assert (ret > 0);
+      /* data */
+      ret2 = write (fd_out, buffer, ret);
+      assert (ret2 == ret);
+    }
+  ret = close (fd_in);
+  assert (ret == 0);
+
+  bss_sbrk_did_unexec = 1;
+  fixup_executable (fd_out);
+  bss_sbrk_did_unexec = 0;
+
+  ret = close (fd_out);
+  assert (ret == 0);
+
+  return (0);
+}

Re: [PATCH] unexec() for Cygwin, try #3

[Harald Maier]

Joe Buehler <jbuehler@hekimian.com> writes:

> - code formatting fixed
> - debugging printf's conditionalized and off by default
> - added support for Xaw3d scrollbar widgets
> - strcasecmp instead of strcmp for .exe suffix
> - added comment about object ordering for link (comment from Stephan Monier)
>
> Please install in CVS for me, I don't have write access (and given how
> little I know about CVS, it would be somewhat dangerous to give it to me...)

Today I tried your patch and _all_ worked fine on cygwin except
patching of the ChangLog files. I crosschecked too building of the W32
version with the gcc mingw port and under GNU/Linux. Here too all
worked fine. So I assume installing of the patch would be fine.

Harald