bug#20555: Emacs 24.2 vs 24.4 on Solaris: M-x shell and "tty => not a tty"

[白井彰 <okshirai@gmail.com>]

I wrote a small program my-emacs-24.5.c, and found that grantpt(fd)
returned -1 after fcntl(fd, F_SETFD, FD_CLOEXEC).
Without fcntl(fd, F_SETFD, FD_CLOEXEC), grantpt(fd) returns 0.

aaa% sh emacs-24.5/build-aux/config.guess
i386-pc-solaris2.10

aaa% ls my-emacs-24.5.c
my-emacs-24.5.c

aaa% make my-emacs-24.5 && ./my-emacs-24.5
cc    -o my-emacs-24.5 my-emacs-24.5.c
call_fcntl_fd_cloexec: 1
fcntl_result: 0
fd: 3
grantpt_result: -1
errno: 13
grantpt: Permission denied

aaa% make my-emacs-24.5 && ./my-emacs-24.5 call_fcntl_fd_cloexec=0
`my-emacs-24.5' is up to date.
call_fcntl_fd_cloexec: 0
fd: 3
grantpt_result: 0

bbb% sh emacs-24.5/build-aux/config.guess
sparc-sun-solaris2.8

bbb% rm my-emacs-24.5
bbb% make my-emacs-24.5 && ./my-emacs-24.5
cc    -o my-emacs-24.5 my-emacs-24.5.c
call_fcntl_fd_cloexec: 1
fcntl_result: 0
fd: 3
grantpt_result: -1
errno: 0
grantpt: Error 0

bbb% make my-emacs-24.5 && ./my-emacs-24.5 call_fcntl_fd_cloexec=0
`my-emacs-24.5' is up to date.
call_fcntl_fd_cloexec: 0
fd: 3
grantpt_result: 0

% cat -n my-emacs-24.5.c
     1  /* 24.5 */
     2
     3  #include <errno.h>
     4  #include <fcntl.h>
     5  #include <stdio.h>
     6  #include <stdlib.h>
     7  #include <string.h>
     8  #include <sys/stat.h>
     9  #include <sys/types.h>
    10  #include <unistd.h>
    11
    12  int call_fcntl_fd_cloexec;
    13
    14  enum { PTY_NAME_SIZE = 24 };
    15
    16  int emacs_open(const char *file, int oflags, int mode) {
    17  #define O_CLOEXEC 0
    18    int fd;
    19    oflags |= O_CLOEXEC;
    20    while ((fd = open(file, oflags, mode)) < 0 && errno == EINTR) abort();
    21    if (! O_CLOEXEC && 0 <= fd) {
    22      fprintf(stderr, "call_fcntl_fd_cloexec: %d\n", call_fcntl_fd_cloexec);
    23      if (call_fcntl_fd_cloexec) {
    24        int fcntl_result = fcntl(fd, F_SETFD, FD_CLOEXEC);
    25        fprintf(stderr, "fcntl_result: %d\n", fcntl_result);
    26      }
    27    }
    28    return fd;
    29  }
    30
    31  static int allocate_pty(char pty_name[PTY_NAME_SIZE]) {
    32    int fd;
    33    strcpy(pty_name, "/dev/ptmx");
    34    fd = emacs_open(pty_name, O_RDWR | O_NONBLOCK, 0);
    35    fprintf(stderr, "fd: %d\n", fd);
    36    if (fd >= 0) {
    37      int grantpt_result = grantpt(fd);
    38      fprintf(stderr, "grantpt_result: %d\n", grantpt_result);
    39      if (grantpt_result != 0) {
    40        fprintf(stderr, "errno: %d\n", errno);
    41        perror("grantpt");
    42      }
    43    }
    44    return fd;
    45  }
    46
    47  int main(int argc, char **argv) {
    48    char pty_name[PTY_NAME_SIZE];
    49    if (argc == 2) call_fcntl_fd_cloexec = 0;
    50    else           call_fcntl_fd_cloexec = 1;
    51    allocate_pty(pty_name);
    52    return 0;
    53  }