Re: Fwd: PATCH - Add cooperative REPL server module

[David Thompson <dthompson2@worcester.edu>]

[-- Attachment #1: Type: text/plain, Size: 5634 bytes --]

Mark H Weaver <mhw@netris.org> writes:

> Interesting.  Does it happen with unmodified stable-2.0?  If so, I think
> we can treat this as an independent bug.

Yes, exact same problem on stable-2.0.

> Can you reproduce the segfault while running meta/gdb-uninstalled-guile
> and get a backtrace?  Alternatively, if you provide enough detail to
> reproduce the segfault, I can track it down.

Here's the output from gdb:

Program received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0x7ffff4ea2700 (LWP 13122)]
get_codepoint (port=port@entry=0xe13ff0, codepoint=codepoint@entry=0x7ffff4ea1334, buf=buf@entry=0x7ffff4ea1330 "(%`", 
    len=len@entry=0x7ffff4ea1338) at ports.c:1460
1460	      update_port_lf (*codepoint, port);
(gdb) bt
#0  get_codepoint (port=port@entry=0xe13ff0, codepoint=codepoint@entry=0x7ffff4ea1334, buf=buf@entry=0x7ffff4ea1330 "(%`", 
    len=len@entry=0x7ffff4ea1338) at ports.c:1460
#1  0x00007ffff7ae93ea in scm_peek_char (port=0xe13ff0) at ports.c:2008
#2  0x00007ffff7b3274d in vm_regular_engine (vm=<optimized out>, program=0x7ffff7dbc958 <scm_peek_char__subr_raw_cell>, argv=<optimized out>, 
    nargs=1) at vm-i-system.c:852
#3  0x00007ffff7b32738 in vm_regular_engine (vm=<optimized out>, program=0x6d27c0, argv=<optimized out>, nargs=2) at vm-i-system.c:855
#4  0x00007ffff7aa9723 in scm_call_4 (proc=0x7ae570, arg1=arg1@entry=0x404, arg2=<optimized out>, arg3=<optimized out>, arg4=<optimized out>)
    at eval.c:507
#5  0x00007ffff7b1c854 in scm_catch_with_pre_unwind_handler (key=key@entry=0x404, thunk=<optimized out>, handler=<optimized out>, 
    pre_unwind_handler=<optimized out>) at throw.c:86
#6  0x00007ffff7b1ca2f in scm_c_catch (tag=tag@entry=0x404, body=body@entry=0x7ffff7aa03f0 <scm_body>, body_data=body_data@entry=0x7ffff4ea1810, 
    handler=handler@entry=0x7ffff7aa03d0 <scm_handler>, handler_data=handler_data@entry=0x7ffff4ea1810, 
    pre_unwind_handler=pre_unwind_handler@entry=0x7ffff7aa0290 <pre_unwind_handler>, 
    pre_unwind_handler_data=pre_unwind_handler_data@entry=0x6d8eb0) at throw.c:213
#7  0x00007ffff7aa086f in scm_i_with_continuation_barrier (body=body@entry=0x7ffff7aa03f0 <scm_body>, body_data=body_data@entry=0x7ffff4ea1810, 
    handler=handler@entry=0x7ffff7aa03d0 <scm_handler>, handler_data=handler_data@entry=0x7ffff4ea1810, 
    pre_unwind_handler=pre_unwind_handler@entry=0x7ffff7aa0290 <pre_unwind_handler>, pre_unwind_handler_data=0x6d8eb0) at continuations.c:449
#8  0x00007ffff7aa08c0 in scm_with_continuation_barrier (proc=<optimized out>) at continuations.c:589
#9  0x00007ffff7b3274d in vm_regular_engine (vm=<optimized out>, program=0x7ffff7db4fc0 <scm_with_continuation_barrier__subr_raw_cell>, 
    argv=<optimized out>, nargs=1) at vm-i-system.c:852
#10 0x00007ffff7aa96de in scm_call_3 (proc=0x7ae570, arg1=arg1@entry=0x404, arg2=arg2@entry=0xe11f00, arg3=arg3@entry=0xcc6440) at eval.c:500
#11 0x00007ffff7b1c7bd in scm_catch (key=key@entry=0x404, thunk=thunk@entry=0xe11f00, handler=handler@entry=0xcc6440) at throw.c:73
#12 0x00007ffff7b1a9f5 in really_launch (d=0x7ffff5ea3780) at threads.c:1009
#13 0x00007ffff7aa013a in c_body (d=0x7ffff4ea1d40) at continuations.c:511
#14 0x00007ffff7b32738 in vm_regular_engine (vm=<optimized out>, program=0x6d27c0, argv=<optimized out>, nargs=2) at vm-i-system.c:855
#15 0x00007ffff7aa9723 in scm_call_4 (proc=0x7ae570, arg1=arg1@entry=0x404, arg2=<optimized out>, arg3=<optimized out>, arg4=<optimized out>)
    at eval.c:507
#16 0x00007ffff7b1c854 in scm_catch_with_pre_unwind_handler (key=key@entry=0x404, thunk=<optimized out>, handler=<optimized out>, 
    pre_unwind_handler=<optimized out>) at throw.c:86
#17 0x00007ffff7b1ca2f in scm_c_catch (tag=tag@entry=0x404, body=body@entry=0x7ffff7aa0130 <c_body>, body_data=body_data@entry=0x7ffff4ea1d40, 
    handler=handler@entry=0x7ffff7aa04e0 <c_handler>, handler_data=handler_data@entry=0x7ffff4ea1d40, 
    pre_unwind_handler=pre_unwind_handler@entry=0x7ffff7aa0290 <pre_unwind_handler>, 
    pre_unwind_handler_data=pre_unwind_handler_data@entry=0x6d8eb0) at throw.c:213
#18 0x00007ffff7aa086f in scm_i_with_continuation_barrier (body=body@entry=0x7ffff7aa0130 <c_body>, body_data=body_data@entry=0x7ffff4ea1d40, 
    handler=handler@entry=0x7ffff7aa04e0 <c_handler>, handler_data=handler_data@entry=0x7ffff4ea1d40, 
    pre_unwind_handler=pre_unwind_handler@entry=0x7ffff7aa0290 <pre_unwind_handler>, pre_unwind_handler_data=0x6d8eb0) at continuations.c:449
#19 0x00007ffff7aa0905 in scm_c_with_continuation_barrier (func=<optimized out>, data=<optimized out>) at continuations.c:545
#20 0x00007ffff7b1a21c in with_guile_and_parent (base=0x7ffff4ea1da0, data=0x7ffff4ea1dd0) at threads.c:908
#21 0x00007ffff6cf3e72 in GC_call_with_stack_base () from /usr/lib/x86_64-linux-gnu/libgc.so.1
#22 0x00007ffff7b19bdc in scm_i_with_guile_and_parent (parent=<optimized out>, data=0x7ffff5ea3780, func=0x7ffff7b1a990 <really_launch>)
    at threads.c:951
---Type <return> to continue, or q <return> to quit---
#23 launch_thread (d=0x7ffff5ea3780) at threads.c:1019
#24 0x00007ffff6cf8b83 in GC_inner_start_routine () from /usr/lib/x86_64-linux-gnu/libgc.so.1
#25 0x00007ffff6cf3e72 in GC_call_with_stack_base () from /usr/lib/x86_64-linux-gnu/libgc.so.1
#26 0x00007ffff72fae0e in start_thread (arg=0x7ffff4ea2700) at pthread_create.c:311
#27 0x00007ffff70300fd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:113

> Please see below for comments.

Thanks for the further review.  Updated patch attached.  My technical
writing skills aren't what they should be, so an extra thanks for
helping with the documentation.

- Dave


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: 0001-Add-cooperative-REPL-server-module.patch --]
[-- Type: text/x-diff, Size: 12671 bytes --]

From 952e2b3f199031896996b33bc058e42586cbc69e Mon Sep 17 00:00:00 2001
From: David Thompson <dthompson2@worcester.edu>
Date: Sun, 19 Jan 2014 13:16:02 -0500
Subject: [PATCH] Add cooperative REPL server module.

* module/system/repl/coop-server.scm: New module.

* module/system/repl/repl.scm (start-repl): Extract body to start-repl*.
  (start-repl*): New procedure.
  (run-repl): Extract body to run-repl*.
  (run-repl*): New procedure.

* module/system/repl/server.scm (run-server): Extract body to
  run-server*.
  (run-server*): New procedure.

* doc/ref/api-evaluation.texi: Add docs.

* module/Makefile.am: Add system/repl/coop-server.scm to SYSTEM_SOURCES.
---
 doc/ref/api-evaluation.texi        |  45 +++++++++++
 module/Makefile.am                 |   3 +-
 module/system/repl/coop-server.scm | 156 +++++++++++++++++++++++++++++++++++++
 module/system/repl/repl.scm        |  11 ++-
 module/system/repl/server.scm      |   5 +-
 5 files changed, 216 insertions(+), 4 deletions(-)
 create mode 100644 module/system/repl/coop-server.scm

diff --git a/doc/ref/api-evaluation.texi b/doc/ref/api-evaluation.texi
index 7d67d9a..27585e6 100644
--- a/doc/ref/api-evaluation.texi
+++ b/doc/ref/api-evaluation.texi
@@ -23,6 +23,7 @@ loading, evaluating, and compiling Scheme code at run time.
 * Local Evaluation::            Evaluation in a local lexical environment.
 * Local Inclusion::             Compile-time inclusion of one file in another.
 * REPL Servers::                Serving a REPL over a socket.
+* Cooperative REPL Servers::    REPL server for single-threaded applications.
 @end menu
 
 
@@ -1281,6 +1282,50 @@ with no arguments.
 Closes the connection on all running server sockets.
 @end deffn
 
+@node Cooperative REPL Servers
+@subsection Cooperative REPL Servers
+
+@cindex Cooperative REPL server
+
+The procedures in this section are provided by
+@lisp
+(use-modules (system repl coop-server))
+@end lisp
+
+Whereas ordinary REPL servers run in their own threads (@pxref{REPL
+Servers}), sometimes it is more convenient to provide REPLs that run at
+specified times within an existing thread, for example in programs
+utilizing an event loop or in single-threaded programs.  This allows for
+safe access and mutation of a program's data structures from the REPL,
+without concern for thread synchronization.
+
+Although the REPLs are run in the thread that calls
+@code{spawn-coop-repl-server} and @code{poll-coop-repl-server},
+dedicated threads are spawned so that the calling thread is not blocked.
+The spawned threads read input for the REPLs and to listen for new
+connections.
+
+Cooperative REPL servers must be polled periodically to evaluate any
+pending expressions by calling @code{poll-coop-repl-server} with the
+object returned from @code{spawn-coop-repl-server}.  The thread that
+calls @code{poll-coop-repl-server} will be blocked for as long as the
+expression takes to be evaluated or if the debugger is entered.
+
+@deffn {Scheme Procedure} spawn-coop-repl-server [server-socket]
+Create and return a new cooperative REPL server object, and spawn a new
+thread to listen for connections on @var{server-socket}.  Proper
+functioning of the REPL server requires that
+@code{poll-coop-repl-server} be called periodically on the returned
+server object.
+@end deffn
+
+@deffn {Scheme Procedure} poll-coop-repl-server coop-server
+Poll the cooperative REPL server @var{coop-server} and apply a pending
+operation if there is one, such as evaluating an expression typed at the
+REPL prompt.  This procedure must be called from the same thread that
+called @code{spawn-coop-repl-server}.
+@end deffn
+
 @c Local Variables:
 @c TeX-master: "guile.texi"
 @c End:
diff --git a/module/Makefile.am b/module/Makefile.am
index 8a7befd..b7960dc 100644
--- a/module/Makefile.am
+++ b/module/Makefile.am
@@ -360,7 +360,8 @@ SYSTEM_SOURCES =				\
   system/repl/common.scm			\
   system/repl/command.scm			\
   system/repl/repl.scm				\
-  system/repl/server.scm
+  system/repl/server.scm			\
+  system/repl/coop-server.scm
 
 LIB_SOURCES =					\
   statprof.scm					\
diff --git a/module/system/repl/coop-server.scm b/module/system/repl/coop-server.scm
new file mode 100644
index 0000000..4c8dc77
--- /dev/null
+++ b/module/system/repl/coop-server.scm
@@ -0,0 +1,156 @@
+;;; Cooperative REPL server
+
+;; Copyright (C)  2014 Free Software Foundation, Inc.
+
+;; This library is free software; you can redistribute it and/or
+;; modify it under the terms of the GNU Lesser General Public
+;; License as published by the Free Software Foundation; either
+;; version 3 of the License, or (at your option) any later version.
+;;
+;; This library is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+;; Lesser General Public License for more details.
+;;
+;; You should have received a copy of the GNU Lesser General Public
+;; License along with this library; if not, write to the Free Software
+;; Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+;; 02110-1301 USA
+
+;;; Code:
+
+(define-module (system repl coop-server)
+  #:use-module (ice-9 match)
+  #:use-module (ice-9 mvars)
+  #:use-module (ice-9 receive)
+  #:use-module (ice-9 threads)
+  #:use-module (srfi srfi-9)
+  #:use-module ((system repl repl)
+                #:select (start-repl* prompting-meta-read))
+  #:use-module ((system repl server)
+                #:select (run-server* make-tcp-server-socket close-socket!))
+  #:export (spawn-coop-repl-server
+            poll-coop-repl-server))
+
+(define-record-type <coop-repl-server>
+  (%make-coop-repl-server eval-mvar)
+  coop-repl-server?
+  (eval-mvar coop-repl-server-eval-mvar))
+
+(define (make-coop-repl-server)
+  (%make-coop-repl-server (new-empty-mvar)))
+
+(define (coop-repl-server-eval coop-server opcode . args)
+  "Put a new instruction with the symbolic name OPCODE and an arbitrary
+number of arguments into the evaluation mvar of COOP-SERVER."
+  (put-mvar (coop-repl-server-eval-mvar coop-server)
+            (cons opcode args)))
+
+(define-record-type <coop-repl>
+  (%make-coop-repl read-mvar cont)
+  coop-repl?
+  (read-mvar coop-repl-read-mvar)
+  (cont coop-repl-cont set-coop-repl-cont!))
+
+(define (make-coop-repl)
+  (%make-coop-repl (new-empty-mvar) #f))
+
+(define (coop-repl-read coop-repl)
+  "Read an expression via the thunk stored in COOP-REPL."
+  ((take-mvar (coop-repl-read-mvar coop-repl))))
+
+(define (store-repl-cont cont coop-repl)
+  "Save the partial continuation CONT within COOP-REPL."
+  (set-coop-repl-cont! coop-repl
+                       (lambda (exp)
+                         (coop-repl-prompt
+                          (lambda () (cont exp))))))
+
+(define (coop-repl-prompt thunk)
+  "Apply THUNK within a prompt for cooperative REPLs."
+  (call-with-prompt 'coop-repl-prompt thunk store-repl-cont))
+
+(define (make-coop-reader coop-repl)
+  "Return a new procedure for reading user input from COOP-REPL.  The
+generated procedure passes the responsibility of reading input to
+another thread via an mvar and aborts the cooperative REPL prompt."
+  (lambda (repl)
+    (put-mvar (coop-repl-read-mvar coop-repl)
+              ;; Need to preserve the REPL stack and current module across
+              ;; threads.
+              (let ((stack (fluid-ref *repl-stack*))
+                    (module (current-module)))
+                (lambda ()
+                  (with-fluids ((*repl-stack* stack))
+                    (set-current-module module)
+                    (prompting-meta-read repl)))))
+    (abort-to-prompt 'coop-repl-prompt coop-repl)))
+
+(define (reader-loop coop-server coop-repl)
+  "Run an unbounded loop that reads an expression for COOP-REPL and
+stores the expression within COOP-SERVER for later evaluation."
+  (coop-repl-server-eval coop-server 'eval coop-repl
+                         (coop-repl-read coop-repl))
+  (reader-loop coop-server coop-repl))
+
+(define (poll-coop-repl-server coop-server)
+  "Test if there is an expression waiting to be evaluated within
+COOP-SERVER and evaluate it if so."
+  (receive (op success?)
+      (try-take-mvar (coop-repl-server-eval-mvar coop-server))
+    (when success?
+      (match op
+        (('new-repl client)
+         (start-repl-client coop-server client))
+        (('eval coop-repl exp)
+         ((coop-repl-cont coop-repl) exp))))))
+
+(define (start-coop-repl coop-server)
+  "Start a new cooperative REPL process for COOP-SERVER."
+  ;; Calling stop-server-and-clients! from a REPL will cause an
+  ;; exception to be thrown when trying to read from the socket that has
+  ;; been closed, so we catch that here.
+  (false-if-exception
+   (let ((coop-repl (make-coop-repl)))
+     (make-thread reader-loop coop-server coop-repl)
+     (start-repl* (current-language) #f (make-coop-reader coop-repl)))))
+
+(define (run-coop-repl-server coop-server server-socket)
+  "Start the cooperative REPL server for COOP-SERVER using the socket
+SERVER-SOCKET."
+  (run-server* server-socket (make-coop-client-proc coop-server)))
+
+(define* (spawn-coop-repl-server
+          #:optional (server-socket (make-tcp-server-socket)))
+  "Return a newly allocated cooperative REPL server and run the server
+in a new thread, making it available over SERVER-SOCKET."
+  (let ((coop-server (make-coop-repl-server)))
+    (make-thread run-coop-repl-server
+                 coop-server
+                 server-socket)
+    coop-server))
+
+(define (make-coop-client-proc coop-server)
+  "Return a new procedure that is used to schedule the creation of a new
+cooperative REPL for COOP-SERVER."
+  (lambda (client addr)
+    (coop-repl-server-eval coop-server 'new-repl client)))
+
+(define (start-repl-client coop-server client)
+  "Run a cooperative REPL for COOP-SERVER within a prompt.  All input
+and output is sent over the socket CLIENT."
+  (with-continuation-barrier
+   (lambda ()
+     (coop-repl-prompt
+      (lambda ()
+        (with-input-from-port client
+          (lambda ()
+            (with-output-to-port client
+              (lambda ()
+                (with-error-to-port client
+                  (lambda ()
+                    (with-fluids ((*repl-stack* '()))
+                      (save-module-excursion
+                       (lambda ()
+                         (start-coop-repl coop-server))))))))))
+        (close-socket! client))))))
diff --git a/module/system/repl/repl.scm b/module/system/repl/repl.scm
index 1649556..20309e3 100644
--- a/module/system/repl/repl.scm
+++ b/module/system/repl/repl.scm
@@ -1,6 +1,7 @@
 ;;; Read-Eval-Print Loop
 
-;; Copyright (C) 2001, 2009, 2010, 2011, 2013 Free Software Foundation, Inc.
+;; Copyright (C) 2001, 2009, 2010, 2011, 2013,
+;;   2014 Free Software Foundation, Inc.
 
 ;; This library is free software; you can redistribute it and/or
 ;; modify it under the terms of the GNU Lesser General Public
@@ -129,10 +130,13 @@
 ;;;
 
 (define* (start-repl #:optional (lang (current-language)) #:key debug)
+  (start-repl* lang debug prompting-meta-read))
+
+(define (start-repl* lang debug prompting-meta-read)
   ;; ,language at the REPL will update the current-language.  Make
   ;; sure that it does so in a new dynamic scope.
   (parameterize ((current-language lang))
-    (run-repl (make-repl lang debug))))
+    (run-repl* (make-repl lang debug) prompting-meta-read)))
 
 ;; (put 'abort-on-error 'scheme-indent-function 1)
 (define-syntax-rule (abort-on-error string exp)
@@ -144,6 +148,9 @@
       (abort))))
 
 (define (run-repl repl)
+  (run-repl* repl prompting-meta-read))
+
+(define (run-repl* repl prompting-meta-read)
   (define (with-stack-and-prompt thunk)
     (call-with-prompt (default-prompt-tag)
                       (lambda () (start-stack #t (thunk)))
diff --git a/module/system/repl/server.scm b/module/system/repl/server.scm
index 2df7564..cfa1261 100644
--- a/module/system/repl/server.scm
+++ b/module/system/repl/server.scm
@@ -1,6 +1,6 @@
 ;;; Repl server
 
-;; Copyright (C)  2003, 2010, 2011 Free Software Foundation, Inc.
+;; Copyright (C)  2003, 2010, 2011, 2014 Free Software Foundation, Inc.
 
 ;; This library is free software; you can redistribute it and/or
 ;; modify it under the terms of the GNU Lesser General Public
@@ -68,6 +68,9 @@
     sock))
 
 (define* (run-server #:optional (server-socket (make-tcp-server-socket)))
+  (run-server* server-socket serve-client))
+
+(define (run-server* server-socket serve-client)
   (define (accept-new-client)
     (catch #t
       (lambda () (accept server-socket))
-- 
1.8.5.2