Stefan,
the patch seems to be working. Thanks for fixing it.
I'll play with it some more and will then commit.

--

       --- michael


On 06/22/2013 05:56 PM, Stefan Monnier wrote:
Heya, just checking back about the status of this bug report. I'm
affected by it and I've been stuck with Emacs 23 since its first arrival
(or else I can jump back on Emacs 24.3.x ditching viper all together,
which might even be a good thing for my saneness :-))
I don't understand all of what Viper does with the ESC key, nor do
I know what are the different cases where ESC has to behave in
a specific way, so it's difficult for me to come up with
a trustworthy patch.

Could you try the patch below under X11 and under a tty, ideally even
using emacsclient to create new X11 and tty frames?

Michael, could you also take a look at this patch?


        Stefan


=== modified file 'lisp/emulation/viper-cmd.el'
--- lisp/emulation/viper-cmd.el	2013-06-18 20:24:44 +0000
+++ lisp/emulation/viper-cmd.el	2013-06-22 21:40:32 +0000
@@ -996,93 +996,7 @@
 	(suspend-emacs))
     (viper-change-state-to-emacs)))
 
-
-;; Intercept ESC sequences on dumb terminals.
-;; Based on the idea contributed by Marcelino Veiga Tuimil <mveiga@dit.upm.es>
-
-;; Check if last key was ESC and if so try to reread it as a function key.
-;; But only if there are characters to read during a very short time.
-;; Returns the last event, if any.
-(defun viper-envelop-ESC-key ()
-  (let ((event last-input-event)
-	(keyseq [nil])
-	(inhibit-quit t))
-    (if (viper-ESC-event-p event)
-	(progn
-	  ;; Some versions of Emacs (eg., 22.50.8 (?)) have a bug, which makes
-          ;; even a single ESC into a fast keyseq. To guard against this, we
-	  ;; added a check if there are other events as well.  Keep the next
-	  ;; line for the next time the bug reappears, so that will remember to
-	  ;; report it.
-	  ;;(if (and (viper-fast-keysequence-p) unread-command-events)
-	  (if (viper-fast-keysequence-p) ;; for Emacsen without the above bug
-	      (progn
-		(let (minor-mode-map-alist emulation-mode-map-alists)
-		  (viper-set-unread-command-events event)
-		  (setq keyseq (read-key-sequence nil 'continue-echo))
-		  ) ; let
-		;; If keyseq translates into something that still has ESC
-		;; at the beginning, separate ESC from the rest of the seq.
-		;; In XEmacs we check for events that are keypress meta-key
-		;; and convert them into [escape key]
-		;;
-		;; This is needed for the following reason:
-		;; If ESC is the first symbol, we interpret it as if the
-		;; user typed ESC and then quickly some other symbols.
-		;; If ESC is not the first one, then the key sequence
-		;; entered was apparently translated into a function key or
-		;; something (e.g., one may have
-		;; (define-key function-key-map "\e[192z" [f11])
-		;; which would translate the escape-sequence generated by
-		;; f11 in an xterm window into the symbolic key f11.
-		;;
-		;; If `first-key' is not an ESC event, we make it into the
-		;; last-command-event in order to pretend that this key was
-		;; pressed.  This is needed to allow arrow keys to be bound to
-		;; macros.  Otherwise, viper-exec-mapped-kbd-macro will think
-		;; that the last event was ESC and so it'll execute whatever is
-		;; bound to ESC. (Viper macros can't be bound to
-		;; ESC-sequences).
-		(let* ((first-key (elt keyseq 0))
-		       (key-mod (event-modifiers first-key)))
-		  (cond ((and (viper-ESC-event-p first-key)
-			      (not (viper-translate-all-ESC-keysequences)))
-			 ;; put keys following ESC on the unread list
-			 ;; and return ESC as the key-sequence
-			 (viper-set-unread-command-events (viper-subseq keyseq 1))
-			 (setq last-input-event event
-			       keyseq (if (featurep 'emacs)
-					  "\e"
-					(vector (character-to-event ?\e)))))
-			((and (featurep 'xemacs)
-			      (key-press-event-p first-key)
-			      (equal '(meta) key-mod))
-			 (viper-set-unread-command-events
-			  (vconcat (vector
-				    (character-to-event (event-key first-key)))
-				   (viper-subseq keyseq 1)))
-			 (setq last-input-event event
-			       keyseq (vector (character-to-event ?\e))))
-			((eventp first-key)
-			 (setq last-command-event
-			       (viper-copy-event first-key)))
-			))
-		) ; end progn
-
-	    ;; this is escape event with nothing after it
-	    ;; put in unread-command-event and then re-read
-	    (viper-set-unread-command-events event)
-	    (setq keyseq (read-key-sequence nil))
-	    ))
-      ;; not an escape event
-      (setq keyseq (vector event)))
-    keyseq))
-
-
-
 ;; Listen to ESC key.
-;; If a sequence of keys starting with ESC is issued with very short delays,
-;; interpret these keys in Emacs mode, so ESC won't be interpreted as a Vi key.
 (defun viper-intercept-ESC-key ()
   "Function that implements ESC key in Viper emulation of Vi."
   (interactive)
@@ -1090,13 +1004,7 @@
   ;; minor-mode map(s) have been temporarily disabled so the ESC
   ;; binding to viper-intercept-ESC-key doesn't hide the binding we're
   ;; looking for (Bug#9146):
-  (let* ((event (viper-envelop-ESC-key))
-	 (cmd (cond ((equal event viper-ESC-key)
-		     'viper-intercept-ESC-key)
-		    ((let ((emulation-mode-map-alists nil))
-		       (key-binding event)))
-		    (t
-		     (error "Viper bell")))))
+  (let* ((cmd 'viper-intercept-ESC-key))
 
     ;; call the actual function to execute ESC (if no other symbols followed)
     ;; or the key bound to the ESC sequence (if the sequence was issued

=== modified file 'lisp/emulation/viper-keym.el'
--- lisp/emulation/viper-keym.el	2013-01-01 09:11:05 +0000
+++ lisp/emulation/viper-keym.el	2013-06-22 21:43:10 +0000
@@ -192,7 +192,7 @@
   :type 'string
   :group 'viper)
 
-(defvar viper-ESC-key (kbd "ESC")
+(defconst viper-ESC-key [escape]
   "Key used to ESC.")
 
 

=== modified file 'lisp/emulation/viper.el'
--- lisp/emulation/viper.el	2013-03-12 02:08:21 +0000
+++ lisp/emulation/viper.el	2013-06-22 21:54:55 +0000
@@ -660,7 +660,7 @@
 undone.
 It also can't undo some Viper settings."
   (interactive)
-
+  (viper-setup-ESC-to-escape nil)
   ;; restore non-viper vars
   (setq-default
    next-line-add-newlines
@@ -825,6 +825,58 @@
   (add-hook 'viper-post-command-hooks 'set-viper-state-in-major-mode t))
 
 
+;;; Handling of tty's ESC event
+
+;; On a tty, an ESC event can either be the user hitting the escape key, or
+;; some element of a byte sequence used to encode for example cursor keys.
+;; So we try to recognize those events that correspond to the escape key and
+;; turn them into `escape' events (same as used under GUIs).  The heuristic we
+;; use to distinguish the two cases is based, as usual, on a timeout, and on
+;; the fact that the special ESC=>escape mapping only takes place if the whole
+;; last key-sequence so far is just [?\e], i.e. either we're still in
+;; read-key-sequence, or the last read-key-sequence only read [?\e], which
+;; should ideally never happen because it should have been mapped to [escape].
+
+(defun viper--tty-ESC-filter (map)
+  (if (and (equal (this-single-command-keys) [?\e])
+           (sit-for (/ viper-fast-keyseq-timeout 1000)))
+      [escape] map))
+
+(defun viper--lookup-key (map key)
+  "Kind of like `lookup-key'.
+Two differences:
+- KEY is a single key, not a sequence.
+- the result is the \"raw\" binding, so it can be a `menu-item', rather than the
+  binding contained in that menu item."
+  (catch 'found
+    (map-keymap (lambda (k b) (if (equal key k) (throw 'found b))) map)))
+
+(defun viper-catch-tty-ESC ()
+  "Setup key mappings of current terminal to turn a tty's ESC into `escape'."
+  (when (memq (terminal-live-p (frame-terminal)) '(t pc))
+    (let ((esc-binding (viper-uncatch-tty-ESC)))
+      (define-key input-decode-map
+        [?\e] `(menu-item "" ,esc-binding :filter viper--tty-ESC-filter)))))
+
+(defun viper-uncatch-tty-ESC ()
+  "Don't hack ESC into `escape' any more."
+  (let ((b (viper--lookup-key input-decode-map ?\e)))
+    (and (eq 'menu-item (car-safe b))
+         (eq 'viper--tty-ESC-filter (nth 4 b))
+         (define-key input-decode-map [?\e] (setq b (nth 2 b))))
+    b))
+
+(defun viper-setup-ESC-to-escape (enable)
+  (if enable
+      (add-hook 'tty-setup-hook 'viper-catch-tty-ESC)
+    (remove-hook 'tty-setup-hook 'viper-catch-tty-ESC))
+  (let ((seen ()))
+    (dolist (frame (frame-list))
+      (let ((terminal (frame-terminal frame)))
+        (unless (memq terminal seen)
+          (push terminal seen)
+          (with-selected-frame frame
+            (if enable (viper-catch-tty-ESC) (viper-uncatch-tty-ESC))))))))
 
 ;; This sets major mode hooks to make them come up in vi-state.
 (defun viper-set-hooks ()
@@ -837,6 +889,8 @@
   (if (eq (default-value 'major-mode) 'fundamental-mode)
       (setq-default major-mode 'viper-mode))
 
+  (viper-setup-ESC-to-escape t)
+
   (add-hook 'change-major-mode-hook 'viper-major-mode-change-sentinel)
   (add-hook 'find-file-hooks 'set-viper-state-in-major-mode)
 
@@ -847,13 +901,6 @@
   (defvar emerge-startup-hook)
   (add-hook 'emerge-startup-hook 'viper-change-state-to-emacs)
 
-  ;; Zap bad bindings in flyspell-mouse-map, which prevent ESC from working
-  ;; over misspelled words (due to the overlay keymaps)
-  (defvar flyspell-mode-hook)
-  (defvar flyspell-mouse-map)
-  (add-hook 'flyspell-mode-hook
-	    (lambda ()
-              (define-key flyspell-mouse-map viper-ESC-key nil)))
   ;; if viper is started from .emacs, it might be impossible to get certain
   ;; info about the display and windows until emacs initialization is complete
   ;; So do it via the window-setup-hook

=== modified file 'lisp/faces.el'
--- lisp/faces.el	2013-05-15 23:55:41 +0000
+++ lisp/faces.el	2013-06-22 20:22:31 +0000
@@ -2126,7 +2126,8 @@
 		       type)
 	(when (fboundp term-init-func)
 	  (funcall term-init-func))
-	(set-terminal-parameter frame 'terminal-initted term-init-func)))))
+        (set-terminal-parameter frame 'terminal-initted term-init-func)
+	(run-hooks 'tty-setup-hook)))))
 
 ;; Called from C function init_display to initialize faces of the
 ;; dumped terminal frame on startup.