From a430631efdfdb1a668f1b0ffd6f6f857e0df3466 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mattias=20Engdeg=C3=A5rd?= <mattiase@acm.org>
Date: Mon, 25 Nov 2019 18:54:20 +0100
Subject: [PATCH] Fine-grained NS modifier key settings (bug#38296)

For the ns-MOD-modifier and ns-right-MOD-modifier variables, MOD
being 'control', 'command', 'alternate' and 'function', allow
values on the form (:ordinary SYMBOL :function :SYMBOL :mouse SYMBOL),
so that the key can be used for different modifiers (or none) in
different contexts.  This is particularly useful for using the macOS
Option key for extended character entry while still using it as an
Emacs modifier for function keys and mouse clicks.

* src/nsterm.m (mod_of_kind, right_mod, nil_or_none): Helper functions.
(EV_MODIFIERS2): Add KIND argument.
(EV_MODIFIERS): Adapt call to EV_MODIFIERS2.
(ns_get_shifted_character): Use correct event kind for modifiers.
(ns-alternate-modifier, ns-right-alternate-modifier)
(ns-command-modifier, ns-right-command-modifier)
(ns-control-modifier, ns-right-control-modifier)
(ns-function-modifier): Rewrite doc strings for new data format.
(QCordinary, QCfunction, QCmouse): Define symbols.
* lisp/cus-start.el: Conform to new data types.
* doc/emacs/macos.texi (Mac / GNUstep Basics)
(Mac / GNUstep Customization): Improved documentation.
* etc/NEWS: Mention the change.
---
 doc/emacs/macos.texi |  60 +++++++++++++----
 etc/NEWS             |   6 ++
 lisp/cus-start.el    |  97 ++++++++++++++++++++++++----
 src/nsterm.m         | 150 +++++++++++++++++++++++++++++--------------
 4 files changed, 241 insertions(+), 72 deletions(-)

diff --git a/doc/emacs/macos.texi b/doc/emacs/macos.texi
index d9920957ad..fe944e411d 100644
--- a/doc/emacs/macos.texi
+++ b/doc/emacs/macos.texi
@@ -48,18 +48,8 @@ Mac / GNUstep Basics
 Emacs provides a set of key bindings using this modifier key that mimic
 other Mac / GNUstep applications (@pxref{Mac / GNUstep Events}).  You
 can change these bindings in the usual way (@pxref{Key Bindings}).
-
-@vindex ns-alternate-modifier
-@vindex ns-right-alternate-modifier
-  The variable @code{ns-right-alternate-modifier} controls the
-behavior of the right @key{Alt} and @key{Option} keys.  These keys
-behave like the left-hand keys if the value is @code{left} (the
-default).  A value of @code{control}, @code{meta}, @code{alt},
-@code{super}, or @code{hyper} makes them behave like the corresponding
-modifier keys; a value of @code{left} means be the same key as
-@code{ns-alternate-modifier}; a value of @code{none} tells Emacs to
-ignore them, in which case you get the default behavior of macOS
-accentuation system from the right @key{Option} key.
+The modifiers themselves can be customised;
+@pxref{Mac / GNUstep Customization}.
 
   @kbd{S-mouse-1} adjusts the region to the click position,
 just like @kbd{mouse-3} (@code{mouse-save-then-kill}); it does not pop
@@ -107,6 +97,52 @@ Mac / GNUstep Customization
 keys and the fullscreen behavior.  To see all such options, use
 @kbd{M-x customize-group @key{RET} ns @key{RET}}.
 
+@subsection Modifier keys
+
+The following variables control the behaviour of the actual modifier
+keys:
+
+@table @code
+@vindex ns-alternate-modifier
+@vindex ns-right-alternate-modifier
+@item ns-alternate-modifier
+@itemx ns-right-alternate-modifier
+The left and right @key{Option} or @key{Alt} keys.
+
+@vindex ns-command-modifier
+@vindex ns-right-command-modifier
+@item ns-command-modifier
+@itemx ns-right-command-modifier
+The left and right @key{Command} keys.
+
+@vindex ns-control-modifier
+@vindex ns-right-control-modifier
+@item ns-control-modifier
+@itemx ns-right-control-modifier
+The left and right @key{Control} keys.
+
+@vindex ns-function-modifier
+@item ns-function-modifier
+The @key{Function} (fn) key.
+@end table
+
+The value of each variable is either a symbol, describing the key for
+any purpose, or a list on the form
+@code{(:ordinary @var{symbol} :function @var{symbol} :mouse @var{symbol})},
+which describes the modifier when used with ordinary keys, function keys
+(that do not produce a character, such as arrow keys), and mouse clicks.
+
+If the @var{symbol} is one of @code{control}, @code{meta}, @code{alt},
+@code{super} or @code{hyper}, this describes the Emacs modifier it
+represents.  If @var{symbol} is @code{none}, Emacs does not use the
+key, which retains its standard behaviour.  For instance, the
+@key{Option} key in macOS is normally used for composing additional
+characters.
+
+The variables for right-hand keys, like @code{ns-right-alternate-modifier},
+may also be set to @code{left}, which means to use the same behaviour as
+the corresponding left-hand key.
+
 @subsection Font Panel
 
 @findex ns-popup-font-panel
diff --git a/etc/NEWS b/etc/NEWS
index edd536f2f0..7056b99878 100644
--- a/etc/NEWS
+++ b/etc/NEWS
@@ -3241,6 +3241,12 @@ Previously it was supported only in the Cygwin-w32 build.
 ** Emacs now handles key combinations involving the macOS "command"
 and "option" modifier keys more correctly.
 
+** MacOS modifier key behaviour is now more adjustable.
+The behaviour of the macOS "Option", "Command", "Control" and
+"Function" keys can now be specified separately for use with
+ordinary keys, function keys and mouse clicks.  This allows using them
+in their standard macOS way for composing characters.
+
 ** The special handling of 'frame-title-format' on NS where setting it
 to 't' would enable the macOS proxy icon has been replaced with a
 separate variable, 'ns-use-proxy-icon'.  'frame-title-format' will now
diff --git a/lisp/cus-start.el b/lisp/cus-start.el
index e4b6d8f2d6..1c497ee5ae 100644
--- a/lisp/cus-start.el
+++ b/lisp/cus-start.el
@@ -424,16 +424,23 @@ minibuffer-prompt-properties--setter
 	     ;; msdos.c
 	     (dos-unsupported-char-glyph display integer)
 	     ;; nsterm.m
-             ;;
-             ;; FIXME: Why does ⌃ use nil instead of none?  Also the
-             ;; description is confusing; setting it to nil disables ⌃
-             ;; entirely.
 	     (ns-control-modifier
 	      ns
-	      (choice (const :tag "No modifier" nil)
+	      (choice (const :tag "No modifier" none)
 		      (const control) (const meta)
 		      (const alt) (const hyper)
-		      (const super)) "23.1")
+		      (const super)
+                      (plist :key-type (choice (const :ordinary)
+                                               (const :function)
+                                               (const :mouse))
+                             :value-type (choice (const control)
+                                                 (const meta)
+                                                 (const alt)
+                                                 (const hyper)
+                                                 (const super)
+                                                 (const :tag "No modifier"
+                                                        none))))
+              "23.1")
 	     (ns-right-control-modifier
 	      ns
 	      (choice (const :tag "No modifier (work as control)" none)
@@ -441,13 +448,35 @@ minibuffer-prompt-properties--setter
 			     left)
 		      (const control) (const meta)
 		      (const alt) (const hyper)
-		      (const super)) "24.1")
+		      (const super)
+                      (plist :key-type (choice (const :ordinary)
+                                               (const :function)
+                                               (const :mouse))
+                             :value-type (choice (const control)
+                                                 (const meta)
+                                                 (const alt)
+                                                 (const hyper)
+                                                 (const super)
+                                                 (const :tag "No modifier"
+                                                        none))))
+              "24.1")
 	     (ns-command-modifier
 	      ns
 	      (choice (const :tag "No modifier (work as layout switch)" none)
 		      (const control) (const meta)
 		      (const alt) (const hyper)
-		      (const super)) "23.1")
+		      (const super)
+                      (plist :key-type (choice (const :ordinary)
+                                               (const :function)
+                                               (const :mouse))
+                             :value-type (choice (const control)
+                                                 (const meta)
+                                                 (const alt)
+                                                 (const hyper)
+                                                 (const super)
+                                                 (const :tag "No modifier"
+                                                        none))))
+              "23.1")
 	     (ns-right-command-modifier
 	      ns
 	      (choice (const :tag "No modifier (work as layout switch)" none)
@@ -455,13 +484,35 @@ minibuffer-prompt-properties--setter
 			     left)
 		      (const control) (const meta)
 		      (const alt) (const hyper)
-		      (const super)) "24.1")
+		      (const super)
+                      (plist :key-type (choice (const :ordinary)
+                                               (const :function)
+                                               (const :mouse))
+                             :value-type (choice (const control)
+                                                 (const meta)
+                                                 (const alt)
+                                                 (const hyper)
+                                                 (const super)
+                                                 (const :tag "No modifier"
+                                                        none))))
+              "24.1")
 	     (ns-alternate-modifier
 	      ns
 	      (choice (const :tag "No modifier (work as alternate/option)" none)
 		      (const control) (const meta)
 		      (const alt) (const hyper)
-		      (const super)) "23.1")
+		      (const super)
+                      (plist :key-type (choice (const :ordinary)
+                                               (const :function)
+                                               (const :mouse))
+                             :value-type (choice (const control)
+                                                 (const meta)
+                                                 (const alt)
+                                                 (const hyper)
+                                                 (const super)
+                                                 (const :tag "No modifier"
+                                                        none))))
+              "23.1")
 	     (ns-right-alternate-modifier
 	      ns
 	      (choice (const :tag "No modifier (work as alternate/option)" none)
@@ -469,13 +520,35 @@ minibuffer-prompt-properties--setter
 			     left)
 		      (const control) (const meta)
 		      (const alt) (const hyper)
-		      (const super)) "23.3")
+		      (const super)
+                      (plist :key-type (choice (const :ordinary)
+                                               (const :function)
+                                               (const :mouse))
+                             :value-type (choice (const control)
+                                                 (const meta)
+                                                 (const alt)
+                                                 (const hyper)
+                                                 (const super)
+                                                 (const :tag "No modifier"
+                                                        none))))
+              "23.3")
 	     (ns-function-modifier
 	      ns
 	      (choice (const :tag "No modifier (work as function)" none)
 		      (const control) (const meta)
 		      (const alt) (const hyper)
-		      (const super)) "23.1")
+		      (const super)
+                      (plist :key-type (choice (const :ordinary)
+                                               (const :function)
+                                               (const :mouse))
+                             :value-type (choice (const control)
+                                                 (const meta)
+                                                 (const alt)
+                                                 (const hyper)
+                                                 (const super)
+                                                 (const :tag "No modifier"
+                                                        none))))
+              "23.1")
 	     (ns-antialias-text ns boolean "23.1")
 	     (ns-auto-hide-menu-bar ns boolean "24.1")
              (ns-confirm-quit ns boolean "25.1")
diff --git a/src/nsterm.m b/src/nsterm.m
index e1d745e332..52a9830be8 100644
--- a/src/nsterm.m
+++ b/src/nsterm.m
@@ -354,6 +354,19 @@ - (NSColor *)colorUsingDefaultColorSpace
 #define NSLeftAlternateKeyMask  (0x000020 | NSEventModifierFlagOption)
 #define NSRightAlternateKeyMask (0x000040 | NSEventModifierFlagOption)
 
+/* MODIFIER if a symbol; otherwise its property KIND, if a symbol.  */
+static Lisp_Object
+mod_of_kind (Lisp_Object modifier, Lisp_Object kind)
+{
+  if (SYMBOLP (modifier))
+    return modifier;
+  else
+    {
+      Lisp_Object val = Fplist_get (modifier, kind);
+      return SYMBOLP (val) ? val : Qnil;
+    }
+}
+
 static unsigned int
 ev_modifiers_helper (unsigned int flags, unsigned int left_mask,
                      unsigned int right_mask, unsigned int either_mask,
@@ -380,30 +393,35 @@ - (NSColor *)colorUsingDefaultColorSpace
   return modifiers;
 }
 
-#define EV_MODIFIERS2(flags)                                            \
+#define EV_MODIFIERS2(flags, kind)                                      \
   (((flags & NSEventModifierFlagHelp) ?                                 \
     hyper_modifier : 0)                                                 \
    | ((flags & NSEventModifierFlagShift) ?                              \
       shift_modifier : 0)                                               \
-   | ((flags & NS_FUNCTION_KEY_MASK) ?                                  \
-      parse_solitary_modifier (ns_function_modifier) : 0)               \
+   | ((flags & NS_FUNCTION_KEY_MASK)                                    \
+      ? parse_solitary_modifier (mod_of_kind (ns_function_modifier,     \
+                                              kind))                    \
+      : 0)                                                              \
    | ev_modifiers_helper (flags, NSLeftControlKeyMask,                  \
                           NSRightControlKeyMask,                        \
                           NSEventModifierFlagControl,                   \
-                          ns_control_modifier,                          \
-                          ns_right_control_modifier)                    \
+                          mod_of_kind (ns_control_modifier, kind),      \
+                          mod_of_kind (ns_right_control_modifier,       \
+                                       kind))                           \
    | ev_modifiers_helper (flags, NSLeftCommandKeyMask,                  \
                           NSRightCommandKeyMask,                        \
                           NSEventModifierFlagCommand,                   \
-                          ns_command_modifier,                          \
-                          ns_right_command_modifier)                    \
+                          mod_of_kind (ns_command_modifier, kind),      \
+                          mod_of_kind (ns_right_command_modifier,       \
+                                       kind))                           \
    | ev_modifiers_helper (flags, NSLeftAlternateKeyMask,                \
                           NSRightAlternateKeyMask,                      \
                           NSEventModifierFlagOption,                    \
-                          ns_alternate_modifier,                        \
-                          ns_right_alternate_modifier))
+                          mod_of_kind (ns_alternate_modifier, kind),    \
+                          mod_of_kind (ns_right_alternate_modifier,     \
+                                       kind)))
 
-#define EV_MODIFIERS(e) EV_MODIFIERS2 ([e modifierFlags])
+#define EV_MODIFIERS(e) EV_MODIFIERS2 ([e modifierFlags], QCmouse)
 
 #define EV_UDMODIFIERS(e)                                      \
     ((([e type] == NSEventTypeLeftMouseDown) ? down_modifier : 0)       \
@@ -2599,6 +2617,18 @@ so some key presses (TAB) are swallowed by the system.  */
 }
 
 #ifdef NS_IMPL_COCOA
+static Lisp_Object
+right_mod (Lisp_Object left, Lisp_Object right)
+{
+  return EQ (right, Qleft) ? left : right;
+}
+
+static bool
+nil_or_none (Lisp_Object val)
+{
+  return NILP (val) || EQ (val, Qnone);
+}
+
 static UniChar
 ns_get_shifted_character (NSEvent *event)
 /* Look up the character corresponding to the key pressed on the
@@ -2630,25 +2660,25 @@ so some key presses (TAB) are swallowed by the system.  */
   NSTRACE ("ns_get_shifted_character");
 
   if ((flags & NSRightAlternateKeyMask) == NSRightAlternateKeyMask
-      && (EQ (ns_right_alternate_modifier, Qnone)
-          || (EQ (ns_right_alternate_modifier, Qleft)
-              && EQ (ns_alternate_modifier, Qnone))))
+      && nil_or_none (mod_of_kind (right_mod (ns_alternate_modifier,
+                                              ns_right_alternate_modifier),
+                                   QCordinary)))
     modifiers |= rightOptionKey;
 
   if ((flags & NSLeftAlternateKeyMask) == NSLeftAlternateKeyMask
-      && EQ (ns_alternate_modifier, Qnone))
+      && nil_or_none (mod_of_kind (ns_alternate_modifier, QCordinary)))
     modifiers |= optionKey;
 
   if ((flags & NSRightCommandKeyMask) == NSRightCommandKeyMask
-      && (EQ (ns_right_command_modifier, Qnone)
-          || (EQ (ns_right_command_modifier, Qleft)
-              && EQ (ns_command_modifier, Qnone))))
+      && nil_or_none (mod_of_kind (right_mod (ns_command_modifier,
+                                              ns_right_command_modifier),
+                                   QCordinary)))
     /* Carbon doesn't differentiate between left and right command
        keys.  */
     modifiers |= cmdKey;
 
   if ((flags & NSLeftCommandKeyMask) == NSLeftCommandKeyMask
-      && EQ (ns_command_modifier, Qnone))
+      && nil_or_none (mod_of_kind (ns_command_modifier, QCordinary)))
     modifiers |= cmdKey;
 
   result = UCKeyTranslate (layout, [event keyCode], kUCKeyActionDown,
@@ -6287,7 +6317,8 @@ In that case we use UCKeyTranslate (ns_get_shifted_character)
          modifier keys, which returns 0 for shift-like modifiers.
          Therefore its return value is the set of control-like
          modifiers.  */
-      emacs_event->modifiers = EV_MODIFIERS2 (flags);
+      Lisp_Object kind = fnKeysym ? QCfunction : QCordinary;
+      emacs_event->modifiers = EV_MODIFIERS2 (flags, kind);
 
       /* Function keys (such as the F-keys, arrow keys, etc.) set
          modifiers as though the fn key has been pressed when it
@@ -6296,7 +6327,9 @@ In that case we use UCKeyTranslate (ns_get_shifted_character)
          <home>).  We need to unset the fn modifier in these cases.
          FIXME: Can we avoid setting it in the first place?  */
       if (fnKeysym && (flags & NS_FUNCTION_KEY_MASK))
-        emacs_event->modifiers ^= parse_solitary_modifier (ns_function_modifier);
+        emacs_event->modifiers
+          ^= parse_solitary_modifier (mod_of_kind (ns_function_modifier,
+                                                   QCfunction));
 
       if (NS_KEYLOG)
         fprintf (stderr, "keyDown: code =%x\tfnKey =%x\tflags = %x\tmods = %x\n",
@@ -9399,57 +9432,75 @@ Convert an X font name (XLFD) to an NS font name.
 
   DEFVAR_LISP ("ns-alternate-modifier", ns_alternate_modifier,
                "This variable describes the behavior of the alternate or option key.\n\
-Set to the symbol control, meta, alt, super, or hyper means it is taken to be\n\
-that key.\n\
-Set to none means that the alternate / option key is not interpreted by Emacs\n\
-at all, allowing it to be used at a lower level for accented character entry.");
+Either SYMBOL, describing the behaviour for any event,\n\
+or (:ordinary SYMBOL :function SYMBOL :mouse SYMBOL), describing behaviour\n\
+separately for ordinary keys, function keys, and mouse events.\n\
+\n\
+Each SYMBOL is `control', `meta', `alt', `super', `hyper' or `none'.\n\
+If `none', the key is ignored by Emacs and retains its standard meaning.");
   ns_alternate_modifier = Qmeta;
 
   DEFVAR_LISP ("ns-right-alternate-modifier", ns_right_alternate_modifier,
                "This variable describes the behavior of the right alternate or option key.\n\
-Set to the symbol control, meta, alt, super, or hyper means it is taken to be\n\
-that key.\n\
-Set to left means be the same key as `ns-alternate-modifier'.\n\
-Set to none means that the alternate / option key is not interpreted by Emacs\n\
-at all, allowing it to be used at a lower level for accented character entry.");
+Either SYMBOL, describing the behaviour for any event,\n\
+or (:ordinary SYMBOL :function SYMBOL :mouse SYMBOL), describing behaviour\n\
+separately for ordinary keys, function keys, and mouse events.\n\
+It can also be `left' to use the value of `ns-alternate-modifier' instead.\n\
+\n\
+Each SYMBOL is `control', `meta', `alt', `super', `hyper' or `none'.\n\
+If `none', the key is ignored by Emacs and retains its standard meaning.");
   ns_right_alternate_modifier = Qleft;
 
   DEFVAR_LISP ("ns-command-modifier", ns_command_modifier,
                "This variable describes the behavior of the command key.\n\
-Set to the symbol control, meta, alt, super, or hyper means it is taken to be\n\
-that key.");
+Either SYMBOL, describing the behaviour for any event,\n\
+or (:ordinary SYMBOL :function SYMBOL :mouse SYMBOL), describing behaviour\n\
+separately for ordinary keys, function keys, and mouse events.\n\
+\n\
+Each SYMBOL is `control', `meta', `alt', `super', `hyper' or `none'.\n\
+If `none', the key is ignored by Emacs and retains its standard meaning.");
   ns_command_modifier = Qsuper;
 
   DEFVAR_LISP ("ns-right-command-modifier", ns_right_command_modifier,
                "This variable describes the behavior of the right command key.\n\
-Set to the symbol control, meta, alt, super, or hyper means it is taken to be\n\
-that key.\n\
-Set to left means be the same key as `ns-command-modifier'.\n\
-Set to none means that the command / option key is not interpreted by Emacs\n\
-at all, allowing it to be used at a lower level for accented character entry.");
+Either SYMBOL, describing the behaviour for any event,\n\
+or (:ordinary SYMBOL :function SYMBOL :mouse SYMBOL), describing behaviour\n\
+separately for ordinary keys, function keys, and mouse events.\n\
+It can also be `left' to use the value of `ns-command-modifier' instead.\n\
+\n\
+Each SYMBOL is `control', `meta', `alt', `super', `hyper' or `none'.\n\
+If `none', the key is ignored by Emacs and retains its standard meaning.");
   ns_right_command_modifier = Qleft;
 
   DEFVAR_LISP ("ns-control-modifier", ns_control_modifier,
                "This variable describes the behavior of the control key.\n\
-Set to the symbol control, meta, alt, super, or hyper means it is taken to be\n\
-that key.");
+Either SYMBOL, describing the behaviour for any event,\n\
+or (:ordinary SYMBOL :function SYMBOL :mouse SYMBOL), describing behaviour\n\
+separately for ordinary keys, function keys, and mouse events.\n\
+\n\
+Each SYMBOL is `control', `meta', `alt', `super', `hyper' or `none'.\n\
+If `none', the key is ignored by Emacs and retains its standard meaning.");
   ns_control_modifier = Qcontrol;
 
   DEFVAR_LISP ("ns-right-control-modifier", ns_right_control_modifier,
                "This variable describes the behavior of the right control key.\n\
-Set to the symbol control, meta, alt, super, or hyper means it is taken to be\n\
-that key.\n\
-Set to left means be the same key as `ns-control-modifier'.\n\
-Set to none means that the control / option key is not interpreted by Emacs\n\
-at all, allowing it to be used at a lower level for accented character entry.");
+Either SYMBOL, describing the behaviour for any event,\n\
+or (:ordinary SYMBOL :function SYMBOL :mouse SYMBOL), describing behaviour\n\
+separately for ordinary keys, function keys, and mouse events.\n\
+It can also be `left' to use the value of `ns-control-modifier' instead.\n\
+\n\
+Each SYMBOL is `control', `meta', `alt', `super', `hyper' or `none'.\n\
+If `none', the key is ignored by Emacs and retains its standard meaning.");
   ns_right_control_modifier = Qleft;
 
   DEFVAR_LISP ("ns-function-modifier", ns_function_modifier,
-               "This variable describes the behavior of the function key (on laptops).\n\
-Set to the symbol control, meta, alt, super, or hyper means it is taken to be\n\
-that key.\n\
-Set to none means that the function key is not interpreted by Emacs at all,\n\
-allowing it to be used at a lower level for accented character entry.");
+               "This variable describes the behavior of the function (fn) key.\n\
+Either SYMBOL, describing the behaviour for any event,\n\
+or (:ordinary SYMBOL :function SYMBOL :mouse SYMBOL), describing behaviour\n\
+separately for ordinary keys, function keys, and mouse events.\n\
+\n\
+Each SYMBOL is `control', `meta', `alt', `super', `hyper' or `none'.\n\
+If `none', the key is ignored by Emacs and retains its standard meaning.");
   ns_function_modifier = Qnone;
 
   DEFVAR_LISP ("ns-antialias-text", ns_antialias_text,
@@ -9529,6 +9580,9 @@ Nil means use fullscreen the old (< 10.7) way.  The old way works better with
 
   DEFSYM (Qcocoa, "cocoa");
   DEFSYM (Qgnustep, "gnustep");
+  DEFSYM (QCordinary, ":ordinary");
+  DEFSYM (QCfunction, ":function");
+  DEFSYM (QCmouse, ":mouse");
 
 #ifdef NS_IMPL_COCOA
   Fprovide (Qcocoa, Qnil);
-- 
2.21.0 (Apple Git-122.2)

