Bugfix for utf-8 XTerm/MinTTY and (set-input-meta-mode t)

Bugfix for utf-8 XTerm/MinTTY and (set-input-meta-mode t)

[Max Mikhanosha]

Emacs incorrectly handles (set-input-meta-mode t) (the meta in the 8th bit of input) when terminal is in UTF-8 mode.

Both XTerm and MinTTY, when configured to send meta modifier as 8th bit while in utf-8 mode, will first add 8th bit, and then encode resulting character with utf-8. For example Meta-X is encoded as ?x+120 = #248 codepoint, encoded as 0xc3,0xb8

But Emacs handles meta modifier in the 8th bit in tty_read_avail_input, before decoding the raw keyboard input.

So it erroneously treats 0xc3,0xb8 input as two ordinary ASCII characters with meta modifier set, stripping the 8th bit and garbling the input.

This problem had existed for a long time, and had frustrated at least a few hundred people, as can be seen by the view count on stackoverflow article that comes up when googling "emacs utf8 xterm"

Below patch fixes this bug, by making 8th bit meta key handling to work correctly in utf8 mode.

I have tested it with xterm and mintty and meta keys, and meta-control keys now work correctly regardless if terminals are in utf-8 mode.

Diff against Emacs-26 branch pasted below

diff --git a/src/coding.c b/src/coding.c
index 078c1c4e6a..743fceb32c 100644
--- a/src/coding.c
+++ b/src/coding.c
@@ -5989,6 +5989,11 @@ raw_text_coding_system_p (struct coding_system *coding)
 	  && coding->encoder == encode_coding_raw_text) ? true : false;
 }

+bool utf_8_input_coding_system_p(struct coding_system *coding)
+{
+  return (coding->decoder == decode_coding_utf_8) ? true : false;
+}
+

 /* If CODING_SYSTEM doesn't specify end-of-line format, return one of
    the subsidiary that has the same eol-spec as PARENT (if it is not
diff --git a/src/coding.h b/src/coding.h
index aab8c2d438..6124330a1f 100644
--- a/src/coding.h
+++ b/src/coding.h
@@ -702,6 +702,7 @@ extern Lisp_Object encode_file_name (Lisp_Object);
 extern Lisp_Object decode_file_name (Lisp_Object);
 extern Lisp_Object raw_text_coding_system (Lisp_Object);
 extern bool raw_text_coding_system_p (struct coding_system *);
+extern bool utf_8_input_coding_system_p (struct coding_system *);
 extern Lisp_Object coding_inherit_eol_type (Lisp_Object, Lisp_Object);
 extern Lisp_Object complement_process_encoding_system (Lisp_Object);

diff --git a/src/keyboard.c b/src/keyboard.c
index aa3448439b..84acf4a998 100644
--- a/src/keyboard.c
+++ b/src/keyboard.c
@@ -2235,14 +2235,16 @@ read_decoded_event_from_main_queue (struct timespec *end_time,
 	return nextevt;		/* No decoding needed.  */
       else
 	{
+	  struct coding_system *coding = TERMINAL_KEYBOARD_CODING (terminal);
+	  bool utf8_input_terminal = utf_8_input_coding_system_p (coding);
 	  int meta_key = terminal->display_info.tty->meta_key;
+
 	  eassert (n < MAX_ENCODED_BYTES);
 	  events[n++] = nextevt;
+
 	  if (NATNUMP (nextevt)
-	      && XINT (nextevt) < (meta_key == 1 ? 0x80 : 0x100))
+	      && XINT (nextevt) < ((meta_key == 1 && !utf8_input_terminal) ? 0x80 : 0x100))
 	    { /* An encoded byte sequence, let's try to decode it.  */
-	      struct coding_system *coding
-		= TERMINAL_KEYBOARD_CODING (terminal);

 	      if (raw_text_coding_system_p (coding))
 		{
@@ -2253,12 +2255,13 @@ read_decoded_event_from_main_queue (struct timespec *end_time,
 		}
 	      else
 		{
+
 		  unsigned char src[MAX_ENCODED_BYTES];
 		  unsigned char dest[MAX_ENCODED_BYTES * MAX_MULTIBYTE_LENGTH];
 		  int i;
 		  for (i = 0; i < n; i++)
 		    src[i] = XINT (events[i]);
-		  if (meta_key != 2)
+		  if (!utf8_input_terminal && meta_key != 2)
 		    for (i = 0; i < n; i++)
 		      src[i] &= ~0x80;
 		  coding->destination = dest;
@@ -2275,8 +2278,21 @@ read_decoded_event_from_main_queue (struct timespec *end_time,
 		      const unsigned char *p = coding->destination;
 		      eassert (coding->carryover_bytes == 0);
 		      n = 0;
-		      while (n < coding->produced_char)
-			events[n++] = make_number (STRING_CHAR_ADVANCE (p));
+                      while (n < coding->produced_char)
+                        {
+                          int c = STRING_CHAR_ADVANCE (p);
+			  if (utf8_input_terminal)
+			    {
+			      /* put meta modifier on the key */
+			      int modifier = 0;
+			      if (meta_key == 1 && c < 0x100 && (c & 0x80))
+				modifier = meta_modifier;
+			      if (meta_key != 2)
+				c &= ~0x80;
+			      c |= modifier;
+			    }
+			  events[n++] = make_number (c);
+                        }
 		    }
 		}
 	    }
@@ -7118,16 +7134,31 @@ tty_read_avail_input (struct terminal *terminal,
 #endif /* not MSDOS */
 #endif /* not WINDOWSNT */

+  bool utf8_input_terminal = utf_8_input_coding_system_p (TERMINAL_KEYBOARD_CODING(terminal));
+
   for (i = 0; i < nread; i++)
     {
       struct input_event buf;
       EVENT_INIT (buf);
       buf.kind = ASCII_KEYSTROKE_EVENT;
       buf.modifiers = 0;
-      if (tty->meta_key == 1 && (cbuf[i] & 0x80))
-        buf.modifiers = meta_modifier;
-      if (tty->meta_key != 2)
-        cbuf[i] &= ~0x80;
+
+      /* Both XTerm and MinTTY in utf8:true + MetaSendEscape:false mode
+         send Meta + ASCII letters by first adding 0x80, and then UTF-8
+         encoding the result.
+
+         Therefore trying to detect 0x80 meta key flag now not only
+         confuses meta key with UTF-8 encoding, but also loses
+         information by stripping the 8th bit from UTF-8 input before
+         decoding
+      */
+      if (!utf8_input_terminal)
+	{
+	  if (tty->meta_key == 1 && (cbuf[i] & 0x80))
+	    buf.modifiers = meta_modifier;
+	  if (tty->meta_key != 2)
+	    cbuf[i] &= ~0x80;
+	}

       buf.code = cbuf[i];
       /* Set the frame corresponding to the active tty.  Note that the

Re: Bugfix for utf-8 XTerm/MinTTY and (set-input-meta-mode t)

[Eli Zaretskii]

> Date: Tue, 01 Jun 2021 16:19:40 +0000
> From: Max Mikhanosha <max.mikhanosha@protonmail.com>
> 
> Both XTerm and MinTTY, when configured to send meta modifier as 8th bit while in utf-8 mode, will first add 8th bit, and then encode resulting character with utf-8. For example Meta-X is encoded as ?x+120 = #248 codepoint, encoded as 0xc3,0xb8

If these terminal emulators send M-x as 248 decimal, then how do they
send the Latin character ø, whose codepoint is 248 decimal?

Re: Bugfix for utf-8 XTerm/MinTTY and (set-input-meta-mode t)

[Andreas Schwab]

On Jun 01 2021, Max Mikhanosha wrote:

> +bool utf_8_input_coding_system_p(struct coding_system *coding)
> +{
> +  return (coding->decoder == decode_coding_utf_8) ? true : false;

The == operator already produces a bool value.

Andreas.

-- 
Andreas Schwab, schwab@linux-m68k.org
GPG Key fingerprint = 7578 EB47 D4E5 4D69 2510  2552 DF73 E780 A9DA AEC1
"And now for something completely different."

Re: Bugfix for utf-8 XTerm/MinTTY and (set-input-meta-mode t)

[Max Mikhanosha]

On Tuesday, June 1st, 2021 at 12:51 PM, Eli Zaretskii <eliz@gnu.org> wrote:

> > Date: Tue, 01 Jun 2021 16:19:40 +0000
>
> > From: Max Mikhanosha max.mikhanosha@protonmail.com
> >
> > Both XTerm and MinTTY, when configured to send meta modifier as 8th bit while in utf-8 mode, will first add 8th bit, and then encode resulting character with utf-8. For example Meta-X is encoded as ?x+120 = #248 codepoint, encoded as 0xc3,0xb8
>
> If these terminal emulators send M-x as 248 decimal, then how do they
> send the Latin character ø, whose codepoint is 248 decimal?

Exactly the same 0xc3,0xb8. Its point of the encode meta-key in the 8th bit mechanics, that you give up ability to type upper half of ASCII table (128-255) instead using them to indicate meta key.  Its a life-safer for those of us using vi, (on in Emacs case evil), otherwise sending meta keys as Esc <key> really interferes with vi mode where Esc key is super-special..  You can still insert these characters by doing quoted-insert, ie C-v M-x and it will insert ø

To illustrate my point, this patch restores the parity/functionality between non-utf-8 enabled and utf-8 enabled terminal

Before the patch, emacs is configured

non-utf8-xterm & emacs with (set-input-meta-mode nil), meta-x produces ø
non-utf8-xterm & emacs with (set-input-meta-mode t), meta-x produces M-x
utf8-xterm & emacs with (set-input-meta-mode nil), meta-x produces ø
utf8-xterm & emacs with (set-input-meta-mode t), meta-x produces weird garbage, C-v M-x produces weird garbage

After the patch

non-utf8-xterm & emacs with (set-input-meta-mode nil), meta-x produces ø
non-utf8-xterm & emacs with (set-input-meta-mode t), meta-x produces M-x, C-v M-x produces ø
utf8-xterm & emacs with (set-input-meta-mode nil), meta-x produces ø
utf8-xterm & emacs with (set-input-meta-mode t), meta-x produces M-x, C-v M-x produces ø

I also typed some stuff in Russian and nothing seem to be broke in UTF8 in general.    If there is a concern that this patch can break something, maybe one way forward is to rework it to add another meaning to meta mode flag, like (set-input-meta-mode 'utf8), which would trigger what my patch is doing.  But current behavior, where meta_key is checked before utf-8 decoding is obviously a bug, as it produces garbage input by treating utf-8 encoding itself as having meta modifiers.

Re: Bugfix for utf-8 XTerm/MinTTY and (set-input-meta-mode t)

[Eli Zaretskii]

> Date: Tue, 01 Jun 2021 19:51:28 +0300
> From: Eli Zaretskii <eliz@gnu.org>
> Cc: emacs-devel@gnu.org
> 
> > Date: Tue, 01 Jun 2021 16:19:40 +0000
> > From: Max Mikhanosha <max.mikhanosha@protonmail.com>
> > 
> > Both XTerm and MinTTY, when configured to send meta modifier as 8th bit while in utf-8 mode, will first add 8th bit, and then encode resulting character with utf-8. For example Meta-X is encoded as ?x+120 = #248 codepoint, encoded as 0xc3,0xb8
> 
> If these terminal emulators send M-x as 248 decimal, then how do they
> send the Latin character ø, whose codepoint is 248 decimal?

And btw, why do you use MetaSendEscape:false with these emulators?  It
sounds like if you set MetaSendEscape:true, your problems will be
solved, no?  FWIW, this is how I use terminal emulators here, for many
years, including with non-ASCII input (and of course Meta-characters),
and I have yet to see a single problem.

Re: Bugfix for utf-8 XTerm/MinTTY and (set-input-meta-mode t)

[Max Mikhanosha]

On Tuesday, June 1st, 2021 at 1:04 PM, Andreas Schwab <schwab@linux-m68k.org> wrote:

> On Jun 01 2021, Max Mikhanosha wrote:
>
> > +bool utf_8_input_coding_system_p(struct coding_system *coding)
> >
> > +{
> >
> > -   return (coding->decoder == decode_coding_utf_8) ? true : false;
>
> The == operator already produces a bool value.

Its a cut-n-paste of  existing function below, where you can make same argument, I figured it was written in that more verbose way for clarity, so did not change it to follow the style.

bool
raw_text_coding_system_p (struct coding_system *coding)
{
  return (coding->decoder == decode_coding_raw_text
          && coding->encoder == encode_coding_raw_text) ? true : false;
}

Re: Bugfix for utf-8 XTerm/MinTTY and (set-input-meta-mode t)

[Eli Zaretskii]

> Date: Tue, 01 Jun 2021 17:28:59 +0000
> From: Max Mikhanosha <max.mikhanosha@protonmail.com>
> Cc: emacs-devel@gnu.org
> 
> > If these terminal emulators send M-x as 248 decimal, then how do they
> > send the Latin character ø, whose codepoint is 248 decimal?
> 
> Exactly the same 0xc3,0xb8. Its point of the encode meta-key in the 8th bit mechanics, that you give up ability to type upper half of ASCII table (128-255) instead using them to indicate meta key.  Its a life-safer for those of us using vi, (on in Emacs case evil), otherwise sending meta keys as Esc <key> really interferes with vi mode where Esc key is super-special..  You can still insert these characters by doing quoted-insert, ie C-v M-x and it will insert ø

I see your point, but I don't think we can then accept this patch as
is.  The patch in effect _forces_ everyone to give up on being able to
type 8-bit Latin characters as soon as they configure keyboard
encoding to be UTF-8, right?  Or am I missing something?

> I also typed some stuff in Russian and nothing seem to be broke in UTF8 in general.    If there is a concern that this patch can break something, maybe one way forward is to rework it to add another meaning to meta mode flag, like (set-input-meta-mode 'utf8), which would trigger what my patch is doing.  But current behavior, where meta_key is checked before utf-8 decoding is obviously a bug, as it produces garbage input by treating utf-8 encoding itself as having meta modifiers.

If something like this is accepted, it must be an optional feature.
And I'm not yet sure we want it even then, it sounds like a kludge to
work around broken terminal emulator configuration and cater to other
editors.

Re: Bugfix for utf-8 XTerm/MinTTY and (set-input-meta-mode t)

[Max Mikhanosha]

On Tuesday, June 1st, 2021 at 1:29 PM, Eli Zaretskii <eliz@gnu.org> wrote:

> And btw, why do you use MetaSendEscape:false with these emulators? It
> sounds like if you set MetaSendEscape:true, your problems will be
> solved, no? FWIW, this is how I use terminal emulators here, for many
> years, including with non-ASCII input (and of course Meta-characters),
> and I have yet to see a single problem.

Its because of vi. Not to start an argument, but evil package for emacs is best of both worlds,
its vim with Elisp scripting, and my brain is already hardwired for vi keys for decades).

When typing fast in vi mode, its really frustrating to be typing, then press Esc o to
exit insert mode and start new line, only for it to be interpreted as M-o.  Its even worse
when typing over a slow-laggy ssh connections.

Thus most vi-junkies have their terminals configured with metaSendEscape:false

Re: Bugfix for utf-8 XTerm/MinTTY and (set-input-meta-mode t)

[Eli Zaretskii]

> Date: Tue, 01 Jun 2021 17:45:22 +0000
> From: Max Mikhanosha <max.mikhanosha@protonmail.com>
> Cc: emacs-devel@gnu.org
> 
> On Tuesday, June 1st, 2021 at 1:29 PM, Eli Zaretskii <eliz@gnu.org> wrote:
> 
> > And btw, why do you use MetaSendEscape:false with these emulators? It
> > sounds like if you set MetaSendEscape:true, your problems will be
> > solved, no? FWIW, this is how I use terminal emulators here, for many
> > years, including with non-ASCII input (and of course Meta-characters),
> > and I have yet to see a single problem.
> 
> Its because of vi. Not to start an argument, but evil package for emacs is best of both worlds,
> its vim with Elisp scripting, and my brain is already hardwired for vi keys for decades).

OK, but then it cannot really be anything but an opt-in behavior.  And
it cannot be hardcoded to depend on UTF-8 being the keyboard encoding,
because if a user sets the keyboard encoding to be UTF-8 (something
very popular these days), it doesn't mean the user gives up the
ability to type Latin characters in the 128 -- 255 range.

Also, you show the results with input-meta-mode nil or t, but the
default value is neither nil nor t.  What is the effect of the patch
when input-meta-mode is at its default value?

Re: Bugfix for utf-8 XTerm/MinTTY and (set-input-meta-mode t)

[Max Mikhanosha]

On Tuesday, June 1st, 2021 at 1:38 PM, Eli Zaretskii <eliz@gnu.org> wrote:

> > Date: Tue, 01 Jun 2021 17:28:59 +0000
>
> > From: Max Mikhanosha max.mikhanosha@protonmail.com
> >
> > Cc: emacs-devel@gnu.org
> >
> > > If these terminal emulators send M-x as 248 decimal, then how do they
> > >
> > > send the Latin character ø, whose codepoint is 248 decimal?
> >
> > Exactly the same 0xc3,0xb8. Its point of the encode meta-key in the 8th bit mechanics, that you give up ability to type upper half of ASCII table (128-255) instead using them to indicate meta key. Its a life-safer for those of us using vi, (on in Emacs case evil), otherwise sending meta keys as Esc <key> really interferes with vi mode where Esc key is super-special.. You can still insert these characters by doing quoted-insert, ie C-v M-x and it will insert ø
>
> I see your point, but I don't think we can then accept this patch as
> is. The patch in effect forces everyone to give up on being able to
> type 8-bit Latin characters as soon as they configure keyboard
> encoding to be UTF-8, right? Or am I missing something?

No there are two separate toggles in terminal programs (metaSendsEscape:false), and in Emacs itself  (set-input-meta-mode t) and (set-input-mode). First one changes how terminals send it, and 2nd one makes Emacs interpret 8th bit as meta key.  Both of these are non-default, for either Xterm or in Emacse

The (set-input-meta-mode META) parameter in emacs can be either NIL, T or something else, you can lookup the docstring yourself.  Default is 0, which sets keyboard->meta_key = 2, and thus does not affect anything.

I just started Emacs with ./emacs -nw --no-init-file in UTF-8 XTerm, did (current-input-mode) which shows (nil nil 0 7), and typed Alt+aoeuid which produces áïåõé as expected, and exactly same as without my patch.

My patch functionality is specifically enabled only when someone customizes their Emacs to set META parameter of (set-input-mode) to T or NIL, but its behavior is exactly the same as in non-UTF-8 mode.  Only thing it does, is make UTF-8 terminals work exactly same as non-UTF-8 terminals, instead of beaing broken

Re: Bugfix for utf-8 XTerm/MinTTY and (set-input-meta-mode t)

[Max Mikhanosha]

On Tuesday, June 1st, 2021 at 1:52 PM, Eli Zaretskii <eliz@gnu.org> wrote:

> OK, but then it cannot really be anything but an opt-in behavior. And
> it cannot be hardcoded to depend on UTF-8 being the keyboard encoding,
> because if a user sets the keyboard encoding to be UTF-8 (something
> very popular these days), it doesn't mean the user gives up the
> ability to type Latin characters in the 128 -- 255 range.

I agree, and I don't really insist on my patch making it in, I understand that you're
reluctant to approve something thats in a hairy area of the code thats very few people
are touching. Reason I posted it, so that if other people having same problem as me
can find the patch easily, if someone is configured their Emacs with 8-bit-meta they
likely to have no problems applying the patch and compiling their own.

> Also, you show the results with input-meta-mode nil or t, but the
> default value is neither nil nor t. What is the effect of the patch
> when input-meta-mode is at its default value?

With default neither nil nor t (ie anything else), it sets kepyboard->meta_key to 2, which means 8-bit clean,
and than neither before my patch nor after my patch 8th bit is never looked at or stripped.

META parameter to keyboard->meta_key mapping in C is:
  NIL   => 0
  T     => 1
  other => 2

This can be trivially seen by grepping kepyboard.c for meta_key, all the if statements referencing meta_key
are either if (meta_key==1), and than if (meta_key != 2) for stripping the 8th bit.

Thus with default value of meta_key==2, there is no change to input decoding either before or after my patch

Re: Bugfix for utf-8 XTerm/MinTTY and (set-input-meta-mode t)

[Eli Zaretskii]

> Date: Tue, 01 Jun 2021 18:01:27 +0000
> From: Max Mikhanosha <max.mikhanosha@protonmail.com>
> Cc: emacs-devel@gnu.org
> 
> My patch functionality is specifically enabled only when someone customizes their Emacs to set META parameter of (set-input-mode) to T or NIL, but its behavior is exactly the same as in non-UTF-8 mode.  Only thing it does, is make UTF-8 terminals work exactly same as non-UTF-8 terminals, instead of beaing broken

But UTF-8 is not the only encoding that uses the 8th bit.  So this
must be a separate setting, unrelated to the encoding being UTF-8 or
not.  What you want is for Emacs to interpret the 8th bit after
decoding characters and not before it as it does now.  So I think a
new non-nil value of input-meta-mode should be introduced to trigger
this behavior, and you should drop the tests of the encoding being
UTF-8, as it's not the only encoding that can be affected.

Re: Bugfix for utf-8 XTerm/MinTTY and (set-input-meta-mode t)

[Max Mikhanosha]

Ok I'll rework the patch along these lines, and submit again later. Do you have a suggestion or input-meta-mode new enum value? My only idea is 'ENCODED, to mean same as T but that 8th bit will be checked after decoding the input with whatever keyboard coding system is.

‐‐‐‐‐‐‐ Original Message ‐‐‐‐‐‐‐

On Tuesday, June 1st, 2021 at 2:18 PM, Eli Zaretskii <eliz@gnu.org> wrote:

> > Date: Tue, 01 Jun 2021 18:01:27 +0000
>
> > From: Max Mikhanosha max.mikhanosha@protonmail.com
> >
> > Cc: emacs-devel@gnu.org
> >
> > My patch functionality is specifically enabled only when someone customizes their Emacs to set META parameter of (set-input-mode) to T or NIL, but its behavior is exactly the same as in non-UTF-8 mode. Only thing it does, is make UTF-8 terminals work exactly same as non-UTF-8 terminals, instead of beaing broken
>
> But UTF-8 is not the only encoding that uses the 8th bit. So this
>
> must be a separate setting, unrelated to the encoding being UTF-8 or
>
> not. What you want is for Emacs to interpret the 8th bit after
>
> decoding characters and not before it as it does now. So I think a
>
> new non-nil value of input-meta-mode should be introduced to trigger
>
> this behavior, and you should drop the tests of the encoding being
>
> UTF-8, as it's not the only encoding that can be affected.

Re: Bugfix for utf-8 XTerm/MinTTY and (set-input-meta-mode t)

[Eli Zaretskii]

> Date: Tue, 01 Jun 2021 18:35:43 +0000
> From: Max Mikhanosha <max.mikhanosha@protonmail.com>
> Cc: emacs-devel@gnu.org
> 
> Ok I'll rework the patch along these lines, and submit again later. Do you have a suggestion or input-meta-mode new enum value? My only idea is 'ENCODED, to mean same as T but that 8th bit will be checked after decoding the input with whatever keyboard coding system is.

'encoded' is okay (just in lower-case).

Re: Bugfix for utf-8 XTerm/MinTTY and (set-input-meta-mode t)

[Stefan Monnier]

> Both XTerm and MinTTY, when configured to send meta modifier as 8th
> bit while in utf-8 mode, will first add 8th bit, and then encode
> resulting character with utf-8. For example Meta-X is encoded
> as ?x+128 = #248 codepoint, encoded as 0xc3,0xb8

How did they end up with that weird design?

I mean they could have made meta toggle the 24th bit, for example, so it
doesn't collide with other existing characters.

This design is quite weird since it breaks all the latin-1 chars of
unicode plus all the uses of meta with non-ASCII chars.

How do they encode M-λ ?
Is it also sent as the same byte-sequence as `?λ + 128 = ?л`  ?

Admittedly, it is better than the "original" meta-as-8th-bit which was
limited to ASCII, but it still seems unnecessarily limited and kludgey.


        Stefan

Re: Bugfix for utf-8 XTerm/MinTTY and (set-input-meta-mode t)

[Max Mikhanosha]

On Tuesday, June 1st, 2021 at 2:46 PM, Eli Zaretskii <eliz@gnu.org> wrote:

> > Date: Tue, 01 Jun 2021 18:35:43 +0000
>
> > From: Max Mikhanosha max.mikhanosha@protonmail.com
> >
> > Cc: emacs-devel@gnu.org
> >
> > Ok I'll rework the patch along these lines, and submit again later. Do you have a suggestion or input-meta-mode new enum value? My only idea is 'ENCODED, to mean same as T but that 8th bit will be checked after decoding the input with whatever keyboard coding system is.
>
> 'encoded' is okay (just in lower-case).

Updated patch pasted below. I've run it through the motions testing all combinations of utf8 xterm, non-utf-8 xterm, with input-meta-mode 'encoded' vs t and so on.   In non-utf-8 xterm 'encoded' behaves exactly as t would.

diff --git a/src/keyboard.c b/src/keyboard.c
index aa3448439b..017e2176ed 100644
--- a/src/keyboard.c
+++ b/src/keyboard.c
@@ -2249,7 +2249,11 @@ read_decoded_event_from_main_queue (struct timespec *end_time,
 		  int i;
 		  if (meta_key != 2)
 		    for (i = 0; i < n; i++)
-		      events[i] = make_number (XINT (events[i]) & ~0x80);
+		      {
+			int c = XINT (events[i]);
+			int modifier = (meta_key == 3 && c < 0x100 && (c & 0x80)) ? meta_modifier : 0;
+			events[i] = make_number ((c & ~0x80) | modifier);
+		      }
 		}
 	      else
 		{
@@ -2258,7 +2262,7 @@ read_decoded_event_from_main_queue (struct timespec *end_time,
 		  int i;
 		  for (i = 0; i < n; i++)
 		    src[i] = XINT (events[i]);
-		  if (meta_key != 2)
+		  if (meta_key < 2) /* input-meta-mode T or NIL */
 		    for (i = 0; i < n; i++)
 		      src[i] &= ~0x80;
 		  coding->destination = dest;
@@ -2276,7 +2280,15 @@ read_decoded_event_from_main_queue (struct timespec *end_time,
 		      eassert (coding->carryover_bytes == 0);
 		      n = 0;
 		      while (n < coding->produced_char)
-			events[n++] = make_number (STRING_CHAR_ADVANCE (p));
+			{
+			  int c = STRING_CHAR_ADVANCE (p);
+			  if (meta_key == 3)
+			    {
+			      int modifier = (c < 0x100 && (c & 0x80)) ? meta_modifier : 0;
+			      c = (c & ~0x80) | modifier;
+			    }
+			  events[n++] = make_number (c);
+			}
 		    }
 		}
 	    }
@@ -7126,7 +7138,7 @@ tty_read_avail_input (struct terminal *terminal,
       buf.modifiers = 0;
       if (tty->meta_key == 1 && (cbuf[i] & 0x80))
         buf.modifiers = meta_modifier;
-      if (tty->meta_key != 2)
+      if (tty->meta_key < 2)
         cbuf[i] &= ~0x80;

       buf.code = cbuf[i];
@@ -10644,6 +10656,10 @@ DEFUN ("set-input-meta-mode", Fset_input_meta_mode, Sset_input_meta_mode, 1, 2,
 If META is t, Emacs will accept 8-bit input, and interpret the 8th
 bit as the Meta modifier.

+If META is 'encoded', Emacs will decode the input with accordance
+to current coding system, and then examine decoded input 8th bit
+and interpret it as the Meta modifier
+
 If META is nil, Emacs will ignore the top bit, on the assumption it is
 parity.

@@ -10671,8 +10687,10 @@ See also `current-input-mode'.  */)
     new_meta = 0;
   else if (EQ (meta, Qt))
     new_meta = 1;
-  else
+  else if (!EQ (meta, Qencoded))
     new_meta = 2;
+  else
+    new_meta = 3;

   if (tty->meta_key != new_meta)
     {
@@ -10754,6 +10772,7 @@ The value is a list of the form (INTERRUPT FLOW META QUIT), where
   FLOW is non-nil if Emacs uses ^S/^Q flow control for output to the
     terminal; this does not apply if Emacs uses interrupt-driven input.
   META is t if accepting 8-bit input with 8th bit as Meta flag.
+    META 'encoded' means the same but 8th bit is checked after coding system
     META nil means ignoring the top bit, on the assumption it is parity.
     META is neither t nor nil if accepting 8-bit input and using
     all 8 bits as the character code.
@@ -10771,7 +10790,8 @@ The elements of this list correspond to the arguments of
       flow = FRAME_TTY (sf)->flow_control ? Qt : Qnil;
       meta = (FRAME_TTY (sf)->meta_key == 2
 	      ? make_number (0)
-	      : (CURTTY ()->meta_key == 1 ? Qt : Qnil));
+	      : (CURTTY ()->meta_key == 1 ? Qt :
+		 (CURTTY ()->meta_key == 3 ? Qencoded : Qnil)));
     }
   else
     {
@@ -11210,6 +11230,9 @@ syms_of_keyboard (void)

   DEFSYM (Qecho_keystrokes, "echo-keystrokes");

+  /* input-meta-mode constant */
+  DEFSYM (Qencoded, "encoded");
+
   Fset (Qinput_method_exit_on_first_char, Qnil);
   Fset (Qinput_method_use_echo_area, Qnil);

Re: Bugfix for utf-8 XTerm/MinTTY and (set-input-meta-mode t)

[Max Mikhanosha]

On Tuesday, June 1st, 2021 at 4:06 PM, Stefan Monnier <monnier@iro.umontreal.ca> wrote:

> > Both XTerm and MinTTY, when configured to send meta modifier as 8th
> > bit while in utf-8 mode, will first add 8th bit, and then encode
> > resulting character with utf-8. For example Meta-X is encoded
> > as ?x+128 = #248 codepoint, encoded as 0xc3,0xb8
>
> How did they end up with that weird design?

It seems to be logical extension to preserve backward compatibility as much
as possible. I mean since without UTF-8, in meta-as-8th-bit mode, Meta-x generates #248,
just having terminal in UTF-8 mode should not change that. And in UTF8, starting
sequences that have 8th bit set indicate start of the encoding, so sending 128-256
as is would be confused for UTF8 sequence by the receiver. Obvious solution would be
to just UTF-8 encode the output that non-utf-8 terminal would be sending.

> I mean they could have made meta toggle the 24th bit, for example, so it
> doesn't collide with other existing characters.

There is XTerm solution for this, called modifyOtherKey resource with new enum, which can be set
so that any modifiers even on ordinary keys like M-x would generate properly structured
ESC[ sequences describing the modifiers. I agree that in perfect world we would have come out
with some binary bitmask solution, rather than current thing where terminal can send you 20 byte
sequence for Ctrl-Alt-Shift-PageDown, but it is what it is.

> This design is quite weird since it breaks all the latin-1 chars of
> unicode plus all the uses of meta with non-ASCII chars.
>
> How do they encode M-λ ?
> Is it also sent as the same byte-sequence as `?λ + 128 = ?л` ?

Unfortunately yes, at least mintty in meta-as-8th bit mode (which is my terminal on cygwin) just
dumbly shoves 8th bit into even wide characters (like when I press Alt+letter in a cyrillic layout), but
my patch does not change that.  TBH Xterm maybe smarter, and just generates English M-x when you press
M-x when on a different keyboard layout, or you can probably make it behave like this with some xkb
config magic. Proper way to support multi-modifier key sequences is by  using modifyOtherKeys:2 but it would
need to have wider adoption than just xterm, hopefully it will percolate to other terminals just as
xterm-direct truecolor mode.

For now i'm pretty happy with meta as 8th bit mode, as it allows me to use stuff like M-C-v that is sent
as a single char (or 2 in utf8 mode), and with a bit of magic M-S-v and such work too, so all my keybindings
are exactly the same regardless if I'm in a terminal or GUI frame.

Re: Bugfix for utf-8 XTerm/MinTTY and (set-input-meta-mode t)

[Andreas Schwab]

On Jun 02 2021, Max Mikhanosha wrote:

> @@ -10644,6 +10656,10 @@ DEFUN ("set-input-meta-mode", Fset_input_meta_mode, Sset_input_meta_mode, 1, 2,
>  If META is t, Emacs will accept 8-bit input, and interpret the 8th
>  bit as the Meta modifier.
>
> +If META is 'encoded', Emacs will decode the input with accordance
> +to current coding system, and then examine decoded input 8th bit
> +and interpret it as the Meta modifier

in accordance to

Andreas.

-- 
Andreas Schwab, schwab@linux-m68k.org
GPG Key fingerprint = 7578 EB47 D4E5 4D69 2510  2552 DF73 E780 A9DA AEC1
"And now for something completely different."

Re: Bugfix for utf-8 XTerm/MinTTY and (set-input-meta-mode t)

[Max Mikhanosha]

On Wednesday, June 2nd, 2021 at 8:16 AM, Andreas Schwab <schwab@linux-m68k.org> wrote:

> > If META is t, Emacs will accept 8-bit input, and interpret the 8th
> > bit as the Meta modifier.
> >
> > +If META is 'encoded', Emacs will decode the input with accordance
> > +to current coding system, and then examine decoded input 8th bit
> > +and interpret it as the Meta modifier
>
> in accordance to

Updated per above. Feel free to do any farther changes to wording or coding style if
you going to merge it, or let me know if you'd like me to improve this in any way.

diff --git a/src/keyboard.c b/src/keyboard.c
index aa3448439b..01f154ec04 100644
--- a/src/keyboard.c
+++ b/src/keyboard.c
@@ -2249,7 +2249,11 @@ read_decoded_event_from_main_queue (struct timespec *end_time,
 		  int i;
 		  if (meta_key != 2)
 		    for (i = 0; i < n; i++)
-		      events[i] = make_number (XINT (events[i]) & ~0x80);
+		      {
+			int c = XINT (events[i]);
+			int modifier = (meta_key == 3 && c < 0x100 && (c & 0x80)) ? meta_modifier : 0;
+			events[i] = make_number ((c & ~0x80) | modifier);
+		      }
 		}
 	      else
 		{
@@ -2258,7 +2262,7 @@ read_decoded_event_from_main_queue (struct timespec *end_time,
 		  int i;
 		  for (i = 0; i < n; i++)
 		    src[i] = XINT (events[i]);
-		  if (meta_key != 2)
+		  if (meta_key < 2) /* input-meta-mode T or NIL */
 		    for (i = 0; i < n; i++)
 		      src[i] &= ~0x80;
 		  coding->destination = dest;
@@ -2276,7 +2280,15 @@ read_decoded_event_from_main_queue (struct timespec *end_time,
 		      eassert (coding->carryover_bytes == 0);
 		      n = 0;
 		      while (n < coding->produced_char)
-			events[n++] = make_number (STRING_CHAR_ADVANCE (p));
+			{
+			  int c = STRING_CHAR_ADVANCE (p);
+			  if (meta_key == 3)
+			    {
+			      int modifier = (c < 0x100 && (c & 0x80)) ? meta_modifier : 0;
+			      c = (c & ~0x80) | modifier;
+			    }
+			  events[n++] = make_number (c);
+			}
 		    }
 		}
 	    }
@@ -7126,7 +7138,7 @@ tty_read_avail_input (struct terminal *terminal,
       buf.modifiers = 0;
       if (tty->meta_key == 1 && (cbuf[i] & 0x80))
         buf.modifiers = meta_modifier;
-      if (tty->meta_key != 2)
+      if (tty->meta_key < 2)
         cbuf[i] &= ~0x80;

       buf.code = cbuf[i];
@@ -10644,6 +10656,10 @@ DEFUN ("set-input-meta-mode", Fset_input_meta_mode, Sset_input_meta_mode, 1, 2,
 If META is t, Emacs will accept 8-bit input, and interpret the 8th
 bit as the Meta modifier.

+If META is 'encoded', Emacs will decode the input in accordance
+to current coding system, and then examine decoded input 8th bit
+and interpret it as the Meta modifier
+
 If META is nil, Emacs will ignore the top bit, on the assumption it is
 parity.

@@ -10671,8 +10687,10 @@ See also `current-input-mode'.  */)
     new_meta = 0;
   else if (EQ (meta, Qt))
     new_meta = 1;
-  else
+  else if (!EQ (meta, Qencoded))
     new_meta = 2;
+  else
+    new_meta = 3;

   if (tty->meta_key != new_meta)
     {
@@ -10754,6 +10772,7 @@ The value is a list of the form (INTERRUPT FLOW META QUIT), where
   FLOW is non-nil if Emacs uses ^S/^Q flow control for output to the
     terminal; this does not apply if Emacs uses interrupt-driven input.
   META is t if accepting 8-bit input with 8th bit as Meta flag.
+    META 'encoded' means the same but 8th bit is checked after coding system
     META nil means ignoring the top bit, on the assumption it is parity.
     META is neither t nor nil if accepting 8-bit input and using
     all 8 bits as the character code.
@@ -10771,7 +10790,8 @@ The elements of this list correspond to the arguments of
       flow = FRAME_TTY (sf)->flow_control ? Qt : Qnil;
       meta = (FRAME_TTY (sf)->meta_key == 2
 	      ? make_number (0)
-	      : (CURTTY ()->meta_key == 1 ? Qt : Qnil));
+	      : (CURTTY ()->meta_key == 1 ? Qt :
+		 (CURTTY ()->meta_key == 3 ? Qencoded : Qnil)));
     }
   else
     {
@@ -11210,6 +11230,9 @@ syms_of_keyboard (void)

   DEFSYM (Qecho_keystrokes, "echo-keystrokes");

+  /* input-meta-mode constant */
+  DEFSYM (Qencoded, "encoded");
+
   Fset (Qinput_method_exit_on_first_char, Qnil);
   Fset (Qinput_method_use_echo_area, Qnil);

Re: Bugfix for utf-8 XTerm/MinTTY and (set-input-meta-mode t)

[Eli Zaretskii]

> Date: Thu, 03 Jun 2021 05:42:57 +0000
> From: Max Mikhanosha <max.mikhanosha@protonmail.com>
> Cc: Eli Zaretskii <eliz@gnu.org>, emacs-devel@gnu.org
> 
> On Wednesday, June 2nd, 2021 at 8:16 AM, Andreas Schwab <schwab@linux-m68k.org> wrote:
> 
> > > If META is t, Emacs will accept 8-bit input, and interpret the 8th
> > > bit as the Meta modifier.
> > >
> > > +If META is 'encoded', Emacs will decode the input with accordance
> > > +to current coding system, and then examine decoded input 8th bit
> > > +and interpret it as the Meta modifier
> >
> > in accordance to
> 
> Updated per above. Feel free to do any farther changes to wording or coding style if
> you going to merge it, or let me know if you'd like me to improve this in any way.

Thanks, I installed this on the master branch.

(The patch didn't apply as-is, because you used old codebase.  Please
in the future use the latest code on the relevant branch, and also
accompany the patches with a ChageLog-style commit log message, as
described in CONTRIBUTE.)