Re: Character literals for Unicode (control) characters

[Philipp Stephani <p.stephani2@gmail.com>]

[-- Attachment #1.1: Type: text/plain, Size: 500 bytes --]

Paul Eggert <eggert@cs.ucla.edu> schrieb am So., 20. März 2016 um 13:58 Uhr:

> Thanks, one thing I didn't notice earlier:
>
> +  xsignal1 (Qinvalid_read_syntax,
> +            CALLN (Fformat, build_pure_c_string ("\\N{%s}"), name));
>
> This can run Emacs out of pure space unnecessarily.  Use AUTO_STRING
> instead of
> build_pure_c_string.
>
> Also, I've lost track of what this patch is building on. Perhaps send all
> the
> patches next time....
>

Done. Attached all patches.

[-- Attachment #1.2: Type: text/html, Size: 807 bytes --]

[-- Attachment #2: 0003-Minor-cleanups-for-character-name-escapes.patch --]
[-- Type: application/octet-stream, Size: 2923 bytes --]

From 30e6d9dd4e83a36fe07bbeae678b3f086773346e Mon Sep 17 00:00:00 2001
From: Philipp Stephani <phst@google.com>
Date: Sun, 6 Mar 2016 19:27:21 +0100
Subject: [PATCH 3/4] Minor cleanups for character name escapes.

* lread.c (init_character_names): Add missing `void'.  Remove
top-level `const'.
(read_escape): Simplify loop a bit.  Remove top-level `const'.
---
 src/lread.c | 27 ++++++++++++---------------
 1 file changed, 12 insertions(+), 15 deletions(-)

diff --git a/src/lread.c b/src/lread.c
index 6e84fc8..4000637 100644
--- a/src/lread.c
+++ b/src/lread.c
@@ -2159,20 +2159,20 @@ static ptrdiff_t max_character_name_length;
 
 /* Initializes `character_names' and `max_character_name_length'.
    Called by `read_escape'.  */
-void init_character_names ()
+void init_character_names (void)
 {
   character_names = CALLN (Fmake_hash_table,
                            QCtest, Qequal,
                            /* Currently around 100,000 Unicode
                               characters are defined.  */
                            QCsize, make_natnum (100000));
-  const Lisp_Object get_property =
+  Lisp_Object get_property =
     Fsymbol_function (intern_c_string ("get-char-code-property"));
   ptrdiff_t length = 0;
   for (int i = 0; i <= MAX_UNICODE_CHAR; ++i)
     {
-      const Lisp_Object code = make_natnum (i);
-      const Lisp_Object name = call2 (get_property, code, Qname);
+      Lisp_Object code = make_natnum (i);
+      Lisp_Object name = call2 (get_property, code, Qname);
       if (NILP (name)) continue;
       CHECK_STRING (name);
       length = max (length, SBYTES (name));
@@ -2418,25 +2418,22 @@ read_escape (Lisp_Object readcharfun, bool stringp)
                character names in e.g. multi-line strings.  */
             if (c_isspace (c))
               {
-                if (! whitespace)
-                  {
-                    whitespace = true;
-                    name[length++] = ' ';
-                  }
+                if (whitespace)
+                  continue;
+                c = ' ';
+                whitespace = true;
               }
             else
-              {
-                whitespace = false;
-                name[length++] = c;
-              }
+              whitespace = false;
+            name[length++] = c;
             if (length >= max_character_name_length)
               invalid_syntax ("Character name too long");
           }
         if (length == 0)
           invalid_syntax ("Empty character name");
         name[length] = 0;
-        const Lisp_Object lisp_name = make_unibyte_string (name, length);
-        const Lisp_Object code =
+        Lisp_Object lisp_name = make_unibyte_string (name, length);
+        Lisp_Object code =
           (length >= 3 && length <= 10 && name[0] == 'U' && name[1] == '+') ?
           /* Code point as U+N, where N is between 1 and 8 hexadecimal
              digits.  */
-- 
2.7.0


[-- Attachment #3: 0001-Implement-named-character-escapes-similar-to-Perl.patch --]
[-- Type: application/octet-stream, Size: 7134 bytes --]

From 22e299cd23a72a072461befa30a04bf557aecac8 Mon Sep 17 00:00:00 2001
From: Philipp Stephani <phst@google.com>
Date: Sun, 6 Mar 2016 16:16:29 +0100
Subject: [PATCH 1/4] Implement named character escapes, similar to Perl

* lread.c (init_character_names): New function.
(read_escape): Read Perl-style named character escape sequences.
(syms_of_lread): Initialize new variable `character_names'.
* test/src/lread-tests.el (lread-char-empty-name): Add test file
for src/lread.c.
---
 src/lread.c             | 96 +++++++++++++++++++++++++++++++++++++++++++++++++
 test/src/lread-tests.el | 54 ++++++++++++++++++++++++++++
 2 files changed, 150 insertions(+)
 create mode 100644 test/src/lread-tests.el

diff --git a/src/lread.c b/src/lread.c
index 25e3ff0..6e84fc8 100644
--- a/src/lread.c
+++ b/src/lread.c
@@ -43,6 +43,7 @@ along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.  */
 #include "systime.h"
 #include "termhooks.h"
 #include "blockinput.h"
+#include <c-ctype.h>
 
 #ifdef MSDOS
 #include "msdos.h"
@@ -2150,6 +2151,36 @@ grow_read_buffer (void)
 			 MAX_MULTIBYTE_LENGTH, -1, 1);
 }
 
+/* Hash table that maps Unicode character names to code points.  */
+static Lisp_Object character_names;
+
+/* Length of the longest Unicode character name, in bytes. */
+static ptrdiff_t max_character_name_length;
+
+/* Initializes `character_names' and `max_character_name_length'.
+   Called by `read_escape'.  */
+void init_character_names ()
+{
+  character_names = CALLN (Fmake_hash_table,
+                           QCtest, Qequal,
+                           /* Currently around 100,000 Unicode
+                              characters are defined.  */
+                           QCsize, make_natnum (100000));
+  const Lisp_Object get_property =
+    Fsymbol_function (intern_c_string ("get-char-code-property"));
+  ptrdiff_t length = 0;
+  for (int i = 0; i <= MAX_UNICODE_CHAR; ++i)
+    {
+      const Lisp_Object code = make_natnum (i);
+      const Lisp_Object name = call2 (get_property, code, Qname);
+      if (NILP (name)) continue;
+      CHECK_STRING (name);
+      length = max (length, SBYTES (name));
+      Fputhash (name, code, character_names);
+    }
+  max_character_name_length = length;
+}
+
 /* Read a \-escape sequence, assuming we already read the `\'.
    If the escape sequence forces unibyte, return eight-bit char.  */
 
@@ -2357,6 +2388,68 @@ read_escape (Lisp_Object readcharfun, bool stringp)
 	return i;
       }
 
+    case 'N':
+      /* Named character.  */
+      {
+        c = READCHAR;
+        if (c != '{')
+          invalid_syntax ("Expected opening brace after \\N");
+        if (NILP (character_names))
+          init_character_names ();
+        USE_SAFE_ALLOCA;
+        char *name = SAFE_ALLOCA (max_character_name_length + 1);
+        bool whitespace = false;
+        ptrdiff_t length = 0;
+        while (true)
+          {
+            c = READCHAR;
+            if (c < 0)
+              end_of_file_error ();
+            if (c == '}')
+              break;
+            if (! c_isascii (c))
+              xsignal1 (Qinvalid_read_syntax,
+                        CALLN (Fformat,
+                               build_pure_c_string ("Non-ASCII character U+%04X"
+                                                    " in character name"),
+                               make_natnum (c)));
+            /* We treat multiple adjacent whitespace characters as a
+               single space character.  This makes it easier to use
+               character names in e.g. multi-line strings.  */
+            if (c_isspace (c))
+              {
+                if (! whitespace)
+                  {
+                    whitespace = true;
+                    name[length++] = ' ';
+                  }
+              }
+            else
+              {
+                whitespace = false;
+                name[length++] = c;
+              }
+            if (length >= max_character_name_length)
+              invalid_syntax ("Character name too long");
+          }
+        if (length == 0)
+          invalid_syntax ("Empty character name");
+        name[length] = 0;
+        const Lisp_Object lisp_name = make_unibyte_string (name, length);
+        const Lisp_Object code =
+          (length >= 3 && length <= 10 && name[0] == 'U' && name[1] == '+') ?
+          /* Code point as U+N, where N is between 1 and 8 hexadecimal
+             digits.  */
+          string_to_number (name + 2, 16, false) :
+          Fgethash (lisp_name, character_names, Qnil);
+        SAFE_FREE ();
+        if (! RANGED_INTEGERP (0, code, MAX_UNICODE_CHAR))
+          xsignal1 (Qinvalid_read_syntax,
+                    CALLN (Fformat,
+                           build_pure_c_string ("\\N{%s}"), lisp_name));
+        return XINT (code);
+      }
+
     default:
       return c;
     }
@@ -4745,4 +4838,7 @@ that are loaded before your customizations are read!  */);
   DEFSYM (Qweakness, "weakness");
   DEFSYM (Qrehash_size, "rehash-size");
   DEFSYM (Qrehash_threshold, "rehash-threshold");
+
+  character_names = Qnil;
+  staticpro (&character_names);
 }
diff --git a/test/src/lread-tests.el b/test/src/lread-tests.el
new file mode 100644
index 0000000..1f87334
--- /dev/null
+++ b/test/src/lread-tests.el
@@ -0,0 +1,54 @@
+;;; lread-tests.el --- tests for lread.c -*- lexical-binding: t; -*-
+
+;; Copyright (C) 2016  Google Inc.
+
+;; Author: Philipp Stephani <phst@google.com>
+
+;; This file is part of GNU Emacs.
+
+;; This program is free software; you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; This program is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+;;; Commentary:
+
+;; Unit tests for code in src/lread.c.
+
+;;; Code:
+
+(ert-deftest lread-char-number ()
+  (should (equal ?\N{U+A817} #xA817)))
+
+(ert-deftest lread-char-name ()
+  (should (equal ?\N{SYLOTI  NAGRI LETTER
+                 DHO}
+                 #xA817)))
+
+(ert-deftest lread-char-invalid-number ()
+  (should-error (read "?\\N{U+110000}") :type 'invalid-read-syntax))
+
+(ert-deftest lread-char-invalid-name ()
+  (should-error (read "?\\N{DOES NOT EXIST}")) :type 'invalid-read-syntax)
+
+(ert-deftest lread-char-non-ascii-name ()
+  (should-error (read "?\\N{LATIN CAPITAL LETTER Ø}")) 'invalid-read-syntax)
+
+(ert-deftest lread-char-empty-name ()
+  (should-error (read "?\\N{}")) 'invalid-read-syntax)
+
+(ert-deftest lread-string-char-number ()
+  (should (equal "a\N{U+A817}b" "a\uA817b")))
+
+(ert-deftest lread-string-char-name ()
+  (should (equal "a\N{SYLOTI NAGRI  LETTER DHO}b" "a\uA817b")))
+
+;;; lread-tests.el ends here
-- 
2.7.0


[-- Attachment #4: 0002-Add-documentation-for-character-name-escapes.patch --]
[-- Type: application/octet-stream, Size: 2348 bytes --]

From d0d5219a358a2d8e853f1ce11cf16fb2629697c6 Mon Sep 17 00:00:00 2001
From: Philipp Stephani <phst@google.com>
Date: Sun, 6 Mar 2016 19:07:06 +0100
Subject: [PATCH 2/4] Add documentation for character name escapes

---
 doc/lispref/nonascii.texi |  2 +-
 doc/lispref/objects.texi  | 10 ++++++++++
 etc/NEWS                  |  5 +++++
 3 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/doc/lispref/nonascii.texi b/doc/lispref/nonascii.texi
index 9cf3b57..66ad9ac 100644
--- a/doc/lispref/nonascii.texi
+++ b/doc/lispref/nonascii.texi
@@ -633,7 +633,7 @@ Character Properties
 @end group
 @group
 ;; U+2163 ROMAN NUMERAL FOUR
-(get-char-code-property ?\u2163 'numeric-value)
+(get-char-code-property ?\N@{ROMAN NUMERAL FOUR@} 'numeric-value)
      @result{} 4
 @end group
 @group
diff --git a/doc/lispref/objects.texi b/doc/lispref/objects.texi
index 3245930..96b334d 100644
--- a/doc/lispref/objects.texi
+++ b/doc/lispref/objects.texi
@@ -387,6 +387,16 @@ General Escape Syntax
 for the character @kbd{C-b}.  Only characters up to octal code 777 can
 be specified this way.
 
+  Fourthly, you can specify characters by their name.  A character
+name escape sequence consists of a backslash, @samp{N@{}, the Unicode
+character name, and @samp{@}}.  Alternatively, you can also put the
+numeric code point value between the braces, using the syntax
+@samp{\N@{U+nnnn@}}, where @samp{nnnn} denotes between one and eight
+hexadecimal digits.  Thus, @samp{?\N@{LATIN CAPITAL LETTER A@}} and
+@samp{?\N@{U+41@}} both denote the character @kbd{A}.  To simplify
+entering multi-line strings, you can replace spaces in the character
+names by arbitrary non-empty sequence of whitespace (e.g., newlines).
+
   These escape sequences may also be used in strings.  @xref{Non-ASCII
 in Strings}.
 
diff --git a/etc/NEWS b/etc/NEWS
index 92d69d2..9c77474 100644
--- a/etc/NEWS
+++ b/etc/NEWS
@@ -159,6 +159,11 @@ that negotiation should complete even on non-blocking sockets.
 `window-pixel-height-before-size-change' allow to detect which window
 changed size when `window-size-change-functions' are run.
 
++++
+** Emacs now supports character name escape sequences in character and
+string literals.  The syntax variants \N{character name} and
+\N{U+code} are supported.
+
 \f
 * Changes in Emacs 25.2 on Non-Free Operating Systems
 
-- 
2.7.0


[-- Attachment #5: 0004-Use-ucs-names-for-character-name-escapes.patch --]
[-- Type: application/octet-stream, Size: 8347 bytes --]

From 173eb8b38b4b495a46bca02779ae590130328175 Mon Sep 17 00:00:00 2001
From: Philipp Stephani <phst@google.com>
Date: Sun, 13 Mar 2016 21:27:30 +0100
Subject: [PATCH 4/4] Use `ucs-names' for character name escapes

* lread.c (invalid_character_name, check_scalar_value)
(parse_code_after_prefix, character_name_to_code): New helper
functions that use `ucs-names' and parsing for CJK ideographs.
(read_escape): Use helper functions.
(syms_of_lread): New symbol `ucs-names'.
* test/src/lread-tests.el: New tests; fix a couple of bugs in
existing tests.
---
 src/lread.c             | 126 +++++++++++++++++++++++++++++++-----------------
 test/src/lread-tests.el |  11 ++++-
 2 files changed, 91 insertions(+), 46 deletions(-)

diff --git a/src/lread.c b/src/lread.c
index 4000637..dc023eb 100644
--- a/src/lread.c
+++ b/src/lread.c
@@ -44,6 +44,7 @@ along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.  */
 #include "termhooks.h"
 #include "blockinput.h"
 #include <c-ctype.h>
+#include <string.h>
 
 #ifdef MSDOS
 #include "msdos.h"
@@ -2151,36 +2152,90 @@ grow_read_buffer (void)
 			 MAX_MULTIBYTE_LENGTH, -1, 1);
 }
 
-/* Hash table that maps Unicode character names to code points.  */
-static Lisp_Object character_names;
+/* Signal an invalid-read-syntax error indicating that the character
+   name in an \N{…} literal is invalid.  */
+static _Noreturn void
+invalid_character_name (Lisp_Object name)
+{
+  xsignal1 (Qinvalid_read_syntax,
+            CALLN (Fformat, build_pure_c_string ("\\N{%s}"), name));
+}
 
-/* Length of the longest Unicode character name, in bytes. */
-static ptrdiff_t max_character_name_length;
+/* Check that CODE is a valid Unicode scalar value, and return its
+   value.  CODE should be parsed from the character name given by
+   NAME.  NAME is used for error messages.  */
+static int
+check_scalar_value (Lisp_Object code, Lisp_Object name)
+{
+  if (! NUMBERP (code))
+    invalid_character_name (name);
+  EMACS_INT i = XINT (code);
+  if (! (0 <= i && i <= MAX_UNICODE_CHAR)
+      /* Don't allow surrogates.  */
+      || (0xD800 <= code && code <= 0xDFFF))
+    invalid_character_name (name);
+  return i;
+}
 
-/* Initializes `character_names' and `max_character_name_length'.
-   Called by `read_escape'.  */
-void init_character_names (void)
+/* If NAME starts with PREFIX, interpret the rest as a hexadecimal
+   number and return its value.  Raise invalid-read-syntax if the
+   number is not a valid scalar value.  Return −1 if NAME doesn’t
+   start with PREFIX.  */
+static int
+parse_code_after_prefix (Lisp_Object name, const char *prefix)
 {
-  character_names = CALLN (Fmake_hash_table,
-                           QCtest, Qequal,
-                           /* Currently around 100,000 Unicode
-                              characters are defined.  */
-                           QCsize, make_natnum (100000));
-  Lisp_Object get_property =
-    Fsymbol_function (intern_c_string ("get-char-code-property"));
-  ptrdiff_t length = 0;
-  for (int i = 0; i <= MAX_UNICODE_CHAR; ++i)
+  ptrdiff_t name_len = SBYTES (name);
+  ptrdiff_t prefix_len = strlen (prefix);
+  /* Allow between one and eight hexadecimal digits after the
+     prefix.  */
+  if (prefix_len < name_len && name_len <= prefix_len + 8
+      && memcmp (SDATA (name), prefix, prefix_len) == 0)
     {
-      Lisp_Object code = make_natnum (i);
-      Lisp_Object name = call2 (get_property, code, Qname);
-      if (NILP (name)) continue;
-      CHECK_STRING (name);
-      length = max (length, SBYTES (name));
-      Fputhash (name, code, character_names);
+      Lisp_Object code = string_to_number (SDATA (name) + prefix_len, 16, false);
+      if (NUMBERP (code))
+        return check_scalar_value (code, name);
     }
-  max_character_name_length = length;
+  return -1;
 }
 
+/* Returns the scalar value that has the Unicode character name NAME.
+   Raises `invalid-read-syntax' if there is no such character.  */
+static int
+character_name_to_code (Lisp_Object name)
+{
+  /* Code point as U+N, where N is between 1 and 8 hexadecimal
+     digits.  */
+  int code = parse_code_after_prefix (name, "U+");
+  if (code >= 0)
+    return code;
+
+  /* CJK ideographs are not contained in the association list returned
+     by `ucs-names'.  But they follow a predictable naming pattern: a
+     fixed prefix plus the hexadecimal codepoint value.  */
+  code = parse_code_after_prefix (name, "CJK IDEOGRAPH-");
+  if (code >= 0)
+    {
+      /* Various ranges of CJK characters; see UnicodeData.txt.  */
+      if ((0x3400 <= code && code <= 0x4DB5)
+          || (0x4E00 <= code && code <= 0x9FD5)
+          || (0x20000 <= code && code <= 0x2A6D6)
+          || (0x2A700 <= code && code <= 0x2B734)
+          || (0x2B740 <= code && code <= 0x2B81D)
+          || (0x2B820 <= code && code <= 0x2CEA1))
+        return code;
+      else
+        invalid_character_name (name);
+    }
+
+  /* Look up the name in the table returned by `ucs-names'.  */
+  Lisp_Object names = call0 (Qucs_names);
+  return check_scalar_value (CDR (Fassoc (name, names)), name);
+}
+
+/* Bound on the length of a Unicode character name.  As of
+   Unicode 9.0.0 the maximum is 83, so this should be safe. */
+enum { UNICODE_CHARACTER_NAME_LENGTH_BOUND = 200 };
+
 /* Read a \-escape sequence, assuming we already read the `\'.
    If the escape sequence forces unibyte, return eight-bit char.  */
 
@@ -2394,10 +2449,7 @@ read_escape (Lisp_Object readcharfun, bool stringp)
         c = READCHAR;
         if (c != '{')
           invalid_syntax ("Expected opening brace after \\N");
-        if (NILP (character_names))
-          init_character_names ();
-        USE_SAFE_ALLOCA;
-        char *name = SAFE_ALLOCA (max_character_name_length + 1);
+        char name[UNICODE_CHARACTER_NAME_LENGTH_BOUND + 1];
         bool whitespace = false;
         ptrdiff_t length = 0;
         while (true)
@@ -2426,25 +2478,12 @@ read_escape (Lisp_Object readcharfun, bool stringp)
             else
               whitespace = false;
             name[length++] = c;
-            if (length >= max_character_name_length)
+            if (length >= sizeof name)
               invalid_syntax ("Character name too long");
           }
         if (length == 0)
           invalid_syntax ("Empty character name");
-        name[length] = 0;
-        Lisp_Object lisp_name = make_unibyte_string (name, length);
-        Lisp_Object code =
-          (length >= 3 && length <= 10 && name[0] == 'U' && name[1] == '+') ?
-          /* Code point as U+N, where N is between 1 and 8 hexadecimal
-             digits.  */
-          string_to_number (name + 2, 16, false) :
-          Fgethash (lisp_name, character_names, Qnil);
-        SAFE_FREE ();
-        if (! RANGED_INTEGERP (0, code, MAX_UNICODE_CHAR))
-          xsignal1 (Qinvalid_read_syntax,
-                    CALLN (Fformat,
-                           build_pure_c_string ("\\N{%s}"), lisp_name));
-        return XINT (code);
+        return character_name_to_code (make_unibyte_string (name, length));
       }
 
     default:
@@ -4836,6 +4875,5 @@ that are loaded before your customizations are read!  */);
   DEFSYM (Qrehash_size, "rehash-size");
   DEFSYM (Qrehash_threshold, "rehash-threshold");
 
-  character_names = Qnil;
-  staticpro (&character_names);
+  DEFSYM (Qucs_names, "ucs-names");
 }
diff --git a/test/src/lread-tests.el b/test/src/lread-tests.el
index 1f87334..ff5d0f6 100644
--- a/test/src/lread-tests.el
+++ b/test/src/lread-tests.el
@@ -40,10 +40,17 @@
   (should-error (read "?\\N{DOES NOT EXIST}")) :type 'invalid-read-syntax)
 
 (ert-deftest lread-char-non-ascii-name ()
-  (should-error (read "?\\N{LATIN CAPITAL LETTER Ø}")) 'invalid-read-syntax)
+  (should-error (read "?\\N{LATIN CAPITAL LETTER Ø}")
+                :type 'invalid-read-syntax))
 
 (ert-deftest lread-char-empty-name ()
-  (should-error (read "?\\N{}")) 'invalid-read-syntax)
+  (should-error (read "?\\N{}") :type 'invalid-read-syntax))
+
+(ert-deftest lread-char-cjk-name ()
+  (should (equal ?\N{CJK IDEOGRAPH-2B734} #x2B734)))
+
+(ert-deftest lread-char-invalid-cjk-name ()
+  (should-error (read "?\\N{CJK IDEOGRAPH-2B735}") :type 'invalid-read-syntax))
 
 (ert-deftest lread-string-char-number ()
   (should (equal "a\N{U+A817}b" "a\uA817b")))
-- 
2.7.0