1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
| | #!/usr/bin/python
# -*- coding: utf-8 -*-
"""generate-special-casing.py --- generate special-casing.h file
Copyright (C) 2016 Free Software Foundation, Inc.
This file is part of GNU Emacs.
GNU Emacs is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or (at
your option) any later version.
GNU Emacs is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>.
"""
import os
import re
import sys
import tempfile
import textwrap
TEMPLATE = '''\
/* Special case mapping rules. Only unconditional entries are included.
This file is automatically generated from SpecialCasing.txt file
distributed with Unicode standard by %(generator)s.
Do not edit manually. */
#include <stdint.h>
struct special_casing_static_asserts {
%(asserts)s
};
typedef %(code_point_type)s special_casing_char_t;
/* Zero-terminated, sorted list. */
static const special_casing_char_t special_casing_code_points[] = {
%(code_points)s
};
/* If buf.len_chars has this bit set, the character maps to itself. */
#define SPECIAL_CASING_NO_CHANGE_BIT 0x80
static const struct casing_str_buf special_casing_entries[] = {
%(entries)s
};
'''
MAX_DATA_BYTES_LENGTH = 6
ASSERTS = (
('casing_str_buf_data_must_be_at_least_%d_chars' % MAX_DATA_BYTES_LENGTH,
'sizeof ((struct casing_str_buf*)0)->data >= %d' % MAX_DATA_BYTES_LENGTH),
('CASE_UP_must_equal_0', 'CASE_UP == 0'),
('CASE_DOWN_must_equal_1', 'CASE_DOWN == 1'),
('CASE_CAPITALIZE_must_equal_2', 'CASE_CAPITALIZE == 2')
)
def encode(code, code_points):
"""Convert a space-separated list of code points into UTF-8 C-string.
Args:
code: Code point this mapping is for.
code_points: A space-separated list of hexadecimal numbers representing
code points in desired representation.
Returns:
A (literal, len_chars, len_bytes) tuple. len_chars may be zero if code
point maps to itself.
"""
code_points = [int(cp, 16) for cp in code_points.split()]
len_chars = len(code_points)
if len_chars == 1 and code_points[0] == code:
len_chars = len_chars | 0x80
val = ''.join(unichr(cp) for cp in code_points).encode('utf-8')
ret = ''
for ch in val:
o = ord(ch)
if o < 32 or o >= 127:
ch = '\\x%02x' % o
ret += ch
return '"%s"' % ret, len_chars, len(val)
def read_entries(fd):
"""Read entries from SpecialCasing.txt file.
Conditional entries are ignored.
Args:
fd: File object to read data from.
Returns:
A list of [code, up_lit, up_len, down_lit, down_len, title_lit,
title_len, comment] lists.
"""
idx_lower = 1
idx_title = 2
idx_upper = 3
# This order must match CASE_UP, CASE_DOWN and CASE_CAPITALIZE_UP from
# casefiddle.c. This ordering (which is different than in SpecialCasing.txt
# file) is checked via static asserts included in the generated file.
indexes = (idx_upper, idx_lower, idx_title)
entries = []
for line in fd:
line = line.strip()
if not line or line[0] == '#':
continue
line = re.split(r';\s*', line)
if len(line) == 6:
# Conditional special casing don’t go into special-casing.h
#sys.stderr.write('make-special-casing: %s: conditions present '
# '(%s), ignoring\n' % (line[0], line[4]))
continue
code = int(line[0], 16)
entry = [code]
for i in indexes:
val = encode(code, line[i])
entry.append(val)
if val:
# The data structure we’re using assumes that all C strings are
# no more than six bytes (excluding NUL terminator). Enforce
# that here.
assert val[2] <= MAX_DATA_BYTES_LENGTH, (code, i, val)
entry.append(line[4].strip(' #').capitalize())
entries.append(entry)
entries.sort()
return entries
def format_output(entries):
# If all code points are 16-bit prefer using uint16_t since it makes the
# array smaller and more cache friendly.
if all(entry[0] <= 0xffff for entry in entries):
cp_type, cp_fmt = 'uint16_t', '%04X'
else:
cp_type, cp_fmt = 'uint32_t', '%06X'
fmt = '0x%sU' % cp_fmt;
code_points = ', '.join(fmt % entry[0] for entry in entries)
lines = []
for entry in entries:
lines.append(' /* U+%s %%s */' % cp_fmt % (entry[0], entry[4]))
for val in entry[1:4]:
lines.append(' { %s, %d, %d },' % val)
lines[-1] = lines[-1].rstrip(',')
return TEMPLATE % {
'generator': os.path.basename(__file__),
'asserts': '\n'.join(' char %s[%s ? 1 : -1];' % p for p in ASSERTS),
'code_point_type': cp_type,
'code_points': textwrap.fill(code_points + ', 0', width=80,
initial_indent=' ',
subsequent_indent=' '),
'entries': '\n'.join(lines)
}
def main(argv):
if len(argv) != 3:
sys.stderr.write('usage: %s SpecialCasing.txt special-casing.h\n' %
argv[0])
sys.exit(1)
with open(argv[1]) as fd:
entries = read_entries(fd)
data = format_output(entries)
with open(argv[2], 'w') as fd:
fd.write(data)
if __name__ == '__main__':
main(sys.argv)
|