about summary refs log tree commit diff
path: root/localedata/unicode-gen/gen_translit_compat.py
blob: d4c3d195ea7651eff83b67561c192528434ab56f (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
#!/usr/bin/python3
# -*- coding: utf-8 -*-
#
# Generate a translit_compat file from a UnicodeData file.
# Copyright (C) 2015-2019 Free Software Foundation, Inc.
# This file is part of the GNU C Library.
#
# The GNU C Library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# The GNU C Library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with the GNU C Library; if not, see
# <https://www.gnu.org/licenses/>.

'''
Generate a translit_compat file from UnicodeData.txt

To see how this script is used, call it with the “-h” option:

    $ ./gen_translit_compat -h
    … prints usage message …
'''

import argparse
import time
import unicode_utils

def read_input_file(filename):
    '''Reads the original glibc translit_compat file to get the
    original head and tail.

    We want to replace only the part of the file between
    “translit_start” and “translit_end”
    '''
    head = tail = ''
    with open(filename, mode='r') as translit_file:
        for line in translit_file:
            head = head + line
            if line.startswith('translit_start'):
                break
        for line in translit_file:
            if line.startswith('translit_end'):
                tail = line
                break
        for line in translit_file:
            tail = tail + line
    return (head, tail)

def output_head(translit_file, unicode_version, head=''):
    '''Write the header of the output file, i.e. the part of the file
    before the “translit_start” line.
    '''
    if ARGS.input_file and head:
        translit_file.write(head)
    else:
        translit_file.write('escape_char /\n')
        translit_file.write('comment_char %\n')
        translit_file.write(unicode_utils.COMMENT_HEADER)
        translit_file.write('\n')
        translit_file.write('% Transliterations of compatibility characters ')
        translit_file.write('and ligatures.\n')
        translit_file.write('% Generated automatically from UnicodeData.txt '
                            + 'by gen_translit_compat.py '
                            + 'on {:s} '.format(time.strftime('%Y-%m-%d'))
                            + 'for Unicode {:s}.\n'.format(unicode_version))
        translit_file.write('\n')
        translit_file.write('LC_CTYPE\n')
        translit_file.write('\n')
        translit_file.write('translit_start\n')

def output_tail(translit_file, tail=''):
    '''Write the tail of the output file'''
    if ARGS.input_file and tail:
        translit_file.write(tail)
    else:
        translit_file.write('translit_end\n')
        translit_file.write('\n')
        translit_file.write('END LC_CTYPE\n')

def compatibility_decompose(code_point):
    '''http://www.unicode.org/reports/tr44/#Character_Decomposition_Mappings

    “The compatibility decomposition is formed by recursively applying
    the canonical and compatibility mappings, then applying the
    Canonical Ordering Algorithm.”

    We don’t do the canonical decomposition here because this is
    done in gen_translit_combining.py to generate translit_combining.

    And we ignore some of the possible compatibility formatting tags
    here. Some of them are used in other translit_* files, not
    translit_compat:

    <font>:   translit_font
    <circle>: translit_circle
    <wide>:   translit_wide
    <narrow>: translit_narrow
    <square>: translit_cjk_compat
    <fraction>: translit_fraction

    And we ignore

    <noBreak>, <initial>, <medial>, <final>, <isolated>

    because they seem to be not useful for transliteration.
    '''
    decomposition = unicode_utils.UNICODE_ATTRIBUTES[
        code_point]['decomposition']
    compatibility_tags = (
        '<compat>', '<super>', '<sub>', '<vertical>')
    for compatibility_tag in compatibility_tags:
        if decomposition.startswith(compatibility_tag):
            decomposition = decomposition[len(compatibility_tag)+1:]
            decomposed_code_points = [int(x, 16)
                                      for x in decomposition.split(' ')]
            if (len(decomposed_code_points) > 1
                    and decomposed_code_points[0] == 0x0020
                    and decomposed_code_points[1] >= 0x0300
                    and decomposed_code_points[1] <= 0x03FF):
                # Decomposes into a space followed by a combining character.
                # This is not useful fo transliteration.
                return []
            else:
                return_value = []
                for index in range(0, len(decomposed_code_points)):
                    cd_code_points = compatibility_decompose(
                        decomposed_code_points[index])
                    if cd_code_points:
                        return_value += cd_code_points
                    else:
                        return_value += [decomposed_code_points[index]]
                return return_value
    return []

def special_decompose(code_point_list):
    '''
    Decompositions which are not in UnicodeData.txt at all but which
    were used in the original translit_compat file in glibc and
    which seem to make sense.  I want to keep the update of
    translit_compat close to the spirit of the original file,
    therefore I added this special decomposition rules here.
    '''
    special_decompose_dict = {
        (0x03BC,): [0x0075], # μ → u
        (0x02BC,): [0x0027], # ʼ → '
    }
    if tuple(code_point_list) in special_decompose_dict:
        return special_decompose_dict[tuple(code_point_list)]
    else:
        return code_point_list

def special_ligature_decompose(code_point):
    '''
    Decompositions for ligatures which are not in UnicodeData.txt at
    all but which were used in the original translit_compat file in
    glibc and which seem to make sense.  I want to keep the update of
    translit_compat close to the spirit of the original file,
    therefore I added these special ligature decomposition rules here.

    '''
    special_ligature_decompose_dict = {
        0x00E6: [0x0061, 0x0065], # æ → ae
        0x00C6: [0x0041, 0x0045], # Æ → AE
        # These following 5 special ligature decompositions were
        # in the original glibc/localedata/locales/translit_compat file
        0x0152: [0x004F, 0x0045], # Œ → OE
        0x0153: [0x006F, 0x0065], # œ → oe
        0x05F0: [0x05D5, 0x05D5], # װ → וו
        0x05F1: [0x05D5, 0x05D9], # ױ → וי
        0x05F2: [0x05D9, 0x05D9], # ײ → יי
        # The following special ligature decompositions were
        # not in the original glibc/localedata/locales/translit_compat file
        # U+04A4 CYRILLIC CAPITAL LIGATURE EN GHE
        # → U+041D CYRILLIC CAPITAL LETTER EN,
        #   U+0413 CYRILLIC CAPITAL LETTER GHE
        0x04A4: [0x041D, 0x0413], # Ҥ → НГ
        # U+04A5 CYRILLIC SMALL LIGATURE EN GHE
        # → U+043D CYRILLIC SMALL LETTER EN,
        #   U+0433 CYRILLIC SMALL LETTER GHE
        0x04A5: [0x043D, 0x0433], # ҥ → нг
        # U+04B4 CYRILLIC CAPITAL LIGATURE TE TSE
        # → U+0422 CYRILLIC CAPITAL LETTER TE,
        #   U+0426 CYRILLIC CAPITAL LETTER TSE
        0x04B4: [0x0422, 0x0426], # Ҵ → ТЦ
        # U+04B5 CYRILLIC SMALL LIGATURE TE TSE
        # → U+0442 CYRILLIC SMALL LETTER TE,
        #   U+0446 CYRILLIC SMALL LETTER TSE
        0x04B5: [0x0442, 0x0446], # ҵ → тц
        # U+04d4 CYRILLIC CAPITAL LIGATURE A IE
        # → U+0410 CYRILLIC CAPITAL LETTER A
        #   U+0415;CYRILLIC CAPITAL LETTER IE
        0x04D4: [0x0410, 0x0415], # Ӕ → АЕ
        # U+04D5 CYRILLIC SMALL LIGATURE A IE
        # → U+0430 CYRILLIC SMALL LETTER A,
        #   U+0435 CYRILLIC SMALL LETTER IE
        0x04D5: [0x0430, 0x0435], # ӕ → ае
        # I am not sure what to do with the following ligatures
        # maybe it makes no sense to decompose them:
        # U+0616 ARABIC SMALL HIGH LIGATURE ALEF WITH LAM WITH YEH
        # U+06d6 ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA
        # U+06d7 ARABIC SMALL HIGH LIGATURE QAF WITH LAM WITH ALEF MAKSURA
        # U+fdfd ARABIC LIGATURE BISMILLAH AR-RAHMAN AR-RAHEEM
        # U+fe20 COMBINING LIGATURE LEFT HALF
        # U+fe21 COMBINING LIGATURE RIGHT HALF
        # U+fe27 COMBINING LIGATURE LEFT HALF BELOW
        # U+fe28 COMBINING LIGATURE RIGHT HALF BELOW
        # U+11176 MAHAJANI LIGATURE SHRI
        # U+1f670 SCRIPT LIGATURE ET ORNAMENT
        # U+1f671 HEAVY SCRIPT LIGATURE ET ORNAMENT
        # U+1f672 LIGATURE OPEN ET ORNAMENT
        # U+1f673 HEAVY LIGATURE OPEN ET ORNAMENT
    }
    if code_point in special_ligature_decompose_dict:
        return special_ligature_decompose_dict[code_point]
    else:
        return [code_point]

def output_transliteration(translit_file):
    '''Write the new transliteration to the output file'''
    translit_file.write('\n')
    for code_point in sorted(unicode_utils.UNICODE_ATTRIBUTES):
        name = unicode_utils.UNICODE_ATTRIBUTES[code_point]['name']
        decomposed_code_points = [compatibility_decompose(code_point)]
        if not decomposed_code_points[0]:
            if special_decompose([code_point]) != [code_point]:
                decomposed_code_points[0] = special_decompose([code_point])
        else:
            special_decomposed_code_points = []
            while True:
                special_decomposed_code_points = special_decompose(
                    decomposed_code_points[-1])
                if (special_decomposed_code_points
                        != decomposed_code_points[-1]):
                    decomposed_code_points.append(
                        special_decomposed_code_points)
                    continue
                special_decomposed_code_points = []
                for decomposed_code_point in decomposed_code_points[-1]:
                    special_decomposed_code_points += special_decompose(
                        [decomposed_code_point])
                if (special_decomposed_code_points
                        == decomposed_code_points[-1]):
                    break
                decomposed_code_points.append(
                    special_decomposed_code_points)
        if decomposed_code_points[0]:
            translit_file.write('% {:s}\n'.format(name))
            translit_file.write('{:s} '.format(
                unicode_utils.ucs_symbol(code_point)))
            for index in range(0, len(decomposed_code_points)):
                if index > 0:
                    translit_file.write(';')
                translit_file.write('"')
                for decomposed_code_point in decomposed_code_points[index]:
                    translit_file.write('{:s}'.format(
                        unicode_utils.ucs_symbol(decomposed_code_point)))
                translit_file.write('"')
            translit_file.write('\n')
        elif 'LIGATURE' in name and 'ARABIC' not in name:
            decomposed_code_points = special_ligature_decompose(code_point)
            if decomposed_code_points[0] != code_point:
                translit_file.write('% {:s}\n'.format(name))
                translit_file.write('{:s} '.format(
                    unicode_utils.ucs_symbol(code_point)))
                translit_file.write('"')
                for decomposed_code_point in decomposed_code_points:
                    translit_file.write('{:s}'.format(
                        unicode_utils.ucs_symbol(decomposed_code_point)))
                translit_file.write('"')
                translit_file.write('\n')
            else:
                print('Warning: unhandled ligature: {:x} {:s}'.format(
                    code_point, name))
    translit_file.write('\n')

if __name__ == "__main__":
    PARSER = argparse.ArgumentParser(
        description='''
        Generate a translit_compat file from UnicodeData.txt.
        ''')
    PARSER.add_argument(
        '-u', '--unicode_data_file',
        nargs='?',
        type=str,
        default='UnicodeData.txt',
        help=('The UnicodeData.txt file to read, '
              + 'default: %(default)s'))
    PARSER.add_argument(
        '-i', '--input_file',
        nargs='?',
        type=str,
        help=''' The original glibc/localedata/locales/translit_compat
        file.''')
    PARSER.add_argument(
        '-o', '--output_file',
        nargs='?',
        type=str,
        default='translit_compat.new',
        help='''The new translit_compat file, default: %(default)s.  If the
        original glibc/localedata/locales/translit_compat file has
        been given as an option, the header up to the
        “translit_start” line and the tail from the “translit_end”
        line to the end of the file will be copied unchanged into the
        output file.  ''')
    PARSER.add_argument(
        '--unicode_version',
        nargs='?',
        required=True,
        type=str,
        help='The Unicode version of the input files used.')
    ARGS = PARSER.parse_args()

    unicode_utils.fill_attributes(ARGS.unicode_data_file)
    HEAD = TAIL = ''
    if ARGS.input_file:
        (HEAD, TAIL) = read_input_file(ARGS.input_file)
    with open(ARGS.output_file, mode='w') as TRANSLIT_FILE:
        output_head(TRANSLIT_FILE, ARGS.unicode_version, head=HEAD)
        output_transliteration(TRANSLIT_FILE)
        output_tail(TRANSLIT_FILE, tail=TAIL)