Update.

2000-01-01  Ulrich Drepper  <drepper@cygnus.com>

	* locale/programs/ld-ctype.c: Implement rest of transliteration
	definition parsing.
	* locale/programs/locfile-kw.gperf: New keyword translit_ignore.
	* locale/programs/locfile-token.h: Add tok_translit_ignore.

4 files changed, 423 insertions, 227 deletions

diff --git a/locale/programs/ld-ctype.c b/locale/programs/ld-ctype.c
index bfaf6c7d09..231df137ea 100644
--- a/locale/programs/ld-ctype.c
+++ b/locale/programs/ld-ctype.c
@@ -85,11 +85,25 @@ struct translit_t
 {
   uint32_t *from;
 
+  const char *fname;
+  size_t lineno;
+
   struct translit_to_t *to;
 
   struct translit_t *next;
 };
 
+struct translit_ignore_t
+{
+  uint32_t from;
+  uint32_t to;
+
+  const char *fname;
+  size_t lineno;
+
+  struct translit_ignore_t *next;
+};
+
 
 /* The real definition of the struct for the LC_CTYPE locale.  */
 struct locale_ctype_t
@@ -138,6 +152,11 @@ struct locale_ctype_t
   const char *translit_copy_locale;
   const char *translit_copy_repertoire;
   struct translit_t *translit;
+  struct translit_ignore_t *translit_ignore;
+
+  uint32_t *default_missing;
+  const char *default_missing_file;
+  size_t default_missing_lineno;
 
   /* The arrays for the binary representation.  */
   uint32_t plane_size;
@@ -162,7 +181,7 @@ struct locale_ctype_t
   size_t translit_from_tbl_size;
   size_t translit_to_tbl_size;
 
-  struct obstack mem_pool;
+  struct obstack mempool;
 };
 
 
@@ -282,7 +301,7 @@ ctype_startup (struct linereader *lr, struct localedef_t *locale,
 	  ctype->map256_collection[1][cnt] = cnt;
 	}
 
-      obstack_init (&ctype->mem_pool);
+      obstack_init (&ctype->mempool);
     }
 }
 
@@ -1537,7 +1556,7 @@ read_widestring (struct linereader *ldfile, struct token *now,
   else if (now->tok == tok_bsymbol)
     {
       /* Get the value from the repertoire.  */
-      wstr = xmalloc (2 * sizeof (uint32_t));
+      wstr = (uint32_t *) xmalloc (2 * sizeof (uint32_t));
       wstr[0] = repertoire_find_value (repertoire, now->val.str.startmb,
 				       now->val.str.lenmb);
       if (wstr[0] == ILLEGAL_CHAR_VALUE)
@@ -1548,7 +1567,7 @@ read_widestring (struct linereader *ldfile, struct token *now,
     }
   else if (now->tok == tok_ucs4)
     {
-      wstr = xmalloc (2 * sizeof (uint32_t));
+      wstr = (uint32_t *) xmalloc (2 * sizeof (uint32_t));
       wstr[0] = now->val.ucs4;
       wstr[1] = 0;
     }
@@ -1570,14 +1589,14 @@ read_widestring (struct linereader *ldfile, struct token *now,
 	/* We cannot proceed, we don't know the UCS4 value.  */
 	return NULL;
 
-      wstr = xmalloc (2 * sizeof (uint32_t));
+      wstr = (uint32_t *) xmalloc (2 * sizeof (uint32_t));
       wstr[0] = seq->ucs4;
       wstr[1] = 0;
     }
   else if (now->tok == tok_string)
     {
       wstr = now->val.str.startwc;
-      if (wstr[0] == 0)
+      if (wstr == NULL || wstr[0] == 0)
 	return NULL;
     }
   else
@@ -1600,7 +1619,7 @@ read_translit_entry (struct linereader *ldfile, struct locale_ctype_t *ctype,
   uint32_t *from_wstr = read_widestring (ldfile, now, charmap, repertoire);
   struct translit_t *result;
   struct translit_to_t **top;
-  struct obstack *ob = &ctype->mem_pool;
+  struct obstack *ob = &ctype->mempool;
   int first;
   int ignore;
 
@@ -1611,6 +1630,8 @@ read_translit_entry (struct linereader *ldfile, struct locale_ctype_t *ctype,
   result = (struct translit_t *) obstack_alloc (ob,
 						sizeof (struct translit_t));
   result->from = from_wstr;
+  result->fname = ldfile->fname;
+  result->lineno = ldfile->lineno;
   result->next = NULL;
   result->to = NULL;
   top = &result->to;
@@ -1673,6 +1694,129 @@ read_translit_entry (struct linereader *ldfile, struct locale_ctype_t *ctype,
 }
 
 
+static void
+read_translit_ignore_entry (struct linereader *ldfile,
+			    struct locale_ctype_t *ctype,
+			    struct charmap_t *charmap,
+			    struct repertoire_t *repertoire)
+{
+  /* We expect a semicolon-separated list of characters we ignore.  We are
+     only interested in the wide character definitions.  These must be
+     single characters, possibly defining a range when an ellipsis is used.  */
+  while (1)
+    {
+      struct token *now = lr_token (ldfile, charmap, repertoire);
+      struct translit_ignore_t *newp;
+      uint32_t from;
+
+      if (now->tok == tok_eol || now->tok == tok_eof)
+	{
+	  lr_error (ldfile,
+		    _("premature end of `translit_ignore' definition"));
+	  return;
+	}
+
+      if (now->tok != tok_bsymbol && now->tok != tok_ucs4)
+	{
+	  lr_error (ldfile, _("syntax error"));
+	  lr_ignore_rest (ldfile, 0);
+	  return;
+	}
+
+      if (now->tok == tok_ucs4)
+	from = now->val.ucs4;
+      else
+	{
+	  /* Try to get the value.  */
+	  from = repertoire_find_value (repertoire, now->val.str.startmb,
+					now->val.str.lenmb);
+	}
+
+      if (from == ILLEGAL_CHAR_VALUE)
+	{
+	  lr_error (ldfile, "invalid character name");
+	  newp = NULL;
+	}
+      else
+	{
+	  newp = (struct translit_ignore_t *)
+	    obstack_alloc (&ctype->mempool, sizeof (struct translit_ignore_t));
+	  newp->from = from;
+	  newp->to = from;
+
+	  newp->next = ctype->translit_ignore;
+	  ctype->translit_ignore = newp;
+	}
+
+      /* Now we expect either a semicolon, an ellipsis, or the end of the
+	 line.  */
+      now = lr_token (ldfile, charmap, repertoire);
+
+      if (now->tok == tok_ellipsis2)
+	{
+	  /* XXX Should we bother implementing `....'?  `...' certainly
+	     will not be implemented.  */
+	  uint32_t to;
+
+	  now = lr_token (ldfile, charmap, repertoire);
+
+	  if (now->tok == tok_eol || now->tok == tok_eof)
+	    {
+	      lr_error (ldfile,
+			_("premature end of `translit_ignore' definition"));
+	      return;
+	    }
+
+	  if (now->tok != tok_bsymbol && now->tok != tok_ucs4)
+	    {
+	      lr_error (ldfile, _("syntax error"));
+	      lr_ignore_rest (ldfile, 0);
+	      return;
+	    }
+
+	  if (now->tok == tok_ucs4)
+	    to = now->val.ucs4;
+	  else
+	    {
+	      /* Try to get the value.  */
+	      to = repertoire_find_value (repertoire, now->val.str.startmb,
+					  now->val.str.lenmb);
+	    }
+
+	  if (to == ILLEGAL_CHAR_VALUE)
+	    lr_error (ldfile, "invalid character name");
+	  else
+	    {
+	      /* Make sure the `to'-value is larger.  */
+	      if (to >= from)
+		newp->to = to;
+	      else
+		lr_error (ldfile, _("\
+to-value <U%0*X> of range is smaller than from-value <U%0*X>"),
+			  (to | from) < 65536 ? 4 : 8, to,
+			  (to | from) < 65536 ? 4 : 8, from);
+	    }
+
+	  /* And the next token.  */
+	  now = lr_token (ldfile, charmap, repertoire);
+	}
+
+      if (now->tok == tok_eol || now->tok == tok_eof)
+	/* We are done.  */
+	return;
+
+      if (now->tok == tok_semicolon)
+	/* Next round.  */
+	continue;
+
+      /* If we come here something is wrong.  */
+      lr_error (ldfile, _("syntax error"));
+      lr_ignore_rest (ldfile, 0);
+      return;
+    }
+}
+
+
 /* The parser for the LC_CTYPE section of the locale definition.  */
 void
 ctype_read (struct linereader *ldfile, struct localedef_t *result,
@@ -2257,6 +2401,45 @@ with character code range values one must use the absolute ellipsis `...'"));
 
 		  /* The rest of the line must be empty.  */
 		  lr_ignore_rest (ldfile, 1);
+
+		  /* Make sure the locale is read.  */
+		  add_to_readlist (LC_CTYPE, ctype->translit_copy_locale,
+				   repertoire_name, 1);
+		  continue;
+		}
+	      else if (now->tok == tok_default_missing)
+		{
+		  uint32_t *wstr;
+
+		  /* We expect a single character or string as the
+		     argument.  */
+		  now = lr_token (ldfile, charmap, NULL);
+		  wstr = read_widestring (ldfile, now, charmap, repertoire);
+
+		  if (wstr != NULL)
+		    {
+		      if (ctype->default_missing != NULL)
+			{
+			  lr_error (ldfile, _("\
+%s: duplicate `default_missing' definition"), "LC_CTYPE");
+			  error_at_line (0, 0, ctype->default_missing_file,
+					 ctype->default_missing_lineno,
+					 _("previous definition was here"));
+			}
+		      else
+			{
+			  ctype->default_missing = wstr;
+			  ctype->default_missing_file = ldfile->fname;
+			  ctype->default_missing_lineno = ldfile->lineno;
+			}
+		    }
+		  lr_ignore_rest (ldfile, 1);
+		  continue;
+		}
+	      else if (now->tok == tok_translit_ignore)
+		{
+		  read_translit_ignore_entry (ldfile, ctype, charmap,
+					      repertoire);
 		  continue;
 		}
 
diff --git a/locale/programs/locfile-kw.gperf b/locale/programs/locfile-kw.gperf
index 91c20891f1..3670548a4c 100644
--- a/locale/programs/locfile-kw.gperf
+++ b/locale/programs/locfile-kw.gperf
@@ -52,6 +52,7 @@ tolower,                tok_tolower,                0
 map,                    tok_map,                    0
 translit_start,         tok_translit_start,         0
 translit_end,           tok_translit_end,           0
+translit_ignore,        tok_translit_ignore,        0
 default_missing,        tok_default_missing,        0
 LC_COLLATE,             tok_lc_collate,             0
 coll_weight_max,        tok_coll_weight_max,        0
diff --git a/locale/programs/locfile-kw.h b/locale/programs/locfile-kw.h
index 811234b25d..a04bc2723c 100644
--- a/locale/programs/locfile-kw.h
+++ b/locale/programs/locfile-kw.h
@@ -24,12 +24,12 @@
 #include "locfile-token.h"
 struct keyword_t ;
 
-#define TOTAL_KEYWORDS 172
+#define TOTAL_KEYWORDS 173
 #define MIN_WORD_LENGTH 3
 #define MAX_WORD_LENGTH 22
 #define MIN_HASH_VALUE 3
-#define MAX_HASH_VALUE 545
-/* maximum key range = 543, duplicates = 0 */
+#define MAX_HASH_VALUE 645
+/* maximum key range = 643, duplicates = 0 */
 
 #ifdef __GNUC__
 __inline
@@ -39,32 +39,32 @@ hash (register const char *str, register unsigned int len)
 {
   static const unsigned short asso_values[] =
     {
-      546, 546, 546, 546, 546, 546, 546, 546, 546, 546,
-      546, 546, 546, 546, 546, 546, 546, 546, 546, 546,
-      546, 546, 546, 546, 546, 546, 546, 546, 546, 546,
-      546, 546, 546, 546, 546, 546, 546, 546, 546, 546,
-      546, 546, 546, 546, 546, 546, 546, 546, 546, 546,
-        5,   0, 546, 546, 546, 546, 546, 546, 546, 546,
-      546, 546, 546, 546, 546,  20, 546,   0,   0,   0,
-        5,  30,   0,   0, 546, 546,   0, 546,   0,   0,
-      546, 546,  10,   0,   5,  10, 546, 546, 546,   0,
-      546, 546, 546, 546, 546,  30, 546,   0,  10, 125,
-        5,   0, 105,  30,   5,  95, 546,   0, 105, 155,
-      135,  50,  75,   0,   5,  45,   0,  55,   0,  30,
-       25,  25,  10, 546, 546, 546, 546, 546, 546, 546,
-      546, 546, 546, 546, 546, 546, 546, 546, 546, 546,
-      546, 546, 546, 546, 546, 546, 546, 546, 546, 546,
-      546, 546, 546, 546, 546, 546, 546, 546, 546, 546,
-      546, 546, 546, 546, 546, 546, 546, 546, 546, 546,
-      546, 546, 546, 546, 546, 546, 546, 546, 546, 546,
-      546, 546, 546, 546, 546, 546, 546, 546, 546, 546,
-      546, 546, 546, 546, 546, 546, 546, 546, 546, 546,
-      546, 546, 546, 546, 546, 546, 546, 546, 546, 546,
-      546, 546, 546, 546, 546, 546, 546, 546, 546, 546,
-      546, 546, 546, 546, 546, 546, 546, 546, 546, 546,
-      546, 546, 546, 546, 546, 546, 546, 546, 546, 546,
-      546, 546, 546, 546, 546, 546, 546, 546, 546, 546,
-      546, 546, 546, 546, 546, 546
+      646, 646, 646, 646, 646, 646, 646, 646, 646, 646,
+      646, 646, 646, 646, 646, 646, 646, 646, 646, 646,
+      646, 646, 646, 646, 646, 646, 646, 646, 646, 646,
+      646, 646, 646, 646, 646, 646, 646, 646, 646, 646,
+      646, 646, 646, 646, 646, 646, 646, 646, 646, 646,
+       10,   0, 646, 646, 646, 646, 646, 646, 646, 646,
+      646, 646, 646, 646, 646,  15, 646,   0,   0,   0,
+       10,  20,   0,   0, 646, 646,   0, 646,   0,   5,
+      646, 646,  10,   0,  10,   5, 646, 646, 646,  20,
+      646, 646, 646, 646, 646,  30, 646,   0,  25, 110,
+        5,   0,  25,   0,   5, 105, 646,  20, 125,  85,
+      175,   0,  75,  10,   5, 125,   0, 135,  45,  15,
+       10,  75,   0, 646, 646, 646, 646, 646, 646, 646,
+      646, 646, 646, 646, 646, 646, 646, 646, 646, 646,
+      646, 646, 646, 646, 646, 646, 646, 646, 646, 646,
+      646, 646, 646, 646, 646, 646, 646, 646, 646, 646,
+      646, 646, 646, 646, 646, 646, 646, 646, 646, 646,
+      646, 646, 646, 646, 646, 646, 646, 646, 646, 646,
+      646, 646, 646, 646, 646, 646, 646, 646, 646, 646,
+      646, 646, 646, 646, 646, 646, 646, 646, 646, 646,
+      646, 646, 646, 646, 646, 646, 646, 646, 646, 646,
+      646, 646, 646, 646, 646, 646, 646, 646, 646, 646,
+      646, 646, 646, 646, 646, 646, 646, 646, 646, 646,
+      646, 646, 646, 646, 646, 646, 646, 646, 646, 646,
+      646, 646, 646, 646, 646, 646, 646, 646, 646, 646,
+      646, 646, 646, 646, 646, 646
     };
   register int hval = len;
 
@@ -106,285 +106,296 @@ locfile_hash (register const char *str, register unsigned int len)
       {"LC_ADDRESS",             tok_lc_address,             0},
       {""},
       {"LC_TELEPHONE",           tok_lc_telephone,           0},
-      {"LC_CTYPE",               tok_lc_ctype,               0},
+      {""},
       {"era_t_fmt",              tok_era_t_fmt,              0},
-      {"LC_COLLATE",             tok_lc_collate,             0},
+      {"LC_NUMERIC",             tok_lc_numeric,             0},
       {"height",                 tok_height,                 0},
       {"LC_IDENTIFICATION",      tok_lc_identification,      0},
-      {""},
+      {"LC_CTYPE",               tok_lc_ctype,               0},
       {"era_d_fmt",              tok_era_d_fmt,              0},
-      {"LC_NUMERIC",             tok_lc_numeric,             0},
-      {""}, {""}, {""},
-      {"UNDEFINED",              tok_undefined,              0},
+      {"graph",                  tok_graph,                  0},
+      {"order_start",            tok_order_start,            0},
+      {"LC_NAME",                tok_lc_name,                0},
       {""},
+      {"UNDEFINED",              tok_undefined,              0},
+      {"LC_COLLATE",             tok_lc_collate,             0},
       {"reorder-end",            tok_reorder_end,            0},
-      {"LC_NAME",                tok_lc_name,                0},
+      {"tolower",                tok_tolower,                0},
       {"reorder-after",          tok_reorder_after,          0},
-      {"LC_MEASUREMENT",         tok_lc_measurement,         0},
-      {""},
-      {"LC_MONETARY",            tok_lc_monetary,            0},
+      {"order_end",              tok_order_end,              0},
       {""},
-      {"day",                    tok_day,                    0},
-      {"week",                   tok_week,                   0},
-      {"t_fmt",                  tok_t_fmt,                  0},
-      {"yesstr",                 tok_yesstr,                 0},
+      {"LC_MESSAGES",            tok_lc_messages,            0},
       {""},
       {"LC_PAPER",               tok_lc_paper,               0},
-      {""},
+      {"LC_MEASUREMENT",         tok_lc_measurement,         0},
+      {"t_fmt",                  tok_t_fmt,                  0},
+      {"IGNORE",                 tok_ignore,                 0},
+      {"forward",                tok_forward,                0},
+      {"fax",                    tok_fax,                    0},
+      {"week",                   tok_week,                   0},
       {"d_fmt",                  tok_d_fmt,                  0},
-      {"LC_MESSAGES",            tok_lc_messages,            0},
+      {"tel_dom_fmt",            tok_tel_dom_fmt,            0},
+      {""}, {""}, {""}, {""},
+      {"era_d_t_fmt",            tok_era_d_t_fmt,            0},
+      {""}, {""}, {""}, {""},
+      {"LC_MONETARY",            tok_lc_monetary,            0},
       {""},
-      {"era_year",               tok_era_year,               0},
-      {""}, {""},
-      {"IGNORE",                 tok_ignore,                 0},
-      {""}, {""}, {""},
-      {"graph",                  tok_graph,                  0},
-      {""}, {""},
       {"backward",               tok_backward,               0},
-      {""}, {""}, {""},
-      {"address",                tok_address,                0},
+      {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
       {""}, {""}, {""}, {""},
-      {"yesexpr",                tok_yesexpr,                0},
-      {"audience",               tok_audience,               0},
-      {""},
-      {"abday",                  tok_abday,                  0},
-      {""}, {""}, {""}, {""}, {""},
-      {"order_start",            tok_order_start,            0},
-      {""}, {""}, {""}, {""}, {""}, {""}, {""},
-      {"order_end",              tok_order_end,              0},
-      {"reorder-sections-end",   tok_reorder_sections_end,   0},
+      {"d_t_fmt",                tok_d_t_fmt,                0},
+      {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
+      {""}, {""}, {""}, {""}, {""}, {""},
+      {"day",                    tok_day,                    0},
       {""},
-      {"reorder-sections-after", tok_reorder_sections_after, 0},
-      {""}, {""},
       {"print",                  tok_print,                  0},
+      {"yesstr",                 tok_yesstr,                 0},
+      {"toupper",                tok_toupper,                0},
+      {"grouping",               tok_grouping,               0},
+      {""}, {""}, {""}, {""},
+      {"era_year",               tok_era_year,               0},
+      {""}, {""}, {""},
+      {"yesexpr",                tok_yesexpr,                0},
+      {""}, {""}, {""}, {""}, {""},
+      {"repertoiremap",          tok_repertoiremap,          0},
       {""}, {""}, {""}, {""}, {""}, {""},
-      {"tolower",                tok_tolower,                0},
-      {""},
-      {"translit_start",         tok_translit_start,         0},
-      {""}, {""},
-      {"translit_end",           tok_translit_end,           0},
-      {""}, {""},
       {"title",                  tok_title,                  0},
       {""}, {""},
-      {"repertoiremap",          tok_repertoiremap,          0},
+      {"timezone",               tok_timezone,               0},
       {""},
       {"digit",                  tok_digit,                  0},
-      {""}, {""},
+      {""},
+      {"contact",                tok_contact,                0},
+      {""},
+      {"from",                   tok_from,                   0},
+      {""},
+      {"country_ab3",            tok_country_ab3,            0},
+      {""}, {""}, {""},
+      {"conversion_rate",        tok_conversion_rate,        0},
+      {"xdigit",                 tok_xdigit,                 0},
+      {"collating-element",      tok_collating_element,      0},
       {"tel",                    tok_tel,                    0},
       {"else",                   tok_else,                   0},
       {"alpha",                  tok_alpha,                  0},
-      {""}, {""},
-      {"timezone",               tok_timezone,               0},
-      {""}, {""}, {""}, {""}, {""}, {""},
-      {"blank",                  tok_blank,                  0},
-      {"tel_dom_fmt",            tok_tel_dom_fmt,            0},
-      {""}, {""}, {""},
-      {"space",                  tok_space,                  0},
-      {"era_d_t_fmt",            tok_era_d_t_fmt,            0},
-      {"duo_valid_to",           tok_duo_valid_to,           0},
+      {"country_ab2",            tok_country_ab2,            0},
       {""}, {""}, {""},
-      {"xdigit",                 tok_xdigit,                 0},
+      {"width",                  tok_width,                  0},
       {""},
-      {"fax",                    tok_fax,                    0},
+      {"address",                tok_address,                0},
+      {""}, {""},
+      {"lower",                  tok_lower,                  0},
+      {"tel_int_fmt",            tok_tel_int_fmt,            0},
       {""},
-      {"punct",                  tok_punct,                  0},
+      {"audience",               tok_audience,               0},
+      {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
       {""},
-      {"toupper",                tok_toupper,                0},
-      {"symbol-equivalence",     tok_symbol_equivalence,     0},
+      {"elif",                   tok_elif,                   0},
+      {""}, {""}, {""}, {""}, {""},
+      {"reorder-sections-end",   tok_reorder_sections_end,   0},
       {""},
-      {"width",                  tok_width,                  0},
-      {"escape_char",            tok_escape_char,            0},
-      {""}, {""},
+      {"reorder-sections-after", tok_reorder_sections_after, 0},
+      {"map",                    tok_map,                    0},
       {"lang_name",              tok_lang_name,              0},
-      {"upper",                  tok_upper,                  0},
+      {"coll_weight_max",        tok_coll_weight_max,        0},
+      {""}, {""}, {""}, {""},
+      {"postal_fmt",             tok_postal_fmt,             0},
+      {""},
+      {"mon_grouping",           tok_mon_grouping,           0},
+      {""},
+      {"translit_start",         tok_translit_start,         0},
+      {"translit_ignore",        tok_translit_ignore,        0},
+      {""},
+      {"translit_end",           tok_translit_end,           0},
+      {"decimal_point",          tok_decimal_point,          0},
+      {""},
+      {"abday",                  tok_abday,                  0},
+      {""}, {""}, {""}, {""},
+      {"ifdef",                  tok_ifdef,                  0},
       {"define",                 tok_define,                 0},
-      {"d_t_fmt",                tok_d_t_fmt,                0},
-      {"grouping",               tok_grouping,               0},
-      {""}, {""}, {""},
       {"lang_ab",                tok_lang_ab,                0},
       {"lang_lib",               tok_lang_lib,               0},
-      {"territory",              tok_territory,              0},
-      {""}, {""},
-      {"abbreviation",           tok_abbreviation,           0},
-      {""}, {""}, {""}, {""}, {""}, {""}, {""},
-      {"alt_digits",             tok_alt_digits,             0},
+      {"copy",                   tok_copy,                   0},
+      {"nostr",                  tok_nostr,                  0},
       {""},
-      {"forward",                tok_forward,                0},
-      {"language",               tok_language,               0},
+      {"mon_decimal_point",      tok_mon_decimal_point,      0},
+      {"category",               tok_category,               0},
       {""},
-      {"lower",                  tok_lower,                  0},
+      {"blank",                  tok_blank,                  0},
+      {""},
+      {"country_post",           tok_country_post,           0},
       {""}, {""},
+      {"t_fmt_ampm",             tok_t_fmt_ampm,             0},
+      {""},
+      {"duo_valid_to",           tok_duo_valid_to,           0},
+      {""}, {""},
+      {"space",                  tok_space,                  0},
+      {""}, {""}, {""}, {""}, {""}, {""},
+      {"abbreviation",           tok_abbreviation,           0},
       {"name_fmt",               tok_name_fmt,               0},
-      {""}, {""}, {""},
+      {""},
+      {"punct",                  tok_punct,                  0},
+      {""},
       {"name_mr",                tok_name_mr,                0},
-      {""}, {""}, {""}, {""},
-      {"contact",                tok_contact,                0},
-      {"thousands_sep",          tok_thousands_sep,          0},
+      {"first_weekday",          tok_first_weekday,          0},
       {""}, {""},
-      {"country_ab3",            tok_country_ab3,            0},
+      {"escape_char",            tok_escape_char,            0},
       {""},
-      {"category",               tok_category,               0},
+      {"first_workday",          tok_first_workday,          0},
+      {"p_sep_by_space",         tok_p_sep_by_space,         0},
+      {"upper",                  tok_upper,                  0},
       {""}, {""},
-      {"country_ab2",            tok_country_ab2,            0},
+      {"symbol-equivalence",     tok_symbol_equivalence,     0},
       {""},
-      {"revision",               tok_revision,               0},
+      {"endif",                  tok_endif,                  0},
+      {"measurement",            tok_measurement,            0},
+      {""}, {""}, {""}, {""},
+      {"country_car",            tok_country_car,            0},
+      {"comment_char",           tok_comment_char,           0},
+      {""}, {""},
+      {"default_missing",        tok_default_missing,        0},
+      {"source",                 tok_source,                 0},
       {""}, {""}, {""}, {""}, {""}, {""},
-      {"nostr",                  tok_nostr,                  0},
-      {""}, {""}, {""},
-      {"copy",                   tok_copy,                   0},
-      {""}, {""}, {""},
       {"outdigit",               tok_outdigit,               0},
       {""}, {""},
-      {"tel_int_fmt",            tok_tel_int_fmt,            0},
-      {""}, {""},
-      {"elif",                   tok_elif,                   0},
-      {""}, {""},
-      {"name_ms",                tok_name_ms,                0},
-      {"name_mrs",               tok_name_mrs,               0},
-      {""}, {""},
-      {"measurement",            tok_measurement,            0},
-      {"collating-element",      tok_collating_element,      0},
+      {"collating-symbol",       tok_collating_symbol,       0},
+      {""}, {""}, {""}, {""}, {""}, {""},
+      {"position",               tok_position,               0},
       {""},
-      {"p_sep_by_space",         tok_p_sep_by_space,         0},
+      {"am_pm",                  tok_am_pm,                  0},
+      {"noexpr",                 tok_noexpr,                 0},
+      {""},
+      {"mon",                    tok_mon,                    0},
+      {"territory",              tok_territory,              0},
+      {"alt_digits",             tok_alt_digits,             0},
+      {""}, {""},
+      {"language",               tok_language,               0},
+      {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
+      {"charconv",               tok_charconv,               0},
+      {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
       {""},
-      {"source",                 tok_source,                 0},
-      {"duo_p_cs_precedes",      tok_duo_p_cs_precedes,      0},
-      {"duo_p_sep_by_space",     tok_duo_p_sep_by_space,     0},
-      {""}, {""}, {""}, {""},
-      {"map",                    tok_map,                    0},
       {"duo_valid_from",         tok_duo_valid_from,         0},
-      {""}, {""}, {""},
-      {"first_weekday",          tok_first_weekday,          0},
       {""},
-      {"conversion_rate",        tok_conversion_rate,        0},
-      {""}, {""},
-      {"first_workday",          tok_first_workday,          0},
-      {""}, {""}, {""}, {""},
-      {"decimal_point",          tok_decimal_point,          0},
-      {""}, {""}, {""},
-      {"duo_int_p_sep_by_space", tok_duo_int_p_sep_by_space, 0},
-      {""}, {""},
-      {"duo_frac_digits",        tok_duo_frac_digits,        0},
+      {"p_sign_posn",            tok_p_sign_posn,            0},
       {""},
-      {"uno_valid_to",           tok_uno_valid_to,           0},
+      {"positive_sign",          tok_positive_sign,          0},
+      {""}, {""}, {""},
+      {"country_name",           tok_country_name,           0},
       {""}, {""},
-      {"default_missing",        tok_default_missing,        0},
+      {"alnum",                  tok_alnum,                  0},
+      {"frac_digits",            tok_frac_digits,            0},
+      {"mon_thousands_sep",      tok_mon_thousands_sep,      0},
+      {""}, {""}, {""}, {""}, {""},
+      {"duo_p_sep_by_space",     tok_duo_p_sep_by_space,     0},
+      {""}, {""}, {""}, {""},
+      {"revision",               tok_revision,               0},
+      {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
       {""},
-      {"country_post",           tok_country_post,           0},
-      {"charconv",               tok_charconv,               0},
-      {"name_miss",              tok_name_miss,              0},
-      {""}, {""}, {""},
-      {"position",               tok_position,               0},
-      {"from",                   tok_from,                   0},
-      {"t_fmt_ampm",             tok_t_fmt_ampm,             0},
-      {"noexpr",                 tok_noexpr,                 0},
-      {""}, {""}, {""},
-      {"coll_weight_max",        tok_coll_weight_max,        0},
-      {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
       {"n_sep_by_space",         tok_n_sep_by_space,         0},
-      {"abmon",                  tok_abmon,                  0},
+      {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
+      {"lang_term",              tok_lang_term,              0},
+      {"duo_frac_digits",        tok_duo_frac_digits,        0},
       {""},
-      {"duo_n_cs_precedes",      tok_duo_n_cs_precedes,      0},
-      {"duo_n_sep_by_space",     tok_duo_n_sep_by_space,     0},
+      {"name_ms",                tok_name_ms,                0},
+      {"name_mrs",               tok_name_mrs,               0},
       {""},
-      {"postal_fmt",             tok_postal_fmt,             0},
-      {"frac_digits",            tok_frac_digits,            0},
-      {"include",                tok_include,                0},
+      {"email",                  tok_email,                  0},
+      {""},
+      {"duo_int_p_sep_by_space", tok_duo_int_p_sep_by_space, 0},
+      {"thousands_sep",          tok_thousands_sep,          0},
+      {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
+      {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
       {""}, {""}, {""},
-      {"duo_int_p_cs_precedes",  tok_duo_int_p_cs_precedes,  0},
+      {"undef",                  tok_undef,                  0},
       {""}, {""}, {""}, {""}, {""}, {""},
-      {"positive_sign",          tok_positive_sign,          0},
-      {"section-symbol",         tok_section_symbol,         0},
+      {"uno_valid_to",           tok_uno_valid_to,           0},
+      {""}, {""}, {""}, {""}, {""}, {""}, {""},
+      {"abmon",                  tok_abmon,                  0},
+      {"country_num",            tok_country_num,            0},
+      {""},
+      {"p_cs_precedes",          tok_p_cs_precedes,          0},
       {""}, {""}, {""},
+      {"duo_p_cs_precedes",      tok_duo_p_cs_precedes,      0},
       {"name_gen",               tok_name_gen,               0},
-      {"duo_currency_symbol",    tok_duo_currency_symbol,    0},
       {""}, {""},
-      {"duo_int_n_sep_by_space", tok_duo_int_n_sep_by_space, 0},
-      {"negative_sign",          tok_negative_sign,          0},
+      {"n_sign_posn",            tok_n_sign_posn,            0},
       {""},
+      {"negative_sign",          tok_negative_sign,          0},
+      {"duo_currency_symbol",    tok_duo_currency_symbol,    0},
+      {""}, {""}, {""}, {""}, {""}, {""}, {""},
+      {"country_isbn",           tok_country_isbn,           0},
+      {""}, {""},
       {"duo_p_sign_posn",        tok_duo_p_sign_posn,        0},
-      {"country_car",            tok_country_car,            0},
-      {"comment_char",           tok_comment_char,           0},
-      {"p_cs_precedes",          tok_p_cs_precedes,          0},
-      {""}, {""}, {""},
-      {"country_name",           tok_country_name,           0},
-      {""},
+      {""}, {""},
+      {"duo_n_sep_by_space",     tok_duo_n_sep_by_space,     0},
+      {""}, {""}, {""}, {""},
+      {"cal_direction",          tok_cal_direction,          0},
       {"duo_int_frac_digits",    tok_duo_int_frac_digits,    0},
-      {"class",                  tok_class,                  0},
-      {"collating-symbol",       tok_collating_symbol,       0},
-      {""}, {""}, {""},
       {"currency_symbol",        tok_currency_symbol,        0},
-      {"p_sign_posn",            tok_p_sign_posn,            0},
-      {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
-      {""},
-      {"mon_thousands_sep",      tok_mon_thousands_sep,      0},
-      {"mon",                    tok_mon,                    0},
       {""}, {""}, {""}, {""}, {""}, {""},
-      {"endif",                  tok_endif,                  0},
-      {""},
-      {"mon_grouping",           tok_mon_grouping,           0},
+      {"include",                tok_include,                0},
+      {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
+      {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
       {""},
-      {"charclass",              tok_charclass,              0},
+      {"duo_int_n_sep_by_space", tok_duo_int_n_sep_by_space, 0},
+      {""}, {""}, {""}, {""}, {""},
+      {"int_p_sep_by_space",     tok_int_p_sep_by_space,     0},
+      {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
       {""},
-      {"duo_int_n_cs_precedes",  tok_duo_int_n_cs_precedes,  0},
-      {""}, {""}, {""}, {""}, {""}, {""}, {""},
       {"uno_valid_from",         tok_uno_valid_from,         0},
-      {""}, {""}, {""}, {""}, {""},
-      {"email",                  tok_email,                  0},
       {""}, {""}, {""}, {""},
-      {"duo_n_sign_posn",        tok_duo_n_sign_posn,        0},
-      {""}, {""},
-      {"n_cs_precedes",          tok_n_cs_precedes,          0},
-      {""}, {""}, {""},
-      {"mon_decimal_point",      tok_mon_decimal_point,      0},
+      {"name_miss",              tok_name_miss,              0},
       {""},
-      {"duo_int_p_sign_posn",    tok_duo_int_p_sign_posn,    0},
-      {""}, {""}, {""}, {""}, {""}, {""},
-      {"n_sign_posn",            tok_n_sign_posn,            0},
-      {""}, {""}, {""}, {""}, {""},
-      {"int_p_cs_precedes",      tok_int_p_cs_precedes,      0},
-      {"int_p_sep_by_space",     tok_int_p_sep_by_space,     0},
-      {""}, {""}, {""}, {""},
-      {"cal_direction",          tok_cal_direction,          0},
-      {"duo_int_curr_symbol",    tok_duo_int_curr_symbol,    0},
-      {"undef",                  tok_undef,                  0},
+      {"duo_int_p_cs_precedes",  tok_duo_int_p_cs_precedes,  0},
       {""}, {""}, {""}, {""},
-      {"int_select",             tok_int_select,             0},
       {"application",            tok_application,            0},
       {""}, {""}, {""},
-      {"ifdef",                  tok_ifdef,                  0},
-      {""},
-      {"country_isbn",           tok_country_isbn,           0},
-      {""}, {""},
-      {"alnum",                  tok_alnum,                  0},
-      {""}, {""}, {""}, {""},
       {"int_frac_digits",        tok_int_frac_digits,        0},
-      {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
+      {""}, {""}, {""}, {""},
       {"int_prefix",             tok_int_prefix,             0},
+      {""}, {""},
+      {"n_cs_precedes",          tok_n_cs_precedes,          0},
+      {"charclass",              tok_charclass,              0},
+      {""}, {""},
+      {"duo_n_cs_precedes",      tok_duo_n_cs_precedes,      0},
+      {""}, {""},
+      {"class",                  tok_class,                  0},
+      {""}, {""}, {""},
+      {"section-symbol",         tok_section_symbol,         0},
+      {""}, {""}, {""}, {""},
+      {"duo_int_curr_symbol",    tok_duo_int_curr_symbol,    0},
+      {""}, {""}, {""}, {""}, {""},
+      {"duo_n_sign_posn",        tok_duo_n_sign_posn,        0},
       {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
-      {"duo_int_n_sign_posn",    tok_duo_int_n_sign_posn,    0},
+      {"duo_int_p_sign_posn",    tok_duo_int_p_sign_posn,    0},
       {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
-      {"lang_term",              tok_lang_term,              0},
+      {""},
+      {"int_select",             tok_int_select,             0},
+      {""},
+      {"int_p_cs_precedes",      tok_int_p_cs_precedes,      0},
+      {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
+      {""}, {""}, {""},
+      {"cntrl",                  tok_cntrl,                  0},
+      {""}, {""}, {""}, {""},
+      {"int_p_sign_posn",        tok_int_p_sign_posn,        0},
       {""}, {""},
-      {"int_n_cs_precedes",      tok_int_n_cs_precedes,      0},
       {"int_n_sep_by_space",     tok_int_n_sep_by_space,     0},
       {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
       {""}, {""},
-      {"am_pm",                  tok_am_pm,                  0},
-      {""}, {""}, {""}, {""},
-      {"cntrl",                  tok_cntrl,                  0},
-      {"country_num",            tok_country_num,            0},
-      {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
-      {"int_p_sign_posn",        tok_int_p_sign_posn,        0},
+      {"int_curr_symbol",        tok_int_curr_symbol,        0},
+      {""}, {""}, {""}, {""}, {""},
+      {"duo_int_n_cs_precedes",  tok_duo_int_n_cs_precedes,  0},
       {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
       {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
-      {""},
-      {"int_curr_symbol",        tok_int_curr_symbol,        0},
       {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
       {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
       {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
+      {""}, {""},
+      {"duo_int_n_sign_posn",    tok_duo_int_n_sign_posn,    0},
       {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
       {""}, {""}, {""},
+      {"int_n_cs_precedes",      tok_int_n_cs_precedes,      0},
+      {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
+      {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""},
       {"int_n_sign_posn",        tok_int_n_sign_posn,        0}
     };
 
diff --git a/locale/programs/locfile-token.h b/locale/programs/locfile-token.h
index 97945f8f1a..e1cd5f7a74 100644
--- a/locale/programs/locfile-token.h
+++ b/locale/programs/locfile-token.h
@@ -88,6 +88,7 @@ enum token_t
   tok_map,
   tok_translit_start,
   tok_translit_end,
+  tok_translit_ignore,
   tok_default_missing,
   tok_lc_collate,
   tok_coll_weight_max,