42175 + 42177 + 42178: avoid localized output from external commands

_call_program and '_arguments --' will call _comp_locale before
calling external command for easier analysis of the output.
This is disabled by passing an option '-l'.

8 files changed, 62 insertions, 19 deletions

diff --git a/ChangeLog b/ChangeLog
index c2b726b21..eb93c5bef 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,13 @@
+2017-12-28  Jun-ichi Takimoto <takimoto-j@kba.biglobe.ne.jp>
+
+	* 42175 + 42177 (+ 24178 by Daniel):
+	Completion/Base/Utility/_arguments,
+	Completion/Base/Utility/_call_program,
+	Completion/Debian/Command/_aptitude, Completion/Unix/Command/_a2ps,
+	Completion/Unix/Command/_subversion, Completion/Unix/Command/_tar,
+	Doc/Zsh/compsys.yo: avoid localized output from external commands
+	for easier analysys by the completion system.
+
 2017-12-27  Jun-ichi Takimoto <takimoto-j@kba.biglobe.ne.jp>
 
 	* 42176: Completion/Unix/Command/_tar: update long options
diff --git a/Completion/Base/Utility/_arguments b/Completion/Base/Utility/_arguments
index d2c0d33de..136dd5826 100644
--- a/Completion/Base/Utility/_arguments
+++ b/Completion/Base/Utility/_arguments
@@ -43,7 +43,7 @@ if (( long )); then
   name="${name//[^a-zA-Z0-9_]/_}"
 
   if (( ! ${(P)+name} )); then
-    local iopts sopts pattern tmpo dir cur cache
+    local iopts sopts lflag pattern tmpo dir cur cache
     typeset -Ua lopts
 
     cache=()
@@ -55,7 +55,12 @@ if (( long )); then
 
     iopts=()
     sopts=()
-    while [[ "$1" = -[is]* ]]; do
+    while [[ "$1" = -[lis]* ]]; do
+      if [[ "$1" = -l ]]; then
+	lflag='-l'
+	shift
+	continue
+      fi
       if [[ "$1" = -??* ]]; then
         tmp="${1[3,-1]}"
         cur=1
@@ -88,7 +93,8 @@ if (( long )); then
     # option up to the end.
 
    tmp=()
-   _call_program options ${~words[1]} --help 2>&1 | while IFS= read -r opt; do
+   _call_program $lflag options ${~words[1]} --help 2>&1 |
+     while IFS= read -r opt; do
      if (( ${#tmp} )); then
        # Previous line had no comment.  Is the current one suitable?
        # It's hard to be sure, but if it there was nothing on the
diff --git a/Completion/Base/Utility/_call_program b/Completion/Base/Utility/_call_program
index 9a44f2d8e..73f3ef6d2 100644
--- a/Completion/Base/Utility/_call_program
+++ b/Completion/Base/Utility/_call_program
@@ -1,6 +1,6 @@
 #autoload +X
 
-local curcontext="${curcontext}" tmp err_fd=-1
+local curcontext="${curcontext}" tmp err_fd=-1 clocale='_comp_locale;'
 local -a prefix
 
 if [[ "$1" = -p ]]; then
@@ -10,6 +10,9 @@ if [[ "$1" = -p ]]; then
     zstyle -t ":completion:${curcontext}:${1}" gain-privileges &&
 	prefix=( $_comp_priv_prefix )
   fi
+elif [[ "$1" = -l ]]; then
+  shift
+  clocale=''
 fi
 
 if (( ${debug_fd:--1} > 2 )) || [[ ! -t 2 ]]
@@ -21,12 +24,12 @@ fi
 
 if zstyle -s ":completion:${curcontext}:${1}" command tmp; then
   if [[ "$tmp" = -* ]]; then
-    eval "$tmp[2,-1]" "$argv[2,-1]"
+    eval $clocale "$tmp[2,-1]" "$argv[2,-1]"
   else
-    eval $prefix "$tmp"
+    eval $clocale $prefix "$tmp"
   fi
 else
-  eval $prefix "$argv[2,-1]"
+  eval $clocale $prefix "$argv[2,-1]"
 fi 2>&$err_fd
 
 } always {
diff --git a/Completion/Debian/Command/_aptitude b/Completion/Debian/Command/_aptitude
index b2b54599f..f79a100a8 100644
--- a/Completion/Debian/Command/_aptitude
+++ b/Completion/Debian/Command/_aptitude
@@ -80,7 +80,7 @@ _arguments -C \
 
 case $state in
   cmds)
-    cmds=( ${${(M)${(f)"$(LC_ALL=C _call_program commands aptitude -h 2>/dev/null)"}:# [^- ][^ ]## *}/(#b) ([^ ]##) ##(- )#([^- ]*)/$match[1]:$match[3]:l})
+    cmds=( ${${(M)${(f)"$(_call_program commands aptitude -h 2>/dev/null)"}:# [^- ][^ ]## *}/(#b) ([^ ]##) ##(- )#([^- ]*)/$match[1]:$match[3]:l})
 
     _describe -t commands 'aptitude command' cmds && ret=0
   ;;
diff --git a/Completion/Unix/Command/_a2ps b/Completion/Unix/Command/_a2ps
index c700c205d..aa2f4c3ba 100644
--- a/Completion/Unix/Command/_a2ps
+++ b/Completion/Unix/Command/_a2ps
@@ -7,7 +7,7 @@ if [[ "$words[1]" != "$_cache_a2ps_cmd" ]]; then
 
   _cache_a2ps_cmd="$words[1]"
 
-  descr=( "${(@)${(f@)$(_call_program features LC_ALL=C $words[1] --list=features)//
+  descr=( "${(@)${(f@)$(_call_program features $words[1] --list=features)//
  /	}:#}" )
 
   _cache_a2ps_values=(
diff --git a/Completion/Unix/Command/_subversion b/Completion/Unix/Command/_subversion
index b4ccccfae..6a35e9947 100644
--- a/Completion/Unix/Command/_subversion
+++ b/Completion/Unix/Command/_subversion
@@ -34,7 +34,7 @@ _svn () {
     typeset -gHA _svn_cmds
     if _cache_invalid svn-cmds || ! _retrieve_cache svn-cmds; then
       _svn_cmds=(
-	${=${(f)${${"$(_comp_locale; _call_program commands svn help)"#l#*Available subcommands:}%%Subversion is a tool*}}/(#s)[[:space:]]#(#b)([a-z]##)[[:space:]]#(\([a-z, ?]##\))#/$match[1] :$match[1]${match[2]:+:${${match[2]//[(),]}// /:}}:}
+	${=${(f)${${"$(_call_program commands svn help)"#l#*Available subcommands:}%%Subversion is a tool*}}/(#s)[[:space:]]#(#b)([a-z]##)[[:space:]]#(\([a-z, ?]##\))#/$match[1] :$match[1]${match[2]:+:${${match[2]//[(),]}// /:}}:}
       )
       if (( $? == 0 )); then
         _store_cache svn-cmds _svn_cmds
@@ -60,14 +60,14 @@ _svn () {
 	if _cache_invalid svn-${cmd}-usage || \
 	    ! _retrieve_cache svn-${cmd}-usage;
 	then
-	  usage=${${(M)${(f)"$(_comp_locale; _call_program options svn help $cmd)"}:#usage:*}#usage:*$cmd] }
+	  usage=${${(M)${(f)"$(_call_program options svn help $cmd)"}:#usage:*}#usage:*$cmd] }
 	  _store_cache svn-${cmd}-usage usage
 	fi
 	if _cache_invalid svn-${cmd}-usage || \
 	    ! _retrieve_cache svn-${cmd}-args;
 	then
 	  args=(
-	    ${=${${${(M)${(f)"$(_comp_locale; _call_program options svn help $cmd)"#(*Valid options:|(#e))}:#* :*}%% #:*}/ (arg|ARG)/:arg:}/(#b)(-##)([[:alpha:]]##) \[--([a-z-]##)\](:arg:)#/(--$match[3])$match[1]$match[2]$match[4] ($match[1]$match[2])--$match[3]$match[4]}
+	    ${=${${${(M)${(f)"$(_call_program options svn help $cmd)"#(*Valid options:|(#e))}:#* :*}%% #:*}/ (arg|ARG)/:arg:}/(#b)(-##)([[:alpha:]]##) \[--([a-z-]##)\](:arg:)#/(--$match[3])$match[1]$match[2]$match[4] ($match[1]$match[2])--$match[3]$match[4]}
 	  )
           while (( idx=$args[(I)*--accept:arg:] )); do
             args[(I)*--accept:arg:]=( --accept'=:automatic conflict resolution action:((working\:working base\:base '"`for i j in p postpone mc mine-conflict tc theirs-conflict mf mine-full tf theirs-full e edit l launch; do print -rn $i\\\\:$j $j\\\\:$j ""; done `"'))' )
@@ -227,7 +227,7 @@ _svnadmin () {
   if [[ -n $state ]] && (( ! $+_svnadmin_cmds )); then
     typeset -gHA _svnadmin_cmds
     _svnadmin_cmds=(
-      ${=${(f)${${"$(_comp_locale; _call_program commands svnadmin help)"#l#*Available subcommands:}}}/(#s)[[:space:]]#(#b)([-a-z]##)[[:space:]]#(\([a-z, ?]##\))#/$match[1] :$match[1]${match[2]:+:${${match[2]//[(),]}// /:}}:}
+      ${=${(f)${${"$(_call_program commands svnadmin help)"#l#*Available subcommands:}}}/(#s)[[:space:]]#(#b)([-a-z]##)[[:space:]]#(\([a-z, ?]##\))#/$match[1] :$match[1]${match[2]:+:${${match[2]//[(),]}// /:}}:}
     )
   fi
 
@@ -242,9 +242,9 @@ _svnadmin () {
       if (( $#cmd )); then
         curcontext="${curcontext%:*:*}:svnadmin-${cmd}:"
 
-        usage=${${(M)${(f)"$(_comp_locale; _call_program options svnadmin help $cmd)"}:#$cmd: usage:*}#$cmd: usage: svnadmin $cmd }
+        usage=${${(M)${(f)"$(_call_program options svnadmin help $cmd)"}:#$cmd: usage:*}#$cmd: usage: svnadmin $cmd }
         args=(
-          ${=${${${(M)${(f)"$(_comp_locale; _call_program options svnadmin help $cmd)"#(*Valid options:|(#e))}:#* :*}%% #:*}/ (arg|ARG)/:arg:}/(#b)-([[:alpha:]]) \[--([a-z-]##)\](:arg:)#/(--$match[2])-$match[1]$match[3] (-$match[1])--$match[2]$match[3]}
+          ${=${${${(M)${(f)"$(_call_program options svnadmin help $cmd)"#(*Valid options:|(#e))}:#* :*}%% #:*}/ (arg|ARG)/:arg:}/(#b)-([[:alpha:]]) \[--([a-z-]##)\](:arg:)#/(--$match[2])-$match[1]$match[3] (-$match[1])--$match[2]$match[3]}
         )
         # All options get {-x+,--long-x=}
         args=( ${args/(#b)(--[A-Za-z0-9-]##):arg:/$match[1]=:arg:} )
@@ -429,7 +429,7 @@ _svn_props() {
 _svn_changelists() {
   local cls
 
-  cls=( ${${${(M)${(f)"$(_comp_locale; _call_program changelists svn status 2>/dev/null)"}:#--- Changelist*}%??}##*\'} )
+  cls=( ${${${(M)${(f)"$(_call_program changelists svn status 2>/dev/null)"}:#--- Changelist*}%??}##*\'} )
   compadd "$@" -a cls && return 0
 }
 
diff --git a/Completion/Unix/Command/_tar b/Completion/Unix/Command/_tar
index ecf02fd83..cff50562e 100644
--- a/Completion/Unix/Command/_tar
+++ b/Completion/Unix/Command/_tar
@@ -99,7 +99,7 @@ if [[ "$PREFIX" = --* ]]; then
 
   # ...long options after `--'.
 
-  _arguments --  '--owner=*:user:_users' \
+  _arguments -- -l '--owner=*:user:_users' \
 		 '--group=*:group:_groups' \
 		 '--atime-preserve*::method:(replace system)' \
 		 '--*-script=NAME:script file:_files' \
diff --git a/Doc/Zsh/compsys.yo b/Doc/Zsh/compsys.yo
index 36afd7305..afe332544 100644
--- a/Doc/Zsh/compsys.yo
+++ b/Doc/Zsh/compsys.yo
@@ -3596,7 +3596,8 @@ findex(_arguments)
 redef(SPACES)(0)(tt(ifztexi(NOTRANS(@ @ @ @ @ @ @ @ @ @ @ ))ifnztexi(           )))
 xitem(tt(_arguments )[ tt(-nswWCRS) ] [ tt(-A) var(pat) ] [ tt(-O) var(name) ] [ tt(-M) var(matchspec) ])
 xitem(SPACES()[ tt(:) ] var(spec) ...)
-item(tt(_arguments) [ var(opt) ... ] tt(-)tt(-) [ tt(-i) var(pats) ] [ tt(-s) var(pair) ] [ var(helpspec) ... ])(
+xitem(tt(_arguments )[ var(opt) ... ] tt(-)tt(-) [ tt(-l) ] [ tt(-i) var(pats) ] [ tt(-s) var(pair) ])
+item(SPACES()[ var(helpspec) ...])(
 This function can be used to give a complete specification for completion
 for a command whose arguments follow standard UNIX option and argument
 conventions.
@@ -4139,6 +4140,12 @@ pattern and the var(action) will be used only directly after the
 `tt(=)', not in the next word.  This is the behaviour of a normal
 specification defined with the form `tt(=-)'.
 
+By default, the command (with the option `tt(--help)') is run after
+resetting all the locale categories (except for tt(LC_CTYPE)) to `tt(C)'.
+If the localized help output is known to work, the option `tt(-l)' can
+be specified after the `tt(_arguments -)tt(-)' so that the command is
+run in the current locale.
+
 The `tt(_arguments -)tt(-)' can be followed by the option `tt(-i)
 var(patterns)' to give patterns for options which are not to be
 completed.  The patterns can be given as the name of an array parameter
@@ -4230,7 +4237,7 @@ The return status of tt(_call_function) itself is zero if the function
 var(name) exists and was called and non-zero otherwise.
 )
 findex(_call_program)
-item(tt(_call_program) [ tt(-p) ] var(tag) var(string) ...)(
+item(tt(_call_program) [ tt(-l) ] [ tt(-p) ] var(tag) var(string) ...)(
 This function provides a mechanism for the user to override the use of an
 external command.  It looks up the tt(command) style with the supplied
 var(tag).  If the style is set, its value is used as the command to
@@ -4239,6 +4246,11 @@ style if set, are concatenated with spaces between them and the resulting
 string is evaluated.  The return status is the return status of the command
 called.
 
+By default, the command is run in an environment where all the locale
+categories (except for tt(LC_CTYPE)) are reset to `tt(C)' by calling the
+utility function tt(_comp_locale) (see below). If the option `tt(-l)' is
+given, the command is run with the current locale.
+
 If the option `tt(-p)' is supplied it indicates that the command
 output is influenced by the permissions it is run with. If the
 tt(gain-privileges) style is set to true, tt(_call_program) will make
@@ -4299,6 +4311,18 @@ This function completes words that are valid at command position: names of
 aliases, builtins, hashed commands, functions, and so on.  With the tt(-e)
 flag, only hashed commands are completed.  The tt(-) flag is ignored.
 )
+findex(_comp_locale)
+item(tt(_comp_locale))(
+This function resets all the locale categories other than tt(LC_CTYPE) to
+`tt(C)' so that the output from external commands can be easily analyzed by
+the completion system. tt(LC_CTYPE) retains the current value (taking
+tt(LC_ALL) and tt(LANG) into account), ensuring that non-ASCII characters
+in file names are still handled properly.
+
+This function should normally be run only in a subshell, because the new
+locale is exported to the environment. Typical usage would be
+`tt($LPAR()_comp_locale; )var(command) ...tt(RPAR())'.
+)
 findex(_completers)
 item(tt(_completers) [ tt(-p) ])(
 This function completes names of completers.