econf: If a host in configuration is encoded IDNA, decode it (#3519)

For example:

hosts:
  - localhost
  - locälhost3
  - xn--loclhost4-x2a

all them are converted to utf8:

ejabberd_option:hosts().
[<<"localhost">>,
 <<"locälhost3"/utf8>>,
 <<"locälhost4"/utf8>>]

mix.exs

@@ -131,7 +131,7 @@ defmodule Ejabberd.MixProject do
      {:pkix, "~> 1.0"},
      {:stringprep, ">= 1.0.26"},
      {:xmpp, ">= 1.11.0"},
-     {:yconf, ">= 1.0.18"}]
+     {:yconf, git: "https://github.com/processone/yconf", ref: "c59f94097af5b78a8c6c72a143dcefaa5b3f406f", override: true}]
     ++ cond_deps()
   end
 

mix.lock

@@ -35,5 +35,5 @@
   "stun": {:hex, :stun, "1.2.20", "62a149cea122a78a104b9e064a12d9e33105b26c23168ecf3aea6e0c26de0748", [:rebar3], [{:fast_tls, "1.1.24", [hex: :fast_tls, repo: "hexpm", optional: false]}, {:p1_utils, "1.0.28", [hex: :p1_utils, repo: "hexpm", optional: false]}], "hexpm", "79e49f826a4f7d522c939ab633d935c79d7d6b229e4cb7e05f62f33b50177414"},
   "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.1", "a48703a25c170eedadca83b11e88985af08d35f37c6f664d6dcfb106a97782fc", [:rebar3], [], "hexpm", "b3a917854ce3ae233619744ad1e0102e05673136776fb2fa76234f3e03b23642"},
   "xmpp": {:hex, :xmpp, "1.11.0", "a3158c486c9b86a7090c361d876db622381f4312ede8c125d7a52ad390387932", [:rebar3], [{:ezlib, "~> 1.0.12", [hex: :ezlib, repo: "hexpm", optional: false]}, {:fast_tls, "~> 1.1.19", [hex: :fast_tls, repo: "hexpm", optional: false]}, {:fast_xml, "~> 1.1.51", [hex: :fast_xml, repo: "hexpm", optional: false]}, {:idna, "~> 6.0", [hex: :idna, repo: "hexpm", optional: false]}, {:p1_utils, "~> 1.0.26", [hex: :p1_utils, repo: "hexpm", optional: false]}, {:stringprep, "~> 1.0.29", [hex: :stringprep, repo: "hexpm", optional: false]}], "hexpm", "34a191d6a3b74e8f0a42346f859e2cab5b3a2ae7e5c28f392e5cb56612e7ce85"},
-  "yconf": {:hex, :yconf, "1.0.20", "f2b38db613fa826966e8d22bdc3e3ebae46919f2a27ab149a5a086c1d99d3bbd", [:rebar3], [{:fast_yaml, "1.0.39", [hex: :fast_yaml, repo: "hexpm", optional: false]}], "hexpm", "f2b3d730756fc2e4afd1c0b0ab6efb99f0e448952d25dc15ed75ac1635bf8882"},
+  "yconf": {:git, "https://github.com/processone/yconf", "c59f94097af5b78a8c6c72a143dcefaa5b3f406f", [ref: "c59f94097af5b78a8c6c72a143dcefaa5b3f406f"]},
 }

rebar.config

@@ -78,7 +78,7 @@
         {if_var_true, stun,
          {stun, "~> 1.2.20", {git, "https://github.com/processone/stun", {tag, "1.2.20"}}}},
         {xmpp, "~> 1.11.0", {git, "https://github.com/processone/xmpp", {tag, "1.11.0"}}},
-        {yconf, "~> 1.0.20", {git, "https://github.com/processone/yconf", {tag, "1.0.20"}}}
+        {yconf, "~> 1.0.20", {git, "https://github.com/processone/yconf", "c59f94097af5b78a8c6c72a143dcefaa5b3f406f"}}
        ]}.
 
 {gitonly_deps, [ejabberd_po]}.

rebar.lock

@@ -28,7 +28,10 @@
  {<<"stun">>,{pkg,<<"stun">>,<<"1.2.20">>},0},
  {<<"unicode_util_compat">>,{pkg,<<"unicode_util_compat">>,<<"0.7.1">>},1},
  {<<"xmpp">>,{pkg,<<"xmpp">>,<<"1.11.0">>},0},
- {<<"yconf">>,{pkg,<<"yconf">>,<<"1.0.20">>},0}]}.
+ {<<"yconf">>,
+  {git,"https://github.com/processone/yconf",
+       {ref,"c59f94097af5b78a8c6c72a143dcefaa5b3f406f"}},
+  0}]}.
 [
 {pkg_hash,[
  {<<"base64url">>, <<"F8C7F2DA04CA9A5D0F5F50258F055E1D699F0E8BF4CFDB30B750865368403CF6">>},
@@ -55,8 +58,7 @@
  {<<"stringprep">>, <<"22F42866B4F6F3C238EA2B9CB6241791184DDEDBAB55E94A025511F46325F3CA">>},
  {<<"stun">>, <<"62A149CEA122A78A104B9E064A12D9E33105B26C23168ECF3AEA6E0C26DE0748">>},
  {<<"unicode_util_compat">>, <<"A48703A25C170EEDADCA83B11E88985AF08D35F37C6F664D6DCFB106A97782FC">>},
- {<<"xmpp">>, <<"A3158C486C9B86A7090C361D876DB622381F4312EDE8C125D7A52AD390387932">>},
- {<<"yconf">>, <<"F2B38DB613FA826966E8D22BDC3E3EBAE46919F2A27AB149A5A086C1D99D3BBD">>}]},
+ {<<"xmpp">>, <<"A3158C486C9B86A7090C361D876DB622381F4312EDE8C125D7A52AD390387932">>}]},
 {pkg_hash_ext,[
  {<<"base64url">>, <<"F9B3ADD4731A02A9B0410398B475B33E7566A695365237A6BDEE1BB447719F5C">>},
  {<<"cache_tab">>, <<"4258009EB050B22AABE0C848E230BBA58401A6895C58C2FF74DFB635E3C35900">>},
@@ -82,6 +84,5 @@
  {<<"stringprep">>, <<"96F8B30BC50887F605B33B46BCA1D248C19A879319B8C482790E3B4DA5DA98C0">>},
  {<<"stun">>, <<"79E49F826A4F7D522C939AB633D935C79D7D6B229E4CB7E05F62F33B50177414">>},
  {<<"unicode_util_compat">>, <<"B3A917854CE3AE233619744AD1E0102E05673136776FB2FA76234F3E03B23642">>},
- {<<"xmpp">>, <<"34A191D6A3B74E8F0A42346F859E2CAB5B3A2AE7E5C28F392E5CB56612E7CE85">>},
- {<<"yconf">>, <<"F2B3D730756FC2E4AFD1C0B0AB6EFB99F0E448952D25DC15ED75AC1635BF8882">>}]}
+ {<<"xmpp">>, <<"34A191D6A3B74E8F0A42346F859E2CAB5B3A2AE7E5C28F392E5CB56612E7CE85">>}]}
 ].

src/econf.erl

@@ -482,6 +482,8 @@ domain() ->
       non_empty(binary()),
       fun(Val) ->
 	      try jid:tolower(jid:decode(Val)) of
+		  {<<"">>, <<"xn--", _/binary>> = Domain, <<"">>} ->
+		      unicode:characters_to_binary(idna:decode(binary_to_list(Domain)), utf8);
 		  {<<"">>, Domain, <<"">>} -> Domain;
 		  _ -> fail({bad_domain, Val})
 	      catch _:{bad_jid, _} ->

src/ejabberd_config.erl

@@ -508,7 +508,7 @@ get_predefined_keywords(Host) ->
             global ->
                 [];
             _ ->
-                [{<<"HOST">>, Host}]
+                [{<<"HOST">>, Host}, {<<"HOST_URL_ENCODE">>, misc:url_encode(Host)}]
         end,
     Home = misc:get_home(),
     ConfigDirPath =

src/ejabberd_http.erl

@@ -718,7 +718,7 @@ file_format_error(Reason) ->
 url_decode_q_split_normalize(Path) ->
     {NPath, Query} = url_decode_q_split(Path),
     LPath = normalize_path([NPE
-		    || NPE <- str:tokens(path_decode(NPath), <<"/">>)]),
+		    || NPE <- str:tokens(misc:uri_decode(NPath), <<"/">>)]),
     {LPath, Query}.
 
 % Code below is taken (with some modifications) from the yaws webserver, which
@@ -746,19 +746,6 @@ url_decode_q_split(<<H, T/binary>>, Acc) when H /= 0 ->
 url_decode_q_split(<<>>, Ack) ->
     {path_norm_reverse(Ack), <<>>}.
 
-%% @doc Decode a part of the URL and return string()
-path_decode(Path) -> path_decode(Path, <<>>).
-
-path_decode(<<$%, Hi, Lo, Tail/binary>>, Acc) ->
-    Hex = list_to_integer([Hi, Lo], 16),
-    if Hex == 0 -> exit(badurl);
-       true -> ok
-    end,
-    path_decode(Tail, <<Acc/binary, Hex>>);
-path_decode(<<H, T/binary>>, Acc) when H /= 0 ->
-    path_decode(T, <<Acc/binary, H>>);
-path_decode(<<>>, Acc) -> Acc.
-
 path_norm_reverse(<<"/", T/binary>>) -> start_dir(0, <<"/">>, T);
 path_norm_reverse(T) -> start_dir(0, <<"">>, T).
 

src/misc.erl

@@ -43,6 +43,7 @@
 	 is_mucsub_message/1, best_match/2, pmap/2, peach/2, format_exception/4,
 	 get_my_ipv4_address/0, get_my_ipv6_address/0, parse_ip_mask/1,
 	 crypto_hmac/3, crypto_hmac/4, uri_parse/1, uri_parse/2, uri_quote/1,
+	 uri_decode/1,
          json_encode/1, json_decode/1,
 	 set_proc_label/1,
 	 match_ip_mask/3, format_hosts_list/1, format_cycle/1, delete_dir/1,
@@ -72,45 +73,11 @@
 -type distance_cache() :: #{{string(), string()} => non_neg_integer()}.
 
 -spec uri_parse(binary()|string()) -> {ok, string(), string(), string(), number(), string(), string()} | {error, term()}.
--ifdef(USE_OLD_HTTP_URI).
-uri_parse(URL) when is_binary(URL) ->
-    uri_parse(binary_to_list(URL));
 uri_parse(URL) ->
-    uri_parse(URL, []).
+    yconf:parse_uri(URL).
 
-uri_parse(URL, Protocols) when is_binary(URL) ->
-    uri_parse(binary_to_list(URL), Protocols);
 uri_parse(URL, Protocols) ->
-    case http_uri:parse(URL, [{scheme_defaults, Protocols}]) of
-	{ok, {Scheme, UserInfo, Host, Port, Path, Query}} ->
-	    {ok, atom_to_list(Scheme), UserInfo, Host, Port, Path, Query};
-	{error, _} = E ->
-	    E
-    end.
-
--else.
-uri_parse(URL) when is_binary(URL) ->
-    uri_parse(binary_to_list(URL));
-uri_parse(URL) ->
-    uri_parse(URL, [{http, 80}, {https, 443}]).
-
-uri_parse(URL, Protocols) when is_binary(URL) ->
-    uri_parse(binary_to_list(URL), Protocols);
-uri_parse(URL, Protocols) ->
-    case uri_string:parse(URL) of
-	#{scheme := Scheme, host := Host, port := Port, path := Path} = M1 ->
-	    {ok, Scheme, maps:get(userinfo, M1, ""), Host, Port, Path, maps:get(query, M1, "")};
-	#{scheme := Scheme, host := Host, path := Path} = M2 ->
-	    case lists:keyfind(list_to_atom(Scheme), 1, Protocols) of
-		{_, Port} ->
-		    {ok, Scheme, maps:get(userinfo, M2, ""), Host, Port, Path, maps:get(query, M2, "")};
-		_ ->
-		    {error, unknown_protocol}
-	    end;
-	{error, Atom, _} ->
-	    {error, Atom}
-    end.
--endif.
+    yconf:parse_uri(URL, Protocols).
 
 -ifdef(OTP_BELOW_25).
 -ifdef(OTP_BELOW_24).
@@ -125,6 +92,26 @@ uri_quote(Data) ->
     uri_string:quote(Data).
 -endif.
 
+%% @doc Decode a part of the URL and return string()
+%% -spec url_decode(binary()) -> bitstring().
+
+-ifdef(OTP_BELOW_24).
+uri_decode(Path) -> uri_decode(Path, <<>>).
+
+uri_decode(<<$%, Hi, Lo, Tail/binary>>, Acc) ->
+    Hex = list_to_integer([Hi, Lo], 16),
+    if Hex == 0 -> exit(badurl);
+       true -> ok
+    end,
+    uri_decode(Tail, <<Acc/binary, Hex>>);
+uri_decode(<<H, T/binary>>, Acc) when H /= 0 ->
+    uri_decode(T, <<Acc/binary, H>>);
+uri_decode(<<>>, Acc) -> Acc.
+-else.
+uri_decode(Path) ->
+    uri_string:percent_decode(Path).
+-endif.
+
 -ifdef(USE_OLD_CRYPTO_HMAC).
 crypto_hmac(Type, Key, Data) -> crypto:hmac(Type, Key, Data).
 crypto_hmac(Type, Key, Data, MacL) -> crypto:hmac(Type, Key, Data, MacL).

src/mod_http_upload.erl

@@ -319,8 +319,10 @@ mod_doc() ->
               desc =>
                   ?T("This option specifies the initial part of the PUT URLs "
                      "used for file uploads. The keyword '@HOST@' is replaced "
-                     "with the virtual host name. NOTE: different virtual "
-                     "hosts cannot use the same PUT URL. "
+                     "with the virtual host name. "
+                     "And '@HOST_URL_ENCODE@' is replaced with the host name encoded for URL, "
+                     "useful when your virtual hosts contain non-latin characters. "
+                     "NOTE: different virtual hosts cannot use the same PUT URL. "
                      "The default value is '\"https://@HOST@:5443/upload\"'.")}},
            {get_url,
             #{value => ?T("URL"),
@@ -531,7 +533,8 @@ process(LocalPath, #request{method = Method, host = Host, ip = IP})
 	   [Method, encode_addr(IP), Host]),
     http_response(404);
 process(_LocalPath, #request{method = 'PUT', host = Host, ip = IP,
-			     length = Length} = Request) ->
+			     length = Length} = Request0) ->
+    Request = Request0#request{host = redecode_url(Host)},
     {Proc, Slot} = parse_http_request(Request),
     try gen_server:call(Proc, {use_slot, Slot, Length}, ?CALL_TIMEOUT) of
 	{ok, Path, FileMode, DirMode, GetPrefix, Thumbnail, CustomHeaders} ->
@@ -571,9 +574,10 @@ process(_LocalPath, #request{method = 'PUT', host = Host, ip = IP,
 		       [encode_addr(IP), Host, Error]),
 	    http_response(500)
     end;
-process(_LocalPath, #request{method = Method, host = Host, ip = IP} = Request)
+process(_LocalPath, #request{method = Method, host = Host, ip = IP} = Request0)
     when Method == 'GET';
 	 Method == 'HEAD' ->
+    Request = Request0#request{host = redecode_url(Host)},
     {Proc, [_UserDir, _RandDir, FileName] = Slot} = parse_http_request(Request),
     try gen_server:call(Proc, get_conf, ?CALL_TIMEOUT) of
 	{ok, DocRoot, CustomHeaders} ->
@@ -907,8 +911,8 @@ mk_slot(Slot, #state{put_url = PutPrefix, get_url = GetPrefix}, XMLNS, Query) ->
     GetURL = str:join([GetPrefix | Slot], <<$/>>),
     mk_slot(PutURL, GetURL, XMLNS, Query);
 mk_slot(PutURL, GetURL, XMLNS, Query) ->
-    PutURL1 = <<(misc:url_encode(PutURL))/binary, Query/binary>>,
-    GetURL1 = misc:url_encode(GetURL),
+    PutURL1 = <<(reencode_url(PutURL))/binary, Query/binary>>,
+    GetURL1 = reencode_url(GetURL),
     case XMLNS of
 	?NS_HTTP_UPLOAD_0 ->
 	    #upload_slot_0{get = GetURL1, put = PutURL1, xmlns = XMLNS};
@@ -916,6 +920,18 @@ mk_slot(PutURL, GetURL, XMLNS, Query) ->
 	    #upload_slot{get = GetURL1, put = PutURL1, xmlns = XMLNS}
     end.
 
+reencode_url(UrlString) ->
+    {ok, _, _, Host, _, _, _} = yconf:parse_uri(UrlString),
+    HostDecoded = misc:uri_decode(Host),
+    HostIdna = idna:encode(HostDecoded),
+    re:replace(UrlString, Host, HostIdna, [{return, binary}]).
+
+redecode_url(UrlString) ->
+    {ok, _, _, HostIdna, _, _, _} = yconf:parse_uri(<<"http://", UrlString/binary>>),
+    HostDecoded = idna:decode(HostIdna),
+    Host = misc:uri_quote(HostDecoded),
+    re:replace(UrlString, HostIdna, Host, [{return, binary}]).
+
 -spec make_user_string(jid(), sha1 | node) -> binary().
 make_user_string(#jid{luser = U, lserver = S}, sha1) ->
     str:sha(<<U/binary, $@, S/binary>>);

src/prosody2ejabberd.erl

@@ -87,8 +87,8 @@ convert_dir(Path, Host, Type) ->
 			      case eval_file(FilePath) of
 				  {ok, Data} ->
 				      Name = iolist_to_binary(filename:rootname(File)),
-				      convert_data(url_decode(Host), Type,
-						   url_decode(Name), Data);
+				      convert_data(misc:uri_decode(Host), Type,
+						   misc:uri_decode(Name), Data);
 				  Err ->
 				      Err
 			      end
@@ -410,16 +410,6 @@ convert_privacy_item({_, Item}) ->
 	      match_presence_in = MatchPresIn,
 	      match_presence_out = MatchPresOut}.
 
-url_decode(Encoded) ->
-    url_decode(Encoded, <<>>).
-url_decode(<<$%, Hi, Lo, Tail/binary>>, Acc) ->
-    Hex = list_to_integer([Hi, Lo], 16),
-    url_decode(Tail, <<Acc/binary, Hex>>);
-url_decode(<<H, Tail/binary>>, Acc) ->
-    url_decode(Tail, <<Acc/binary, H>>);
-url_decode(<<>>, Acc) ->
-    Acc.
-
 decode_pubsub_host(Host) ->
     try jid:decode(Host) of
 	#jid{luser = <<>>, lserver = LServer} -> LServer;