yetangitu/ejabberd
1
0
Fork 0
mirror of https://github.com/processone/ejabberd synced 2025-10-03 17:59:31 +02:00
Code Issues Releases Wiki Activity

mod_antispam: add format instructions

Browse source
This commit is contained in:
Stefan Strigler 2025-06-05 14:31:02 +02:00
parent 639147be41
commit 34b40aec66
3 changed files with 675 additions and 519 deletions
Download patch file Download diff file Expand all files Collapse all files

427
src/mod_antispam.erl
View file

@ -98,6 +98,8 @@
-define(DEFAULT_RTBL_DOMAINS_NODE, <<"spam_source_domains">>). -define(DEFAULT_RTBL_DOMAINS_NODE, <<"spam_source_domains">>).
-define(DEFAULT_CACHE_SIZE, 10000). -define(DEFAULT_CACHE_SIZE, 10000).
%% @format-begin
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% gen_mod callbacks. %% gen_mod callbacks.
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
@ -134,34 +136,28 @@ depends(_Host, _Opts) ->
-spec mod_opt_type(atom()) -> econf:validator(). -spec mod_opt_type(atom()) -> econf:validator().
mod_opt_type(spam_domains_file) -> mod_opt_type(spam_domains_file) ->
econf:either( econf:either(
econf:enum([none]), econf:enum([none]), econf:file());
econf:file());
mod_opt_type(whitelist_domains_file) -> mod_opt_type(whitelist_domains_file) ->
econf:either( econf:either(none, econf:binary());
none,
econf:binary());
mod_opt_type(spam_dump_file) -> mod_opt_type(spam_dump_file) ->
econf:either( econf:either(
econf:enum([none]), econf:enum([none]), econf:binary());
econf:binary());
mod_opt_type(spam_jids_file) -> mod_opt_type(spam_jids_file) ->
econf:either( econf:either(
econf:enum([none]), econf:enum([none]), econf:file());
econf:file());
mod_opt_type(spam_urls_file) -> mod_opt_type(spam_urls_file) ->
econf:either( econf:either(
econf:enum([none]), econf:enum([none]), econf:file());
econf:file());
mod_opt_type(access_spam) -> mod_opt_type(access_spam) ->
econf:acl(); econf:acl();
mod_opt_type(cache_size) -> mod_opt_type(cache_size) ->
econf:pos_int(unlimited); econf:pos_int(unlimited);
mod_opt_type(rtbl_host) -> mod_opt_type(rtbl_host) ->
econf:either( econf:either(
econf:enum([none]), econf:enum([none]), econf:host());
econf:host());
mod_opt_type(rtbl_domains_node) -> mod_opt_type(rtbl_domains_node) ->
econf:non_empty(econf:binary()). econf:non_empty(
econf:binary()).
-spec mod_options(binary()) -> [{atom(), any()}]. -spec mod_options(binary()) -> [{atom(), any()}].
mod_options(_Host) -> mod_options(_Host) ->
@ -175,7 +171,8 @@ mod_options(_Host) ->
{rtbl_host, none}, {rtbl_host, none},
{rtbl_domains_node, ?DEFAULT_RTBL_DOMAINS_NODE}]. {rtbl_domains_node, ?DEFAULT_RTBL_DOMAINS_NODE}].
mod_doc() -> #{}. mod_doc() ->
#{}.
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% gen_server callbacks. %% gen_server callbacks.
@ -184,25 +181,29 @@ mod_doc() -> #{}.
init([Host, Opts]) -> init([Host, Opts]) ->
process_flag(trap_exit, true), process_flag(trap_exit, true),
DumpFile = expand_host(gen_mod:get_opt(spam_dump_file, Opts), Host), DumpFile = expand_host(gen_mod:get_opt(spam_dump_file, Opts), Host),
Files = #{domains => gen_mod:get_opt(spam_domains_file, Opts), Files =
#{domains => gen_mod:get_opt(spam_domains_file, Opts),
jid => gen_mod:get_opt(spam_jids_file, Opts), jid => gen_mod:get_opt(spam_jids_file, Opts),
url => gen_mod:get_opt(spam_urls_file, Opts), url => gen_mod:get_opt(spam_urls_file, Opts),
whitelist_domains => gen_mod:get_opt(whitelist_domains_file, Opts)}, whitelist_domains => gen_mod:get_opt(whitelist_domains_file, Opts)},
try read_files(Files) of try read_files(Files) of
#{jid := JIDsSet, url := URLsSet, domains := SpamDomainsSet, whitelist_domains := WhitelistDomains} -> #{jid := JIDsSet,
ejabberd_hooks:add(s2s_in_handle_info, Host, ?MODULE, url := URLsSet,
s2s_in_handle_info, 90), domains := SpamDomainsSet,
ejabberd_hooks:add(s2s_receive_packet, Host, ?MODULE, whitelist_domains := WhitelistDomains} ->
s2s_receive_packet, 50), ejabberd_hooks:add(s2s_in_handle_info, Host, ?MODULE, s2s_in_handle_info, 90),
ejabberd_hooks:add(sm_receive_packet, Host, ?MODULE, ejabberd_hooks:add(s2s_receive_packet, Host, ?MODULE, s2s_receive_packet, 50),
sm_receive_packet, 50), ejabberd_hooks:add(sm_receive_packet, Host, ?MODULE, sm_receive_packet, 50),
ejabberd_hooks:add(reopen_log_hook, ?MODULE, ejabberd_hooks:add(reopen_log_hook, ?MODULE, reopen_log, 50),
reopen_log, 50), ejabberd_hooks:add(local_send_to_resource_hook,
ejabberd_hooks:add(local_send_to_resource_hook, Host, Host,
mod_antispam_rtbl, pubsub_event_handler, 50), mod_antispam_rtbl,
pubsub_event_handler,
50),
RTBLHost = gen_mod:get_opt(rtbl_host, Opts), RTBLHost = gen_mod:get_opt(rtbl_host, Opts),
RTBLDomainsNode = gen_mod:get_opt(rtbl_domains_node, Opts), RTBLDomainsNode = gen_mod:get_opt(rtbl_domains_node, Opts),
InitState0 = #state{host = Host, InitState0 =
#state{host = Host,
jid_set = JIDsSet, jid_set = JIDsSet,
url_set = URLsSet, url_set = URLsSet,
max_cache_size = gen_mod:get_opt(cache_size, Opts), max_cache_size = gen_mod:get_opt(cache_size, Opts),
@ -213,8 +214,8 @@ init([Host, Opts]) ->
mod_antispam_rtbl:request_blocked_domains(RTBLHost, RTBLDomainsNode, Host), mod_antispam_rtbl:request_blocked_domains(RTBLHost, RTBLDomainsNode, Host),
InitState = init_open_dump_file(DumpFile, InitState0), InitState = init_open_dump_file(DumpFile, InitState0),
{ok, InitState} {ok, InitState}
catch {Op, File, Reason} when Op == open; catch
Op == read -> {Op, File, Reason} when Op == open; Op == read ->
?CRITICAL_MSG("Cannot ~s ~s: ~s", [Op, File, format_error(Reason)]), ?CRITICAL_MSG("Cannot ~s ~s: ~s", [Op, File, format_error(Reason)]),
{stop, config_error} {stop, config_error}
end. end.
@ -231,16 +232,18 @@ init_open_dump_file(DumpFile, State) ->
end, end,
open_dump_file(DumpFile, State). open_dump_file(DumpFile, State).
-spec handle_call(term(), {pid(), term()}, state()) -spec handle_call(term(), {pid(), term()}, state()) ->
-> {reply, {spam_filter, term()}, state()} | {noreply, state()}. {reply, {spam_filter, term()}, state()} | {noreply, state()}.
handle_call({check_jid, From}, _From, #state{jid_set = JIDsSet} = State) -> handle_call({check_jid, From}, _From, #state{jid_set = JIDsSet} = State) ->
{Result, State1} = filter_jid(From, JIDsSet, State), {Result, State1} = filter_jid(From, JIDsSet, State),
{reply, {spam_filter, Result}, State1}; {reply, {spam_filter, Result}, State1};
handle_call({check_body, URLs, JIDs, From}, _From, handle_call({check_body, URLs, JIDs, From},
_From,
#state{url_set = URLsSet, jid_set = JIDsSet} = State) -> #state{url_set = URLsSet, jid_set = JIDsSet} = State) ->
{Result1, State1} = filter_body(URLs, URLsSet, From, State), {Result1, State1} = filter_body(URLs, URLsSet, From, State),
{Result2, State2} = filter_body(JIDs, JIDsSet, From, State1), {Result2, State2} = filter_body(JIDs, JIDsSet, From, State1),
Result = if Result1 == spam -> Result =
if Result1 == spam ->
Result1; Result1;
true -> true ->
Result2 Result2
@ -263,18 +266,30 @@ handle_call({drop_from_cache, JID}, _From, State) ->
{reply, {spam_filter, Result}, State1}; {reply, {spam_filter, Result}, State1};
handle_call(get_cache, _From, #state{jid_cache = Cache} = State) -> handle_call(get_cache, _From, #state{jid_cache = Cache} = State) ->
{reply, {spam_filter, maps:to_list(Cache)}, State}; {reply, {spam_filter, maps:to_list(Cache)}, State};
handle_call({add_blocked_domain, Domain}, _From, #state{blocked_domains = BlockedDomains} = State) -> handle_call({add_blocked_domain, Domain},
_From,
#state{blocked_domains = BlockedDomains} = State) ->
BlockedDomains1 = maps:merge(BlockedDomains, #{Domain => true}), BlockedDomains1 = maps:merge(BlockedDomains, #{Domain => true}),
Txt = format("~s added to blocked domains", [Domain]), Txt = format("~s added to blocked domains", [Domain]),
{reply, {spam_filter, {ok, Txt}}, State#state{blocked_domains = BlockedDomains1}}; {reply, {spam_filter, {ok, Txt}}, State#state{blocked_domains = BlockedDomains1}};
handle_call({remove_blocked_domain, Domain}, _From, #state{blocked_domains = BlockedDomains} = State) -> handle_call({remove_blocked_domain, Domain},
_From,
#state{blocked_domains = BlockedDomains} = State) ->
BlockedDomains1 = maps:remove(Domain, BlockedDomains), BlockedDomains1 = maps:remove(Domain, BlockedDomains),
Txt = format("~s removed from blocked domains", [Domain]), Txt = format("~s removed from blocked domains", [Domain]),
{reply, {spam_filter, {ok, Txt}}, State#state{blocked_domains = BlockedDomains1}}; {reply, {spam_filter, {ok, Txt}}, State#state{blocked_domains = BlockedDomains1}};
handle_call(get_blocked_domains, _From, #state{blocked_domains = BlockedDomains, whitelist_domains = WhitelistDomains} = State) -> handle_call(get_blocked_domains,
_From,
#state{blocked_domains = BlockedDomains, whitelist_domains = WhitelistDomains} =
State) ->
{reply, {blocked_domains, maps:merge(BlockedDomains, WhitelistDomains)}, State}; {reply, {blocked_domains, maps:merge(BlockedDomains, WhitelistDomains)}, State};
handle_call({is_blocked_domain, Domain}, _From, #state{blocked_domains = BlockedDomains, whitelist_domains = WhitelistDomains} = State) -> handle_call({is_blocked_domain, Domain},
{reply, maps:get(Domain, maps:merge(BlockedDomains, WhitelistDomains), false) =/= false, State}; _From,
#state{blocked_domains = BlockedDomains, whitelist_domains = WhitelistDomains} =
State) ->
{reply,
maps:get(Domain, maps:merge(BlockedDomains, WhitelistDomains), false) =/= false,
State};
handle_call(Request, From, State) -> handle_call(Request, From, State) ->
?ERROR_MSG("Got unexpected request from ~p: ~p", [From, Request]), ?ERROR_MSG("Got unexpected request from ~p: ~p", [From, Request]),
{noreply, State}. {noreply, State}.
@ -287,26 +302,27 @@ handle_cast({dump, XML}, #state{dump_fd = Fd} = State) ->
ok -> ok ->
ok; ok;
{error, Reason} -> {error, Reason} ->
?ERROR_MSG("Cannot write spam to dump file: ~s", ?ERROR_MSG("Cannot write spam to dump file: ~s", [file:format_error(Reason)])
[file:format_error(Reason)])
end, end,
{noreply, State}; {noreply, State};
handle_cast({reload, NewOpts, OldOpts}, handle_cast({reload, NewOpts, OldOpts},
#state{host = Host, #state{host = Host,
rtbl_host = OldRTBLHost, rtbl_host = OldRTBLHost,
rtbl_domains_node = OldRTBLDomainsNode, rtbl_domains_node = OldRTBLDomainsNode,
rtbl_retry_timer = RTBLRetryTimer} = State) -> rtbl_retry_timer = RTBLRetryTimer} =
State) ->
misc:cancel_timer(RTBLRetryTimer), misc:cancel_timer(RTBLRetryTimer),
State1 = case {gen_mod:get_opt(spam_dump_file, OldOpts), State1 =
gen_mod:get_opt(spam_dump_file, NewOpts)} of case {gen_mod:get_opt(spam_dump_file, OldOpts), gen_mod:get_opt(spam_dump_file, NewOpts)}
of
{OldDumpFile, NewDumpFile} when NewDumpFile /= OldDumpFile -> {OldDumpFile, NewDumpFile} when NewDumpFile /= OldDumpFile ->
close_dump_file(expand_host(OldDumpFile, Host), State), close_dump_file(expand_host(OldDumpFile, Host), State),
open_dump_file(expand_host(NewDumpFile, Host), State); open_dump_file(expand_host(NewDumpFile, Host), State);
{_OldDumpFile, _NewDumpFile} -> {_OldDumpFile, _NewDumpFile} ->
State State
end, end,
State2 = case {gen_mod:get_opt(cache_size, OldOpts), State2 =
gen_mod:get_opt(cache_size, NewOpts)} of case {gen_mod:get_opt(cache_size, OldOpts), gen_mod:get_opt(cache_size, NewOpts)} of
{OldMax, NewMax} when NewMax < OldMax -> {OldMax, NewMax} when NewMax < OldMax ->
shrink_cache(State1#state{max_cache_size = NewMax}); shrink_cache(State1#state{max_cache_size = NewMax});
{OldMax, NewMax} when NewMax > OldMax -> {OldMax, NewMax} when NewMax > OldMax ->
@ -315,7 +331,8 @@ handle_cast({reload, NewOpts, OldOpts},
State1 State1
end, end,
ok = mod_antispam_rtbl:unsubscribe(OldRTBLHost, OldRTBLDomainsNode, Host), ok = mod_antispam_rtbl:unsubscribe(OldRTBLHost, OldRTBLDomainsNode, Host),
Files = #{domains => gen_mod:get_opt(spam_domains_file, NewOpts), Files =
#{domains => gen_mod:get_opt(spam_domains_file, NewOpts),
jid => gen_mod:get_opt(spam_jids_file, NewOpts), jid => gen_mod:get_opt(spam_jids_file, NewOpts),
url => gen_mod:get_opt(spam_urls_file, NewOpts), url => gen_mod:get_opt(spam_urls_file, NewOpts),
whitelist_domains => gen_mod:get_opt(whitelist_domains_file, NewOpts)}, whitelist_domains => gen_mod:get_opt(whitelist_domains_file, NewOpts)},
@ -326,7 +343,8 @@ handle_cast({reload, NewOpts, OldOpts},
{noreply, State3#state{rtbl_host = RTBLHost, rtbl_domains_node = RTBLDomainsNode}}; {noreply, State3#state{rtbl_host = RTBLHost, rtbl_domains_node = RTBLDomainsNode}};
handle_cast(reopen_log, State) -> handle_cast(reopen_log, State) ->
{noreply, reopen_dump_file(State)}; {noreply, reopen_dump_file(State)};
handle_cast({update_blocked_domains, NewItems}, #state{blocked_domains = BlockedDomains} = State) -> handle_cast({update_blocked_domains, NewItems},
#state{blocked_domains = BlockedDomains} = State) ->
{noreply, State#state{blocked_domains = maps:merge(BlockedDomains, NewItems)}}; {noreply, State#state{blocked_domains = maps:merge(BlockedDomains, NewItems)}};
handle_cast(Request, State) -> handle_cast(Request, State) ->
?ERROR_MSG("Got unexpected request from: ~p", [Request]), ?ERROR_MSG("Got unexpected request from: ~p", [Request]),
@ -334,24 +352,32 @@ handle_cast(Request, State) ->
-spec handle_info(term(), state()) -> {noreply, state()}. -spec handle_info(term(), state()) -> {noreply, state()}.
handle_info({iq_reply, timeout, blocked_domains}, State) -> handle_info({iq_reply, timeout, blocked_domains}, State) ->
?WARNING_MSG("Fetching blocked domains failed: fetch timeout. Retrying in 60 seconds", []), ?WARNING_MSG("Fetching blocked domains failed: fetch timeout. Retrying in 60 seconds",
{noreply, State#state{rtbl_retry_timer = erlang:send_after(60000, self(), request_blocked_domains)}}; []),
{noreply,
State#state{rtbl_retry_timer =
erlang:send_after(60000, self(), request_blocked_domains)}};
handle_info({iq_reply, #iq{type = error} = IQ, blocked_domains}, State) -> handle_info({iq_reply, #iq{type = error} = IQ, blocked_domains}, State) ->
?WARNING_MSG("Fetching blocked domains failed: ~p. Retrying in 60 seconds", ?WARNING_MSG("Fetching blocked domains failed: ~p. Retrying in 60 seconds",
[xmpp:format_stanza_error(xmpp:get_error(IQ))]), [xmpp:format_stanza_error(
{noreply, State#state{rtbl_retry_timer = erlang:send_after(60000, self(), request_blocked_domains)}}; xmpp:get_error(IQ))]),
{noreply,
State#state{rtbl_retry_timer =
erlang:send_after(60000, self(), request_blocked_domains)}};
handle_info({iq_reply, IQReply, blocked_domains}, handle_info({iq_reply, IQReply, blocked_domains},
#state{blocked_domains = OldBlockedDomains, #state{blocked_domains = OldBlockedDomains,
rtbl_host = RTBLHost, rtbl_host = RTBLHost,
rtbl_domains_node = RTBLDomainsNode, rtbl_domains_node = RTBLDomainsNode,
host = Host} = State) -> host = Host} =
State) ->
case mod_antispam_rtbl:parse_blocked_domains(IQReply) of case mod_antispam_rtbl:parse_blocked_domains(IQReply) of
undefined -> undefined ->
?WARNING_MSG("Fetching initial list failed: invalid result payload", []), ?WARNING_MSG("Fetching initial list failed: invalid result payload", []),
{noreply, State#state{rtbl_retry_timer = undefined}}; {noreply, State#state{rtbl_retry_timer = undefined}};
NewBlockedDomains -> NewBlockedDomains ->
ok = mod_antispam_rtbl:subscribe(RTBLHost, RTBLDomainsNode, Host), ok = mod_antispam_rtbl:subscribe(RTBLHost, RTBLDomainsNode, Host),
{noreply, State#state{rtbl_retry_timer = undefined, {noreply,
State#state{rtbl_retry_timer = undefined,
rtbl_subscribed = true, rtbl_subscribed = true,
blocked_domains = maps:merge(OldBlockedDomains, NewBlockedDomains)}} blocked_domains = maps:merge(OldBlockedDomains, NewBlockedDomains)}}
end; end;
@ -359,7 +385,9 @@ handle_info({iq_reply, timeout, subscribe_result}, State) ->
?WARNING_MSG("Subscription error: request timeout", []), ?WARNING_MSG("Subscription error: request timeout", []),
{noreply, State#state{rtbl_subscribed = false}}; {noreply, State#state{rtbl_subscribed = false}};
handle_info({iq_reply, #iq{type = error} = IQ, subscribe_result}, State) -> handle_info({iq_reply, #iq{type = error} = IQ, subscribe_result}, State) ->
?WARNING_MSG("Subscription error: ~p", [xmpp:format_stanza_error(xmpp:get_error(IQ))]), ?WARNING_MSG("Subscription error: ~p",
[xmpp:format_stanza_error(
xmpp:get_error(IQ))]),
{noreply, State#state{rtbl_subscribed = false}}; {noreply, State#state{rtbl_subscribed = false}};
handle_info({iq_reply, IQReply, subscribe_result}, State) -> handle_info({iq_reply, IQReply, subscribe_result}, State) ->
?DEBUG("Got subscribe result: ~p", [IQReply]), ?DEBUG("Got subscribe result: ~p", [IQReply]),
@ -368,7 +396,11 @@ handle_info({iq_reply, _IQReply, unsubscribe_result}, State) ->
%% FIXME: we should check it's true (of type `result`, not `error`), but at that point, what %% FIXME: we should check it's true (of type `result`, not `error`), but at that point, what
%% would we do? %% would we do?
{noreply, State#state{rtbl_subscribed = false}}; {noreply, State#state{rtbl_subscribed = false}};
handle_info(request_blocked_domains, #state{host = Host, rtbl_host = RTBLHost, rtbl_domains_node = RTBLDomainsNode} = State) -> handle_info(request_blocked_domains,
#state{host = Host,
rtbl_host = RTBLHost,
rtbl_domains_node = RTBLDomainsNode} =
State) ->
mod_antispam_rtbl:request_blocked_domains(RTBLHost, RTBLDomainsNode, Host), mod_antispam_rtbl:request_blocked_domains(RTBLHost, RTBLDomainsNode, Host),
{noreply, State}; {noreply, State};
handle_info(Info, State) -> handle_info(Info, State) ->
@ -380,25 +412,25 @@ terminate(Reason,
#state{host = Host, #state{host = Host,
rtbl_host = RTBLHost, rtbl_host = RTBLHost,
rtbl_domains_node = RTBLDomainsNode, rtbl_domains_node = RTBLDomainsNode,
rtbl_retry_timer = RTBLRetryTimer} = State) -> rtbl_retry_timer = RTBLRetryTimer} =
State) ->
?DEBUG("Stopping spam filter process for ~s: ~p", [Host, Reason]), ?DEBUG("Stopping spam filter process for ~s: ~p", [Host, Reason]),
misc:cancel_timer(RTBLRetryTimer), misc:cancel_timer(RTBLRetryTimer),
DumpFile = gen_mod:get_module_opt(Host, ?MODULE, spam_dump_file), DumpFile = gen_mod:get_module_opt(Host, ?MODULE, spam_dump_file),
DumpFile1 = expand_host(DumpFile, Host), DumpFile1 = expand_host(DumpFile, Host),
close_dump_file(DumpFile1, State), close_dump_file(DumpFile1, State),
ejabberd_hooks:delete(s2s_receive_packet, Host, ?MODULE, ejabberd_hooks:delete(s2s_receive_packet, Host, ?MODULE, s2s_receive_packet, 50),
s2s_receive_packet, 50), ejabberd_hooks:delete(sm_receive_packet, Host, ?MODULE, sm_receive_packet, 50),
ejabberd_hooks:delete(sm_receive_packet, Host, ?MODULE, ejabberd_hooks:delete(s2s_in_handle_info, Host, ?MODULE, s2s_in_handle_info, 90),
sm_receive_packet, 50), ejabberd_hooks:delete(local_send_to_resource_hook,
ejabberd_hooks:delete(s2s_in_handle_info, Host, ?MODULE, Host,
s2s_in_handle_info, 90), mod_antispam_rtbl,
ejabberd_hooks:delete(local_send_to_resource_hook, Host, pubsub_event_handler,
mod_antispam_rtbl, pubsub_event_handler, 50), 50),
mod_antispam_rtbl:unsubscribe(RTBLHost, RTBLDomainsNode,Host), mod_antispam_rtbl:unsubscribe(RTBLHost, RTBLDomainsNode, Host),
case gen_mod:is_loaded_elsewhere(Host, ?MODULE) of case gen_mod:is_loaded_elsewhere(Host, ?MODULE) of
false -> false ->
ejabberd_hooks:delete(reopen_log_hook, ?MODULE, ejabberd_hooks:delete(reopen_log_hook, ?MODULE, reopen_log, 50);
reopen_log, 50);
true -> true ->
ok ok
end. end.
@ -412,8 +444,8 @@ code_change(_OldVsn, #state{host = Host} = State, _Extra) ->
%% Hook callbacks. %% Hook callbacks.
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
-spec s2s_receive_packet({stanza() | drop, s2s_in_state()}) -spec s2s_receive_packet({stanza() | drop, s2s_in_state()}) ->
-> {stanza() | drop, s2s_in_state()} | {stop, {drop, s2s_in_state()}}. {stanza() | drop, s2s_in_state()} | {stop, {drop, s2s_in_state()}}.
s2s_receive_packet({A, State}) -> s2s_receive_packet({A, State}) ->
case sm_receive_packet(A) of case sm_receive_packet(A) of
{stop, drop} -> {stop, drop} ->
@ -427,13 +459,14 @@ sm_receive_packet(drop = Acc) ->
Acc; Acc;
sm_receive_packet(#message{from = From, sm_receive_packet(#message{from = From,
to = #jid{lserver = LServer} = To, to = #jid{lserver = LServer} = To,
type = Type} = Msg) type = Type} =
when Type /= groupchat, Msg)
Type /= error -> when Type /= groupchat, Type /= error ->
do_check(From, To, LServer, Msg); do_check(From, To, LServer, Msg);
sm_receive_packet(#presence{from = From, sm_receive_packet(#presence{from = From,
to = #jid{lserver = LServer} = To, to = #jid{lserver = LServer} = To,
type = subscribe} = Presence) -> type = subscribe} =
Presence) ->
do_check(From, To, LServer, Presence); do_check(From, To, LServer, Presence);
sm_receive_packet(Acc) -> sm_receive_packet(Acc) ->
Acc. Acc.
@ -463,8 +496,8 @@ check_stanza(LServer, From, #message{body = Body}) ->
check_stanza(_, _, _) -> check_stanza(_, _, _) ->
ham. ham.
-spec s2s_in_handle_info(s2s_in_state(), any()) -spec s2s_in_handle_info(s2s_in_state(), any()) ->
-> s2s_in_state() | {stop, s2s_in_state()}. s2s_in_state() | {stop, s2s_in_state()}.
s2s_in_handle_info(State, {_Ref, {spam_filter, _}}) -> s2s_in_handle_info(State, {_Ref, {spam_filter, _}}) ->
?DEBUG("Dropping expired spam filter result", []), ?DEBUG("Dropping expired spam filter result", []),
{stop, State}; {stop, State};
@ -476,7 +509,8 @@ reopen_log() ->
lists:foreach(fun(Host) -> lists:foreach(fun(Host) ->
Proc = get_proc_name(Host), Proc = get_proc_name(Host),
gen_server:cast(Proc, reopen_log) gen_server:cast(Proc, reopen_log)
end, get_spam_filter_hosts()). end,
get_spam_filter_hosts()).
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% Internal functions. %% Internal functions.
@ -492,8 +526,11 @@ needs_checking(#jid{lserver = FromHost} = From, #jid{lserver = LServer} = To) ->
false; false;
deny -> deny ->
?DEBUG("Spam is filtered for ~s", [jid:encode(To)]), ?DEBUG("Spam is filtered for ~s", [jid:encode(To)]),
not mod_roster:is_subscribed(From, To) andalso not mod_roster:is_subscribed(From, To)
not mod_roster:is_subscribed(jid:make(<<>>, FromHost), To) % likely a gateway andalso not
mod_roster:is_subscribed(
jid:make(<<>>, FromHost),
To) % likely a gateway
end; end;
false -> false ->
?DEBUG("~s not loaded for ~s", [?MODULE, LServer]), ?DEBUG("~s not loaded for ~s", [?MODULE, LServer]),
@ -503,7 +540,10 @@ needs_checking(#jid{lserver = FromHost} = From, #jid{lserver = LServer} = To) ->
-spec check_from(binary(), jid()) -> ham | spam. -spec check_from(binary(), jid()) -> ham | spam.
check_from(Host, From) -> check_from(Host, From) ->
Proc = get_proc_name(Host), Proc = get_proc_name(Host),
LFrom = {_, FromDomain, _} = jid:remove_resource(jid:tolower(From)), LFrom =
{_, FromDomain, _} =
jid:remove_resource(
jid:tolower(From)),
try try
case gen_server:call(Proc, {is_blocked_domain, FromDomain}) of case gen_server:call(Proc, {is_blocked_domain, FromDomain}) of
true -> true ->
@ -516,7 +556,8 @@ check_from(Host, From) ->
Result Result
end end
end end
catch exit:{timeout, _} -> catch
exit:{timeout, _} ->
?WARNING_MSG("Timeout while checking ~s against list of blocked domains or spammers", ?WARNING_MSG("Timeout while checking ~s against list of blocked domains or spammers",
[jid:encode(From)]), [jid:encode(From)]),
ham ham
@ -530,11 +571,14 @@ check_body(Host, From, Body) ->
ham; ham;
{URLs, JIDs} -> {URLs, JIDs} ->
Proc = get_proc_name(Host), Proc = get_proc_name(Host),
LFrom = jid:remove_resource(jid:tolower(From)), LFrom =
jid:remove_resource(
jid:tolower(From)),
try gen_server:call(Proc, {check_body, URLs, JIDs, LFrom}) of try gen_server:call(Proc, {check_body, URLs, JIDs, LFrom}) of
{spam_filter, Result} -> {spam_filter, Result} ->
Result Result
catch exit:{timeout, _} -> catch
exit:{timeout, _} ->
?WARNING_MSG("Timeout while checking body", []), ?WARNING_MSG("Timeout while checking body", []),
ham ham
end end
@ -558,17 +602,20 @@ resolve_redirects(Host, URLs) ->
try gen_server:call(Proc, {resolve_redirects, URLs}) of try gen_server:call(Proc, {resolve_redirects, URLs}) of
{spam_filter, ResolvedURLs} -> {spam_filter, ResolvedURLs} ->
ResolvedURLs ResolvedURLs
catch exit:{timeout, _} -> catch
exit:{timeout, _} ->
?WARNING_MSG("Timeout while resolving redirects: ~p", [URLs]), ?WARNING_MSG("Timeout while resolving redirects: ~p", [URLs]),
URLs URLs
end. end.
-spec do_resolve_redirects([url()], [url()]) -> [url()]. -spec do_resolve_redirects([url()], [url()]) -> [url()].
do_resolve_redirects([], Result) -> Result; do_resolve_redirects([], Result) ->
Result;
do_resolve_redirects([URL | Rest], Acc) -> do_resolve_redirects([URL | Rest], Acc) ->
case case httpc:request(get,
httpc:request(get, {URL, [{"user-agent", "curl/8.7.1"}]}, {URL, [{"user-agent", "curl/8.7.1"}]},
[{autoredirect, false}, {timeout, ?HTTPC_TIMEOUT}], []) [{autoredirect, false}, {timeout, ?HTTPC_TIMEOUT}],
[])
of of
{ok, {{_, StatusCode, _}, Headers, _Body}} when StatusCode >= 300, StatusCode < 400 -> {ok, {{_, StatusCode, _}, Headers, _Body}} when StatusCode >= 300, StatusCode < 400 ->
Location = proplists:get_value("location", Headers), Location = proplists:get_value("location", Headers),
@ -588,8 +635,7 @@ extract_jids(Body) ->
Options = [global, {capture, all, binary}], Options = [global, {capture, all, binary}],
case re:run(Body, RE, Options) of case re:run(Body, RE, Options) of
{match, Captured} when is_list(Captured) -> {match, Captured} when is_list(Captured) ->
{jids, lists:filtermap(fun try_decode_jid/1, {jids, lists:filtermap(fun try_decode_jid/1, lists:flatten(Captured))};
lists:flatten(Captured))};
nomatch -> nomatch ->
none none
end. end.
@ -598,8 +644,11 @@ extract_jids(Body) ->
try_decode_jid(S) -> try_decode_jid(S) ->
try jid:decode(S) of try jid:decode(S) of
#jid{} = JID -> #jid{} = JID ->
{true, jid:remove_resource(jid:tolower(JID))} {true,
catch _:{bad_jid, _} -> jid:remove_resource(
jid:tolower(JID))}
catch
_:{bad_jid, _} ->
false false
end. end.
@ -623,8 +672,10 @@ filter_jid(From, Set, #state{host = Host} = State) ->
end. end.
-spec filter_body({urls, [url()]} | {jids, [ljid()]} | none, -spec filter_body({urls, [url()]} | {jids, [ljid()]} | none,
url_set() | jid_set(), jid(), state()) url_set() | jid_set(),
-> {ham | spam, state()}. jid(),
state()) ->
{ham | spam, state()}.
filter_body({_, Addrs}, Set, From, #state{host = Host} = State) -> filter_body({_, Addrs}, Set, From, #state{host = Host} = State) ->
case lists:any(fun(Addr) -> sets:is_element(Addr, Set) end, Addrs) of case lists:any(fun(Addr) -> sets:is_element(Addr, Set) end, Addrs) of
true -> true ->
@ -638,11 +689,14 @@ filter_body({_, Addrs}, Set, From, #state{host = Host} = State) ->
filter_body(none, _Set, _From, State) -> filter_body(none, _Set, _From, State) ->
{ham, State}. {ham, State}.
-spec reload_files(#{Type :: atom() => filename()}, state()) -spec reload_files(#{Type :: atom() => filename()}, state()) ->
-> {ok | {error, binary()}, state()}. {ok | {error, binary()}, state()}.
reload_files(Files, #state{host = Host, blocked_domains = BlockedDomains} = State) -> reload_files(Files, #state{host = Host, blocked_domains = BlockedDomains} = State) ->
try read_files(Files) of try read_files(Files) of
#{jid := JIDsSet, url := URLsSet, domains := SpamDomainsSet, whitelist_domains := WhitelistDomains} -> #{jid := JIDsSet,
url := URLsSet,
domains := SpamDomainsSet,
whitelist_domains := WhitelistDomains} ->
case sets_equal(JIDsSet, State#state.jid_set) of case sets_equal(JIDsSet, State#state.jid_set) of
true -> true ->
?INFO_MSG("Reloaded spam JIDs for ~s (unchanged)", [Host]); ?INFO_MSG("Reloaded spam JIDs for ~s (unchanged)", [Host]);
@ -655,15 +709,14 @@ reload_files(Files, #state{host = Host, blocked_domains = BlockedDomains} = Stat
false -> false ->
?INFO_MSG("Reloaded spam URLs for ~s (changed)", [Host]) ?INFO_MSG("Reloaded spam URLs for ~s (changed)", [Host])
end, end,
{ok, State#state{jid_set = JIDsSet, {ok,
State#state{jid_set = JIDsSet,
url_set = URLsSet, url_set = URLsSet,
blocked_domains = maps:merge(BlockedDomains, set_to_map(SpamDomainsSet)), blocked_domains = maps:merge(BlockedDomains, set_to_map(SpamDomainsSet)),
whitelist_domains = set_to_map(WhitelistDomains, false)} whitelist_domains = set_to_map(WhitelistDomains, false)}}
} catch
catch {Op, File, Reason} when Op == open; {Op, File, Reason} when Op == open; Op == read ->
Op == read -> Txt = format("Cannot ~s ~s for ~s: ~s", [Op, File, Host, format_error(Reason)]),
Txt = format("Cannot ~s ~s for ~s: ~s",
[Op, File, Host, format_error(Reason)]),
?ERROR_MSG("~s", [Txt]), ?ERROR_MSG("~s", [Txt]),
{{error, Txt}, State} {{error, Txt}, State}
end. end.
@ -674,37 +727,44 @@ set_to_map(Set) ->
set_to_map(Set, V) -> set_to_map(Set, V) ->
sets:fold(fun(K, M) -> M#{K => V} end, #{}, Set). sets:fold(fun(K, M) -> M#{K => V} end, #{}, Set).
-spec read_files(#{Type => filename()}) -> #{jid => jid_set(), url => url_set(), Type => sets:set(binary())} -spec read_files(#{Type => filename()}) ->
#{jid => jid_set(),
url => url_set(),
Type => sets:set(binary())}
when Type :: atom(). when Type :: atom().
read_files(Files) -> read_files(Files) ->
maps:map(fun(Type, Filename) -> maps:map(fun(Type, Filename) -> read_file(Filename, line_parser(Type)) end, Files).
read_file(Filename, line_parser(Type))
end,
Files).
-spec line_parser(Type :: atom()) -> fun((binary()) -> binary()). -spec line_parser(Type :: atom()) -> fun((binary()) -> binary()).
line_parser(jid) -> fun parse_jid/1; line_parser(jid) ->
line_parser(url) -> fun parse_url/1; fun parse_jid/1;
line_parser(_) -> fun trim/1. line_parser(url) ->
fun parse_url/1;
line_parser(_) ->
fun trim/1.
-spec read_file(filename(), fun((binary()) -> ljid() | url())) -spec read_file(filename(), fun((binary()) -> ljid() | url())) -> jid_set() | url_set().
-> jid_set() | url_set().
read_file(none, _ParseLine) -> read_file(none, _ParseLine) ->
sets:new(); sets:new();
read_file(File, ParseLine) -> read_file(File, ParseLine) ->
case file:open(File, [read, binary, raw, {read_ahead, 65536}]) of case file:open(File, [read, binary, raw, {read_ahead, 65536}]) of
{ok, Fd} -> {ok, Fd} ->
try read_line(Fd, ParseLine, sets:new()) try
catch throw:E -> throw({read, File, E}) read_line(Fd, ParseLine, sets:new())
after ok = file:close(Fd) catch
E ->
throw({read, File, E})
after
ok = file:close(Fd)
end; end;
{error, Reason} -> {error, Reason} ->
throw({open, File, Reason}) throw({open, File, Reason})
end. end.
-spec read_line(file:io_device(), fun((binary()) -> ljid() | url()), -spec read_line(file:io_device(),
jid_set() | url_set()) fun((binary()) -> ljid() | url()),
-> jid_set() | url_set(). jid_set() | url_set()) ->
jid_set() | url_set().
read_line(Fd, ParseLine, Set) -> read_line(Fd, ParseLine, Set) ->
case file:read_line(Fd) of case file:read_line(Fd) of
{ok, Line} -> {ok, Line} ->
@ -719,8 +779,10 @@ read_line(Fd, ParseLine, Set) ->
parse_jid(S) -> parse_jid(S) ->
try jid:decode(trim(S)) of try jid:decode(trim(S)) of
#jid{} = JID -> #jid{} = JID ->
jid:remove_resource(jid:tolower(JID)) jid:remove_resource(
catch _:{bad_jid, _} -> jid:tolower(JID))
catch
_:{bad_jid, _} ->
throw({bad_jid, S}) throw({bad_jid, S})
end. end.
@ -741,16 +803,22 @@ trim(S) ->
re:replace(S, <<"\\s+$">>, <<>>, [{return, binary}]). re:replace(S, <<"\\s+$">>, <<>>, [{return, binary}]).
-spec reject(stanza()) -> ok. -spec reject(stanza()) -> ok.
reject(#message{from = From, to = To, type = Type, lang = Lang} = Msg) reject(#message{from = From,
when Type /= groupchat, to = To,
Type /= error -> type = Type,
lang = Lang} =
Msg)
when Type /= groupchat, Type /= error ->
?INFO_MSG("Rejecting unsolicited message from ~s to ~s", ?INFO_MSG("Rejecting unsolicited message from ~s to ~s",
[jid:encode(From), jid:encode(To)]), [jid:encode(From), jid:encode(To)]),
Txt = <<"Your message is unsolicited">>, Txt = <<"Your message is unsolicited">>,
Err = xmpp:err_policy_violation(Txt, Lang), Err = xmpp:err_policy_violation(Txt, Lang),
maybe_dump_spam(Msg), maybe_dump_spam(Msg),
ejabberd_router:route_error(Msg, Err); ejabberd_router:route_error(Msg, Err);
reject(#presence{from = From, to = To, lang = Lang} = Presence) -> reject(#presence{from = From,
to = To,
lang = Lang} =
Presence) ->
?INFO_MSG("Rejecting unsolicited presence from ~s to ~s", ?INFO_MSG("Rejecting unsolicited presence from ~s to ~s",
[jid:encode(From), jid:encode(To)]), [jid:encode(From), jid:encode(To)]),
Txt = <<"Your traffic is unsolicited">>, Txt = <<"Your traffic is unsolicited">>,
@ -797,7 +865,8 @@ maybe_dump_spam(#message{to = #jid{lserver = LServer}} = Msg) ->
Proc = get_proc_name(LServer), Proc = get_proc_name(LServer),
Time = erlang:timestamp(), Time = erlang:timestamp(),
Msg1 = misc:add_delay_info(Msg, By, Time), Msg1 = misc:add_delay_info(Msg, By, Time),
XML = fxml:element_to_binary(xmpp:encode(Msg1)), XML = fxml:element_to_binary(
xmpp:encode(Msg1)),
gen_server:cast(Proc, {dump, XML}). gen_server:cast(Proc, {dump, XML}).
-spec get_proc_name(binary()) -> atom(). -spec get_proc_name(binary()) -> atom().
@ -860,7 +929,9 @@ shrink_cache(#state{jid_cache = Cache, max_cache_size = MaxSize} = State) ->
ShrinkedSize = round(MaxSize / 2), ShrinkedSize = round(MaxSize / 2),
N = map_size(Cache) - ShrinkedSize, N = map_size(Cache) - ShrinkedSize,
L = lists:keysort(2, maps:to_list(Cache)), L = lists:keysort(2, maps:to_list(Cache)),
Cache1 = maps:from_list(lists:nthtail(N, L)), Cache1 =
maps:from_list(
lists:nthtail(N, L)),
State#state{jid_cache = Cache1}. State#state{jid_cache = Cache1}.
-spec expire_cache(integer(), state()) -> {{ok, binary()}, state()}. -spec expire_cache(integer(), state()) -> {{ok, binary()}, state()}.
@ -893,65 +964,82 @@ drop_from_cache(LJID, #state{jid_cache = Cache} = State) ->
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
-spec get_commands_spec() -> [ejabberd_commands()]. -spec get_commands_spec() -> [ejabberd_commands()].
get_commands_spec() -> get_commands_spec() ->
[#ejabberd_commands{name = reload_spam_filter_files, tags = [filter], [#ejabberd_commands{name = reload_spam_filter_files,
tags = [filter],
desc = "Reload spam JID/URL files", desc = "Reload spam JID/URL files",
module = ?MODULE, function = reload_spam_filter_files, module = ?MODULE,
function = reload_spam_filter_files,
args = [{host, binary}], args = [{host, binary}],
result = {res, rescode}}, result = {res, rescode}},
#ejabberd_commands{name = get_spam_filter_cache, tags = [filter], #ejabberd_commands{name = get_spam_filter_cache,
tags = [filter],
desc = "Show spam filter cache contents", desc = "Show spam filter cache contents",
module = ?MODULE, function = get_spam_filter_cache, module = ?MODULE,
function = get_spam_filter_cache,
args = [{host, binary}], args = [{host, binary}],
result = {spammers, {list, {spammer, {tuple, result =
[{jid, string}, {timestamp, integer}]}}}}}, {spammers,
#ejabberd_commands{name = expire_spam_filter_cache, tags = [filter], {list, {spammer, {tuple, [{jid, string}, {timestamp, integer}]}}}}},
#ejabberd_commands{name = expire_spam_filter_cache,
tags = [filter],
desc = "Remove old/unused spam JIDs from cache", desc = "Remove old/unused spam JIDs from cache",
module = ?MODULE, function = expire_spam_filter_cache, module = ?MODULE,
function = expire_spam_filter_cache,
args = [{host, binary}, {seconds, integer}], args = [{host, binary}, {seconds, integer}],
result = {res, restuple}}, result = {res, restuple}},
#ejabberd_commands{name = add_to_spam_filter_cache, tags = [filter], #ejabberd_commands{name = add_to_spam_filter_cache,
tags = [filter],
desc = "Add JID to spam filter cache", desc = "Add JID to spam filter cache",
module = ?MODULE, module = ?MODULE,
function = add_to_spam_filter_cache, function = add_to_spam_filter_cache,
args = [{host, binary}, {jid, binary}], args = [{host, binary}, {jid, binary}],
result = {res, restuple}}, result = {res, restuple}},
#ejabberd_commands{name = drop_from_spam_filter_cache, tags = [filter], #ejabberd_commands{name = drop_from_spam_filter_cache,
tags = [filter],
desc = "Drop JID from spam filter cache", desc = "Drop JID from spam filter cache",
module = ?MODULE, module = ?MODULE,
function = drop_from_spam_filter_cache, function = drop_from_spam_filter_cache,
args = [{host, binary}, {jid, binary}], args = [{host, binary}, {jid, binary}],
result = {res, restuple}}, result = {res, restuple}},
#ejabberd_commands{name = get_blocked_domains, tags = [filter], #ejabberd_commands{name = get_blocked_domains,
tags = [filter],
desc = "Get list of domains being blocked", desc = "Get list of domains being blocked",
module = ?MODULE, module = ?MODULE,
function = get_blocked_domains, function = get_blocked_domains,
args = [{host, binary}], args = [{host, binary}],
result = {blocked_domains, {list, {jid, string}}}}, result = {blocked_domains, {list, {jid, string}}}},
#ejabberd_commands{name = add_blocked_domain, tags = [filter], #ejabberd_commands{name = add_blocked_domain,
tags = [filter],
desc = "Add domain to list of blocked domains", desc = "Add domain to list of blocked domains",
module = ?MODULE, module = ?MODULE,
function = add_blocked_domain, function = add_blocked_domain,
args = [{host, binary}, {domain, binary}], args = [{host, binary}, {domain, binary}],
result = {res, restuple}}, result = {res, restuple}},
#ejabberd_commands{name = remove_blocked_domain, tags = [filter], #ejabberd_commands{name = remove_blocked_domain,
tags = [filter],
desc = "Remove domain from list of blocked domains", desc = "Remove domain from list of blocked domains",
module = ?MODULE, module = ?MODULE,
function = remove_blocked_domain, function = remove_blocked_domain,
args = [{host, binary}, {domain, binary}], args = [{host, binary}, {domain, binary}],
result = {res, restuple}} result = {res, restuple}}].
].
for_all_hosts(F, A) -> for_all_hosts(F, A) ->
try lists:map( try lists:map(fun(Host) -> apply(F, [Host | A]) end, get_spam_filter_hosts()) of
fun(Host) ->
apply(F, [Host | A])
end, get_spam_filter_hosts()) of
List -> List ->
case lists:filter(fun({error, _}) -> true; (_) -> false end, List) of case lists:filter(fun ({error, _}) ->
[] -> hd(List); true;
Errors -> hd(Errors) (_) ->
false
end,
List)
of
[] ->
hd(List);
Errors ->
hd(Errors)
end end
catch error:{badmatch, {error, _Reason} = Error} -> catch
error:{badmatch, {error, _Reason} = Error} ->
Error Error
end. end.
@ -961,7 +1049,8 @@ try_call_by_host(Host, Call) ->
try gen_server:call(Proc, Call, ?COMMAND_TIMEOUT) of try gen_server:call(Proc, Call, ?COMMAND_TIMEOUT) of
Result -> Result ->
Result Result
catch exit:{noproc, _} -> catch
exit:{noproc, _} ->
{error, "Not configured for " ++ binary_to_list(Host)}; {error, "Not configured for " ++ binary_to_list(Host)};
exit:{timeout, _} -> exit:{timeout, _} ->
{error, "Timeout while querying ejabberd"} {error, "Timeout while querying ejabberd"}
@ -972,7 +1061,8 @@ reload_spam_filter_files(<<"global">>) ->
for_all_hosts(fun reload_spam_filter_files/1, []); for_all_hosts(fun reload_spam_filter_files/1, []);
reload_spam_filter_files(Host) -> reload_spam_filter_files(Host) ->
LServer = jid:nameprep(Host), LServer = jid:nameprep(Host),
Files = #{domains => gen_mod:get_module_opt(LServer, ?MODULE, spam_domains_file), Files =
#{domains => gen_mod:get_module_opt(LServer, ?MODULE, spam_domains_file),
jid => gen_mod:get_module_opt(LServer, ?MODULE, spam_jids_file), jid => gen_mod:get_module_opt(LServer, ?MODULE, spam_jids_file),
url => gen_mod:get_module_opt(LServer, ?MODULE, spam_urls_file)}, url => gen_mod:get_module_opt(LServer, ?MODULE, spam_urls_file)},
case try_call_by_host(Host, {reload_files, Files}) of case try_call_by_host(Host, {reload_files, Files}) of
@ -988,7 +1078,13 @@ reload_spam_filter_files(Host) ->
get_blocked_domains(Host) -> get_blocked_domains(Host) ->
case try_call_by_host(Host, get_blocked_domains) of case try_call_by_host(Host, get_blocked_domains) of
{blocked_domains, BlockedDomains} -> {blocked_domains, BlockedDomains} ->
maps:keys(maps:filter(fun(_, false) -> false; (_, _) -> true end, BlockedDomains)); maps:keys(
maps:filter(fun (_, false) ->
false;
(_, _) ->
true
end,
BlockedDomains));
{error, _R} = Error -> {error, _R} = Error ->
Error Error
end. end.
@ -1015,13 +1111,11 @@ remove_blocked_domain(Host, Domain) ->
Error Error
end. end.
-spec get_spam_filter_cache(binary()) -spec get_spam_filter_cache(binary()) -> [{binary(), integer()}] | {error, string()}.
-> [{binary(), integer()}] | {error, string()}.
get_spam_filter_cache(Host) -> get_spam_filter_cache(Host) ->
case try_call_by_host(Host, get_cache) of case try_call_by_host(Host, get_cache) of
{spam_filter, Cache} -> {spam_filter, Cache} ->
[{jid:encode(JID), TS + erlang:time_offset(second)} || [{jid:encode(JID), TS + erlang:time_offset(second)} || {JID, TS} <- Cache];
{JID, TS} <- Cache];
{error, _R} = Error -> {error, _R} = Error ->
Error Error
end. end.
@ -1037,20 +1131,24 @@ expire_spam_filter_cache(Host, Age) ->
Error Error
end. end.
-spec add_to_spam_filter_cache(binary(), binary()) -> [{binary(), integer()}] | {error, string()}. -spec add_to_spam_filter_cache(binary(), binary()) ->
[{binary(), integer()}] | {error, string()}.
add_to_spam_filter_cache(<<"global">>, JID) -> add_to_spam_filter_cache(<<"global">>, JID) ->
for_all_hosts(fun add_to_spam_filter_cache/2, [JID]); for_all_hosts(fun add_to_spam_filter_cache/2, [JID]);
add_to_spam_filter_cache(Host, EncJID) -> add_to_spam_filter_cache(Host, EncJID) ->
try jid:decode(EncJID) of try jid:decode(EncJID) of
#jid{} = JID -> #jid{} = JID ->
LJID = jid:remove_resource(jid:tolower(JID)), LJID =
jid:remove_resource(
jid:tolower(JID)),
case try_call_by_host(Host, {add_to_cache, LJID}) of case try_call_by_host(Host, {add_to_cache, LJID}) of
{spam_filter, {Status, Txt}} -> {spam_filter, {Status, Txt}} ->
{Status, binary_to_list(Txt)}; {Status, binary_to_list(Txt)};
{error, _R} = Error -> {error, _R} = Error ->
Error Error
end end
catch _:{bad_jid, _} -> catch
_:{bad_jid, _} ->
{error, "Not a valid JID: " ++ binary_to_list(EncJID)} {error, "Not a valid JID: " ++ binary_to_list(EncJID)}
end. end.
@ -1060,13 +1158,16 @@ drop_from_spam_filter_cache(<<"global">>, JID) ->
drop_from_spam_filter_cache(Host, EncJID) -> drop_from_spam_filter_cache(Host, EncJID) ->
try jid:decode(EncJID) of try jid:decode(EncJID) of
#jid{} = JID -> #jid{} = JID ->
LJID = jid:remove_resource(jid:tolower(JID)), LJID =
jid:remove_resource(
jid:tolower(JID)),
case try_call_by_host(Host, {drop_from_cache, LJID}) of case try_call_by_host(Host, {drop_from_cache, LJID}) of
{spam_filter, {Status, Txt}} -> {spam_filter, {Status, Txt}} ->
{Status, binary_to_list(Txt)}; {Status, binary_to_list(Txt)};
{error, _R} = Error -> {error, _R} = Error ->
Error Error
end end
catch _:{bad_jid, _} -> catch
_:{bad_jid, _} ->
{error, "Not a valid JID: " ++ binary_to_list(EncJID)} {error, "Not a valid JID: " ++ binary_to_list(EncJID)}
end. end.

43
src/mod_antispam_rtbl.erl
View file

@ -38,11 +38,15 @@
subscribe/3, subscribe/3,
unsubscribe/3]). unsubscribe/3]).
%% @format-begin
subscribe(RTBLHost, RTBLDomainsNode, From) -> subscribe(RTBLHost, RTBLDomainsNode, From) ->
FromJID = service_jid(From), FromJID = service_jid(From),
SubIQ = #iq{type = set, to = jid:make(RTBLHost), from = FromJID, SubIQ =
sub_els = [ #iq{type = set,
#pubsub{subscribe = #ps_subscribe{jid = FromJID, node = RTBLDomainsNode}}]}, to = jid:make(RTBLHost),
from = FromJID,
sub_els = [#pubsub{subscribe = #ps_subscribe{jid = FromJID, node = RTBLDomainsNode}}]},
?DEBUG("Sending subscription request:~n~p", [xmpp:encode(SubIQ)]), ?DEBUG("Sending subscription request:~n~p", [xmpp:encode(SubIQ)]),
ejabberd_router:route_iq(SubIQ, subscribe_result, self()). ejabberd_router:route_iq(SubIQ, subscribe_result, self()).
@ -51,19 +55,22 @@ unsubscribe(none, _PSNode, _From) ->
ok; ok;
unsubscribe(RTBLHost, RTBLDomainsNode, From) -> unsubscribe(RTBLHost, RTBLDomainsNode, From) ->
FromJID = jid:make(From), FromJID = jid:make(From),
SubIQ = #iq{type = set, to = jid:make(RTBLHost), from = FromJID, SubIQ =
sub_els = [ #iq{type = set,
#pubsub{unsubscribe = #ps_unsubscribe{jid = FromJID, node = RTBLDomainsNode}}]}, to = jid:make(RTBLHost),
from = FromJID,
sub_els =
[#pubsub{unsubscribe = #ps_unsubscribe{jid = FromJID, node = RTBLDomainsNode}}]},
ejabberd_router:route_iq(SubIQ, unsubscribe_result, self()). ejabberd_router:route_iq(SubIQ, unsubscribe_result, self()).
-spec request_blocked_domains(binary() | none, binary(), binary()) -> ok. -spec request_blocked_domains(binary() | none, binary(), binary()) -> ok.
request_blocked_domains(none, _PSNode, _From) -> request_blocked_domains(none, _PSNode, _From) ->
ok; ok;
request_blocked_domains(RTBLHost, RTBLDomainsNode, From) -> request_blocked_domains(RTBLHost, RTBLDomainsNode, From) ->
IQ = #iq{type = get, from = jid:make(From), IQ = #iq{type = get,
from = jid:make(From),
to = jid:make(RTBLHost), to = jid:make(RTBLHost),
sub_els = [ sub_els = [#pubsub{items = #ps_items{node = RTBLDomainsNode}}]},
#pubsub{items = #ps_items{node = RTBLDomainsNode}}]},
?DEBUG("Requesting RTBL blocked domains from ~s:~n~p", [RTBLHost, xmpp:encode(IQ)]), ?DEBUG("Requesting RTBL blocked domains from ~s:~n~p", [RTBLHost, xmpp:encode(IQ)]),
ejabberd_router:route_iq(IQ, blocked_domains, self()). ejabberd_router:route_iq(IQ, blocked_domains, self()).
@ -83,7 +90,10 @@ parse_blocked_domains(#iq{to = #jid{lserver = LServer}, type = result} = IQ) ->
parse_pubsub_event(#message{to = #jid{lserver = LServer}} = Msg) -> parse_pubsub_event(#message{to = #jid{lserver = LServer}} = Msg) ->
RTBLDomainsNode = gen_mod:get_module_opt(LServer, ?SERVICE_MODULE, rtbl_domains_node), RTBLDomainsNode = gen_mod:get_module_opt(LServer, ?SERVICE_MODULE, rtbl_domains_node),
case xmpp:get_subtag(Msg, #ps_event{}) of case xmpp:get_subtag(Msg, #ps_event{}) of
#ps_event{items = #ps_items{node = RTBLDomainsNode, items = Items, retract = RetractIds}} -> #ps_event{items =
#ps_items{node = RTBLDomainsNode,
items = Items,
retract = RetractIds}} ->
maps:merge(retract_items(RetractIds), parse_items(Items)); maps:merge(retract_items(RetractIds), parse_items(Items));
Other -> Other ->
?WARNING_MSG("Couldn't extract items: ~p", [Other]), ?WARNING_MSG("Couldn't extract items: ~p", [Other]),
@ -92,11 +102,12 @@ parse_pubsub_event(#message{to = #jid{lserver = LServer}} = Msg) ->
-spec parse_items([ps_item()]) -> #{binary() => any()}. -spec parse_items([ps_item()]) -> #{binary() => any()}.
parse_items(Items) -> parse_items(Items) ->
lists:foldl( lists:foldl(fun(#ps_item{id = ID}, Acc) ->
fun(#ps_item{id = ID}, Acc) ->
%% TODO extract meta/extra instructions %% TODO extract meta/extra instructions
maps:put(ID, true, Acc) maps:put(ID, true, Acc)
end, #{}, Items). end,
#{},
Items).
-spec retract_items([binary()]) -> #{binary() => false}. -spec retract_items([binary()]) -> #{binary() => false}.
retract_items(Ids) -> retract_items(Ids) ->
@ -112,8 +123,10 @@ service_jid(Host) ->
-spec pubsub_event_handler(stanza()) -> drop | stanza(). -spec pubsub_event_handler(stanza()) -> drop | stanza().
pubsub_event_handler(#message{from = FromJid, pubsub_event_handler(#message{from = FromJid,
to = #jid{lserver = LServer, to =
lresource = <<?SERVICE_JID_PREFIX, _/binary>>}} = Msg) -> #jid{lserver = LServer,
lresource = <<?SERVICE_JID_PREFIX, _/binary>>}} =
Msg) ->
?DEBUG("Got RTBL message:~n~p", [Msg]), ?DEBUG("Got RTBL message:~n~p", [Msg]),
From = jid:encode(FromJid), From = jid:encode(FromJid),
case gen_mod:get_module_opt(LServer, ?SERVICE_MODULE, rtbl_host) of case gen_mod:get_module_opt(LServer, ?SERVICE_MODULE, rtbl_host) of

92
test/antispam_tests.erl
View file

@ -31,6 +31,8 @@
my_muc_jid/1, get_features/2, set_opt/3]). my_muc_jid/1, get_features/2, set_opt/3]).
-include("suite.hrl"). -include("suite.hrl").
%% @format-begin
%%%=================================================================== %%%===================================================================
%%% API %%% API
%%%=================================================================== %%%===================================================================
@ -38,7 +40,8 @@
%%% Single tests %%% Single tests
%%%=================================================================== %%%===================================================================
single_cases() -> single_cases() ->
{antispam_single, [sequence], {antispam_single,
[sequence],
[single_test(spam_files), [single_test(spam_files),
single_test(blocked_domains), single_test(blocked_domains),
single_test(jid_cache), single_test(jid_cache),
@ -49,24 +52,39 @@ spam_files(Config) ->
To = my_jid(Config), To = my_jid(Config),
SpamJID = jid:make(<<"spammer_jid">>, <<"localhost">>, <<"spam_client">>), SpamJID = jid:make(<<"spammer_jid">>, <<"localhost">>, <<"spam_client">>),
SpamJIDMsg = #message{from = SpamJID, to = To, type = chat, body = [#text{data = <<"hello world">>}]}, SpamJIDMsg =
#message{from = SpamJID,
to = To,
type = chat,
body = [#text{data = <<"hello world">>}]},
is_spam(SpamJIDMsg), is_spam(SpamJIDMsg),
Spammer = jid:make(<<"spammer">>, <<"localhost">>, <<"spam_client">>), Spammer = jid:make(<<"spammer">>, <<"localhost">>, <<"spam_client">>),
NoSpamMsg = #message{from = Spammer, to = To, type = chat, body = [#text{data = <<"hello world">>}]}, NoSpamMsg =
#message{from = Spammer,
to = To,
type = chat,
body = [#text{data = <<"hello world">>}]},
is_not_spam(NoSpamMsg), is_not_spam(NoSpamMsg),
SpamMsg = #message{from = Spammer, to = To, type = chat, body = [#text{data = <<"hello world\nhttps://spam.domain.url">>}]}, SpamMsg =
#message{from = Spammer,
to = To,
type = chat,
body = [#text{data = <<"hello world\nhttps://spam.domain.url">>}]},
is_spam(SpamMsg), is_spam(SpamMsg),
%% now check this mischief is in jid_cache %% now check this mischief is in jid_cache
is_spam(NoSpamMsg), is_spam(NoSpamMsg),
mod_antispam:drop_from_spam_filter_cache(Host, jid:to_string(Spammer)), mod_antispam:drop_from_spam_filter_cache(Host, jid:to_string(Spammer)),
is_not_spam(NoSpamMsg), is_not_spam(NoSpamMsg),
?retry(100, 10, ?retry(100, 10, ?match(true, (has_spam_domain(<<"spam_domain.org">>))(Host))),
?match(true, (has_spam_domain(<<"spam_domain.org">>))(Host))),
SpamDomain = jid:make(<<"spammer">>, <<"spam_domain.org">>, <<"spam_client">>), SpamDomain = jid:make(<<"spammer">>, <<"spam_domain.org">>, <<"spam_client">>),
SpamDomainMsg = #message{from = SpamDomain, to = To, type = chat, body = [#text{data = <<"hello world">>}]}, SpamDomainMsg =
#message{from = SpamDomain,
to = To,
type = chat,
body = [#text{data = <<"hello world">>}]},
is_spam(SpamDomainMsg), is_spam(SpamDomainMsg),
?match({ok, _}, mod_antispam:remove_blocked_domain(Host, <<"spam_domain.org">>)), ?match({ok, _}, mod_antispam:remove_blocked_domain(Host, <<"spam_domain.org">>)),
?match([], mod_antispam:get_blocked_domains(Host)), ?match([], mod_antispam:get_blocked_domains(Host)),
@ -78,7 +96,10 @@ blocked_domains(Config) ->
?match([], mod_antispam:get_blocked_domains(Host)), ?match([], mod_antispam:get_blocked_domains(Host)),
SpamFrom = jid:make(<<"spammer">>, <<"spam.domain">>, <<"spam_client">>), SpamFrom = jid:make(<<"spammer">>, <<"spam.domain">>, <<"spam_client">>),
To = my_jid(Config), To = my_jid(Config),
Msg = #message{from = SpamFrom, to = To, type = chat, body = [#text{data = <<"hello world">>}]}, Msg = #message{from = SpamFrom,
to = To,
type = chat,
body = [#text{data = <<"hello world">>}]},
is_not_spam(Msg), is_not_spam(Msg),
?match({ok, _}, mod_antispam:add_blocked_domain(<<"global">>, <<"spam.domain">>)), ?match({ok, _}, mod_antispam:add_blocked_domain(<<"global">>, <<"spam.domain">>)),
is_spam(Msg), is_spam(Msg),
@ -102,7 +123,10 @@ jid_cache(Config) ->
Host = ?config(server, Config), Host = ?config(server, Config),
SpamFrom = jid:make(<<"spammer">>, Host, <<"spam_client">>), SpamFrom = jid:make(<<"spammer">>, Host, <<"spam_client">>),
To = my_jid(Config), To = my_jid(Config),
Msg = #message{from = SpamFrom, to = To, type = chat, body = [#text{data = <<"hello world">>}]}, Msg = #message{from = SpamFrom,
to = To,
type = chat,
body = [#text{data = <<"hello world">>}]},
is_not_spam(Msg), is_not_spam(Msg),
mod_antispam:add_to_spam_filter_cache(Host, jid:to_string(SpamFrom)), mod_antispam:add_to_spam_filter_cache(Host, jid:to_string(SpamFrom)),
is_spam(Msg), is_spam(Msg),
@ -112,25 +136,45 @@ jid_cache(Config) ->
rtbl_domains(Config) -> rtbl_domains(Config) ->
Host = ?config(server, Config), Host = ?config(server, Config),
RTBLHost = jid:to_string(suite:pubsub_jid(Config)), RTBLHost =
jid:to_string(
suite:pubsub_jid(Config)),
RTBLDomainsNode = <<"spam_source_domains">>, RTBLDomainsNode = <<"spam_source_domains">>,
OldOpts = gen_mod:get_module_opts(Host, mod_antispam), OldOpts = gen_mod:get_module_opts(Host, mod_antispam),
NewOpts = maps:merge(OldOpts, #{rtbl_host => RTBLHost, rtbl_domains_node => RTBLDomainsNode}), NewOpts =
maps:merge(OldOpts, #{rtbl_host => RTBLHost, rtbl_domains_node => RTBLDomainsNode}),
Owner = jid:make(?config(user, Config), ?config(server, Config), <<>>), Owner = jid:make(?config(user, Config), ?config(server, Config), <<>>),
{result, _} = mod_pubsub:create_node(RTBLHost, ?config(server, Config), RTBLDomainsNode, Owner, <<"flat">>), {result, _} =
{result, _} = mod_pubsub:publish_item(RTBLHost, ?config(server, Config), RTBLDomainsNode, Owner, <<"spam.source.domain">>, mod_pubsub:create_node(RTBLHost,
[xmpp:encode(#ps_item{id = <<"spam.source.domain">>, sub_els = []})]), ?config(server, Config),
RTBLDomainsNode,
Owner,
<<"flat">>),
{result, _} =
mod_pubsub:publish_item(RTBLHost,
?config(server, Config),
RTBLDomainsNode,
Owner,
<<"spam.source.domain">>,
[xmpp:encode(#ps_item{id = <<"spam.source.domain">>,
sub_els = []})]),
mod_antispam:reload(Host, OldOpts, NewOpts), mod_antispam:reload(Host, OldOpts, NewOpts),
?match({ok, _}, mod_antispam:remove_blocked_domain(Host, <<"spam_domain.org">>)), ?match({ok, _}, mod_antispam:remove_blocked_domain(Host, <<"spam_domain.org">>)),
?retry(100, 10, ?retry(100,
10,
?match([<<"spam.source.domain">>], mod_antispam:get_blocked_domains(Host))), ?match([<<"spam.source.domain">>], mod_antispam:get_blocked_domains(Host))),
{result, _} = mod_pubsub:publish_item(RTBLHost, ?config(server, Config), RTBLDomainsNode, Owner, <<"spam.source.another">>, {result, _} =
[xmpp:encode(#ps_item{id = <<"spam.source.another">>, sub_els = []})]), mod_pubsub:publish_item(RTBLHost,
?retry(100, 10, ?config(server, Config),
?match(true, (has_spam_domain(<<"spam.source.another">>))(Host))), RTBLDomainsNode,
{result, _} = mod_pubsub:delete_item(RTBLHost, RTBLDomainsNode, Owner, <<"spam.source.another">>, true), Owner,
?retry(100, 10, <<"spam.source.another">>,
?match(false, (has_spam_domain(<<"spam.source.another">>))(Host))), [xmpp:encode(#ps_item{id = <<"spam.source.another">>,
sub_els = []})]),
?retry(100, 10, ?match(true, (has_spam_domain(<<"spam.source.another">>))(Host))),
{result, _} =
mod_pubsub:delete_item(RTBLHost, RTBLDomainsNode, Owner, <<"spam.source.another">>, true),
?retry(100, 10, ?match(false, (has_spam_domain(<<"spam.source.another">>))(Host))),
{result, _} = mod_pubsub:delete_node(RTBLHost, RTBLDomainsNode, Owner), {result, _} = mod_pubsub:delete_node(RTBLHost, RTBLDomainsNode, Owner),
disconnect(Config). disconnect(Config).
@ -141,9 +185,7 @@ single_test(T) ->
list_to_atom("antispam_" ++ atom_to_list(T)). list_to_atom("antispam_" ++ atom_to_list(T)).
has_spam_domain(Domain) -> has_spam_domain(Domain) ->
fun(Host) -> fun(Host) -> lists:member(Domain, mod_antispam:get_blocked_domains(Host)) end.
lists:member(Domain, mod_antispam:get_blocked_domains(Host))
end.
is_not_spam(Msg) -> is_not_spam(Msg) ->
?match({Msg, undefined}, mod_antispam:s2s_receive_packet({Msg, undefined})). ?match({Msg, undefined}, mod_antispam:s2s_receive_packet({Msg, undefined})).