1
0
Fork 0
mirror of https://github.com/deltachat/deltachat-core.git synced 2025-10-06 12:00:11 +02:00

Rewrite code to get new messages from IMAP-server, avoid storing invalid lastseenuid.

This commit is contained in:
B. Petersen 2018-02-12 17:24:00 +01:00
parent 36dbb69a78
commit 15a8ac7f44
3 changed files with 123 additions and 116 deletions

View file

@ -74,6 +74,44 @@ static int is_error(mrimap_t* ths, int code)
} }
static void get_config_lastseenuid(mrimap_t* imap, const char* folder, uint32_t* uidvalidity, uint32_t* lastseenuid)
{
*uidvalidity = 0;
*lastseenuid = 0;
char* key = mr_mprintf("imap.mailbox.%s", folder);
char* val1 = imap->m_get_config(imap, key, NULL), *val2 = NULL, *val3 = NULL;
if( val1 )
{
/* the entry has the format `imap.mailbox.<folder>=<uidvalidity>:<lastseenuid>` */
val2 = strchr(val1, ':');
if( val2 )
{
*val2 = 0;
val2++;
val3 = strchr(val2, ':');
if( val3 ) { *val3 = 0; /* ignore everything bethind an optional second colon to allow future enhancements */ }
*uidvalidity = atol(val1);
*lastseenuid = atol(val2);
}
}
free(val1); /* val2 and val3 are only pointers inside val1 and MUST NOT be free()'d */
free(key);
}
static void set_config_lastseenuid(mrimap_t* imap, const char* folder, uint32_t uidvalidity, uint32_t lastseenuid)
{
char* key = mr_mprintf("imap.mailbox.%s", folder);
char* val = mr_mprintf("%lu:%lu", uidvalidity, lastseenuid);
imap->m_set_config(imap, key, val);
free(val);
free(key);
}
/******************************************************************************* /*******************************************************************************
* Handle folders * Handle folders
******************************************************************************/ ******************************************************************************/
@ -305,9 +343,9 @@ static int init_chat_folders__(mrimap_t* ths)
/* Subscribe to the created folder. Otherwise, although a top-level folder, if clients use LSUB for listing, the created folder may be hidden. /* Subscribe to the created folder. Otherwise, although a top-level folder, if clients use LSUB for listing, the created folder may be hidden.
(we could also do this directly after creation, however, we forgot this in versions <v0.1.19 */ (we could also do this directly after creation, however, we forgot this in versions <v0.1.19 */
if( chats_folder && ths->m_get_config_int(ths, "imap.subscribedToChats", 0)==0 ) { if( chats_folder && ths->m_get_config(ths, "imap.subscribedToChats", NULL)==NULL ) {
mailimap_subscribe(ths->m_hEtpan, chats_folder); mailimap_subscribe(ths->m_hEtpan, chats_folder);
ths->m_set_config_int(ths, "imap.subscribedToChats", 1); ths->m_set_config(ths, "imap.subscribedToChats", "1");
} }
if( chats_folder ) { if( chats_folder ) {
@ -553,16 +591,15 @@ cleanup:
} }
static int fetch_from_single_folder(mrimap_t* ths, const char* folder, uint32_t uidvalidity) static int fetch_from_single_folder(mrimap_t* ths, const char* folder)
{ {
int r, handle_locked = 0, log_summary = 1; int r, handle_locked = 0;
clist* fetch_result = NULL; uint32_t uidvalidity = 0;
uint32_t out_largetst_uid = 0; uint32_t lastseenuid = 0, new_lastseenuid = 0;
size_t read_cnt = 0, read_errors = 0; clist* fetch_result = NULL;
clistiter* cur; size_t read_cnt = 0, read_errors = 0;
clistiter* cur;
uint32_t lastuid = 0; /* The last uid fetched, we fetch from lastuid+1. If 0, we get some of the newest ones. */ struct mailimap_set* set;
char* lastuid_config_key = NULL;
if( ths==NULL ) { if( ths==NULL ) {
goto cleanup; goto cleanup;
@ -575,87 +612,68 @@ static int fetch_from_single_folder(mrimap_t* ths, const char* folder, uint32_t
goto cleanup; goto cleanup;
} }
if( uidvalidity ) if( select_folder__(ths, folder)==0 ) {
{ mrmailbox_log_warning(ths->m_mailbox, 0, "Cannot select folder \"%s\".", folder);
lastuid_config_key = mr_mprintf("imap.lastuid.%lu.%s", goto cleanup;
(unsigned long)uidvalidity, folder); /* RFC3501: UID are unique and should grow only, for mailbox recreation etc. UIDVALIDITY changes. */
lastuid = ths->m_get_config_int(ths, lastuid_config_key, 0);
if( lastuid > 0 ) {
struct mailimap_set* set = mailimap_set_new_interval(lastuid+1, 0);
r = mailimap_uid_fetch(ths->m_hEtpan, set, ths->m_fetch_type_uid, &fetch_result); /* execute UID FETCH from:to command, result includes the given UIDs */
mailimap_set_free(set);
}
else {
/* fall back to init behaviour below */
free(lastuid_config_key);
lastuid_config_key = NULL;
lastuid = 0;
}
mrmailbox_log_info(ths->m_mailbox, 0, "%s=%lu", lastuid_config_key, (unsigned long)lastuid);
} }
if( lastuid == 0 ) /* compare last seen UIDVALIDITY against the current one */
get_config_lastseenuid(ths, folder, &uidvalidity, &lastseenuid);
if( uidvalidity != ths->m_hEtpan->imap_selection_info->sel_uidvalidity )
{ {
if( select_folder__(ths, folder)==0 ) { /* first time this folder is selected or UIDVALIDITY has changed, init lastseenuid and save it to config */
mrmailbox_log_warning(ths->m_mailbox, 0, "Cannot select folder \"%s\".", folder); mrmailbox_log_info(ths->m_mailbox, 0, "Init lastseenuid and attach it to UIDVALIDITY for folder \"%s\".", folder);
log_summary = 0; if( ths->m_hEtpan->imap_selection_info->sel_uidvalidity <= 0 ) {
mrmailbox_log_error(ths->m_mailbox, 0, "Cannot get UIDVALIDITY for folder \"%s\".", folder);
goto cleanup; goto cleanup;
} }
lastuid_config_key = mr_mprintf("imap.lastuid.%lu.%s", if( ths->m_hEtpan->imap_selection_info->sel_has_exists ) {
(unsigned long)ths->m_hEtpan->imap_selection_info->sel_uidvalidity, folder); /* RFC3501: UIDs are unique and should grow only, for mailbox recreation etc. UIDVALIDITY changes. */ if( ths->m_hEtpan->imap_selection_info->sel_exists <= 0 ) {
lastuid = ths->m_get_config_int(ths, lastuid_config_key, 0); mrmailbox_log_info(ths->m_mailbox, 0, "Folder \"%s\" is empty.", folder);
mrmailbox_log_info(ths->m_mailbox, 0, "%s=%lu (validity read from folder)", lastuid_config_key, (unsigned long)lastuid);
if( lastuid == 0 ) {
if( ths->m_hEtpan->imap_selection_info->sel_uidnext != 0 ) {
lastuid = ths->m_hEtpan->imap_selection_info->sel_uidnext - 1; /* this is not always exact, however, as we only use this as "range start", this is no real problem */
ths->m_set_config_int(ths, lastuid_config_key, lastuid); /* write back the UID as otherwise we'll never receive new messages as UIDNEXT grows as messages come in */
}
}
if( lastuid > 0 )
{
/* Get messages with an ID larger than the one we got last time */
/* check if the predicted "next uid on insert" is equal to the one we start for fetching
-> no new messages (this check only works for the folder with the last message, however, most times this is INBOX - and the check is cheap)
--> unfortunately, this check does not work with our cached select!
if( ths->m_hEtpan->imap_selection_info->sel_uidnext == lastuid+1 ) {
goto cleanup; goto cleanup;
} }
*/ /* `FETCH <message sequence number> (UID)` */
set = mailimap_set_new_single(ths->m_hEtpan->imap_selection_info->sel_exists);
struct mailimap_set* set = mailimap_set_new_interval(lastuid+1, 0);
r = mailimap_uid_fetch(ths->m_hEtpan, set, ths->m_fetch_type_uid, &fetch_result); /* execute UID FETCH from:to command, result includes the given UIDs */
mailimap_set_free(set);
} }
else else {
{ /* `FETCH * (UID)` - according to RFC 3501, `*` represents the largest message sequence number; if the mailbox is empty,
/* fetch the newest message to get the UID */ an error resp. an empty list is returned. */
struct mailimap_set* set = mailimap_set_new_single(0); // 0=*=the _largest_ index (results in `FETCH * (UID)` to get the latest message) mrmailbox_log_info(ths->m_mailbox, 0, "EXISTS is missing for folder \"%s\", using fallback.", folder);
r = mailimap_fetch(ths->m_hEtpan, set, ths->m_fetch_type_uid, &fetch_result); /* execute FETCH from:to command, result includes the given index */ set = mailimap_set_new_single(0);
mailimap_set_free(set);
#if 0
/* fetch the last 200 messages by one-based-index.
This implementation seems to make problems if we cannot get the count for any reasons ... see mailbox.org ... */
int32_t i_first = 1, i_last = 200; /* if we cannot get the count, we start with the oldest messages; normally, this should not happen */
if( ths->m_hEtpan->imap_selection_info->sel_has_exists ) {
i_last = ths->m_hEtpan->imap_selection_info->sel_exists;
i_first = MR_MAX(i_last-200, 1);
}
struct mailimap_set* set = mailimap_set_new_interval(i_first, i_last);
r = mailimap_fetch(ths->m_hEtpan, set, ths->m_fetch_type_uid, &fetch_result); /* execute FETCH from:to command, result includes the given index */
mailimap_set_free(set);
#endif
} }
r = mailimap_fetch(ths->m_hEtpan, set, ths->m_fetch_type_uid, &fetch_result);
mailimap_set_free(set);
if( is_error(ths, r) || fetch_result==NULL || (cur=clist_begin(fetch_result))==NULL ) {
mrmailbox_log_info(ths->m_mailbox, 0, "Empty result returned for folder \"%s\".", folder);
goto cleanup; /* this might happen if the mailbox is empty an EXISTS does not work */
}
struct mailimap_msg_att* msg_att = (struct mailimap_msg_att*)clist_content(cur);
lastseenuid = peek_uid(msg_att);
mailimap_fetch_list_free(fetch_result);
fetch_result = NULL;
if( lastseenuid <= 0 ) {
mrmailbox_log_error(ths->m_mailbox, 0, "Cannot get largest UID for folder \"%s\"", folder);
goto cleanup;
}
/* if the UIDVALIDITY has _changed_, decrease lastseenuid by one to avoid gaps (well add 1 below) */
if( uidvalidity > 0 && lastseenuid > 1 ) {
lastseenuid -= 1;
}
/* store calculated uidvalidity/lastseenuid */
uidvalidity = ths->m_hEtpan->imap_selection_info->sel_uidvalidity;
set_config_lastseenuid(ths, folder, uidvalidity, lastseenuid);
} }
/* fetch messages with larger UID than the last one seen (`UID FETCH lastseenuid+1:*)`, see RFC 4549 */
set = mailimap_set_new_interval(lastseenuid+1, 0);
r = mailimap_uid_fetch(ths->m_hEtpan, set, ths->m_fetch_type_uid, &fetch_result);
mailimap_set_free(set);
UNLOCK_HANDLE UNLOCK_HANDLE
if( is_error(ths, r) || fetch_result == NULL ) if( is_error(ths, r) || fetch_result == NULL )
@ -666,7 +684,6 @@ static int fetch_from_single_folder(mrimap_t* ths, const char* folder, uint32_t
goto cleanup; /* the folder is simply empty, this is no error */ goto cleanup; /* the folder is simply empty, this is no error */
} }
mrmailbox_log_warning(ths->m_mailbox, 0, "Cannot fetch message list from folder \"%s\".", folder); mrmailbox_log_warning(ths->m_mailbox, 0, "Cannot fetch message list from folder \"%s\".", folder);
log_summary = 0;
goto cleanup; goto cleanup;
} }
@ -675,31 +692,28 @@ static int fetch_from_single_folder(mrimap_t* ths, const char* folder, uint32_t
{ {
struct mailimap_msg_att* msg_att = (struct mailimap_msg_att*)clist_content(cur); /* mailimap_msg_att is a list of attributes: list is a list of message attributes */ struct mailimap_msg_att* msg_att = (struct mailimap_msg_att*)clist_content(cur); /* mailimap_msg_att is a list of attributes: list is a list of message attributes */
uint32_t cur_uid = peek_uid(msg_att); uint32_t cur_uid = peek_uid(msg_att);
if( cur_uid /*&& (lastuid==0 || cur_uid>lastuid)*/ ) if( cur_uid > 0
/* normally, the "cur_uid>lastuid" is not needed, however, some server return some smaller IDs && cur_uid!=lastseenuid /* `UID FETCH <lastseenuid+1>:*` may include lastseenuid if "*" == lastseenuid */ )
under some circumstances. Mailcore2 does the same check, see "if (uid < fromUID) {..}"@IMAPSession::fetchMessageNumberUIDMapping()@MCIMAPSession.cpp
however, more worse, _if_ we have this check, for servers which _do_ emit smaller UIDs, this avoids getting new messages.
all in all, this seems to require more research. */
{ {
read_cnt++; read_cnt++;
if( fetch_single_msg(ths, folder, cur_uid, 0) == 0 ) { if( fetch_single_msg(ths, folder, cur_uid, 0) == 0/* 0=try again later*/ ) {
read_errors++; read_errors++;
} }
else if( cur_uid > out_largetst_uid ) { else if( cur_uid > new_lastseenuid ) {
out_largetst_uid = cur_uid; new_lastseenuid = cur_uid;
} }
} }
} }
if( !read_errors && out_largetst_uid > 0 ) { if( !read_errors && new_lastseenuid > 0 ) {
ths->m_set_config_int(ths, lastuid_config_key, out_largetst_uid); set_config_lastseenuid(ths, folder, uidvalidity, new_lastseenuid);
} }
/* done */ /* done */
cleanup: cleanup:
UNLOCK_HANDLE UNLOCK_HANDLE
if( log_summary )
{ {
char* temp = mr_mprintf("%i mails read from \"%s\" with %i errors.", (int)read_cnt, folder, (int)read_errors); char* temp = mr_mprintf("%i mails read from \"%s\" with %i errors.", (int)read_cnt, folder, (int)read_errors);
if( read_errors ) { if( read_errors ) {
@ -715,10 +729,6 @@ cleanup:
mailimap_fetch_list_free(fetch_result); mailimap_fetch_list_free(fetch_result);
} }
if( lastuid_config_key ) {
free(lastuid_config_key);
}
return read_cnt; return read_cnt;
} }
@ -742,7 +752,7 @@ static int fetch_from_all_folders(mrimap_t* ths)
{ {
mrimapfolder_t* folder = (mrimapfolder_t*)clist_content(cur); mrimapfolder_t* folder = (mrimapfolder_t*)clist_content(cur);
if( folder->m_meaning == MEANING_INBOX ) { if( folder->m_meaning == MEANING_INBOX ) {
total_cnt += fetch_from_single_folder(ths, folder->m_name_to_select, 0); total_cnt += fetch_from_single_folder(ths, folder->m_name_to_select);
} }
} }
@ -753,7 +763,7 @@ static int fetch_from_all_folders(mrimap_t* ths)
mrmailbox_log_info(ths->m_mailbox, 0, "Folder \"%s\" ignored.", folder->m_name_utf8); mrmailbox_log_info(ths->m_mailbox, 0, "Folder \"%s\" ignored.", folder->m_name_utf8);
} }
else if( folder->m_meaning != MEANING_INBOX ) { else if( folder->m_meaning != MEANING_INBOX ) {
total_cnt += fetch_from_single_folder(ths, folder->m_name_to_select, 0); total_cnt += fetch_from_single_folder(ths, folder->m_name_to_select);
} }
} }
@ -789,7 +799,6 @@ static void* watch_thread_entry_point(void* entry_arg)
**********************************************************************/ **********************************************************************/
int r, r2; int r, r2;
uint32_t uidvalidity;
fetch_from_all_folders(ths); /* the initial fetch from all folders is needed as this will init the folder UIDs (see fetch_from_single_folder() if lastuid is unset) */ fetch_from_all_folders(ths); /* the initial fetch from all folders is needed as this will init the folder UIDs (see fetch_from_single_folder() if lastuid is unset) */
last_fullread_time = time(NULL); last_fullread_time = time(NULL);
@ -805,7 +814,6 @@ static void* watch_thread_entry_point(void* entry_arg)
do_fetch = 0; do_fetch = 0;
force_sleep = SLEEP_ON_ERROR_SECONDS; force_sleep = SLEEP_ON_ERROR_SECONDS;
uidvalidity = 0;
setup_handle_if_needed__(ths); setup_handle_if_needed__(ths);
if( ths->m_idle_set_up==0 && ths->m_hEtpan && ths->m_hEtpan->imap_stream ) { if( ths->m_idle_set_up==0 && ths->m_hEtpan && ths->m_hEtpan->imap_stream ) {
@ -824,7 +832,6 @@ static void* watch_thread_entry_point(void* entry_arg)
if( select_folder__(ths, "INBOX") ) if( select_folder__(ths, "INBOX") )
{ {
uidvalidity = ths->m_hEtpan->imap_selection_info->sel_uidvalidity;
r = mailimap_idle(ths->m_hEtpan); r = mailimap_idle(ths->m_hEtpan);
if( !is_error(ths, r) ) if( !is_error(ths, r) )
{ {
@ -885,7 +892,7 @@ static void* watch_thread_entry_point(void* entry_arg)
} }
if( do_fetch == 1 ) { if( do_fetch == 1 ) {
fetch_from_single_folder(ths, "INBOX", uidvalidity); fetch_from_single_folder(ths, "INBOX");
} }
else if( do_fetch == 2 ) { else if( do_fetch == 2 ) {
fetch_from_all_folders(ths); fetch_from_all_folders(ths);
@ -919,7 +926,7 @@ static void* watch_thread_entry_point(void* entry_arg)
UNLOCK_HANDLE UNLOCK_HANDLE
if( do_fetch == 1 ) { if( do_fetch == 1 ) {
if( fetch_from_single_folder(ths, "INBOX", 0) > 0 ) { if( fetch_from_single_folder(ths, "INBOX") > 0 ) {
last_message_time = now; last_message_time = now;
} }
} }
@ -1342,7 +1349,7 @@ int mrimap_is_connected(mrimap_t* ths)
******************************************************************************/ ******************************************************************************/
mrimap_t* mrimap_new(mr_get_config_int_t get_config_int, mr_set_config_int_t set_config_int, mr_receive_imf_t receive_imf, void* userData, mrmailbox_t* mailbox) mrimap_t* mrimap_new(mr_get_config_t get_config, mr_set_config_t set_config, mr_receive_imf_t receive_imf, void* userData, mrmailbox_t* mailbox)
{ {
mrimap_t* ths = NULL; mrimap_t* ths = NULL;
@ -1353,8 +1360,8 @@ mrimap_t* mrimap_new(mr_get_config_int_t get_config_int, mr_set_config_int_t set
ths->m_log_connect_errors = 1; ths->m_log_connect_errors = 1;
ths->m_mailbox = mailbox; ths->m_mailbox = mailbox;
ths->m_get_config_int = get_config_int; ths->m_get_config = get_config;
ths->m_set_config_int = set_config_int; ths->m_set_config = set_config;
ths->m_receive_imf = receive_imf; ths->m_receive_imf = receive_imf;
ths->m_userData = userData; ths->m_userData = userData;

View file

@ -37,8 +37,8 @@ typedef struct mrimap_t mrimap_t;
#define MR_IMAP_SEEN 0x0001L #define MR_IMAP_SEEN 0x0001L
typedef int32_t (*mr_get_config_int_t)(mrimap_t*, const char*, int32_t); typedef char* (*mr_get_config_t) (mrimap_t*, const char*, const char*);
typedef void (*mr_set_config_int_t)(mrimap_t*, const char*, int32_t); typedef void (*mr_set_config_t) (mrimap_t*, const char*, const char*);
typedef void (*mr_receive_imf_t) (mrimap_t*, const char* imf_raw_not_terminated, size_t imf_raw_bytes, const char* server_folder, uint32_t server_uid, uint32_t flags); typedef void (*mr_receive_imf_t) (mrimap_t*, const char* imf_raw_not_terminated, size_t imf_raw_bytes, const char* server_folder, uint32_t server_uid, uint32_t flags);
@ -85,8 +85,8 @@ typedef struct mrimap_t
struct mailimap_fetch_type* m_fetch_type_body; struct mailimap_fetch_type* m_fetch_type_body;
struct mailimap_fetch_type* m_fetch_type_flags; struct mailimap_fetch_type* m_fetch_type_flags;
mr_get_config_int_t m_get_config_int; mr_get_config_t m_get_config;
mr_set_config_int_t m_set_config_int; mr_set_config_t m_set_config;
mr_receive_imf_t m_receive_imf; mr_receive_imf_t m_receive_imf;
void* m_userData; void* m_userData;
mrmailbox_t* m_mailbox; mrmailbox_t* m_mailbox;
@ -95,7 +95,7 @@ typedef struct mrimap_t
} mrimap_t; } mrimap_t;
mrimap_t* mrimap_new (mr_get_config_int_t, mr_set_config_int_t, mr_receive_imf_t, void* userData, mrmailbox_t*); mrimap_t* mrimap_new (mr_get_config_t, mr_set_config_t, mr_receive_imf_t, void* userData, mrmailbox_t*);
void mrimap_unref (mrimap_t*); void mrimap_unref (mrimap_t*);
int mrimap_connect (mrimap_t*, const mrloginparam_t*); int mrimap_connect (mrimap_t*, const mrloginparam_t*);

View file

@ -834,19 +834,19 @@ static uintptr_t cb_dummy(mrmailbox_t* mailbox, int event, uintptr_t data1, uint
{ {
return 0; return 0;
} }
static int32_t cb_get_config_int(mrimap_t* imap, const char* key, int32_t value) static char* cb_get_config(mrimap_t* imap, const char* key, const char* def)
{ {
mrmailbox_t* mailbox = (mrmailbox_t*)imap->m_userData; mrmailbox_t* mailbox = (mrmailbox_t*)imap->m_userData;
mrsqlite3_lock(mailbox->m_sql); mrsqlite3_lock(mailbox->m_sql);
int32_t ret = mrsqlite3_get_config_int__(mailbox->m_sql, key, value); char* ret = mrsqlite3_get_config__(mailbox->m_sql, key, def);
mrsqlite3_unlock(mailbox->m_sql); mrsqlite3_unlock(mailbox->m_sql);
return ret; return ret;
} }
static void cb_set_config_int(mrimap_t* imap, const char* key, int32_t def) static void cb_set_config(mrimap_t* imap, const char* key, const char* value)
{ {
mrmailbox_t* mailbox = (mrmailbox_t*)imap->m_userData; mrmailbox_t* mailbox = (mrmailbox_t*)imap->m_userData;
mrsqlite3_lock(mailbox->m_sql); mrsqlite3_lock(mailbox->m_sql);
mrsqlite3_set_config_int__(mailbox->m_sql, key, def); mrsqlite3_set_config__(mailbox->m_sql, key, value);
mrsqlite3_unlock(mailbox->m_sql); mrsqlite3_unlock(mailbox->m_sql);
} }
static void cb_receive_imf(mrimap_t* imap, const char* imf_raw_not_terminated, size_t imf_raw_bytes, const char* server_folder, uint32_t server_uid, uint32_t flags) static void cb_receive_imf(mrimap_t* imap, const char* imf_raw_not_terminated, size_t imf_raw_bytes, const char* server_folder, uint32_t server_uid, uint32_t flags)
@ -903,7 +903,7 @@ mrmailbox_t* mrmailbox_new(mrmailboxcb_t cb, void* userdata, const char* os_name
ths->m_sql = mrsqlite3_new(ths); ths->m_sql = mrsqlite3_new(ths);
ths->m_cb = cb? cb : cb_dummy; ths->m_cb = cb? cb : cb_dummy;
ths->m_userdata = userdata; ths->m_userdata = userdata;
ths->m_imap = mrimap_new(cb_get_config_int, cb_set_config_int, cb_receive_imf, (void*)ths, ths); ths->m_imap = mrimap_new(cb_get_config, cb_set_config, cb_receive_imf, (void*)ths, ths);
ths->m_smtp = mrsmtp_new(ths); ths->m_smtp = mrsmtp_new(ths);
ths->m_os_name = strdup_keep_null(os_name); ths->m_os_name = strdup_keep_null(os_name);