let the mimefactory just use In-Reply-To & Co. from the database, store In-Reply-To & Co. on receiving messages

src/dc_mimefactory.c

@@ -58,8 +58,8 @@ void dc_mimefactory_empty(dc_mimefactory_t* factory)
 	dc_chat_unref(factory->chat);
 	factory->chat = NULL;
 
-	free(factory->predecessor);
-	factory->predecessor = NULL;
+	free(factory->in_reply_to);
+	factory->in_reply_to = NULL;
 
 	free(factory->references);
 	factory->references = NULL;
@@ -174,57 +174,16 @@ int dc_mimefactory_load_msg(dc_mimefactory_t* factory, uint32_t msg_id)
 				}
 			}
 
-			/* Get a predecessor of the mail to send.
-			For simplicity, we use the last message send not by us.
-			This is not 100% accurate and may even be a newer message if first sending fails and new messages arrive -
-			however, as we currently only use it to identifify answers from different email addresses, this is sufficient.
-
-			Our first idea was to write the predecessor to the `In-Reply-To:` header, however, this results
-			in infinite depth thread views eg. in thunderbird.  Maybe we can work around this issue by using only one
-			predecessor anchor a day, however, for the moment, we just use the `Chat-Predecessor` header that does not
-			disturb other mailers.
-
-			Finally, maybe the Predecessor/In-Reply-To header is not needed for all answers but only to the first ones -
-			or after the sender has changes its email address. */
 			stmt = dc_sqlite3_prepare(context->sql,
-				"SELECT rfc724_mid FROM msgs WHERE timestamp=(SELECT max(timestamp) FROM msgs WHERE chat_id=? AND from_id!=?);");
-			sqlite3_bind_int  (stmt, 1, factory->msg->chat_id);
-			sqlite3_bind_int  (stmt, 2, DC_CONTACT_ID_SELF);
+				"SELECT mime_in_reply_to, mime_references FROM msgs WHERE id=?");
+			sqlite3_bind_int  (stmt, 1, factory->msg->id);
 			if (sqlite3_step(stmt)==SQLITE_ROW) {
-				factory->predecessor = dc_strdup_keep_null((const char*)sqlite3_column_text(stmt, 0));
+				factory->in_reply_to = dc_strdup((const char*)sqlite3_column_text(stmt, 0));
+				factory->references  = dc_strdup((const char*)sqlite3_column_text(stmt, 1));
 			}
 			sqlite3_finalize(stmt);
 			stmt = NULL;
 
-			/* get a References:-header: either the same as the last one or a random one.
-			To avoid endless nested threads, we do not use In-Reply-To: here but link subsequent mails to the same reference.
-			This "same reference" is re-calculated after 24 hours to avoid completely different messages being linked to an old context.
-
-			Regarding multi-client: Different clients will create difference References:-header, maybe we will sync these headers some day,
-			however one could also see this as a feature :) (there may be different contextes on different clients)
-			(also, the References-header is not the most important thing, and, at least for now, we do not want to make things too complicated.  */
-			time_t prev_msg_time = 0;
-			stmt = dc_sqlite3_prepare(context->sql,
-				"SELECT max(timestamp) FROM msgs WHERE chat_id=? AND id!=?");
-			sqlite3_bind_int  (stmt, 1, factory->msg->chat_id);
-			sqlite3_bind_int  (stmt, 2, factory->msg->id);
-			if (sqlite3_step(stmt)==SQLITE_ROW) {
-				prev_msg_time = sqlite3_column_int64(stmt, 0);
-			}
-			sqlite3_finalize(stmt);
-			stmt = NULL;
-
-			#define NEW_THREAD_THRESHOLD 24*60*60
-			if (prev_msg_time!=0 && factory->msg->timestamp - prev_msg_time < NEW_THREAD_THRESHOLD) {
-				factory->references = dc_param_get(factory->chat->param, DC_PARAM_REFERENCES, NULL);
-			}
-
-			if (factory->references==NULL) {
-				factory->references = dc_create_dummy_references_mid();
-				dc_param_set(factory->chat->param, DC_PARAM_REFERENCES, factory->references);
-				dc_chat_update_param(factory->chat);
-			}
-
 			success = 1;
 			factory->loaded = DC_MF_MSG_LOADED;
 			factory->timestamp = factory->msg->timestamp;
@@ -507,14 +466,18 @@ int dc_mimefactory_render(dc_mimefactory_t* factory)
 		}
 
 		clist* references_list = NULL;
-		if (factory->references) {
-			references_list = clist_new();
-			clist_append(references_list,  (void*)dc_strdup(factory->references));
+		if (factory->references && factory->references[0]) {
+			references_list = dc_str_to_clist(factory->references, " ");
+		}
+
+		clist* in_reply_to_list = NULL;
+		if (factory->in_reply_to && factory->in_reply_to[0]) {
+			in_reply_to_list = dc_str_to_clist(factory->in_reply_to, " ");
 		}
 
 		imf_fields = mailimf_fields_new_with_data_all(mailimf_get_date(factory->timestamp), from,
 			NULL /* sender */, NULL /* reply-to */,
-			to, NULL /* cc */, NULL /* bcc */, dc_strdup(factory->rfc724_mid), NULL /* in-reply-to */,
+			to, NULL /* cc */, NULL /* bcc */, dc_strdup(factory->rfc724_mid), in_reply_to_list,
 			references_list /* references */,
 			NULL /* subject set later */);
 
@@ -527,9 +490,6 @@ int dc_mimefactory_render(dc_mimefactory_t* factory)
 			factory->context->os_name? factory->context->os_name : "")));
 
 		mailimf_fields_add(imf_fields, mailimf_field_new_custom(strdup("Chat-Version"), strdup("1.0"))); /* mark message as being sent by a messenger */
-		if (factory->predecessor) {
-			mailimf_fields_add(imf_fields, mailimf_field_new_custom(strdup("Chat-Predecessor"), strdup(factory->predecessor)));
-		}
 
 		if (factory->req_mdn) {
 			/* we use "Chat-Disposition-Notification-To" as replies to "Disposition-Notification-To" are weird in many cases, are just freetext and/or do not follow any standard. */

src/dc_mimefactory.h

@@ -43,7 +43,7 @@ typedef struct dc_mimefactory_t {
 	dc_msg_t*     msg;
 	dc_chat_t*    chat;
 	int           increation;
-	char*         predecessor;
+	char*         in_reply_to;
 	char*         references;
 	int           req_mdn;
 

src/dc_param.h

@@ -42,7 +42,6 @@ typedef struct dc_param_t
 #define DC_PARAM_SERVER_UID        'z'  /* for jobs */
 #define DC_PARAM_TIMES             't'  /* for jobs: times a job was tried */
 
-#define DC_PARAM_REFERENCES        'R'  /* for groups and chats: References-header last used for a chat */
 #define DC_PARAM_UNPROMOTED        'U'  /* for groups */
 #define DC_PARAM_PROFILE_IMAGE     'i'  /* for groups and contacts */
 #define DC_PARAM_SELFTALK          'K'  /* for chats */

src/dc_receive_imf.c

@@ -944,6 +944,8 @@ void dc_receive_imf(dc_context_t* context, const char* imf_raw_not_terminated, s
 	dc_mimeparser_t* mime_parser = dc_mimeparser_new(context->blobdir, context);
 	int              transaction_pending = 0;
 	const struct mailimf_field* field;
+	char*            mime_in_reply_to = NULL;
+	char*            mime_references = NULL;
 
 	carray*          created_db_entries = carray_new(16);
 	int              create_event_to_send = DC_EVENT_MSGS_CHANGED;
@@ -1267,6 +1269,24 @@ void dc_receive_imf(dc_context_t* context, const char* imf_raw_not_terminated, s
 				}
 			}
 
+			if ((field=dc_mimeparser_lookup_field(mime_parser, "In-Reply-To"))!=NULL
+			 && field->fld_type==MAILIMF_FIELD_IN_REPLY_TO)
+			{
+				struct mailimf_in_reply_to* fld_in_reply_to = field->fld_data.fld_in_reply_to;
+				if (fld_in_reply_to) {
+					mime_in_reply_to = dc_str_from_clist(field->fld_data.fld_in_reply_to->mid_list, " ");
+				}
+			}
+
+			if ((field=dc_mimeparser_lookup_field(mime_parser, "References"))!=NULL
+			 && field->fld_type==MAILIMF_FIELD_REFERENCES)
+			{
+				struct mailimf_references* fld_references = field->fld_data.fld_references;
+				if (fld_references) {
+					mime_references = dc_str_from_clist(field->fld_data.fld_references->mid_list, " ");
+				}
+			}
+
 			/* fine, so far.  now, split the message into simple parts usable as "short messages"
 			and add them to the database (mails sent by other messenger clients should result
 			into only one message; mails sent by other clients may result in several messages (eg. one per attachment)) */
@@ -1274,8 +1294,9 @@ void dc_receive_imf(dc_context_t* context, const char* imf_raw_not_terminated, s
 			stmt = dc_sqlite3_prepare(context->sql,
 				"INSERT INTO msgs (rfc724_mid, server_folder, server_uid, chat_id, from_id, to_id,"
 				" timestamp, timestamp_sent, timestamp_rcvd, type, state, msgrmsg, "
-				" txt, txt_raw, param, bytes, hidden, mime_headers)"
-				" VALUES (?,?,?,?,?,?, ?,?,?,?,?,?, ?,?,?,?,?,?);");
+				" txt, txt_raw, param, bytes, hidden, mime_headers, "
+				" mime_in_reply_to, mime_references)"
+				" VALUES (?,?,?,?,?,?, ?,?,?,?,?,?, ?,?,?,?,?,?, ?,?);");
 			for (i = 0; i < icnt; i++)
 			{
 				dc_mimepart_t* part = (dc_mimepart_t*)carray_get(mime_parser->parts, i);
@@ -1310,6 +1331,8 @@ void dc_receive_imf(dc_context_t* context, const char* imf_raw_not_terminated, s
 				sqlite3_bind_int  (stmt, 16, part->bytes);
 				sqlite3_bind_int  (stmt, 17, hidden);
 				sqlite3_bind_text (stmt, 18, save_mime_headers? imf_raw_not_terminated : NULL, header_bytes, SQLITE_STATIC);
+				sqlite3_bind_text (stmt, 19, mime_in_reply_to, -1, SQLITE_STATIC);
+				sqlite3_bind_text (stmt, 20, mime_references, -1, SQLITE_STATIC);
 				if (sqlite3_step(stmt)!=SQLITE_DONE) {
 					dc_log_info(context, 0, "Cannot write DB.");
 					goto cleanup; /* i/o error - there is nothing more we can do - in other cases, we try to write at least an empty record */
@@ -1458,6 +1481,8 @@ cleanup:
 
 	dc_mimeparser_unref(mime_parser);
 	free(rfc724_mid);
+	free(mime_in_reply_to);
+	free(mime_references);
 	dc_array_unref(to_ids);
 
 	if (created_db_entries) {

src/dc_sqlite3.c

@@ -473,6 +473,17 @@ int dc_sqlite3_open(dc_sqlite3_t* sql, const char* dbfile, int flags)
 			}
 		#undef NEW_DB_VERSION
 
+		#define NEW_DB_VERSION 46
+			if (dbversion < NEW_DB_VERSION)
+			{
+				dc_sqlite3_execute(sql, "ALTER TABLE msgs ADD COLUMN mime_in_reply_to TEXT;");
+				dc_sqlite3_execute(sql, "ALTER TABLE msgs ADD COLUMN mime_references TEXT;");
+
+				dbversion = NEW_DB_VERSION;
+				dc_sqlite3_set_config_int(sql, "dbversion", NEW_DB_VERSION);
+			}
+		#undef NEW_DB_VERSION
+
 
 		// (2) updates that require high-level objects
 		// (the structure is complete now and all objects are usable)

src/dc_tools.c

@@ -898,16 +898,6 @@ char* dc_create_id(void)
 }
 
 
-char* dc_create_dummy_references_mid()
-{
-	char* msgid = dc_create_id();
-	char* ret = NULL;
-	ret = dc_mprintf("Rf.%s@mr.thread", msgid);
-	free(msgid);
-	return ret;
-}
-
-
 char* dc_create_outgoing_rfc724_mid(const char* grpid, const char* from_addr)
 {
 	/* Function generates a Message-ID that can be used for a new outgoing message.

src/dc_tools.h

@@ -67,7 +67,6 @@ time_t dc_create_smeared_timestamps  (dc_context_t*, int count);
 /* Message-ID tools */
 #define DC_CREATE_ID_LEN                   11
 char* dc_create_id                         (void);
-char* dc_create_dummy_references_mid       (void);
 char* dc_create_incoming_rfc724_mid        (time_t message_timestamp, uint32_t contact_id_from, dc_array_t* contact_ids_to);
 char* dc_create_outgoing_rfc724_mid        (const char* grpid, const char* addr);
 char* dc_extract_grpid_from_rfc724_mid     (const char* rfc724_mid);