Issue #427: avoid late generation of mime-structure

MIME mails for both messages and MDNs are stored in the blobdir.
There are still two job types due to their different priorities, but
they are processed by the same function now (dc_job_do_DC_JOB_SEND).

python/src/deltachat/chatting.py

@@ -184,7 +184,7 @@ class Chat(object):
         msg_id = lib.dc_send_msg(self._dc_context, 0, message._dc_msg)
         if msg_id == 0:
             raise ValueError("message could not be sent")
-        return message.from_db(self._dc_context, msg_id)
+        return Message.from_db(self._dc_context, msg_id)
 
     def get_messages(self):
         """ return list of messages in this chat.

python/tests/test_increation.py

@@ -39,12 +39,14 @@ class TestInCreation:
         shutil.copy(data.get_path("d.png"), path)
         sent_original = chat.send_prepared(prepared_original)
         assert sent_original.id == prepared_original.id
-        assert sent_original.get_state().is_out_pending()
+        state = sent_original.get_state()
+        assert state.is_out_pending() or state.is_out_delivered()
         wait_msgs_changed(ac1, chat.id, sent_original.id)
 
         lp.sec("expect the forwarded message to be sent now too")
         wait_msgs_changed(ac1, chat2.id, forwarded_id)
-        assert ac1.get_message_by_id(forwarded_id).get_state().is_out_pending()
+        state = ac1.get_message_by_id(forwarded_id).get_state()
+        assert state.is_out_pending() or state.is_out_delivered()
 
         lp.sec("wait for the messages to be delivered to SMTP")
         ev = ac1._evlogger.get_matching("DC_EVENT_MSG_DELIVERED")

src/dc_chat.c

@@ -2428,7 +2428,7 @@ static uint32_t prepare_msg_common(dc_context_t* context, uint32_t chat_id, dc_m
 			goto cleanup;
 		}
 
-		if (dc_msg_is_increation(msg) && !dc_is_blobdir_path(context, pathNfilename)) {
+		if (state==DC_STATE_OUT_PREPARING && !dc_is_blobdir_path(context, pathNfilename)) {
 			dc_log_error(context, 0, "Files must be created in the blob-directory.");
 			goto cleanup;
 		}
@@ -2545,19 +2545,6 @@ uint32_t dc_prepare_msg(dc_context_t* context, uint32_t chat_id, dc_msg_t* msg)
  * dc_send_msg(context, chat_id, msg);
  * ~~~
  *
- * You can even call this function if the file to be sent is still in creation.
- * For this purpose, create a file with the additional extension `.increation`
- * beside the file to sent. Once you're done with creating the file, delete the
- * increation-file and the message will really be sent.
- * This is useful as the user can already send the next messages while
- * eg. the recoding of a video is not yet finished. Or the user can even forward
- * the message with the file being still in creation to other groups.
- *
- * Files being sent with the increation-method must be placed in the
- * blob directory, see dc_get_blobdir().
- * If the increation-method is not used - which is probably the normal case -
- * the file is copied to the blob directory if it is not yet there.
- *
  * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
  * @param chat_id Chat ID to send the message to.
@@ -2588,7 +2575,10 @@ uint32_t dc_send_msg(dc_context_t* context, uint32_t chat_id, dc_msg_t* msg)
 		return 0;
 	}
 
-	dc_job_add(context, DC_JOB_SEND_MSG_TO_SMTP, msg->id, NULL, 0);
+	// create message file and submit SMTP job
+	if (!dc_job_send_msg(context, msg->id)) {
+		return 0;
+	}
 
 	context->cb(context, DC_EVENT_MSGS_CHANGED, msg->chat_id, msg->id);
 
@@ -2777,7 +2767,7 @@ void dc_forward_msgs(dc_context_t* context, const uint32_t* msg_ids, int msg_cnt
 			}
 			else {
 				new_msg_id = prepare_msg_raw(context, chat, msg, curr_timestamp++, msg->state);
-				dc_job_add(context, DC_JOB_SEND_MSG_TO_SMTP, new_msg_id, NULL, 0);
+				dc_job_send_msg(context, new_msg_id);
 			}
 
 			carray_add(created_db_entries, (void*)(uintptr_t)chat_id, NULL);

src/dc_job.c

@@ -9,6 +9,9 @@
 #include "dc_mimefactory.h"
 
 
+static void dc_send_mdn(dc_context_t* context, uint32_t msg_id);
+
+
 /*******************************************************************************
  * IMAP-jobs
  ******************************************************************************/
@@ -140,7 +143,7 @@ static void dc_job_do_DC_JOB_MARKSEEN_MSG_ON_IMAP(dc_context_t* context, dc_job_
 			case DC_FAILED:       goto cleanup;
 			case DC_RETRY_LATER:  dc_job_try_again_later(job, DC_STANDARD_DELAY, NULL); goto cleanup;
 			case DC_ALREADY_DONE: break;
-			case DC_SUCCESS:      dc_job_add(context, DC_JOB_SEND_MDN, msg->id, NULL, 0); break;
+			case DC_SUCCESS:      dc_send_mdn(context, msg->id); break;
 		}
 	}
 
@@ -194,38 +197,73 @@ cleanup:
  ******************************************************************************/
 
 
-static void dc_job_do_DC_JOB_SEND_MSG_TO_SMTP(dc_context_t* context, dc_job_t* job)
+/**
+ * Store the MIME message in a file and send it later with a new SMTP job.
+ *
+ * @param context The context object as created by dc_context_new()
+ * @param action One of the DC_JOB_SEND_ constants
+ * @param mimefactory An instance of dc_mimefactory_t with a loaded and rendered message or MDN
+ * @return 1=success, 0=error
+ */
+static int dc_add_smtp_job(dc_context_t* context, int action, dc_mimefactory_t* mimefactory)
 {
 	char*            pathNfilename = NULL;
+	char*            blobdir = dc_get_blobdir(context);
+	int              success = 0;
+	char*            recipients = NULL;
+	dc_param_t*      param = dc_param_new();
+
+	// find a free file name in the blob directory
+	pathNfilename = dc_get_fine_pathNfilename(context, blobdir, mimefactory->rfc724_mid);
+	if (!pathNfilename) {
+		dc_log_error(context, 0, "Could not find free file name for message with ID <%s>.", mimefactory->rfc724_mid);
+		goto cleanup;
+	}
+
+	// write the file
+	if (!dc_write_file(context, pathNfilename, mimefactory->out->str, mimefactory->out->len)) {
+		dc_log_error(context, 0, "Could not write message <%s> to \"%s\".", mimefactory->rfc724_mid, pathNfilename);
+		goto cleanup;
+	}
+
+	// store recipients in job param
+	recipients = clist_join(mimefactory->recipients_addr, ' ');
+	dc_param_set(param, DC_PARAM_FILE, pathNfilename);
+	dc_param_set(param, DC_PARAM_RECIPIENTS, recipients);
+
+	dc_job_add(context, action, mimefactory->loaded==DC_MF_MSG_LOADED ? mimefactory->msg->id : 0, param->packed, 0);
+
+	success = 1;
+
+cleanup:
+	dc_param_unref(param);
+	free(recipients);
+	free(blobdir);
+	free(pathNfilename);
+	return success;
+}
+
+
+/**
+ * Create an SMTP job to send a message from the DB
+ *
+ * @param context The context object as created by dc_context_new()
+ * @param msg_id The ID of the message to send
+ * @return 1=success, 0=error
+ */
+int dc_job_send_msg(dc_context_t* context, uint32_t msg_id)
+{
+	int success = 0;
 	dc_mimefactory_t mimefactory;
 	dc_mimefactory_init(&mimefactory, context);
 
-	/* connect to SMTP server, if not yet done */
-	if (!dc_smtp_is_connected(context->smtp)) {
-		dc_loginparam_t* loginparam = dc_loginparam_new();
-			dc_loginparam_read(loginparam, context->sql, "configured_");
-			int connected = dc_smtp_connect(context->smtp, loginparam);
-		dc_loginparam_unref(loginparam);
-		if (!connected) {
-			dc_job_try_again_later(job, DC_STANDARD_DELAY, NULL);
-			goto cleanup;
-		}
-	}
-
 	/* load message data */
-	if (!dc_mimefactory_load_msg(&mimefactory, job->foreign_id)
+	if (!dc_mimefactory_load_msg(&mimefactory, msg_id)
 	 || mimefactory.from_addr==NULL) {
 		dc_log_warning(context, 0, "Cannot load data to send, maybe the message is deleted in between.");
 		goto cleanup; // no redo, no IMAP. moreover, as the data does not exist, there is no need in calling dc_set_msg_failed()
 	}
 
-	/* check if the message is ready (normally, only video files may be delayed this way) */
-	if (mimefactory.increation) {
-		dc_log_info(context, 0, "File is in creation, retrying later.");
-		dc_job_try_again_later(job, DC_INCREATION_POLL, NULL);
-		goto cleanup;
-	}
-
 	/* set width/height of images, if not yet done */
 	if (DC_MSG_NEEDS_ATTACHMENT(mimefactory.msg->type)) {
 		char* pathNfilename = dc_param_get(mimefactory.msg->param, DC_PARAM_FILE, NULL);
@@ -245,18 +283,19 @@ static void dc_job_do_DC_JOB_SEND_MSG_TO_SMTP(dc_context_t* context, dc_job_t* j
 				dc_msg_save_param_to_disk(mimefactory.msg);
 			}
 		}
+		free(pathNfilename);
 	}
 
-	/* send message */
+	/* create message */
 	{
 		if (!dc_mimefactory_render(&mimefactory)) {
-			dc_set_msg_failed(context, job->foreign_id, mimefactory.error);
+			dc_set_msg_failed(context, msg_id, mimefactory.error);
 			goto cleanup; // no redo, no IMAP - this will also fail next time
 		}
 
 		/* have we guaranteed encryption but cannot fulfill it for any reason? Do not send the message then.*/
 		if (dc_param_get_int(mimefactory.msg->param, DC_PARAM_GUARANTEE_E2EE, 0) && !mimefactory.out_encrypted) {
-			dc_set_msg_failed(context, job->foreign_id, "End-to-end-encryption unavailable unexpectedly.");
+			dc_set_msg_failed(context, msg_id, "End-to-end-encryption unavailable unexpectedly.");
 			goto cleanup; /* unrecoverable */
 		}
 
@@ -265,28 +304,14 @@ static void dc_job_do_DC_JOB_SEND_MSG_TO_SMTP(dc_context_t* context, dc_job_t* j
 			clist_append(mimefactory.recipients_names, NULL);
 			clist_append(mimefactory.recipients_addr,  (void*)dc_strdup(mimefactory.from_addr));
 		}
-
-		if (!dc_smtp_send_msg(context->smtp, mimefactory.recipients_addr, mimefactory.out->str, mimefactory.out->len)) {
-			if (MAILSMTP_ERROR_EXCEED_STORAGE_ALLOCATION==context->smtp->error_etpan
-			 || MAILSMTP_ERROR_INSUFFICIENT_SYSTEM_STORAGE==context->smtp->error_etpan) {
-				dc_set_msg_failed(context, job->foreign_id, context->smtp->error);
-			}
-			else {
-				dc_smtp_disconnect(context->smtp);
-				dc_job_try_again_later(job, DC_AT_ONCE, context->smtp->error);
-			}
-			goto cleanup;
-		}
 	}
 
-	/* done */
 	dc_sqlite3_begin_transaction(context->sql);
 
 		if (mimefactory.out_gossiped) {
 			dc_set_gossiped_timestamp(context, mimefactory.msg->chat_id, time(NULL));
 		}
 
-		dc_update_msg_state(context, mimefactory.msg->id, DC_STATE_OUT_DELIVERED);
 		if (mimefactory.out_encrypted && dc_param_get_int(mimefactory.msg->param, DC_PARAM_GUARANTEE_E2EE, 0)==0) {
 			dc_param_set_int(mimefactory.msg->param, DC_PARAM_GUARANTEE_E2EE, 1); /* can upgrade to E2EE - fine! */
 			dc_msg_save_param_to_disk(mimefactory.msg);
@@ -297,26 +322,26 @@ static void dc_job_do_DC_JOB_SEND_MSG_TO_SMTP(dc_context_t* context, dc_job_t* j
 
 	dc_sqlite3_commit(context->sql);
 
-	context->cb(context, DC_EVENT_MSG_DELIVERED, mimefactory.msg->chat_id, mimefactory.msg->id);
+	success = dc_add_smtp_job(context, DC_JOB_SEND_MSG_TO_SMTP, &mimefactory);
 
 cleanup:
 	dc_mimefactory_empty(&mimefactory);
-	free(pathNfilename);
+	return success;
 }
 
 
-static void dc_job_do_DC_JOB_SEND_MDN(dc_context_t* context, dc_job_t* job)
+static void dc_job_do_DC_JOB_SEND(dc_context_t* context, dc_job_t* job)
 {
-	dc_mimefactory_t mimefactory;
+	char*         filename = NULL;
-	dc_mimefactory_init(&mimefactory, context);
+	void*         buf = NULL;
-
+	size_t        buf_bytes = 0;
-	if (context==NULL || context->magic!=DC_CONTEXT_MAGIC || job==NULL) {
+	char*         recipients = NULL;
-		return;
+	clist*        recipients_list = clist_new();
-	}
+	char*         saveptr = NULL;
+	sqlite3_stmt* stmt = NULL;
 
 	/* connect to SMTP server, if not yet done */
-	if (!dc_smtp_is_connected(context->smtp))
+	if (!dc_smtp_is_connected(context->smtp)) {
-	{
 		dc_loginparam_t* loginparam = dc_loginparam_new();
 			dc_loginparam_read(loginparam, context->sql, "configured_");
 			int connected = dc_smtp_connect(context->smtp, loginparam);
@@ -327,18 +352,83 @@ static void dc_job_do_DC_JOB_SEND_MDN(dc_context_t* context, dc_job_t* job)
 		}
 	}
 
-    if (!dc_mimefactory_load_mdn(&mimefactory, job->foreign_id)
+	/* load message data */
+	filename = dc_param_get(job->param, DC_PARAM_FILE, NULL);
+	if (!filename) {
+		goto cleanup;
+	}
+	if (!dc_read_file(context, filename, &buf, &buf_bytes)) {
+		goto cleanup;
+	}
+
+	/* get the list of recipients */
+	recipients = dc_param_get(job->param, DC_PARAM_RECIPIENTS, NULL);
+	if (!recipients) {
+		goto cleanup;
+	}
+	for (char* r = strtok_r(recipients, " ", &saveptr); r; r = strtok_r(NULL, " ", &saveptr)) {
+		clist_append(recipients_list, r);
+	}
+
+	/* send message */
+	{
+		if (!dc_smtp_send_msg(context->smtp, recipients_list, buf, buf_bytes)) {
+			if (job->foreign_id && (
+			    MAILSMTP_ERROR_EXCEED_STORAGE_ALLOCATION==context->smtp->error_etpan
+			 || MAILSMTP_ERROR_INSUFFICIENT_SYSTEM_STORAGE==context->smtp->error_etpan)) {
+				dc_set_msg_failed(context, job->foreign_id, context->smtp->error);
+			}
+			else {
+				dc_smtp_disconnect(context->smtp);
+				dc_job_try_again_later(job, DC_AT_ONCE, context->smtp->error);
+			}
+			goto cleanup;
+		}
+	}
+
+	/* delete the stores message file */
+	dc_delete_file(context, filename);
+	/* an error still means the message is sent, so the rest continues */
+
+	/* done */
+	if (job->foreign_id) {
+		dc_update_msg_state(context, job->foreign_id, DC_STATE_OUT_DELIVERED);
+
+		/* find the chat ID */
+		stmt = dc_sqlite3_prepare(context->sql, "SELECT chat_id FROM msgs WHERE id=?");
+		sqlite3_bind_int(stmt, 1, job->foreign_id);
+		/* a deleted message results in an event with chat_id==0, rather than no event at all */
+		int chat_id = sqlite3_step(stmt)==SQLITE_ROW ? sqlite3_column_int(stmt, 0) : 0;
+
+		context->cb(context, DC_EVENT_MSG_DELIVERED, chat_id, job->foreign_id);
+	}
+
+cleanup:
+	sqlite3_finalize(stmt);
+	clist_free(recipients_list); // not clist_free_content() because strtok_r() returns pointers into recipients
+	free(recipients);
+	free(buf);
+	free(filename);
+}
+
+
+static void dc_send_mdn(dc_context_t* context, uint32_t msg_id)
+{
+	dc_mimefactory_t mimefactory;
+	dc_mimefactory_init(&mimefactory, context);
+
+	if (context==NULL || context->magic!=DC_CONTEXT_MAGIC) {
+		return;
+	}
+
+    if (!dc_mimefactory_load_mdn(&mimefactory, msg_id)
      || !dc_mimefactory_render(&mimefactory)) {
 		goto cleanup;
     }
 
 	//char* t1=dc_null_terminate(mimefactory.out->str,mimefactory.out->len);printf("~~~~~MDN~~~~~\n%s\n~~~~~/MDN~~~~~",t1);free(t1); // DEBUG OUTPUT
 
-	if (!dc_smtp_send_msg(context->smtp, mimefactory.recipients_addr, mimefactory.out->str, mimefactory.out->len)) {
+	dc_add_smtp_job(context, DC_JOB_SEND_MDN, &mimefactory);
-		dc_smtp_disconnect(context->smtp);
-		dc_job_try_again_later(job, DC_AT_ONCE, NULL);
-		goto cleanup;
-	}
 
 cleanup:
 	dc_mimefactory_empty(&mimefactory);
@@ -571,12 +661,12 @@ static void dc_job_perform(dc_context_t* context, int thread, int probe_network)
 			job.try_again = DC_DONT_TRY_AGAIN; // this can be modified by a job using dc_job_try_again_later()
 
 			switch (job.action) {
-				case DC_JOB_SEND_MSG_TO_SMTP:     dc_job_do_DC_JOB_SEND_MSG_TO_SMTP     (context, &job); break;
+				case DC_JOB_SEND_MSG_TO_SMTP:     dc_job_do_DC_JOB_SEND                 (context, &job); break;
 				case DC_JOB_DELETE_MSG_ON_IMAP:   dc_job_do_DC_JOB_DELETE_MSG_ON_IMAP   (context, &job); break;
 				case DC_JOB_MARKSEEN_MSG_ON_IMAP: dc_job_do_DC_JOB_MARKSEEN_MSG_ON_IMAP (context, &job); break;
 				case DC_JOB_MARKSEEN_MDN_ON_IMAP: dc_job_do_DC_JOB_MARKSEEN_MDN_ON_IMAP (context, &job); break;
 				case DC_JOB_MOVE_MSG:             dc_job_do_DC_JOB_MOVE_MSG             (context, &job); break;
-				case DC_JOB_SEND_MDN:             dc_job_do_DC_JOB_SEND_MDN             (context, &job); break;
+				case DC_JOB_SEND_MDN:             dc_job_do_DC_JOB_SEND                 (context, &job); break;
 				case DC_JOB_CONFIGURE_IMAP:       dc_job_do_DC_JOB_CONFIGURE_IMAP       (context, &job); break;
 				case DC_JOB_IMEX_IMAP:            dc_job_do_DC_JOB_IMEX_IMAP            (context, &job); break;
 				case DC_JOB_HOUSEKEEPING:         dc_housekeeping                       (context);       break;

src/dc_job.h

@@ -57,6 +57,8 @@ struct _dc_job
 void     dc_job_add                   (dc_context_t*, int action, int foreign_id, const char* param, int delay);
 void     dc_job_kill_action           (dc_context_t*, int action); /* delete all pending jobs with the given action */
 
+int      dc_job_send_msg              (dc_context_t*, uint32_t msg_id); /* special case for DC_JOB_SEND_MSG_TO_SMTP */
+
 #define  DC_DONT_TRY_AGAIN           0
 #define  DC_AT_ONCE                 -1
 #define  DC_INCREATION_POLL          2 // this value does not increase the number of tries

src/dc_param.h

@@ -26,7 +26,7 @@ struct _dc_param
 };
 
 
-#define DC_PARAM_FILE              'f'  /* for msgs */
+#define DC_PARAM_FILE              'f'  /* for msgs and jobs */
 #define DC_PARAM_WIDTH             'w'  /* for msgs */
 #define DC_PARAM_HEIGHT            'h'  /* for msgs */
 #define DC_PARAM_DURATION          'd'  /* for msgs */
@@ -47,6 +47,7 @@ struct _dc_param
 #define DC_PARAM_SERVER_FOLDER     'Z'  /* for jobs */
 #define DC_PARAM_SERVER_UID        'z'  /* for jobs */
 #define DC_PARAM_ALSO_MOVE         'M'  /* for jobs */
+#define DC_PARAM_RECIPIENTS        'R'  /* for jobs: space-separated list of message recipients */
 
 #define DC_PARAM_UNPROMOTED        'U'  /* for groups */
 #define DC_PARAM_PROFILE_IMAGE     'i'  /* for groups and contacts */

src/dc_sqlite3.c

@@ -947,6 +947,10 @@ void dc_housekeeping(dc_context_t* context)
 		"   AND type!=" DC_STRINGIFY(DC_MSG_TEXT) ";",
 		DC_PARAM_FILE);
 
+	maybe_add_from_param(context, &files_in_use,
+		"SELECT param FROM jobs;",
+		DC_PARAM_FILE);
+
 	maybe_add_from_param(context, &files_in_use,
 		"SELECT param FROM chats;",
 		DC_PARAM_PROFILE_IMAGE);

src/dc_tools.c

@@ -658,6 +658,33 @@ int clist_search_string_nocase(const clist* haystack, const char* needle)
 }
 
 
+char* clist_join(const clist* list, char separator)
+{
+	size_t total = 0;
+	for (clistiter* iter=clist_begin(list); iter!=NULL; iter=clist_next(iter)) {
+		char* str = clist_content(iter);
+		total += strlen(str) + 1;  // the 1 is for separators and \0
+	}
+	if (total==0) {
+		return dc_strdup("");
+	}
+	char* result = malloc(total);
+	if (!result) {
+		exit(55); /* cannot allocate little memoty, unrecoverable error */
+	}
+	char* p = result;
+	for (clistiter* iter=clist_begin(list); iter!=NULL; iter=clist_next(iter)) {
+		const char* str = clist_content(iter);
+		size_t len = strlen(str);
+		memcpy(p, str, len);
+		p += len;
+		*(p++) = separator;
+	}
+	*(--p) = 0;
+	return result;
+}
+
+
 /*******************************************************************************
  * date/time tools
  ******************************************************************************/

src/dc_tools.h

@@ -54,6 +54,7 @@ char*   encode_base64              (const char * in, int len);
 /* clist tools */
 void    clist_free_content         (const clist*); /* calls free() for each item content */
 int     clist_search_string_nocase (const clist*, const char* str);
+char*   clist_join                 (const clist*, char separator);
 
 /* date/time tools */
 #define                    DC_INVALID_TIMESTAMP               (-1)