move outgoing files to blobdir; use relative paths for blobdirs. closes #256

src/dc_chat.c

@@ -257,7 +257,10 @@ char* dc_chat_get_profile_image(const dc_chat_t* chat)
 		return NULL;
 	}
 
-	return dc_param_get(chat->param, DC_PARAM_PROFILE_IMAGE, NULL);
+	char* profile_image_rel = dc_param_get(chat->param, DC_PARAM_PROFILE_IMAGE, NULL);
+	char* profile_image_abs = dc_get_abs_path(chat->context, profile_image_rel);
+	free(profile_image_rel);
+	return profile_image_abs;
 }
 
 
@@ -1612,6 +1615,7 @@ int dc_set_chat_profile_image(dc_context_t* context, uint32_t chat_id, const cha
 	int        success = 0;
 	dc_chat_t* chat = dc_chat_new(context);
 	dc_msg_t*  msg = dc_msg_new(context);
+	char*      new_image_rel = NULL;
 
 	if (context==NULL || context->magic!=DC_CONTEXT_MAGIC || chat_id<=DC_CHAT_ID_LAST_SPECIAL) {
 		goto cleanup;
@@ -1627,7 +1631,14 @@ int dc_set_chat_profile_image(dc_context_t* context, uint32_t chat_id, const cha
 		goto cleanup; /* we shoud respect this - whatever we send to the group, it gets discarded anyway! */
 	}
 
-	dc_param_set(chat->param, DC_PARAM_PROFILE_IMAGE, new_image/*may be NULL*/);
+	if (new_image) {
+		new_image_rel = dc_strdup(new_image);
+		if (!dc_make_rel_and_copy(context, &new_image_rel)) {
+			goto cleanup;
+		}
+	}
+
+	dc_param_set(chat->param, DC_PARAM_PROFILE_IMAGE, new_image_rel/*may be NULL*/);
 	if (!dc_chat_update_param(chat)) {
 		goto cleanup;
 	}
@@ -1636,9 +1647,9 @@ int dc_set_chat_profile_image(dc_context_t* context, uint32_t chat_id, const cha
 	if (DO_SEND_STATUS_MAILS)
 	{
 		dc_param_set_int(msg->param, DC_PARAM_CMD,     DC_CMD_GROUPIMAGE_CHANGED);
-		dc_param_set    (msg->param, DC_PARAM_CMD_ARG, new_image);
+		dc_param_set    (msg->param, DC_PARAM_CMD_ARG, new_image_rel);
 		msg->type = DC_MSG_TEXT;
-		msg->text = dc_stock_str(context, new_image? DC_STR_MSGGRPIMGCHANGED : DC_STR_MSGGRPIMGDELETED);
+		msg->text = dc_stock_str(context, new_image_rel? DC_STR_MSGGRPIMGCHANGED : DC_STR_MSGGRPIMGDELETED);
 		msg->id = dc_send_msg(context, chat_id, msg);
 		context->cb(context, DC_EVENT_MSGS_CHANGED, chat_id, msg->id);
 	}
@@ -1649,6 +1660,7 @@ int dc_set_chat_profile_image(dc_context_t* context, uint32_t chat_id, const cha
 cleanup:
 	dc_chat_unref(chat);
 	dc_msg_unref(msg);
+	free(new_image_rel);
 	return success;
 }
 
@@ -2071,6 +2083,11 @@ uint32_t dc_send_msg(dc_context_t* context, uint32_t chat_id, dc_msg_t* msg)
 		pathNfilename = dc_param_get(msg->param, DC_PARAM_FILE, NULL);
 		if (pathNfilename)
 		{
+			if (!dc_make_rel_and_copy(context, &pathNfilename)) {
+				goto cleanup;
+			}
+			dc_param_set(msg->param, DC_PARAM_FILE, pathNfilename);
+
 			/* Got an attachment. Take care, the file may not be ready in this moment!
 			This is useful eg. if a video should be sent and already shown as "being processed" in the chat.
 			In this case, the user should create an `.increation`; when the file is deleted later on, the message is sent.

src/dc_imex.c

@@ -489,7 +489,7 @@ char* dc_initiate_key_transfer(dc_context_t* context)
 
 	CHECK_EXIT
 
-	if ((setup_file_name=dc_get_fine_pathNfilename(context, context->blobdir, "autocrypt-setup-message.html"))==NULL
+	if ((setup_file_name=dc_get_fine_pathNfilename(context, "$BLOBDIR", "autocrypt-setup-message.html"))==NULL
 	 || !dc_write_file(context, setup_file_name, setup_file_content, strlen(setup_file_content))) {
 		goto cleanup;
 	}
@@ -969,7 +969,6 @@ static int export_backup(dc_context_t* context, const char* dir)
 
 	/* done - set some special config values (do this last to avoid importing crashed backups) */
 	dc_sqlite3_set_config_int(dest_sql, "backup_time", now);
-	dc_sqlite3_set_config    (dest_sql, "backup_for", context->blobdir);
 
 	context->cb(context, DC_EVENT_IMEX_FILE_WRITTEN, (uintptr_t)dest_pathNfilename, 0);
 	success = 1;
@@ -995,24 +994,8 @@ cleanup:
  ******************************************************************************/
 
 
-static void ensure_no_slash(char* path)
-{
-	int path_len = strlen(path);
-	if (path_len > 0) {
-		if (path[path_len-1]=='/'
-		 || path[path_len-1]=='\\') {
-			path[path_len-1] = 0;
-		}
-	}
-}
-
-
 static int import_backup(dc_context_t* context, const char* backup_to_import)
 {
-	/* command for testing eg.
-	imex import-backup /home/bpetersen/temp/delta-chat-2017-11-14.bak
-	*/
-
 	int           success = 0;
 	int           processed_files_cnt = 0;
 	int           total_files_cnt = 0;
@@ -1030,9 +1013,6 @@ static int import_backup(dc_context_t* context, const char* backup_to_import)
 
 	/* close and delete the original file - FIXME: we should import to a .bak file and rename it on success. however, currently it is not clear it the import exists in the long run (may be replaced by a restore-from-imap) */
 
-//dc_sqlite3_lock(context->sql);  // TODO: check if this works while threads running
-//locked = 1;
-
 	if (dc_sqlite3_is_open(context->sql)) {
 		dc_sqlite3_close(context->sql);
 	}
@@ -1091,32 +1071,6 @@ static int import_backup(dc_context_t* context, const char* backup_to_import)
 	dc_sqlite3_execute(context->sql, "DROP TABLE backup_blobs;");
 	dc_sqlite3_execute(context->sql, "VACUUM;");
 
-	/* rewrite references to the blobs */
-	repl_from = dc_sqlite3_get_config(context->sql, "backup_for", NULL);
-	if (repl_from && strlen(repl_from)>1 && context->blobdir && strlen(context->blobdir)>1)
-	{
-		ensure_no_slash(repl_from);
-		repl_to = dc_strdup(context->blobdir);
-		ensure_no_slash(repl_to);
-
-		dc_log_info(context, 0, "Rewriting paths from '%s' to '%s' ...", repl_from, repl_to);
-
-		assert( 'f'==DC_PARAM_FILE);
-		assert( 'i'==DC_PARAM_PROFILE_IMAGE);
-
-		char* q3 = sqlite3_mprintf("UPDATE msgs SET param=replace(param, 'f=%q/', 'f=%q/');", repl_from, repl_to); /* cannot use dc_mprintf() because of "%q" */
-			dc_sqlite3_execute(context->sql, q3);
-		sqlite3_free(q3);
-
-		q3 = sqlite3_mprintf("UPDATE chats SET param=replace(param, 'i=%q/', 'i=%q/');", repl_from, repl_to);
-			dc_sqlite3_execute(context->sql, q3);
-		sqlite3_free(q3);
-
-		q3 = sqlite3_mprintf("UPDATE contacts SET param=replace(param, 'i=%q/', 'i=%q/');", repl_from, repl_to);
-			dc_sqlite3_execute(context->sql, q3);
-		sqlite3_free(q3);
-	}
-
 	success = 1;
 
 cleanup:
@@ -1125,8 +1079,6 @@ cleanup:
 	free(repl_to);
 	sqlite3_finalize(stmt);
 
-// if (locked) { dc_sqlite3_unlock(context->sql); }  // TODO: check if this works while threads running
-
 	return success;
 }
 

src/dc_job.c

@@ -167,7 +167,7 @@ static void dc_job_do_DC_JOB_DELETE_MSG_ON_IMAP(dc_context_t* context, dc_job_t*
 
 	char* pathNfilename = dc_param_get(msg->param, DC_PARAM_FILE, NULL);
 	if (pathNfilename) {
-		if (strncmp(context->blobdir, pathNfilename, strlen(context->blobdir))==0)
+		if (strncmp("$BLOBDIR", pathNfilename, 8)==0)
 		{
 			char* strLikeFilename = dc_mprintf("%%f=%s%%", pathNfilename);
 			stmt = dc_sqlite3_prepare(context->sql,

src/dc_mimefactory.c

@@ -455,7 +455,7 @@ static struct mailmime* build_body_file(const dc_msg_t* msg, const char* base_na
 
 	mime_sub = mailmime_new_empty(content, mime_fields);
 
-	mailmime_set_body_file(mime_sub, dc_strdup(pathNfilename));
+	mailmime_set_body_file(mime_sub, dc_get_abs_path(msg->context, pathNfilename));
 
 	if (ret_file_name_as_sent) {
 		*ret_file_name_as_sent = dc_strdup(filename_to_send);

src/dc_mimeparser.c

@@ -951,7 +951,7 @@ static void do_add_single_file_part(dc_mimeparser_t* parser, int msg_type, int m
 	char*          pathNfilename = NULL;
 
 	/* create a free file name to use */
-	if ((pathNfilename=dc_get_fine_pathNfilename(parser->context, parser->blobdir, desired_filename))==NULL) {
+	if ((pathNfilename=dc_get_fine_pathNfilename(parser->context, "$BLOBDIR", desired_filename))==NULL) {
 		goto cleanup;
 	}
 

src/dc_msg.c

@@ -286,16 +286,20 @@ char* dc_msg_get_text(const dc_msg_t* msg)
  */
 char* dc_msg_get_file(const dc_msg_t* msg)
 {
-	char* ret = NULL;
+	char* file_rel = NULL;
+	char* file_abs = NULL;
 
 	if (msg==NULL || msg->magic!=DC_MSG_MAGIC) {
 		goto cleanup;
 	}
 
-	ret = dc_param_get(msg->param, DC_PARAM_FILE, NULL);
+	if ((file_rel = dc_param_get(msg->param, DC_PARAM_FILE, NULL))!=NULL) {
+		file_abs = dc_get_abs_path(msg->context, file_rel);
+	}
 
 cleanup:
-	return ret? ret : dc_strdup(NULL);
+	free(file_rel);
+	return file_abs? file_abs : dc_strdup(NULL);
 }
 
 
@@ -1613,10 +1617,10 @@ char* dc_get_msg_info(dc_context_t* context, uint32_t msg_id)
 	}
 
 	/* add file info */
-	if ((p=dc_param_get(msg->param, DC_PARAM_FILE, NULL))!=NULL) {
+	if ((p=dc_msg_get_file(msg))!=NULL && p[0]) {
 		dc_strbuilder_catf(&ret, "\nFile: %s, %i bytes\n", p, (int)dc_get_filebytes(context, p));
-		free(p);
 	}
+	free(p);
 
 	if (msg->type!=DC_MSG_TEXT) {
 		p = NULL;

src/dc_sqlite3.c

@@ -20,6 +20,7 @@
  ******************************************************************************/
 
 
+#include <assert.h>
 #include "dc_context.h"
 #include "dc_apeerstate.h"
 
@@ -288,9 +289,13 @@ int dc_sqlite3_open(dc_sqlite3_t* sql, const char* dbfile, int flags)
 		}
 
 		// (1) update low-level database structure.
-		// this should be done before updates that use high-level objects that rely themselves on the low-level structure.
+		// this should be done before updates that use high-level objects that
+		// rely themselves on the low-level structure.
+		// --------------------------------------------------------------------
+
 		int dbversion = dbversion_before_update;
 		int recalc_fingerprints = 0;
+		int update_file_paths = 0;
 
 		#define NEW_DB_VERSION 1
 			if (dbversion < NEW_DB_VERSION)
@@ -451,7 +456,20 @@ int dc_sqlite3_open(dc_sqlite3_t* sql, const char* dbfile, int flags)
 			}
 		#undef NEW_DB_VERSION
 
-		// (2) updates that require high-level objects (the structure is complete now and all objects are usable)
+		#define NEW_DB_VERSION 41
+			if (dbversion < NEW_DB_VERSION)
+			{
+				update_file_paths = 1;
+
+				dbversion = NEW_DB_VERSION;
+				dc_sqlite3_set_config_int(sql, "dbversion", NEW_DB_VERSION);
+			}
+		#undef NEW_DB_VERSION
+
+		// (2) updates that require high-level objects
+		// (the structure is complete now and all objects are usable)
+		// --------------------------------------------------------------------
+
 		if (recalc_fingerprints)
 		{
 			sqlite3_stmt* stmt = dc_sqlite3_prepare(sql, "SELECT addr FROM acpeerstates;");
@@ -465,6 +483,28 @@ int dc_sqlite3_open(dc_sqlite3_t* sql, const char* dbfile, int flags)
 				}
 			sqlite3_finalize(stmt);
 		}
+
+		if (update_file_paths)
+		{
+			// versions before 2018-08 save the absolute paths in the database files at "param.f=";
+			// for newer versions, we copy files always to the blob directory and store relative paths.
+			// this snippet converts older databases and can be removed after some time.
+			char* repl_from = dc_sqlite3_get_config(sql, "backup_for", sql->context->blobdir);
+			dc_ensure_no_slash(repl_from);
+
+			assert('f'==DC_PARAM_FILE);
+			char* q3 = sqlite3_mprintf("UPDATE msgs SET param=replace(param, 'f=%q/', 'f=$BLOBDIR/');", repl_from);
+			dc_sqlite3_execute(sql, q3);
+			sqlite3_free(q3);
+
+			assert('i'==DC_PARAM_PROFILE_IMAGE);
+			q3 = sqlite3_mprintf("UPDATE chats SET param=replace(param, 'i=%q/', 'i=$BLOBDIR/');", repl_from);
+			dc_sqlite3_execute(sql, q3);
+			sqlite3_free(q3);
+
+			free(repl_from);
+			dc_sqlite3_set_config(sql, "backup_for", NULL);
+		}
 	}
 
 	dc_log_info(sql->context, 0, "Opened \"%s\".", dbfile);

src/dc_tools.c

@@ -1405,3 +1405,61 @@ cleanup:
 	free(pathNfolder_wo_slash);
 	return ret;
 }
+
+
+void dc_make_rel_path(dc_context_t* context, char** path)
+{
+	if (context==NULL || path==NULL || *path==NULL) {
+		return;
+	}
+
+	if (strncmp(*path, context->blobdir, strlen(context->blobdir))==0) {
+		dc_str_replace(path, context->blobdir, "$BLOBDIR");
+	}
+}
+
+
+/**
+ * Copy a file to the blob directory, if needed.
+ *
+ * @param context The context object as returned from dc_context_new().
+ * @param path[in,out] The path, may be modified to a relative path
+ *     starting with `$BLOBDIR`.
+ * @return 1=success file may or may not be copied, 0=error
+ */
+int dc_make_rel_and_copy(dc_context_t* context, char** path)
+{
+	int   success = 0;
+	char* filename = NULL;
+	char* blobdir_path = NULL;
+
+	if (context==NULL || path==NULL || *path==NULL) {
+		goto cleanup;
+	}
+
+	if ((strncmp(*path, context->blobdir, strlen(context->blobdir))==0)
+	 || (strncmp(*path, "$BLOBDIR", 8)==0)) {
+		dc_make_rel_path(context, path);
+		success = 1; // file is already in blobdir
+		goto cleanup;
+	}
+
+	if ((filename=dc_get_filename(*path))==NULL
+	 || (blobdir_path=dc_get_fine_pathNfilename(context, "$BLOBDIR", filename))==NULL
+	 || !dc_copy_file(context, *path, blobdir_path)) {
+		goto cleanup;
+	}
+
+	context->cb(context, DC_EVENT_FILE_COPIED, (uintptr_t)(*path), 0);
+
+	free(*path);
+	*path = blobdir_path;
+	blobdir_path = NULL;
+	dc_make_rel_path(context, path);
+    success = 1;
+
+cleanup:
+	free(blobdir_path);
+	free(filename);
+	return success;
+}

src/dc_tools.h

@@ -110,7 +110,8 @@ int      dc_create_folder           (dc_context_t*, const char* pathNfilename);
 int      dc_write_file              (dc_context_t*, const char* pathNfilename, const void* buf, size_t buf_bytes);
 int      dc_read_file               (dc_context_t*, const char* pathNfilename, void** buf, size_t* buf_bytes);
 char*    dc_get_fine_pathNfilename  (dc_context_t*, const char* pathNfolder, const char* desired_name);
-
+void     dc_make_rel_path           (dc_context_t*, char** pathNfilename);
+int      dc_make_rel_and_copy       (dc_context_t*, char** pathNfilename);
 
 /* macros */
 #define DC_QUOTEHELPER(name) #name

src/deltachat.h

@@ -792,13 +792,30 @@ time_t          dc_lot_get_timestamp     (const dc_lot_t*);
  * A typical purpose for a handler of this event may be to make the file public to some system
  * services.
  *
- * @param data1 (const char*) File name
+ * @param data1 (const char*) Path and file name
  * @param data2 0
  * @return 0
  */
 #define DC_EVENT_IMEX_FILE_WRITTEN        2052
 
 
+/**
+ * A file has been copied. Files given to dc_set_chat_profile_image(),
+ * dc_send_msg() and related functions are copied to the internal blob directory
+ * unless they are already there.
+ *
+ * After copying, this event is sent; from that moment on,
+ * the given file is no longer needed by the library and it is safe to delete it
+ * (eg. in case it was generated for sending only - if you send images from the
+ * gallery you may not want to delete them afterwards).
+ *
+ * @param data1 (const char*) Path and file name
+ * @param data2 0
+ * @return 0
+ */
+#define DC_EVENT_FILE_COPIED              2055
+
+
 /**
  * Progress information of a secure-join handshake from the view of the inviter
  * (Alice, the person who shows the QR code).
@@ -892,7 +909,7 @@ time_t          dc_lot_get_timestamp     (const dc_lot_t*);
  */
 
 
-#define DC_EVENT_DATA1_IS_STRING(e)  ((e)==DC_EVENT_HTTP_GET || (e)==DC_EVENT_IMEX_FILE_WRITTEN)
+#define DC_EVENT_DATA1_IS_STRING(e)  ((e)==DC_EVENT_HTTP_GET || (e)==DC_EVENT_IMEX_FILE_WRITTEN || (e)==DC_EVENT_FILE_COPIED)
 #define DC_EVENT_DATA2_IS_STRING(e)  ((e)==DC_EVENT_INFO || (e)==DC_EVENT_WARNING || (e)==DC_EVENT_ERROR)
 #define DC_EVENT_RETURNS_INT         ((e)==DC_EVENT_IS_OFFLINE)
 #define DC_EVENT_RETURNS_STRING(e)   ((e)==DC_EVENT_GET_QUANTITY_STRING || (e)==DC_EVENT_GET_STRING || (e)==DC_EVENT_HTTP_GET)