From cc2e0ce73697259ef5ac50d0a716d308b6abdf13 Mon Sep 17 00:00:00 2001
From: Jean-Francois Dockes <jfd@recoll.org>
Date: Fri, 18 Oct 2013 13:05:53 +0200
Subject: [PATCH] stringToTokens bug with skipinit 0 (never used)

---
 src/utils/smallut.cpp | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/src/utils/smallut.cpp b/src/utils/smallut.cpp
index 945fe6a8..4997ed0a 100644
--- a/src/utils/smallut.cpp
+++ b/src/utils/smallut.cpp
@@ -365,18 +365,28 @@ void stringToTokens(const string& str, vector<string>& tokens,
 {
     string::size_type startPos = 0, pos;
 
-    for (pos = 0;;) { 
-        // Skip initial delims, break if this eats all.
-        if (skipinit && 
-	    (startPos = str.find_first_not_of(delims, pos)) == string::npos)
-	    break;
+    // Skip initial delims, return empty if this eats all.
+    if (skipinit && 
+	(startPos = str.find_first_not_of(delims, 0)) == string::npos) {
+	return;
+    }
+    while (startPos < str.size()) { 
         // Find next delimiter or end of string (end of token)
         pos = str.find_first_of(delims, startPos);
-        // Add token to the vector. Note: token cant be empty here
-	if (pos == string::npos)
+
+        // Add token to the vector and adjust start
+	if (pos == string::npos) {
 	    tokens.push_back(str.substr(startPos));
-	else
+	    break;
+	} else if (pos == startPos) {
+	    // Dont' push empty tokens after first
+	    if (tokens.empty())
+		tokens.push_back(string());
+	    startPos = ++pos;
+	} else {
 	    tokens.push_back(str.substr(startPos, pos - startPos));
+	    startPos = ++pos;
+	}
     }
 }