Windows: manage timeouts, time and size limits

src/filters/rclexecm.py

@@ -58,12 +58,34 @@ class RclExecM:
             import msvcrt
             msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
             msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
+        self.debugfile = None
+        if self.debugfile:
+            self.errfout = open(self.debugfile, "ab")
+        else:
+            self.errfout = sys.stderr
         
     def rclog(self, s, doexit = 0, exitvalue = 1):
-        print("RCLMFILT: %s: %s" % (self.myname, s), file=sys.stderr)
+        print("RCLMFILT: %s: %s" % (self.myname, s), file=self.errfout)
         if doexit:
             sys.exit(exitvalue)
 
+    def breakwrite(self, outfile, data):
+        if sys.platform != "win32":
+            outfile.write(data)
+        else:
+            total = len(data)
+            bs = 4*1024
+            offset = 0
+            while total > 0:
+                if total < bs:
+                    tow = total
+                else:
+                    tow = bs
+                #self.rclog("Total %d Writing %d to stdout: %s" % (total,tow,data[offset:offset+tow]))
+                outfile.write(data[offset:offset+tow])
+                offset += tow
+                total -= tow
+                
     # Note: tried replacing this with a multiple replacer according to
     #  http://stackoverflow.com/a/15221068, which was **10 times** slower
     def htmlescape(self, txt):
@@ -119,7 +141,7 @@ class RclExecM:
                 docdata = docdata.encode("UTF-8")
 
             print("Document: %d" % len(docdata))
-            sys.stdout.write(docdata)
+            self.breakwrite(sys.stdout, docdata)
 
             if len(ipath):
                 print("Ipath: %d" % len(ipath))
@@ -332,7 +354,7 @@ def main(proto, extract):
             else:
                 bdata = data
             if debugDumpData or actAsSingle:
-                sys.stdout.write(bdata)
+                proto.breakwrite(sys.stdout, bdata)
                 print()
             sys.exit(0)
         else:
@@ -351,7 +373,7 @@ def main(proto, extract):
             else:
                 bdata = data
             if debugDumpData:
-                sys.stdout.write(bdata)
+                proto.breakwrite(sys.stdout, bdata)
                 print()
             if eof != RclExecM.noteof:
                 sys.exit(0)

src/librecoll/librecoll.pro

@@ -13,6 +13,7 @@ DEFINES += LIBRECOLL_LIBRARY BUILDING_RECOLL
 DEFINES -= UNICODE
 DEFINES -= _UNICODE
 DEFINES += _MBCS
+DEFINES += PSAPI_VERSION=1
 
 SOURCES += \
 ../aspell/rclaspell.cpp \
@@ -119,7 +120,7 @@ windows{
         # Visual Studio
     }
   LIBS += c:/recolldeps/xapian/xapian-core-1.2.21/.libs/libxapian-22.dll \
-       c:/recolldeps/zlib-1.2.8/zlib1.dll -liconv -lshlwapi -lkernel32
+       c:/recolldeps/zlib-1.2.8/zlib1.dll -liconv -lshlwapi -lpsapi -lkernel32
 }
 
 unix {

src/windows/execmd_w.cpp

@@ -11,6 +11,7 @@
 #include "safesysstat.h"
 #include "safeunistd.h"
 #include "safewindows.h"
+#include <psapi.h>
 #include "smallut.h"
 #include "pathut.h"
 
@@ -89,7 +90,7 @@ static string argvToCmdLine(const string& cmd, const vector<string>& args)
 }
 
 // Merge the father environment with the variable specified in m_env
-char *mergeEnvironment(const STD_UNORDERED_MAP<string, string>& addenv)
+static char *mergeEnvironment(const STD_UNORDERED_MAP<string, string>& addenv)
 {
     // Parse existing environment.
     char *envir = GetEnvironmentStrings();
@@ -235,13 +236,27 @@ static WaitResult Wait(HANDLE hdl, int timeout)
     return Timeout;
 }
 
+static int getVMMBytes(HANDLE hProcess)
+{
+    PROCESS_MEMORY_COUNTERS pmc;
+    const int MB = 1024 * 1024;
+    if (GetProcessMemoryInfo(hProcess, &pmc, sizeof(pmc))) {
+        LOGDEB2(("ExecCmd: getVMMBytes paged Kbs %d non paged %d Kbs\n",
+                 int(pmc.QuotaPagedPoolUsage/1024),
+                 int(pmc.QuotaNonPagedPoolUsage/1024)));
+        return int(pmc.QuotaPagedPoolUsage /MB +
+                   pmc.QuotaNonPagedPoolUsage / MB);
+    }
+    return -1;
+}
+
 ////////////////////////////////////////////////////////
 // ExecCmd:
 
 class ExecCmd::Internal {
 public:
     Internal()
-        : m_advise(0), m_provide(0), m_timeoutMs(1000) {
+        : m_advise(0), m_provide(0), m_timeoutMs(1000), m_rlimit_as_mbytes(0) {
         reset();
     }
 
@@ -249,6 +264,7 @@ public:
     ExecCmdAdvise   *m_advise;
     ExecCmdProvide  *m_provide;
     int              m_timeoutMs;
+    int m_rlimit_as_mbytes;
 
     // We need buffered I/O for getline. The Unix version uses netcon's
     string m_buf;	// Buffer. Only used when doing getline()s
@@ -278,6 +294,7 @@ public:
     bool preparePipes(bool has_input, HANDLE *hChildInput,
                       bool has_output, HANDLE *hChildOutput,
                       HANDLE *hChildError);
+    bool tooBig();
 };
 
 // ExecCmd resource releaser class. Using a separate object makes it
@@ -455,7 +472,21 @@ void ExecCmd::putenv(const string &name, const string& value)
 
 void ExecCmd::setrlimit_as(int mbytes)
 {
-	// Later maybe
+    m->m_rlimit_as_mbytes = mbytes;
+}
+
+bool ExecCmd::Internal::tooBig()
+{
+    if (m_rlimit_as_mbytes <= 0)
+        return false;
+    int mbytes = getVMMBytes(m_piProcInfo.hProcess);
+    if (mbytes > m_rlimit_as_mbytes) {
+        LOGINFO(("ExecCmd:: process mbytes %d > set limit %d\n",
+                mbytes, m_rlimit_as_mbytes));
+        m_killRequest = true;
+        return true;
+    }
+    return false;
 }
 
 bool ExecCmd::Internal::preparePipes(bool has_input,HANDLE *hChildInput, 
@@ -781,11 +812,11 @@ int ExecCmd::receive(string& data, int cnt)
         }
     }
     while (true) {
-        const int BUFSIZE = 8192;
+        const int BUFSIZE = 4096;
         CHAR chBuf[BUFSIZE];
         int toread = cnt > 0 ? MIN(cnt - totread, BUFSIZE) : BUFSIZE;
         BOOL bSuccess = ReadFile(m->m_hOutputRead, chBuf, toread,
-				            NULL, &m->m_oOutputRead);
+                                 NULL, &m->m_oOutputRead);
         DWORD err = GetLastError();
         LOGDEB1(("receive: ReadFile: success %d err %d\n",
                  int(bSuccess), int(err)));
@@ -795,7 +826,8 @@ int ExecCmd::receive(string& data, int cnt)
             break;
         }
 
-        WaitResult waitRes = Wait(m->m_oOutputRead.hEvent, 1000);
+    waitagain:
+        WaitResult waitRes = Wait(m->m_oOutputRead.hEvent, m->m_timeoutMs);
         if (waitRes == Ok) {
             DWORD dwRead;
             if (!GetOverlappedResult(m->m_hOutputRead, &m->m_oOutputRead,
@@ -820,6 +852,13 @@ int ExecCmd::receive(string& data, int cnt)
             }
             break;
         } else if (waitRes == Timeout) {
+            LOGDEB0(("ExecCmd::receive: timeout (%d mS)\n", m->m_timeoutMs));
+            if (m->tooBig()) {
+                if (!CancelIo(m->m_hOutputRead)) {
+                    printError("CancelIo");
+                }
+                return -1;
+            }
             // We only want to cancel if m_advise says so here.
             if (m->m_advise) {
                 try {
@@ -838,6 +877,7 @@ int ExecCmd::receive(string& data, int cnt)
                 }
                 break;
             }
+            goto waitagain;
         }
         if ((cnt == 0 && totread > 0) || (cnt > 0 && totread == cnt))
             break;
@@ -901,7 +941,19 @@ int ExecCmd::wait()
     DWORD exit_code = -1;
     if (!m->m_killRequest && m->m_piProcInfo.hProcess) {
         // Wait until child process exits.
-        WaitForSingleObject(m->m_piProcInfo.hProcess, INFINITE);
+        while (WaitForSingleObject(m->m_piProcInfo.hProcess, m->m_timeoutMs)
+               == WAIT_TIMEOUT) {
+            LOGDEB(("ExecCmd::wait: timeout (ok)\n"));
+            if (m->m_advise) {
+                m->m_advise->newData(0);
+            }
+            if (m->tooBig()) {
+                // Let cleaner work to kill the child
+                m->m_killRequest = true;
+                return -1;
+            }
+        }
+
         exit_code = 0;
         GetExitCodeProcess(m->m_piProcInfo.hProcess, &exit_code);
         // Clean up, here to avoid cleaner trying to kill the now

src/windows/mkinstdir.sh

@@ -17,6 +17,9 @@ ANTIWORD=c:/recolldeps/antiword
 CONFIGURATION=Debug
 PLATFORM=x64
 
+GUIBIN=c:/recoll/src/build-librecoll-Desktop_Qt_5_5_0_MinGW_32bit-Debug/debug/librecoll.dll
+GUILIB=c:/Users/Bill/recoll/src/build-recoll-Desktop_Qt_5_5_0_MinGW_32bit-Debug/debug/recoll.exe
+
 
 ################
 # Script:
@@ -58,6 +61,9 @@ copyrecoll()
     cp $RECOLL/qtgui/mtpics/*  $DESTDIR/Share/images
     
     cp $RECOLL/qtgui/i18n/*.qm $DESTDIR/Share/translations
+
+    cp $GUIBIN $DESTDIR || fatal copy recoll.exe
+    cp $GUILIB $DESTDIR || fatal copy Gui librecoll
 }
 
 copyantiword()