Use recursive mutex on log calls because some call arguments can themselves call the log. Caused actual deadlocks in recoll 1.23 GUI with high verbosity

fix typo in mimeview (TB opens eml not emf files)

.hgignore

@@ -2,117 +2,71 @@ syntax: glob
 *.dep
 *.dep.stamp
 *.o 
+.libs
+*.la
+stamp-h1
+.dirstamp
+libtool
+.deps
+*.lo
 *~
 \#*
+*.obj
+*.sdf
+*.tlog
+*.lib
+*.idb
+*.log
+*.pdb
+.vs
+*.exe
+*.ilk
 ptrans
-src/Makefile
-src/autom4te.cache
-src/bincimapmime/alldeps
-src/common/alldeps
-src/common/autoconfig.h
-src/common/rclversion.h
+src/aclocal.m4
+src/compile
+src/config.guess
+src/config.sub
+src/depcomp
+src/install-sh
+src/Makefile.in
+src/ltmain.sh
+src/m4/libtool.m4
+src/m4/ltoptions.m4
+src/m4/ltsugar.m4
+src/m4/ltversion.m4
+src/m4/lt~obsolete.m4
+src/missing
 src/config.log
 src/config.status
 src/configure
-src/desktop/unity-lens-recoll/Makefile
-src/desktop/unity-lens-recoll/autom4te.cache
-src/desktop/unity-lens-recoll/bin/unity-recoll-daemon
-src/desktop/unity-lens-recoll/config.log
-src/desktop/unity-lens-recoll/config.status
-src/desktop/unity-lens-recoll/data/recoll.lens
-src/desktop/unity-lens-recoll/data/unity-lens-recoll.service
-src/doc/user/HTML.manifest
-src/doc/user/RCL.INDEXING.CONFIG.html
-src/doc/user/RCL.INDEXING.EXTATTR.html
-src/doc/user/RCL.INDEXING.EXTTAGS.html
-src/doc/user/RCL.INDEXING.MONITOR.html
-src/doc/user/RCL.INDEXING.PERIODIC.html
-src/doc/user/RCL.INDEXING.STORAGE.html
-src/doc/user/RCL.INDEXING.WEBQUEUE.html
-src/doc/user/RCL.INDEXING.html
-src/doc/user/RCL.INSTALL.BUILDING.html
-src/doc/user/RCL.INSTALL.CONFIG.html
-src/doc/user/RCL.INSTALL.EXTERNAL.html
-src/doc/user/RCL.INSTALL.html
-src/doc/user/RCL.INTRODUCTION.RECOLL.html
-src/doc/user/RCL.INTRODUCTION.SEARCH.html
-src/doc/user/RCL.INTRODUCTION.html
-src/doc/user/RCL.KICKER-APPLET.html
-src/doc/user/RCL.PROGRAM.API.html
-src/doc/user/RCL.PROGRAM.FIELDS.html
-src/doc/user/RCL.PROGRAM.html
-src/doc/user/RCL.SEARCH.ANCHORWILD.html
-src/doc/user/RCL.SEARCH.CASEDIAC.html
-src/doc/user/RCL.SEARCH.COMMANDLINE.html
-src/doc/user/RCL.SEARCH.COMPLEX.html
-src/doc/user/RCL.SEARCH.CUSTOM.html
-src/doc/user/RCL.SEARCH.DESKTOP.html
-src/doc/user/RCL.SEARCH.HISTORY.html
-src/doc/user/RCL.SEARCH.KIO.html
-src/doc/user/RCL.SEARCH.LANG.html
-src/doc/user/RCL.SEARCH.MULTIDB.html
-src/doc/user/RCL.SEARCH.PREVIEW.html
-src/doc/user/RCL.SEARCH.PTRANS.html
-src/doc/user/RCL.SEARCH.RESLIST.html
-src/doc/user/RCL.SEARCH.SORT.html
-src/doc/user/RCL.SEARCH.TERMEXPLORER.html
-src/doc/user/RCL.SEARCH.TIPS.html
-src/doc/user/RCL.SEARCH.WILDCARDS.html
-src/doc/user/RCL.SEARCH.html
-src/doc/user/RCL.SEARCHKCL.html
-src/doc/user/RCL.SEARCHKIO.SEARCHABLEDOCS.html
-src/doc/user/RCL.SEARCHKIO.html
-src/doc/user/index.html
-src/doc/user/usermanual-xml.html
-src/doc/user/usermanual.aux
-src/doc/user/usermanual.html
-src/doc/user/usermanual.html-text
-src/doc/user/usermanual.log
-src/doc/user/usermanual.out
+src/Makefile
+src/autom4te.cache
+src/common/autoconfig.h
+src/common/rclversion.h
+src/doc/user/webhelp/docs/*
 src/doc/user/usermanual.pdf
-src/doc/user/usermanual.tex-pdf
-src/doc/user/usermanual.tex-pdf-tmp
-src/doc/user/usermanual.txt
 src/filters/rclexecm.pyc
 src/filters/rcllatinclass.pyc
-src/index/alldeps
-src/index/alldeps.stamp
-src/index/recollindex
-src/internfile/alldeps
+src/recollindex
 src/kde/kioslave/kio_recoll/builddir
-src/lib/Makefile
-src/lib/alldeps
-src/lib/librecoll.a
-src/lib/librecoll.so*
-src/lib/mkMake
-src/mk/localdefs
-src/mk/sysconf
+src/python/recoll/Makefile
 src/python/recoll/build
 src/python/recoll/recoll/__init__.pyc
 src/python/recoll/setup.py
 src/python/samples/recollgui/rclmain.py
 src/python/samples/recollgui/rclmain.pyc
+src/recollq
+src/xadump
 src/qtgui/.moc/*
 src/qtgui/.obj/*
 src/qtgui/.ui/*
 src/qtgui/Makefile
 src/qtgui/qrc_recoll.cpp
 src/qtgui/recoll
-src/qtgui/recoll.app
 src/qtgui/recoll.pro
-src/query/alldeps
-src/query/location.hh
-src/query/position.hh
-src/query/recollq
-src/query/stack.hh
-src/query/wasaparse.output
-src/query/wasaparse.tab.cpp
-src/query/wasaparse.tab.h
-src/query/xadump
-src/recollinstall
+src/qtgui/recoll.app
 src/sampleconf/rclmon.sh
 src/sampleconf/recoll.conf
-src/utils/alldeps
 tests/casediac/aspdict.en.rws
 tests/casediac/idxstatus.txt
 tests/casediac/index.pid
@@ -137,3 +91,7 @@ tests/xattr/mimeview
 website/usermanual/*
 website/idxthreads/forkingRecoll.html
 website/idxthreads/xapDocCopyCrash.html
+website/pages/recoll-mingw.html
+website/pages/recoll-webui-install-wsgi.html
+website/pages/recoll-windows.html
+website/pages/recoll-windows-faq.html

packaging/debian/buildppa.sh

@@ -4,7 +4,7 @@
 # For the kio: (and kdesdk?)
 # sudo apt-get install pkg-kde-tools  cdbs
 
-RCLVERS=1.21.0
+RCLVERS=1.22.4
 LENSVERS=1.19.10.3543
 SCOPEVERS=1.20.2.4
 PPAVERS=1
@@ -19,7 +19,7 @@ case $RCLVERS in
     1.14*) PPANAME=recoll-ppa;;
     *)     PPANAME=recoll15-ppa;;
 esac
-PPANAME=recollexp-ppa
+#PPANAME=recollexp-ppa
 echo "PPA: $PPANAME. Type CR if Ok, else ^C"
 read rep
 
@@ -38,10 +38,12 @@ check_recoll_orig()
     fi
 }
 
-####### QT4
+# Note: recoll 1.22+ builds on precise fail. precise stays at 1.21
+
+####### QT
 debdir=debian
 # Note: no new releases for lucid: no webkit. Or use old debianrclqt4 dir.
-#series="precise trusty utopic vivid"
+series="trusty xenial yakkety"
 series=
 
 if test "X$series" != X ; then
@@ -70,8 +72,8 @@ for series in $series ; do
 done
 
 ### KIO
-series="precise trusty utopic vivid"
-series=vivid
+series="trusty xenial yakkety"
+series="xenial yakkety"
 
 debdir=debiankio
 topdir=kio-recoll-${RCLVERS}
@@ -89,18 +91,23 @@ if test "X$series" != X ; then
         cd ..
     fi
 fi
-for series in $series ; do
+for svers in $series ; do
 
   rm -rf $topdir/debian
   cp -rp ${debdir}/ $topdir/debian || exit 1
 
-  sed -e s/SERIES/$series/g \
+  sed -e s/SERIES/$svers/g \
       -e s/PPAVERS/${PPAVERS}/g \
           < ${debdir}/changelog > $topdir/debian/changelog ;
-
+  if test $svers = "trusty" ; then
+      mv -f $topdir/debian/control-4 $topdir/debian/control
+      mv -f $topdir/debian/rules-4 $topdir/debian/rules
+  else
+      rm -f $topdir/debian/control-4 $topdir/debian/rules-4
+  fi
   (cd $topdir;debuild -S -sa) || exit 1
 
-  dput $PPANAME kio-recoll_${RCLVERS}-0~ppa${PPAVERS}~${series}1_source.changes
+  dput $PPANAME kio-recoll_${RCLVERS}-0~ppa${PPAVERS}~${svers}1_source.changes
 
 done
 
@@ -140,7 +147,7 @@ for series in $series ; do
 done
 
 ### Unity Scope
-series="trusty utopic vivid"
+series="trusty xenial yakkety"
 series=
 
 debdir=debianunityscope
@@ -154,10 +161,15 @@ if test "X$series" != X ; then
         if test -f unity-scope-recoll-${SCOPEVERS}.tar.gz ; then
             mv unity-scope-recoll-${SCOPEVERS}.tar.gz \
                 unity-scope-recoll_${SCOPEVERS}.orig.tar.gz
+        else
+            if test -f $RCLDOWNLOAD/unity-scope-recoll-${SCOPEVERS}.tar.gz;then
+                cp -p $RCLDOWNLOAD/unity-scope-recoll-${SCOPEVERS}.tar.gz \
+                   unity-scope-recoll_${SCOPEVERS}.orig.tar.gz || fatal copy
             else
                 fatal "Can find neither " \
                       "unity-scope-recoll_${SCOPEVERS}.orig.tar.gz nor " \
-                "unity-scope-recoll-${SCOPEVERS}.tar.gz"
+                      "$RCLDOWNLOAD/unity-scope-recoll-${SCOPEVERS}.tar.gz"
+            fi
         fi
     fi
     test -d $topdir ||  tar xvzf unity-scope-recoll_${SCOPEVERS}.orig.tar.gz \

packaging/debian/debian/changelog

@@ -1,3 +1,63 @@
+recoll (1.22.4-1~ppaPPAVERS~SERIES1) SERIES; urgency=low
+                
+  * Fix advanced search 'start search' not doing anything with qt5
+
+ -- Jean-Francois Dockes <jf@dockes.org>  Tue, 25 Nov 2016 11:38:00 +0100
+
+recoll (1.22.3-1~ppaPPAVERS~SERIES1) SERIES; urgency=low
+                
+  * 
+
+ -- Jean-Francois Dockes <jf@dockes.org>  Tue, 21 Jun 2016 15:11:00 +0200
+
+recoll (1.22.2-1~ppaPPAVERS~SERIES1) SERIES; urgency=low
+                
+  * Packaging fixes.
+
+ -- Jean-Francois Dockes <jf@dockes.org>  Thu, 16 Jun 2016 09:11:00 +0200
+
+recoll (1.22.1-1~ppaPPAVERS~SERIES1) SERIES; urgency=low
+                
+  * Fixed two GUI crashes
+  * Small other fixes
+
+ -- Jean-Francois Dockes <jf@dockes.org>  Tue, 14 Jun 2016 17:31:00 +0200
+
+recoll (1.22.0-1~ppaPPAVERS~SERIES1) SERIES; urgency=low
+                
+  * New build based on autotools
+  * Synonyms
+  * Windows port
+
+ -- Jean-Francois Dockes <jf@dockes.org>  Tue, 05 Apr 2016 16:01:00 +0200
+
+recoll (1.21.5-1~ppaPPAVERS~SERIES1) SERIES; urgency=low
+                
+  * Fix query language parser for multiple mime or rclcat entries
+
+ -- Jean-Francois Dockes <jf@dockes.org>  Fri, 29 Jan 2016 14:48:00 +0200
+
+recoll (1.21.4-1~ppaPPAVERS~SERIES1) SERIES; urgency=low
+                
+  * Show confirmation dialog when opening temp files + small bug fixes
+
+ -- Jean-Francois Dockes <jf@dockes.org>  Tue, 12 Jan 2016 16:19:00 +0200
+
+recoll (1.21.3-1~ppaPPAVERS~SERIES1) SERIES; urgency=low
+                
+  * Fix webcache size being capped at 1 GB
+
+ -- Jean-Francois Dockes <jf@dockes.org>  Sat, 31 Oct 2015 09:26:00 +0200
+
+recoll (1.21.2-1~ppaPPAVERS~SERIES1) SERIES; urgency=low
+                
+  * New special indexing dialog in GUI
+  * Fix advanced search dialog "Any Clause" mode
+  * Fixed a few bounds issues catched by Windows VC++
+  * Miscellaneous other minor fixes
+
+ -- Jean-Francois Dockes <jf@dockes.org>  Tue, 01 Oct 2015 09:40:00 +0200
+
 recoll (1.21.0-1~ppaPPAVERS~SERIES1) SERIES; urgency=low
                 
   * New bison-based query parser.

packaging/debian/debian/control

@@ -4,7 +4,7 @@ Priority: optional
 Maintainer: Jean-Francois Dockes <jfd@recoll.org>
 Build-Depends: autotools-dev,
                debhelper (>= 7),
-               hardening-wrapper,
+               dh-python,
                bison,
                libqt4-dev,
                libqtwebkit-dev,
@@ -13,7 +13,7 @@ Build-Depends: autotools-dev,
                libz-dev,
                python-all-dev (>= 2.6.6-3~),
                python3-all-dev,
-Standards-Version: 3.9.4
+Standards-Version: 3.9.6
 X-Python-Version: >= 2.7
 Homepage: http://www.lesbonscomptes.com/recoll
 Vcs-Git: git://anonscm.debian.org/collab-maint/recoll.git
@@ -21,10 +21,10 @@ Vcs-Browser: http://anonscm.debian.org/gitweb/?p=collab-maint/recoll.git;a=summa
 
 Package: recoll
 Architecture: any
-Depends: ${misc:Depends}, ${shlibs:Depends}
-Recommends: python-recoll, aspell, python, xdg-utils, xsltproc
+Depends: ${misc:Depends}, ${shlibs:Depends}, python
+Recommends: python-recoll, aspell, xdg-utils, xsltproc,
+            python-libxml2, python-libxslt1
 Suggests: antiword,
-          catdoc,
           ghostscript,
           libimage-exiftool-perl,
           poppler-utils,

packaging/debian/debian/copyright

@@ -83,42 +83,11 @@ License: LGPL-2+
  can be found in `/usr/share/common-licenses/LGPL-2' and
  `/usr/share/common-licenses/LGPL-2.1' and `/usr/share/common-licenses/LGPL-3'.
 
-Files: qtgui/q3richtext_p.h
-Copyright: 1992-2007, Trolltech ASA. All rights reserved
-License:
- This file is part of the Qt3Support module of the Qt Toolkit.
- .
- This file may be used under the terms of the GNU General Public License
- version 2.0 as published by the Free Software Foundation and appearing in the
- file LICENSE.GPL included in the packaging of this file.  Please review the
- following information to ensure GNU General Public Licensing requirements will
- be met: http://trolltech.com/products/qt/licenses/licensing/opensource/
- .
- If you are unsure which license is appropriate for your use, please review the
- following information:
- http://trolltech.com/products/qt/licenses/licensing/licensingoverview
- or contact the sales department at sales@trolltech.com.
- .
- In addition, as a special exception, Trolltech gives you certain additional
- rights. These rights are described in the Trolltech GPL Exception version 1.0,
- which can be found at http://www.trolltech.com/products/qt/gplexception/
- and in the file GPL_EXCEPTION.txt in this package.
- .
- In addition, as a special exception, Trolltech, as the sole copyright
- holder for Qt Designer, grants users of the Qt/Eclipse Integration
- plug-in the right for the Qt/Eclipse Integration to link to
- functionality provided by Qt Designer and its related libraries.
- .
- Trolltech reserves all rights not expressly granted herein.
- .
- Trolltech ASA (c) 2007
- .
- This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
- WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
-
 Files: common/uproplist.h
 Copyright: 1991-2006, Unicode, Inc.
-License:
+License: Unicode
+
+License: Unicode
  All rights reserved. Distributed under the Terms of Use in
  http://www.unicode.org/copyright.html
  .
@@ -153,7 +122,9 @@ License:
 
 Files: utils/md5.*
 Copyright: 1991-1992, RSA Data Security, Inc. All rights reserved.
-License:
+License: RSA
+
+License:RSA
  MD5C.C - RSA Data Security, Inc., MD5 message-digest algorithm
  .
  License to copy and use this software is granted provided that it is
@@ -176,7 +147,9 @@ License:
 Files: desktop/xdg-utils-1.0.1/*
 Copyright: 2006, Kevin Krammer <kevin.krammer@gmx.at>,
                  Jeremy White <jwhite@codeweavers.com>.
-License:
+License: MIT
+
+License: MIT
  Permission is hereby granted, free of charge, to any person obtaining a copy
  of this software and associated documentation files (the "Software"), to deal
  in the Software without restriction, including without limitation the rights
@@ -197,6 +170,8 @@ License:
 
 Files: index/csguess.cpp
 Copyright: 2000-2004, Mikio Hirabayashi
+License: LGPL-2.1+
+
 License: LGPL-2.1+
  This file is part of QDBM, Quick Database Manager.
  .

packaging/debian/debian/patches/fix-python-install.patch

@@ -1,10 +0,0 @@
---- a/recollinstall.in
-+++ b/recollinstall.in
-@@ -149,5 +149,5 @@
-     ${datadir}/recoll/translations/recoll_zh.qm || exit 1
- 
- 
--@NOPYTHON@(cd python/recoll;python setup.py install \
--@NOPYTHON@ --prefix=${REALPREFIX} ${ROOTFORPYTHON} ${OPTSFORPYTHON})
-+#@NOPYTHON@(cd python/recoll;python setup.py install \
-+#@NOPYTHON@ --prefix=${REALPREFIX} ${ROOTFORPYTHON} ${OPTSFORPYTHON})

packaging/debian/debian/patches/series

@@ -1 +0,0 @@
-fix-python-install.patch

packaging/debian/debian/rules

@@ -3,16 +3,17 @@
 # Uncomment this to turn on verbose mode.
 #export DH_VERBOSE=1
 
-export DEB_BUILD_HARDENING=1
+DPKG_EXPORT_BUILDFLAGS = 1
+include /usr/share/dpkg/buildflags.mk
 
 DEB_HOST_GNU_TYPE   ?= $(shell dpkg-architecture -qDEB_HOST_GNU_TYPE)
 DEB_BUILD_GNU_TYPE  ?= $(shell dpkg-architecture -qDEB_BUILD_GNU_TYPE)
 
-CFLAGS = -Wall -g
-#LDFLAGS = -Wl,-z,defs
+CFLAGS += -Wall -g
+#LDFLAGS += -Wl,-z,defs
 
 #build qt4 UI only
-#export QMAKE=qmake-qt4
+export QMAKE=qmake-qt4
 
 ifneq (,$(findstring noopt,$(DEB_BUILD_OPTIONS)))
 	CFLAGS += -O0
@@ -33,7 +34,7 @@ build-arch: build-stamp
 build-indep: build-stamp
 build-stamp: config.status
 	dh_testdir
-	$(MAKE)
+	$(MAKE) -j 5
 	touch $@
 
 clean:
@@ -48,10 +49,12 @@ install:
 	dh_testroot
 	dh_prep
 	dh_installdirs
-	$(MAKE) STRIP=ls prefix=$(CURDIR)/debian/tmp/usr install
-	# Work around recollinstall forgetting the PPT text-extract utility
-	install -m 0444 filters/msodump.zip $(CURDIR)/debian/tmp/usr/share/recoll/filters
-	install -m 0555 filters/ppt-dump.py $(CURDIR)/debian/tmp/usr/share/recoll/filters
+	$(MAKE) prefix=$(CURDIR)/debian/tmp/usr install
+	chmod a+x $(CURDIR)/debian/tmp/usr/share/recoll/examples/rclmon.sh
+	chmod a-x $(CURDIR)/debian/tmp/usr/share/recoll/filters/rclxslt.py
+	chmod a-x $(CURDIR)/debian/tmp/usr/share/recoll/filters/rclexec1.py
+	chmod a-x $(CURDIR)/debian/tmp/usr/share/recoll/filters/rclexec1.py
+	rm -f $(CURDIR)/debian/tmp/usr/lib/recoll/librecoll.la
 	(cd python/recoll; python setup.py install \
 	                                   --install-layout=deb \
 	                                   --prefix=$(CURDIR)/debian/tmp/usr )

packaging/debian/debiankio/changelog

@@ -1,3 +1,21 @@
+kio-recoll (1.22.4-0~ppaPPAVERS~SERIES1) SERIES; urgency=low
+
+  * Switch to kf5 for appropriate series. Update to 1.22.4 (same as .3)
+
+ -- Jean-Francois Dockes <jf@dockes.org>  Sat, 26 Nov 2016 09:46:00 +0100
+
+kio-recoll (1.22.3-0~ppaPPAVERS~SERIES1) SERIES; urgency=low
+
+  * No changes specific to the KIO, just use the 1.22 lib. Update to .3
+
+ -- Jean-Francois Dockes <jf@dockes.org>  Fri, 25 Nov 2016 10:59:00 +0100
+
+kio-recoll (1.22.2-0~ppaPPAVERS~SERIES1) SERIES; urgency=low
+
+  * No changes specific to the KIO, just use the 1.22 lib.
+
+ -- Jean-Francois Dockes <jf@dockes.org>  Tue, 15 Jun 2016 09:57:00 +0200
+
 kio-recoll (1.21.0-0~ppaPPAVERS~SERIES1) SERIES; urgency=low
 
   * No changes specific to the KIO, just use the 1.21 lib.

packaging/debian/debiankio/control

@@ -4,20 +4,24 @@ Priority: extra
 Maintainer: Jean-Francois Dockes <jfd@recoll.org>
 Build-Depends: cdbs, 
                cmake,
+               libtool,
                debhelper (>= 7), 
                bison,
                kdelibs5-dev (>= 4:4.2.2),
+               qtbase5-dev,
+               extra-cmake-modules,
+               kio-dev,
                pkg-kde-tools (>= 0.4.0), 
                libxapian-dev, 
                libz-dev
-Standards-Version: 3.9.5
+Standards-Version: 3.9.7
 Homepage: http://www.recoll.org/
 
 Package: kio-recoll
 Architecture: any
 Depends: ${misc:Depends}, ${shlibs:Depends}, recoll
-Description: A Recoll KIO slave for KDE 4
- A Recoll KIO slave for KDE 4, allows performing a Recoll search by
+Description: Recoll KIO slave for KDE 4
+ Recoll KIO slave for KDE 4, allows performing a Recoll search by
  entering an appropriate URL in a KDE open dialog, or with an HTML-based
  interface displayed in Konqueror.  
  The HTML-based interface is similar to the Recoll GUI QT-based interface,

packaging/debian/debiankio/control-4

@@ -0,0 +1,33 @@
+Source: kio-recoll
+Section: kde
+Priority: extra
+Maintainer: Jean-Francois Dockes <jfd@recoll.org>
+Build-Depends: cdbs, 
+               cmake,
+               libtool,
+               debhelper (>= 7), 
+               bison,
+               kdelibs5-dev (>= 4:4.2.2), 
+               pkg-kde-tools (>= 0.4.0), 
+               libxapian-dev, 
+               libz-dev
+Standards-Version: 3.9.5
+Homepage: http://www.recoll.org/
+
+Package: kio-recoll
+Architecture: any
+Depends: ${misc:Depends}, ${shlibs:Depends}, recoll
+Description: A Recoll KIO slave for KDE 4
+ A Recoll KIO slave for KDE 4, allows performing a Recoll search by
+ entering an appropriate URL in a KDE open dialog, or with an HTML-based
+ interface displayed in Konqueror.  
+ The HTML-based interface is similar to the Recoll GUI QT-based interface,
+ slightly less powerful. It allows performing a search while staying fully
+ within the KDE framework: drag and drop from the result list works
+ normally and you have your normal choice of applications for opening files. 
+ An alternative interface uses a directory view of search results. Due to
+ limitations in the current KIO slave interface, it is currently not
+ obviously useful.
+ The interface is described in more detail inside a help file which you can
+ access by entering recoll:/ inside the konqueror URL line (this works only
+ if the recoll KIO slave has been previously installed). 

packaging/debian/debiankio/copyright

@@ -54,7 +54,7 @@ Copyright (c) 1991-2004 Unicode, Inc.
 
   COPYRIGHT AND PERMISSION NOTICE
   
-  Copyright © 1991-2006 Unicode, Inc. All rights reserved. Distributed under
+  Copyright © 1991-2006 Unicode, Inc. All rights reserved. Distributed under
   the Terms of Use in http://www.unicode.org/copyright.html. 
   
   Permission is hereby granted, free of charge, to any person obtaining a

packaging/debian/debiankio/patches/fix-cmake-build-parser.patch

@@ -1,21 +0,0 @@
-diff -r 1e9d7ca73884 src/kde/kioslave/kio_recoll/CMakeLists.txt
---- a/kde/kioslave/kio_recoll/CMakeLists.txt	Tue Jun 16 15:06:07 2015 +0200
-+++ b/kde/kioslave/kio_recoll/CMakeLists.txt	Wed Jun 17 09:08:36 2015 +0200
-@@ -57,11 +57,16 @@
- 
- kde4_add_plugin(kio_recoll ${kio_recoll_SRCS})
- 
-+add_custom_target(parser
-+                 COMMAND make wasaparse.tab.cpp
-+                 WORKING_DIRECTORY ${rcltop}/query
-+)
- add_custom_target(rcllib
-                  COMMAND make librecoll.a
-                  WORKING_DIRECTORY ${rcltop}/lib
- )
--add_dependencies(kio_recoll rcllib)
-+add_dependencies(rcllib parser)
-+add_dependencies(kio_recoll parser rcllib)
- 
- target_link_libraries(kio_recoll recoll xapian z ${EXTRA_LIBS} ${KDE4_KIO_LIBS})
- 

packaging/debian/debiankio/patches/series

@@ -1 +0,0 @@
-fix-cmake-build-parser.patch

packaging/debian/debiankio/rules-4

@@ -0,0 +1,6 @@
+#!/usr/bin/make -f
+
+include /usr/share/cdbs/1/rules/debhelper.mk
+include /usr/share/pkg-kde-tools/makefiles/1/cdbs/kde.mk
+
+DEB_SRCDIR = kde/kioslave/kio_recoll-kde4

packaging/rpm/kio_recoll-opensuse.spec

@@ -0,0 +1,85 @@
+#
+# spec file for package kio_recoll
+#
+# Copyright (c) 2013 SUSE LINUX Products GmbH, Nuernberg, Germany.
+#
+# All modifications and additions to the file contributed by third parties
+# remain the property of their copyright owners, unless otherwise agreed
+# upon. The license for this file, and modifications and additions to the
+# file, is the same license as for the pristine package itself (unless the
+# license for the pristine package is not an Open Source License, in which
+# case the license is the MIT License). An "Open Source License" is a
+# license that conforms to the Open Source Definition (Version 1.9)
+# published by the Open Source Initiative.
+
+# Please submit bugfixes or comments via http://bugs.opensuse.org/
+#
+
+%define rname recoll
+
+Name:           kio_recoll
+Version:        1.21.6
+Release:        0
+Summary:        Extended Search
+License:        GPL-2.0+
+Summary:        KIO slave for the Recoll full text search tool
+Group:          Productivity/Text/Utilities
+Url:            http://www.lesbonscomptes.com/recoll/
+Source:         http://www.lesbonscomptes.com/recoll/%{rname}-%{version}.tar.gz
+BuildRequires:  libkde4-devel
+BuildRequires:  kio-devel
+BuildRequires:  libxapian-devel
+BuildRequires:  recoll = %{version}
+Requires:       recoll = %{version}
+BuildRoot:      %{_tmppath}/%{name}-%{version}-build
+
+%description
+Recoll is a personal full text search tool for Unix/Linux.
+
+This package provides the kio-slave
+
+
+%prep
+%setup -q -n %{rname}-%{version}
+
+%build
+pushd kde/kioslave/kio_recoll
+%cmake_kde4 -d build
+%make_jobs
+popd
+pushd kde/kioslave/kio_recoll-kde4
+%cmake_kde4 -d build
+%make_jobs
+
+%install
+pushd kde/kioslave/kio_recoll
+%kde4_makeinstall -C build
+popd
+pushd kde/kioslave/kio_recoll-kde4
+%kde4_makeinstall -C build
+
+
+%files
+%defattr(-,root,root)
+%{_libdir}/qt5/plugins/kio_recoll.so
+%{_datadir}/kio_recoll/
+%{_datadir}/kservices5/*.protocol
+%{_libdir}/kde4/kio_recoll.so
+%{_datadir}/kde4/apps/kio_recoll/help.html
+%{_datadir}/kde4/apps/kio_recoll/welcome.html
+%{_datadir}/kde4/services/recoll.protocol
+%{_datadir}/kde4/services/recollf.protocol
+
+%changelog
+* Tue Apr 05 2016 Jean-Francois Dockes <jfd@recoll.org> 1.21.6-0
+- Also build kde5 versions: works with Dolphin. Keep kde4 version for
+  Konqueror
+* Sun Mar 18 2012 Jean-Francois Dockes <jfd@recoll.org> 1.17.0-0
+- 1.17.0
+* Mon May 02 2011 Jean-Francois Dockes <jfd@recoll.org> 1.16.2-0
+- 1.16.2
+* Mon May 02 2011 Jean-Francois Dockes <jfd@recoll.org> 1.15.8-0
+- 1.15.8
+* Sun Mar 06 2011 Jean-Francois Dockes <jfd@recoll.org> 1.15.5-0
+- Initial spec file for kio
+

packaging/rpm/recoll-opensuse.spec

@@ -0,0 +1,123 @@
+#
+# spec file for package recoll
+#
+# Copyright (c) 2013 SUSE LINUX Products GmbH, Nuernberg, Germany.
+#
+# All modifications and additions to the file contributed by third parties
+# remain the property of their copyright owners, unless otherwise agreed
+# upon. The license for this file, and modifications and additions to the
+# file, is the same license as for the pristine package itself (unless the
+# license for the pristine package is not an Open Source License, in which
+# case the license is the MIT License). An "Open Source License" is a
+# license that conforms to the Open Source Definition (Version 1.9)
+# published by the Open Source Initiative.
+
+# Please submit bugfixes or comments via http://bugs.opensuse.org/
+#
+
+
+Name:           recoll
+Version:        1.21.6
+Release:        0
+Summary:        Extended Search
+License:        GPL-2.0+
+Summary:        Extended Search
+Group:          Productivity/Text/Utilities
+Url:            http://www.lesbonscomptes.com/recoll/
+Source:         http://www.lesbonscomptes.com/recoll/%{name}-%{version}.tar.gz
+BuildRequires:  aspell-devel
+%if 0%{?suse_version} > 1320
+BuildRequires:  pkgconfig(Qt5Core)
+BuildRequires:  pkgconfig(Qt5WebKit)
+BuildRequires:  pkgconfig(Qt5WebKitWidgets)
+BuildRequires:  pkgconfig(Qt5Xml)
+BuildRequires:  pkgconfig(Qt5Network)
+BuildRequires:  pkgconfig(Qt5PrintSupport)
+%else
+BuildRequires:  libQtWebKit-devel
+BuildRequires:  libqt4-devel
+%endif
+BuildRequires:  bison
+BuildRequires:  libxapian-devel
+BuildRequires:  python-devel
+BuildRequires:  update-desktop-files
+# Recommends for the helpers.
+Recommends:     antiword
+Recommends:     poppler-tools
+Recommends:     libwpd-tools
+Recommends:     libxslt
+Recommends:     catdoc
+Recommends:     djvulibre
+Recommends:     python-mutagen
+Recommends:     exiftool
+Recommends:     perl-Image-ExifTool
+Recommends:     python-base
+Recommends:     unrar
+Recommends:     python-rarfile
+Recommends:     python-PyCHM
+Recommends:     unrtf
+Recommends:     pstotext
+# Default requires for recoll itself
+Requires:       sed
+Requires:       awk
+Requires:       aspell
+Requires:       python
+Suggests:       kio_recoll
+BuildRoot:      %{_tmppath}/%{name}-%{version}-build
+
+%description
+Recoll is a personal full text search tool for Unix/Linux.
+
+It is based on the very strong Xapian back-end, for which it provides a feature-rich yet easy to use front-end with a QT graphical interface.
+
+Recoll is free, open source, and licensed under the GPL. The current version is 1.12.0 .
+Features:
+
+* Easy installation, few dependencies. No database daemon, web server, desktop environment or exotic language necessary.
+* Will run on most unix-based systems
+* Qt-based GUI. Can use either Qt 3 or Qt 4.
+* Searches most common document types, emails and their attachments.
+* Powerful query facilities, with boolean searches, phrases, proximity, wildcards, filter on file types and directory tree.
+* Multi-language and multi-character set with Unicode based internals.
+* (more detail)
+
+%prep
+%setup -q
+
+%build
+%if 0%{?suse_version} > 1320
+export QMAKE=qmake-qt5
+%endif
+%configure --with-inotify
+make %{?_smp_mflags}
+
+%install
+make DESTDIR=%{buildroot} install %{?_smp_mflags}
+
+%post -p /sbin/ldconfig
+
+%postun -p /sbin/ldconfig
+
+%files
+%defattr(-,root,root)
+%dir %{_libdir}/recoll
+%dir %{_datadir}/appdata
+%{_bindir}/recoll
+%{_bindir}/recollindex
+%{_datadir}/recoll
+%{_datadir}/applications/recoll-searchgui.desktop
+%{_datadir}/appdata/recoll.appdata.xml
+%{_datadir}/icons/hicolor
+%{_datadir}/pixmaps/recoll.png
+%{_libdir}/recoll/librecoll.so.%{version}
+%{_libdir}/python2.7/site-packages/recoll
+%{_libdir}/python2.7/site-packages/Recoll-1.0-py2.7.egg-info
+%{_mandir}/man1/recoll.1.gz
+%{_mandir}/man1/recollq.1.gz
+%{_mandir}/man1/recollindex.1.gz
+%{_mandir}/man5/recoll.conf.5.gz
+
+%changelog
+* Tue Apr 05 2016 Jean-Francois Dockes <jfd@recoll.org> 1.21.6-0
+- Recoll 1.21.6, esp. for the kde5 kio
+

packaging/rpm/recollfedora.spec

@@ -1,18 +1,24 @@
 Summary:        Desktop full text search tool with Qt GUI
 Name:           recoll
-Version:        1.20.6
+Version:        1.21.6
 Release:        1%{?dist}
 Group:          Applications/Databases
 License:        GPLv2+
 URL:            http://www.lesbonscomptes.com/recoll/
 Source0:        http://www.lesbonscomptes.com/recoll/recoll-%{version}.tar.gz
-BuildRequires:  qt-devel
-BuildRequires:  qtwebkit-devel
-BuildRequires:  python-devel
-BuildRequires:  zlib-devel
+# Source10:       qmake-qt4.sh
 BuildRequires:  aspell-devel
-BuildRequires:  xapian-core-devel
+BuildRequires:  bison
 BuildRequires:  desktop-file-utils
+# kio
+BuildRequires:  kdelibs4-devel
+BuildRequires:  qt5-qtbase-devel
+BuildRequires:  qt5-qtwebkit-devel
+BuildRequires:  extra-cmake-modules
+BuildRequires:  kf5-kio-devel
+BuildRequires:  python2-devel
+BuildRequires:  xapian-core-devel
+BuildRequires:  zlib-devel
 Requires:       xdg-utils
 
 %description
@@ -20,15 +26,30 @@ Recoll is a personal full text search package for Linux, FreeBSD and
 other Unix systems. It is based on a very strong back end (Xapian), for
 which it provides an easy to use, feature-rich, easy administration
 interface.
-%global __provides_exclude_from ^%{_libdir}/recoll/librecoll\\.so.*$
-%global __requires_exclude      ^librecoll\\.so.*$
+
+%package       kio
+Summary:       KIO support for recoll
+Group:         Applications/Databases
+Requires:      %{name} = %{version}-%{release}
+
+%description   kio
+The recoll KIO slave allows performing a recoll search by entering an
+appropriate URL in a KDE open dialog, or with an HTML-based interface
+displayed in Konqueror.
 
 %prep
 %setup -q -n %{name}-%{version}
 chmod 0644 utils/{conftree.cpp,conftree.h,debuglog.cpp,debuglog.h}
 
 %build
-export QMAKE=qmake-qt4
+CFLAGS="%{optflags}"; export CFLAGS
+CXXFLAGS="%{optflags}"; export CXXFLAGS
+LDFLAGS="%{?__global_ldflags}"; export LDFLAGS
+
+# force use of custom/local qmake, to inject proper build flags (above)
+#install -m755 -D %{SOURCE10} qmake-qt4.sh
+export QMAKE=qmake-qt5
+
 %configure
 make %{?_smp_mflags}
 
@@ -46,6 +67,28 @@ rm -f %{buildroot}/usr/share/recoll/filters/xdg-open
 chmod 0755 %{buildroot}/usr/share/recoll/filters/rclexecm.py
 chmod 0755 %{buildroot}%{_libdir}/recoll/librecoll.so.*
 
+# kio_recoll -kde5
+(
+mkdir kde/kioslave/kio_recoll/build && pushd kde/kioslave/kio_recoll/build
+%cmake ..
+make %{?_smp_mflags} VERBOSE=1
+make install DESTDIR=%{buildroot}
+popd
+)
+
+# kio_recoll -kde4
+(
+mkdir kde/kioslave/kio_recoll-kde4/build && \
+      pushd kde/kioslave/kio_recoll-kde4/build
+%cmake ..
+make %{?_smp_mflags} VERBOSE=1
+make install DESTDIR=%{buildroot}
+popd
+)
+
+mkdir -p %{buildroot}%{_sysconfdir}/ld.so.conf.d
+echo "%{_libdir}/recoll" > %{buildroot}%{_sysconfdir}/ld.so.conf.d/%{name}-%{_arch}.conf
+
 %post
 touch --no-create %{_datadir}/icons/hicolor
 if [ -x %{_bindir}/gtk-update-icon-cache ] ; then
@@ -54,6 +97,7 @@ fi
 if [ -x %{_bindir}/update-desktop-database ] ; then
   %{_bindir}/update-desktop-database &> /dev/null
 fi
+/sbin/ldconfig
 exit 0
 
 %postun
@@ -64,10 +108,12 @@ fi
 if [ -x %{_bindir}/update-desktop-database ] ; then
   %{_bindir}/update-desktop-database &> /dev/null
 fi
+/sbin/ldconfig
 exit 0
 
 %files
 %doc COPYING ChangeLog README
+%{_sysconfdir}/ld.so.conf.d/%{name}-%{_arch}.conf
 %{_bindir}/%{name}
 %{_bindir}/%{name}index
 %{_datadir}/%{name}
@@ -79,18 +125,52 @@ exit 0
 %{python_sitearch}/recoll
 %{python_sitearch}/Recoll*.egg-info
 %{_mandir}/man1/%{name}.1*
-%{_mandir}/man1/recollq.1*
+%{_mandir}/man1/%{name}q.1*
 %{_mandir}/man1/%{name}index.1*
 %{_mandir}/man5/%{name}.conf.5*
 
-%changelog
-* Sat Apr 25 2015 Jean-Francois Dockes <jf@dockes.org> - 1.20.6-1
-- 1.20.6
+%files kio
+%{_libdir}/kde4/kio_recoll.so
+%{_libdir}/qt5/plugins/kio_recoll.so
+%{_datadir}/kde4/apps/kio_recoll/
+%{_datadir}/kde4/services/recoll.protocol
+%{_datadir}/kde4/services/recollf.protocol
+%{_datadir}/kio_recoll/help.html
+%{_datadir}/kio_recoll/welcome.html
+%{_datadir}/kservices5/recoll.protocol
+%{_datadir}/kservices5/recollf.protocol
 
-* Fri Dec 19 2014 Jean-Francois Dockes <jf@dockes.org> - 1.20.1-1
+%changelog
+* Mon Jan 18 2016 Terje Rosten <terje.rosten@ntnu.no> - 1.21.4-1
+- 1.21.4
+
+* Sat Oct 31 2015 Terje Rosten <terje.rosten@ntnu.no> - 1.21.3-1
+- 1.21.3
+
+* Sat Oct 03 2015 Terje Rosten <terje.rosten@ntnu.no> - 1.21.2-1
+- 1.21.2
+
+* Thu Jun 18 2015 Fedora Release Engineering <rel-eng@lists.fedoraproject.org> - 1.20.6-2
+- Rebuilt for https://fedoraproject.org/wiki/Fedora_23_Mass_Rebuild
+
+* Tue May 05 2015 Terje Rosten <terje.rosten@ntnu.no> - 1.20.6-1
+- 1.20.6
+- Fixes bz#1218161 and bz#1204464
+
+* Sat May 02 2015 Kalev Lember <kalevlember@gmail.com> - 1.20.5-3
+- Rebuilt for GCC 5 C++11 ABI change
+
+* Sat Apr 11 2015 Terje Rosten <terje.rosten@ntnu.no> - 1.20.5-2
+- Add KIO subpackage
+
+* Tue Apr 07 2015 Terje Rosten <terje.rosten@ntnu.no> - 1.20.5-1
+- 1.20.5
+- Include kio support (bz#1203257)
+
+* Sat Jan 10 2015 Terje Rosten <terje.rosten@ntnu.no> - 1.20.1-1
 - 1.20.1
 
-* Sun Nov 09 2014 Jean-Francois Dockes <jf@dockes.org> - 1.19.14p2-1
+* Sun Nov 09 2014 Terje Rosten <terje.rosten@ntnu.no> - 1.19.14p2-1
 - 1.19.14p2
 
 * Sun Aug 17 2014 Fedora Release Engineering <rel-eng@lists.fedoraproject.org> - 1.19.13-3
@@ -221,4 +301,3 @@ exit 0
 
 * Wed Feb  1 2006 Jean-Francois Dockes <jfd@recoll.org> 1.2.0-1
 - initial packaging
-

src/Makefile.am

@@ -0,0 +1,654 @@
+
+CXXFLAGS ?= @CXXFLAGS@
+LIBXAPIAN=@LIBXAPIAN@
+XAPIANCXXFLAGS=@XAPIANCXXFLAGS@
+LIBICONV=@LIBICONV@
+INCICONV=@INCICONV@
+LIBFAM = @LIBFAM@
+RCLLIBVERSION=@RCLLIBVERSION@
+X_CFLAGS=@X_CFLAGS@
+X_PRE_LIBS=@X_PRE_LIBS@
+X_LIBS=@X_LIBS@
+X_EXTRA_LIBS=@X_EXTRA_LIBS@
+X_LIBX11=@X_LIBX11@
+DEFS=@DEFS@
+
+COMMONCPPFLAGS = -I. \
+    -I$(top_srcdir)/aspell \
+    -I$(top_srcdir)/bincimapmime \
+    -I$(top_srcdir)/common \
+    -I$(top_srcdir)/index \
+    -I$(top_srcdir)/internfile \
+    -I$(top_srcdir)/rcldb \
+    -I$(top_srcdir)/unac \
+    -I$(top_srcdir)/utils \
+    -I$(top_srcdir)/xaposix \
+    -DBUILDING_RECOLL
+
+AM_CPPFLAGS = -Wall -Wno-unused -std=c++11 \
+    $(COMMONCPPFLAGS) \
+    $(INCICONV) \
+    $(XAPIANCXXFLAGS) \
+    $(X_CFLAGS) \
+    -DRECOLL_DATADIR=\"${pkgdatadir}\" \
+    -D_GNU_SOURCE \
+    $(DEFS)
+
+ACLOCAL_AMFLAGS = -I m4
+
+if NOTHREADS
+    LIBTHREADS=
+else
+    LIBTHREADS= $(LIBSYSTHREADS)
+endif
+
+librcldir = $(libdir)/recoll
+librcl_LTLIBRARIES = librecoll.la
+
+librecoll_la_SOURCES = \
+aspell/aspell-local.h \
+aspell/rclaspell.cpp \
+aspell/rclaspell.h \
+bincimapmime/config.h \
+bincimapmime/convert.cc \
+bincimapmime/convert.h \
+bincimapmime/mime-inputsource.h \
+bincimapmime/mime-parsefull.cc \
+bincimapmime/mime-parseonlyheader.cc \
+bincimapmime/mime-printbody.cc \
+bincimapmime/mime-utils.h \
+bincimapmime/mime.cc \
+bincimapmime/mime.h \
+common/beaglequeuecache.cpp \
+common/beaglequeuecache.h \
+common/conf_post.h \
+common/cstr.cpp \
+common/cstr.h \
+common/rclconfig.cpp \
+common/rclconfig.h \
+common/rclinit.cpp \
+common/rclinit.h \
+common/syngroups.cpp \
+common/syngroups.h \
+common/textsplit.cpp \
+common/textsplit.h \
+common/unacpp.cpp \
+common/unacpp.h \
+common/uproplist.h \
+common/utf8fn.cpp \
+common/utf8fn.h \
+index/beaglequeue.cpp \
+index/beaglequeue.h \
+index/bglfetcher.cpp \
+index/bglfetcher.h \
+index/checkretryfailed.cpp \
+index/checkretryfailed.h \
+index/exefetcher.cpp \
+index/exefetcher.h \
+index/fetcher.cpp \
+index/fetcher.h \
+index/fsfetcher.cpp \
+index/fsfetcher.h \
+index/fsindexer.cpp \
+index/fsindexer.h \
+index/indexer.cpp \
+index/indexer.h \
+index/mimetype.cpp \
+index/mimetype.h \
+index/rclmon.h \
+index/recollindex.h \
+index/subtreelist.cpp \
+index/subtreelist.h \
+internfile/Filter.h \
+internfile/extrameta.cpp \
+internfile/extrameta.h \
+internfile/htmlparse.cpp \
+internfile/htmlparse.h \
+internfile/indextext.h \
+internfile/internfile.cpp \
+internfile/internfile.h \
+internfile/mh_exec.cpp \
+internfile/mh_exec.h \
+internfile/mh_execm.cpp \
+internfile/mh_execm.h \
+internfile/mh_html.cpp \
+internfile/mh_html.h \
+internfile/mh_mail.cpp \
+internfile/mh_mail.h \
+internfile/mh_mbox.cpp \
+internfile/mh_mbox.h \
+internfile/mh_null.h \
+internfile/mh_symlink.h \
+internfile/mh_text.cpp \
+internfile/mh_text.h \
+internfile/mh_unknown.h \
+internfile/mimehandler.cpp \
+internfile/mimehandler.h \
+internfile/myhtmlparse.cpp \
+internfile/myhtmlparse.h \
+internfile/txtdcode.cpp \
+internfile/uncomp.cpp \
+internfile/uncomp.h \
+query/docseq.cpp \
+query/docseq.h \
+query/docseqdb.cpp \
+query/docseqdb.h \
+query/docseqdocs.h \
+query/docseqhist.cpp \
+query/docseqhist.h \
+query/dynconf.cpp \
+query/dynconf.h \
+query/filtseq.cpp \
+query/filtseq.h \
+query/plaintorich.cpp \
+query/plaintorich.h \
+query/recollq.cpp \
+query/recollq.h \
+query/reslistpager.cpp \
+query/reslistpager.h \
+query/sortseq.cpp \
+query/sortseq.h \
+query/wasaparse.ypp \
+query/wasaparseaux.cpp \
+query/wasaparserdriver.h \
+query/wasatorcl.h \
+rcldb/daterange.cpp \
+rcldb/daterange.h \
+rcldb/expansiondbs.cpp \
+rcldb/expansiondbs.h \
+rcldb/rclabstract.cpp \
+rcldb/rcldb.cpp \
+rcldb/rcldb.h \
+rcldb/rcldb_p.h \
+rcldb/rcldoc.cpp \
+rcldb/rcldoc.h \
+rcldb/rcldups.cpp \
+rcldb/rclquery.cpp \
+rcldb/rclquery.h \
+rcldb/rclquery_p.h \
+rcldb/rclterms.cpp \
+rcldb/searchdata.cpp \
+rcldb/searchdata.h \
+rcldb/searchdatatox.cpp \
+rcldb/searchdataxml.cpp \
+rcldb/stemdb.cpp \
+rcldb/stemdb.h \
+rcldb/stoplist.cpp \
+rcldb/stoplist.h \
+rcldb/synfamily.cpp \
+rcldb/synfamily.h \
+rcldb/termproc.h \
+rcldb/xmacros.h \
+unac/unac.cpp \
+unac/unac.h \
+unac/unac_version.h \
+utils/appformime.cpp \
+utils/appformime.h \
+utils/base64.cpp \
+utils/base64.h \
+utils/cancelcheck.cpp \
+utils/cancelcheck.h \
+utils/chrono.h \
+utils/chrono.cpp \
+utils/circache.cpp \
+utils/circache.h \
+utils/closefrom.cpp \
+utils/closefrom.h \
+utils/conftree.cpp \
+utils/conftree.h \
+utils/copyfile.cpp \
+utils/copyfile.h \
+utils/cpuconf.cpp \
+utils/cpuconf.h \
+utils/ecrontab.cpp \
+utils/ecrontab.h \
+utils/execmd.cpp \
+utils/execmd.h \
+utils/fileudi.cpp \
+utils/fileudi.h \
+utils/fstreewalk.cpp \
+utils/fstreewalk.h \
+utils/hldata.h \
+utils/hldata.cpp \
+utils/idfile.cpp \
+utils/idfile.h \
+utils/listmem.cpp \
+utils/listmem.h \
+utils/log.cpp \
+utils/log.h \
+utils/md5.cpp \
+utils/md5.h \
+utils/md5ut.cpp \
+utils/md5ut.h \
+utils/mimeparse.cpp \
+utils/mimeparse.h \
+utils/netcon.cpp \
+utils/netcon.h \
+utils/pathut.cpp \
+utils/pathut.h \
+utils/pxattr.cpp \
+utils/pxattr.h \
+utils/rclionice.cpp \
+utils/rclionice.h \
+utils/rclutil.h \
+utils/rclutil.cpp \
+utils/readfile.cpp \
+utils/readfile.h \
+utils/refcntr.h \
+utils/smallut.cpp \
+utils/smallut.h \
+utils/strmatcher.cpp \
+utils/strmatcher.h \
+utils/transcode.cpp \
+utils/transcode.h \
+utils/utf8iter.h \
+utils/wipedir.cpp \
+utils/wipedir.h \
+utils/workqueue.h \
+xaposix/safefcntl.h \
+xaposix/safesysstat.h \
+xaposix/safesyswait.h \
+xaposix/safeunistd.h
+
+BUILT_SOURCES = query/wasaparse.cpp
+AM_YFLAGS = -d
+
+# We use -release: the lib is only shared
+# between recoll programs from the same release.
+# -version-info $(VERSION_INFO)
+librecoll_la_LDFLAGS = -release $(VERSION) \
+    -Wl,--no-undefined -Wl,--warn-unresolved-symbols
+
+librecoll_la_LIBADD = $(LIBXAPIAN) $(LIBICONV) $(LIBTHREADS)
+
+# There is probably a better way to do this. The KIO needs to be linked
+# with librecoll, but librecoll is installed into a non-standard place
+# (/usr/lib/recoll). Debian packaging has something against setting an
+# rpath on the kio (cause it's not the same package as the lib), so I don't
+# know how to link it dynamically. The other thing I don't know is how to
+# force automake to build a static lib with the PIC objects. So the
+# following target, which is only used from the KIO build, deletes any .a
+# and .so and rebuilds the .a with the pic objs (the kio build calls
+# configured --disable-static).
+# Of course this is very uncomfortably close to automake/libtool internals
+# and may not work on all systems.
+PicStatic: $(librecoll_la_OBJECTS)
+	rm -f .libs/librecoll.a
+	rm -f .libs/librecoll.so
+	libtool --tag=LD --mode=link gcc -g -O -o librecoll.la \
+		$(librecoll_la_OBJECTS)
+
+bin_PROGRAMS = recollindex
+if MAKECMDLINE
+    bin_PROGRAMS += recollq
+endif
+
+if MAKEXADUMP
+   bin_PROGRAMS += xadump
+endif
+
+recollindex_SOURCES = \
+                    index/recollindex.cpp \
+                    index/rclmonprc.cpp \
+                    index/rclmonrcv.cpp \
+                    utils/x11mon.cpp \
+                    utils/x11mon.h 
+recollindex_LDADD = librecoll.la $(X_LIBX11)
+
+recollq_SOURCES = query/recollqmain.cpp
+recollq_LDADD = librecoll.la
+
+xadump_SOURCES = query/xadump.cpp
+xadump_LDADD = librecoll.la $(LIBXAPIAN) $(LIBICONV)
+
+# Note: I'd prefer the generated query parser files not to be distributed
+# at all, but failed to achieve this
+EXTRA_DIST = \
+bincimapmime/00README.recoll bincimapmime/AUTHORS bincimapmime/COPYING \
+\
+desktop/hotrecoll.py \
+desktop/recoll.appdata.xml \
+desktop/recollindex.desktop \
+desktop/recoll_index_on_ac.sh \
+desktop/recoll-searchgui.desktop \
+desktop/recoll.png desktop/recoll.svg desktop/recoll.xcf \
+\
+doc/man \
+doc/prog \
+doc/user/*.html doc/user/*.css doc/user/*.txt doc/user/*.xml doc/user/Makefile \
+\
+filters \
+\
+index/rclmon.sh \
+\
+kde/kioslave/kio_recoll/00README.txt \
+kde/kioslave/kio_recoll/CMakeLists.txt \
+kde/kioslave/kio_recoll/data/help.html \
+kde/kioslave/kio_recoll/data/searchable.html \
+kde/kioslave/kio_recoll/data/welcome.html \
+kde/kioslave/kio_recoll/dirif.cpp \
+kde/kioslave/kio_recoll/htmlif.cpp \
+kde/kioslave/kio_recoll/kio_recoll.cpp \
+kde/kioslave/kio_recoll/kio_recoll.h \
+kde/kioslave/kio_recoll/recollf.protocol \
+kde/kioslave/kio_recoll/recollnolist.protocol \
+kde/kioslave/kio_recoll/recoll.protocol \
+\
+query/location.hh  query/position.hh  query/stack.hh \
+\
+qtgui/advsearch.ui \
+qtgui/advsearch_w.cpp \
+qtgui/advsearch_w.h \
+qtgui/advshist.cpp \
+qtgui/advshist.h \
+qtgui/confgui/confgui.cpp \
+qtgui/confgui/confgui.h \
+qtgui/confgui/confguiindex.cpp \
+qtgui/confgui/confguiindex.h \
+qtgui/confgui/conflinkrcl.h \
+qtgui/confgui/main.cpp \
+qtgui/crontool.cpp \
+qtgui/crontool.h \
+qtgui/crontool.ui \
+qtgui/firstidx.h \
+qtgui/firstidx.ui \
+qtgui/fragbuts.cpp \
+qtgui/fragbuts.h \
+qtgui/guiutils.cpp \
+qtgui/guiutils.h \
+qtgui/i18n/recoll_cs.qm qtgui/i18n/recoll_cs.ts \
+qtgui/i18n/recoll_da.qm qtgui/i18n/recoll_da.ts \
+qtgui/i18n/recoll_de.qm qtgui/i18n/recoll_de.ts \
+qtgui/i18n/recoll_el.qm qtgui/i18n/recoll_el.ts \
+qtgui/i18n/recoll_es.qm qtgui/i18n/recoll_es.ts \
+qtgui/i18n/recoll_fr.qm qtgui/i18n/recoll_fr.ts \
+qtgui/i18n/recoll_it.qm qtgui/i18n/recoll_it.ts \
+qtgui/i18n/recoll_lt.qm qtgui/i18n/recoll_lt.ts \
+qtgui/i18n/recoll_pl.qm qtgui/i18n/recoll_pl.ts \
+qtgui/i18n/recoll_ru.qm qtgui/i18n/recoll_ru.ts \
+qtgui/i18n/recoll_tr.qm qtgui/i18n/recoll_tr.ts \
+qtgui/i18n/recoll_uk.qm qtgui/i18n/recoll_uk.ts \
+qtgui/i18n/recoll_xx.qm qtgui/i18n/recoll_xx.ts \
+qtgui/i18n/recoll_zh.qm qtgui/i18n/recoll_zh.ts \
+qtgui/i18n/recoll_zh_CN.qm qtgui/i18n/recoll_zh_CN.ts \
+qtgui/idxsched.h \
+qtgui/idxsched.ui \
+qtgui/images/asearch.png \
+qtgui/images/cancel.png \
+qtgui/images/close.png \
+qtgui/images/code-block.png \
+qtgui/images/down.png \
+qtgui/images/firstpage.png \
+qtgui/images/history.png \
+qtgui/images/nextpage.png \
+qtgui/images/prevpage.png \
+qtgui/images/recoll.icns \
+qtgui/images/recoll.png \
+qtgui/images/sortparms.png \
+qtgui/images/spell.png \
+qtgui/images/table.png \
+qtgui/images/up.png \
+qtgui/main.cpp \
+qtgui/mtpics/License_sidux.txt \
+qtgui/mtpics/README \
+qtgui/mtpics/aptosid-book.png \
+qtgui/mtpics/aptosid-manual-copyright.txt \
+qtgui/mtpics/aptosid-manual.png \
+qtgui/mtpics/archive.png \
+qtgui/mtpics/book.png \
+qtgui/mtpics/bookchap.png \
+qtgui/mtpics/document.png \
+qtgui/mtpics/drawing.png \
+qtgui/mtpics/emblem-symbolic-link.png \
+qtgui/mtpics/folder.png \
+qtgui/mtpics/html.png \
+qtgui/mtpics/image.png \
+qtgui/mtpics/message.png \
+qtgui/mtpics/mozilla_doc.png \
+qtgui/mtpics/pdf.png \
+qtgui/mtpics/pidgin.png \
+qtgui/mtpics/postscript.png \
+qtgui/mtpics/presentation.png \
+qtgui/mtpics/sidux-book.png \
+qtgui/mtpics/soffice.png \
+qtgui/mtpics/source.png \
+qtgui/mtpics/sownd.png \
+qtgui/mtpics/spreadsheet.png \
+qtgui/mtpics/text-x-python.png \
+qtgui/mtpics/txt.png \
+qtgui/mtpics/video.png \
+qtgui/mtpics/wordprocessing.png \
+qtgui/multisave.cpp \
+qtgui/multisave.h \
+qtgui/preview_load.cpp \
+qtgui/preview_load.h \
+qtgui/preview_plaintorich.cpp \
+qtgui/preview_plaintorich.h \
+qtgui/preview_w.cpp \
+qtgui/preview_w.h \
+qtgui/ptrans.ui \
+qtgui/ptrans_w.cpp \
+qtgui/ptrans_w.h \
+qtgui/rclhelp.cpp \
+qtgui/rclhelp.h \
+qtgui/rclm_idx.cpp \
+qtgui/rclm_preview.cpp \
+qtgui/rclm_saveload.cpp \
+qtgui/rclm_view.cpp \
+qtgui/rclm_wins.cpp \
+qtgui/rclmain.ui \
+qtgui/rclmain_w.cpp \
+qtgui/rclmain_w.h \
+qtgui/rclzg.cpp \
+qtgui/rclzg.h \
+qtgui/recoll.h \
+qtgui/recoll.pro.in \
+qtgui/recoll.qrc \
+qtgui/reslist.cpp \
+qtgui/reslist.h \
+qtgui/respopup.cpp \
+qtgui/respopup.h \
+qtgui/restable.cpp \
+qtgui/restable.h \
+qtgui/restable.ui \
+qtgui/rtitool.cpp \
+qtgui/rtitool.h \
+qtgui/rtitool.ui \
+qtgui/searchclause_w.cpp \
+qtgui/searchclause_w.h \
+qtgui/snippets.ui \
+qtgui/snippets_w.cpp \
+qtgui/snippets_w.h \
+qtgui/specialindex.h \
+qtgui/specialindex.ui \
+qtgui/spell.ui \
+qtgui/spell_w.cpp \
+qtgui/spell_w.h \
+qtgui/ssearch_w.cpp \
+qtgui/ssearch_w.h \
+qtgui/ssearchb.ui \
+qtgui/systray.cpp \
+qtgui/systray.h \
+qtgui/ui_rclmain.h-4.5 \
+qtgui/uiprefs.ui \
+qtgui/uiprefs_w.cpp \
+qtgui/uiprefs_w.h \
+qtgui/viewaction.ui \
+qtgui/viewaction_w.cpp \
+qtgui/viewaction_w.h \
+qtgui/webcache.ui \
+qtgui/webcache.cpp \
+qtgui/webcache.h \
+qtgui/widgets/editdialog.h \
+qtgui/widgets/editdialog.ui \
+qtgui/widgets/listdialog.h \
+qtgui/widgets/listdialog.ui \
+qtgui/widgets/qxtconfirmationmessage.cpp \
+qtgui/widgets/qxtconfirmationmessage.h \
+qtgui/widgets/qxtglobal.h \
+qtgui/xmltosd.cpp \
+qtgui/xmltosd.h \
+\
+python/README.txt \
+python/recoll/Makefile.in \
+python/recoll/pyrclextract.cpp \
+python/recoll/pyrecoll.cpp \
+python/recoll/pyrecoll.h \
+python/recoll/recoll/__init__.py \
+python/recoll/recoll/rclconfig.py \
+python/recoll/setup.py.in \
+python/samples/docdups.py \
+python/samples/mutt-recoll.py \
+python/samples/rcldlkp.py \
+python/samples/rclmbox.py \
+python/samples/recollgui/Makefile \
+python/samples/recollgui/qrecoll.py \
+python/samples/recollgui/rclmain.ui \
+python/samples/recollq.py \
+python/samples/recollqsd.py \
+\
+ \
+sampleconf/fields sampleconf/fragbuts.xml sampleconf/mimeconf \
+sampleconf/mimemap sampleconf/mimeview sampleconf/mimeview.mac \
+sampleconf/recoll.conf sampleconf/recoll.qss \
+\
+unac/AUTHORS unac/COPYING unac/README unac/README.recoll unac/unac.c \
+\
+VERSION
+
+# EXTRA_DIST: The Php Code does not build anymore. No need to ship it until
+# someone fixes it:
+# php/00README.txt php/recoll/config.m4 php/recoll/make.sh 
+# php/recoll/php_recoll.h php/recoll/recoll.cpp php/sample/shell.php 
+
+if MAKEPYTHON
+all-local: recollpython
+recollpython: librecoll.la
+	${MAKE} -C python/recoll libdir=$(libdir)
+install-exec-local: recollpython-install
+recollpython-install:
+	(cd python/recoll; \
+        if test -f /etc/debian_version ; then \
+            OPTSFORPYTHON=--install-layout=deb; \
+        fi; \
+        set -x; \
+        python setup.py install \
+         --prefix=${prefix} --root=$${DESTDIR:-/} $${OPTSFORPYTHON})
+clean-local: recollpython-clean
+recollpython-clean:
+	rm -rf python/recoll/build
+endif
+
+if MAKEQT
+all-local: recollqt
+recollqt: librecoll.la
+	(cd $(QTGUI); ${QMAKE} PREFIX=${prefix} recoll.pro)
+	$(MAKE) -C $(QTGUI) LFLAGS="$(LDFLAGS)" prefix=$(prefix) \
+                exec_prefix=$(exec_prefix) libdir=$(libdir)
+clean-local: recollqt-clean
+recollqt-clean:
+	-$(MAKE) -C $(QTGUI) clean
+install-exec-local: recollqt-install
+recollqt-install:
+	$(MAKE) -C $(QTGUI) LFLAGS="$(LDFLAGS)" INSTALL_ROOT=$(DESTDIR) \
+                prefix=$(prefix) exec_prefix=$(exec_prefix) libdir=$(libdir) \
+                install
+endif
+
+defconfdir = $(pkgdatadir)/examples
+defconf_DATA = \
+desktop/recollindex.desktop \
+index/rclmon.sh \
+sampleconf/fragbuts.xml \
+sampleconf/fields \
+sampleconf/recoll.conf \
+sampleconf/mimeconf \
+sampleconf/recoll.qss \
+sampleconf/mimemap \
+sampleconf/mimeview 
+
+filterdir = $(pkgdatadir)/filters
+filter_DATA = \
+desktop/hotrecoll.py \
+filters/rcl7z \
+filters/rclabw \
+filters/rclaptosidman \
+filters/rclaudio \
+filters/rclcheckneedretry.sh \
+filters/rclchm \
+filters/rcldia \
+filters/rcldjvu.py \
+filters/rcldoc.py \
+filters/rcldvi \
+filters/rclepub \
+filters/rclepub1 \
+filters/rclexec1.py \
+filters/rclexecm.py \
+filters/rclfb2 \
+filters/rclgaim \
+filters/rclgnm \
+filters/rclics \
+filters/rclimg \
+filters/rclimg.py \
+filters/rclinfo \
+filters/rclkar \
+filters/rclkwd \
+filters/rcllatinclass.py \
+filters/rcllatinstops.zip \
+filters/rcllyx \
+filters/rclman \
+filters/rclpdf.py \
+filters/rclokulnote \
+filters/rclopxml.py \
+filters/rclppt.py \
+filters/rclps \
+filters/rclpurple \
+filters/rclpython \
+filters/rclrar \
+filters/rclrtf.py \
+filters/rclscribus \
+filters/rclshowinfo \
+filters/rclsiduxman \
+filters/rclsoff.py \
+filters/rclsvg.py \
+filters/rcltar \
+filters/rcltex \
+filters/rcltext.py \
+filters/rcluncomp \
+filters/rcluncomp.py \
+filters/rclwar \
+filters/rclwpd \
+filters/rclxls.py \
+filters/rclxml.py \
+filters/rclxmp.py \
+filters/rclxslt.py \
+filters/rclzip \
+filters/ppt-dump.py \
+filters/xls-dump.py \
+filters/xlsxmltocsv.py \
+filters/msodump.zip \
+python/recoll/recoll/rclconfig.py 
+
+install-data-hook: 
+	(cd $(DESTDIR)/$(filterdir); \
+	chmod a+x rcl* ppt-dump.py xls-dump.py xlsxmltocsv.py hotrecoll.py; \
+	chmod 0644 msodump.zip rclexecm.py rcllatinstops.zip rclconfig.py) 
+
+if MAKEUSERDOC
+rdocdir = $(pkgdatadir)/doc
+rdoc_DATA = doc/user/usermanual.html doc/user/docbook-xsl.css 
+doc/user/usermanual.html: doc/user/usermanual.xml
+	mkdir -p doc/user
+	test -f doc/user/Makefile  || \
+		cp -p $(top_srcdir)/doc/user/Makefile doc/user
+	$(MAKE) -C doc/user VPATH=$(VPATH):$(VPATH)/doc/user usermanual.html
+endif
+
+dist_man1_MANS = doc/man/recoll.1 doc/man/recollq.1 doc/man/recollindex.1
+dist_man5_MANS = doc/man/recoll.conf.5
+
+dist-hook:
+	modified=`hg status | grep -v /$(distdir)/|grep -v debianunitysco`; \
+	if test ! -z "$$modified"; then \
+            echo Local directory is modified: $$modified ; exit 1; fi
+	hg tag -f -m "Release $(VERSION) tagged" RECOLL-$(VERSION)

src/Makefile.in

@@ -1,92 +0,0 @@
-# Copyright (C) 2005 J.F.Dockes
-
-prefix = @prefix@
-exec_prefix = @exec_prefix@
-
-bindir = @bindir@
-libdir = @libdir@
-datadir = @datadir@
-datarootdir = @datarootdir@
-mandir = @mandir@
-
-QMAKE = @QMAKE@
-QTGUI = @QTGUI@
-
-RCLLIBVERSION=@RCLLIBVERSION@
-
-all: configure mk/sysconf 
-	${MAKE} -C query wasaparse.tab.cpp
-	(cd lib; sh mkMake)
-	${MAKE} -C lib
-	${MAKE} -C index depend recollindex
-	@NOQTMAKE@(cd $(QTGUI); ${QMAKE} recoll.pro)
-	@NOQTMAKE@${MAKE} -C $(QTGUI) LFLAGS="$(LDFLAGS)" depth=.. prefix=$(prefix) exec_prefix=$(exec_prefix) libdir=$(libdir)
-	@NOPYTHON@${MAKE} -C python/recoll libdir=$(libdir)
-	${MAKE} -C query recollq xadump
-
-mk/sysconf:
-	@echo "You need to run configure first" ; exit 1
-
-static: mk/sysconf
-	${MAKE} -C lib
-	rm -f index/recollindex
-	${MAKE} -C index BSTATIC=-Wl,-Bstatic BDYNAMIC=-Wl,-Bdynamic \
-            LIBXAPIANSTATICEXTRA="@LIBXAPIANSTATICEXTRA@" \
-	       recollindex 
-	@NOQTMAKE@(cd $(QTGUI); $(QMAKE) recoll.pro)
-	@NOQTMAKE@rm -f $(QTGUI)/recoll
-	@NOQTMAKE@${MAKE} -C $(QTGUI) BSTATIC=-Wl,-Bstatic \
-	@NOQTMAKE@   BDYNAMIC=-Wl,-Bdynamic depth=.. \
-	@NOQTMAKE@   LIBXAPIANSTATICEXTRA="@LIBXAPIANSTATICEXTRA@"
-
-clean:
-	${MAKE} -C common clean
-	${MAKE} -C index clean
-	${MAKE} -C internfile clean
-	${MAKE} -C query clean
-	${MAKE} -C utils clean
-	-${MAKE} -C lib clean
-	-${MAKE} -C python/recoll clean
-	@NOQTMAKE@@-${MAKE} -C $(QTGUI) clean
-	rm -f qtgui/Makefile qtgui/confgui/Makefile qtgui/recoll
-	rm -f filters/rclexecm.pyc
-	rm -rf qtgui/.moc qtgui/.ui qtgui/confgui/.moc qtgui/confgui/.ui
-	rm -rf qtgui/.obj qtgui/.moc qtgui/.ui 
-	rm -rf python/recoll/build
-	rm -rf php/recoll/build php/recoll/include php/recoll/modules
-	rm -rf $(QTGUI)/recoll.app
-
-# Note: we don't remove the top Makefile, to keep the "clean" targets
-# available but a "Make" won't work without a configure anyway
-distclean: clean
-	${MAKE} -C query distclean
-	-${MAKE} -C python/recoll distclean
-	rm -f mk/sysconf mk/localdefs sampleconf/recoll.conf \
-	   qtgui/recoll.pro \
-	   config.log config.status \
-	   recollinstall \
-	   lib/*.dep lib/mkMake lib/Makefile common/autoconfig.h
-	rm -f common/rclversion.h
-	rm -f index/alldeps lib/alldeps query/alldeps \
-            bincimapmime/alldeps common/alldeps internfile/alldeps utils/alldeps
-	rm -rf autom4te.cache
-
-maintainer-clean: distclean
-	rm -f doc/user/*.html* doc/user/*.txt doc/user/HTML.manifest
-	rm -f qtgui/i18n/*.qm
-
-# recollinstall can be executed by the user and will compute 'normal'
-# values for bindir etc., relative to the prefix argument.  When executed
-# here, we pass a bunch of variables in the environment, the values will
-# override the computed defaults.
-install: all
-	DESTDIR=${DESTDIR} bindir=${bindir} datadir=${datadir} \
-          libdir=${libdir} mandir=${mandir} \
-	  /bin/sh ./recollinstall ${prefix}
-
-configure: VERSION configure.ac
-	rm -f configure
-	autoconf
-	@echo run configure and make again
-	exit 1
-.PHONY: all static clean distclean install

src/VERSION

@@ -1 +1 @@
-1.21.1
+1.23.0

src/aspell/Makefile

@@ -1,19 +1,12 @@
-depth = ..
-include $(depth)/mk/sysconf
-
 PROGS = rclaspell
-SRCS = rclaspell.cpp
-
-all: depend $(PROGS) librecoll
+all: $(PROGS)
 
 RCLASPELL_OBJS= trrclaspell.o 
 rclaspell : $(RCLASPELL_OBJS)
 	$(CXX) $(ALL_CXXFLAGS) -o rclaspell $(RCLASPELL_OBJS) \
-          $(LIBRECOLL)  $(LIBXAPIAN) $(LIBICONV)
+          $(LIBRECOLL)
 trrclaspell.o : rclaspell.cpp
 	$(CXX) $(ALL_CXXFLAGS) -DTEST_RCLASPELL -c -o trrclaspell.o \
 	       rclaspell.cpp
 
-include $(depth)/mk/commontargets
-
-include alldeps
+include ../utils/utmkdefs.mk

src/aspell/rclaspell.cpp

@@ -21,6 +21,8 @@
 
 #ifdef RCL_USE_ASPELL
 
+#include <mutex>
+
 #include <unistd.h>
 #include <dlfcn.h>
 #include <stdlib.h>
@@ -30,9 +32,8 @@
 #include "pathut.h"
 #include "execmd.h"
 #include "rclaspell.h"
-#include "debuglog.h"
+#include "log.h"
 #include "unacpp.h"
-#include "ptmutex.h"
 
 using namespace std;
 
@@ -63,7 +64,7 @@ public:
 
 };
 static AspellApi aapi;
-static PTMutexInit o_aapi_mutex;
+static std::mutex o_aapi_mutex;
 
 #define NMTOPTR(NM, TP)							\
     if ((aapi.NM = TP dlsym(m_data->m_handle, #NM)) == 0) {		\
@@ -84,7 +85,7 @@ public:
         : m_handle(0), m_speller(0) 
     {}
     ~AspellData() {
-        LOGDEB2(("~AspellData\n"));
+        LOGDEB2("~AspellData\n" );
 	if (m_handle) {
 	    dlclose(m_handle);
             m_handle = 0;
@@ -93,7 +94,7 @@ public:
             // Dumps core if I do this?? 
             //aapi.delete_aspell_speller(m_speller);
             m_speller = 0;
-            LOGDEB2(("~AspellData: speller done\n"));
+            LOGDEB2("~AspellData: speller done\n" );
         }
     }
 
@@ -114,7 +115,7 @@ Aspell::~Aspell()
 
 bool Aspell::init(string &reason)
 {
-    PTMutexLocker locker(o_aapi_mutex);
+    std::unique_lock<std::mutex> locker(o_aapi_mutex);
     deleteZ(m_data);
 
     // Language: we get this from the configuration, else from the NLS
@@ -220,8 +221,8 @@ bool Aspell::ok() const
 
 string Aspell::dicPath()
 {
-    return path_cat(m_config->getConfDir(), 
-		    string("aspdict.") + m_lang + string(".rws"));
+    string ccdir = m_config->getAspellcacheDir();
+    return path_cat(ccdir, string("aspdict.") + m_lang + string(".rws"));
 }
 
 
@@ -240,9 +241,9 @@ public:
     {}
     void newData() {
 	while (m_db.termWalkNext(m_tit, *m_input)) {
-	    LOGDEB2(("Aspell::buildDict: term: [%s]\n", m_input->c_str()));
+	    LOGDEB2("Aspell::buildDict: term: ["  << (m_input) << "]\n" );
 	    if (!Rcl::Db::isSpellingCandidate(*m_input)) {
-		LOGDEB2(("Aspell::buildDict: SKIP\n"));
+		LOGDEB2("Aspell::buildDict: SKIP\n" );
 		continue;
 	    }
 	    if (!o_index_stripchars) {
@@ -253,7 +254,7 @@ public:
 	    }
 	    // Got a non-empty sort-of appropriate term, let's send it to
 	    // aspell
-	    LOGDEB2(("Apell::buildDict: SEND\n"));
+	    LOGDEB2("Apell::buildDict: SEND\n" );
 	    m_input->append("\n");
 	    return;
 	}
@@ -268,6 +269,9 @@ bool Aspell::buildDict(Rcl::Db &db, string &reason)
     if (!ok())
 	return false;
 
+    string addCreateParam;
+    m_config->getConfParam("aspellAddCreateParam", addCreateParam);
+
     // We create the dictionary by executing the aspell command:
     // aspell --lang=[lang] create master [dictApath]
     string cmdstring(m_data->m_exec);
@@ -277,6 +281,10 @@ bool Aspell::buildDict(Rcl::Db &db, string &reason)
     cmdstring += string(" ") + string("--lang=") + m_lang;
     args.push_back("--encoding=utf-8");
     cmdstring += string(" ") + "--encoding=utf-8";
+    if (!addCreateParam.empty()) {
+	args.push_back(addCreateParam);
+	cmdstring += string(" ") + addCreateParam;
+    }
     args.push_back("create");
     cmdstring += string(" ") + "create";
     args.push_back("master");
@@ -363,7 +371,7 @@ bool Aspell::make_speller(string& reason)
 
 bool Aspell::check(const string &iterm, string& reason)
 {
-    LOGDEB2(("Aspell::check [%s]\n", iterm.c_str()));
+    LOGDEB2("Aspell::check ["  << (iterm) << "]\n" );
     string mterm(iterm);
 
     if (!ok() || !make_speller(reason))
@@ -374,7 +382,7 @@ bool Aspell::check(const string &iterm, string& reason)
     if (!o_index_stripchars) {
 	string lower;
 	if (!unacmaybefold(mterm, lower, "UTF-8", UNACOP_FOLD)) {
-	    LOGERR(("Aspell::check : cant lowercase input\n"));
+	    LOGERR("Aspell::check : cant lowercase input\n" );
 	    return false;
 	}
 	mterm.swap(lower);
@@ -406,7 +414,7 @@ bool Aspell::suggest(Rcl::Db &db, const string &_term,
     if (!o_index_stripchars) {
 	string lower;
 	if (!unacmaybefold(mterm, lower, "UTF-8", UNACOP_FOLD)) {
-	    LOGERR(("Aspell::check : cant lowercase input\n"));
+	    LOGERR("Aspell::check : cant lowercase input\n" );
 	    return false;
 	}
 	mterm.swap(lower);
@@ -577,3 +585,4 @@ int main(int argc, char **argv)
 #endif // RCL_USE_ASPELL
 
 #endif // TEST_RCLASPELL test driver
+

src/autogen.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+aclocal
+libtoolize --copy
+automake --add-missing --force-missing --copy
+autoconf

src/bincimapmime/Makefile

@@ -1,31 +1,11 @@
-# @(#$Id: Makefile,v 1.6 2006-01-19 12:01:42 dockes Exp $  (C) 2005 J.F.Dockes
-
-depth = ..
-include $(depth)/mk/sysconf
-
-LIBS = libmime.a
 PROGS = trbinc
-
-all: depend $(LIBS) 
-
-SRCS = mime-parsefull.cc mime-parseonlyheader.cc \
-     mime-printbody.cc mime.cc \
-     convert.cc iodevice.cc iofactory.cc
-
-OBJS = mime-parsefull.o mime-parseonlyheader.o \
-     mime-printbody.o mime.o  \
-     convert.o iodevice.o iofactory.o
-
-libmime.a : $(OBJS)
-	 $(AR) ru libmime.a $(OBJS)
-
-.cc.o:
-	$(CXX) $(ALL_CXXFLAGS) -c $<
+all: $(PROGS)
 
 TRBINCOBJS = trbinc.o
 trbinc: trbinc.o
-	$(CXX) -o trbinc trbinc.o libmime.a 
+	$(CXX) -o trbinc trbinc.o $(LIBRECOLL)
 
-include $(depth)/mk/commontargets
+trbinc.o: trbinc.cc
+	$(CXX) $(ALL_CXXFLAGS) -c -o trbinc.o trbinc.cc
 
-include alldeps
+include ../utils/utmkdefs.mk

src/bincimapmime/convert.cc

@@ -23,10 +23,6 @@
  *  Foundation, Inc., 59 Temple Street #330, Boston, MA 02111-1307, USA.
  *  --------------------------------------------------------------------
  */
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
 #include "convert.h"
 #include <string>
 
@@ -47,7 +43,7 @@ BincStream::~BincStream(void)
 }
 
 //------------------------------------------------------------------------
-string BincStream::popString(unsigned int size)
+string BincStream::popString(std::string::size_type size)
 {
   if (size > nstr.length())
     size = nstr.length();

src/bincimapmime/convert.h

@@ -23,19 +23,15 @@
  *  Foundation, Inc., 59 Temple Street #330, Boston, MA 02111-1307, USA.
  *  --------------------------------------------------------------------
  */
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
 #ifndef convert_h_included
 #define convert_h_included
+#include <stddef.h>
 #include <string>
 #include <vector>
 #include <iomanip>
 #include <iostream>
 
 #include <stdio.h>
-#include <sys/stat.h>
 #include <cstdlib>
 #include <cstring>
 
@@ -98,7 +94,7 @@ namespace Binc {
     std::string tmp;
     for (std::string::const_iterator i = s.begin();
 	 i != s.end() && i + 1 != s.end(); i += 2) {
-      int n;
+      ptrdiff_t n;
       unsigned char c = *i;
       unsigned char d = *(i + 1);
       
@@ -127,7 +123,7 @@ namespace Binc {
     for (std::string::const_iterator i = s_in.begin(); i != s_in.end(); ++i) {
       unsigned char c = (unsigned char)*i;
       if (c <= 31 || c >= 127 || c == '\"' || c == '\\')
-	return "{" + toString(s_in.length()) + "}\r\n" + s_in;
+	return "{" + toString((unsigned long)s_in.length()) + "}\r\n" + s_in;
     }
     
     return "\"" + s_in + "\"";
@@ -150,7 +146,7 @@ namespace Binc {
   //----------------------------------------------------------------------
   inline void chomp(std::string &s_in, const std::string &chars = " \t\r\n")
   {
-    int n = s_in.length();
+    std::string::size_type n = s_in.length();
     while (n > 1 && chars.find(s_in[n - 1]) != std::string::npos)
       s_in.resize(n-- - 1);
   }
@@ -295,7 +291,7 @@ namespace Binc {
     BincStream &operator << (char t);
 
     //--
-    std::string popString(unsigned int size);
+    std::string popString(std::string::size_type size);
 
     //--
     char popChar(void);

src/bincimapmime/iodevice.cc

@@ -1,322 +0,0 @@
-/*-*-mode:c++-*-*/
-/*  --------------------------------------------------------------------
- *  Filename:
- *    src/iodevice.cc
- *  
- *  Description:
- *    Implementation of the IODevice class.
- *  --------------------------------------------------------------------
- *  Copyright 2002, 2003 Andreas Aardal Hanssen
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Street #330, Boston, MA 02111-1307, USA.
- *  --------------------------------------------------------------------
- */
-#include "iodevice.h"
-#include "convert.h" // BincStream
-//#include "session.h" // getEnv/hasEnv
-
-#include <stdlib.h>
-#include <unistd.h>
-
-#ifndef NO_NAMESPACES
-using namespace ::std;
-using namespace ::Binc;
-#endif /* NO_NAMESPACES */
-
-//------------------------------------------------------------------------
-IODevice::IODevice(int f) : flags(f | IsEnabled),
-			      maxInputBufferSize(0),
-			      maxOutputBufferSize(0),
-			      timeout(0), 
-			      readCount(0), writeCount(0),
-			      outputLevel(ErrorLevel),
-			      outputLevelLimit(ErrorLevel),
-			      error(Unknown), errorString("Unknown error"),
-			      dumpfd(0)
-{
-}
-
-//------------------------------------------------------------------------
-IODevice::~IODevice(void)
-{
-}
-
-//------------------------------------------------------------------------
-IODevice &IODevice::operator <<(ostream &(*source)(ostream &))
-{
-  if (!(flags & IsEnabled) || outputLevel > outputLevelLimit)
-    return *this;
-
-  static std::ostream &(*endl_funcptr)(ostream &) = endl;
-  
-  if (source != endl_funcptr)
-    return *this;
-
-  outputBuffer << "\r\n";
-
-  if (dumpfd)
-    ::write(dumpfd, "\r\n", 2);
-
-  if (flags & FlushesOnEndl)
-    flush();
-  else if (flags & HasOutputLimit)
-    if (outputBuffer.getSize() > maxOutputBufferSize)
-      flush();
-
-  return *this;
-}
-
-//------------------------------------------------------------------------
-bool IODevice::canRead(void) const
-{
-  return false;
-}
-
-//------------------------------------------------------------------------
-void IODevice::clear()
-{
-  if (!(flags & IsEnabled))
-    return;
-
-  inputBuffer.clear();
-  outputBuffer.clear();
-}
-
-//------------------------------------------------------------------------
-bool IODevice::flush()
-{
-  if (!(flags & IsEnabled))
-    return true;
-
-  WriteResult writeResult = WriteWait;
-  do {
-    unsigned int s = outputBuffer.getSize();
-    if (s == 0)
-      break;
-
-    if (!waitForWrite())
-      return false;
-    
-    writeResult = write();
-    if (writeResult == WriteError)
-      return false;
-
-    writeCount += s - outputBuffer.getSize();
-  } while (outputBuffer.getSize() > 0 && writeResult == WriteWait);
-
-  outputBuffer.clear();
-  return true;
-}
-
-//------------------------------------------------------------------------
-void IODevice::setFlags(unsigned int f)
-{
-  flags |= f;
-}
-
-//------------------------------------------------------------------------
-void IODevice::clearFlags(unsigned int f)
-{
-  flags &= ~f;
-}
-
-//------------------------------------------------------------------------
-void IODevice::setMaxInputBufferSize(unsigned int max)
-{
-  maxInputBufferSize = max;
-}
-
-//------------------------------------------------------------------------
-void IODevice::setMaxOutputBufferSize(unsigned int max)
-{
-  maxOutputBufferSize = max;
-}
-
-//------------------------------------------------------------------------
-void IODevice::setTimeout(unsigned int t)
-{
-  timeout = t;
-
-  if (t)
-    flags |= HasTimeout;
-  else
-    flags &= ~HasTimeout;
-}
-
-//------------------------------------------------------------------------
-unsigned int IODevice::getTimeout(void) const
-{
-  return timeout;
-}
-
-//------------------------------------------------------------------------
-void IODevice::setOutputLevel(LogLevel level)
-{
-  outputLevel = level;
-}
-
-//------------------------------------------------------------------------
-IODevice::LogLevel IODevice::getOutputLevel(void) const
-{
-  return outputLevel;
-}
-
-//------------------------------------------------------------------------
-void IODevice::setOutputLevelLimit(LogLevel level)
-{
-  outputLevelLimit = level;
-}
-
-//------------------------------------------------------------------------
-IODevice::LogLevel IODevice::getOutputLevelLimit(void) const
-{
-  return outputLevelLimit;
-}
-
-//------------------------------------------------------------------------
-bool IODevice::readStr(string *dest, unsigned int max)
-{
-  // If max is 0, fill the input buffer once only if it's empty.
-  if (!max && inputBuffer.getSize() == 0 && !fillInputBuffer())
-    return false;
-
-  // If max is != 0, wait until we have max.
-  while (max && inputBuffer.getSize() < max) {
-      if (!fillInputBuffer())
-	  return false;
-  }
-
-  unsigned int bytesToRead = max ? max : inputBuffer.getSize();
-  *dest += inputBuffer.str().substr(0, bytesToRead);
-  if (dumpfd) {
-      ::write(dumpfd, inputBuffer.str().substr(0, bytesToRead).c_str(), 
-	      bytesToRead);
-  }
-  
-  inputBuffer.popString(bytesToRead);
-  readCount += bytesToRead;
-  return true;
-}
-
-//------------------------------------------------------------------------
-bool IODevice::readChar(char *dest)
-{
-  if (inputBuffer.getSize() == 0 && !fillInputBuffer())
-    return false;
-
-  char c = inputBuffer.popChar();
-  if (dest)
-    *dest = c;
-  if (dumpfd)
-    ::write(dumpfd, &c, 1);
-
-  ++readCount;
-  return true;
-}
-
-//------------------------------------------------------------------------
-void IODevice::unreadChar(char c)
-{
-  inputBuffer.unpopChar(c);
-}
-
-//------------------------------------------------------------------------
-void IODevice::unreadStr(const string &s)
-{
-  inputBuffer.unpopStr(s);
-}
-
-//------------------------------------------------------------------------
-bool IODevice::skipTo(char c)
-{
-  char dest = '\0';
-  do {
-    if (!readChar(&dest))
-      return false;
-    if (dumpfd)
-      ::write(dumpfd, &dest, 1);
-  } while (c != dest);
-
-  return true;
-}
-
-//------------------------------------------------------------------------
-string IODevice::service(void) const
-{
-  return "nul";
-}
-
-//------------------------------------------------------------------------
-bool IODevice::waitForWrite(void) const
-{
-  return false;
-}
-
-//------------------------------------------------------------------------
-bool IODevice::waitForRead(void) const
-{
-  return false;
-}
-
-//------------------------------------------------------------------------
-IODevice::WriteResult IODevice::write(void)
-{
-  return WriteError;
-}
-
-//------------------------------------------------------------------------
-bool IODevice::fillInputBuffer(void)
-{
-  return false;
-}
-
-//------------------------------------------------------------------------
-IODevice::Error IODevice::getLastError(void) const
-{
-  return error;
-}
-
-//------------------------------------------------------------------------
-string IODevice::getLastErrorString(void) const
-{
-  return errorString;
-}
-
-//------------------------------------------------------------------------
-unsigned int IODevice::getReadCount(void) const
-{
-  return readCount;
-}
-
-//------------------------------------------------------------------------
-unsigned int IODevice::getWriteCount(void) const
-{
-  return writeCount;
-}
-
-//------------------------------------------------------------------------
-void IODevice::enableProtocolDumping(void)
-{
-#if 0
-  BincStream ss;
-  ss << "/tmp/bincimap-dump-" << (int) time(0) << "-" 
-     << Session::getInstance().getIP() << "-XXXXXX";
-  char *safename = strdup(ss.str().c_str());
-  dumpfd = mkstemp(safename);
-  if (dumpfd == -1)
-    dumpfd = 0;
-  delete safename;
-#endif
-}

src/bincimapmime/iodevice.h

@@ -1,401 +0,0 @@
-/*-*-mode:c++;c-basic-offset:2-*-*/
-/*  --------------------------------------------------------------------
- *  Filename:
- *    src/iodevice.h
- *  
- *  Description:
- *    Declaration of the IODevice class.
- *  --------------------------------------------------------------------
- *  Copyright 2002, 2003 Andreas Aardal Hanssen
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Street #330, Boston, MA 02111-1307, USA.
- *  --------------------------------------------------------------------
- */
-#ifndef iodevice_h_included
-#define iodevice_h_included
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#include "convert.h" // BincStream
-#include <string>
-#include <unistd.h> // ::write
-
-namespace Binc {
-  /*!
-    \class IODevice
-    \brief The IODevice class provides a framework for reading and
-    writing to device.
-
-    Implement new devices by inheriting this class and overloading all
-    virtual methods.
-
-    service() returns the service that the specific device is used
-    for. Two values are "log" and "client".
-
-    \sa IOFactory, MultilogDevice, SyslogDevice, StdIODevice, SSLDevice
-  */
-  class IODevice {
-  public:
-    /*!
-      Standard options for an IODevice.
-    */
-    enum Flags {
-      None = 0,
-      FlushesOnEndl = 1 << 0,
-      HasInputLimit = 1 << 1,
-      HasOutputLimit = 1 << 2,
-      IsEnabled = 1 << 3,
-      HasTimeout = 1 << 4
-    };
-
-    /*!
-      Errors from when an operation returned false.
-    */
-    enum Error {
-      Unknown,
-      Timeout
-    };
-
-    /*!
-      Constructs an invalid IODevice.      
-
-      Instances of IODevice perform no operations, and all boolean
-      functions always return false. This constructor is only useful
-      if called from a subclass that reimplements all virtual methods.
-    */
-    IODevice(int f = 0);
-
-    /*!
-      Destructs an IODevice; does nothing.
-    */
-    virtual ~IODevice(void);
-
-    /*!
-      Clears all data in the input and output buffers.
-    */
-    void clear(void);
-
-    /*!
-      Sets one or more flags.
-      \param f A bitwise OR of flags from the Flags enum.
-    */
-    void setFlags(unsigned int f);
-
-    /*!
-      Clears one or more flags.
-      \param f A bitwise OR of flags from the Flags enum.
-    */
-    void clearFlags(unsigned int f);
-
-    /*!
-      Sets the maximum allowed input buffer size. If this size is
-      non-zero and exceeded, reading from the device will fail. This
-      functionality is used to prevent clients from forcing this class
-      to consume so much memory that the program crashes.
-
-      Setting the max input buffer size to 0 disables the input size
-      limit.
-
-      \param max The maximum input buffer size in bytes.
-    */
-    void setMaxInputBufferSize(unsigned int max);
-
-    /*!
-      Sets the maximum allowed output buffer size. If this size is
-      non-zero and exceeded, flush() is called implicitly.
-
-      Setting the max output buffer size to 0 disables the output size
-      limit. This is generally discouraged.
-
-      As a contrast to setMaxInputBufferSize(), this function is used
-      to bundle up consequent write calls, allowing more efficient use
-      of the underlying device as larger blocks of data are written at
-      a time.
-
-      \param max The maximum output buffer size in bytes.
-    */
-    void setMaxOutputBufferSize(unsigned int max);
-
-    /*!
-      Sets the device's internal timeout in seconds. This timeout is
-      used both when waiting for data to read and for waiting for the
-      ability to write.
-
-      If this timeout is exceeded, the read or write function that
-      triggered the timeout will fail.
-
-      Setting the timeout to 0 disables the timeout.
-
-      \param t The timeout in seconds.
-      \sa getTimeout()
-    */
-    void setTimeout(unsigned int t);
-
-    /*!
-      Returns the timeout in seconds, or 0 if there is no timeout.
-
-      \sa setTimeout()
-    */
-    unsigned int getTimeout(void) const;
-
-    enum LogLevel {
-      ErrorLevel,
-      InfoLevel,
-      WarningLevel,
-      DebugLevel
-    };
-    
-    /*!
-      Sets the output level for the following write operations on this
-      device.
-
-      The output level is a number which gives the following write
-      operations a priority. You can use setOutputLevelLimit() to
-      filter the write operations valid for different operating modes.
-      This enables you to have certain write operations ignored.
-
-      For instance, if the output level is set to 0, then "Hello" is
-      written, and the output level is set to 1, followed by writing
-      "Daisy", the output level limit value will decive wether only
-      "Hello" is written, or if also "Daisy" is written.
-
-      A low value of the level gives higher priority, and a high level
-      will give low priority. The default value is 0, and write
-      operations that are done with output level 0 are never ignored.
-
-      \param level The output level
-      \sa getOutputLevel(), setOutputLevelLimit()
-    */
-    void setOutputLevel(LogLevel level);
-
-    /*!
-      Returns the current output level.
-
-      \sa setOutputLevel()
-    */
-    LogLevel getOutputLevel(void) const;
-
-    /*!
-      Sets the current output level limit. Write operations with a
-      level higher than the output level limit are ignored.
-
-      \param level The output level limit
-      \sa setOutputLevel()
-    */
-    void setOutputLevelLimit(LogLevel level);
-
-    /*!
-      Returns the current output level limit.
-
-      \sa setOutputLevelLimit()
-    */
-    LogLevel getOutputLevelLimit(void) const;
-
-    /*!
-      Returns the number of bytes that have been read from this device
-      since it was created.
-    */
-    unsigned int getReadCount(void) const;
-
-    /*!
-      Returns the number of bytes that have been written to this
-      device since it was created.
-    */
-    unsigned int getWriteCount(void) const;
-
-    /*!
-      Calling this function enables the built-in protocol dumping feature in
-      the device. All input and output to this device will be dumped to a file
-      in /tmp.
-    */
-    void enableProtocolDumping(void);
-
-    /*!
-      Writes data to the device. Depending on the value of the max
-      output buffer size, the data may not be written immediately.
-
-      \sa setMaxOutputBufferSize()
-    */
-    template <class T> IODevice &operator <<(const T &source);
-
-    /*!
-      Writes data to the device. This function specializes on standard
-      ostream derivates, such as std::endl.
-     */
-    IODevice &operator <<(std::ostream &(*source)(std::ostream &));
-
-    /*!
-      Returns true if data can be read from the device; otherwise
-      returns false.
-    */
-    virtual bool canRead(void) const;
-
-    /*!
-      Reads data from the device, and stores this in a string. Returns
-      true on success; otherwise returns false.
-      
-      \param dest The incoming data is stored in this string.
-      \param max No more than this number of bytes is read from the
-      device.
-    */
-    bool readStr(std::string *dest, unsigned int max = 0);
-
-    /*!
-      Reads exactly one byte from the device and stores this in a
-      char. Returns true on success; otherwise returns false.
-
-      \param dest The incoming byte is stored in this char.
-    */
-    bool readChar(char *dest = 0);
-
-    /*!
-      FIXME: add docs
-    */
-    void unreadChar(char c);
-
-    /*!
-      FIXME: add docs
-    */
-    void unreadStr(const std::string &s);
-
-    /*!
-      Reads characters from the device, until and including one
-      certain character is found. All read characters are discarded.
-
-      This function can be used to skip to the beginning of a line,
-      with the terminating character being '\n'.
-
-      \param The certain character.
-    */
-    bool skipTo(char c);
-
-    /*!
-      Flushes the output buffer. Writes all data in the output buffer
-      to the device.
-    */
-    bool flush(void);
-
-    /*!
-      Returns the type of error that most recently occurred.
-    */
-    Error getLastError(void) const;
-
-    /*!
-      Returns a human readable description of the error that most
-      recently occurred. If no known error has occurred, this method
-      returns "Unknown error".
-    */
-    std::string getLastErrorString(void) const;
-
-    /*!
-      Returns the type of service provided by this device. Two valid
-      return values are "client" and "log".
-    */
-    virtual std::string service(void) const;
-    
-  protected:
-    /*!
-      Waits until data can be written to the device. If the timeout is
-      0, this function waits indefinitely. Otherwise, it waits until
-      the timeout has expired.
-
-      If this function returns true, data can be written to the
-      device; otherwise, getLastError() must be checked to determine
-      whether a timeout occurred or whether an error with the device
-      prevents further writing.
-    */
-    virtual bool waitForWrite(void) const;
-
-    /*!
-      Waits until data can be read from the device.
-
-      \sa waitForWrite()
-    */
-    virtual bool waitForRead(void) const;
-
-    /*!
-      Types of results from a write. 
-    */
-    enum WriteResult {
-      WriteWait = 0,
-      WriteDone = 1 << 0,
-      WriteError = 1 << 1
-    };
-
-    /*!
-      Writes as much data as possible to the device. If some but not
-      all data was written, returns WriteWait. If all data was
-      written, returns WriteDone.  If an error occurred, returns
-      WriteError.
-    */
-    virtual WriteResult write(void);
-
-    /*!
-      Reads data from the device, and stores it in the input buffer.
-      Returns true on success; otherwise returns false.
-      
-      This method will fail if there is no more data available, if a
-      timeout occurred or if an error with the device prevents more
-      data from being read.
-
-      The number of bytes read from the device is undefined.
-    */
-    virtual bool fillInputBuffer(void);
-
-    BincStream inputBuffer;
-    BincStream outputBuffer;
-
-  protected:
-    unsigned int flags;
-    unsigned int maxInputBufferSize;
-    unsigned int maxOutputBufferSize;
-
-    unsigned int timeout;
-
-    unsigned int readCount;
-    unsigned int writeCount;
-
-    LogLevel outputLevel;
-    LogLevel outputLevelLimit;
-
-    mutable Error error;
-    mutable std::string errorString;
-
-    int dumpfd;
-  };
-
-  //----------------------------------------------------------------------
-  template <class T> IODevice &IODevice::operator <<(const T &source)
-  {
-    if ((flags & IsEnabled) && outputLevel <= outputLevelLimit) {
-      outputBuffer << source;
-
-      if (dumpfd) {
-	BincStream ss;
-	ss << source;
-	::write(dumpfd, ss.str().c_str(), ss.getSize());
-      }
-
-      if (flags & HasInputLimit)
-	if (outputBuffer.getSize() > maxOutputBufferSize)
-	  flush();
-    }
-
-    return *this;
-  }
-}
-
-#endif

src/bincimapmime/iofactory.cc

@@ -1,87 +0,0 @@
-/*-*-mode:c++-*-*/
-/*  --------------------------------------------------------------------
- *  Filename:
- *    src/iofactory.cc
- *  
- *  Description:
- *    Implementation of the IOFactory class.
- *  --------------------------------------------------------------------
- *  Copyright 2002, 2003 Andreas Aardal Hanssen
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Street #330, Boston, MA 02111-1307, USA.
- *  --------------------------------------------------------------------
- */
-#include "iofactory.h"
-#include "iodevice.h"
-
-#ifndef NO_NAMESPACES
-using namespace ::Binc;
-using namespace ::std;
-#endif /* NO_NAMESPACES */
-
-//------------------------------------------------------------------------
-IOFactory::IOFactory(void)
-{
-}
-
-//------------------------------------------------------------------------
-IOFactory::~IOFactory(void)
-{
-}
-
-//------------------------------------------------------------------------
-IOFactory &IOFactory::getInstance(void)
-{
-  static IOFactory ioFactory;
-  return ioFactory;
-}
-
-//------------------------------------------------------------------------
-void IOFactory::addDevice(IODevice *dev)
-{
-  IODevice *ioDevice = IOFactory::getInstance().devices[dev->service()];
-
-  // FIXME: Delete correct object. Now, only IODevice's destructor is
-  // called, and only IODevice's memory is freed.
-  if (ioDevice)
-    delete ioDevice;
-
-  IOFactory::getInstance().devices[dev->service()] = dev;
-}
-
-//------------------------------------------------------------------------
-IODevice &IOFactory::getClient(void)
-{
-  static IODevice nulDevice;
-
-  IOFactory &ioFactory = IOFactory::getInstance();
-
-  if (ioFactory.devices.find("client") != ioFactory.devices.end())
-    return *ioFactory.devices["client"];
-
-  return nulDevice;
-}
-
-//------------------------------------------------------------------------
-IODevice &IOFactory::getLogger(void)
-{
-  static IODevice nulDevice;
-
-  IOFactory &ioFactory = IOFactory::getInstance();
-
-  if (ioFactory.devices.find("log") != ioFactory.devices.end())
-    return *ioFactory.devices["log"];
-  return nulDevice;
-}

src/bincimapmime/iofactory.h

@@ -1,69 +0,0 @@
-/*-*-mode:c++-*-*/
-/*  --------------------------------------------------------------------
- *  Filename:
- *    src/iofactory.h
- *  
- *  Description:
- *    Declaration of the IOFactory class.
- *  --------------------------------------------------------------------
- *  Copyright 2002, 2003 Andreas Aardal Hanssen
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Street #330, Boston, MA 02111-1307, USA.
- *  --------------------------------------------------------------------
- */
-#ifndef IOFACTORY_H_INCLUDED
-#define IOFACTORY_H_INCLUDED
-#include <map>
-#include <string>
-
-#include "iodevice.h"
-
-namespace Binc {
-  class IOFactory {
-  public:
-    ~IOFactory(void);
-
-    static void addDevice(IODevice *dev);
-    static IOFactory &getInstance(void);
-    static IODevice &getClient(void);
-    static IODevice &getLogger(void);
-
-  private:
-    IOFactory(void);
-    
-    std::map<std::string, IODevice *> devices;
-  };
-}
-
-#define bincClient \
-  IOFactory::getClient()
-
-#if !defined (DEBUG)
-#define bincError if (false) std::cout
-#define bincWarning if (false) std::cout
-#define bincDebug if (false) std::cout
-#else
-#define bincError \
-  IOFactory::getLogger().setOutputLevel(IODevice::ErrorLevel);IOFactory::getLogger()
-#define bincWarning \
-  IOFactory::getLogger().setOutputLevel(IODevice::WarningLevel);IOFactory::getLogger()
-#define bincDebug \
-  IOFactory::getLogger().setOutputLevel(IODevice::DebugLevel);IOFactory::getLogger()
-#endif
-
-#define bincInfo \
-  IOFactory::getLogger().setOutputLevel(IODevice::InfoLevel);IOFactory::getLogger()
-
-#endif

src/bincimapmime/mime-inputsource.h

@@ -25,13 +25,19 @@
  */
 #ifndef mime_inputsource_h_included
 #define mime_inputsource_h_included
+#include "autoconfig.h"
+// Data source for MIME parser
 
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
+// Note about large files: we might want to change the unsigned int
+// used for offsets into an off_t for intellectual satisfaction, but
+// in the context of recoll, we could only get into trouble if a
+// *single message* exceeded 2GB, which seems rather unlikely. When
+// parsing a mailbox files, we read each message in memory and use the
+// stream input source (from a memory buffer, no file offsets). When
+// parsing a raw message file, it's only one message.
 
 #include <string.h>
-#include <unistd.h>
+#include "safeunistd.h"
 
 #include <iostream>
 
@@ -43,7 +49,7 @@ namespace Binc {
     inline MimeInputSource(int fd, unsigned int start = 0);
     virtual inline ~MimeInputSource(void);
 
-    virtual inline size_t fillRaw(char *raw, size_t nbytes);
+    virtual inline ssize_t fillRaw(char *raw, size_t nbytes);
     virtual inline void reset(void);
 
     virtual inline bool fillInputBuffer(void);
@@ -81,7 +87,7 @@ namespace Binc {
   {
   }
 
-  inline size_t MimeInputSource::fillRaw(char *raw, size_t nbytes)
+  inline ssize_t MimeInputSource::fillRaw(char *raw, size_t nbytes)
   {
       return read(fd, raw, nbytes);
   }
@@ -173,7 +179,7 @@ namespace Binc {
     class MimeInputSourceStream : public MimeInputSource {
   public:
     inline MimeInputSourceStream(istream& s, unsigned int start = 0);
-    virtual inline size_t fillRaw(char *raw, size_t nb);
+    virtual inline ssize_t fillRaw(char *raw, size_t nb);
     virtual inline void reset(void);
   private:
       istream& s;
@@ -185,7 +191,7 @@ namespace Binc {
   {
   }
 
-  inline size_t MimeInputSourceStream::fillRaw(char *raw, size_t nb)
+  inline ssize_t MimeInputSourceStream::fillRaw(char *raw, size_t nb)
   {
     // Why can't streams tell how many characters were actually read
     // when hitting eof ?
@@ -193,16 +199,16 @@ namespace Binc {
     s.seekg(0, ios::end);
     std::streampos lst = s.tellg();
     s.seekg(st);
-    size_t nbytes = lst - st;
+    size_t nbytes = size_t(lst - st);
     if (nbytes > nb) {
 	nbytes = nb;
     }
     if (nbytes <= 0) {
-	return (size_t)-1;
+	return (ssize_t)-1;
     }
 
     s.read(raw, nbytes);
-    return nbytes;
+    return static_cast<ssize_t>(nbytes);
   }
 
   inline void MimeInputSourceStream::reset(void)

src/bincimapmime/mime-parsefull.cc

@@ -23,10 +23,6 @@
  *  Foundation, Inc., 59 Temple Street #330, Boston, MA 02111-1307, USA.
  *  --------------------------------------------------------------------
  */
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
 #include <string.h>
 #include <ctype.h>
 #include <stdio.h>
@@ -306,9 +302,9 @@ void Binc::MimePart::parseMessageRFC822(vector<Binc::MimePart> *members,
 bool Binc::MimePart::skipUntilBoundary(const string &delimiter,
 				       unsigned int *nlines, bool *eof)
 {
-  int endpos = delimiter.length();
+  string::size_type endpos = delimiter.length();
   char *delimiterqueue = 0;
-  int delimiterpos = 0;
+  string::size_type delimiterpos = 0;
   const char *delimiterStr = delimiter.c_str();
   if (delimiter != "") {
     delimiterqueue = new char[endpos];
@@ -340,7 +336,7 @@ bool Binc::MimePart::skipUntilBoundary(const string &delimiter,
       delimiterpos = 0;
       
     if (compareStringToQueue(delimiterStr, delimiterqueue,
-			     delimiterpos, endpos)) {
+			     delimiterpos, int(endpos))) {
       foundBoundary = true;
       break;
     }
@@ -451,7 +447,7 @@ void Binc::MimePart::parseMultipart(const string &boundary,
   skipUntilBoundary(delimiter, nlines, eof);
 
   if (!eof)
-    *boundarysize = delimiter.size();
+    *boundarysize = int(delimiter.size());
 
   postBoundaryProcessing(eof, nlines, boundarysize, foundendofpart);
 
@@ -484,7 +480,7 @@ void Binc::MimePart::parseMultipart(const string &boundary,
     skipUntilBoundary(delimiter, nlines, eof);
 
     if (!*eof)
-      *boundarysize = delimiter.size();
+      *boundarysize = int(delimiter.size());
 
     postBoundaryProcessing(eof, nlines, boundarysize, foundendofpart);
   }
@@ -528,7 +524,7 @@ void Binc::MimePart::parseSinglePart(const string &toboundary,
   //    *boundarysize = _toboundary.length();
 
   char *boundaryqueue = 0;
-  int endpos = _toboundary.length();
+  size_t endpos = _toboundary.length();
   if (toboundary != "") {
     boundaryqueue = new char[endpos];
     memset(boundaryqueue, 0, endpos);
@@ -540,7 +536,7 @@ void Binc::MimePart::parseSinglePart(const string &toboundary,
   string line;
   bool toboundaryIsEmpty = (toboundary == "");
   char c;
-  int boundarypos = 0;
+  string::size_type boundarypos = 0;
   while (mimeSource->getChar(&c)) {
     if (c == '\n') { ++*nbodylines; ++*nlines; }
 
@@ -553,8 +549,8 @@ void Binc::MimePart::parseSinglePart(const string &toboundary,
       boundarypos = 0;
       
     if (compareStringToQueue(_toboundaryStr, boundaryqueue,
-			     boundarypos, endpos)) {
-      *boundarysize = _toboundary.length();
+			     boundarypos, int(endpos))) {
+      *boundarysize = static_cast<int>(_toboundary.length());
       break;
     }
   }

src/bincimapmime/mime-parseonlyheader.cc

@@ -23,10 +23,6 @@
  *  Foundation, Inc., 59 Temple Street #330, Boston, MA 02111-1307, USA.
  *  --------------------------------------------------------------------
  */
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
 #include "mime.h"
 #include "mime-utils.h"
 #include "mime-inputsource.h"
@@ -119,7 +115,7 @@ int Binc::MimePart::doParseOnlyHeader(MimeInputSource *ms,
       if (c == '\n') ++nlines;
       if (c == ':') break;
       if (c == '\n') {
-	for (int i = name.length() - 1; i >= 0; --i)
+    for (int i = int(name.length()) - 1; i >= 0; --i)
 	  mimeSource->ungetChar();
 
 	quit = true;

src/bincimapmime/mime-printbody.cc

@@ -23,52 +23,14 @@
  *  Foundation, Inc., 59 Temple Street #330, Boston, MA 02111-1307, USA.
  *  --------------------------------------------------------------------
  */
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
 
 #include "mime.h"
 #include "mime-utils.h"
 #include "mime-inputsource.h"
 
-#include "convert.h"
-#include "iodevice.h"
-#include "iofactory.h"
-
 #include <string>
-#include <vector>
-#include <map>
-#include <exception>
-#include <iostream>
 
-#include <string.h>
-#include <ctype.h>
-#include <stdio.h>
-#include <errno.h>
-
-#ifndef NO_NAMESPACES
 using namespace ::std;
-#endif /* NO_NAMESPACES */
-
-//------------------------------------------------------------------------
-void Binc::MimePart::printBody(IODevice &output,
-			       unsigned int startoffset,
-			       unsigned int length) const
-{
-  mimeSource->reset();
-  mimeSource->seek(bodystartoffsetcrlf + startoffset);
-
-  if (startoffset + length > bodylength)
-    length = bodylength - startoffset;
-
-  char c = '\0';
-  for (unsigned int i = 0; i < length; ++i) {
-    if (!mimeSource->getChar(&c))
-      break;
-
-    output << (char)c;
-  }
-}
 
 void Binc::MimePart::getBody(string &s,
 			     unsigned int startoffset,

src/bincimapmime/mime-utils.h

@@ -25,11 +25,6 @@
  */
 #ifndef mime_utils_h_included
 #define mime_utils_h_included
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
 #include <string.h>
 #include <ctype.h>
 #include <stdio.h>

src/bincimapmime/mime.cc

@@ -23,10 +23,6 @@
  *  Foundation, Inc., 59 Temple Street #330, Boston, MA 02111-1307, USA.
  *  --------------------------------------------------------------------
  */
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
 #include <string.h>
 #include <ctype.h>
 #include <stdio.h>

src/bincimapmime/mime.h

@@ -23,10 +23,6 @@
  *  Foundation, Inc., 59 Temple Street #330, Boston, MA 02111-1307, USA.
  *  --------------------------------------------------------------------
  */
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
 #ifndef mime_h_included
 #define mime_h_included
 #include <string>

src/bincimapmime/trbinc.cc

@@ -1,4 +1,4 @@
-/* Copyright (C) 2006 J.F.Dockes */
+/* Copyright (C) 2006 J.F.Dockes
  *   This program is free software; you can redistribute it and/or modify
  *   it under the terms of the GNU General Public License as published by
  *   the Free Software Foundation; either version 2 of the License, or
@@ -17,10 +17,10 @@
 
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <errno.h>
+#include "safefcntl.h"
+#include "safeunistd.h"
 #include <string.h>
-#include <fcntl.h>
 
 #include <sstream>
 

src/common/Makefile

@@ -1,33 +1,33 @@
-depth = ..
-include $(depth)/mk/sysconf
 
-# Only test executables get build in here
-PROGS = unacpp textsplit rclconfig
+PROGS = unacpp textsplit rclconfig syngroups
 
-all: librecoll $(PROGS) 
+all: $(PROGS) 
 
 UNACPP_OBJS= trunacpp.o
 unacpp : $(UNACPP_OBJS)
-	$(CXX) $(ALL_CXXFLAGS) -o unacpp $(UNACPP_OBJS) \
-	       $(LIBRECOLL)  $(LIBICONV)
+	$(CXX) $(ALL_CXXFLAGS) -o unacpp $(UNACPP_OBJS) $(LIBRECOLL)
 trunacpp.o : unacpp.cpp unacpp.h
 	$(CXX) $(ALL_CXXFLAGS) -DTEST_UNACPP -c -o trunacpp.o unacpp.cpp
 
 TEXTSPLIT_OBJS= trtextsplit.o  
 textsplit : $(TEXTSPLIT_OBJS)
-	$(CXX) $(ALL_CXXFLAGS) -o textsplit $(TEXTSPLIT_OBJS) \
-        $(LIBRECOLL) $(LIBICONV)
+	$(CXX) $(ALL_CXXFLAGS) -o textsplit $(TEXTSPLIT_OBJS) $(LIBRECOLL)
 trtextsplit.o : textsplit.cpp 
 	$(CXX) $(ALL_CXXFLAGS) -DTEST_TEXTSPLIT -c -o trtextsplit.o \
 	       textsplit.cpp
 
 RCLCONFIG_OBJS= trrclconfig.o 
 rclconfig : $(RCLCONFIG_OBJS) 
-	$(CXX) $(ALL_CXXFLAGS) -o rclconfig $(RCLCONFIG_OBJS) \
-	       $(LIBRECOLL) $(LIBICONV) $(LIBSYS)
+	$(CXX) $(ALL_CXXFLAGS) -o rclconfig $(RCLCONFIG_OBJS) $(LIBRECOLL) 
 trrclconfig.o : rclconfig.cpp 
 	$(CXX) $(ALL_CXXFLAGS) -DTEST_RCLCONFIG -c -o trrclconfig.o \
 	       rclconfig.cpp
 
-include $(depth)/mk/commontargets
+SYNGROUPS_OBJS= trsyngroups.o 
+syngroups : $(SYNGROUPS_OBJS) 
+	$(CXX) $(ALL_CXXFLAGS) -o syngroups $(SYNGROUPS_OBJS) $(LIBRECOLL) 
+trsyngroups.o : syngroups.cpp 
+	$(CXX) $(ALL_CXXFLAGS) -DTEST_SYNGROUPS -c -o trsyngroups.o \
+	       syngroups.cpp
 
+include ../utils/utmkdefs.mk

src/common/autoconfig-win.h

@@ -0,0 +1,183 @@
+/* Manually edited version of autoconfig.h for windows. Many things are
+overriden in the c++ code by ifdefs _WIN32 anyway  */
+#ifndef _AUTOCONFIG_H_INCLUDED
+#define _AUTOCONFIG_H_INCLUDED
+/* Define if building universal (internal helper macro) */
+/* #undef AC_APPLE_UNIVERSAL_BUILD */
+
+/* Path to the aspell api include file */
+/* #undef ASPELL_INCLUDE "aspell-local.h" */
+
+/* Path to the aspell program */
+/* #define ASPELL_PROG "/usr/bin/aspell" */
+
+/* No X11 session monitoring support */
+#define DISABLE_X11MON
+
+/* Path to the fam api include file */
+/* #undef FAM_INCLUDE */
+
+/* Path to the file program */
+#define FILE_PROG "/usr/bin/file"
+
+/* "Have C++0x" */
+#define HAVE_CXX0X_UNORDERED 1
+
+/* Define to 1 if you have the <dlfcn.h> header file. */
+#define HAVE_DLFCN_H 1
+
+/* Define to 1 if you have the <inttypes.h> header file. */
+#define HAVE_INTTYPES_H 1
+
+/* Define to 1 if you have the `dl' library (-ldl). */
+#define HAVE_LIBDL 1
+
+/* Define to 1 if you have the `z' library (-lz). */
+#define HAVE_LIBZ 1
+
+/* Define to 1 if you have the <memory.h> header file. */
+#define HAVE_MEMORY_H 1
+
+/* Define to 1 if you have the `mkdtemp' function. */
+/* #undef HAVE_MKDTEMP */
+
+/* Define to 1 if you have the `posix_spawn,' function. */
+/* #undef HAVE_POSIX_SPAWN_ */
+
+/* Define to 1 if you have the `setrlimit' function. */
+#define HAVE_SETRLIMIT 1
+
+/* Has std::shared_ptr */
+#define HAVE_SHARED_PTR_STD
+
+/* Has std::tr1::shared_ptr */
+/* #undef HAVE_SHARED_PTR_TR1 */
+
+/* Define to 1 if you have the <spawn.h> header file. */
+#define HAVE_SPAWN_H 1
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#define HAVE_STDINT_H 1
+
+/* Define to 1 if you have the <stdlib.h> header file. */
+#define HAVE_STDLIB_H 1
+
+/* Define to 1 if you have the <strings.h> header file. */
+#define HAVE_STRINGS_H 1
+
+/* Define to 1 if you have the <string.h> header file. */
+#define HAVE_STRING_H 1
+
+/* Define to 1 if you have the <sys/mount.h> header file. */
+/* #undef HAVE_SYS_MOUNT_H */
+
+/* Define to 1 if you have the <sys/param.h,> header file. */
+/* #undef HAVE_SYS_PARAM_H_ */
+
+/* Define to 1 if you have the <sys/statfs.h> header file. */
+/* #undef HAVE_SYS_STATFS_H */
+
+/* Define to 1 if you have the <sys/statvfs.h> header file. */
+/* #undef HAVE_SYS_STATVFS_H */
+
+/* Define to 1 if you have the <sys/stat.h> header file. */
+#define HAVE_SYS_STAT_H 1
+
+/* Define to 1 if you have the <sys/types.h> header file. */
+#define HAVE_SYS_TYPES_H 1
+
+/* Define to 1 if you have the <sys/vfs.h> header file. */
+/* #undef HAVE_SYS_VFS_H */
+
+/* "Have tr1" */
+/* #undef HAVE_TR1_UNORDERED */
+
+/* Define to 1 if you have the <unistd.h> header file. */
+/* #undef HAVE_UNISTD_H */
+
+/* Use multiple threads for indexing */
+#define IDX_THREADS 1
+
+/* Define to the sub-directory in which libtool stores uninstalled libraries.
+   */
+#define LT_OBJDIR ".libs/"
+
+/* Define to the address where bug reports for this package should be sent. */
+#define PACKAGE_BUGREPORT ""
+
+/* Define to the full name of this package. */
+#define PACKAGE_NAME "Recoll"
+
+/* Define to the full name and version of this package. */
+#define PACKAGE_STRING "Recoll 1.23.0-x"
+
+/* Define to the one symbol short name of this package. */
+#define PACKAGE_TARNAME "recoll"
+
+/* Define to the home page for this package. */
+#define PACKAGE_URL ""
+
+/* Define to the version of this package. */
+#define PACKAGE_VERSION "1.23.0-x"
+
+/* putenv parameter is const */
+/* #undef PUTENV_ARG_CONST */
+
+/* iconv parameter 2 is const char** */
+#define RCL_ICONV_INBUF_CONST 1
+
+/* Real time monitoring option */
+#undef RCL_MONITOR
+
+/* Split camelCase words */
+/* #undef RCL_SPLIT_CAMELCASE */
+
+/* Compile the aspell interface */
+/* #undef RCL_USE_ASPELL */
+
+/* Compile the fam interface */
+/* #undef RCL_USE_FAM */
+
+/* Compile the inotify interface */
+#define RCL_USE_INOTIFY 1
+
+/* Define to 1 if you have the ANSI C header files. */
+#define STDC_HEADERS 1
+
+/* Use posix_spawn() */
+/* #undef USE_POSIX_SPAWN */
+
+/* Enable using the system's 'file' command to id mime if we fail internally
+   */
+/* #undef USE_SYSTEM_FILE_COMMAND */
+
+/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most
+   significant byte first (like Motorola and SPARC, unlike Intel). */
+#if defined AC_APPLE_UNIVERSAL_BUILD
+# if defined __BIG_ENDIAN__
+#  define WORDS_BIGENDIAN 1
+# endif
+#else
+# ifndef WORDS_BIGENDIAN
+/* #  undef WORDS_BIGENDIAN */
+# endif
+#endif
+
+/* Define to 1 if the X Window System is missing or not being used. */
+/* #undef X_DISPLAY_MISSING */
+
+/* Enable large inode numbers on Mac OS X 10.5.  */
+#ifndef _DARWIN_USE_64_BIT_INODE
+# define _DARWIN_USE_64_BIT_INODE 1
+#endif
+
+/* Number of bits in a file offset, on hosts where this is settable. */
+/* #undef _FILE_OFFSET_BITS */
+
+/* Define for large files, on AIX-style hosts. */
+/* #undef _LARGE_FILES */
+
+#define DISABLE_WEB_INDEXER
+
+#include "conf_post.h"
+#endif // already included

src/common/autoconfig.h.in

@@ -1,5 +1,8 @@
 /* common/autoconfig.h.in.  Generated from configure.ac by autoheader.  */
 
+/* Define if building universal (internal helper macro) */
+#undef AC_APPLE_UNIVERSAL_BUILD
+
 /* Path to the aspell api include file */
 #undef ASPELL_INCLUDE
 
@@ -15,23 +18,45 @@
 /* Path to the file program */
 #undef FILE_PROG
 
-/* Define to 1 if you have the <spawn.h> header file. */
-#undef HAVE_SPAWN_H
-#undef HAVE_POSIX_SPAWN
-#undef USE_POSIX_SPAWN
+/* "Have C++0x" */
+#undef HAVE_CXX0X_UNORDERED
 
-/* Define to 1 if you have the setrlimit() call. */
-#undef HAVE_SETRLIMIT
+/* Define to 1 if you have the <dlfcn.h> header file. */
+#undef HAVE_DLFCN_H
 
 /* Define to 1 if you have the <inttypes.h> header file. */
 #undef HAVE_INTTYPES_H
 
+/* Define to 1 if you have the `dl' library (-ldl). */
+#undef HAVE_LIBDL
+
+/* Define to 1 if you have the `pthread' library (-lpthread). */
+#undef HAVE_LIBPTHREAD
+
+/* Define to 1 if you have the `z' library (-lz). */
+#undef HAVE_LIBZ
+
 /* Define to 1 if you have the <memory.h> header file. */
 #undef HAVE_MEMORY_H
 
 /* Define to 1 if you have the `mkdtemp' function. */
 #undef HAVE_MKDTEMP
 
+/* Define to 1 if you have the `posix_spawn,' function. */
+#undef HAVE_POSIX_SPAWN_
+
+/* Define to 1 if you have the `setrlimit' function. */
+#undef HAVE_SETRLIMIT
+
+/* Has std::shared_ptr */
+#undef HAVE_SHARED_PTR_STD
+
+/* Has std::tr1::shared_ptr */
+#undef HAVE_SHARED_PTR_TR1
+
+/* Define to 1 if you have the <spawn.h> header file. */
+#undef HAVE_SPAWN_H
+
 /* Define to 1 if you have the <stdint.h> header file. */
 #undef HAVE_STDINT_H
 
@@ -47,6 +72,9 @@
 /* Define to 1 if you have the <sys/mount.h> header file. */
 #undef HAVE_SYS_MOUNT_H
 
+/* Define to 1 if you have the <sys/param.h,> header file. */
+#undef HAVE_SYS_PARAM_H_
+
 /* Define to 1 if you have the <sys/statfs.h> header file. */
 #undef HAVE_SYS_STATFS_H
 
@@ -62,9 +90,18 @@
 /* Define to 1 if you have the <sys/vfs.h> header file. */
 #undef HAVE_SYS_VFS_H
 
+/* "Have tr1" */
+#undef HAVE_TR1_UNORDERED
+
 /* Define to 1 if you have the <unistd.h> header file. */
 #undef HAVE_UNISTD_H
 
+/* Use multiple threads for indexing */
+#undef IDX_THREADS
+
+/* Define to the sub-directory where libtool stores uninstalled libraries. */
+#undef LT_OBJDIR
+
 /* Define to the address where bug reports for this package should be sent. */
 #undef PACKAGE_BUGREPORT
 
@@ -77,6 +114,9 @@
 /* Define to the one symbol short name of this package. */
 #undef PACKAGE_TARNAME
 
+/* Define to the home page for this package. */
+#undef PACKAGE_URL
+
 /* Define to the version of this package. */
 #undef PACKAGE_VERSION
 
@@ -101,23 +141,40 @@
 /* Compile the inotify interface */
 #undef RCL_USE_INOTIFY
 
-/* Use multiple threads for indexing */
-#undef IDX_THREADS
-
 /* Define to 1 if you have the ANSI C header files. */
 #undef STDC_HEADERS
 
+/* Use posix_spawn() */
+#undef USE_POSIX_SPAWN
+
 /* Enable using the system's 'file' command to id mime if we fail internally
    */
 #undef USE_SYSTEM_FILE_COMMAND
 
+/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most
+   significant byte first (like Motorola and SPARC, unlike Intel). */
+#if defined AC_APPLE_UNIVERSAL_BUILD
+# if defined __BIG_ENDIAN__
+#  define WORDS_BIGENDIAN 1
+# endif
+#else
+# ifndef WORDS_BIGENDIAN
+#  undef WORDS_BIGENDIAN
+# endif
+#endif
+
 /* Define to 1 if the X Window System is missing or not being used. */
 #undef X_DISPLAY_MISSING
 
+/* Enable large inode numbers on Mac OS X 10.5.  */
+#ifndef _DARWIN_USE_64_BIT_INODE
+# define _DARWIN_USE_64_BIT_INODE 1
+#endif
+
+/* Number of bits in a file offset, on hosts where this is settable. */
 #undef _FILE_OFFSET_BITS
+
+/* Define for large files, on AIX-style hosts. */
 #undef _LARGE_FILES
 
-#undef WORDS_BIGENDIAN
-
-#undef HAVE_TR1_UNORDERED
-#undef HAVE_CXX0X_UNORDERED
+#include "conf_post.h"

src/common/beaglequeuecache.cpp

@@ -20,7 +20,7 @@
 #include "cstr.h"
 #include "beaglequeuecache.h"
 #include "circache.h"
-#include "debuglog.h"
+#include "log.h"
 #include "rclconfig.h"
 #include "pathut.h"
 #include "rcldoc.h"
@@ -29,24 +29,16 @@ const string cstr_bgc_mimetype("mimetype");
 
 BeagleQueueCache::BeagleQueueCache(RclConfig *cnf) 
 {
-    string ccdir;
-    cnf->getConfParam("webcachedir", ccdir);
-    if (ccdir.empty())
-        ccdir = "webcache";
-    ccdir = path_tildexpand(ccdir);
-    // If not an absolute path, compute relative to config dir
-    if (ccdir.at(0) != '/')
-        ccdir = path_cat(cnf->getConfDir(), ccdir);
+    string ccdir = cnf->getWebcacheDir();
 
     int maxmbs = 40;
     cnf->getConfParam("webcachemaxmbs", &maxmbs);
     if ((m_cache = new CirCache(ccdir)) == 0) {
-	LOGERR(("BeagleQueueCache: cant create CirCache object\n"));
+	LOGERR("BeagleQueueCache: cant create CirCache object\n" );
 	return;
     }
     if (!m_cache->create(off_t(maxmbs)*1000*1024, CirCache::CC_CRUNIQUE)) {
-	LOGERR(("BeagleQueueCache: cache file creation failed: %s\n",
-		m_cache->getReason().c_str()));
+	LOGERR("BeagleQueueCache: cache file creation failed: "  << (m_cache->getReason()) << "\n" );
 	delete m_cache;
 	m_cache = 0;
 	return;
@@ -66,11 +58,11 @@ bool BeagleQueueCache::getFromCache(const string& udi, Rcl::Doc &dotdoc,
     string dict;
 
     if (m_cache == 0) {
-	LOGERR(("BeagleQueueCache::getFromCache: cache is null\n"));
+	LOGERR("BeagleQueueCache::getFromCache: cache is null\n" );
 	return false;
     }
-    if (!m_cache->get(udi, dict, data)) {
-	LOGDEB(("BeagleQueueCache::getFromCache: get failed\n"));
+    if (!m_cache->get(udi, dict, &data)) {
+	LOGDEB("BeagleQueueCache::getFromCache: get failed\n" );
 	return false;
     }
 
@@ -93,3 +85,4 @@ bool BeagleQueueCache::getFromCache(const string& udi, Rcl::Doc &dotdoc,
     dotdoc.meta[Rcl::Doc::keyudi] = udi;
     return true;
 }
+

src/common/conf_post.h

@@ -0,0 +1,51 @@
+
+#ifdef _WIN32
+#include "safewindows.h"
+
+
+#ifdef _MSC_VER
+// gmtime is supposedly thread-safe on windows
+#define gmtime_r(A, B) gmtime(A)
+#define localtime_r(A,B) localtime(A)
+typedef int mode_t;
+#define fseeko _fseeki64
+#define ftello (off_t)_ftelli64
+#define ftruncate _chsize_s
+#define PATH_MAX MAX_PATH
+#define RCL_ICONV_INBUF_CONST 1
+#define HAVE_STRUCT_TIMESPEC
+#define strdup _strdup
+#define timegm _mkgmtime
+
+#else // End _MSC_VER -> Gminw
+
+#undef RCL_ICONV_INBUF_CONST
+#define timegm portable_timegm
+
+#endif // GMinw only
+
+typedef int pid_t;
+inline int readlink(const char *cp, void *buf, int cnt)
+{
+    return -1;
+}
+
+#define MAXPATHLEN PATH_MAX
+typedef DWORD32 u_int32_t;
+typedef DWORD64 u_int64_t;
+typedef unsigned __int8 u_int8_t;
+typedef int ssize_t;
+#define strncasecmp _strnicmp
+#define strcasecmp _stricmp
+#define chdir _chdir
+
+#define R_OK 4
+#define W_OK 2
+#ifndef X_OK
+#define X_OK 4
+#endif
+
+#define S_ISLNK(X) false
+#define lstat stat
+
+#endif // _WIN32

src/common/cstr.h

@@ -80,7 +80,6 @@ DEF_CSTR(dj_keycontent, "content");
 DEF_CSTR(dj_keyipath, "ipath");
 DEF_CSTR(dj_keymd5, "md5");
 DEF_CSTR(dj_keymt, "mimetype");
-DEF_CSTR(dj_keydocsize, "docsize");
 DEF_CSTR(dj_keyanc, "rclanc");
 
 #endif /* _CSTR_H_INCLUDED_ */

src/common/rclconfig.cpp

@@ -17,42 +17,42 @@
 #ifndef TEST_RCLCONFIG
 #include "autoconfig.h"
 
-#include <unistd.h>
-#include <fcntl.h>
 #include <stdio.h>
 #include <errno.h>
+#ifndef _WIN32
 #include <langinfo.h>
-#include <limits.h>
 #include <sys/param.h>
-
-#include <algorithm>
-#include <list>
-using std::list;
-
-#include <sys/types.h>
-#include <sys/stat.h>
+#else
+#include "wincodepages.h"
+#endif
+#include <limits.h>
+#include "safesysstat.h"
+#include "safeunistd.h"
 #ifdef __FreeBSD__
 #include <osreldate.h>
 #endif
 
+#include <algorithm>
+#include <list>
 #include <iostream>
 #include <sstream>
 #include <cstdlib>
 #include <cstring>
-using namespace std;
 
 #include "cstr.h"
 #include "pathut.h"
+#include "rclutil.h"
 #include "rclconfig.h"
 #include "conftree.h"
-#include "debuglog.h"
+#include "log.h"
 #include "smallut.h"
 #include "textsplit.h"
 #include "readfile.h"
 #include "fstreewalk.h"
 #include "cpuconf.h"
+#include "execmd.h"
 
-typedef pair<int,int> RclPII;
+using namespace std;
 
 // Static, logically const, RclConfig members are initialized once from the
 // first object build during process initialization.
@@ -67,10 +67,9 @@ string RclConfig::o_origcwd;
 
 bool ParamStale::needrecompute()
 {
-    LOGDEB2(("ParamStale:: needrecompute. parent gen %d mine %d\n", 
-	     parent->m_keydirgen, savedkeydirgen));
+    LOGDEB2("ParamStale:: needrecompute. parent gen "  << (parent->m_keydirgen) << " mine "  << (savedkeydirgen) << "\n" );
     if (active && parent->m_keydirgen != savedkeydirgen) {
-	LOGDEB2(("ParamState:: needrecompute. conffile %p\n", conffile));
+	LOGDEB2("ParamState:: needrecompute. conffile "  << (conffile) << "\n" );
 
         savedkeydirgen = parent->m_keydirgen;
         string newvalue;
@@ -79,8 +78,7 @@ bool ParamStale::needrecompute()
         conffile->get(paramname, newvalue, parent->m_keydir);
         if (newvalue.compare(savedvalue)) {
             savedvalue = newvalue;
-	    LOGDEB2(("ParamState:: needrecompute. return true newvalue [%s]\n",
-		     newvalue.c_str()));
+	    LOGDEB2("ParamState:: needrecompute. return true newvalue ["  << (newvalue) << "]\n" );
             return true;
         }
     }
@@ -113,18 +111,14 @@ void RclConfig::zeroMe() {
     m_ptrans = 0;
     m_stopsuffixes = 0;
     m_maxsufflen = 0;
-
-    m_oldstpsuffstate.init(0);
-    m_stpsuffstate.init(0);
-    m_skpnstate.init(0);
-    m_rmtstate.init(0);
-    m_xmtstate.init(0);
-    m_mdrstate.init(0);
+    initParamStale(0, 0);
 }
 
 bool RclConfig::isDefaultConfig() const
 {
-    string defaultconf = path_cat(path_canon(path_home()), ".recoll/");
+    string defaultconf = path_cat(path_homedata(),
+                                  path_defaultrecollconfsubdir());
+    path_catslash(defaultconf);
     string specifiedconf = path_canon(m_confdir);
     path_catslash(specifiedconf);
     return !defaultconf.compare(specifiedconf);
@@ -150,14 +144,7 @@ RclConfig::RclConfig(const string *argcnf)
     }
 
     // Compute our data dir name, typically /usr/local/share/recoll
-    const char *cdatadir = getenv("RECOLL_DATADIR");
-    if (cdatadir == 0) {
-	// If not in environment, use the compiled-in constant. 
-	m_datadir = RECOLL_DATADIR;
-    } else {
-	m_datadir = cdatadir;
-    }
-
+    m_datadir = path_pkgdatadir();
     // We only do the automatic configuration creation thing for the default
     // config dir, not if it was specified through -c or RECOLL_CONFDIR
     bool autoconfdir = false;
@@ -176,7 +163,7 @@ RclConfig::RclConfig(const string *argcnf)
 	    m_confdir = path_canon(cp);
 	} else {
 	    autoconfdir = true;
-	    m_confdir = path_cat(path_home(), ".recoll/");
+	    m_confdir = path_cat(path_homedata(), path_defaultrecollconfsubdir());
 	}
     }
 
@@ -184,7 +171,7 @@ RclConfig::RclConfig(const string *argcnf)
     // want to avoid the imperfect test in isDefaultConfig() if we actually know
     // this is the default conf
     if (!autoconfdir && !isDefaultConfig()) {
-	if (access(m_confdir.c_str(), 0) < 0) {
+	if (!path_exists(m_confdir)) {
 	    m_reason = "Explicitly specified configuration "
 		"directory must exist"
 		" (won't be automatically created). Use mkdir first";
@@ -192,7 +179,7 @@ RclConfig::RclConfig(const string *argcnf)
 	}
     }
 
-    if (access(m_confdir.c_str(), 0) < 0) {
+    if (!path_exists(m_confdir)) {
 	if (!initUserConfig()) 
 	    return;
     }
@@ -204,6 +191,7 @@ RclConfig::RclConfig(const string *argcnf)
     // is called from the main thread at once, by constructing a config
     // from recollinit
     if (o_localecharset.empty()) {
+#ifndef _WIN32
 	const char *cp;
 	cp = nl_langinfo(CODESET);
 	// We don't keep US-ASCII. It's better to use a superset
@@ -221,8 +209,11 @@ RclConfig::RclConfig(const string *argcnf)
 	    // Use cp1252 instead of iso-8859-1, it's a superset.
 	    o_localecharset = string(cstr_cp1252);
 	}
-	LOGDEB1(("RclConfig::getDefCharset: localecharset [%s]\n",
-		 o_localecharset.c_str()));
+#else
+        o_localecharset = winACPName();
+#endif
+	LOGDEB1("RclConfig::getDefCharset: localecharset ["  <<
+                o_localecharset << "]\n");
     }
 
     const char *cp;
@@ -276,20 +267,14 @@ RclConfig::RclConfig(const string *argcnf)
 	return;
 
     // Default is no threading
-    m_thrConf = create_vector<RclPII>
-	(RclPII(-1, 0))(RclPII(-1, 0))(RclPII(-1, 0));
+    m_thrConf = {{-1, 0}, {-1, 0}, {-1, 0}};
 
     m_ptrans = new ConfSimple(path_cat(m_confdir, "ptrans").c_str());
 
     m_ok = true;
     setKeyDir(cstr_null);
 
-    m_oldstpsuffstate.init(mimemap);
-    m_stpsuffstate.init(m_conf);
-    m_skpnstate.init(m_conf);
-    m_rmtstate.init(m_conf);
-    m_xmtstate.init(m_conf);
-    m_mdrstate.init(m_conf);
+    initParamStale(m_conf, mimemap);
 
     return;
 }
@@ -305,24 +290,19 @@ bool RclConfig::updateMainConfig()
 	stringsToString(m_cdirs, where);
 	m_reason = string("No/bad main configuration file in: ") + where;
 	m_ok = false;
-        m_skpnstate.init(0);
-        m_rmtstate.init(0);
-        m_xmtstate.init(0);
-        m_mdrstate.init(0);
+        initParamStale(0, 0);
 	return false;
     }
 
     delete m_conf;
     m_conf = newconf;
 
-    m_skpnstate.init(m_conf);
-    m_rmtstate.init(m_conf);
-    m_xmtstate.init(m_conf);
-    m_mdrstate.init(m_conf);
+    initParamStale(m_conf, mimemap);
 
     setKeyDir(cstr_null);
-    bool nocjk = false;
-    if (getConfParam("nocjk", &nocjk) && nocjk == true) {
+
+    bool bvalue = false;
+    if (getConfParam("nocjk", &bvalue) && bvalue == true) {
 	TextSplit::cjkProcessing(false);
     } else {
 	int ngramlen;
@@ -333,14 +313,18 @@ bool RclConfig::updateMainConfig()
 	}
     }
 
-    bool nonum = false;
-    if (getConfParam("nonumbers", &nonum) && nonum == true) {
+    bvalue = false;
+    if (getConfParam("nonumbers", &bvalue) && bvalue == true) {
 	TextSplit::noNumbers();
     }
 
-    bool fnmpathname = true;
-    if (getConfParam("skippedPathsFnmPathname", &fnmpathname)
-	&& fnmpathname == false) {
+    bvalue = false;
+    if (getConfParam("dehyphenate", &bvalue)) {
+	TextSplit::deHyphenate(bvalue);
+    }
+
+    bvalue = true;
+    if (getConfParam("skippedPathsFnmPathname", &bvalue) && bvalue == false) {
 	FsTreeWalker::setNoFnmPathname();
     }
 
@@ -351,6 +335,9 @@ bool RclConfig::updateMainConfig()
 	m_index_stripchars_init = 1;
     }
 
+    if (getConfParam("cachedir", m_cachedir)) {
+        m_cachedir = path_canon(path_tildexpand(m_cachedir));
+    }
     return true;
 }
 
@@ -433,8 +420,7 @@ bool RclConfig::getConfParam(const string &name, vector<int> *vip,
 	char *ep;
 	vip->push_back(strtol(vs[i].c_str(), &ep, 0));
 	if (ep == vs[i].c_str()) {
-	    LOGDEB(("RclConfig::getConfParam: bad int value in [%s]\n",
-		    name.c_str()));
+	    LOGDEB("RclConfig::getConfParam: bad int value in ["  << (name) << "]\n" );
 	    return false;
 	}
     }
@@ -444,24 +430,27 @@ bool RclConfig::getConfParam(const string &name, vector<int> *vip,
 void RclConfig::initThrConf()
 {
     // Default is no threading
-    m_thrConf = create_vector<RclPII>
-	(RclPII(-1, 0))(RclPII(-1, 0))(RclPII(-1, 0));
+    m_thrConf = {{-1, 0}, {-1, 0}, {-1, 0}};
 
     vector<int> vq;
     vector<int> vt;
     if (!getConfParam("thrQSizes", &vq)) {
-	LOGINFO(("RclConfig::initThrConf: no thread info (queues)\n"));
+	LOGINFO("RclConfig::initThrConf: no thread info (queues)\n" );
 	goto out;
     }
 
     // If the first queue size is 0, autoconf is requested.
     if (vq.size() > 0 && vq[0] == 0) {
-	LOGDEB(("RclConfig::initThrConf: autoconf requested\n"));
 	CpuConf cpus;
 	if (!getCpuConf(cpus) || cpus.ncpus < 1) {
-	    LOGERR(("RclConfig::initThrConf: could not retrieve cpu conf\n"));
+	    LOGERR("RclConfig::initThrConf: could not retrieve cpu conf\n" );
 	    cpus.ncpus = 1;
 	}
+        if (cpus.ncpus != 1) {
+            LOGDEB("RclConfig::initThrConf: autoconf requested. " <<
+                   cpus.ncpus << " concurrent threads available.\n");
+        }
+
 	// Arbitrarily set threads config based on number of CPUS. This also
 	// depends on the IO setup actually, so we're bound to be wrong...
 	if (cpus.ncpus == 1) {
@@ -469,14 +458,11 @@ void RclConfig::initThrConf()
 	    // it seems that the best config here is no threading
 	} else if (cpus.ncpus < 4) {
 	    // Untested so let's guess...
-	    m_thrConf = create_vector<RclPII>
-		(RclPII(2, 2))(RclPII(2, 2))(RclPII(2, 1));
+            m_thrConf = {{2, 2}, {2, 2}, {2, 1}};
 	} else if (cpus.ncpus < 6) {
-	    m_thrConf = create_vector<RclPII>
-		(RclPII(2, 4))(RclPII(2, 2))(RclPII(2, 1));
+	    m_thrConf = {{2, 4}, {2, 2}, {2, 1}};
 	} else {
-	    m_thrConf = create_vector<RclPII>
-		(RclPII(2, 5))(RclPII(2, 3))(RclPII(2, 1));
+	    m_thrConf = {{2, 5}, {2, 3}, {2, 1}};
 	}
 	goto out;
     } else if (vq.size() > 0 && vq[0] < 0) {
@@ -485,19 +471,19 @@ void RclConfig::initThrConf()
     }
 
     if (!getConfParam("thrTCounts", &vt) ) {
-	LOGINFO(("RclConfig::initThrConf: no thread info (threads)\n"));
+	LOGINFO("RclConfig::initThrConf: no thread info (threads)\n" );
 	goto out;
     }
 
     if (vq.size() != 3 || vt.size() != 3) {
-	LOGINFO(("RclConfig::initThrConf: bad thread info vector sizes\n"));
+	LOGINFO("RclConfig::initThrConf: bad thread info vector sizes\n" );
 	goto out;
     }
 
     // Normal case: record info from config
     m_thrConf.clear();
     for (unsigned int i = 0; i < 3; i++) {
-	m_thrConf.push_back(RclPII(vq[i], vt[i]));
+	m_thrConf.push_back({vq[i], vt[i]});
     }
 
 out:
@@ -507,14 +493,13 @@ out:
 	    ") ";
     }
 
-    LOGDEB(("RclConfig::initThrConf: chosen config (ql,nt): %s\n", 
-	    sconf.str().c_str()));
+    LOGDEB("RclConfig::initThrConf: chosen config (ql,nt): "  << (sconf.str()) << "\n" );
 }
 
 pair<int,int> RclConfig::getThrConf(ThrStage who) const
 {
     if (m_thrConf.size() != 3) {
-	LOGERR(("RclConfig::getThrConf: bad data in rclconfig\n"));
+	LOGERR("RclConfig::getThrConf: bad data in rclconfig\n" );
 	return pair<int,int>(-1,-1);
     }
     return m_thrConf[who];
@@ -524,8 +509,7 @@ vector<string> RclConfig::getTopdirs() const
 {
     vector<string> tdl;
     if (!getConfParam("topdirs", &tdl)) {
-	LOGERR(("RclConfig::getTopdirs: no top directories in config or "
-		"bad list format\n"));
+	LOGERR("RclConfig::getTopdirs: no top directories in config or bad list format\n" );
 	return tdl;
     }
 
@@ -536,6 +520,11 @@ vector<string> RclConfig::getTopdirs() const
     return tdl;
 }
 
+const string& RclConfig::getLocaleCharset()
+{
+    return o_localecharset;
+}
+
 // Get charset to be used for transcoding to utf-8 if unspecified by doc
 // For document contents:
 //  If defcharset was set (from the config or a previous call, this
@@ -609,7 +598,7 @@ typedef multiset<SfString, SuffCmp> SuffixStore;
 
 bool RclConfig::inStopSuffixes(const string& fni)
 {
-    LOGDEB2(("RclConfig::inStopSuffixes(%s)\n", fni.c_str()));
+    LOGDEB2("RclConfig::inStopSuffixes("  << (fni) << ")\n" );
     // Beware: both needrecompute() need to be called always hence the
     // bizarre way we do things
     bool needrecompute = m_stpsuffstate.needrecompute();
@@ -618,7 +607,7 @@ bool RclConfig::inStopSuffixes(const string& fni)
 	// Need to initialize the suffixes
         delete STOPSUFFIXES;
 	if ((m_stopsuffixes = new SuffixStore) == 0) {
-	    LOGERR(("RclConfig::inStopSuffixes: out of memory\n"));
+	    LOGERR("RclConfig::inStopSuffixes: out of memory\n" );
 	    return false;
 	}
         // Let the old customisation have priority: if recoll_noindex
@@ -634,7 +623,7 @@ bool RclConfig::inStopSuffixes(const string& fni)
 	     it != stoplist.end(); it++) {
 	    STOPSUFFIXES->insert(SfString(stringtolower(*it)));
 	    if (m_maxsufflen < it->length())
-		m_maxsufflen = it->length();
+		m_maxsufflen = int(it->length());
 	}
     }
 
@@ -645,11 +634,10 @@ bool RclConfig::inStopSuffixes(const string& fni)
     stringtolower(fn);
     SuffixStore::const_iterator it = STOPSUFFIXES->find(fn);
     if (it != STOPSUFFIXES->end()) {
-	LOGDEB2(("RclConfig::inStopSuffixes: Found (%s) [%s]\n", 
-		fni.c_str(), (*it).m_str.c_str()));
+	LOGDEB2("RclConfig::inStopSuffixes: Found ("  << (fni) << ") ["  << ((*it).m_str) << "]\n" );
 	return true;
     } else {
-	LOGDEB2(("RclConfig::inStopSuffixes: not found [%s]\n", fni.c_str()));
+	LOGDEB2("RclConfig::inStopSuffixes: not found ["  << (fni) << "]\n" );
 	return false;
     }
 }
@@ -726,18 +714,18 @@ string RclConfig::getMimeHandlerDef(const string &mtype, bool filtertypes)
         }
         if (!m_restrictMTypes.empty() && 
             !m_restrictMTypes.count(stringtolower(mtype))) {
-            LOGDEB2(("RclConfig::getMimeHandlerDef: not in mime type list\n"));
+            LOGDEB2("RclConfig::getMimeHandlerDef: not in mime type list\n" );
             return hs;
         }
         if (!m_excludeMTypes.empty() && 
             m_excludeMTypes.count(stringtolower(mtype))) {
-            LOGDEB2(("RclConfig::getMimeHandlerDef: in excluded mime list\n"));
+            LOGDEB2("RclConfig::getMimeHandlerDef: in excluded mime list\n" );
             return hs;
         }
     }
 
     if (!mimeconf->get(mtype, hs, "index")) {
-	LOGDEB1(("getMimeHandler: no handler for '%s'\n", mtype.c_str()));
+	LOGDEB1("getMimeHandler: no handler for '"  << (mtype) << "'\n" );
     }
     return hs;
 }
@@ -824,11 +812,11 @@ bool RclConfig::getMissingHelperDesc(string& out) const
 
 void RclConfig::storeMissingHelperDesc(const string &s)
 {
-    string fmiss = path_cat(getConfDir(), "missing");
+    string fmiss = path_cat(getCacheDir(), "missing");
     FILE *fp = fopen(fmiss.c_str(), "w");
     if (fp) {
 	if (s.size() > 0 && fwrite(s.c_str(), s.size(), 1, fp) != 1) {
-            LOGERR(("storeMissingHelperDesc: fwrite failed\n"));
+            LOGERR("storeMissingHelperDesc: fwrite failed\n" );
         }
 	fclose(fp);
     }
@@ -838,7 +826,7 @@ void RclConfig::storeMissingHelperDesc(const string &s)
 // things for speed (theses are used a lot during indexing)
 bool RclConfig::readFieldsConfig(const string& cnferrloc)
 {
-    LOGDEB2(("RclConfig::readFieldsConfig\n"));
+    LOGDEB2("RclConfig::readFieldsConfig\n" );
     m_fields = new ConfStack<ConfSimple>("fields", m_cdirs, true);
     if (m_fields == 0 || !m_fields->ok()) {
 	m_reason = string("No/bad fields file in: ") + cnferrloc;
@@ -856,8 +844,8 @@ bool RclConfig::readFieldsConfig(const string& cnferrloc)
 	ConfSimple attrs;
 	FieldTraits ft;
 	if (!valueSplitAttributes(val, ft.pfx, attrs)) {
-	    LOGERR(("readFieldsConfig: bad config line for [%s]: [%s]\n", 
-		    it->c_str(), val.c_str()));
+	    LOGERR("readFieldsConfig: bad config line for ["  << *it <<
+                   "]: [" << val << "]\n");
 	    return 0;
 	}
 	string tval;
@@ -867,9 +855,11 @@ bool RclConfig::readFieldsConfig(const string& cnferrloc)
 	    ft.boost = atof(tval.c_str());
 	if (attrs.get("pfxonly", tval))
 	    ft.pfxonly = stringToBool(tval);
+	if (attrs.get("noterms", tval))
+	    ft.noterms = stringToBool(tval);
 	m_fldtotraits[stringtolower(*it)] = ft;
-	LOGDEB2(("readFieldsConfig: [%s] -> [%s] %d %.1f\n", 
-		it->c_str(), ft.pfx.c_str(), ft.wdfinc, ft.boost));
+	LOGDEB2("readFieldsConfig: ["  << *it << "] -> ["  << ft.pfx <<
+                "] " << ft.wdfinc << " " << ft.boost << "\n");
     }
 
     // Add prefixes for aliases and build alias-to-canonic map while
@@ -915,9 +905,8 @@ bool RclConfig::readFieldsConfig(const string& cnferrloc)
 #if 0
     for (map<string, FieldTraits>::const_iterator it = m_fldtotraits.begin();
 	 it != m_fldtotraits.end(); it++) {
-	LOGDEB(("readFieldsConfig: [%s] -> [%s] %d %.1f\n", 
-		it->c_str(), it->second.pfx.c_str(), it->second.wdfinc, 
-		it->second.boost));
+	LOGDEB("readFieldsConfig: ["  << *it << "] -> ["  << it->second.pfx <<
+               "] " << it->second.wdfinc << " "  << it->second.boost << "\n");
     }
 #endif
 
@@ -950,12 +939,10 @@ bool RclConfig::getFieldTraits(const string& _fld, const FieldTraits **ftpp,
     map<string, FieldTraits>::const_iterator pit = m_fldtotraits.find(fld);
     if (pit != m_fldtotraits.end()) {
 	*ftpp = &pit->second;
-	LOGDEB1(("RclConfig::getFieldTraits: [%s]->[%s]\n", 
-		 _fld.c_str(), pit->second.pfx.c_str()));
+	LOGDEB1("RclConfig::getFieldTraits: ["  << (_fld) << "]->["  << (pit->second.pfx) << "]\n" );
 	return true;
     } else {
-	LOGDEB1(("RclConfig::getFieldTraits: no prefix for field [%s]\n",
-		 fld.c_str()));
+	LOGDEB1("RclConfig::getFieldTraits: no prefix for field ["  << (fld) << "]\n" );
 	*ftpp = 0;
 	return false;
     }
@@ -977,11 +964,10 @@ string RclConfig::fieldCanon(const string& f) const
     string fld = stringtolower(f);
     map<string, string>::const_iterator it = m_aliastocanon.find(fld);
     if (it != m_aliastocanon.end()) {
-	LOGDEB1(("RclConfig::fieldCanon: [%s] -> [%s]\n", 
-		 f.c_str(), it->second.c_str()));
+	LOGDEB1("RclConfig::fieldCanon: ["  << (f) << "] -> ["  << (it->second) << "]\n" );
 	return it->second;
     }
-    LOGDEB1(("RclConfig::fieldCanon: [%s] -> [%s]\n", f.c_str(), fld.c_str()));
+    LOGDEB1("RclConfig::fieldCanon: ["  << (f) << "] -> ["  << (fld) << "]\n" );
     return fld;
 }
 
@@ -990,8 +976,7 @@ string RclConfig::fieldQCanon(const string& f) const
     string fld = stringtolower(f);
     map<string, string>::const_iterator it = m_aliastoqcanon.find(fld);
     if (it != m_aliastoqcanon.end()) {
-	LOGDEB1(("RclConfig::fieldQCanon: [%s] -> [%s]\n", 
-                f.c_str(), it->second.c_str()));
+	LOGDEB1("RclConfig::fieldQCanon: ["  << (f) << "] -> ["  << (it->second) << "]\n" );
 	return it->second;
     }
     return fieldCanon(f);
@@ -1037,8 +1022,7 @@ bool RclConfig::setMimeViewerAllEx(const string& allex)
 string RclConfig::getMimeViewerDef(const string &mtype, const string& apptag,
 				   bool useall) const
 {
-    LOGDEB2(("RclConfig::getMimeViewerDef: mtype [%s] apptag [%s]\n",
-	     mtype.c_str(), apptag.c_str()));
+    LOGDEB2("RclConfig::getMimeViewerDef: mtype ["  << (mtype) << "] apptag ["  << (apptag) << "]\n" );
     string hs;
     if (mimeview == 0)
 	return hs;
@@ -1141,38 +1125,98 @@ string RclConfig::getMimeIconPath(const string &mtype, const string &apptag)
     return path_cat(iconpath, iconname) + ".png";
 }
 
+// Return path defined by varname. May be absolute or relative to
+// confdir, with default in confdir
+string RclConfig::getConfdirPath(const char *varname, const char *dflt) const
+{
+    string result;
+    if (!getConfParam(varname, result)) {
+	result = path_cat(getConfDir(), dflt);
+    } else {
+	result = path_tildexpand(result);
+	// If not an absolute path, compute relative to config dir
+	if (!path_isabsolute(result)) {
+	    result = path_cat(getConfDir(), result);
+	}
+    }
+    return path_canon(result);
+
+}
+
+string RclConfig::getCacheDir() const
+{
+    return m_cachedir.empty() ? getConfDir() : m_cachedir;
+}
+
+// Return path defined by varname. May be absolute or relative to
+// confdir, with default in confdir
+string RclConfig::getCachedirPath(const char *varname, const char *dflt) const
+{
+    string result;
+    if (!getConfParam(varname, result)) {
+	result = path_cat(getCacheDir(), dflt);
+    } else {
+	result = path_tildexpand(result);
+	// If not an absolute path, compute relative to cache dir
+	if (!path_isabsolute(result)) {
+	    result = path_cat(getCacheDir(), result);
+	}
+    }
+    return path_canon(result);
+}
+
 string RclConfig::getDbDir() const
 {
-    string dbdir;
-    if (!getConfParam("dbdir", dbdir)) {
-	LOGERR(("RclConfig::getDbDir: no db directory in configuration\n"));
-    } else {
-	dbdir = path_tildexpand(dbdir);
-	// If not an absolute path, compute relative to config dir
-	if (dbdir.at(0) != '/') {
-	    LOGDEB1(("Dbdir not abs, catting with confdir\n"));
-	    dbdir = path_cat(getConfDir(), dbdir);
+    return getCachedirPath("dbdir", "xapiandb");
 }
+string RclConfig::getWebcacheDir() const
+{
+    return getCachedirPath("webcachedir", "webcache");
 }
-    LOGDEB1(("RclConfig::getDbDir: dbdir: [%s]\n", dbdir.c_str()));
-    return path_canon(dbdir);
+string RclConfig::getMboxcacheDir() const
+{
+    return getCachedirPath("mboxcachedir", "mboxcache");
+}
+string RclConfig::getAspellcacheDir() const
+{
+    return getCachedirPath("aspellDicDir", "");
+}
+
+string RclConfig::getStopfile() const
+{
+    return getConfdirPath("stoplistfile", "stoplist.txt");
+}
+
+string RclConfig::getSynGroupsFile() const
+{
+    return getConfdirPath("syngroupsfile", "syngroups.txt");
+}
+
+// The index status file is fast changing, so it's possible to put it outside
+// of the config directory (for ssds, not sure this is really useful).
+// To enable being quite xdg-correct we should add a getRundirPath()
+string RclConfig::getIdxStatusFile() const
+{
+    return getCachedirPath("idxstatusfile", "idxstatus.txt");
+}
+string RclConfig::getPidfile() const
+{
+    return path_cat(getCacheDir(), "index.pid");
 }
 
 void RclConfig::urlrewrite(const string& dbdir, string& url) const
 {
-    LOGDEB2(("RclConfig::urlrewrite: dbdir [%s] url [%s]\n",
-	    dbdir.c_str(), url.c_str()));
+    LOGDEB2("RclConfig::urlrewrite: dbdir ["  << (dbdir) << "] url ["  << (url) << "]\n" );
 
     // Do path translations exist for this index ?
     if (m_ptrans == 0 || !m_ptrans->hasSubKey(dbdir)) {
-	LOGDEB2(("RclConfig::urlrewrite: no paths translations (m_ptrans %p)\n",
-		m_ptrans));
+	LOGDEB2("RclConfig::urlrewrite: no paths translations (m_ptrans "  << (m_ptrans) << ")\n" );
 	return;
     }
 
     string path = fileurltolocalpath(url);
     if (path.empty()) {
-	LOGDEB2(("RclConfig::urlrewrite: not file url\n"));
+	LOGDEB2("RclConfig::urlrewrite: not file url\n" );
 	return;
     }
 
@@ -1186,7 +1230,7 @@ void RclConfig::urlrewrite(const string& dbdir, string& url) const
 	    // This call always succeeds because the key comes from getNames()
 	    if (m_ptrans->get(*it, npath, dbdir)) { 
 		path = path.replace(0, it->size(), npath);
-		url = "file://" + path;
+		url = path_pathtofileurl(path);
 	    }
 	    break;
 	}
@@ -1210,32 +1254,6 @@ bool RclConfig::sourceChanged() const
     return false;
 }
 
-string RclConfig::getStopfile() const
-{
-    return path_cat(getConfDir(), "stoplist.txt");
-}
-string RclConfig::getPidfile() const
-{
-    return path_cat(getConfDir(), "index.pid");
-}
-
-// The index status file is fast changing, so it's possible to put it outside
-// of the config directory (for ssds, not sure this is really useful).
-string RclConfig::getIdxStatusFile() const
-{
-    string path;
-    if (!getConfParam("idxstatusfile", path)) {
-	return path_cat(getConfDir(), "idxstatus.txt");
-    } else {
-	path = path_tildexpand(path);
-	// If not an absolute path, compute relative to config dir
-	if (path.at(0) != '/') {
-	    path = path_cat(getConfDir(), path);
-	}
-	return path_canon(path);
-    }
-}
-
 string RclConfig::getWebQueueDir() const
 {
     string webqueuedir;
@@ -1263,6 +1281,9 @@ vector<string> RclConfig::getSkippedPaths() const
     // don't do this.
     skpl.push_back(getDbDir());
     skpl.push_back(getConfDir());
+    if (getCacheDir().compare(getConfDir())) {
+        skpl.push_back(getCacheDir());
+    }
     // And the web queue dir
     skpl.push_back(getWebQueueDir());
     for (vector<string>::iterator it = skpl.begin(); it != skpl.end(); it++) {
@@ -1300,46 +1321,46 @@ vector<string> RclConfig::getDaemSkippedPaths() const
 }
 
 
-// Look up an executable filter.  We look in $RECOLL_FILTERSDIR,
-// filtersdir in config file, then let the system use the PATH
+// Look up an executable filter.  We add $RECOLL_FILTERSDIR,
+// and filtersdir from the config file to the PATH, then use execmd::which()
 string RclConfig::findFilter(const string &icmd) const
 {
     // If the path is absolute, this is it
-    if (icmd[0] == '/')
+    if (path_isabsolute(icmd))
 	return icmd;
 
+    const char *cp = getenv("PATH");
+    if (!cp) //??
+        cp = "";
+    string PATH(cp);
+
+    // For historical reasons: check in personal config directory
+    PATH = getConfDir() + path_PATHsep() + PATH;
+
+    string temp;
+    // Prepend $datadir/filters
+    temp = path_cat(m_datadir, "filters");
+    PATH = temp + path_PATHsep() + PATH;
+
+    // Prepend possible configuration parameter?
+    if (getConfParam(string("filtersdir"), temp)) {
+        temp = path_tildexpand(temp);
+        PATH = temp + path_PATHsep() + PATH;
+    }
+
+    // Prepend possible environment variable
+    if ((cp = getenv("RECOLL_FILTERSDIR"))) {
+        PATH = string(cp) + path_PATHsep() + PATH;
+    } 
+
     string cmd;
-    const char *cp;
-
-    // Filters dir from environment ?
-    if ((cp = getenv("RECOLL_FILTERSDIR"))) {
-	cmd = path_cat(cp, icmd);
-	if (access(cmd.c_str(), X_OK) == 0)
+    if (ExecCmd::which(icmd, cmd, PATH.c_str())) {
         return cmd;
-    } 
-    // Filters dir as configuration parameter?
-    if (getConfParam(string("filtersdir"), cmd)) {
-	cmd = path_cat(cmd, icmd);
-	if (access(cmd.c_str(), X_OK) == 0)
-	    return cmd;
-    } 
-
-    // Filters dir as datadir subdir. Actually the standard case, but
-    // this is normally the same value found in config file (previous step)
-    cmd = path_cat(m_datadir, "filters");
-    cmd = path_cat(cmd, icmd);
-    if (access(cmd.c_str(), X_OK) == 0)
-	return cmd;
-
-    // Last resort for historical reasons: check in personal config
-    // directory
-    cmd = path_cat(getConfDir(), icmd);
-    if (access(cmd.c_str(), X_OK) == 0)
-	return cmd;
-
+    } else {
         // Let the shell try to find it...
         return icmd;
     }
+}
 
 /** 
  * Return decompression command line for given mime type
@@ -1354,7 +1375,7 @@ bool RclConfig::getUncompressor(const string &mtype, vector<string>& cmd) const
     vector<string> tokens;
     stringToStrings(hs, tokens);
     if (tokens.empty()) {
-	LOGERR(("getUncompressor: empty spec for mtype %s\n", mtype.c_str()));
+	LOGERR("getUncompressor: empty spec for mtype "  << (mtype) << "\n" );
 	return false;
     }
     vector<string>::iterator it = tokens.begin();
@@ -1363,7 +1384,23 @@ bool RclConfig::getUncompressor(const string &mtype, vector<string>& cmd) const
     if (stringlowercmp("uncompress", *it++)) 
 	return false;
     cmd.clear();
-    cmd.push_back(findFilter(*it++));
+    cmd.push_back(findFilter(*it));
+
+    // Special-case python and perl on windows: we need to also locate the
+    // first argument which is the script name "python somescript.py". 
+    // On Unix, thanks to #!, we usually just run "somescript.py", but need
+    // the same change if we ever want to use the same cmdling as windows
+    if (!stringlowercmp("python", *it) || !stringlowercmp("perl", *it)) {
+        it++;
+        if (tokens.size() < 3) {
+            LOGERR("getUncpressor: python/perl cmd: no script?. ["  << (mtype) << "]\n" );
+        } else {
+            *it = findFilter(*it);
+        }
+    } else {
+        it++;
+    }
+    
     cmd.insert(cmd.end(), it, tokens.end());
     return true;
 }
@@ -1399,7 +1436,7 @@ bool RclConfig::initUserConfig()
 
     // Use protective 700 mode to create the top configuration
     // directory: documents can be reconstructed from index data.
-    if (access(m_confdir.c_str(), 0) < 0 && 
+    if (!path_exists(m_confdir) && 
 	mkdir(m_confdir.c_str(), 0700) < 0) {
 	m_reason += string("mkdir(") + m_confdir + ") failed: " + 
 	    strerror(errno);
@@ -1408,7 +1445,7 @@ bool RclConfig::initUserConfig()
     string lang = localelang();
     for (int i = 0; i < ncffiles; i++) {
 	string dst = path_cat(m_confdir, string(configfiles[i])); 
-	if (access(dst.c_str(), 0) < 0) {
+	if (!path_exists(dst)) {
 	    FILE *fp = fopen(dst.c_str(), "w");
 	    if (fp) {
 		fprintf(fp, "%s\n", blurb);
@@ -1451,6 +1488,7 @@ void RclConfig::initFrom(const RclConfig& r)
 	return;
     m_reason = r.m_reason;
     m_confdir = r.m_confdir;
+    m_cachedir = r.m_cachedir;
     m_datadir = r.m_datadir;
     m_keydir = r.m_keydir;
     m_cdirs = r.m_cdirs;
@@ -1476,16 +1514,21 @@ void RclConfig::initFrom(const RclConfig& r)
     m_maxsufflen = r.m_maxsufflen;
     m_defcharset = r.m_defcharset;
 
-    m_oldstpsuffstate.init(mimemap);
-    m_stpsuffstate.init(m_conf);
-    m_skpnstate.init(m_conf);
-    m_rmtstate.init(m_conf);
-    m_xmtstate.init(m_conf);
-    m_mdrstate.init(m_conf);
+    initParamStale(m_conf, mimemap);
 
     m_thrConf = r.m_thrConf;
 }
 
+void RclConfig::initParamStale(ConfNull *cnf, ConfNull *mimemap)
+{
+    m_oldstpsuffstate.init(mimemap);
+    m_stpsuffstate.init(cnf);
+    m_skpnstate.init(cnf);
+    m_rmtstate.init(cnf);
+    m_xmtstate.init(cnf);
+    m_mdrstate.init(cnf);
+}
+
 #else // -> Test
 
 #include <stdio.h>
@@ -1497,7 +1540,8 @@ void RclConfig::initFrom(const RclConfig& r)
 
 using namespace std;
 
-#include "debuglog.h"
+#include "log.h"
+
 #include "rclinit.h"
 #include "rclconfig.h"
 #include "cstr.h"
@@ -1672,3 +1716,4 @@ int main(int argc, char **argv)
 }
 
 #endif // TEST_RCLCONFIG
+

src/common/rclconfig.h

@@ -16,13 +16,14 @@
  */
 #ifndef _RCLCONFIG_H_INCLUDED_
 #define _RCLCONFIG_H_INCLUDED_
+#include "autoconfig.h"
 
 #include <string>
 #include <vector>
 #include <set>
 #include <utility>
 #include <map>
-#include "unordered_defs.h"
+#include <unordered_set>
 
 using std::string;
 using std::vector;
@@ -30,7 +31,6 @@ using std::pair;
 using std::set;
 using std::map;
 
-
 #include "conftree.h"
 #include "smallut.h"
 
@@ -66,9 +66,9 @@ struct FieldTraits {
     int    wdfinc; // Index time term frequency increment (default 1)
     double boost; // Query time boost (default 1.0)
     bool   pfxonly; // Suppress prefix-less indexing
-
+    bool   noterms; // Don't add term to highlight data (e.g.: rclbes)
     FieldTraits() 
-        : wdfinc(1), boost(1.0), pfxonly(false)
+        : wdfinc(1), boost(1.0), pfxonly(false), noterms(false)
         {}
 };
 
@@ -109,6 +109,7 @@ class RclConfig {
      *  constructor it it is the default one (~/.recoll) and it did 
      *  not exist yet. */
     string getConfDir() const {return m_confdir;}
+    string getCacheDir() const;
 
     /** Check if the config files were modified since we read them */
     bool sourceChanged() const;
@@ -171,10 +172,17 @@ class RclConfig {
      * need for other status */
     vector<string> getTopdirs() const;
 
-    /** Get database directory */
+    string getConfdirPath(const char *varname, const char *dflt) const;
+    string getCachedirPath(const char *varname, const char *dflt) const;
+    /** Get database and other directories */
     string getDbDir() const;
+    string getWebcacheDir() const;
+    string getMboxcacheDir() const;
+    string getAspellcacheDir() const;
     /** Get stoplist file name */
     string getStopfile() const;
+    /** Get synonym groups file name */
+    string getSynGroupsFile() const;
     /** Get indexing pid file name */
     string getPidfile() const;
     /** Get indexing status file name */
@@ -231,6 +239,9 @@ class RclConfig {
     static bool valueSplitAttributes(const string& whole, string& value, 
 				     ConfSimple& attrs) ;
 
+    /** Return the locale's character set */
+    static const std::string& getLocaleCharset();
+    
     /** Return icon path for mime type and tag */
     string getMimeIconPath(const string &mt, const string& apptag) const;
 
@@ -307,7 +318,7 @@ class RclConfig {
     string findFilter(const string& cmd) const;
 
     /** Thread config init is not done automatically because not all
-	programs need it and it uses debuglog so that it's better to
+	programs need it and it uses the debug log so that it's better to
 	call it after primary init */
     void initThrConf();
 
@@ -330,6 +341,11 @@ class RclConfig {
     int m_ok;
     string m_reason;    // Explanation for bad state
     string m_confdir;   // User directory where the customized files are stored
+    // Normally same as confdir. Set to store all bulk data elsewhere.
+    // Provides defaults top location for dbdir, webcachedir,
+    // mboxcachedir, aspellDictDir, which can still be used to
+    // override.
+    string m_cachedir;  
     string m_datadir;   // Example: /usr/local/share/recoll
     string m_keydir;    // Current directory used for parameter fetches.
     int    m_keydirgen; // To help with knowing when to update computed data.
@@ -365,10 +381,10 @@ class RclConfig {
     static string o_localecharset;
     // Limiting set of mime types to be processed. Normally empty.
     ParamStale    m_rmtstate;
-    STD_UNORDERED_SET<string>   m_restrictMTypes; 
+    std::unordered_set<string>   m_restrictMTypes; 
     // Exclusion set of mime types. Normally empty
     ParamStale    m_xmtstate;
-    STD_UNORDERED_SET<string>   m_excludeMTypes; 
+    std::unordered_set<string>   m_excludeMTypes; 
 
     vector<pair<int, int> > m_thrConf;
 
@@ -379,6 +395,8 @@ class RclConfig {
 
     /** Create initial user configuration */
     bool initUserConfig();
+    /** Init all ParamStale members */
+    void initParamStale(ConfNull *cnf, ConfNull *mimemap);
     /** Copy from other */
     void initFrom(const RclConfig& r);
     /** Init pointers to 0 */

src/common/rclinit.cpp

@@ -17,49 +17,61 @@
 #include "autoconfig.h"
 
 #include <stdio.h>
+#ifdef _WIN32
+#include "safewindows.h"
+#endif
 #include <signal.h>
 #include <locale.h>
-#include <pthread.h>
 #include <cstdlib>
 #if !defined(PUTENV_ARG_CONST)
 #include <string.h>
 #endif
 
-#include "debuglog.h"
+#include <thread>
+
+#include "log.h"
 #include "rclconfig.h"
 #include "rclinit.h"
 #include "pathut.h"
+#include "rclutil.h"
 #include "unac.h"
 #include "smallut.h"
 #include "execmd.h"
 
-static const int catchedSigs[] = {SIGINT, SIGQUIT, SIGTERM, SIGUSR1, SIGUSR2};
+std::thread::id mainthread_id;
 
-static pthread_t mainthread_id;
+// Signal etc. processing. We want to be able to properly close the
+// index if we are currently writing to it.
+//
+// This is active if the sigcleanup parameter to recollinit is set,
+// which only recollindex does. We arrange for the handler to be
+// called when process termination is requested either by the system
+// or a user keyboard intr.
+//
+// The recollindex handler just sets a global termination flag (plus
+// the cancelcheck thing), which are tested in all timeout loops
+// etc. When the flag is seen, the main thread processing returns, and
+// recollindex calls exit().
+//
+// The other parameter, to recollinit(), cleanup, is set as an
+// atexit() routine, it does the job of actually signalling the
+// workers to stop and tidy up. It's automagically called by exit().
 
+#ifndef _WIN32
 static void siglogreopen(int)
 {
     if (recoll_ismainthread())
-	DebugLog::reopen();
+        Logger::getTheLog("")->reopen("");
 }
 
-RclConfig *recollinit(RclInitFlags flags, 
-		      void (*cleanup)(void), void (*sigcleanup)(int), 
-		      string &reason, const string *argcnf)
-{
-    if (cleanup)
-	atexit(cleanup);
-
-    // We ignore SIGPIPE always. All pieces of code which can write to a pipe
-    // must check write() return values.
-    signal(SIGPIPE, SIG_IGN);
-    
-    // Make sure the locale is set. This is only for converting file names 
-    // to utf8 for indexing.
-    setlocale(LC_CTYPE, "");
-
 // We would like to block SIGCHLD globally, but we can't because
 // QT uses it. Have to block it inside execmd.cpp
+static const int catchedSigs[] = {SIGINT, SIGQUIT, SIGTERM, SIGUSR1, SIGUSR2};
+void initAsyncSigs(void (*sigcleanup)(int))
+{
+    // We ignore SIGPIPE always. All pieces of code which can write to a pipe
+    // must check write() return values.
+    signal(SIGPIPE, SIG_IGN);
 
     // Install app signal handler
     if (sigcleanup) {
@@ -75,46 +87,6 @@ RclConfig *recollinit(RclInitFlags flags,
 	    }
     }
 
-    DebugLog::getdbl()->setloglevel(DEBDEB1);
-    DebugLog::setfilename("stderr");
-    if (getenv("RECOLL_LOGDATE"))
-        DebugLog::getdbl()->logdate(1);
-
-    RclConfig *config = new RclConfig(argcnf);
-    if (!config || !config->ok()) {
-	reason = "Configuration could not be built:\n";
-	if (config)
-	    reason += config->getReason();
-	else
-	    reason += "Out of memory ?";
-	return 0;
-    }
-
-    // Retrieve the log file name and level
-    string logfilename, loglevel;
-    if (flags & RCLINIT_DAEMON) {
-	config->getConfParam(string("daemlogfilename"), logfilename);
-	config->getConfParam(string("daemloglevel"), loglevel);
-    }
-    if (logfilename.empty())
-	config->getConfParam(string("logfilename"), logfilename);
-    if (loglevel.empty())
-	config->getConfParam(string("loglevel"), loglevel);
-
-    // Initialize logging
-    if (!logfilename.empty()) {
-	logfilename = path_tildexpand(logfilename);
-	// If not an absolute path or , compute relative to config dir
-	if (logfilename.at(0) != '/' && 
-	    !DebugLog::DebugLog::isspecialname(logfilename.c_str())) {
-	    logfilename = path_cat(config->getConfDir(), logfilename);
-	}
-	DebugLog::setfilename(logfilename.c_str());
-    }
-    if (!loglevel.empty()) {
-	int lev = atoi(loglevel.c_str());
-	DebugLog::getdbl()->setloglevel(lev);
-    }
     // Install log rotate sig handler
     {
 	struct sigaction action;
@@ -127,18 +99,228 @@ RclConfig *recollinit(RclInitFlags flags,
 	    }
 	}
     }
+}
+void recoll_exitready()
+{
+}
+
+#else // _WIN32 ->
+
+// Windows signals etc.
+//
+// ^C can be caught by the signal() emulation, but not ^Break
+// apparently, which is why we use the native approach too
+//
+// When a keyboard interrupt occurs, windows creates a thread inside
+// the process and calls the handler. The process exits when the
+// handler returns or after at most 10S
+//
+// This should also work, with different signals (CTRL_LOGOFF_EVENT,
+// CTRL_SHUTDOWN_EVENT) when the user exits or the system shuts down).
+//
+// Unfortunately, this is not the end of the story. It seems that in
+// recent Windows version "some kinds" of apps will not reliably
+// receive the signals. "Some kind" is variably defined, for example a
+// simple test program works when built with vs 2015, but not
+// mingw. See the following discussion thread for tentative
+// explanations, it seems that importing or not from user32.dll is the
+// determining factor.
+// https://social.msdn.microsoft.com/Forums/windowsdesktop/en-US/abf09824-4e4c-4f2c-ae1e-5981f06c9c6e/windows-7-console-application-has-no-way-of-trapping-logoffshutdown-event?forum=windowscompatibility
+// In any case, it appears that the only reliable way to be advised of
+// system shutdown or user exit is to create an "invisible window" and
+// process window messages, which we now do.
+
+static void (*l_sigcleanup)(int);
+static HANDLE eWorkFinished = INVALID_HANDLE_VALUE;
+
+static BOOL WINAPI CtrlHandler(DWORD fdwCtrlType)
+{
+    LOGDEB("CtrlHandler\n" );
+    if (l_sigcleanup == 0)
+        return FALSE;
+
+    switch(fdwCtrlType) { 
+    case CTRL_C_EVENT: 
+    case CTRL_CLOSE_EVENT: 
+    case CTRL_BREAK_EVENT: 
+    case CTRL_LOGOFF_EVENT: 
+    case CTRL_SHUTDOWN_EVENT:
+    {
+        l_sigcleanup(SIGINT);
+        LOGDEB0("CtrlHandler: waiting for exit ready\n" );
+	DWORD res = WaitForSingleObject(eWorkFinished, INFINITE);
+	if (res != WAIT_OBJECT_0) {
+            LOGERR("CtrlHandler: exit ack wait failed\n" );
+	}
+        LOGDEB0("CtrlHandler: got exit ready event, exiting\n" );
+        return TRUE;
+    }
+    default: 
+        return FALSE; 
+    } 
+} 
+
+LRESULT CALLBACK MainWndProc(HWND hwnd , UINT msg , WPARAM wParam,
+                             LPARAM lParam)
+{
+    switch (msg) {
+    case WM_QUERYENDSESSION:
+    case WM_ENDSESSION:
+    case WM_DESTROY:
+    case WM_CLOSE:
+    {
+        l_sigcleanup(SIGINT);
+        LOGDEB("MainWndProc: got end message, waiting for work finished\n" );
+	DWORD res = WaitForSingleObject(eWorkFinished, INFINITE);
+	if (res != WAIT_OBJECT_0) {
+            LOGERR("MainWndProc: exit ack wait failed\n" );
+	}
+        LOGDEB("MainWindowProc: got exit ready event, exiting\n" );
+        return TRUE;
+    }
+    default:
+        return DefWindowProc(hwnd, msg, wParam, lParam);
+    }
+    return TRUE;
+}
+
+bool CreateInvisibleWindow()
+{
+    HWND hwnd;
+    WNDCLASS wc = {0};
+
+    wc.lpfnWndProc = (WNDPROC)MainWndProc;
+    wc.hInstance = GetModuleHandle(NULL);
+    wc.hIcon = LoadIcon(GetModuleHandle(NULL), "TestWClass");
+    wc.lpszClassName = "TestWClass";
+    RegisterClass(&wc);
+
+    hwnd =
+        CreateWindowEx(0, "TestWClass", "TestWClass", WS_OVERLAPPEDWINDOW,
+                       CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT,
+                       CW_USEDEFAULT, (HWND) NULL, (HMENU) NULL,
+                       GetModuleHandle(NULL), (LPVOID) NULL);
+    if (!hwnd) {
+        return FALSE;
+    }
+    return TRUE;
+}
+
+DWORD WINAPI RunInvisibleWindowThread(LPVOID lpParam)
+{
+    MSG msg;
+    CreateInvisibleWindow();
+    while (GetMessage(&msg, (HWND) NULL , 0 , 0)) {
+        TranslateMessage(&msg);
+        DispatchMessage(&msg);
+    }
+    return 0;
+}
+
+static const int catchedSigs[] = {SIGINT, SIGTERM};
+void initAsyncSigs(void (*sigcleanup)(int))
+{
+    DWORD tid;
+    // Install app signal handler
+    if (sigcleanup) {
+        l_sigcleanup = sigcleanup;
+	for (unsigned int i = 0; i < sizeof(catchedSigs) / sizeof(int); i++) {
+	    if (signal(catchedSigs[i], SIG_IGN) != SIG_IGN) {
+		signal(catchedSigs[i], sigcleanup);
+	    }
+        }
+    }
+    HANDLE hInvisiblethread =
+        CreateThread(NULL, 0, RunInvisibleWindowThread, NULL, 0, &tid);
+    SetConsoleCtrlHandler((PHANDLER_ROUTINE)CtrlHandler, TRUE);
+    eWorkFinished = CreateEvent(NULL, TRUE, FALSE, NULL);
+    if (eWorkFinished == INVALID_HANDLE_VALUE) {
+        LOGERR("initAsyncSigs: error creating exitready event\n" );
+    }
+}
+void recoll_exitready()
+{
+    LOGDEB("recoll_exitready()\n" );
+    if (!SetEvent(eWorkFinished)) {
+        LOGERR("recoll_exitready: SetEvent failed\n" );
+    }
+}
+
+#endif
+
+RclConfig *recollinit(RclInitFlags flags, 
+		      void (*cleanup)(void), void (*sigcleanup)(int), 
+		      string &reason, const string *argcnf)
+{
+    if (cleanup)
+	atexit(cleanup);
+
+    // Make sure the locale is set. This is only for converting file names 
+    // to utf8 for indexing.
+    setlocale(LC_CTYPE, "");
+
+    Logger::getTheLog("")->setLogLevel(Logger::LLDEB1);
+
+    initAsyncSigs(sigcleanup);
+    
+    RclConfig *config = new RclConfig(argcnf);
+    if (!config || !config->ok()) {
+	reason = "Configuration could not be built:\n";
+	if (config)
+	    reason += config->getReason();
+	else
+	    reason += "Out of memory ?";
+	return 0;
+    }
+
+    // Retrieve the log file name and level. Daemon and batch indexing
+    // processes may use specific values, else fall back on common
+    // ones.
+    string logfilename, loglevel;
+    if (flags & RCLINIT_DAEMON) {
+	config->getConfParam(string("daemlogfilename"), logfilename);
+	config->getConfParam(string("daemloglevel"), loglevel);
+    }
+    if ((flags & RCLINIT_IDX) && logfilename.empty())
+	config->getConfParam(string("idxlogfilename"), logfilename);
+    if ((flags & RCLINIT_IDX) && loglevel.empty()) 
+	config->getConfParam(string("idxloglevel"), loglevel);
+
+    if (logfilename.empty())
+	config->getConfParam(string("logfilename"), logfilename);
+    if (loglevel.empty())
+	config->getConfParam(string("loglevel"), loglevel);
+
+    // Initialize logging
+    if (!logfilename.empty()) {
+	logfilename = path_tildexpand(logfilename);
+	// If not an absolute path or , compute relative to config dir
+	if (!path_isabsolute(logfilename) &&
+            logfilename.compare("stderr")) {
+	    logfilename = path_cat(config->getConfDir(), logfilename);
+	}
+        Logger::getTheLog("")->reopen(logfilename);
+    }
+    if (!loglevel.empty()) {
+	int lev = atoi(loglevel.c_str());
+        Logger::getTheLog("")->setLogLevel(Logger::LogLevel(lev));
+    }
 
     // Make sure the locale charset is initialized (so that multiple
     // threads don't try to do it at once).
     config->getDefCharset();
 
-    mainthread_id = pthread_self();
+    mainthread_id = std::this_thread::get_id();
 
-    // Init unac locking
-    unac_init_mt();
     // Init smallut and pathut static values
     pathut_init_mt();
     smallut_init_mt();
+    rclutil_init_mt();
+    
+    // Init execmd.h static PATH and PATHELT splitting
+    {string bogus;
+        ExecCmd::which("nosuchcmd", bogus);
+    }
     
     // Init Unac translation exceptions
     string unacex;
@@ -151,23 +333,24 @@ RclConfig *recollinit(RclInitFlags flags,
     // Keep threads init behind log init, but make sure it's done before
     // we do the vfork choice ! The latter is not used any more actually, 
     // we always use vfork except if forbidden by config.
+    if ((flags & RCLINIT_IDX)) {
         config->initThrConf();
+    }
 
     bool novfork;
     config->getConfParam("novfork", &novfork);
     if (novfork) {
-	LOGDEB0(("rclinit: will use fork() for starting commands\n"));
+	LOGDEB0("rclinit: will use fork() for starting commands\n" );
         ExecCmd::useVfork(false);
     } else {
-	LOGDEB0(("rclinit: will use vfork() for starting commands\n"));
+	LOGDEB0("rclinit: will use vfork() for starting commands\n" );
 	ExecCmd::useVfork(true);
     }
 #endif
 
     int flushmb;
     if (config->getConfParam("idxflushmb", &flushmb) && flushmb > 0) {
-	LOGDEB1(("rclinit: idxflushmb=%d, set XAPIAN_FLUSH_THRESHOLD to 10E6\n",
-		 flushmb));
+	LOGDEB1("rclinit: idxflushmb="  << (flushmb) << ", set XAPIAN_FLUSH_THRESHOLD to 10E6\n" );
 	static const char *cp = "XAPIAN_FLUSH_THRESHOLD=1000000";
 #ifdef PUTENV_ARG_CONST
 	::putenv(cp);
@@ -179,10 +362,11 @@ RclConfig *recollinit(RclInitFlags flags,
     return config;
 }
 
-// Signals are handled by the main thread. All others should call this routine
-// to block possible signals
+// Signals are handled by the main thread. All others should call this
+// routine to block possible signals
 void recoll_threadinit()
 {
+#ifndef _WIN32
     sigset_t sset;
     sigemptyset(&sset);
 
@@ -190,11 +374,19 @@ void recoll_threadinit()
 	sigaddset(&sset, catchedSigs[i]);
     sigaddset(&sset, SIGHUP);
     pthread_sigmask(SIG_BLOCK, &sset, 0);
+#else
+    // Not sure that this is needed at all or correct under windows.
+    for (unsigned int i = 0; i < sizeof(catchedSigs) / sizeof(int); i++) {
+        if (signal(catchedSigs[i], SIG_IGN) != SIG_IGN) {
+            signal(catchedSigs[i], SIG_IGN);
+        }
+    }
+#endif
 }
 
 bool recoll_ismainthread()
 {
-    return pthread_equal(pthread_self(), mainthread_id);
+    return std::this_thread::get_id() == mainthread_id;
 }
 
 

src/common/rclinit.h

@@ -18,9 +18,6 @@
 #define _RCLINIT_H_INCLUDED_
 
 #include <string>
-#ifndef NO_NAMESPACES
-using std::string;
-#endif
 
 class RclConfig;
 /**
@@ -30,7 +27,8 @@ class RclConfig;
  * up the global signal handling. other threads must call recoll_threadinit()
  * when starting.
  *
- * @param flags   misc modifiers
+ * @param flags   misc modifiers. These are currently only used to customize
+ *      the log file and verbosity.
  * @param cleanup function to call before exiting (atexit)
  * @param sigcleanup function to call on terminal signal (INT/HUP...) This
  *       should typically set a flag which tells the program (recoll,
@@ -41,12 +39,14 @@ class RclConfig;
  *               default and environment
  * @return the parsed configuration.
  */
-enum RclInitFlags {RCLINIT_NONE=0, RCLINIT_DAEMON=1};
+enum RclInitFlags {RCLINIT_NONE = 0, RCLINIT_DAEMON = 1, RCLINIT_IDX = 2};
 extern RclConfig *recollinit(RclInitFlags flags,
                              void (*cleanup)(void), void (*sigcleanup)(int),
-			     string &reason, const string *argcnf = 0);
+                             std::string& reason, const string *argcnf = 0);
 inline RclConfig *recollinit(void (*cleanup)(void), void (*sigcleanup)(int),
-			     string &reason, const string *argcnf = 0) {
+                             std::string& reason,
+                             const std::string *argcnf = 0)
+{
     return recollinit(RCLINIT_NONE, cleanup, sigcleanup, reason, argcnf);
 }
 
@@ -57,4 +57,9 @@ extern void recoll_threadinit();
 // Check if main thread
 extern bool recoll_ismainthread();
 
+// Should be called while exiting asap when critical cleanup (db
+// close) has been performed. Only useful for the indexer (writes to
+// the db), and only actually does something on Windows.
+extern void recoll_exitready();
+
 #endif /* _RCLINIT_H_INCLUDED_ */

src/common/syngroups.cpp

@@ -0,0 +1,250 @@
+/* Copyright (C) 2014 J.F.Dockes
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation; either version 2 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the
+ *   Free Software Foundation, Inc.,
+ *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ */
+#ifndef TEST_SYNGROUPS
+#include "autoconfig.h"
+
+#include "syngroups.h"
+
+#include "log.h"
+#include "smallut.h"
+
+#include <errno.h>
+#include <unordered_map>
+#include <fstream>
+#include <iostream>
+#include <cstring>
+
+using namespace std;
+
+// Note that we are storing each term twice. I don't think that the
+// size could possibly be a serious issue, but if it was, we could
+// reduce the storage a bit by storing (short hash)-> vector<int>
+// correspondances in the direct map, and then checking all the
+// resulting groups for the input word.
+//
+// As it is, a word can only index one group (the last it is found
+// in). It can be part of several groups though (appear in
+// expansions). I really don't know what we should do with multiple
+// groups anyway
+class SynGroups::Internal {
+public:
+    Internal() : ok(false) {
+    }
+    bool ok;
+    // Term to group num 
+    std::unordered_map<string, unsigned int> terms;
+    // Group num to group
+    vector<vector<string> > groups;
+};
+
+bool SynGroups::ok() 
+{
+    return m && m->ok;
+}
+
+SynGroups::~SynGroups()
+{
+    delete m;
+}
+
+SynGroups::SynGroups()
+    : m(new Internal)
+{
+}
+
+bool SynGroups::setfile(const string& fn)
+{
+    LOGDEB("SynGroups::setfile("  << (fn) << ")\n" );
+    if (!m) {
+        m = new Internal;
+        if (!m) {
+            LOGERR("SynGroups:setfile:: new Internal failed: no mem ?\n" );
+            return false;
+        }
+    }
+
+    if (fn.empty()) {
+        delete m;
+        m = 0;
+	return true;
+    }
+
+    ifstream input;
+    input.open(fn.c_str(), ios::in);
+    if (!input.is_open()) {
+	LOGERR("SynGroups:setfile:: could not open "  << (fn) << " errno "  << (errno) << "\n" );
+	return false;
+    }	    
+
+    string cline;
+    bool appending = false;
+    string line;
+    bool eof = false;
+    int lnum = 0;
+
+    for (;;) {
+        cline.clear();
+	getline(input, cline);
+	if (!input.good()) {
+	    if (input.bad()) {
+                LOGERR("Syngroup::setfile("  << (fn) << "):Parse: input.bad()\n" );
+		return false;
+	    }
+	    // Must be eof ? But maybe we have a partial line which
+	    // must be processed. This happens if the last line before
+	    // eof ends with a backslash, or there is no final \n
+            eof = true;
+	}
+	lnum++;
+
+        {
+            string::size_type pos = cline.find_last_not_of("\n\r");
+            if (pos == string::npos) {
+                cline.clear();
+            } else if (pos != cline.length()-1) {
+                cline.erase(pos+1);
+            }
+        }
+
+	if (appending)
+	    line += cline;
+	else
+	    line = cline;
+
+	// Note that we trim whitespace before checking for backslash-eol
+	// This avoids invisible whitespace problems.
+	trimstring(line);
+	if (line.empty() || line.at(0) == '#') {
+            if (eof)
+                break;
+	    continue;
+	}
+	if (line[line.length() - 1] == '\\') {
+	    line.erase(line.length() - 1);
+	    appending = true;
+	    continue;
+	}
+	appending = false;
+
+	vector<string> words;
+	if (!stringToStrings(line, words)) {
+	    LOGERR("SynGroups:setfile: "  << (fn) << ": bad line "  << (lnum) << ": "  << (line) << "\n" );
+	    continue;
+	}
+
+	if (words.empty())
+	    continue;
+	if (words.size() == 1) {
+	    LOGERR("Syngroup::setfile("  << (fn) << "):single term group at line "  << (lnum) << " ??\n" );
+	    continue;
+	}
+
+	m->groups.push_back(words);
+	for (vector<string>::const_iterator it = words.begin();
+	     it != words.end(); it++) {
+	    m->terms[*it] = m->groups.size()-1;
+	}
+	LOGDEB1("SynGroups::setfile: group: ["  << (stringsToString(m->groups.back())) << "]\n" );
+    }
+    m->ok = true;
+    return true;
+}
+
+vector<string> SynGroups::getgroup(const string& term)
+{
+    vector<string> ret;
+    if (!ok())
+	return ret;
+
+    std::unordered_map<string, unsigned int>::const_iterator it1 =
+        m->terms.find(term);
+    if (it1 == m->terms.end()) {
+	LOGDEB1("SynGroups::getgroup: ["  << (term) << "] not found in direct map\n" );
+	return ret;
+    }
+
+    unsigned int idx = it1->second;
+    if (idx >= m->groups.size()) {
+        LOGERR("SynGroups::getgroup: line index higher than line count !\n" );
+        return ret;
+    }
+    return m->groups[idx];
+}
+
+#else
+
+#include "syngroups.h"
+#include "log.h"
+
+
+#include <string>
+#include <iostream>
+#include <vector>
+#include <cstdlib>
+#include <cstdio>
+
+using namespace std;
+
+static char *thisprog;
+
+static char usage [] =
+    "syngroups <synfilename> <word>\n"
+    "  \n\n"
+    ;
+static void Usage(void)
+{
+    fprintf(stderr, "%s: usage:\n%s", thisprog, usage);
+    exit(1);
+}
+
+static int     op_flags;
+#define OPT_MOINS 0x1
+#define OPT_s     0x2
+#define OPT_b     0x4
+
+int main(int argc, char **argv)
+{
+    thisprog = argv[0];
+    argc--; argv++;
+
+    if (argc != 2) {
+        Usage();
+    }
+    string fn = *argv++;argc--;
+    string word = *argv++;argc--;
+
+    DebugLog::getdbl()->setloglevel(DEBDEB1);
+    DebugLog::setfilename("stderr");
+    SynGroups syns;
+    syns.setfile(fn);
+    if (!syns.ok()) {
+	cerr << "Initialization failed\n";
+	return 1;
+    }
+
+    vector<string> group = syns.getgroup(word);
+    cout << group.size() << " terms in group\n";
+    for (vector<string>::const_iterator it = group.begin();
+	 it != group.end(); it++) {
+	cout << "[" << *it << "] ";
+    }
+    cout << endl;
+    return 0;
+}
+
+#endif
+

src/common/syngroups.h

@@ -0,0 +1,41 @@
+/* Copyright (C) 2015 J.F.Dockes
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation; either version 2 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the
+ *   Free Software Foundation, Inc.,
+ *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ */
+
+#ifndef _SYNGROUPS_H_INCLUDED_
+#define _SYNGROUPS_H_INCLUDED_
+
+#include <string>
+#include <vector>
+
+// Manage synonym groups. This is very different from stemming and
+// case/diac expansion because there is no reference form: all terms
+// in a group are equivalent.
+class SynGroups {
+public:
+    SynGroups();
+    ~SynGroups();
+    bool setfile(const std::string& fname);
+    std::vector<std::string> getgroup(const std::string& term);
+    bool ok();
+private:
+    class Internal;
+    Internal *m;
+    SynGroups(const SynGroups&);
+    SynGroups& operator=(const SynGroups&);
+};
+
+#endif /* _SYNGROUPS_H_INCLUDED_ */

src/common/textsplit.cpp

@@ -24,15 +24,15 @@
 #include <string>
 #include <algorithm>
 #include <cstring>
-#include "unordered_defs.h"
-using namespace std;
+#include <unordered_set>
 
 #include "textsplit.h"
-#include "debuglog.h"
+#include "log.h"
 //#define UTF8ITER_CHECK
 #include "utf8iter.h"
 #include "uproplist.h"
 
+using namespace std;
 
 /**
  * Splitting a text into words. The code in this file works with utf-8
@@ -62,9 +62,9 @@ static int charclasses[charclasses_size];
 // management of Unicode properties, but seems to do the job well
 // enough in most common cases
 static vector<unsigned int> vpuncblocks;
-static STD_UNORDERED_SET<unsigned int> spunc;
-static STD_UNORDERED_SET<unsigned int> visiblewhite;
-static STD_UNORDERED_SET<unsigned int> sskip;
+static std::unordered_set<unsigned int> spunc;
+static std::unordered_set<unsigned int> visiblewhite;
+static std::unordered_set<unsigned int> sskip;
 
 class CharClassInit {
 public:
@@ -142,6 +142,8 @@ static inline int whatcc(unsigned int c)
 	} else {
 	    vector<unsigned int>::iterator it = 
 		lower_bound(vpuncblocks.begin(), vpuncblocks.end(), c);
+		if (it == vpuncblocks.end())
+			return LETTER;
 	    if (c == *it)
 		return SPACE;
 	    if ((it - vpuncblocks.begin()) % 2 == 1) {
@@ -212,15 +214,16 @@ bool TextSplit::isCJK(int c)
 bool          TextSplit::o_processCJK = true;
 unsigned int  TextSplit::o_CJKNgramLen = 2;
 bool          TextSplit::o_noNumbers = false;
+bool          TextSplit::o_deHyphenate = false;
 
 // Final term checkpoint: do some checking (the kind which is simpler
 // to do here than in the main loop), then send term to our client.
 inline bool TextSplit::emitterm(bool isspan, string &w, int pos, 
-				int btstart, int btend)
+				size_t btstart, size_t btend)
 {
-    LOGDEB2(("TextSplit::emitterm: [%s] pos %d\n", w.c_str(), pos));
+    LOGDEB2("TextSplit::emitterm: ["  << (w) << "] pos "  << (pos) << "\n" );
 
-    unsigned int l = w.length();
+    int l = int(w.length());
 
 #ifdef TEXTSPLIT_STATS
     // Update word length statistics. Do this before we filter out
@@ -229,7 +232,7 @@ inline bool TextSplit::emitterm(bool isspan, string &w, int pos,
 	m_stats.newsamp(m_wordChars);
 #endif
 
-    if (l > 0 && l < (unsigned)m_maxWordLength) {
+    if (l > 0 && l < m_maxWordLength) {
 	// 1 byte word: we index single ascii letters and digits, but
 	// nothing else. We might want to turn this into a test for a
 	// single utf8 character instead ?
@@ -244,12 +247,12 @@ inline bool TextSplit::emitterm(bool isspan, string &w, int pos,
 	    }
 	}
 	if (pos != m_prevpos || l != m_prevlen) {
-	    bool ret = takeword(w, pos, btstart, btend);
+	    bool ret = takeword(w, pos, int(btstart), int(btend));
 	    m_prevpos = pos;
-	    m_prevlen = w.length();
+	    m_prevlen = int(w.length());
 	    return ret;
 	}
-	LOGDEB2(("TextSplit::emitterm:dup: [%s] pos %d\n", w.c_str(), pos));
+	LOGDEB2("TextSplit::emitterm:dup: ["  << (w) << "] pos "  << (pos) << "\n" );
     }
     return true;
 }
@@ -292,7 +295,7 @@ bool TextSplit::span_is_acronym(string *acronym)
 
 // Generate terms from span. Have to take into account the
 // flags: ONLYSPANS, NOSPANS, noNumbers
-bool TextSplit::words_from_span(int bp)
+bool TextSplit::words_from_span(size_t bp)
 {
 #if 0
     cerr << "Span: [" << m_span << "] " << " w_i_s size: " << 
@@ -304,18 +307,30 @@ bool TextSplit::words_from_span(int bp)
     }
     cerr << endl;
 #endif
-    unsigned int spanwords = m_words_in_span.size();
+    int spanwords = int(m_words_in_span.size());
     int pos = m_spanpos;
     // Byte position of the span start
-    int spboffs = bp - m_span.size();
+    size_t spboffs = bp - m_span.size();
 
-    for (unsigned int i = 0; 
+    if (o_deHyphenate && spanwords == 2 && 
+	m_span[m_words_in_span[0].second] == '-') {
+	unsigned int s0 = m_words_in_span[0].first;
+	unsigned int l0 = m_words_in_span[0].second - m_words_in_span[0].first;
+	unsigned int s1 = m_words_in_span[1].first;
+	unsigned int l1 = m_words_in_span[1].second - m_words_in_span[1].first;
+	string word = m_span.substr(s0, l0) + m_span.substr(s1, l1);
+	if (l0 && l1) 
+	    emitterm(false, word,
+		     m_spanpos, spboffs, spboffs + m_words_in_span[1].second);
+    }
+
+    for (int i = 0; 
          i < ((m_flags&TXTS_ONLYSPANS) ? 1 : spanwords); 
-         i++, pos++) {
+         i++) {
 
         int deb = m_words_in_span[i].first;
-
-        for (unsigned int j = ((m_flags&TXTS_ONLYSPANS) ? spanwords-1 : i);
+        bool noposinc = m_words_in_span[i].second == deb;
+        for (int j = ((m_flags&TXTS_ONLYSPANS) ? spanwords-1 : i);
              j < ((m_flags&TXTS_NOSPANS) ? i+1 : spanwords);
              j++) {
 
@@ -328,6 +343,8 @@ bool TextSplit::words_from_span(int bp)
             if (!emitterm(j != i+1, word, pos, spboffs+deb, spboffs+fin))
                 return false;
         }
+        if (!noposinc)
+            ++pos;
     }
     return true;
 }
@@ -349,12 +366,10 @@ bool TextSplit::words_from_span(int bp)
  * @param spanerase Set if the current span is at its end. Process it.
  * @param bp        The current BYTE position in the stream
  */
-inline bool TextSplit::doemit(bool spanerase, int bp)
+inline bool TextSplit::doemit(bool spanerase, size_t _bp)
 {
-    LOGDEB2(("TextSplit::doemit: sper %d bp %d spp %d spanwords %u wS %d wL %d "
-            "inn %d span [%s]\n",
-            spanerase, bp, m_spanpos, m_words_in_span.size(), 
-            m_wordStart, m_wordLen, m_inNumber, m_span.c_str()));
+    int bp = int(_bp);
+    LOGDEB2("TextSplit::doemit: sper "  << (spanerase) << " bp "  << (bp) << " spp "  << (m_spanpos) << " spanwords "  << (m_words_in_span.size()) << " wS "  << (m_wordStart) << " wL "  << (m_wordLen) << " inn "  << (m_inNumber) << " span ["  << (m_span) << "]\n" );
 
     if (m_wordLen) {
         // We have a current word. Remember it
@@ -391,8 +406,8 @@ inline bool TextSplit::doemit(bool spanerase, int bp)
 	    case '\'':
 		m_span.resize(m_span.length()-1);
                 if (m_words_in_span.size() &&
-                    m_words_in_span.back().second > m_span.size())
-                    m_words_in_span.back().second = m_span.size();
+                    m_words_in_span.back().second > int(m_span.size()))
+                    m_words_in_span.back().second = int(m_span.size());
 		if (--bp < 0) 
 		    bp = 0;
 		break;
@@ -409,7 +424,7 @@ inline bool TextSplit::doemit(bool spanerase, int bp)
 
     } else {
     
-	m_wordStart = m_span.length();
+	m_wordStart = int(m_span.length());
 
     }
 
@@ -450,12 +465,12 @@ static inline bool isdigit(int what, unsigned int flgs)
  */
 bool TextSplit::text_to_words(const string &in)
 {
-    LOGDEB1(("TextSplit::text_to_words: docjk %d (%d) %s%s%s [%s]\n", 
-	     o_processCJK, o_CJKNgramLen,
-	     m_flags & TXTS_NOSPANS ? " nospans" : "",
-	     m_flags & TXTS_ONLYSPANS ? " onlyspans" : "",
-	     m_flags & TXTS_KEEPWILD ? " keepwild" : "",
-	     in.substr(0,50).c_str()));
+    LOGDEB1("TextSplit::text_to_words: docjk " << o_processCJK << "(" <<
+            o_CJKNgramLen <<  ")" << 
+            (m_flags & TXTS_NOSPANS ? " nospans" : "") << 
+            (m_flags & TXTS_ONLYSPANS ? " onlyspans" : "") << 
+            (m_flags & TXTS_KEEPWILD ? " keepwild" : "") << 
+            "[" << in.substr(0,50) << "]\n");
 
     if (in.empty())
 	return true;
@@ -477,7 +492,7 @@ bool TextSplit::text_to_words(const string &in)
 	nonalnumcnt++;
 
 	if (c == (unsigned int)-1) {
-	    LOGERR(("Textsplit: error occured while scanning UTF-8 string\n"));
+	    LOGERR("Textsplit: error occured while scanning UTF-8 string\n" );
 	    return false;
 	}
 
@@ -491,7 +506,7 @@ bool TextSplit::text_to_words(const string &in)
 
 	    // Hand off situation to the cjk routine.
 	    if (!cjk_to_words(&it, &c)) {
-		LOGERR(("Textsplit: scan error in cjk handler\n"));
+		LOGERR("Textsplit: scan error in cjk handler\n" );
 		return false;
 	    }
 
@@ -626,8 +641,12 @@ bool TextSplit::text_to_words(const string &in)
                     // Check for number like .1
                     if (isdigit(nextwhat, m_flags)) {
                         m_inNumber = true;
-                    }
                         m_wordLen += it.appendchartostring(m_span);
+                    } else {
+                        m_words_in_span.
+                            push_back(pair<int,int>(m_wordStart, m_wordStart));
+                        m_wordStart += it.appendchartostring(m_span);
+                    }
                     STATS_INC_WORDCHARS;
                     break;
                 }
@@ -655,16 +674,16 @@ bool TextSplit::text_to_words(const string &in)
 	    }
 	    break;
 
-	case '#': 
+	case '#':  {
+	    int w = whatcc(it[it.getCpos()+1]);
 	    // Keep it only at the beginning of a word (hashtag), 
-            if (m_wordLen == 0) {
+            if (m_wordLen == 0 && isalphanum(w, m_flags)) {
                 m_wordLen += it.appendchartostring(m_span);
                 STATS_INC_WORDCHARS;
                 break;
             }
             // or at the end (special case for c# ...)
 	    if (m_wordLen > 0) {
-		int w = whatcc(it[it.getCpos()+1]);
 		if (w == SPACE || w == '\n' || w == '\r') {
 		    m_wordLen += it.appendchartostring(m_span);
 		    STATS_INC_WORDCHARS;
@@ -672,6 +691,7 @@ bool TextSplit::text_to_words(const string &in)
 		}
 	    }
 	    goto SPACE;
+	}
 	    break;
 
 	case '\n':
@@ -782,7 +802,7 @@ bool TextSplit::text_to_words(const string &in)
 // be better off converting the whole buffer to utf32 on entry...
 bool TextSplit::cjk_to_words(Utf8Iter *itp, unsigned int *cp)
 {
-    LOGDEB1(("cjk_to_words: m_wordpos %d\n", m_wordpos));
+    LOGDEB1("cjk_to_words: m_wordpos "  << (m_wordpos) << "\n" );
     Utf8Iter &it = *itp;
 
     // We use an offset buffer to remember the starts of the utf-8
@@ -816,16 +836,16 @@ bool TextSplit::cjk_to_words(Utf8Iter *itp, unsigned int *cp)
 	}
 
 	// Take note of byte offset for this character.
-	boffs[nchars-1] = it.getBpos();
+	boffs[nchars-1] = int(it.getBpos());
 
 	// Output all new ngrams: they begin at each existing position
 	// and end after the new character. onlyspans->only output
 	// maximum words, nospans=> single chars
 	if (!(m_flags & TXTS_ONLYSPANS) || nchars == o_CJKNgramLen) {
-	    unsigned int btend = it.getBpos() + it.getBlen();
-	    unsigned int loopbeg = (m_flags & TXTS_NOSPANS) ? nchars-1 : 0;
-	    unsigned int loopend = (m_flags & TXTS_ONLYSPANS) ? 1 : nchars;
-	    for (unsigned int i = loopbeg; i < loopend; i++) {
+	    int btend = int(it.getBpos() + it.getBlen());
+	    int loopbeg = (m_flags & TXTS_NOSPANS) ? nchars-1 : 0;
+	    int loopend = (m_flags & TXTS_ONLYSPANS) ? 1 : nchars;
+	    for (int i = loopbeg; i < loopend; i++) {
 		if (!takeword(it.buffer().substr(boffs[i], 
 						       btend-boffs[i]),
 				m_wordpos - (nchars-i-1), boffs[i], btend)) {
@@ -846,7 +866,7 @@ bool TextSplit::cjk_to_words(Utf8Iter *itp, unsigned int *cp)
     // If onlyspans is set, there may be things to flush in the buffer
     // first
     if ((m_flags & TXTS_ONLYSPANS) && nchars > 0 && nchars != o_CJKNgramLen)  {
-	unsigned int btend = it.getBpos(); // Current char is out
+	int btend = int(it.getBpos()); // Current char is out
 	if (!takeword(it.buffer().substr(boffs[0], btend-boffs[0]),
 			    m_wordpos - nchars,
 			    boffs[0], btend)) {
@@ -885,9 +905,8 @@ bool TextSplit::hasVisibleWhite(const string &in)
     Utf8Iter it(in);
     for (; !it.eof(); it++) {
 	unsigned int c = (unsigned char)*it;
-	LOGDEB3(("TextSplit::hasVisibleWhite: testing 0x%04x\n", c));
 	if (c == (unsigned int)-1) {
-	    LOGERR(("hasVisibleWhite: error while scanning UTF-8 string\n"));
+	    LOGERR("hasVisibleWhite: error while scanning UTF-8 string\n" );
 	    return false;
 	}
 	if (visiblewhite.find(c) != visiblewhite.end())
@@ -908,10 +927,8 @@ template <class T> bool u8stringToStrings(const string &s, T &tokens)
 	unsigned int c = *it;
 	if (visiblewhite.find(c) != visiblewhite.end()) 
 	    c = ' ';
-	LOGDEB3(("TextSplit::stringToStrings: 0x%04x\n", c));
 	if (c == (unsigned int)-1) {
-	    LOGERR(("TextSplit::stringToStrings: error while "
-		    "scanning UTF-8 string\n"));
+	    LOGERR("TextSplit::stringToStrings: error while scanning UTF-8 string\n" );
 	    return false;
 	}
 
@@ -988,7 +1005,8 @@ bool TextSplit::stringToStrings(const string &s, vector<string> &tokens)
 
 #include "textsplit.h"
 #include "readfile.h"
-#include "debuglog.h"
+#include "log.h"
+
 #include "transcode.h"
 #include "unacpp.h"
 #include "termproc.h"
@@ -1147,8 +1165,6 @@ int main(int argc, char **argv)
 	    }
     b1: argc--; argv++;
     }
-    DebugLog::getdbl()->setloglevel(DEBDEB1);
-    DebugLog::setfilename("stderr");
 
     TextSplit::Flags flags = TextSplit::TXTS_NONE;
 
@@ -1221,3 +1237,4 @@ int main(int argc, char **argv)
     }    
 }
 #endif // TEST
+

src/common/textsplit.h

@@ -22,10 +22,6 @@
 #include <string>
 #include <vector>
 
-using std::string;
-using std::vector;
-using std::pair;
-
 class Utf8Iter;
 
 /** 
@@ -56,6 +52,13 @@ public:
 	o_noNumbers = true;
     }
 
+    // Given [co-worker] as input, do we also generate [coworker] ?
+    // Set by rclconfig
+    static bool o_deHyphenate;
+    static void deHyphenate(bool on) {
+	o_deHyphenate = on;
+    }
+
     enum Flags {
         // Default: will return spans and words (a_b, a, b)
         TXTS_NONE = 0, 
@@ -77,10 +80,10 @@ public:
     virtual ~TextSplit() {}
 
     /** Split text, emit words and positions. */
-    virtual bool text_to_words(const string &in);
+    virtual bool text_to_words(const std::string &in);
 
     /** Process one output word: to be implemented by the actual user class */
-    virtual bool takeword(const string& term, 
+    virtual bool takeword(const std::string& term, 
 			  int pos,  // term pos
 			  int bts,  // byte offset of first char in term
 			  int bte   // byte offset of first char after term
@@ -96,10 +99,10 @@ public:
     // Static utility functions:
 
     /** Count words in string, as the splitter would generate them */
-    static int countWords(const string &in, Flags flgs = TXTS_ONLYSPANS);
+    static int countWords(const std::string &in, Flags flgs = TXTS_ONLYSPANS);
 
     /** Check if this is visibly not a single block of text */
-    static bool hasVisibleWhite(const string &in);
+    static bool hasVisibleWhite(const std::string &in);
 
     /** Split text span into strings, at white space, allowing for substrings
      * quoted with " . Escaping with \ works as usual inside the quoted areas.
@@ -108,7 +111,7 @@ public:
      * non-utf-8 input (iso-8859 config files work ok). This hopefully
      * handles all Unicode whitespace, but needs correct utf-8 input
      */
-    static bool stringToStrings(const string &s, vector<string> &tokens);
+    static bool stringToStrings(const std::string &s, std::vector<std::string> &tokens);
 
     /** Is char CJK ? */
     static bool isCJK(int c);
@@ -179,9 +182,9 @@ private:
     int           m_maxWordLength;
 
     // Current span. Might be jf.dockes@wanadoo.f
-    string        m_span; 
+    std::string        m_span; 
 
-    vector <pair<unsigned int, unsigned int> > m_words_in_span;
+    std::vector <std::pair<int, int> > m_words_in_span;
 
     // Current word: no punctuation at all in there. Byte offset
     // relative to the current span and byte length
@@ -198,7 +201,7 @@ private:
     // It may happen that our cleanup would result in emitting the
     // same term twice. We try to avoid this
     int           m_prevpos;
-    unsigned int  m_prevlen;
+    int           m_prevlen;
 
 #ifdef TEXTSPLIT_STATS
     // Stats counters. These are processed in TextSplit rather than by a 
@@ -212,11 +215,11 @@ private:
     // This processes cjk text:
     bool cjk_to_words(Utf8Iter *it, unsigned int *cp);
 
-    bool emitterm(bool isspan, string &term, int pos, int bs, int be);
-    bool doemit(bool spanerase, int bp);
+    bool emitterm(bool isspan, std::string &term, int pos, size_t bs,size_t be);
+    bool doemit(bool spanerase, size_t bp);
     void discardspan();
     bool span_is_acronym(std::string *acronym);
-    bool words_from_span(int bp);
+    bool words_from_span(size_t bp);
 };
 
 #endif /* _TEXTSPLIT_H_INCLUDED_ */

src/common/unacpp.cpp

@@ -24,7 +24,7 @@
 
 #include "unacpp.h"
 #include "unac.h"
-#include "debuglog.h"
+#include "log.h"
 #include "utf8iter.h"
 
 bool unacmaybefold(const string &in, string &out, 
@@ -68,7 +68,7 @@ bool unacmaybefold(const string &in, string &out,
 // testing user-entered terms, so we don't really care.
 bool unaciscapital(const string& in)
 {
-    LOGDEB2(("unaciscapital: [%s]\n", in.c_str()));
+    LOGDEB2("unaciscapital: ["  << (in) << "]\n" );
     if (in.empty())
 	return false;
     Utf8Iter it(in);
@@ -77,7 +77,7 @@ bool unaciscapital(const string& in)
 
     string lower;
     if (!unacmaybefold(shorter, lower, "UTF-8", UNACOP_FOLD)) {
-	LOGINFO(("unaciscapital: unac/fold failed for [%s]\n", in.c_str()));
+	LOGINFO("unaciscapital: unac/fold failed for ["  << (in) << "]\n" );
 	return false;
     } 
     Utf8Iter it1(lower);
@@ -88,13 +88,13 @@ bool unaciscapital(const string& in)
 }
 bool unachasuppercase(const string& in)
 {
-    LOGDEB2(("unachasuppercase: [%s]\n", in.c_str()));
+    LOGDEB2("unachasuppercase: ["  << (in) << "]\n" );
     if (in.empty())
 	return false;
 
     string lower;
     if (!unacmaybefold(in, lower, "UTF-8", UNACOP_FOLD)) {
-	LOGINFO(("unachasuppercase: unac/fold failed for [%s]\n", in.c_str()));
+	LOGINFO("unachasuppercase: unac/fold failed for ["  << (in) << "]\n" );
 	return false;
     } 
     if (lower != in)
@@ -104,13 +104,13 @@ bool unachasuppercase(const string& in)
 }
 bool unachasaccents(const string& in)
 {
-    LOGDEB2(("unachasaccents: [%s]\n", in.c_str()));
+    LOGDEB2("unachasaccents: ["  << (in) << "]\n" );
     if (in.empty())
 	return false;
 
     string noac;
     if (!unacmaybefold(in, noac, "UTF-8", UNACOP_UNAC)) {
-	LOGINFO(("unachasaccents: unac/unac failed for [%s]\n", in.c_str()));
+	LOGINFO("unachasaccents: unac/unac failed for ["  << (in) << "]\n" );
 	return false;
     } 
     if (noac != in)
@@ -247,3 +247,4 @@ int main(int argc, char **argv)
 }
 
 #endif
+

src/common/unordered_defs.h

@@ -1,19 +0,0 @@
-
-#include "autoconfig.h"
-
-#ifdef  HAVE_CXX0X_UNORDERED
-#  include <unordered_map>
-#  include <unordered_set>
-#  define STD_UNORDERED_MAP std::unordered_map
-#  define STD_UNORDERED_SET std::unordered_set
-#elif defined(HAVE_TR1_UNORDERED)
-#  include <tr1/unordered_map>
-#  include <tr1/unordered_set>
-#  define STD_UNORDERED_MAP std::tr1::unordered_map
-#  define STD_UNORDERED_SET std::tr1::unordered_set
-#else
-#  include <map>
-#  include <set>
-#  define STD_UNORDERED_MAP std::map
-#  define STD_UNORDERED_SET std::set
-#endif

src/common/utf8fn.cpp

@@ -0,0 +1,24 @@
+#include "utf8fn.h"
+#include "rclconfig.h"
+#include "transcode.h"
+#include "log.h"
+
+using namespace std;
+
+string compute_utf8fn(const RclConfig *config, const string& ifn, bool simple)
+{
+    string charset = config->getDefCharset(true);
+    string utf8fn; 
+    int ercnt;
+    string lfn(simple ? path_getsimple(ifn) : ifn);
+    if (!transcode(lfn, utf8fn, charset, "UTF-8", &ercnt)) {
+	LOGERR("compute_utf8fn: fn transcode failure from ["  << charset <<
+               "] to UTF-8 for: [" << lfn << "]\n");
+    } else if (ercnt) {
+	LOGDEB("compute_utf8fn: "  << ercnt << " transcode errors from [" <<
+               charset << "] to UTF-8 for: ["  << lfn << "]\n");
+    }
+    LOGDEB1("compute_utf8fn: transcoded from ["  << lfn << "] to ["  <<
+            utf8fn << "] ("  << charset << "->"  << "UTF-8)\n");
+    return utf8fn;
+}

src/common/utf8fn.h

@@ -0,0 +1,16 @@
+#ifndef _UTF8FN_H_
+#define _UTF8FN_H_
+
+#include <string>
+
+class RclConfig;
+
+// Translate file name/path to utf8 for indexing.
+// 
+// @param simple If true we extract and process only the simple file name
+// (ignore the path)
+std::string compute_utf8fn(const RclConfig *config, const std::string& ifn,
+                           bool simple);
+
+#endif // _UTF8FN_H_
+

src/configure.ac

@@ -1,34 +1,24 @@
 AC_INIT([Recoll], m4_esyscmd_s(cat VERSION))
 AC_CONFIG_HEADERS([common/autoconfig.h])
+AH_BOTTOM([#include "conf_post.h"])
 AC_PREREQ(2.53)
 AC_CONFIG_SRCDIR(index/recollindex.cpp)
 
+AM_INIT_AUTOMAKE([1.10 no-define subdir-objects foreign])
+AC_DISABLE_STATIC
+LT_INIT
+AC_CONFIG_MACRO_DIR([m4])
+
 AC_PROG_CXX
 if test C$CXX = C ; then
    AC_MSG_ERROR([C++ compiler needed. Please install one (ie: gnu g++)])
 fi
 
+AC_PROG_YACC
+
+AC_PROG_LIBTOOL
 AC_C_BIGENDIAN
 
-sys=`uname | tr / _ | awk -F_ '{print $1}'`
-
-if test ! -f mk/$sys ; then
-   AC_MSG_NOTICE([
-    No system configuration file found in mk/ for uname = '$sys'. 
-    Trying with Default file. 
-    If the build fails, you'll need to write a configuration file, starting 
-    from one of the existing ones.])
-
-    sys=Default   
-fi
-(cd mk; rm -f sysconf; ln -s $sys sysconf)
-
-# There a few Recoll users on Mac OS X and a few things are just not worth
-# supporting
-if test X$sys = XDarwin ; then
-   NODYNLIB=#
-fi
-
 AC_SYS_LARGEFILE
 
 # OpenBSD needs sys/param.h for mount.h to compile
@@ -52,6 +42,21 @@ AC_CHECK_HEADER(tr1/unordered_map,[AC_DEFINE([HAVE_TR1_UNORDERED],
     [],["Have tr1"])],[])
 AC_CHECK_HEADER(unordered_map,[AC_DEFINE([HAVE_CXX0X_UNORDERED],
     [],["Have C++0x"])],[])
+AC_TRY_COMPILE([
+    #include <memory>
+  ],[
+    std::shared_ptr<int> ptr;
+  ], rcl_shared_ptr_std="1", rcl_shared_ptr_std="0")
+AC_TRY_COMPILE([
+    #include <tr1/memory>
+  ],[
+    std::tr1::shared_ptr<int> ptr;
+  ], rcl_shared_ptr_tr1="1", rcl_shared_ptr_tr1="0")
+if test X$rcl_shared_ptr_std = X1; then 
+   AC_DEFINE(HAVE_SHARED_PTR_STD, [], [Has std::shared_ptr])
+elif test X$rcl_shared_ptr_tr1 = X1; then 
+   AC_DEFINE(HAVE_SHARED_PTR_TR1, [], [Has std::tr1::shared_ptr])
+fi
 AC_LANG_POP([C++])
 
 AC_CHECK_HEADERS([sys/mount.h sys/statfs.h sys/statvfs.h sys/vfs.h], [], [],
@@ -199,12 +204,9 @@ AC_ARG_ENABLE(idxthreads,
     AC_HELP_STRING([--disable-idxthreads],
    [Disable multithread indexing.]),
         idxthreadsEnabled=$enableval, idxthreadsEnabled=yes)
-
+AM_CONDITIONAL(NOTHREADS, [test X$idxthreadsEnabled = Xno])
 if test X$idxthreadsEnabled = Xyes ; then
   AC_DEFINE(IDX_THREADS, 1, [Use multiple threads for indexing])
-  NOTHREADS=""
-else
-  NOTHREADS="#"
 fi
 
 # Enable CamelCase word splitting. This is optional because it causes 
@@ -221,54 +223,27 @@ AC_ARG_ENABLE(camelcase,
    "mysql manual" (in phrases only and you could raise the phrase slack to
    get a match).]),
         camelcaseEnabled=$enableval, camelcaseEnabled=no)
-
 if test X$camelcaseEnabled = Xyes ; then
   AC_DEFINE(RCL_SPLIT_CAMELCASE, 1, [Split camelCase words])
 fi
 
-# Disable building the python module. This is built by default, because
-# it's really the easiest way to interface and extend recoll. It forces PIC
-# objects for everything (indexing performance impact: 1%), because it's
-# just not worth building the lib twice
-# You can still have a non-pic recoll with:
-#   configure --disable-python-module; make; make install;make clean
-#   configure; make; cd python/recoll; make install
-#
+# Disable building the python module.
 if test X$sys != XDarwin ; then
   AC_ARG_ENABLE(python-module,
     AC_HELP_STRING([--disable-python-module],
     [Do not build the Python module.]),
         pythonEnabled=$enableval, pythonEnabled=yes)
-if test X$pythonEnabled = Xyes ; then
-  NOPYTHON=""
 else
-  NOPYTHON="#"
-fi
-else
-  NOPYTHON="#"
+  pythonEnabled=no
 fi
 
-# Build PIC objects for the library ?
-AC_ARG_ENABLE(pic,
-    AC_HELP_STRING([--disable-pic],
-   [Do not compile library objects as position independant code. 
-    This is incompatible with the php or python extensions.]),
-        picEnabled=$enableval, picEnabled=forpython)
-case $picEnabled in
-forpython) picEnabled=$pythonEnabled; NOPIC=$NOPYTHON;;
-yes) NOPIC="";;
-*) NOPIC="#";;
-esac
+AM_CONDITIONAL(MAKEPYTHON, [test X$pythonEnabled = Xyes])
 
-if test X$pythonEnabled = Xyes -a X$picEnabled != Xyes; then
-   AC_MSG_ERROR([Python build needs PIC library])
-fi
-
-if test X$NOPIC != X; then
-    NODYNLIB=#
-fi
 
 AC_CHECK_FUNCS(mkdtemp)
+AC_CHECK_LIB([pthread], [pthread_create], [], [])
+AC_CHECK_LIB([dl], [dlopen], [], [])
+AC_CHECK_LIB([z], [zlibVersion], [], [])
 
 ##### Look for iconv. This can exist in either libc (ie: Linux, solaris) or
 ##### libiconv. We'd need a --with-libiconv= option
@@ -340,7 +315,7 @@ fi
 
 #### Look for Xapian. Done in a strange way to work around autoconf
 # cache
-XAPIAN_CONFIG=no
+XAPIAN_CONFIG=${XAPIAN_CONFIG:-no}
 if test "$XAPIAN_CONFIG" = "no"; then 
     AC_PATH_PROG(XAPIAN_CONFIG0, [xapian-config], no)
     XAPIAN_CONFIG=$XAPIAN_CONFIG0
@@ -373,8 +348,6 @@ for i in $LIBXAPIAN ; do
     esac
 done
 LIBXAPIAN=$tmpxaplib
-# Also recent xapian libs need lz even when they think they don't...
-LIBXAPIAN="$LIBXAPIAN -lz"
 LIBXAPIANDIR=`$XAPIAN_CONFIG --libs | awk '{print $1}'`
 case A"$LIBXAPIANDIR" in
   A-L*) LIBXAPIANDIR=`echo $LIBXAPIANDIR | sed -e 's/-L//'`;;
@@ -388,6 +361,20 @@ XAPIANCXXFLAGS=`$XAPIAN_CONFIG --cxxflags`
 #echo LIBXAPIANSTATICEXTRA: $LIBXAPIANSTATICEXTRA
 #echo XAPIANCXXFLAGS: $XAPIANCXXFLAGS
 
+AC_ARG_ENABLE(xadump, 
+    AC_HELP_STRING([--enable-xadump],
+   [Enable building the xadump low level Xapian access program.]),
+        enableXADUMP=$enableval, enableXADUMP="no")
+AM_CONDITIONAL(MAKEXADUMP, [test X$enableXADUMP = Xyes])
+
+AC_ARG_ENABLE(userdoc,
+    AC_HELP_STRING([--disable-userdoc],
+       [Disable building the user manual. (Avoids the need for docbook xml/xsl files and TeX tools.]),
+        enableUserdoc=$enableval, enableUserdoc="yes")
+AM_CONDITIONAL(MAKEUSERDOC, [test X$enableUserdoc = Xyes])
+
+
+
 #### QT
 # The way qt and its tools (qmake especially) are installed is very
 # different between systems (and maybe qt versions)
@@ -413,13 +400,22 @@ AC_ARG_ENABLE(qtgui,
     AC_HELP_STRING([--disable-qtgui],
    [Disable the QT-based graphical user interface.]),
         enableQT=$enableval, enableQT="yes")
+AM_CONDITIONAL(MAKEQT, [test X$enableQT = Xyes])
 
-if test "$enableQT" != "yes" ; then
-   NOQTMAKE="#"
-   NOCMDLINE=""
+AC_ARG_ENABLE(recollq, 
+    AC_HELP_STRING([--enable-recollq],
+   [Enable building the recollq command line query tool (recoll -t without
+   need for Qt). This is done by default if --disable-qtgui is set but this
+   option enables forcing it.]),
+        enableRECOLLQ=$enableval, enableRECOLLQ="no")
+if test X"$enableRECOLLQ" != X ; then
+    AM_CONDITIONAL(MAKECMDLINE, [test X$enableRECOLLQ = Xyes])
 else
-  NOQTMAKE=""
-  NOCMDLINE="#"
+    AM_CONDITIONAL(MAKECMDLINE, [test X$enableQT = Xno])
+fi
+
+
+if test X$enableQT = Xyes ; then
 
   if test X$QTDIR != X ; then
      PATH=$PATH:$QTDIR/bin
@@ -449,15 +445,13 @@ else
      QMAKE="${QMAKE} -spec macx-g++"
   fi
   
-  # Discriminate qt3/4. Qt3 qmake prints its version on stderr but we don't
-  # depend on this. We try to detect the qt 4 version string instead.
+  # Check Qt version
   qmakevers="`${QMAKE} --version 2>&1`"
   #echo "qmake version: $qmakevers"
   v4=`expr "$qmakevers" : '.*Qt[ ][ ]*version[ ][ ]*4.*'`
   v5=`expr "$qmakevers" : '.*Qt[ ][ ]*version[ ][ ]*5.*'`
   if test X$v4 = X0 -a X$v5 = X0; then 
      AC_MSG_ERROR([qmake seems to be using Qt version 3 which is not supported any more])
-     QTGUI=qtgui
   else
     if test X$v4 != X0 ; then
        AC_MSG_NOTICE([using qt version 4 user interface])
@@ -467,27 +461,6 @@ else
     QTGUI=qtgui
   fi
 
-  cd $QTGUI
-  # We just want a .pro file: no problem with unsubstituted variables at 
-  # this point.
-  test -f recoll.pro && chmod +w recoll.pro
-  cp recoll.pro.in recoll.pro
-  #echo QMAKE ${QMAKE}
-  ${QMAKE} recoll.pro
-  if test $? != 0 ; then
-     AC_MSG_ERROR([Cannot use qmake to generate a Makefile. Maybe you need to
-  check the QTDIR and QMAKESPEC environment variables?])
-  fi
-  # is QTDIR set and do we actually need it ?
-  if test X$QTDIR = X ; then
-     QTDIRNEEDED=`grep INCPATH Makefile | grep = | grep QTDIR`
-     if test "X$QTDIRNEEDED" != "X" ; then
-         AC_MSG_ERROR([You need to set the QTDIR variable to point to the QT
-  installation. If there is no default mkspecs, you should also set QMAKESPEC])
-     fi
-  fi
-  cd ..
-
 
  ##### Using Qt webkit for reslist display? Else Qt textbrowser
   AC_ARG_ENABLE(webkit,
@@ -503,8 +476,6 @@ else
    QMAKE_DISABLE_WEBKIT=""
   fi
 
-
-
  ##### Using QZeitGeist lib ? Default no for now
   AC_ARG_WITH(qzeitgeist,
     AC_HELP_STRING([--with-qzeitgeist],
@@ -551,19 +522,21 @@ fi
 #echo X_CFLAGS "'$X_CFLAGS'" X_PRE_LIBS "'$X_PRE_LIBS'" X_LIBS \
 #      "'$X_LIBS'" X_LIBX11 "'$X_LIBX11'" X_EXTRA_LIBS "'$X_EXTRA_LIBS'"
 
-
-# We have to expand prefix in here, couldn't find a way to do it inside 
-# the qt gui .pro file or Makefile. This just means that you can't change
-# prefix at build time. It works at install time because we dont' use the
-# qtgui Makefile
+# For communicating the value of RECOLL_DATADIR to non-make-based
+# subpackages like python-recoll, we have to expand prefix in here, because
+# things like "datadir = ${prefix}/share" (which is what we'd get by
+# expanding @datadir@) don't mean a thing in Python... I guess we could
+# have a piece of shell-script text to be substituted into and executed by
+# setup.py for getting the value of pkgdatadir, but really...
 m_prefix=$prefix
 test "X$m_prefix" = "XNONE" && m_prefix=/usr/local
 m_datadir=${m_prefix}/share
-QTRECOLL_DATADIR=${m_datadir}/recoll
+RECOLL_DATADIR=${m_datadir}/recoll
 
-RCLVERSION=`cat VERSION`
+RCLVERSION=$PACKAGE_VERSION
 RCLLIBVERSION=$RCLVERSION
 
+AC_SUBST(RECOLL_DATADIR)
 AC_SUBST(X_CFLAGS)
 AC_SUBST(X_PRE_LIBS)
 AC_SUBST(X_LIBS)
@@ -577,20 +550,12 @@ AC_SUBST(LIBXAPIANSTATICEXTRA)
 AC_SUBST(LIBFAM)
 AC_SUBST(QMAKE)
 AC_SUBST(QTGUI)
-AC_SUBST(QTRECOLL_DATADIR)
 AC_SUBST(XAPIANCXXFLAGS)
-AC_SUBST(HAVE_MKDTEMP)
-AC_SUBST(NOQTMAKE)
-AC_SUBST(NOCMDLINE)
 AC_SUBST(QMAKE_ENABLE_WEBKIT)
 AC_SUBST(QMAKE_DISABLE_WEBKIT)
 AC_SUBST(QMAKE_ENABLE_ZEITGEIST)
 AC_SUBST(QMAKE_DISABLE_ZEITGEIST)
 AC_SUBST(LIBQZEITGEIST)
-AC_SUBST(NOPIC)
-AC_SUBST(NOTHREADS)
-AC_SUBST(NOPYTHON)
-AC_SUBST(NODYNLIB)
 AC_SUBST(RCLVERSION)
 AC_SUBST(RCLLIBVERSION)
 
@@ -598,23 +563,7 @@ AC_SUBST(RCLLIBVERSION)
 # changing it unless necessary
 AC_CONFIG_FILES(Makefile)
 AC_CONFIG_FILES(common/rclversion.h)
-AC_CONFIG_FILES(lib/mkMake)
-AC_CONFIG_FILES(mk/localdefs.new:mk/localdefs.in)
 AC_CONFIG_FILES(python/recoll/setup.py)
-AC_CONFIG_FILES(recollinstall)
-AC_CONFIG_FILES(sampleconf/recoll.conf)
-
-for d in bincimapmime index lib query
-do 
-    rm -f $d/alldeps.stamp
-    cp -f /dev/null $d/alldeps
-done
+AC_CONFIG_FILES(python/recoll/Makefile)
 
 AC_OUTPUT
-
-if cmp -s mk/localdefs mk/localdefs.new ; then
-  rm -f mk/localdefs.new
-else
-  mv -f mk/localdefs.new mk/localdefs
-fi
-

src/doc/man/recoll.1

@@ -72,7 +72,11 @@ simple search query.
 .PP
 If 
 .B \-t
-is specified, the Graphical User Interface will not be started, and results
+is specified, or if
+.B recoll
+is called as
+.B recollq
+(through a link), the Graphical User Interface will not be started, and results
 will be printed to the standard output. Additional options understood by
 the 
 .B recollq

src/doc/man/recoll.conf.5

@@ -54,315 +54,565 @@ Where values are lists, white space is used for separation, and elements with
 embedded spaces can be quoted with double-quotes.
 .SH OPTIONS
 .TP
-.BI "topdirs = "  directories
-Specifies the list of directories to index (recursively). 
+.BI "topdirs = "string
+Space-separated list of files or
+directories to recursively index. Default to ~ (indexes
+$HOME). You can use symbolic links in the list, they will be followed,
+independantly of the value of the followLinks variable.
 .TP
-.BI "skippedNames = " patterns
-A space-separated list of patterns for names of files or directories that
-should be completely ignored. The list defined in the default file is:
-.sp
-.nf
-*~ #* bin CVS  Cache caughtspam  tmp
+.BI "skippedNames = "string
+Files and directories which should be ignored. 
+White space separated list of wildcard patterns (simple ones, not paths,
+must contain no / ), which will be tested against file and directory
+names.  The list in the default configuration does not exclude hidden
+directories (names beginning with a dot), which means that it may index
+quite a few things that you do not want. On the other hand, email user
+agents like Thunderbird usually store messages in hidden directories, and
+you probably want this indexed. One possible solution is to have '.*' in
+'skippedNames', and add things like '~/.thunderbird' '~/.evolution' to
+'topdirs'.  Not even the file names are indexed for patterns in this
+list, see the 'noContentSuffixes' variable for an alternative approach
+which indexes the file names. Can be redefined for any
+subtree.
+.TP
+.BI "noContentSuffixes = "string
+List of name endings (not necessarily dot-separated suffixes) for
+which we don't try MIME type identification, and don't uncompress or
+index content. Only the names will be indexed. This
+complements the now obsoleted recoll_noindex list from the mimemap file,
+which will go away in a future release (the move from mimemap to
+recoll.conf allows editing the list through the GUI). This is different
+from skippedNames because these are name ending matches only (not
+wildcard patterns), and the file name itself gets indexed normally. This
+can be redefined for subdirectories.
+.TP
+.BI "skippedPaths = "string
+Paths we should not go into. Space-separated list of
+wildcard expressions for filesystem paths. Can contain files and
+directories. The database and configuration directories will
+automatically be added. The expressions are matched using 'fnmatch(3)'
+with the FNM_PATHNAME flag set by default. This means that '/' characters
+must be matched explicitely. You can set 'skippedPathsFnmPathname' to 0
+to disable the use of FNM_PATHNAME (meaning that '/*/dir3' will match
+'/dir1/dir2/dir3').  The default value contains the usual mount point for
+removable media to remind you that it is a bad idea to have Recoll work
+on these (esp. with the monitor: media gets indexed on mount, all data
+gets erased on unmount).  Explicitely adding '/media/xxx' to the topdirs
+will override this.
+.TP
+.BI "skippedPathsFnmPathname = "bool
+Set to 0 to
+override use of FNM_PATHNAME for matching skipped
+paths. 
+.TP
+.BI "daemSkippedPaths = "string
+skippedPaths equivalent specific to
+real time indexing. This enables having parts of the tree
+which are initially indexed but not monitored. If daemSkippedPaths is
+not set, the daemon uses skippedPaths.
+.TP
+.BI "zipSkippedNames = "string
+Space-separated list of wildcard expressions for names that should
+be ignored inside zip archives. This is used directly by
+the zip handler, and has a function similar to skippedNames, but works
+independantly. Can be redefined for subdirectories. Supported by recoll
+1.20 and newer. See
+https://bitbucket.org/medoc/recoll/wiki/Filtering%20out%20Zip%20archive%20members
 
-.fi
-The list can be redefined for subdirectories, but is only actually changed
-for the top level ones in 
-.I topdirs
 .TP
-.BI "skippedPaths = " patterns
-A space-separated list of patterns for paths the indexer should not descend
-into. Together with topdirs, this allows pruning the indexed tree to one's
-content.
-.B daemSkippedPaths 
-can be used to define a specific value for the real time indexing monitor.
+.BI "followLinks = "bool
+Follow symbolic links during
+indexing. The default is to ignore symbolic links to avoid
+multiple indexing of linked files. No effort is made to avoid duplication
+when this option is set to true. This option can be set individually for
+each of the 'topdirs' members by using sections. It can not be changed
+below the 'topdirs' level. Links in the 'topdirs' list itself are always
+followed.
 .TP
-.BI "skippedPathsFnmPathname = " 0/1
-The values in the *skippedPaths variables are matched by default with
-fnmatch(3), with the FNM_PATHNAME and FNM_LEADING_DIR flags. This means
-that '/' characters must be matched explicitly. You can set
-skippedPathsFnmPathname to 0 to disable the use of FNM_PATHNAME (meaning
-that /*/dir3 will match /dir1/dir2/dir3). 
+.BI "indexedmimetypes = "string
+Restrictive list of
+indexed mime types. Normally not set (in which case all
+supported types are indexed). If it is set,
+only the types from the list will have their contents indexed. The names
+will be indexed anyway if indexallfilenames is set (default). MIME
+type names should be taken from the mimemap file. Can be redefined for
+subtrees.
 .TP
-.BI "followLinks = " boolean
-Specifies if the indexer should follow
-symbolic links while walking the file tree. The default is
-to ignore symbolic links to avoid multiple indexing of
-linked files. No effort is made to avoid duplication when
-this option is set to true. This option can be set
-individually for each of the 
-.I topdirs
-members by using sections. It can not be changed below the
-.I topdirs
-level.
+.BI "excludedmimetypes = "string
+List of excluded MIME
+types. Lets you exclude some types from indexing. Can be
+redefined for subtrees.
 .TP
-.BI "indexedmimetypes = " list
-Recoll normally indexes any file which it knows how to read. This list lets
-you restrict the indexed mime types to what you specify. If the variable is
-unspecified or the list empty (the default), all supported types are
-processed.
+.BI "compressedfilemaxkbs = "int
+Size limit for compressed
+files. We need to decompress these in a
+temporary directory for identification, which can be wasteful in some
+cases. Limit the waste. Negative means no limit. 0 results in no
+processing of any compressed file. Default 50 MB.
 .TP
-.BI "compressedfilemaxkbs = " value
-Size limit for compressed (.gz or .bz2) files. These need to be
-decompressed in a temporary directory for identification, which can be very
-wasteful if 'uninteresting' big compressed files are present.  Negative
-means no limit, 0 means no processing of any compressed file. Defaults 
-to \-1.
+.BI "textfilemaxmbs = "int
+Size limit for text
+files. Mostly for skipping monster
+logs. Default 20 MB.
 .TP
-.BI "textfilemaxmbs = " value
-Maximum size for text files. Very big text files are often uninteresting
-logs. Set to \-1 to disable (default 20MB). 
+.BI "indexallfilenames = "bool
+Index the file names of
+unprocessed files Index the names of files the contents of
+which we don't index because of an excluded or unsupported MIME
+type.
 .TP
-.BI "textfilepagekbs = " value
-If this is set to other than \-1, text files will be indexed as multiple
-documents of the given page size. This may be useful if you do want to
-index very big text files as it will both reduce memory usage at index time
-and help with loading data to the preview window. A size of a few megabytes
-would seem reasonable (default: 1000 : 1MB).
+.BI "usesystemfilecommand = "bool
+Use a system command
+for file MIME type guessing as a final step in file type
+identification This is generally useful, but will usually
+cause the indexing of many bogus 'text' files. See 'systemfilecommand'
+for the command used.
 .TP
-.BI "membermaxkbs = " "value in kilobytes"
-This defines the maximum size for an archive member (zip, tar or rar at
-the moment). Bigger entries will be skipped. Current default: 50000 (50 MB).
+.BI "systemfilecommand = "string
+Command used to guess
+MIME types if the internal methods fails This should be a
+"file -i" workalike.  The file path will be added as a last parameter to
+the command line. 'xdg-mime' works better than the traditional 'file'
+command, and is now the configured default (with a hard-coded fallback to
+'file')
 .TP
-.BI "indexallfilenames = " boolean
-Recoll indexes file names into a special section of the database to allow
-specific file names searches using wild cards. This parameter decides if
-file name indexing is performed only for files with mime types that would
-qualify them for full text indexing, or for all files inside
-the selected subtrees, independent of mime type.
+.BI "processwebqueue = "bool
+Decide if we process the
+Web queue. The queue is a directory where the Recoll Web
+browser plugins create the copies of visited pages.
 .TP
-.BI "usesystemfilecommand = " boolean
-Decide if we use the 
-.B "file \-i"
-system command as a final step for determining the mime type for a file
-(the main procedure uses suffix associations as defined in the 
-.B mimemap 
-file). This can be useful for files with suffixless names, but it will
-also cause the indexing of many bogus "text" files.
+.BI "textfilepagekbs = "int
+Page size for text
+files. If this is set, text/plain files will be divided
+into documents of approximately this size. Will reduce memory usage at
+index time and help with loading data in the preview window at query
+time. Particularly useful with very big files, such as application or
+system logs. Also see textfilemaxmbs and
+compressedfilemaxkbs.
 .TP
-.BI "processbeaglequeue = " 0/1
-If this is set, process the directory where Beagle Web browser plugins copy
-visited pages for indexing. Of course, Beagle MUST NOT be running, else
-things will behave strangely. 
+.BI "membermaxkbs = "int
+Size limit for archive
+members. This is passed to the filters in the environment
+as RECOLL_FILTER_MAXMEMBERKB.
 .TP
-.BI "beaglequeuedir = " directory path
-The path to the Beagle indexing queue. This is hard-coded in the Beagle
-plugin as ~/.beagle/ToIndex so there should be no need to change it. 
-.TP 
-.BI "indexStripChars = " 0/1
-Decide if we strip characters of diacritics and convert them to lower-case
-before terms are indexed. If we don't, searches sensitive to case and
-diacritics can be performed, but the index will be bigger, and some
-marginal weirdness may sometimes occur. The default is a stripped index
-(indexStripChars = 1) for now. When using multiple indexes for a search,
+.BI "indexStripChars = "bool
+Decide if we store
+character case and diacritics in the index. If we do,
+searches sensitive to case and diacritics can be performed, but the index
+will be bigger, and some marginal weirdness may sometimes occur. The
+default is a stripped index. When using multiple indexes for a search,
 this parameter must be defined identically for all. Changing the value
 implies an index reset.
 .TP
-.BI "maxTermExpand = " value
-Maximum expansion count for a single term (e.g.: when using wildcards). The
-default of 10000 is reasonable and will avoid queries that appear frozen
-while the engine is walking the term list. 
+.BI "nonumbers = "bool
+Decides if terms will be
+generated for numbers. For example "123", "1.5e6",
+192.168.1.4, would not be indexed if nonumbers is set ("value123" would
+still be). Numbers are often quite interesting to search for, and this
+should probably not be set except for special situations, ie, scientific
+documents with huge amounts of numbers in them, where setting nonumbers
+will reduce the index size. This can only be set for a whole index, not
+for a subtree.
 .TP
-.BI "maxXapianClauses = " value
-Maximum number of elementary clauses we can add to a single Xapian
-query. In some cases, the result of term expansion can be multiplicative,
-and we want to avoid using excessive memory. The default of 100 000 should
-be both high enough in most cases and compatible with current typical
-hardware configurations. 
+.BI "dehyphenate = "bool
+Determines if we index
+'coworker' also when the input is 'co-worker'. This is new
+in version 1.22, and on by default. Setting the variable to off allows
+restoring the previous behaviour.
 .TP
-.BI "nonumbers = " 0/1
-If this set to true, no terms will be generated for numbers. For example
-"123", "1.5e6", 192.168.1.4, would not be indexed ("value123" would still
-be). Numbers are often quite interesting to search for, and this should
-probably not be set except for special situations, ie, scientific documents
-with huge amounts of numbers in them. This can only be set for a whole
-index, not for a subtree. 
+.BI "nocjk = "bool
+Decides if specific East Asian
+(Chinese Korean Japanese) characters/word splitting is turned
+off. This will save a small amount of CPU if you have no CJK
+documents. If your document base does include such text but you are not
+interested in searching it, setting nocjk may be a
+significant time and space saver.
 .TP
-.BI "nocjk = " boolean
-If this set to true, specific east asian (Chinese Korean Japanese)
-characters/word splitting is turned off. This will save a small amount of
-cpu if you have no CJK documents. If your document base does include such
-text but you are not interested in searching it, setting
-.I nocjk
-may be a significant time and space saver.
+.BI "cjkngramlen = "int
+This lets you adjust the size of
+n-grams used for indexing CJK text. The default value of 2 is
+probably appropriate in most cases. A value of 3 would allow more precision
+and efficiency on longer words, but the index will be approximately twice
+as large.
 .TP
-.BI "cjkngramlen = " value
-This lets you adjust the size of n-grams used for indexing CJK text. The
-default value of 2 is probably appropriate in most cases. A value of 3
-would allow more precision and efficiency on longer words, but the index
-will be approximately twice as large.
+.BI "indexstemminglanguages = "string
+Languages for which to create stemming expansion
+data. Stemmer names can be found by executing 'recollindex
+-l', or this can also be set from a list in the GUI.
 .TP
-.BI "indexstemminglanguages = " languages
-A list of languages for which the stem expansion databases will be
-built. See recollindex(1) for possible values.
+.BI "defaultcharset = "string
+Default character
+set. This is used for files which do not contain a
+character set definition (e.g.: text/plain). Values found inside files,
+e.g. a 'charset' tag in HTML documents, will override it. If this is not
+set, the default character set is the one defined by the NLS environment
+($LC_ALL, $LC_CTYPE, $LANG), or ultimately iso-8859-1 (cp-1252 in fact).
+If for some reason you want a general default which does not match your
+LANG and is not 8859-1, use this variable. This can be redefined for any
+sub-directory.
 .TP
-.BI "defaultcharset = " charset
-The name of the character set used for files that do not contain a
-character set definition (ie: plain text files). This can be redefined for
-any subdirectory.
+.BI "unac_except_trans = "string
+A list of characters,
+encoded in UTF-8, which should be handled specially
+when converting text to unaccented lowercase. For
+example, in Swedish, the letter a with diaeresis has full alphabet
+citizenship and should not be turned into an a.
+Each element in the space-separated list has the special character as
+first element and the translation following. The handling of both the
+lowercase and upper-case versions of a character should be specified, as
+appartenance to the list will turn-off both standard accent and case
+processing. The value is global and affects both indexing and querying.
+Examples:
+Swedish:
+unac_except_trans = ää Ää öö Öö üü Üü ßss œoe Œoe æae Æae ffff fifi flfl åå Åå
+. German:
+unac_except_trans = ää Ää öö Öö üü Üü ßss œoe Œoe æae Æae ffff fifi flfl
+In French, you probably want to decompose oe and ae and nobody would type
+a German ß
+unac_except_trans = ßss œoe Œoe æae Æae ffff fifi flfl
+. The default for all until someone protests follows. These decompositions
+are not performed by unac, but it is unlikely that someone would type the
+composed forms in a search.
+unac_except_trans = ßss œoe Œoe æae Æae ffff fifi flfl
 .TP
-.BI "unac_except_trans = " "list of utf-8 groups"
-This is a list of characters, encoded in UTF-8, which should be handled
-specially when converting text to unaccented lowercase. For example, in
-Swedish, the letter "a with diaeresis" has full alphabet citizenship and
-should not be turned into an a. 
-.br
-Each element in the space-separated list has the special character as first
-element and the translation following. The handling of both the lowercase
-and upper-case versions of a character should be specified, as appartenance
-to the list will turn-off both standard accent and case processing.
-.br
-Note that the translation is not limited to a single character.
-.br
-This parameter cannot be redefined for subdirectories, it is global,
-because there is no way to do otherwise when querying. If you have document
-sets which would need different values, you will have to index and query
-them separately.
+.BI "maildefcharset = "string
+Overrides the default
+character set for email messages which don't specify
+one. This is mainly useful for readpst (libpst) dumps,
+which are utf-8 but do not say so.
 .TP
-.BI "maildefcharset = " character set name
-This can be used to define the default character set specifically for email
-messages which don't specify it. This is mainly useful for readpst (libpst)
-dumps, which are utf-8 but do not say so. 
+.BI "localfields = "string
+Set fields on all files
+(usually of a specific fs area). Syntax is the usual:
+name = value ; attr1 = val1 ; [...]
+value is empty so this needs an initial semi-colon. This is useful, e.g.,
+for setting the rclaptg field for application selection inside
+mimeview.
 .TP
-.BI "localfields = " "fieldname = value:..."
-This allows setting fields for all documents under a given
-directory. Typical usage would be to set an "rclaptg" field, to be used in
-mimeview to select a specific viewer. If several fields are to be set, they
-should be separated with a colon (':') character (which there is currently
-no way to escape). Ie: localfields= rclaptg=gnus:other = val, then select
-specifier viewer with mimetype|tag=... in mimeview. 
+.BI "testmodifusemtime = "bool
+Use mtime instead of
+ctime to test if a file has been modified. The time is used
+in addition to the size, which is always used.
+Setting this can reduce re-indexing on systems where extended attributes
+are used (by some other application), but not indexed, because changing
+extended attributes only affects ctime.
+Notes:
+- This may prevent detection of change in some marginal file rename cases
+(the target would need to have the same size and mtime).
+- You should probably also set noxattrfields to 1 in this case, except if
+you still prefer to perform xattr indexing, for example if the local
+file update pattern makes it of value (as in general, there is a risk
+for pure extended attributes updates without file modification to go
+undetected). Perform a full index reset after changing this.
+
 .TP
-.BI "dbdir = " directory
-The name of the Xapian database directory. It will be created if needed
-when the database is initialized. If this is not an absolute pathname, it
-will be taken relative to the configuration directory.
+.BI "noxattrfields = "bool
+Disable extended attributes
+conversion to metadata fields. This probably needs to be
+set if testmodifusemtime is set.
 .TP
-.BI "idxstatusfile = " "file path"
-The name of the scratch file where the indexer process updates its
-status. Default: idxstatus.txt inside the configuration directory. 
+.BI "metadatacmds = "string
+Define commands to
+gather external metadata, e.g. tmsu tags. 
+There can be several entries, separated by semi-colons, each defining
+which field name the data goes into and the command to use. Don't forget the
+initial semi-colon. All the field names must be different. You can use
+aliases in the "field" file if necessary.
+As a not too pretty hack conceded to convenience, any field name
+beginning with "rclmulti" will be taken as an indication that the command
+returns multiple field values inside a text blob formatted as a recoll
+configuration file ("fieldname = fieldvalue" lines). The rclmultixx name
+will be ignored, and field names and values will be parsed from the data.
+Example: metadatacmds = ; tags = tmsu tags %f; rclmulti1 = cmdOutputsConf %f
+
 .TP
-.BI "maxfsoccuppc = " percentnumber
-Maximum file system occupation before we
-stop indexing. The value is a percentage, corresponding to
-what the "Capacity" df output column shows.  The default
+.BI "cachedir = "dfn
+Top directory for Recoll data. Recoll data
+directories are normally located relative to the configuration directory
+(e.g. ~/.recoll/xapiandb, ~/.recoll/mboxcache). If 'cachedir' is set, the
+directories are stored under the specified value instead (e.g. if
+cachedir is ~/.cache/recoll, the default dbdir would be
+~/.cache/recoll/xapiandb).  This affects dbdir, webcachedir,
+mboxcachedir, aspellDicDir, which can still be individually specified to
+override cachedir.  Note that if you have multiple configurations, each
+must have a different cachedir, there is no automatic computation of a
+subpath under cachedir.
+.TP
+.BI "maxfsoccuppc = "int
+Maximum file system occupation
+over which we stop indexing. The value is a percentage,
+corresponding to what the "Capacity" df output column shows. The default
 value is 0, meaning no checking.
 .TP
-.BI "mboxcachedir = " "directory path"
-The directory where mbox message offsets cache files are held. This is
-normally $RECOLL_CONFDIR/mboxcache, but it may be useful to share a
-directory between different configurations. 
+.BI "xapiandb = "dfn
+Xapian database directory
+location. This will be created on first indexing. If the
+value is not an absolute path, it will be interpreted as relative to
+cachedir if set, or the configuration directory (-c argument or
+$RECOLL_CONFDIR).  If nothing is specified, the default is then
+~/.recoll/xapiandb/
 .TP
-.BI "mboxcacheminmbs = " "value in megabytes"
-The minimum mbox file size over which we cache the offsets. There is really no sense in caching offsets for small files. The default is 5 MB.
+.BI "idxstatusfile = "fn
+Name of the scratch file where the indexer process updates its
+status. Default: idxstatus.txt inside the configuration
+directory.
 .TP
-.BI "webcachedir = " "directory path"
-This is only used by the Beagle web browser plugin indexing code, and
-defines where the cache for visited pages will live. Default:
+.BI "mboxcachedir = "dfn
+Directory location for storing mbox message offsets cache
+files. This is normally 'mboxcache' under cachedir if set,
+or else under the configuration directory, but it may be useful to share
+a directory between different configurations.
+.TP
+.BI "mboxcacheminmbs = "int
+Minimum mbox file size over which we cache the offsets. There is really no sense in caching offsets for small files. The
+default is 5 MB.
+.TP
+.BI "webcachedir = "dfn
+Directory where we store the archived web pages. This is only used by the web history indexing code
+Default: cachedir/webcache if cachedir is set, else
 $RECOLL_CONFDIR/webcache
 .TP
-.BI "webcachemaxmbs = " "value in megabytes"
-This is only used by the Beagle web browser plugin indexing code, and
-defines the maximum size for the web page cache. Default: 40 MB. 
+.BI "webcachemaxmbs = "int
+Maximum size in MB of the Web archive. This is only used by the web history indexing code.
+Default: 40 MB.
+Reducing the size will not physically truncate the file.
 .TP
-.BI "idxflushmb = " megabytes
-Threshold (megabytes of new text data)
-where we flush from memory to disk index. Setting this can
-help control memory usage. A value of 0 means no explicit
-flushing, letting Xapian use its own default, which is
-flushing every 10000 documents (or XAPIAN_FLUSH_THRESHOLD), meaning that
-memory usage depends on average document size. The default value is 10.
+.BI "webqueuedir = "fn
+The path to the Web indexing queue. This is
+hard-coded in the plugin as ~/.recollweb/ToIndex so there should be no
+need or possibility to change it.
 .TP
-.BI "autodiacsens = " 0/1
-IF the index is not stripped, decide if we automatically trigger diacritics
-sensitivity if the search term has accented characters (not in
-unac_except_trans). Else you need to use the query language and the D
-modifier to specify diacritics sensitivity. Default is no. 
+.BI "aspellDicDir = "dfn
+Aspell dictionary storage directory location. The
+aspell dictionary (aspdict.(lang).rws) is normally stored in the
+directory specified by cachedir if set, or under the configuration
+directory.
 .TP
-.BI "autocasesens = " 0/1
-IF the index is not stripped, decide if we automatically trigger character
-case sensitivity if the search term has upper-case characters in any but
-the first position. Else you need to use the query language and the C
-modifier to specify character-case sensitivity. Default is yes. 
+.BI "filtersdir = "dfn
+Directory location for executable input handlers. If
+RECOLL_FILTERSDIR is set in the environment, we use it instead. Defaults
+to $prefix/share/recoll/filters. Can be redefined for
+subdirectories.
 .TP
-.BI "loglevel = " value
-Verbosity level for recoll and recollindex. A value of 4 lists quite a lot of
-debug/information messages. 3 lists only errors. 
-.B daemloglevel
-can be used to specify a different value for the real-time indexing daemon.
+.BI "iconsdir = "dfn
+Directory location for icons. The only reason to
+change this would be if you want to change the icons displayed in the
+result list. Defaults to $prefix/share/recoll/images
 .TP
-.BI "logfilename = " file
-Where should the messages go. 'stderr' can be used as a special value.
-.B daemlogfilename
-can be used to specify a different value for the real-time indexing daemon.
+.BI "idxflushmb = "int
+Threshold (megabytes of new data) where we flush from memory to
+disk index. Setting this allows some control over memory
+usage by the indexer process. A value of 0 means no explicit flushing,
+which lets Xapian perform its own thing, meaning flushing every
+$XAPIAN_FLUSH_THRESHOLD documents created, modified or deleted: as memory
+usage depends on average document size, not only document count, the
+Xapian approach is is not very useful, and you should let Recoll manage
+the flushes.  The default value of idxflushmb is 10 MB, and may be a bit
+low. If you are looking for maximum speed, you may want to experiment
+with values between 20 and
+80. In my experience, values beyond 100 are always counterproductive. If
+you find otherwise, please drop me a note.
 .TP
-.BI "mondelaypatterns = " "list of patterns"
-This allows specify wildcard path patterns (processed with fnmatch(3) with
-0 flag), to match files which change too often and for which a delay should
-be observed before re-indexing. This is a space-separated list, each entry
-being a pattern and a time in seconds, separated by a colon. You can use
-double quotes if a path entry contains white space. Example: 
-.sp
-mondelaypatterns = *.log:20 "this one has spaces*:10"
+.BI "filtermaxseconds = "int
+Maximum external filter execution time in
+seconds. Default 1200 (20mn). Set to 0 for no limit. This
+is mainly to avoid infinite loops in postscript files
+(loop.ps)
 .TP
-.BI "monixinterval = " "value in seconds
-Minimum interval (seconds) for processing the indexing queue. The real time
-monitor does not process each event when it comes in, but will wait this
-time for the queue to accumulate to diminish overhead and in order to
-aggregate multiple events to the same file. Default 30 S. 
+.BI "filtermaxmbytes = "int
+Maximum virtual memory space for filter processes
+(setrlimit(RLIMIT_AS)), in megabytes. Note that this
+includes any mapped libs (there is no reliable Linux way to limit the
+data space only), so we need to be a bit generous here. Anything over
+2000 will be ignored on 32 bits machines.
 .TP
-.BI "monauxinterval = " "value in seconds
-Period (in seconds) at which the real time monitor will regenerate the
-auxiliary databases (spelling, stemming) if needed. The default is one
-hour. 
+.BI "thrQSizes = "string
+Stage input queues configuration. There are three
+internal queues in the indexing pipeline stages (file data extraction,
+terms generation, index update). This parameter defines the queue depths
+for each stage (three integer values). If a value of -1 is given for a
+given stage, no queue is used, and the thread will go on performing the
+next stage. In practise, deep queues have not been shown to increase
+performance. Default: a value of 0 for the first queue tells Recoll to
+perform autoconfiguration based on the detected number of CPUs (no need
+for the two other values in this case).  Use thrQSizes = -1 -1 -1 to
+disable multithreading entirely.
 .TP
-.BI "monioniceclass, monioniceclassdata"
-These allow defining the ionice class and data used by the indexer (default
-class 3, no data). 
+.BI "thrTCounts = "string
+Number of threads used for each indexing stage. The
+three stages are: file data extraction, terms generation, index
+update). The use of the counts is also controlled by some special values
+in thrQSizes: if the first queue depth is 0, all counts are ignored
+(autoconfigured); if a value of -1 is used for a queue depth, the
+corresponding thread count is ignored. It makes no sense to use a value
+other than 1 for the last stage because updating the Xapian index is
+necessarily single-threaded (and protected by a mutex).
 .TP
-.BI "filtermaxseconds = " "value in seconds"
-Maximum filter execution time, after which it is aborted. Some postscript
-programs just loop... 
+.BI "loglevel = "int
+Log file verbosity 1-6. A value of 2 will print
+only errors and warnings. 3 will print information like document updates,
+4 is quite verbose and 6 very verbose.
 .TP
-.BI "filtersdir = " directory
-A directory to search for the external filter scripts used to index some
-types of files. The value should not be changed, except if you want to
-modify one of the default scripts. The value can be redefined for any
-subdirectory. 
+.BI "logfilename = "fn
+Log file destination. Use 'stderr' (default) to write to the
+console. 
 .TP
-.BI "iconsdir = " directory
-The name of the directory where 
-.B recoll
-result list icons are stored. You can change this if you want different
-images.
+.BI "idxloglevel = "int
+Override loglevel for the indexer. 
 .TP
-.BI "idxabsmlen = " value
-Recoll stores an abstract for each indexed file inside the database. The
-text can come from an actual 'abstract' section in the document or will
-just be the beginning of the document. It is stored in the index so that it
-can be displayed inside the result lists without decoding the original
-file. The
-.I idxabsmlen
-parameter defines the size of the stored abstract. The default value is 250
+.BI "idxlogfilename = "fn
+Override logfilename for the indexer. 
+.TP
+.BI "daemloglevel = "int
+Override loglevel for the indexer in real time
+mode. The default is to use the idx... values if set, else
+the log... values.
+.TP
+.BI "daemlogfilename = "fn
+Override logfilename for the indexer in real time
+mode. The default is to use the idx... values if set, else
+the log... values.
+.TP
+.BI "idxrundir = "dfn
+Indexing process current directory. The input
+handlers sometimes leave temporary files in the current directory, so it
+makes sense to have recollindex chdir to some temporary directory. If the
+value is empty, the current directory is not changed. If the
+value is (literal) tmp, we use the temporary directory as set by the
+environment (RECOLL_TMPDIR else TMPDIR else /tmp). If the value is an
+absolute path to a directory, we go there.
+.TP
+.BI "checkneedretryindexscript = "fn
+Script used to heuristically check if we need to retry indexing
+files which previously failed.  The default script checks
+the modified dates on /usr/bin and /usr/local/bin. A relative path will
+be looked up in the filters dirs, then in the path. Use an absolute path
+to do otherwise.
+.TP
+.BI "recollhelperpath = "string
+Additional places to search for helper executables. This is only used on Windows for now.
+.TP
+.BI "idxabsmlen = "int
+Length of abstracts we store while indexing. Recoll stores an abstract for each indexed file.
+The text can come from an actual 'abstract' section in the
+document or will just be the beginning of the document. It is stored in
+the index so that it can be displayed inside the result lists without
+decoding the original file. The idxabsmlen parameter
+defines the size of the stored abstract. The default value is 250
 bytes. The search interface gives you the choice to display this stored
 text or a synthetic abstract built by extracting text around the search
 terms. If you always prefer the synthetic abstract, you can reduce this
 value and save a little space.
 .TP
-.BI "aspellLanguage = " lang
-Language definitions to use when creating the aspell dictionary.  The value
-must match a set of aspell language definition files. You can type "aspell
-config" to see where these are installed (look for data-dir). The default
-if the variable is not set is to use your desktop national language
-environment to guess the value.
+.BI "idxmetastoredlen = "int
+Truncation length of stored metadata fields. This
+does not affect indexing (the whole field is processed anyway), just the
+amount of data stored in the index for the purpose of displaying fields
+inside result lists or previews. The default value is 150 bytes which
+may be too low if you have custom fields.
 .TP
-.BI "noaspell = " boolean
-If this is set, the aspell dictionary generation is turned off. Useful for
-cases where you don't need the functionality or when it is unusable because
-aspell crashes during dictionary generation.
+.BI "aspellLanguage = "string
+Language definitions to use when creating the aspell
+dictionary. The value must match a set of aspell language
+definition files. You can type "aspell dicts"  to see a list The default
+if this is not set is to use the NLS environment to guess the
+value.
 .TP
-.BI "mhmboxquirks = " flags
-This allows definining location-related quirks for the mailbox
-handler. Currently only the tbird flag is defined, and it should be set for
-directories which hold Thunderbird data, as their folder format is weird. 
+.BI "aspellAddCreateParam = "string
+Additional option and parameter to aspell dictionary creation
+command. Some aspell packages may need an additional option
+(e.g. on Debian Jessie: --local-data-dir=/usr/lib/aspell). See Debian bug
+772415.
+.TP
+.BI "aspellKeepStderr = "bool
+Set this to have a look at aspell dictionary creation
+errors. There are always many, so this is mostly for
+debugging.
+.TP
+.BI "noaspell = "bool
+Disable aspell use. The aspell dictionary generation
+takes time, and some combinations of aspell version, language, and local
+terms, result in aspell crashing, so it sometimes makes sense to just
+disable the thing.
+.TP
+.BI "monauxinterval = "int
+Auxiliary database update interval. The real time
+indexer only updates the auxiliary databases (stemdb, aspell)
+periodically, because it would be too costly to do it for every document
+change. The default period is one hour.
+.TP
+.BI "monixinterval = "int
+Minimum interval (seconds) between processings of the indexing
+queue. The real time indexer does not process each event
+when it comes in, but lets the queue accumulate, to diminish overhead and
+to aggregate multiple events affecting the same file. Default 30
+S.
+.TP
+.BI "mondelaypatterns = "string
+Timing parameters for the real time indexing. Definitions for files which get a longer delay before reindexing
+is allowed. This is for fast-changing files, that should only be
+reindexed once in a while. A list of wildcardPattern:seconds pairs. The
+patterns are matched with fnmatch(pattern, path, 0) You can quote entries
+containing white space with double quotes (quote the whole entry, not the
+pattern). The default is empty.
+Example: mondelaypatterns = *.log:20 "*with spaces.*:30"
+.TP
+.BI "monioniceclass = "int
+ionice class for the real time indexing process On platforms where this is supported. The default value is
+3.
+.TP
+.BI "monioniceclassdata = "string
+ionice class parameter for the real time indexing process. On platforms where this is supported. The default is
+empty.
+.TP
+.BI "autodiacsens = "bool
+auto-trigger diacritics sensitivity (raw index only). IF the index is not stripped, decide if we automatically trigger
+diacritics sensitivity if the search term has accented characters (not in
+unac_except_trans). Else you need to use the query language and the "D"
+modifier to specify diacritics sensitivity. Default is no.
+.TP
+.BI "autocasesens = "bool
+auto-trigger case sensitivity (raw index only). IF
+the index is not stripped (see indexStripChars), decide if we
+automatically trigger character case sensitivity if the search term has
+upper-case characters in any but the first position. Else you need to use
+the query language and the "C" modifier to specify character-case
+sensitivity. Default is yes.
+.TP
+.BI "maxTermExpand = "int
+Maximum query expansion count
+for a single term (e.g.: when using wildcards). This only
+affects queries, not indexing. We used to not limit this at all (except
+for filenames where the limit was too low at 1000), but it is
+unreasonable with a big index. Default 10000.
+.TP
+.BI "maxXapianClauses = "int
+Maximum number of clauses
+we add to a single Xapian query. This only affects queries,
+not indexing. In some cases, the result of term expansion can be
+multiplicative, and we want to avoid eating all the memory. Default
+50000.
+.TP
+.BI "snippetMaxPosWalk = "int
+Maximum number of positions we walk while populating a snippet for
+the result list. The default of 1,000,000 may be
+insufficient for very big documents, the consequence would be snippets
+with possibly meaning-altering missing words.
+.TP
+.BI "pdfocr = "bool
+Attempt OCR of PDF files with no text content if both tesseract and
+pdftoppm are installed. The default is off because OCR is so
+very slow.
+.TP
+.BI "pdfattach = "bool
+Enable PDF attachment extraction by executing pdftk (if
+available). This is
+normally disabled, because it does slow down PDF indexing a bit even if
+not one attachment is ever found.
+.TP
+.BI "mhmboxquirks = "string
+Enable thunderbird/mozilla-seamonkey mbox format quirks Set this for the directory where the email mbox files are
+stored.
 
 .SH SEE ALSO
 .PP 

src/doc/man/recollindex.1

@@ -67,8 +67,18 @@ recollindex \- indexing command for the Recoll full text search system
 .B \-Z
 ]
 [
+.B \-K
+]
+[
+.B \-e
+]
+[
 .B \-f
 ]
+[
+.B \-p 
+pattern
+]
 <dirpath>
 .br
 .B recollindex 
@@ -208,6 +218,14 @@ control system).
 will erase data for individual files from the database. The stem expansion
 databases will not be updated.
 .PP
+Options
+.B
+\-i
+and
+.B
+\-e
+can be combined. This will first perform the purge, then the indexing.
+.PP
 With options 
 .B \-i 
 or 
@@ -215,24 +233,30 @@ or
 , if no file names are given on the command line, they
 will be read from stdin, so that you could for example run:
 .PP
-find /path/to/dir \-print | recollindex \-e 
-.PP
-followed by 
-.PP
-find /path/to/dir \-print | recollindex \-i
+find /path/to/dir \-print | recollindex \-e \-i
 .PP
 to force the reindexing of a directory tree (which has to exist inside the
 file system area defined by
 .I topdirs 
 in recoll.conf). You could mostly accomplish the same thing with
 .PP
-.B find /path/to/dir \-print | recollindex \-f \-Z 
+find /path/to/dir \-print | recollindex \-Z \-i
+.PP
+The latter will perform a less thorough job of purging stale sub-documents
+though.
 .PP
 .B recollindex \-r
-mostly works like \-i, but the parameter is a single directory, which will
+mostly works like 
+.B \-i
+, but the parameter is a single directory, which will
 be recursively updated. This mostly does nothing more than 
 .B find topdir | recollindex \-i
-but it may be more convenient to use when started from another program.
+but it may be more convenient to use when started from another
+program. This retries failed files by default, use option
+.B \-K
+to change. One or multiple 
+.B \-p
+options can be used to set shell-type selection patterns (e.g.: *.pdf).
 .PP
 .B recollindex \-l 
 will list the names of available language stemmers.

src/doc/user/Makefile

@@ -17,11 +17,16 @@ commonoptions=--stringparam section.autolabel 1 \
   --stringparam generate.toc "book toc,title,figure,table,example,equation"
 
 
-all: usermanual.html index.html usermanual.pdf
+# index.html chunk format target replaced by nicer webhelp (needs separate
+# make) in webhelp/ subdir
+all: usermanual.html webh usermanual.pdf
+
+webh:
+	make -C webhelp
         
 usermanual.html: usermanual.xml
-	xsltproc ${commonoptions} \
-            -o tmpfile.html "${XSLDIR}/html/docbook.xsl" usermanual.xml
+	xsltproc --xinclude ${commonoptions} \
+            -o tmpfile.html "${XSLDIR}/html/docbook.xsl" $<
 	-tidy -indent tmpfile.html > usermanual.html
 	rm -f tmpfile.html
 
@@ -29,10 +34,10 @@ index.html: usermanual.xml
 	xsltproc ${commonoptions} \
             --stringparam use.id.as.filename 1 \
             --stringparam root.filename index \
-            "${XSLDIR}/html/chunk.xsl" usermanual.xml
+            "${XSLDIR}/html/chunk.xsl" $<
 
 usermanual.pdf: usermanual.xml
-	dblatex usermanual.xml 
+	dblatex $<
 
 clean:
 	rm -f RCL.*.html usermanual.pdf usermanual.html index.html tmpfile.html

src/doc/user/recoll.conf.xml

@@ -0,0 +1,589 @@
+<?xml version="1.0"?>
+<sect2 id="RCL.INSTALL.CONFIG.RECOLLCONF">
+<title>Recoll main configuration file, recoll.conf </title>
+<sect3 id="RCL.INSTALL.CONFIG.RECOLLCONF.WHATDOCS">
+<title>Parameters affecting what documents we index </title>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.TOPDIRS">
+<term><varname>topdirs</varname></term>
+<listitem><para>Space-separated list of files or
+directories to recursively index. Default to ~ (indexes
+$HOME). You can use symbolic links in the list, they will be followed,
+independantly of the value of the followLinks variable.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.SKIPPEDNAMES">
+<term><varname>skippedNames</varname></term>
+<listitem><para>Files and directories which should be ignored. 
+White space separated list of wildcard patterns (simple ones, not paths,
+must contain no / ), which will be tested against file and directory
+names.  The list in the default configuration does not exclude hidden
+directories (names beginning with a dot), which means that it may index
+quite a few things that you do not want. On the other hand, email user
+agents like Thunderbird usually store messages in hidden directories, and
+you probably want this indexed. One possible solution is to have ".*" in
+"skippedNames", and add things like "~/.thunderbird" "~/.evolution" to
+"topdirs".  Not even the file names are indexed for patterns in this
+list, see the "noContentSuffixes" variable for an alternative approach
+which indexes the file names. Can be redefined for any
+subtree.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.NOCONTENTSUFFIXES">
+<term><varname>noContentSuffixes</varname></term>
+<listitem><para>List of name endings (not necessarily dot-separated suffixes) for
+which we don't try MIME type identification, and don't uncompress or
+index content. Only the names will be indexed. This
+complements the now obsoleted recoll_noindex list from the mimemap file,
+which will go away in a future release (the move from mimemap to
+recoll.conf allows editing the list through the GUI). This is different
+from skippedNames because these are name ending matches only (not
+wildcard patterns), and the file name itself gets indexed normally. This
+can be redefined for subdirectories.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.SKIPPEDPATHS">
+<term><varname>skippedPaths</varname></term>
+<listitem><para>Paths we should not go into. Space-separated list of
+wildcard expressions for filesystem paths. Can contain files and
+directories. The database and configuration directories will
+automatically be added. The expressions are matched using 'fnmatch(3)'
+with the FNM_PATHNAME flag set by default. This means that '/' characters
+must be matched explicitely. You can set 'skippedPathsFnmPathname' to 0
+to disable the use of FNM_PATHNAME (meaning that '/*/dir3' will match
+'/dir1/dir2/dir3').  The default value contains the usual mount point for
+removable media to remind you that it is a bad idea to have Recoll work
+on these (esp. with the monitor: media gets indexed on mount, all data
+gets erased on unmount).  Explicitely adding '/media/xxx' to the topdirs
+will override this.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.SKIPPEDPATHSFNMPATHNAME">
+<term><varname>skippedPathsFnmPathname</varname></term>
+<listitem><para>Set to 0 to
+override use of FNM_PATHNAME for matching skipped
+paths. </para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.DAEMSKIPPEDPATHS">
+<term><varname>daemSkippedPaths</varname></term>
+<listitem><para>skippedPaths equivalent specific to
+real time indexing. This enables having parts of the tree
+which are initially indexed but not monitored. If daemSkippedPaths is
+not set, the daemon uses skippedPaths.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.ZIPSKIPPEDNAMES">
+<term><varname>zipSkippedNames</varname></term>
+<listitem><para>Space-separated list of wildcard expressions for names that should
+be ignored inside zip archives. This is used directly by
+the zip handler, and has a function similar to skippedNames, but works
+independantly. Can be redefined for subdirectories. Supported by recoll
+1.20 and newer. See
+https://bitbucket.org/medoc/recoll/wiki/Filtering%20out%20Zip%20archive%20members
+</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.FOLLOWLINKS">
+<term><varname>followLinks</varname></term>
+<listitem><para>Follow symbolic links during
+indexing. The default is to ignore symbolic links to avoid
+multiple indexing of linked files. No effort is made to avoid duplication
+when this option is set to true. This option can be set individually for
+each of the 'topdirs' members by using sections. It can not be changed
+below the 'topdirs' level. Links in the 'topdirs' list itself are always
+followed.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.INDEXEDMIMETYPES">
+<term><varname>indexedmimetypes</varname></term>
+<listitem><para>Restrictive list of
+indexed mime types. Normally not set (in which case all
+supported types are indexed). If it is set,
+only the types from the list will have their contents indexed. The names
+will be indexed anyway if indexallfilenames is set (default). MIME
+type names should be taken from the mimemap file. Can be redefined for
+subtrees.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.EXCLUDEDMIMETYPES">
+<term><varname>excludedmimetypes</varname></term>
+<listitem><para>List of excluded MIME
+types. Lets you exclude some types from indexing. Can be
+redefined for subtrees.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.COMPRESSEDFILEMAXKBS">
+<term><varname>compressedfilemaxkbs</varname></term>
+<listitem><para>Size limit for compressed
+files. We need to decompress these in a
+temporary directory for identification, which can be wasteful in some
+cases. Limit the waste. Negative means no limit. 0 results in no
+processing of any compressed file. Default 50 MB.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.TEXTFILEMAXMBS">
+<term><varname>textfilemaxmbs</varname></term>
+<listitem><para>Size limit for text
+files. Mostly for skipping monster
+logs. Default 20 MB.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.INDEXALLFILENAMES">
+<term><varname>indexallfilenames</varname></term>
+<listitem><para>Index the file names of
+unprocessed files Index the names of files the contents of
+which we don't index because of an excluded or unsupported MIME
+type.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.USESYSTEMFILECOMMAND">
+<term><varname>usesystemfilecommand</varname></term>
+<listitem><para>Use a system command
+for file MIME type guessing as a final step in file type
+identification This is generally useful, but will usually
+cause the indexing of many bogus 'text' files. See 'systemfilecommand'
+for the command used.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.SYSTEMFILECOMMAND">
+<term><varname>systemfilecommand</varname></term>
+<listitem><para>Command used to guess
+MIME types if the internal methods fails This should be a
+"file -i" workalike.  The file path will be added as a last parameter to
+the command line. 'xdg-mime' works better than the traditional 'file'
+command, and is now the configured default (with a hard-coded fallback to
+'file')</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.PROCESSWEBQUEUE">
+<term><varname>processwebqueue</varname></term>
+<listitem><para>Decide if we process the
+Web queue. The queue is a directory where the Recoll Web
+browser plugins create the copies of visited pages.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.TEXTFILEPAGEKBS">
+<term><varname>textfilepagekbs</varname></term>
+<listitem><para>Page size for text
+files. If this is set, text/plain files will be divided
+into documents of approximately this size. Will reduce memory usage at
+index time and help with loading data in the preview window at query
+time. Particularly useful with very big files, such as application or
+system logs. Also see textfilemaxmbs and
+compressedfilemaxkbs.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.MEMBERMAXKBS">
+<term><varname>membermaxkbs</varname></term>
+<listitem><para>Size limit for archive
+members. This is passed to the filters in the environment
+as RECOLL_FILTER_MAXMEMBERKB.</para></listitem></varlistentry>
+</sect3>
+<sect3 id="RCL.INSTALL.CONFIG.RECOLLCONF.TERMS">
+<title>Parameters affecting how we generate terms </title>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.INDEXSTRIPCHARS">
+<term><varname>indexStripChars</varname></term>
+<listitem><para>Decide if we store
+character case and diacritics in the index. If we do,
+searches sensitive to case and diacritics can be performed, but the index
+will be bigger, and some marginal weirdness may sometimes occur. The
+default is a stripped index. When using multiple indexes for a search,
+this parameter must be defined identically for all. Changing the value
+implies an index reset.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.NONUMBERS">
+<term><varname>nonumbers</varname></term>
+<listitem><para>Decides if terms will be
+generated for numbers. For example "123", "1.5e6",
+192.168.1.4, would not be indexed if nonumbers is set ("value123" would
+still be). Numbers are often quite interesting to search for, and this
+should probably not be set except for special situations, ie, scientific
+documents with huge amounts of numbers in them, where setting nonumbers
+will reduce the index size. This can only be set for a whole index, not
+for a subtree.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.DEHYPHENATE">
+<term><varname>dehyphenate</varname></term>
+<listitem><para>Determines if we index
+'coworker' also when the input is 'co-worker'. This is new
+in version 1.22, and on by default. Setting the variable to off allows
+restoring the previous behaviour.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.NOCJK">
+<term><varname>nocjk</varname></term>
+<listitem><para>Decides if specific East Asian
+(Chinese Korean Japanese) characters/word splitting is turned
+off. This will save a small amount of CPU if you have no CJK
+documents. If your document base does include such text but you are not
+interested in searching it, setting nocjk may be a
+significant time and space saver.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.CJKNGRAMLEN">
+<term><varname>cjkngramlen</varname></term>
+<listitem><para>This lets you adjust the size of
+n-grams used for indexing CJK text. The default value of 2 is
+probably appropriate in most cases. A value of 3 would allow more precision
+and efficiency on longer words, but the index will be approximately twice
+as large.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.INDEXSTEMMINGLANGUAGES">
+<term><varname>indexstemminglanguages</varname></term>
+<listitem><para>Languages for which to create stemming expansion
+data. Stemmer names can be found by executing 'recollindex
+-l', or this can also be set from a list in the GUI.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.DEFAULTCHARSET">
+<term><varname>defaultcharset</varname></term>
+<listitem><para>Default character
+set. This is used for files which do not contain a
+character set definition (e.g.: text/plain). Values found inside files,
+e.g. a 'charset' tag in HTML documents, will override it. If this is not
+set, the default character set is the one defined by the NLS environment
+($LC_ALL, $LC_CTYPE, $LANG), or ultimately iso-8859-1 (cp-1252 in fact).
+If for some reason you want a general default which does not match your
+LANG and is not 8859-1, use this variable. This can be redefined for any
+sub-directory.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.UNAC_EXCEPT_TRANS">
+<term><varname>unac_except_trans</varname></term>
+<listitem><para>A list of characters,
+encoded in UTF-8, which should be handled specially
+when converting text to unaccented lowercase. For
+example, in Swedish, the letter a with diaeresis has full alphabet
+citizenship and should not be turned into an a.
+Each element in the space-separated list has the special character as
+first element and the translation following. The handling of both the
+lowercase and upper-case versions of a character should be specified, as
+appartenance to the list will turn-off both standard accent and case
+processing. The value is global and affects both indexing and querying.
+Examples:
+Swedish:
+unac_except_trans = ää Ää öö Öö üü Üü ßss œoe Œoe æae Æae ffff fifi flfl åå Åå
+. German:
+unac_except_trans = ää Ää öö Öö üü Üü ßss œoe Œoe æae Æae ffff fifi flfl
+In French, you probably want to decompose oe and ae and nobody would type
+a German ß
+unac_except_trans = ßss œoe Œoe æae Æae ffff fifi flfl
+. The default for all until someone protests follows. These decompositions
+are not performed by unac, but it is unlikely that someone would type the
+composed forms in a search.
+unac_except_trans = ßss œoe Œoe æae Æae ffff fifi flfl</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.MAILDEFCHARSET">
+<term><varname>maildefcharset</varname></term>
+<listitem><para>Overrides the default
+character set for email messages which don't specify
+one. This is mainly useful for readpst (libpst) dumps,
+which are utf-8 but do not say so.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.LOCALFIELDS">
+<term><varname>localfields</varname></term>
+<listitem><para>Set fields on all files
+(usually of a specific fs area). Syntax is the usual:
+name = value ; attr1 = val1 ; [...]
+value is empty so this needs an initial semi-colon. This is useful, e.g.,
+for setting the rclaptg field for application selection inside
+mimeview.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.TESTMODIFUSEMTIME">
+<term><varname>testmodifusemtime</varname></term>
+<listitem><para>Use mtime instead of
+ctime to test if a file has been modified. The time is used
+in addition to the size, which is always used.
+Setting this can reduce re-indexing on systems where extended attributes
+are used (by some other application), but not indexed, because changing
+extended attributes only affects ctime.
+Notes:
+- This may prevent detection of change in some marginal file rename cases
+(the target would need to have the same size and mtime).
+- You should probably also set noxattrfields to 1 in this case, except if
+you still prefer to perform xattr indexing, for example if the local
+file update pattern makes it of value (as in general, there is a risk
+for pure extended attributes updates without file modification to go
+undetected). Perform a full index reset after changing this.
+</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.NOXATTRFIELDS">
+<term><varname>noxattrfields</varname></term>
+<listitem><para>Disable extended attributes
+conversion to metadata fields. This probably needs to be
+set if testmodifusemtime is set.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.METADATACMDS">
+<term><varname>metadatacmds</varname></term>
+<listitem><para>Define commands to
+gather external metadata, e.g. tmsu tags. 
+There can be several entries, separated by semi-colons, each defining
+which field name the data goes into and the command to use. Don't forget the
+initial semi-colon. All the field names must be different. You can use
+aliases in the "field" file if necessary.
+As a not too pretty hack conceded to convenience, any field name
+beginning with "rclmulti" will be taken as an indication that the command
+returns multiple field values inside a text blob formatted as a recoll
+configuration file ("fieldname = fieldvalue" lines). The rclmultixx name
+will be ignored, and field names and values will be parsed from the data.
+Example: metadatacmds = ; tags = tmsu tags %f; rclmulti1 = cmdOutputsConf %f
+</para></listitem></varlistentry>
+</sect3>
+<sect3 id="RCL.INSTALL.CONFIG.RECOLLCONF.STORE">
+<title>Parameters affecting where and how we store things </title>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.CACHEDIR">
+<term><varname>cachedir</varname></term>
+<listitem><para>Top directory for Recoll data. Recoll data
+directories are normally located relative to the configuration directory
+(e.g. ~/.recoll/xapiandb, ~/.recoll/mboxcache). If 'cachedir' is set, the
+directories are stored under the specified value instead (e.g. if
+cachedir is ~/.cache/recoll, the default dbdir would be
+~/.cache/recoll/xapiandb).  This affects dbdir, webcachedir,
+mboxcachedir, aspellDicDir, which can still be individually specified to
+override cachedir.  Note that if you have multiple configurations, each
+must have a different cachedir, there is no automatic computation of a
+subpath under cachedir.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.MAXFSOCCUPPC">
+<term><varname>maxfsoccuppc</varname></term>
+<listitem><para>Maximum file system occupation
+over which we stop indexing. The value is a percentage,
+corresponding to what the "Capacity" df output column shows. The default
+value is 0, meaning no checking.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.XAPIANDB">
+<term><varname>xapiandb</varname></term>
+<listitem><para>Xapian database directory
+location. This will be created on first indexing. If the
+value is not an absolute path, it will be interpreted as relative to
+cachedir if set, or the configuration directory (-c argument or
+$RECOLL_CONFDIR).  If nothing is specified, the default is then
+~/.recoll/xapiandb/</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.IDXSTATUSFILE">
+<term><varname>idxstatusfile</varname></term>
+<listitem><para>Name of the scratch file where the indexer process updates its
+status. Default: idxstatus.txt inside the configuration
+directory.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.MBOXCACHEDIR">
+<term><varname>mboxcachedir</varname></term>
+<listitem><para>Directory location for storing mbox message offsets cache
+files. This is normally 'mboxcache' under cachedir if set,
+or else under the configuration directory, but it may be useful to share
+a directory between different configurations.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.MBOXCACHEMINMBS">
+<term><varname>mboxcacheminmbs</varname></term>
+<listitem><para>Minimum mbox file size over which we cache the offsets. There is really no sense in caching offsets for small files. The
+default is 5 MB.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.WEBCACHEDIR">
+<term><varname>webcachedir</varname></term>
+<listitem><para>Directory where we store the archived web pages. This is only used by the web history indexing code
+Default: cachedir/webcache if cachedir is set, else
+$RECOLL_CONFDIR/webcache</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.WEBCACHEMAXMBS">
+<term><varname>webcachemaxmbs</varname></term>
+<listitem><para>Maximum size in MB of the Web archive. This is only used by the web history indexing code.
+Default: 40 MB.
+Reducing the size will not physically truncate the file.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.WEBQUEUEDIR">
+<term><varname>webqueuedir</varname></term>
+<listitem><para>The path to the Web indexing queue. This is
+hard-coded in the plugin as ~/.recollweb/ToIndex so there should be no
+need or possibility to change it.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.ASPELLDICDIR">
+<term><varname>aspellDicDir</varname></term>
+<listitem><para>Aspell dictionary storage directory location. The
+aspell dictionary (aspdict.(lang).rws) is normally stored in the
+directory specified by cachedir if set, or under the configuration
+directory.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.FILTERSDIR">
+<term><varname>filtersdir</varname></term>
+<listitem><para>Directory location for executable input handlers. If
+RECOLL_FILTERSDIR is set in the environment, we use it instead. Defaults
+to $prefix/share/recoll/filters. Can be redefined for
+subdirectories.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.ICONSDIR">
+<term><varname>iconsdir</varname></term>
+<listitem><para>Directory location for icons. The only reason to
+change this would be if you want to change the icons displayed in the
+result list. Defaults to $prefix/share/recoll/images</para></listitem></varlistentry>
+</sect3>
+<sect3 id="RCL.INSTALL.CONFIG.RECOLLCONF.PERFS">
+<title>Parameters affecting indexing performance and resource usage </title>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.IDXFLUSHMB">
+<term><varname>idxflushmb</varname></term>
+<listitem><para>Threshold (megabytes of new data) where we flush from memory to
+disk index. Setting this allows some control over memory
+usage by the indexer process. A value of 0 means no explicit flushing,
+which lets Xapian perform its own thing, meaning flushing every
+$XAPIAN_FLUSH_THRESHOLD documents created, modified or deleted: as memory
+usage depends on average document size, not only document count, the
+Xapian approach is is not very useful, and you should let Recoll manage
+the flushes. The program compiled value is 0. The configured default
+value (from this file) is 10 MB, and will be too low in many cases (it is
+chosen to conserve memory). If you are looking
+for maximum speed, you may want to experiment with values between 20 and
+200. In my experience, values beyond this are always counterproductive. If
+you find otherwise, please drop me a note.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.FILTERMAXSECONDS">
+<term><varname>filtermaxseconds</varname></term>
+<listitem><para>Maximum external filter execution time in
+seconds. Default 1200 (20mn). Set to 0 for no limit. This
+is mainly to avoid infinite loops in postscript files
+(loop.ps)</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.FILTERMAXMBYTES">
+<term><varname>filtermaxmbytes</varname></term>
+<listitem><para>Maximum virtual memory space for filter processes
+(setrlimit(RLIMIT_AS)), in megabytes. Note that this
+includes any mapped libs (there is no reliable Linux way to limit the
+data space only), so we need to be a bit generous here. Anything over
+2000 will be ignored on 32 bits machines.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.THRQSIZES">
+<term><varname>thrQSizes</varname></term>
+<listitem><para>Stage input queues configuration. There are three
+internal queues in the indexing pipeline stages (file data extraction,
+terms generation, index update). This parameter defines the queue depths
+for each stage (three integer values). If a value of -1 is given for a
+given stage, no queue is used, and the thread will go on performing the
+next stage. In practise, deep queues have not been shown to increase
+performance. Default: a value of 0 for the first queue tells Recoll to
+perform autoconfiguration based on the detected number of CPUs (no need
+for the two other values in this case).  Use thrQSizes = -1 -1 -1 to
+disable multithreading entirely.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.THRTCOUNTS">
+<term><varname>thrTCounts</varname></term>
+<listitem><para>Number of threads used for each indexing stage. The
+three stages are: file data extraction, terms generation, index
+update). The use of the counts is also controlled by some special values
+in thrQSizes: if the first queue depth is 0, all counts are ignored
+(autoconfigured); if a value of -1 is used for a queue depth, the
+corresponding thread count is ignored. It makes no sense to use a value
+other than 1 for the last stage because updating the Xapian index is
+necessarily single-threaded (and protected by a mutex).</para></listitem></varlistentry>
+</sect3>
+<sect3 id="RCL.INSTALL.CONFIG.RECOLLCONF.MISC">
+<title>Miscellaneous parameters </title>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.LOGLEVEL">
+<term><varname>loglevel</varname></term>
+<listitem><para>Log file verbosity 1-6. A value of 2 will print
+only errors and warnings. 3 will print information like document updates,
+4 is quite verbose and 6 very verbose.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.LOGFILENAME">
+<term><varname>logfilename</varname></term>
+<listitem><para>Log file destination. Use 'stderr' (default) to write to the
+console. </para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.IDXLOGLEVEL">
+<term><varname>idxloglevel</varname></term>
+<listitem><para>Override loglevel for the indexer. </para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.IDXLOGFILENAME">
+<term><varname>idxlogfilename</varname></term>
+<listitem><para>Override logfilename for the indexer. </para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.DAEMLOGLEVEL">
+<term><varname>daemloglevel</varname></term>
+<listitem><para>Override loglevel for the indexer in real time
+mode. The default is to use the idx... values if set, else
+the log... values.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.DAEMLOGFILENAME">
+<term><varname>daemlogfilename</varname></term>
+<listitem><para>Override logfilename for the indexer in real time
+mode. The default is to use the idx... values if set, else
+the log... values.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.IDXRUNDIR">
+<term><varname>idxrundir</varname></term>
+<listitem><para>Indexing process current directory. The input
+handlers sometimes leave temporary files in the current directory, so it
+makes sense to have recollindex chdir to some temporary directory. If the
+value is empty, the current directory is not changed. If the
+value is (literal) tmp, we use the temporary directory as set by the
+environment (RECOLL_TMPDIR else TMPDIR else /tmp). If the value is an
+absolute path to a directory, we go there.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.CHECKNEEDRETRYINDEXSCRIPT">
+<term><varname>checkneedretryindexscript</varname></term>
+<listitem><para>Script used to heuristically check if we need to retry indexing
+files which previously failed.  The default script checks
+the modified dates on /usr/bin and /usr/local/bin. A relative path will
+be looked up in the filters dirs, then in the path. Use an absolute path
+to do otherwise.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.RECOLLHELPERPATH">
+<term><varname>recollhelperpath</varname></term>
+<listitem><para>Additional places to search for helper executables. This is only used on Windows for now.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.IDXABSMLEN">
+<term><varname>idxabsmlen</varname></term>
+<listitem><para>Length of abstracts we store while indexing. Recoll stores an abstract for each indexed file.
+The text can come from an actual 'abstract' section in the
+document or will just be the beginning of the document. It is stored in
+the index so that it can be displayed inside the result lists without
+decoding the original file. The idxabsmlen parameter
+defines the size of the stored abstract. The default value is 250
+bytes. The search interface gives you the choice to display this stored
+text or a synthetic abstract built by extracting text around the search
+terms. If you always prefer the synthetic abstract, you can reduce this
+value and save a little space.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.IDXMETASTOREDLEN">
+<term><varname>idxmetastoredlen</varname></term>
+<listitem><para>Truncation length of stored metadata fields. This
+does not affect indexing (the whole field is processed anyway), just the
+amount of data stored in the index for the purpose of displaying fields
+inside result lists or previews. The default value is 150 bytes which
+may be too low if you have custom fields.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.ASPELLLANGUAGE">
+<term><varname>aspellLanguage</varname></term>
+<listitem><para>Language definitions to use when creating the aspell
+dictionary. The value must match a set of aspell language
+definition files. You can type "aspell dicts"  to see a list The default
+if this is not set is to use the NLS environment to guess the
+value.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.ASPELLADDCREATEPARAM">
+<term><varname>aspellAddCreateParam</varname></term>
+<listitem><para>Additional option and parameter to aspell dictionary creation
+command. Some aspell packages may need an additional option
+(e.g. on Debian Jessie: --local-data-dir=/usr/lib/aspell). See Debian bug
+772415.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.ASPELLKEEPSTDERR">
+<term><varname>aspellKeepStderr</varname></term>
+<listitem><para>Set this to have a look at aspell dictionary creation
+errors. There are always many, so this is mostly for
+debugging.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.NOASPELL">
+<term><varname>noaspell</varname></term>
+<listitem><para>Disable aspell use. The aspell dictionary generation
+takes time, and some combinations of aspell version, language, and local
+terms, result in aspell crashing, so it sometimes makes sense to just
+disable the thing.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.MONAUXINTERVAL">
+<term><varname>monauxinterval</varname></term>
+<listitem><para>Auxiliary database update interval. The real time
+indexer only updates the auxiliary databases (stemdb, aspell)
+periodically, because it would be too costly to do it for every document
+change. The default period is one hour.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.MONIXINTERVAL">
+<term><varname>monixinterval</varname></term>
+<listitem><para>Minimum interval (seconds) between processings of the indexing
+queue. The real time indexer does not process each event
+when it comes in, but lets the queue accumulate, to diminish overhead and
+to aggregate multiple events affecting the same file. Default 30
+S.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.MONDELAYPATTERNS">
+<term><varname>mondelaypatterns</varname></term>
+<listitem><para>Timing parameters for the real time indexing. Definitions for files which get a longer delay before reindexing
+is allowed. This is for fast-changing files, that should only be
+reindexed once in a while. A list of wildcardPattern:seconds pairs. The
+patterns are matched with fnmatch(pattern, path, 0) You can quote entries
+containing white space with double quotes (quote the whole entry, not the
+pattern). The default is empty.
+Example: mondelaypatterns = *.log:20 "*with spaces.*:30"</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.MONIONICECLASS">
+<term><varname>monioniceclass</varname></term>
+<listitem><para>ionice class for the real time indexing process On platforms where this is supported. The default value is
+3.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.MONIONICECLASSDATA">
+<term><varname>monioniceclassdata</varname></term>
+<listitem><para>ionice class parameter for the real time indexing process. On platforms where this is supported. The default is
+empty.</para></listitem></varlistentry>
+</sect3>
+<sect3 id="RCL.INSTALL.CONFIG.RECOLLCONF.QUERY">
+<title>Query-time parameters (no impact on the index) </title>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.AUTODIACSENS">
+<term><varname>autodiacsens</varname></term>
+<listitem><para>auto-trigger diacritics sensitivity (raw index only). IF the index is not stripped, decide if we automatically trigger
+diacritics sensitivity if the search term has accented characters (not in
+unac_except_trans). Else you need to use the query language and the "D"
+modifier to specify diacritics sensitivity. Default is no.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.AUTOCASESENS">
+<term><varname>autocasesens</varname></term>
+<listitem><para>auto-trigger case sensitivity (raw index only). IF
+the index is not stripped (see indexStripChars), decide if we
+automatically trigger character case sensitivity if the search term has
+upper-case characters in any but the first position. Else you need to use
+the query language and the "C" modifier to specify character-case
+sensitivity. Default is yes.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.MAXTERMEXPAND">
+<term><varname>maxTermExpand</varname></term>
+<listitem><para>Maximum query expansion count
+for a single term (e.g.: when using wildcards). This only
+affects queries, not indexing. We used to not limit this at all (except
+for filenames where the limit was too low at 1000), but it is
+unreasonable with a big index. Default 10000.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.MAXXAPIANCLAUSES">
+<term><varname>maxXapianClauses</varname></term>
+<listitem><para>Maximum number of clauses
+we add to a single Xapian query. This only affects queries,
+not indexing. In some cases, the result of term expansion can be
+multiplicative, and we want to avoid eating all the memory. Default
+50000.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.SNIPPETMAXPOSWALK">
+<term><varname>snippetMaxPosWalk</varname></term>
+<listitem><para>Maximum number of positions we walk while populating a snippet for
+the result list. The default of 1,000,000 may be
+insufficient for very big documents, the consequence would be snippets
+with possibly meaning-altering missing words.</para></listitem></varlistentry>
+</sect3>
+<sect3 id="RCL.INSTALL.CONFIG.RECOLLCONF.PDF">
+<title>Parameters for the PDF input script </title>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.PDFOCR">
+<term><varname>pdfocr</varname></term>
+<listitem><para>Attempt OCR of PDF files with no text content if both tesseract and
+pdftoppm are installed. The default is off because OCR is so
+very slow.</para></listitem></varlistentry>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.PDFATTACH">
+<term><varname>pdfattach</varname></term>
+<listitem><para>Enable PDF attachment extraction by executing pdftk (if
+available). This is
+normally disabled, because it does slow down PDF indexing a bit even if
+not one attachment is ever found.</para></listitem></varlistentry>
+</sect3>
+<sect3 id="RCL.INSTALL.CONFIG.RECOLLCONF.SPECLOCATIONS">
+<title>Parameters set for specific locations </title>
+<varlistentry id="RCL.INSTALL.CONFIG.RECOLLCONF.MHMBOXQUIRKS">
+<term><varname>mhmboxquirks</varname></term>
+<listitem><para>Enable thunderbird/mozilla-seamonkey mbox format quirks Set this for the directory where the email mbox files are
+stored.</para></listitem></varlistentry>
+</sect3>
+</sect2>

src/doc/user/webhelp/00README.txt

@@ -0,0 +1,13 @@
+
+01-2016:
+Webhelp is not packaged on Debian. To setup the build:
+
+ - Get a docbook-xsl tar distribution (e.g. docbook-xsl-1.79.1.tar.bz2)
+ - Copy webhelp/xsl to the docbook dist:
+   /usr/share/xml/docbook/stylesheet/docbook-xsl/webhelp/xsl
+ - Possibly adjust
+  DOCBOOK_DIST := /usr/share/xml/docbook/stylesheet/docbook-xsl/ 
+  in Makefile (if other system with docbook installed elsewhere).
+
+Did not try to get the search to work (needs lucene etc.)
+

src/doc/user/webhelp/Makefile

@@ -0,0 +1,114 @@
+# Configuration
+# The name of the source DocBook xml file
+INPUT_XML = ../usermanual.xml
+
+# The makefile assumes that you have a 
+# directory named images that contains 
+# your images. It copies this to the 
+# output directory
+USER_IMAGES_PARENT_DIR=docsrc
+
+# Name of the desired output directory
+# This will be created if it doesn't exist
+OUTPUT_DIR = docs
+
+# A list of files to exclude from indexing
+INDEXER_EXCLUDED_FILES = ix01.html
+
+# Profiling params. For more information on 
+# profiling (conditional text) and DocBook documents, see
+# http://www.sagehill.net/docbookxsl/Profiling.html
+PROFILE.ARCH = ""
+PROFILE.AUDIENCE = ""
+PROFILE.CONDITION = ""
+PROFILE.CONFORMANCE = ""
+PROFILE.LANG = ""
+PROFILE.OS = ""
+PROFILE.REVISION = ""
+PROFILE.REVISIONFLAG = ""
+PROFILE.ROLE = ""
+PROFILE.SECURITY = ""
+PROFILE.STATUS = ""
+PROFILE.USERLEVEL = ""
+PROFILE.VENDOR = ""
+PROFILE.WORDSIZE = ""
+PROFILE.ATTRIBUTE = ""
+PROFILE.VALUE = ""
+
+# Use this variable to pass in other stringparams
+# to the xsltproc pass that generates DocBook output.
+# For example:
+# OTHER_XSLTPROC_ARGS = --stringparam example.param ""
+OTHER_XSLTPROC_ARGS = 
+
+# Path to the DocBook Distribution that 
+# contains the xslts etc.
+DOCBOOK_DIST := /usr/share/xml/docbook/stylesheet/docbook-xsl/
+
+# =================================================
+# You probably don't need to change anything below
+# unless you choose to add a validation step.
+# ================================================
+DOCBOOK_EXTENSIONS_DIR = $(DOCBOOK_DIST)/extensions
+INDEXER_JAR   := $(DOCBOOK_EXTENSIONS_DIR)/webhelpindexer.jar
+TAGSOUP_JAR   := $(DOCBOOK_EXTENSIONS_DIR)/tagsoup-1.2.1.jar
+LUCENE_ANALYZER_JAR   := $(DOCBOOK_EXTENSIONS_DIR)/lucene-analyzers-3.0.0.jar
+LUCENE_CORE_JAR   := $(DOCBOOK_EXTENSIONS_DIR)/lucene-core-3.0.0.jar
+
+classpath := $(INDEXER_JAR):$(TAGSOUP_JAR):$(LUCENE_ANALYZER_JAR):$(LUCENE_CORE_JAR)
+
+all: copyfiles docs/index.html # index
+
+${OUTPUT_DIR}/favicon.ico: template/favicon.ico
+	cp -p  template/favicon.ico ${OUTPUT_DIR}/favicon.ico
+${OUTPUT_DIR}/common/main.js: template/common/main.js
+	cp -rp template/common ${OUTPUT_DIR}
+
+copyfiles: ${OUTPUT_DIR}/favicon.ico ${OUTPUT_DIR}/common/main.js
+
+# test ! -d $(USER_IMAGES_PARENT_DIR)/images/ || \
+#     cp -rp $(USER_IMAGES_PARENT_DIR)/images ${OUTPUT_DIR}/images
+
+docs/index.html: ${INPUT_XML}
+	xsltproc  --xinclude --output xincluded-profiled.xml  \
+                  --stringparam  profile.arch ${PROFILE.ARCH} \
+                  --stringparam  profile.audience ${PROFILE.AUDIENCE} \
+                  --stringparam  profile.condition ${PROFILE.CONDITION} \
+                  --stringparam  profile.conformance ${PROFILE.CONFORMANCE} \
+                  --stringparam  profile.lang ${PROFILE.LANG} \
+                  --stringparam  profile.os ${PROFILE.OS} \
+                  --stringparam  profile.revision ${PROFILE.REVISION} \
+                  --stringparam  profile.revisionflag ${PROFILE.REVISIONFLAG} \
+                  --stringparam  profile.role ${PROFILE.ROLE} \
+                  --stringparam  profile.security ${PROFILE.SECURITY} \
+                  --stringparam  profile.status ${PROFILE.STATUS} \
+                  --stringparam  profile.userlevel ${PROFILE.USERLEVEL} \
+                  --stringparam  profile.vendor ${PROFILE.VENDOR} \
+                  --stringparam  profile.wordsize ${PROFILE.WORDSIZE} \
+                  --stringparam  profile.attribute ${PROFILE.ATTRIBUTE} \
+                  --stringparam  profile.value ${PROFILE.VALUE} \
+                  ${DOCBOOK_DIST}/profiling/profile.xsl  \
+                  ${INPUT_XML}
+	xsltproc ${OTHER_XSLTPROC_ARGS} \
+                ${DOCBOOK_DIST}/webhelp/xsl/webhelp.xsl \
+                xincluded-profiled.xml 
+	rm xincluded-profiled.xml
+
+index:
+	java \
+                -DhtmlDir=$(OUTPUT_DIR) \
+                -DindexerLanguage=en \
+                -DhtmlExtension=html \
+                -DdoStem=true \
+                -DindexerExcludedFiles=$(INDEXER_EXCLUDED_FILES) \
+                -Dorg.xml.sax.driver=org.ccil.cowan.tagsoup.Parser \
+                -Djavax.xml.parsers.SAXParserFactory=org.ccil.cowan.tagsoup.jaxp.SAXFactoryImpl \
+                -classpath $(classpath) \
+                com.nexwave.nquindexer.IndexerMain
+
+	cp -r template/search/* ${OUTPUT_DIR}/search
+
+clean:
+	$(RM) -r ${OUTPUT_DIR}
+	mkdir -p $(OUTPUT_DIR)
+

src/doc/user/webhelp/template/common/browserDetect.js

@@ -0,0 +1,116 @@
+var BrowserDetect = {
+	init: function () {
+		this.browser = this.searchString(this.dataBrowser) || "An unknown browser";
+		this.version = this.searchVersion(navigator.userAgent)
+			|| this.searchVersion(navigator.appVersion)
+			|| "an unknown version";
+		this.OS = this.searchString(this.dataOS) || "an unknown OS";
+	},
+	searchString: function (data) {
+		for (var i=0;i<data.length;i++)	{
+			var dataString = data[i].string;
+			var dataProp = data[i].prop;
+			this.versionSearchString = data[i].versionSearch || data[i].identity;
+			if (dataString) {
+				if (dataString.indexOf(data[i].subString) != -1)
+					return data[i].identity;
+			}
+			else if (dataProp)
+				return data[i].identity;
+		}
+	},
+	searchVersion: function (dataString) {
+		var index = dataString.indexOf(this.versionSearchString);
+		if (index == -1) return;
+		return parseFloat(dataString.substring(index+this.versionSearchString.length+1));
+	},
+	dataBrowser: [
+		{
+			string: navigator.userAgent,
+			subString: "Chrome",
+			identity: "Chrome"
+		},
+		{ 	string: navigator.userAgent,
+			subString: "OmniWeb",
+			versionSearch: "OmniWeb/",
+			identity: "OmniWeb"
+		},
+		{
+			string: navigator.vendor,
+			subString: "Apple",
+			identity: "Safari",
+			versionSearch: "Version"
+		},
+		{
+			prop: window.opera,
+			identity: "Opera"
+		},
+		{
+			string: navigator.vendor,
+			subString: "iCab",
+			identity: "iCab"
+		},
+		{
+			string: navigator.vendor,
+			subString: "KDE",
+			identity: "Konqueror"
+		},
+		{
+			string: navigator.userAgent,
+			subString: "Firefox",
+			identity: "Firefox"
+		},
+		{
+			string: navigator.vendor,
+			subString: "Camino",
+			identity: "Camino"
+		},
+		{		// for newer Netscapes (6+)
+			string: navigator.userAgent,
+			subString: "Netscape",
+			identity: "Netscape"
+		},
+		{
+			string: navigator.userAgent,
+			subString: "MSIE",
+			identity: "Explorer",
+			versionSearch: "MSIE"
+		},
+		{
+			string: navigator.userAgent,
+			subString: "Gecko",
+			identity: "Mozilla",
+			versionSearch: "rv"
+		},
+		{ 		// for older Netscapes (4-)
+			string: navigator.userAgent,
+			subString: "Mozilla",
+			identity: "Netscape",
+			versionSearch: "Mozilla"
+		}
+	],
+	dataOS : [
+		{
+			string: navigator.platform,
+			subString: "Win",
+			identity: "Windows"
+		},
+		{
+			string: navigator.platform,
+			subString: "Mac",
+			identity: "Mac"
+		},
+		{
+			   string: navigator.userAgent,
+			   subString: "iPhone",
+			   identity: "iPhone/iPod"
+	    },
+		{
+			string: navigator.platform,
+			subString: "Linux",
+			identity: "Linux"
+		}
+	]
+
+};
+BrowserDetect.init();

src/doc/user/webhelp/template/common/css/ie.css

@@ -0,0 +1,13 @@
+.statustext{
+    filter: progid:DXImageTransform.Microsoft.BasicImage(rotation=1)
+            progid:DXImageTransform.Microsoft.BasicImage(opacity=.5);  
+    width: 100%;
+    height: 30px;
+    right: -5px;
+    top:105px;
+    /* left: 280px; */ /* change to -12px; when sidebar is collapsed */
+}
+
+body #content{                      
+   /* padding-top: 136px;*/
+}

src/doc/user/webhelp/template/common/css/positioning.css

@@ -0,0 +1,377 @@
+tr th .added { color: #E6E6FA; } 
+tr th .changed {color: #99ff99; }
+div.added tr, div.added    { background-color: #E6E6FA; }
+div.deleted tr, div.deleted  { text-decoration: line-through;
+               background-color: #FF7F7F; }
+div.changed tr, div.changed  { background-color: #99ff99; }
+div.off      {  }
+
+span.added   { background-color: #E6E6FA; }
+span.deleted { text-decoration: line-through;
+               background-color: #FF7F7F; }
+span.changed { background-color: #99ff99; }
+span.off     {  }
+
+
+body { font: 12px Verdana, Geneva, sans-serif; }
+p, ul, ol, li { font: 10pt Verdana, Geneva, sans-serif; }
+h1 { font: 15pt Arial, Helvetica, geneva;
+     color: black!important;
+}
+h2 { font: normal 12pt Arial, Helvetica, geneva; }
+
+#header {
+    background: url("../images/header-bg.png") scroll top left repeat-x #4d8c83;
+    position: fixed;
+    width: 100%;
+    height: 99px;
+    top: 0;
+    right: 0;
+    bottom: auto;
+    left: 0;
+    border-bottom: 1px solid #bbc4c5;
+    z-index: 2000;
+}
+
+#header h1 {
+    margin-left: 310px;
+    position: fixed;
+    top: 20px;
+    left: -15px;
+    color: #404040 !important;
+}
+
+
+#header h1 {
+	margin-top: 2px;
+}
+
+
+p.breadcrumbs {
+    margin-top: 30px;
+    margin-left: 310px;
+}
+
+#header img {
+    float: left;
+    margin-left: 20px;
+    margin-top: 12px;
+}
+
+#header p.breadcrumbs a {
+    color: #bbb;
+}
+
+#leftnavigation {
+    overflow: auto;
+    position: fixed;
+    height: auto;
+    top:100px;
+    /*right:10px;*/
+    /*left:10px;*/
+    bottom: 0;
+    left: 0;
+    width:inherit;
+    z-index: 1500;
+    border-right:2px solid #bbc4c5;
+    padding:1px;
+    background-color: #ededed!important;
+}
+
+#treeDiv {
+    overflow: auto;
+   /* position: fixed;*/
+    height: auto;
+    top: 136px;
+    bottom: 0;
+    left: 0;
+   /* width: 18%;*/
+    z-index: 1500;
+   /* border-right:2px solid #CCCCCC;
+    background-color: #f0f0f0!important;*/
+}
+
+/*#searchDiv {
+    overflow: auto;
+    position: fixed;
+    height: auto;
+    top: 138px;
+    bottom: 0;
+    left: 0;
+    width: 243px;
+    z-index: 1500;
+    border-right:2px solid #CCCCCC;
+    background-color: #f0f0f0!important;
+}*/
+
+#content {
+    position: relative;
+    top: 90px;  /*left: 240px;*/
+    right: auto;   bottom: 20px;  
+    /*margin: 0px 0px 0px 280px;*/
+    width: auto;
+    height: inherit;
+    padding-left: 5px;
+    padding-right: 30px;
+    border-left: 1px solid #cccccc;
+    overflow :scroll;
+    overflow-x:hidden;
+    z-index: 1000;
+ 
+}
+
+#navheader {
+    position: fixed;
+    background: #DCDCDC;
+    padding-left: 10px;
+    right: 0px;
+    top: 10px;
+    text-align: right;
+}
+
+#content h1, #content h2 { 
+color: #404040 !important; 
+font-size: 170%;
+font-weight: normal;
+}
+.navfooter { bottom: 2%; }
+.highlight { background-color: #c5d3c3; }
+.highlightButton{ font-size: 0; }
+
+/*  Show Hide TOC tree */
+.pointLeft {
+    padding-right: 15px;
+    display: block;
+    cursor: pointer;
+}
+.pointRight {
+    padding-right: 15px;
+    display: block;
+    cursor: pointer;
+}
+
+/* Search results Styling */
+.searchExpression {
+    color: #0050A0;
+    background-color: #EBEFF8;
+    font-size: 12pt;
+}
+.searchresult li a {
+    text-decoration: none;
+    color: #0050A0;
+}
+.searchresult li { color: #0050A0; }
+.shortdesclink { color: gray; font-size: 9pt; }
+.searchText { float:left;width:150px; }
+.searchButton {
+    padding: 2px 12px 2px 12px;
+    background-color:#bbb;
+    border:#bbb solid 1pt;
+    font-weight: bold;
+    font-size: 10pt
+}
+.searchButton:hover{
+    background-color: #cccccc;
+}
+.searchFieldSet {}
+
+.title, div.toc>p{ font-weight: bold; }	
+
+p.breadcrumbs {
+        display: inline;
+	margin-bottom: 0px;
+	margin-top: 33px;
+}
+
+p.breadcrumbs a {
+	padding-right: 12px;
+	margin-right: 5px;
+	text-decoration: none;
+	color: #575757;
+	text-transform: uppercase;
+	font-size: 10px;
+}
+
+p.breadcrumbs a:first-child {background: url(../images/breadcrumb-arrow-white.png) no-repeat right center;}
+
+p.breadcrumbs a:hover {text-decoration: underline;}
+
+#star ul.star { 
+    LIST-STYLE: none; 
+    MARGIN: 0; 
+    PADDING: 0; 
+    WIDTH: 85px;
+    /* was 100 */ 
+    HEIGHT: 20px;
+    LEFT: 1px; 
+    TOP: -5px; 
+    POSITION: relative; 
+    FLOAT: right; 
+    BACKGROUND: url('../images/starsSmall.png') repeat-x 0 -25px;
+}
+#star li { 
+    PADDING: 0; 
+    MARGIN: 0; 
+    FLOAT: right; 
+    DISPLAY: block; 
+    WIDTH: 85px;
+    /* was 100 */
+    HEIGHT: 20px; 
+    TEXT-DECORATION: none; 
+    text-indent: -9000px; 
+    Z-INDEX: 20; 
+    POSITION: absolute; 
+    PADDING: 0; 
+}
+#star li.curr { 
+    BACKGROUND: url('../images/starsSmall.png') left 25px; 
+    FONT-SIZE: 1px; 
+}
+
+table.navLinks {margin-right: 20px;}
+
+table.navLinks td a {
+	text-decoration: none;
+	text-transform: uppercase;
+	color: black;
+	font-size: 11px;
+}
+
+a.navLinkPrevious {
+	padding-left: 12px;
+	background: url(../images/previous-arrow.png) no-repeat left center;
+}
+
+a.navLinkNext {
+	padding-right: 12px;
+	background: url(../images/next-arrow.png) no-repeat right center;
+}
+
+a#showHideButton {
+	padding-left: 20px;
+	background: url(../images/sidebar.png) no-repeat left center;
+}
+
+	
+.filetree li span a { color: #777; }
+
+#treediv { -webkit-box-shadow: #CCC 0px 1px 2px 0px inset; }
+
+.legal, .legal *{
+ color: #555;
+ text-align: center;
+ padding-bottom: 10px;
+}
+
+.internal { color : #0000CC;}
+
+.writeronly {color : red;}
+
+.remark, .remark .added, .remark .changed, .remark .deleted{ background: yellow;} 
+
+tr th, tr th .internal, tr th .added, tr th .changed {
+	background: #00589E;
+	color: white;
+	font-weight: bold;
+	text-align: left;
+}
+
+.statustext{
+    position:fixed;
+    top:105px;
+    width: 0%;
+    height: 0%;
+    opacity: .3;
+    -webkit-transform: rotate(90deg);
+    -moz-transform: rotate(90deg);
+    -o-transform: rotate(90deg);
+    white-space: nowrap;
+    color: red;
+    font-weight: bold;
+    font-size: 2em;
+    margin-top: 30px;
+}
+
+#toolbar {
+	width: 100%;
+	height: 33px;
+	position: fixed;
+	top: 93px;
+	z-index: 99;
+	left: 280px;
+	color: #333;
+	line-height: 28px;
+	padding-left: 10px;
+}
+
+#toolbar-left {
+	position: relative;
+	left: 0px;
+}
+ 
+body p.breadcrumbs {
+	margin: 0px;
+	padding: 0px;
+	line-height: 28px;
+}
+
+/*body #content {
+	position: static;
+	margin-top: 126px;
+	top: 0px;
+}*/
+
+body.sidebar #toolbar{left: 0px;}
+
+body.sidebar #toolbar-left{left: 0px;}
+
+div#toolbar-left img {vertical-align: text-top;}
+
+div.note *, div.caution *, div.important *, div.tip *, div.warning * {
+    background: inherit !important;
+    color: inherit !important;
+    border: inherit  !important;    
+}
+
+#content table thead, #content table th{
+    background: gray;
+    color: white;
+    font-weight: bold;
+}
+
+#content table caption{font-weight: bold;}
+
+#content table td, #content table {border: 1px solid black;}
+
+#content table td, #content table th { padding: 5px;}
+
+#content table {margin-bottom: 20px;}
+
+*[align = 'center']{ text-align: center;}
+
+#content .qandaset>table, #content .qandaset>table td, #content .calloutlist table, #content .calloutlist table td, #content .navfooter table, #content .navfooter table td {
+    border: 0px solid;
+}
+
+#sidebar { display: none }
+
+@media print {
+
+    body * {
+        visibility: hidden;
+    }
+
+    #content, #content * {
+        visibility: visible;
+    }
+
+   #sidebar, .navfooter {
+       display: none;
+   }
+
+   #content {
+	margin: 0 0 0 0;
+    }
+
+}
+