Make sure to execute python2 scripts with python2

tests/pythonapi/doc.py

@@ -0,0 +1,53 @@
+import sys
+from recoll import recoll
+
+if sys.version_info[0] >= 3:
+    ISP3 = True
+else:
+    ISP3 = False
+
+def utf8string(s):
+    if ISP3:
+        return s
+    else:
+        return s.encode('utf8')
+if ISP3:
+    def u(x):
+        return x
+else:
+    import codecs
+    def u(x):
+        return codecs.unicode_escape_decode(x)[0]
+
+db = recoll.connect()
+query = db.query()
+
+nres = query.execute("testfield:testfieldvalue1", stemming=0)
+qs = "Xapian query: [%s]" % query.getxquery()
+print(utf8string(qs))
+
+print("Result count: %d %d" % (nres, query.rowcount))
+
+for doc in query:
+    print("doc.title: [%s]"%utf8string(doc.title))
+    print("doc.testfield: [%s]"%utf8string(doc.testfield))
+    for fld in ('title', 'testfield', 'filename'):
+        print("getattr(doc, %s) -> [%s]"%(fld,utf8string(getattr(doc, fld))))
+        print("doc.get(%s) -> [%s]"%(fld,utf8string(doc.get(fld))))
+    print("\nfor fld in sorted(doc.keys()):")
+    for fld in sorted(doc.keys()):
+        print(utf8string("[%s] -> [%s]" % (fld, getattr(doc, fld))))
+    print("\nfor k,v in sorted(doc.items().items()):")
+    for k,v in sorted(doc.items().items(), key=lambda itm: itm[0]):
+        print(utf8string("[%s] -> [%s]" % (k, v)))
+
+print("\nAccented query:")
+uqs = u('title:"\u00e9t\u00e9 \u00e0 no\u00ebl"')
+print("User query [%s]"%utf8string(uqs))
+nres = query.execute(uqs, stemming=0)
+#nres = query.execute('title:"ete a noel"', stemming=0)
+qs = "Xapian query: [%s]" % query.getxquery()
+print(utf8string(qs))
+print("nres %d" %(nres,))
+doc = query.fetchone()
+print("doc.title: [%s]"%utf8string(doc.title))

tests/pythonapi/extract.py

@@ -0,0 +1,45 @@
+import sys
+import hashlib
+from recoll import recoll
+from recoll import rclextract
+
+if sys.version_info[0] >= 3:
+    ISP3 = True
+else:
+    ISP3 = False
+
+def utf8string(s):
+    if ISP3:
+        return s
+    else:
+        return s.encode('utf8')
+
+db = recoll.connect()
+query = db.query()
+
+# This normally has only one result, a well-known html file
+nres = query.execute("HtmlAttachment_uniqueTerm", stemming=0)
+print("Result count: %d %d" % (nres, query.rowcount))
+doc = query.fetchone()
+xtrac = rclextract.Extractor(doc)
+doc = xtrac.textextract(doc.ipath)
+print("Text length: %d"%len(doc.text))
+
+refdigest = 'bfbb63f7a245c31767585b45014dbd07'
+
+# This normally has 2 results, one of which is a pdf attachment.
+nres = query.execute("population_size_cultural_transmission", stemming=0)
+for doc in query:
+    if doc.mimetype == 'application/pdf':
+        xtrac = rclextract.Extractor(doc)
+        filename = xtrac.idoctofile(doc.ipath, doc.mimetype)
+        f = open(filename, 'rb')
+        data = f.read()
+        f.close()
+        m = hashlib.md5()
+        m.update(data)
+        digest = m.hexdigest()
+        print(digest)
+        if digest != refdigest:
+            print("extract.py: wrong digest for extracted file!")
+            

tests/pythonapi/pythonapi.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+# Test the Python API
+
+thisdir=`dirname $0`
+topdir=$thisdir/..
+. $topdir/shared.sh
+
+initvariables $0
+
+xrun()
+{
+    echo $*
+    $*
+}
+
+(
+    t2=$toptmp/python2out
+    t3=$toptmp/python3out
+    for i in *.py;do python $i ;done > $t2
+    for i in *.py;do python3 $i ;done > $t3
+
+    if ! cmp $t2 $t3 ; then 
+        echo "Python2 and Python 3 outputs differ: $t2 $t3"
+    fi
+    for i in *.py;do xrun python $i ;done
+) 2> $mystderr | egrep -v '^Recoll query: ' > $mystdout
+
+diff -w ${myname}.txt $mystdout > $mydiffs 2>&1
+checkresult

tests/pythonapi/pythonapi.txt

@@ -0,0 +1,79 @@
+python doc.py
+Xapian query: [(TSTFLDtestfieldvalue1:(wqf=11))]
+Result count: 1 1
+doc.title: [HTML fields test file: été à noël]
+doc.testfield: [testfieldvalue1]
+getattr(doc, title) -> [HTML fields test file: été à noël]
+doc.get(title) -> [HTML fields test file: été à noël]
+getattr(doc, testfield) -> [testfieldvalue1]
+doc.get(testfield) -> [testfieldvalue1]
+getattr(doc, filename) -> [htmlfield1.html]
+doc.get(filename) -> [htmlfield1.html]
+
+for fld in sorted(doc.keys()):
+[abstract] -> [ ThisIsTheFieldHtmlTestFile]
+[caption] -> [HTML fields test file: été à noël]
+[dbytes] -> [27]
+[fbytes] -> [267]
+[filename] -> [htmlfield1.html]
+[fmtime] -> [01383154417]
+[ipath] -> []
+[mtime] -> [01383154417]
+[mtype] -> [text/html]
+[origcharset] -> [utf-8]
+[pcbytes] -> [267]
+[rcludi] -> [/home/dockes/projets/fulltext/testrecoll/html/htmlfield1.html|]
+[relevancyrating] -> [100%]
+[sig] -> [2671402138880]
+[testfield] -> [testfieldvalue1]
+[title] -> [HTML fields test file: été à noël]
+[url] -> [file:///home/dockes/projets/fulltext/testrecoll/html/htmlfield1.html]
+
+for k,v in sorted(doc.items().items()):
+[abstract] -> [ ThisIsTheFieldHtmlTestFile]
+[caption] -> [HTML fields test file: été à noël]
+[dbytes] -> [27]
+[fbytes] -> [267]
+[filename] -> [htmlfield1.html]
+[fmtime] -> [01383154417]
+[ipath] -> []
+[mtime] -> [01383154417]
+[mtype] -> [text/html]
+[origcharset] -> [utf-8]
+[pcbytes] -> [267]
+[rcludi] -> [/home/dockes/projets/fulltext/testrecoll/html/htmlfield1.html|]
+[relevancyrating] -> [100%]
+[sig] -> [2671402138880]
+[testfield] -> [testfieldvalue1]
+[title] -> [HTML fields test file: été à noël]
+[url] -> [file:///home/dockes/projets/fulltext/testrecoll/html/htmlfield1.html]
+
+Accented query:
+User query [title:"été à noël"]
+Xapian query: [(10 * (Sete PHRASE 3 Sa PHRASE 3 Snoel))]
+nres 1
+doc.title: [HTML fields test file: été à noël]
+python extract.py
+Result count: 1 1
+Text length: 3457
+bfbb63f7a245c31767585b45014dbd07
+python simple.py
+Xapian query: [(huniique:(wqf=11))]
+Result count: 2 2
+for i in range(nres):
+Home.ics
+unique.txt
+
+for doc in query:
+Home.ics
+unique.txt
+
+Catched bad mode. (ok)
+python url.py
+Xapian query: [((latin1name_uniquexxx:(wqf=11) AND XPiso8859name))]
+Result count: 1 1
+iso8859-1_????????.txt
+Contents: [LATIN1NAME_UNIQUEXXX 
+ Contenu du fichier dont le nom est:
+àáâãäåæç
+]

tests/pythonapi/simple.py

@@ -0,0 +1,40 @@
+import sys
+from recoll import recoll
+
+if sys.version_info[0] >= 3:
+    ISP3 = True
+else:
+    ISP3 = False
+
+def utf8string(s):
+    if s is None:
+        return "None"
+    if ISP3:
+        return s
+    else:
+        return s.encode('utf8')
+
+db = recoll.connect()
+query = db.query()
+
+nres = query.execute("huniique", stemlang="english")
+qs = "Xapian query: [%s]" % query.getxquery()
+print(utf8string(qs))
+
+print("Result count: %d %d" % (nres, query.rowcount))
+
+print("for i in range(nres):")
+for i in range(nres):
+    doc = query.fetchone()
+    print(utf8string(doc.filename))
+
+query.scroll(0, 'absolute')
+print("\nfor doc in query:")
+for doc in query:
+    print(utf8string(doc.filename))
+
+try:
+    query.scroll(0, 'badmode')
+except:
+    print("\nCatched bad mode. (ok)")
+    

tests/pythonapi/url.py

@@ -0,0 +1,47 @@
+import sys
+from recoll import recoll
+
+# Test the doc.getbinurl() method.
+# Select file with a binary name (actually iso8859-1), open it and
+# convert/print the contents (also iso8859-1)
+
+if sys.version_info[0] >= 3:
+    ISP3 = True
+else:
+    ISP3 = False
+
+def utf8string(s):
+    if ISP3:
+        return s
+    else:
+        return s.encode('utf8')
+if ISP3:
+    def u(x):
+        return x
+else:
+    import codecs
+    def u(x):
+        return codecs.unicode_escape_decode(x)[0]
+
+db = recoll.connect()
+query = db.query()
+
+# This should select a file with an iso8859-1 file name
+nres = query.execute("LATIN1NAME_UNIQUEXXX dir:iso8859name", stemming=0)
+qs = "Xapian query: [%s]" % query.getxquery()
+print(utf8string(qs))
+
+print("Result count: %d %d" % (nres, query.rowcount))
+
+for doc in query:
+    print(utf8string(doc.filename))
+    burl = doc.getbinurl()
+    bytesname = burl[7:]
+    f = open(bytesname, 'rb')
+    s = f.read()
+    f.close()
+    if ISP3:
+        content = str(s, "iso8859-1")
+    else:
+        content = unicode(s, "iso8859-1")
+    print("Contents: [%s]"%utf8string(content))