yetangitu/recoll
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Also index non-html files from the web queue and fix the Open operation for them

Browse source
This commit is contained in:
Jean-Francois Dockes 2015-07-24 16:30:13 +02:00
parent 7cc5f9e039
commit be51bf5ef0
8 changed files with 204 additions and 142 deletions
Download patch file Download diff file Expand all files Collapse all files

15
src/index/beaglequeue.cpp
View file

@ -217,12 +217,6 @@ bool BeagleQueueIndexer::indexFromCache(const string& udi)
// Just index the dotdoc
dotdoc.meta[Rcl::Doc::keybcknd] = "BGL";
return m_db->addOrUpdate(udi, cstr_null, dotdoc);
} else if (stringlowercmp("webhistory", dotdoc.meta[Rcl::Doc::keybght]) ||
(dotdoc.mimetype.compare("text/html") &&
dotdoc.mimetype.compare(cstr_textplain))) {
LOGDEB(("BeagleQueueIndexer: skipping: hittype %s mimetype %s\n",
dotdoc.meta[Rcl::Doc::keybght].c_str(), dotdoc.mimetype.c_str()));
return true;
} else {
Rcl::Doc doc;
FileInterner interner(data, m_config,
@ -404,7 +398,6 @@ BeagleQueueIndexer::processone(const string &path,
char ascdate[30];
sprintf(ascdate, "%ld", long(stp->st_mtime));
// We only process bookmarks or text/html and text/plain files.
if (!stringlowercmp("bookmark", dotdoc.meta[Rcl::Doc::keybght])) {
// For bookmarks, we just index the doc that was built from the
// metadata.
@ -422,14 +415,6 @@ BeagleQueueIndexer::processone(const string &path,
if (!m_db->addOrUpdate(udi, cstr_null, dotdoc))
return FsTreeWalker::FtwError;
} else if (stringlowercmp("webhistory", dotdoc.meta[Rcl::Doc::keybght]) ||
(dotdoc.mimetype.compare("text/html") &&
dotdoc.mimetype.compare(cstr_textplain))) {
LOGDEB(("BeagleQueueIndexer: skipping: hittype %s mimetype %s\n",
dotdoc.meta[Rcl::Doc::keybght].c_str(), dotdoc.mimetype.c_str()));
// Unlink them anyway
dounlink = true;
goto out;
} else {
Rcl::Doc doc;
// Store the dotdoc fields in the future doc. In case someone wants

128
src/internfile/internfile.cpp
View file

@ -380,20 +380,12 @@ TempFile FileInterner::dataToTempFile(const string& dt, const string& mt)
temp->getreason().c_str()));
return TempFile();
}
int fd = open(temp->filename(), O_WRONLY);
if (fd < 0) {
LOGERR(("FileInterner::dataToTempFile: open(%s) failed errno %d\n",
temp->filename(), errno));
string reason;
if (!stringtofile(dt, temp->filename(), reason)) {
LOGERR(("FileInterner::dataToTempFile: stringtofile: %s\n",
reason.c_str()));
return TempFile();
}
if (write(fd, dt.c_str(), dt.length()) != (int)dt.length()) {
close(fd);
LOGERR(("FileInterner::dataToTempFile: write to %s failed errno %d\n",
temp->filename(), errno));
return TempFile();
}
close(fd);
return temp;
}
@ -892,7 +884,20 @@ static string urltolocalpath(string url)
return url.substr(7, string::npos);
}
// Extract subdoc out of multidoc into temporary file.
bool FileInterner::tempFileForMT(TempFile& otemp, RclConfig* cnf,
const string& mimetype)
{
TempFile temp(new TempFileInternal(
cnf->getSuffixFromMimeType(mimetype)));
if (!temp->ok()) {
LOGERR(("FileInterner::interntofile: can't create temp file\n"));
return false;
}
otemp = temp;
return true;
}
// Extract document (typically subdoc of multidoc) into temporary file.
// We do the usual internfile stuff: create a temporary directory,
// then create an interner and call internfile. The target mtype is set to
// the input mtype, so that no data conversion is performed.
@ -901,22 +906,20 @@ static string urltolocalpath(string url)
// - The internfile temporary directory gets destroyed by its destructor
// - The output temporary file which is held in a reference-counted
// object and will be deleted when done with.
// This DOES NOT work with a non-internal file (because at least one conversion
// is always performed).
//
// If the ipath is null, maybe we're called because the file is not
// stored in the regular file system. We use the docfetcher to get a
// copy (in topdocToFile())
//
// We currently don't handle the case of an internal doc of a non-fs document.
bool FileInterner::idocToFile(TempFile& otemp, const string& tofile,
RclConfig *cnf, const Rcl::Doc& idoc)
{
LOGDEB(("FileInterner::idocToFile\n"));
// idoc.dump();
if (idoc.ipath.empty()) {
LOGDEB(("FileInterner::idocToFile: not a sub-document !\n"));
// We could do a copy here but it's much more complicated than
// it seems because the source is not necessarily a simple
// depending on the backend. Until we fix the Internfile
// constructor to not do the first conversion, it's much saner
// to just return an error
return false;
return topdocToFile(otemp, tofile, cnf, idoc);
}
// We set FIF_forPreview for consistency with the previous version
@ -927,6 +930,54 @@ bool FileInterner::idocToFile(TempFile& otemp, const string& tofile,
return interner.interntofile(otemp, tofile, idoc.ipath, idoc.mimetype);
}
bool FileInterner::topdocToFile(TempFile& otemp, const string& tofile,
RclConfig *cnf, const Rcl::Doc& idoc)
{
DocFetcher *fetcher = docFetcherMake(idoc);
if (fetcher == 0) {
LOGERR(("FileInterner::idocToFile no backend\n"));
return false;
}
DocFetcher::RawDoc rawdoc;
if (!fetcher->fetch(cnf, idoc, rawdoc)) {
LOGERR(("FileInterner::idocToFile fetcher failed\n"));
return false;
}
const char *filename = "";
TempFile temp;
if (tofile.empty()) {
if (!tempFileForMT(temp, cnf, idoc.mimetype)) {
return false;
}
filename = temp->filename();
} else {
filename = tofile.c_str();
}
string reason;
switch (rawdoc.kind) {
case DocFetcher::RawDoc::RDK_FILENAME:
if (!copyfile(rawdoc.data.c_str(), filename, reason)) {
LOGERR(("FileInterner::idocToFile: copyfile: %s\n",
reason.c_str()));
return false;
}
break;
case DocFetcher::RawDoc::RDK_DATA:
if (!stringtofile(rawdoc.data, filename, reason)) {
LOGERR(("FileInterner::idocToFile: stringtofile: %s\n",
reason.c_str()));
return false;
}
break;
default:
LOGERR(("FileInterner::FileInterner(idoc): bad rawdoc kind ??\n"));
}
if (tofile.empty())
otemp = temp;
return true;
}
bool FileInterner::interntofile(TempFile& otemp, const string& tofile,
const string& ipath, const string& mimetype)
{
@ -952,35 +1003,22 @@ bool FileInterner::interntofile(TempFile& otemp, const string& tofile,
doc.mimetype = "text/html";
}
string filename;
const char *filename;
TempFile temp;
if (tofile.empty()) {
TempFile temp1(new TempFileInternal(
m_cfg->getSuffixFromMimeType(mimetype)));
temp = temp1;
if (!temp->ok()) {
LOGERR(("FileInterner::interntofile: can't create temp file\n"));
return false;
}
if (!tempFileForMT(temp, m_cfg, mimetype)) {
return false;
}
filename = temp->filename();
} else {
filename = tofile;
filename = tofile.c_str();
}
int fd = open(filename.c_str(), O_WRONLY|O_CREAT, 0600);
if (fd < 0) {
LOGERR(("FileInterner::interntofile: open(%s) failed errno %d\n",
filename.c_str(), errno));
string reason;
if (!stringtofile(doc.text, filename, reason)) {
LOGERR(("FileInterner::interntofile: stringtofile : %s\n",
reason.c_str()));
return false;
}
const string& dt = doc.text;
if (write(fd, dt.c_str(), dt.length()) != (int)dt.length()) {
close(fd);
LOGERR(("FileInterner::interntofile: write to %s failed errno %d\n",
filename.c_str(), errno));
return false;
}
close(fd);
if (tofile.empty())
otemp = temp;

4
src/internfile/internfile.h
View file

@ -287,6 +287,10 @@ class FileInterner {
int addHandler();
void checkExternalMissing(const string& msg, const string& mt);
void processNextDocError(Rcl::Doc &doc);
static bool tempFileForMT(TempFile& otemp, RclConfig *cnf,
const std::string& mimetype);
static bool topdocToFile(TempFile& otemp, const std::string& tofile,
RclConfig *cnf, const Rcl::Doc& idoc);
};

18
src/qtgui/rclm_saveload.cpp
View file

@ -31,6 +31,7 @@
#include "readfile.h"
#include "xmltosd.h"
#include "searchdata.h"
#include "copyfile.h"
using namespace std;
using namespace Rcl;
@ -90,23 +91,12 @@ void RclMain::saveLastQuery()
string tofile((const char *)s.toLocal8Bit());
LOGDEB(("RclMain::saveLastQuery: XML: [%s]\n", xml.c_str()));
int fd = ::open(tofile.c_str(), O_WRONLY|O_CREAT|O_TRUNC, 0600);
if (fd < 0) {
QMessageBox::warning(this, tr("Open failed"),
tr("Could not open/create file"));
return;
}
if (::write(fd, xml.c_str(), xml.size()) != int(xml.size())) {
::close(fd);
string reason;
if (!stringtofile(xml, tofile.c_str(), reason)) {
QMessageBox::warning(this, tr("Write failed"),
tr("Could not write to file"));
return;
}
if (::close(fd) != 0) {
QMessageBox::warning(this, tr("Close failed"), tr("File close error"));
return;
}
return;
}

13
src/qtgui/rclm_view.cpp
View file

@ -291,11 +291,20 @@ void RclMain::startNativeViewer(Rcl::Doc doc, int pagenum, QString term)
LOGDEB(("RclMain::startNV: groksipath %d wantsf %d wantsparentf %d\n",
groksipath, wantsfile, wantsparentfile));
bool notinfs = false;
{
string backend;
doc.getmeta(Rcl::Doc::keybcknd, &backend);
if (!backend.empty() && backend.compare("FS"))
notinfs = true;
}
// If the command wants a file but this is not a file url, or
// there is an ipath that it won't understand, we need a temp file:
theconfig->setKeyDir(path_getfather(fn));
if (((wantsfile || wantsparentfile) && fn.empty()) ||
(!groksipath && !doc.ipath.empty())) {
if (notinfs ||
((wantsfile || wantsparentfile) && fn.empty()) ||
(!groksipath && !doc.ipath.empty()) ) {
TempFile temp;
Rcl::Doc& thedoc = wantsparentfile ? pdoc : doc;
if (!FileInterner::idocToFile(temp, string(), theconfig, thedoc)) {

8
src/qtgui/rtitool.cpp
View file

@ -111,17 +111,13 @@ void RTIToolW::accept()
dir = path_cat(dir, "autostart");
mkdir(dir.c_str(), 0700);
int fd = ::open(autostartfile.c_str(), O_WRONLY|O_CREAT, 0644);
if (fd < 0 || ::write(fd, text.c_str(), size_t(text.size()))
!= ssize_t(text.size()) || ::close(fd) != 0) {
if (fd >=0)
::close(fd);
string reason;
if (!stringtofile(text, autostartfile.c_str(), reason)) {
QString msg = tr("Can't create: ") +
QString::fromLocal8Bit(autostartfile.c_str());
QMessageBox::warning(0, tr("Warning"), msg, QMessageBox::Ok);
return;
}
::close(fd);
if (nowCB->isChecked()) {
ExecCmd cmd;

156
src/utils/copyfile.cpp
View file

@ -35,55 +35,91 @@ using namespace std;
bool copyfile(const char *src, const char *dst, string &reason, int flags)
{
int sfd = -1;
int dfd = -1;
bool ret = false;
char buf[CPBSIZ];
int oflags = O_WRONLY|O_CREAT|O_TRUNC;
int sfd = -1;
int dfd = -1;
bool ret = false;
char buf[CPBSIZ];
int oflags = O_WRONLY|O_CREAT|O_TRUNC;
LOGDEB(("copyfile: %s to %s\n", src, dst));
LOGDEB(("copyfile: %s to %s\n", src, dst));
if ((sfd = open(src, O_RDONLY)) < 0) {
reason += string("open ") + src + ": " + strerror(errno);
goto out;
}
if ((sfd = ::open(src, O_RDONLY)) < 0) {
reason += string("open ") + src + ": " + strerror(errno);
goto out;
}
if (flags & COPYFILE_EXCL) {
oflags |= O_EXCL;
}
if (flags & COPYFILE_EXCL) {
oflags |= O_EXCL;
}
if ((dfd = open(dst, oflags, 0644)) < 0) {
reason += string("open/creat ") + dst + ": " + strerror(errno);
// If we fail because of an open/truncate error, we do not want to unlink
// the file, we might succeed...
flags |= COPYFILE_NOERRUNLINK;
goto out;
}
if ((dfd = ::open(dst, oflags, 0644)) < 0) {
reason += string("open/creat ") + dst + ": " + strerror(errno);
// If we fail because of an open/truncate error, we do not
// want to unlink the file, we might succeed...
flags |= COPYFILE_NOERRUNLINK;
goto out;
}
for (;;) {
int didread;
didread = read(sfd, buf, CPBSIZ);
if (didread < 0) {
reason += string("read src ") + src + ": " + strerror(errno);
goto out;
}
if (didread == 0)
break;
if (write(dfd, buf, didread) != didread) {
reason += string("write dst ") + src + ": " + strerror(errno);
goto out;
}
}
for (;;) {
int didread;
didread = ::read(sfd, buf, CPBSIZ);
if (didread < 0) {
reason += string("read src ") + src + ": " + strerror(errno);
goto out;
}
if (didread == 0)
break;
if (::write(dfd, buf, didread) != didread) {
reason += string("write dst ") + src + ": " + strerror(errno);
goto out;
}
}
ret = true;
out:
if (ret == false && !(flags&COPYFILE_NOERRUNLINK))
unlink(dst);
if (sfd >= 0)
close(sfd);
if (dfd >= 0)
close(dfd);
return ret;
ret = true;
out:
if (ret == false && !(flags&COPYFILE_NOERRUNLINK))
::unlink(dst);
if (sfd >= 0)
::close(sfd);
if (dfd >= 0)
::close(dfd);
return ret;
}
bool stringtofile(const string& dt, const char *dst, string& reason,
int flags)
{
LOGDEB(("stringtofile:\n"));
int dfd = -1;
bool ret = false;
int oflags = O_WRONLY|O_CREAT|O_TRUNC;
LOGDEB(("stringtofile: %u bytes to %s\n", (unsigned int)dt.size(), dst));
if (flags & COPYFILE_EXCL) {
oflags |= O_EXCL;
}
if ((dfd = ::open(dst, oflags, 0644)) < 0) {
reason += string("open/creat ") + dst + ": " + strerror(errno);
// If we fail because of an open/truncate error, we do not
// want to unlink the file, we might succeed...
flags |= COPYFILE_NOERRUNLINK;
goto out;
}
if (::write(dfd, dt.c_str(), size_t(dt.size())) != ssize_t(dt.size())) {
reason += string("write dst ") + ": " + strerror(errno);
goto out;
}
ret = true;
out:
if (ret == false && !(flags&COPYFILE_NOERRUNLINK))
::unlink(dst);
if (dfd >= 0)
::close(dfd);
return ret;
}
bool renameormove(const char *src, const char *dst, string &reason)
@ -117,13 +153,13 @@ bool renameormove(const char *src, const char *dst, string &reason)
// of reasons
if ((st1.st_mode & 0777) != (st.st_mode & 0777)) {
if (chmod(dst, st.st_mode&0777) != 0) {
reason += string("Chmod ") + dst + "Error : " + strerror(errno);
}
reason += string("Chmod ") + dst + "Error : " + strerror(errno);
}
}
if (st.st_uid != st1.st_uid || st.st_gid != st1.st_gid) {
if (chown(dst, st.st_uid, st.st_gid) != 0) {
reason += string("Chown ") + dst + "Error : " + strerror(errno);
}
reason += string("Chown ") + dst + "Error : " + strerror(errno);
}
}
struct timeval times[2];
times[0].tv_sec = st.st_atime;
@ -161,11 +197,11 @@ static int op_flags;
static const char *thisprog;
static char usage [] =
"trcopyfile [-m] src dst\n"
" -m : move instead of copying\n"
" -e : fail if dest exists (only for copy)\n"
"\n"
;
"trcopyfile [-m] src dst\n"
" -m : move instead of copying\n"
" -e : fail if dest exists (only for copy)\n"
"\n"
;
static void
Usage(void)
{
@ -185,9 +221,9 @@ int main(int argc, const char **argv)
Usage();
while (**argv)
switch (*(*argv)++) {
case 'm': op_flags |= OPT_m; break;
case 'e': op_flags |= OPT_e; break;
default: Usage(); break;
case 'm': op_flags |= OPT_m; break;
case 'e': op_flags |= OPT_e; break;
default: Usage(); break;
}
argc--; argv++;
}
@ -211,11 +247,11 @@ int main(int argc, const char **argv)
cerr << reason << endl;
exit(1);
} else {
cout << "Succeeded" << endl;
if (!reason.empty()) {
cout << "Warnings: " << reason << endl;
}
exit(0);
cout << "Succeeded" << endl;
if (!reason.empty()) {
cout << "Warnings: " << reason << endl;
}
exit(0);
}
}

4
src/utils/copyfile.h
View file

@ -34,6 +34,10 @@ enum CopyfileFlags {COPYFILE_NONE = 0,
extern bool copyfile(const char *src, const char *dst, std::string &reason,
int flags = 0);
/** Save c++ string to file */
extern bool stringtofile(const std::string& dt, const char *dst,
std::string& reason, int flags = 0);
/** Try to rename src. If this fails (different devices) copy then unlink src */
extern bool renameormove(const char *src, const char *dst, std::string &reason);