/* Copyright (C) 2006 J.F.Dockes
 *   This program is free software; you can redistribute it and/or modify
 *   it under the terms of the GNU General Public License as published by
 *   the Free Software Foundation; either version 2 of the License, or
 *   (at your option) any later version.
 *
 *   This program is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *   GNU General Public License for more details.
 *
 *   You should have received a copy of the GNU General Public License
 *   along with this program; if not, write to the
 *   Free Software Foundation, Inc.,
 *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 */

// Handle translation from rcl's SearchData structures to Xapian Queries

#include "autoconfig.h"

#include <stdio.h>

#include <string>
#include <vector>
#include <algorithm>
#include <sstream>
#include <iostream>
using namespace std;

#include "xapian.h"

#include "cstr.h"
#include "rcldb.h"
#include "rcldb_p.h"
#include "searchdata.h"
#include "debuglog.h"
#include "smallut.h"
#include "textsplit.h"
#include "unacpp.h"
#include "utf8iter.h"
#include "stoplist.h"
#include "rclconfig.h"
#include "termproc.h"
#include "synfamily.h"
#include "stemdb.h"
#include "expansiondbs.h"
#include "base64.h"
#include "daterange.h"

namespace Rcl {

typedef  vector<SearchDataClause *>::iterator qlist_it_t;
typedef  vector<SearchDataClause *>::const_iterator qlist_cit_t;

void SearchData::commoninit()
{
    m_haveDates = false;
    m_maxSize = size_t(-1);
    m_minSize = size_t(-1);
    m_haveWildCards = false;
    m_autodiacsens = false;
    m_autocasesens = true;
    m_maxexp = 10000;
    m_maxcl = 100000;
    m_softmaxexpand = -1;
}

SearchData::~SearchData() 
{
    LOGDEB0(("SearchData::~SearchData\n"));
    for (qlist_it_t it = m_query.begin(); it != m_query.end(); it++)
        delete *it;
}

// This is called by the GUI simple search if the option is set: add
// (OR) phrase to a query (if it is simple enough) so that results
// where the search terms are close and in order will come up on top.
// We remove very common terms from the query to avoid performance issues.
bool SearchData::maybeAddAutoPhrase(Rcl::Db& db, double freqThreshold)
{
    LOGDEB0(("SearchData::maybeAddAutoPhrase()\n"));
    // cerr << "BEFORE SIMPLIFY\n"; dump(cerr);
    simplify();
    // cerr << "AFTER SIMPLIFY\n"; dump(cerr);

    if (!m_query.size()) {
        LOGDEB2(("SearchData::maybeAddAutoPhrase: empty query\n"));
        return false;
    }

    string field;
    vector<string> words;
    // Walk the clause list. If this is not an AND list, we find any
    // non simple clause or different field names, bail out.
    for (qlist_it_t it = m_query.begin(); it != m_query.end(); it++) {
        SClType tp = (*it)->m_tp;
        if (tp != SCLT_AND) {
            LOGDEB2(("SearchData::maybeAddAutoPhrase: wrong tp %d\n", tp));
            return false;
        }
        SearchDataClauseSimple *clp = 
            dynamic_cast<SearchDataClauseSimple*>(*it);
        if (clp == 0) {
            LOGDEB2(("SearchData::maybeAddAutoPhrase: dyncast failed\n"));
            return false;
        }
        if (it == m_query.begin()) {
            field = clp->getfield();
        } else {
            if (clp->getfield().compare(field)) {
                LOGDEB2(("SearchData::maybeAddAutoPhrase: diff. fields\n"));
                return false;
            }
        }

        // If there are wildcards or quotes in there, bail out
        if (clp->gettext().find_first_of("\"*[?") != string::npos) { 
            LOGDEB2(("SearchData::maybeAddAutoPhrase: wildcards\n"));
            return false;
        }

        // Do a simple word-split here, not the full-blown
        // textsplit. Spans of stopwords should not be trimmed later
        // in this function, they will be properly split when the
        // phrase gets processed by toNativeQuery() later on.
        vector<string> wl;
        stringToStrings(clp->gettext(), wl);
        words.insert(words.end(), wl.begin(), wl.end());
    }


    // Trim the word list by eliminating very frequent terms
    // (increasing the slack as we do it):
    int slack = 0;
    int doccnt = db.docCnt();
    if (!doccnt)
        doccnt = 1;
    string swords;
    for (vector<string>::iterator it = words.begin(); 
         it != words.end(); it++) {
        double freq = double(db.termDocCnt(*it)) / doccnt;
        if (freq < freqThreshold) {
            if (!swords.empty())
                swords.append(1, ' ');
            swords += *it;
        } else {
            LOGDEB0(("SearchData::Autophrase: [%s] too frequent (%.2f %%)\n", 
                    it->c_str(), 100 * freq));
            slack++;
        }
    }
    
    // We can't make a phrase with a single word :)
    int nwords = TextSplit::countWords(swords);
    if (nwords <= 1) {
        LOGDEB2(("SearchData::maybeAddAutoPhrase: ended with 1 word\n"));
        return false;
    }

    // Increase the slack: we want to be a little more laxist than for
    // an actual user-entered phrase
    slack += 1 + nwords / 3;
    
    m_autophrase = RefCntr<SearchDataClauseDist>(
        new SearchDataClauseDist(SCLT_PHRASE, swords, slack, field));
    return true;
}

// Add clause to current list. OR lists cant have EXCL clauses.
bool SearchData::addClause(SearchDataClause* cl)
{
    if (m_tp == SCLT_OR && cl->getexclude()) {
        LOGERR(("SearchData::addClause: cant add EXCL to OR list\n"));
        m_reason = "No Negative (AND_NOT) clauses allowed in OR queries";
        return false;
    }
    cl->setParent(this);
    m_haveWildCards = m_haveWildCards || cl->m_haveWildCards;
    m_query.push_back(cl);
    return true;
}

// Am I a file name only search ? This is to turn off term highlighting
bool SearchData::fileNameOnly() 
{
    for (qlist_it_t it = m_query.begin(); it != m_query.end(); it++)
        if (!(*it)->isFileName())
            return false;
    return true;
}

void SearchData::simplify()
{
    for (unsigned int i = 0; i < m_query.size(); i++) {
        if (m_query[i]->m_tp != SCLT_SUB)
            continue;
        //C[est ce dyncast qui crashe??
        SearchDataClauseSub *clsubp = 
            dynamic_cast<SearchDataClauseSub*>(m_query[i]);
        if (clsubp == 0) {
            // ??
            continue;
        }
        if (clsubp->getSub()->m_tp != m_tp)
            continue;

        clsubp->getSub()->simplify();

        // If this subquery has special attributes, it's not a
        // candidate for collapsing
        if (!clsubp->getSub()->m_filetypes.empty() || 
            !clsubp->getSub()->m_nfiletypes.empty() ||
            clsubp->getSub()->m_haveDates || 
            clsubp->getSub()->m_maxSize != size_t(-1) ||
            clsubp->getSub()->m_minSize != size_t(-1) ||
            clsubp->getSub()->m_haveWildCards)
            continue;

        bool allsametp = true;
        for (qlist_it_t it1 = clsubp->getSub()->m_query.begin(); 
             it1 != clsubp->getSub()->m_query.end(); it1++) {
            // We want all AND or OR clause, and same as our conjunction
            if (((*it1)->getTp() != SCLT_AND && (*it1)->getTp() != SCLT_OR) || 
                (*it1)->getTp() != m_tp) {
                allsametp = false;
                break;
            }
        }
        if (!allsametp)
            continue;

        // All ok: delete the clause_sub, and insert the queries from
        // its searchdata in its place
        m_query.erase(m_query.begin() + i);
        m_query.insert(m_query.begin() + i, 
                       clsubp->getSub()->m_query.begin(),
                       clsubp->getSub()->m_query.end());
        for (unsigned int j = i; 
             j < i + clsubp->getSub()->m_query.size(); j++) {
            m_query[j]->setParent(this);
        }
        i += clsubp->getSub()->m_query.size() - 1;

        // We don't want the clauses to be deleted when the parent is, as we
        // know own them.
        clsubp->getSub()->m_query.clear();
        delete clsubp;
    }
}

bool SearchData::singleSimple()
{
    if (m_query.size() != 1 || !m_filetypes.empty() || !m_nfiletypes.empty() ||
        m_haveDates || m_maxSize != size_t(-1) || m_minSize != size_t(-1) ||
        m_haveWildCards)
        return false;
    SearchDataClause *clp = *m_query.begin();
    if (clp->getTp() != SCLT_AND && clp->getTp() != SCLT_OR) {
        return false;
    }
    return true;
}

// Extract all term data
void SearchData::getTerms(HighlightData &hld) const
{
    for (qlist_cit_t it = m_query.begin(); it != m_query.end(); it++)
        (*it)->getTerms(hld);
    return;
}

void SearchData::dump(ostream& o) const
{
    o << "SearchData: " << " qs " << int(m_query.size()) << 
        " ft " << m_filetypes.size() << " nft " << m_nfiletypes.size() << 
        " hd " << m_haveDates << " maxs " << int(m_maxSize) << " mins " << 
        int(m_minSize) << " wc " << m_haveWildCards << "\n";
    for (std::vector<SearchDataClause*>::const_iterator it =
             m_query.begin(); it != m_query.end(); it++) {
        (*it)->dump(o);
        o << "\n";
    }
    o << "\n";
}

void SearchDataClause::dump(ostream& o) const
{
    o << "SearchDataClause??";
}

void SearchDataClauseSimple::dump(ostream& o) const
{
    o << "ClauseSimple: ";
    if (m_exclude)
        o << "- ";
    o << "[" ;
    if (!m_field.empty())
        o << m_field << " : ";
    o << m_text << "]";
}

void SearchDataClauseFilename::dump(ostream& o) const
{
    o << "ClauseFN: ";
    if (m_exclude)
        o << " - ";
    o << "[" << m_text << "]";
}

void SearchDataClausePath::dump(ostream& o) const
{
    o << "ClausePath: ";
    if (m_exclude)
        o << " - ";
    o << "[" << m_text << "]";
}

void SearchDataClauseDist::dump(ostream& o) const
{
    if (m_tp == SCLT_NEAR)
        o << "ClauseDist: NEAR: ";
    else
        o << "ClauseDist: PHRA: ";
            
    if (m_exclude)
        o << " - ";
    o << "[";
    if (!m_field.empty())
        o << m_field << " : ";
    o << m_text << "]";
}

void SearchDataClauseSub::dump(ostream& o) const
{
    o << "ClauseSub {\n";
    m_sub.getconstptr()->dump(o);
    o << "}";
}

} // Namespace Rcl