data struct for synonym groups
This commit is contained in:
Jean-Francois Dockes
parent
776539571b
commit
b1c5f5b96e
2 changed files with 264 additions and 0 deletions
224
src/common/syngroups.cpp
Normal file
224
src/common/syngroups.cpp
Normal file
|
|
@ -0,0 +1,224 @@
|
|||
/* Copyright (C) 2014 J.F.Dockes
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 2 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the
|
||||
* Free Software Foundation, Inc.,
|
||||
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
*/
|
||||
#ifndef TEST_SYNGROUPS
|
||||
#include "autoconfig.h"
|
||||
|
||||
#include "syngroups.h"
|
||||
|
||||
#include "debuglog.h"
|
||||
#include "smallut.h"
|
||||
|
||||
#include <errno.h>
|
||||
#include UNORDERED_MAP_INCLUDE
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <cstring>
|
||||
|
||||
using namespace std;
|
||||
|
||||
class SynGroups::Internal {
|
||||
public:
|
||||
Internal() : ok(false) {
|
||||
}
|
||||
bool ok;
|
||||
// Term to group num
|
||||
STD_UNORDERED_MAP<string, int> terms;
|
||||
// Group num to group
|
||||
STD_UNORDERED_MAP<int, vector<string> > groups;
|
||||
};
|
||||
bool SynGroups::ok()
|
||||
{
|
||||
return m && m->ok;
|
||||
}
|
||||
SynGroups::~SynGroups()
|
||||
{
|
||||
delete m;
|
||||
}
|
||||
|
||||
const int LL = 1024;
|
||||
|
||||
SynGroups::SynGroups(const string& fn)
|
||||
: m(new Internal)
|
||||
{
|
||||
if (!m) {
|
||||
LOGERR(("SynGroups::SynGroups:: new Internal failed: no mem ?\n"));
|
||||
return;
|
||||
}
|
||||
|
||||
ifstream input;
|
||||
input.open(fn.c_str(), ios::in);
|
||||
if (!input.is_open()) {
|
||||
LOGERR(("SynGroups::SynGroups:: could not open %s errno %d\n",
|
||||
fn.c_str(), errno));
|
||||
return;
|
||||
}
|
||||
|
||||
char cline[LL];
|
||||
bool appending = false;
|
||||
string line;
|
||||
bool eof = false;
|
||||
int lnum = 0;
|
||||
|
||||
for (;;) {
|
||||
cline[0] = 0;
|
||||
input.getline(cline, LL-1);
|
||||
if (!input.good()) {
|
||||
if (input.bad()) {
|
||||
LOGDEB(("Parse: input.bad()\n"));
|
||||
return;
|
||||
}
|
||||
// Must be eof ? But maybe we have a partial line which
|
||||
// must be processed. This happens if the last line before
|
||||
// eof ends with a backslash, or there is no final \n
|
||||
eof = true;
|
||||
}
|
||||
lnum++;
|
||||
|
||||
{
|
||||
int ll = strlen(cline);
|
||||
while (ll > 0 && (cline[ll-1] == '\n' || cline[ll-1] == '\r')) {
|
||||
cline[ll-1] = 0;
|
||||
ll--;
|
||||
}
|
||||
}
|
||||
|
||||
if (appending)
|
||||
line += cline;
|
||||
else
|
||||
line = cline;
|
||||
|
||||
// Note that we trim whitespace before checking for backslash-eol
|
||||
// This avoids invisible whitespace problems.
|
||||
trimstring(line);
|
||||
if (line.empty() || line.at(0) == '#') {
|
||||
if (eof)
|
||||
break;
|
||||
continue;
|
||||
}
|
||||
if (line[line.length() - 1] == '\\') {
|
||||
line.erase(line.length() - 1);
|
||||
appending = true;
|
||||
continue;
|
||||
}
|
||||
appending = false;
|
||||
|
||||
vector<string> words;
|
||||
if (!stringToStrings(line, words)) {
|
||||
LOGERR(("SynGroups::SynGroups: %s: bad line %d: %s\n",
|
||||
fn.c_str(), lnum, line.c_str()));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (words.empty())
|
||||
continue;
|
||||
if (words.size() == 1) {
|
||||
LOGDEB(("SynGroups::SynGroups: single term group at line %d ??\n",
|
||||
lnum));
|
||||
continue;
|
||||
}
|
||||
|
||||
m->groups[lnum] = words;
|
||||
for (vector<string>::const_iterator it = words.begin();
|
||||
it != words.end(); it++) {
|
||||
m->terms[*it] = lnum;
|
||||
}
|
||||
}
|
||||
m->ok = true;
|
||||
}
|
||||
|
||||
vector<string> SynGroups::getgroup(const string& term)
|
||||
{
|
||||
vector<string> ret;
|
||||
if (!ok())
|
||||
return ret;
|
||||
|
||||
STD_UNORDERED_MAP<string, int>::const_iterator it1 = m->terms.find(term);
|
||||
if (it1 == m->terms.end()) {
|
||||
LOGDEB1(("SynGroups::getgroup: [%s] not found in direct map\n",
|
||||
term.c_str()));
|
||||
return ret;
|
||||
}
|
||||
|
||||
// Group num to group
|
||||
STD_UNORDERED_MAP<int, vector<string> >::const_iterator it2 =
|
||||
m->groups.find(it1->second);
|
||||
|
||||
if (it2 == m->groups.end()) {
|
||||
LOGERR(("SynGroups::getgroup: %s found in direct map but no group\n",
|
||||
term.c_str()));
|
||||
return ret;
|
||||
}
|
||||
return it2->second;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
#include "syngroups.h"
|
||||
|
||||
#include <string>
|
||||
#include <iostream>
|
||||
#include <vector>
|
||||
#include <cstdlib>
|
||||
#include <cstdio>
|
||||
|
||||
using namespace std;
|
||||
|
||||
static char *thisprog;
|
||||
|
||||
static char usage [] =
|
||||
"syngroups <synfilename> <word>\n"
|
||||
" \n\n"
|
||||
;
|
||||
static void Usage(void)
|
||||
{
|
||||
fprintf(stderr, "%s: usage:\n%s", thisprog, usage);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
static int op_flags;
|
||||
#define OPT_MOINS 0x1
|
||||
#define OPT_s 0x2
|
||||
#define OPT_b 0x4
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
thisprog = argv[0];
|
||||
argc--; argv++;
|
||||
|
||||
if (argc != 2) {
|
||||
Usage();
|
||||
}
|
||||
string fn = *argv++;argc--;
|
||||
string word = *argv++;argc--;
|
||||
|
||||
SynGroups syns(fn);
|
||||
if (!syns.ok()) {
|
||||
cerr << "Initialization failed\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
vector<string> group = syns.getgroup(word);
|
||||
cout << group.size() << " terms in group\n";
|
||||
for (vector<string>::const_iterator it = group.begin();
|
||||
it != group.end(); it++) {
|
||||
cout << *it << " ";
|
||||
}
|
||||
cout << endl;
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif
|
||||
40
src/common/syngroups.h
Normal file
40
src/common/syngroups.h
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
/* Copyright (C) 2015 J.F.Dockes
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 2 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the
|
||||
* Free Software Foundation, Inc.,
|
||||
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
*/
|
||||
|
||||
#ifndef _SYNGROUPS_H_INCLUDED_
|
||||
#define _SYNGROUPS_H_INCLUDED_
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
// Manage synonym groups. This is very different from stemming and
|
||||
// case/diac expansion because there is no reference form: all terms
|
||||
// in a group are equivalent.
|
||||
class SynGroups {
|
||||
public:
|
||||
SynGroups(const std::string& fname);
|
||||
~SynGroups();
|
||||
std::vector<std::string> getgroup(const std::string& term);
|
||||
bool ok();
|
||||
private:
|
||||
class Internal;
|
||||
Internal *m;
|
||||
SynGroups(const SynGroups&);
|
||||
SynGroups& operator=(const SynGroups&);
|
||||
};
|
||||
|
||||
#endif /* _SYNGROUPS_H_INCLUDED_ */
|
||||
Loading…
Add table
Add a link
Reference in a new issue