--- /dev/null
+#ifdef HAVE_CONFIG_H
+# include "config.h"
+#endif
+
+#include <algorithm>
+#include <cstring>
+#include <cctype>
+
+#include <sys/stat.h>
+#include <zlib.h>
+#include <glib/gstdio.h>
+
+#include "distance.h"
+#include "file.hpp"
+#include "mapfile.hpp"
+
+#include "lib.h"
+
+// Notice: read src/tools/DICTFILE_FORMAT for the dictionary
+// file's format information!
+
+
+static inline bool bIsVowel(gchar inputchar)
+{
+ gchar ch = g_ascii_toupper(inputchar);
+ return ( ch == 'A' || ch == 'E' || ch == 'I' || ch == 'O' || ch == 'U' );
+}
+
+static bool bIsPureEnglish(const gchar *str)
+{
+ // i think this should work even when it is UTF8 string :).
+ for (int i = 0; str[i] != 0; i++)
+ //if(str[i]<0)
+ //if(str[i]<32 || str[i]>126) // tab equal 9,so this is not OK.
+ // Better use isascii() but not str[i]<0 while char is default unsigned in arm
+ if (!isascii(str[i]))
+ return false;
+ return true;
+}
+
+static inline gint stardict_strcmp(const gchar *s1, const gchar *s2)
+{
+ gint a = g_ascii_strcasecmp(s1, s2);
+ if (a == 0)
+ return strcmp(s1, s2);
+ else
+ return a;
+}
+
+bool DictInfo::load_from_ifo_file(const std::string& ifofilename,
+ bool istreedict)
+{
+ ifo_file_name = ifofilename;
+ gchar *buffer;
+ if (!g_file_get_contents(ifofilename.c_str(), &buffer, NULL, NULL))
+ return false;
+
+#define TREEDICT_MAGIC_DATA "StarDict's treedict ifo file\nversion=2.4.2\n"
+#define DICT_MAGIC_DATA "StarDict's dict ifo file\nversion=2.4.2\n"
+
+ const gchar *magic_data = istreedict ? TREEDICT_MAGIC_DATA : DICT_MAGIC_DATA;
+ if (!g_str_has_prefix(buffer, magic_data))
+ {
+ g_free(buffer);
+ return false;
+ }
+
+ gchar *p1, *p2, *p3;
+
+ p1 = buffer + strlen(magic_data) - 1;
+
+ p2 = strstr(p1, "\nwordcount=");
+ if (!p2)
+ {
+ g_free(buffer);
+ return false;
+ }
+
+ p3 = strchr(p2 + sizeof("\nwordcount=") - 1, '\n');
+ gchar *tmpstr = (gchar *)g_memdup(p2 + sizeof("\nwordcount=") - 1, p3 - (p2 + sizeof("\nwordcount=") - 1) + 1);
+ tmpstr[p3 - (p2 + sizeof("\nwordcount=") - 1)] = '\0';
+ wordcount = atol(tmpstr);
+ g_free(tmpstr);
+
+ if (istreedict)
+ {
+ p2 = strstr(p1, "\ntdxfilesize=");
+ if (!p2)
+ {
+ g_free(buffer);
+ return false;
+ }
+ p3 = strchr(p2 + sizeof("\ntdxfilesize=") - 1, '\n');
+ tmpstr = (gchar *)g_memdup(p2 + sizeof("\ntdxfilesize=") - 1, p3 - (p2 + sizeof("\ntdxfilesize=") - 1) + 1);
+ tmpstr[p3 - (p2 + sizeof("\ntdxfilesize=") - 1)] = '\0';
+ index_file_size = atol(tmpstr);
+ g_free(tmpstr);
+ }
+ else
+ {
+
+ p2 = strstr(p1, "\nidxfilesize=");
+ if (!p2)
+ {
+ g_free(buffer);
+ return false;
+ }
+
+ p3 = strchr(p2 + sizeof("\nidxfilesize=") - 1, '\n');
+ tmpstr = (gchar *)g_memdup(p2 + sizeof("\nidxfilesize=") - 1, p3 - (p2 + sizeof("\nidxfilesize=") - 1) + 1);
+ tmpstr[p3 - (p2 + sizeof("\nidxfilesize=") - 1)] = '\0';
+ index_file_size = atol(tmpstr);
+ g_free(tmpstr);
+ }
+
+ p2 = strstr(p1, "\nbookname=");
+
+ if (!p2)
+ {
+ g_free(buffer);
+ return false;
+ }
+
+ p2 = p2 + sizeof("\nbookname=") - 1;
+ p3 = strchr(p2, '\n');
+ bookname.assign(p2, p3 - p2);
+
+ p2 = strstr(p1, "\nauthor=");
+ if (p2)
+ {
+ p2 = p2 + sizeof("\nauthor=") - 1;
+ p3 = strchr(p2, '\n');
+ author.assign(p2, p3 - p2);
+ }
+
+ p2 = strstr(p1, "\nemail=");
+ if (p2)
+ {
+ p2 = p2 + sizeof("\nemail=") - 1;
+ p3 = strchr(p2, '\n');
+ email.assign(p2, p3 - p2);
+ }
+
+ p2 = strstr(p1, "\nwebsite=");
+ if (p2)
+ {
+ p2 = p2 + sizeof("\nwebsite=") - 1;
+ p3 = strchr(p2, '\n');
+ website.assign(p2, p3 - p2);
+ }
+
+ p2 = strstr(p1, "\ndate=");
+ if (p2)
+ {
+ p2 = p2 + sizeof("\ndate=") - 1;
+ p3 = strchr(p2, '\n');
+ date.assign(p2, p3 - p2);
+ }
+
+ p2 = strstr(p1, "\ndescription=");
+ if (p2)
+ {
+ p2 = p2 + sizeof("\ndescription=") - 1;
+ p3 = strchr(p2, '\n');
+ description.assign(p2, p3 - p2);
+ }
+
+ p2 = strstr(p1, "\nsametypesequence=");
+ if (p2)
+ {
+ p2 += sizeof("\nsametypesequence=") - 1;
+ p3 = strchr(p2, '\n');
+ sametypesequence.assign(p2, p3 - p2);
+ }
+
+ g_free(buffer);
+
+ return true;
+}
+//===================================================================
+DictBase::DictBase()
+{
+ dictfile = NULL;
+ cache_cur = 0;
+}
+
+DictBase::~DictBase()
+{
+ if (dictfile)
+ fclose(dictfile);
+}
+
+gchar* DictBase::GetWordData(guint32 idxitem_offset, guint32 idxitem_size)
+{
+ for (int i = 0; i < WORDDATA_CACHE_NUM; i++)
+ if (cache[i].data && cache[i].offset == idxitem_offset)
+ return cache[i].data;
+
+ if (dictfile)
+ fseek(dictfile, idxitem_offset, SEEK_SET);
+
+ gchar *data;
+ if (!sametypesequence.empty())
+ {
+ gchar *origin_data = (gchar *)g_malloc(idxitem_size);
+
+ if (dictfile)
+ fread(origin_data, idxitem_size, 1, dictfile);
+ else
+ dictdzfile->read(origin_data, idxitem_offset, idxitem_size);
+
+ guint32 data_size;
+ gint sametypesequence_len = sametypesequence.length();
+ //there have sametypesequence_len char being omitted.
+ data_size = idxitem_size + sizeof(guint32) + sametypesequence_len;
+ //if the last item's size is determined by the end up '\0',then +=sizeof(gchar);
+ //if the last item's size is determined by the head guint32 type data,then +=sizeof(guint32);
+ switch (sametypesequence[sametypesequence_len - 1])
+ {
+ case 'm':
+ case 't':
+ case 'y':
+ case 'l':
+ case 'g':
+ case 'x':
+ data_size += sizeof(gchar);
+ break;
+ case 'W':
+ case 'P':
+ data_size += sizeof(guint32);
+ break;
+ default:
+ if (g_ascii_isupper(sametypesequence[sametypesequence_len - 1]))
+ data_size += sizeof(guint32);
+ else
+ data_size += sizeof(gchar);
+ break;
+ }
+ data = (gchar *)g_malloc(data_size);
+ gchar *p1, *p2;
+ p1 = data + sizeof(guint32);
+ p2 = origin_data;
+ guint32 sec_size;
+ //copy the head items.
+ for (int i = 0; i < sametypesequence_len - 1; i++)
+ {
+ *p1 = sametypesequence[i];
+ p1 += sizeof(gchar);
+ switch (sametypesequence[i])
+ {
+ case 'm':
+ case 't':
+ case 'y':
+ case 'l':
+ case 'g':
+ case 'x':
+ sec_size = strlen(p2) + 1;
+ memcpy(p1, p2, sec_size);
+ p1 += sec_size;
+ p2 += sec_size;
+ break;
+ case 'W':
+ case 'P':
+ sec_size = *reinterpret_cast<guint32 *>(p2);
+ sec_size += sizeof(guint32);
+ memcpy(p1, p2, sec_size);
+ p1 += sec_size;
+ p2 += sec_size;
+ break;
+ default:
+ if (g_ascii_isupper(sametypesequence[i]))
+ {
+ sec_size = *reinterpret_cast<guint32 *>(p2);
+ sec_size += sizeof(guint32);
+ }
+ else
+ {
+ sec_size = strlen(p2) + 1;
+ }
+ memcpy(p1, p2, sec_size);
+ p1 += sec_size;
+ p2 += sec_size;
+ break;
+ }
+ }
+ //calculate the last item 's size.
+ sec_size = idxitem_size - (p2 - origin_data);
+ *p1 = sametypesequence[sametypesequence_len - 1];
+ p1 += sizeof(gchar);
+ switch (sametypesequence[sametypesequence_len - 1])
+ {
+ case 'm':
+ case 't':
+ case 'y':
+ case 'l':
+ case 'g':
+ case 'x':
+ memcpy(p1, p2, sec_size);
+ p1 += sec_size;
+ *p1 = '\0'; //add the end up '\0';
+ break;
+ case 'W':
+ case 'P':
+ *reinterpret_cast<guint32 *>(p1) = sec_size;
+ p1 += sizeof(guint32);
+ memcpy(p1, p2, sec_size);
+ break;
+ default:
+ if (g_ascii_isupper(sametypesequence[sametypesequence_len - 1]))
+ {
+ *reinterpret_cast<guint32 *>(p1) = sec_size;
+ p1 += sizeof(guint32);
+ memcpy(p1, p2, sec_size);
+ }
+ else
+ {
+ memcpy(p1, p2, sec_size);
+ p1 += sec_size;
+ *p1 = '\0';
+ }
+ break;
+ }
+ g_free(origin_data);
+ *reinterpret_cast<guint32 *>(data) = data_size;
+ }
+ else
+ {
+ data = (gchar *)g_malloc(idxitem_size + sizeof(guint32));
+ if (dictfile)
+ fread(data + sizeof(guint32), idxitem_size, 1, dictfile);
+ else
+ dictdzfile->read(data + sizeof(guint32), idxitem_offset, idxitem_size);
+ *reinterpret_cast<guint32 *>(data) = idxitem_size + sizeof(guint32);
+ }
+ g_free(cache[cache_cur].data);
+
+ cache[cache_cur].data = data;
+ cache[cache_cur].offset = idxitem_offset;
+ cache_cur++;
+ if (cache_cur == WORDDATA_CACHE_NUM)
+ cache_cur = 0;
+ return data;
+}
+
+inline bool DictBase::containSearchData()
+{
+ if (sametypesequence.empty())
+ return true;
+
+ return sametypesequence.find_first_of("mlgxty") != std::string::npos;
+}
+
+bool DictBase::SearchData(std::vector<std::string> &SearchWords, guint32 idxitem_offset, guint32 idxitem_size, gchar *origin_data)
+{
+ int nWord = SearchWords.size();
+ std::vector<bool> WordFind(nWord, false);
+ int nfound = 0;
+
+ if (dictfile)
+ fseek(dictfile, idxitem_offset, SEEK_SET);
+ if (dictfile)
+ fread(origin_data, idxitem_size, 1, dictfile);
+ else
+ dictdzfile->read(origin_data, idxitem_offset, idxitem_size);
+ gchar *p = origin_data;
+ guint32 sec_size;
+ int j;
+ if (!sametypesequence.empty())
+ {
+ gint sametypesequence_len = sametypesequence.length();
+ for (int i = 0; i < sametypesequence_len - 1; i++)
+ {
+ switch (sametypesequence[i])
+ {
+ case 'm':
+ case 't':
+ case 'y':
+ case 'l':
+ case 'g':
+ case 'x':
+ for (j = 0; j < nWord; j++)
+ if (!WordFind[j] && strstr(p, SearchWords[j].c_str()))
+ {
+ WordFind[j] = true;
+ ++nfound;
+ }
+
+
+ if (nfound == nWord)
+ return true;
+ sec_size = strlen(p) + 1;
+ p += sec_size;
+ break;
+ default:
+ if (g_ascii_isupper(sametypesequence[i]))
+ {
+ sec_size = *reinterpret_cast<guint32 *>(p);
+ sec_size += sizeof(guint32);
+ }
+ else
+ {
+ sec_size = strlen(p) + 1;
+ }
+ p += sec_size;
+ }
+ }
+ switch (sametypesequence[sametypesequence_len - 1])
+ {
+ case 'm':
+ case 't':
+ case 'y':
+ case 'l':
+ case 'g':
+ case 'x':
+ sec_size = idxitem_size - (p - origin_data);
+ for (j = 0; j < nWord; j++)
+ if (!WordFind[j] &&
+ g_strstr_len(p, sec_size, SearchWords[j].c_str()))
+ {
+ WordFind[j] = true;
+ ++nfound;
+ }
+
+
+ if (nfound == nWord)
+ return true;
+ break;
+ }
+ }
+ else
+ {
+ while (guint32(p - origin_data) < idxitem_size)
+ {
+ switch (*p)
+ {
+ case 'm':
+ case 't':
+ case 'y':
+ case 'l':
+ case 'g':
+ case 'x':
+ for (j = 0; j < nWord; j++)
+ if (!WordFind[j] && strstr(p, SearchWords[j].c_str()))
+ {
+ WordFind[j] = true;
+ ++nfound;
+ }
+
+ if (nfound == nWord)
+ return true;
+ sec_size = strlen(p) + 1;
+ p += sec_size;
+ break;
+ default:
+ if (g_ascii_isupper(*p))
+ {
+ sec_size = *reinterpret_cast<guint32 *>(p);
+ sec_size += sizeof(guint32);
+ }
+ else
+ {
+ sec_size = strlen(p) + 1;
+ }
+ p += sec_size;
+ }
+ }
+ }
+ return false;
+}
+
+class offset_index : public index_file
+{
+ public:
+ offset_index() : idxfile(NULL)
+ {}
+ ~offset_index();
+ bool load(const std::string& url, gulong wc, gulong fsize);
+ const gchar *get_key(glong idx);
+ void get_data(glong idx);
+ const gchar *get_key_and_data(glong idx);
+ bool lookup(const char *str, glong &idx);
+ private:
+ static const gint ENTR_PER_PAGE = 32;
+ static const char *CACHE_MAGIC;
+
+ std::vector<guint32> wordoffset;
+ FILE *idxfile;
+ gulong wordcount;
+
+ gchar wordentry_buf[256 + sizeof(guint32)*2]; // The length of "word_str" should be less than 256. See src/tools/DICTFILE_FORMAT.
+ struct index_entry
+ {
+ glong idx;
+ std::string keystr;
+ void assign(glong i, const std::string& str)
+ {
+ idx = i;
+ keystr.assign(str);
+ }
+ };
+ index_entry first, last, middle, real_last;
+
+ struct page_entry
+ {
+ gchar *keystr;
+ guint32 off, size;
+ };
+ std::vector<gchar> page_data;
+ struct page_t
+ {
+ glong idx;
+ page_entry entries[ENTR_PER_PAGE];
+
+ page_t(): idx( -1)
+ {}
+ void fill(gchar *data, gint nent, glong idx_);
+ }
+ page;
+ gulong load_page(glong page_idx);
+ const gchar *read_first_on_page_key(glong page_idx);
+ const gchar *get_first_on_page_key(glong page_idx);
+ bool load_cache(const std::string& url);
+ bool save_cache(const std::string& url);
+ static strlist_t get_cache_variant(const std::string& url);
+};
+
+const char *offset_index::CACHE_MAGIC = "StarDict's Cache, Version: 0.1";
+
+class wordlist_index : public index_file
+{
+ public:
+ wordlist_index() : idxdatabuf(NULL)
+ {}
+ ~wordlist_index();
+ bool load(const std::string& url, gulong wc, gulong fsize);
+ const gchar *get_key(glong idx);
+ void get_data(glong idx);
+ const gchar *get_key_and_data(glong idx);
+ bool lookup(const char *str, glong &idx);
+ private:
+ gchar *idxdatabuf;
+ std::vector<gchar *> wordlist;
+};
+
+void offset_index::page_t::fill(gchar *data, gint nent, glong idx_)
+{
+ idx = idx_;
+ gchar *p = data;
+ glong len;
+ for (gint i = 0; i < nent; ++i)
+ {
+ entries[i].keystr = p;
+ len = strlen(p);
+ p += len + 1;
+ entries[i].off = g_ntohl(*reinterpret_cast<guint32 *>(p));
+ p += sizeof(guint32);
+ entries[i].size = g_ntohl(*reinterpret_cast<guint32 *>(p));
+ p += sizeof(guint32);
+ }
+}
+
+offset_index::~offset_index()
+{
+ if (idxfile)
+ fclose(idxfile);
+}
+
+inline const gchar *offset_index::read_first_on_page_key(glong page_idx)
+{
+ fseek(idxfile, wordoffset[page_idx], SEEK_SET);
+ guint page_size = wordoffset[page_idx + 1] - wordoffset[page_idx];
+ fread(wordentry_buf, std::min<guint>(sizeof(wordentry_buf), page_size), 1, idxfile); //TODO: check returned values, deal with word entry that strlen>255.
+ return wordentry_buf;
+}
+
+inline const gchar *offset_index::get_first_on_page_key(glong page_idx)
+{
+ if (page_idx < middle.idx)
+ {
+ if (page_idx == first.idx)
+ return first.keystr.c_str();
+ return read_first_on_page_key(page_idx);
+ }
+ else if (page_idx > middle.idx)
+ {
+ if (page_idx == last.idx)
+ return last.keystr.c_str();
+ return read_first_on_page_key(page_idx);
+ }
+ else
+ return middle.keystr.c_str();
+}
+
+bool offset_index::load_cache(const std::string& url)
+{
+ strlist_t vars = get_cache_variant(url);
+
+ for (strlist_t::const_iterator it = vars.begin(); it != vars.end(); ++it)
+ {
+ struct stat idxstat, cachestat;
+ if (g_stat(url.c_str(), &idxstat) != 0 ||
+ g_stat(it->c_str(), &cachestat) != 0)
+ continue;
+ if (cachestat.st_mtime < idxstat.st_mtime)
+ continue;
+ MapFile mf;
+ if (!mf.open(it->c_str(), cachestat.st_size))
+ continue;
+ if (strncmp(mf.begin(), CACHE_MAGIC, strlen(CACHE_MAGIC)) != 0)
+ continue;
+ memcpy(&wordoffset[0], mf.begin() + strlen(CACHE_MAGIC), wordoffset.size()*sizeof(wordoffset[0]));
+ return true;
+
+ }
+
+ return false;
+}
+
+strlist_t offset_index::get_cache_variant(const std::string& url)
+{
+ strlist_t res;
+ res.push_back(url + ".oft");
+ if (!g_file_test(g_get_user_cache_dir(), G_FILE_TEST_EXISTS) &&
+ g_mkdir(g_get_user_cache_dir(), 0700) == -1)
+ return res;
+
+ std::string cache_dir = std::string(g_get_user_cache_dir()) + G_DIR_SEPARATOR_S + "sdcv";
+
+ if (!g_file_test(cache_dir.c_str(), G_FILE_TEST_EXISTS))
+ {
+ if (g_mkdir(cache_dir.c_str(), 0700) == -1)
+ return res;
+ }
+ else if (!g_file_test(cache_dir.c_str(), G_FILE_TEST_IS_DIR))
+ return res;
+
+ gchar *base = g_path_get_basename(url.c_str());
+ res.push_back(cache_dir + G_DIR_SEPARATOR_S + base + ".oft");
+ g_free(base);
+ return res;
+}
+
+bool offset_index::save_cache(const std::string& url)
+{
+ strlist_t vars = get_cache_variant(url);
+ for (strlist_t::const_iterator it = vars.begin(); it != vars.end(); ++it)
+ {
+ FILE *out = fopen(it->c_str(), "wb");
+ if (!out)
+ continue;
+ if (fwrite(CACHE_MAGIC, 1, strlen(CACHE_MAGIC), out) != strlen(CACHE_MAGIC))
+ continue;
+ if (fwrite(&wordoffset[0], sizeof(wordoffset[0]), wordoffset.size(), out) != wordoffset.size())
+ continue;
+ fclose(out);
+ printf("save to cache %s\n", url.c_str());
+ return true;
+ }
+ return false;
+}
+
+bool offset_index::load(const std::string& url, gulong wc, gulong fsize)
+{
+ wordcount = wc;
+ gulong npages = (wc - 1) / ENTR_PER_PAGE + 2;
+ wordoffset.resize(npages);
+ if (!load_cache(url))
+ { //map file will close after finish of block
+ MapFile map_file;
+ if (!map_file.open(url.c_str(), fsize))
+ return false;
+ const gchar *idxdatabuffer = map_file.begin();
+
+ const gchar *p1 = idxdatabuffer;
+ gulong index_size;
+ guint32 j = 0;
+ for (guint32 i = 0; i < wc; i++)
+ {
+ index_size = strlen(p1) + 1 + 2 * sizeof(guint32);
+ if (i % ENTR_PER_PAGE == 0)
+ {
+ wordoffset[j] = p1 - idxdatabuffer;
+ ++j;
+ }
+ p1 += index_size;
+ }
+ wordoffset[j] = p1 - idxdatabuffer;
+ if (!save_cache(url))
+ fprintf(stderr, "cache update failed\n");
+ }
+
+ if (!(idxfile = fopen(url.c_str(), "rb")))
+ {
+ wordoffset.resize(0);
+ return false;
+ }
+
+ first.assign(0, read_first_on_page_key(0));
+ last.assign(wordoffset.size() - 2, read_first_on_page_key(wordoffset.size() - 2));
+ middle.assign((wordoffset.size() - 2) / 2, read_first_on_page_key((wordoffset.size() - 2) / 2));
+ real_last.assign(wc - 1, get_key(wc - 1));
+
+ return true;
+}
+
+inline gulong offset_index::load_page(glong page_idx)
+{
+ gulong nentr = ENTR_PER_PAGE;
+ if (page_idx == glong(wordoffset.size() - 2))
+ if ((nentr = wordcount % ENTR_PER_PAGE) == 0)
+ nentr = ENTR_PER_PAGE;
+
+
+ if (page_idx != page.idx)
+ {
+ page_data.resize(wordoffset[page_idx + 1] - wordoffset[page_idx]);
+ fseek(idxfile, wordoffset[page_idx], SEEK_SET);
+ fread(&page_data[0], 1, page_data.size(), idxfile);
+ page.fill(&page_data[0], nentr, page_idx);
+ }
+
+ return nentr;
+}
+
+const gchar *offset_index::get_key(glong idx)
+{
+ load_page(idx / ENTR_PER_PAGE);
+ glong idx_in_page = idx % ENTR_PER_PAGE;
+ wordentry_offset = page.entries[idx_in_page].off;
+ wordentry_size = page.entries[idx_in_page].size;
+
+ return page.entries[idx_in_page].keystr;
+}
+
+void offset_index::get_data(glong idx)
+{
+ get_key(idx);
+}
+
+const gchar *offset_index::get_key_and_data(glong idx)
+{
+ return get_key(idx);
+}
+
+bool offset_index::lookup(const char *str, glong &idx)
+{
+ bool bFound = false;
+ glong iFrom;
+ glong iTo = wordoffset.size() - 2;
+ gint cmpint;
+ glong iThisIndex;
+ if (stardict_strcmp(str, first.keystr.c_str()) < 0)
+ {
+ idx = 0;
+ return false;
+ }
+ else if (stardict_strcmp(str, real_last.keystr.c_str()) > 0)
+ {
+ idx = INVALID_INDEX;
+ return false;
+ }
+ else
+ {
+ iFrom = 0;
+ iThisIndex = 0;
+ while (iFrom <= iTo)
+ {
+ iThisIndex = (iFrom + iTo) / 2;
+ cmpint = stardict_strcmp(str, get_first_on_page_key(iThisIndex));
+ if (cmpint > 0)
+ iFrom = iThisIndex + 1;
+ else if (cmpint < 0)
+ iTo = iThisIndex - 1;
+ else
+ {
+ bFound = true;
+ break;
+ }
+ }
+ if (!bFound)
+ idx = iTo; //prev
+ else
+ idx = iThisIndex;
+ }
+ if (!bFound)
+ {
+ gulong netr = load_page(idx);
+ iFrom = 1; // Needn't search the first word anymore.
+ iTo = netr - 1;
+ iThisIndex = 0;
+ while (iFrom <= iTo)
+ {
+ iThisIndex = (iFrom + iTo) / 2;
+ cmpint = stardict_strcmp(str, page.entries[iThisIndex].keystr);
+ if (cmpint > 0)
+ iFrom = iThisIndex + 1;
+ else if (cmpint < 0)
+ iTo = iThisIndex - 1;
+ else
+ {
+ bFound = true;
+ break;
+ }
+ }
+ idx *= ENTR_PER_PAGE;
+ if (!bFound)
+ idx += iFrom; //next
+ else
+ idx += iThisIndex;
+ }
+ else
+ {
+ idx *= ENTR_PER_PAGE;
+ }
+ return bFound;
+}
+
+wordlist_index::~wordlist_index()
+{
+ g_free(idxdatabuf);
+}
+
+bool wordlist_index::load(const std::string& url, gulong wc, gulong fsize)
+{
+ gzFile in = gzopen(url.c_str(), "rb");
+ if (in == NULL)
+ return false;
+
+ idxdatabuf = (gchar *)g_malloc(fsize);
+
+ gulong len = gzread(in, idxdatabuf, fsize);
+ gzclose(in);
+
+ if (len != fsize)
+ return false;
+
+ wordlist.resize(wc + 1);
+ gchar *p1 = idxdatabuf;
+ guint32 i;
+ for (i = 0; i < wc; i++)
+ {
+ wordlist[i] = p1;
+ p1 += strlen(p1) + 1 + 2 * sizeof(guint32);
+ }
+ wordlist[wc] = p1;
+
+ return true;
+}
+
+const gchar *wordlist_index::get_key(glong idx)
+{
+ return wordlist[idx];
+}
+
+void wordlist_index::get_data(glong idx)
+{
+ gchar *p1 = wordlist[idx] + strlen(wordlist[idx]) + sizeof(gchar);
+ wordentry_offset = g_ntohl(*reinterpret_cast<guint32 *>(p1));
+ p1 += sizeof(guint32);
+ wordentry_size = g_ntohl(*reinterpret_cast<guint32 *>(p1));
+}
+
+const gchar *wordlist_index::get_key_and_data(glong idx)
+{
+ get_data(idx);
+ return get_key(idx);
+}
+
+bool wordlist_index::lookup(const char *str, glong &idx)
+{
+ bool bFound = false;
+ glong iTo = wordlist.size() - 2;
+
+ if (stardict_strcmp(str, get_key(0)) < 0)
+ {
+ idx = 0;
+ }
+ else if (stardict_strcmp(str, get_key(iTo)) > 0)
+ {
+ idx = INVALID_INDEX;
+ }
+ else
+ {
+ glong iThisIndex = 0;
+ glong iFrom = 0;
+ gint cmpint;
+ while (iFrom <= iTo)
+ {
+ iThisIndex = (iFrom + iTo) / 2;
+ cmpint = stardict_strcmp(str, get_key(iThisIndex));
+ if (cmpint > 0)
+ iFrom = iThisIndex + 1;
+ else if (cmpint < 0)
+ iTo = iThisIndex - 1;
+ else
+ {
+ bFound = true;
+ break;
+ }
+ }
+ if (!bFound)
+ idx = iFrom; //next
+ else
+ idx = iThisIndex;
+ }
+ return bFound;
+}
+
+//===================================================================
+bool Dict::load(const std::string& ifofilename)
+{
+ gulong idxfilesize;
+ if (!load_ifofile(ifofilename, idxfilesize))
+ return false;
+
+ std::string fullfilename(ifofilename);
+ fullfilename.replace(fullfilename.length() - sizeof("ifo") + 1, sizeof("ifo") - 1, "dict.dz");
+
+ if (g_file_test(fullfilename.c_str(), G_FILE_TEST_EXISTS))
+ {
+ dictdzfile.reset(new dictData);
+ if (!dictdzfile->open(fullfilename, 0))
+ {
+ //g_print("open file %s failed!\n",fullfilename);
+ return false;
+ }
+ }
+ else
+ {
+ fullfilename.erase(fullfilename.length() - sizeof(".dz") + 1, sizeof(".dz") - 1);
+ dictfile = fopen(fullfilename.c_str(), "rb");
+ if (!dictfile)
+ {
+ //g_print("open file %s failed!\n",fullfilename);
+ return false;
+ }
+ }
+
+ fullfilename = ifofilename;
+ fullfilename.replace(fullfilename.length() - sizeof("ifo") + 1, sizeof("ifo") - 1, "idx.gz");
+
+ if (g_file_test(fullfilename.c_str(), G_FILE_TEST_EXISTS))
+ {
+ idx_file.reset(new wordlist_index);
+ }
+ else
+ {
+ fullfilename.erase(fullfilename.length() - sizeof(".gz") + 1, sizeof(".gz") - 1);
+ idx_file.reset(new offset_index);
+ }
+
+ if (!idx_file->load(fullfilename, wordcount, idxfilesize))
+ return false;
+
+ //g_print("bookname: %s , wordcount %lu\n", bookname.c_str(), narticles());
+ return true;
+}
+
+bool Dict::load_ifofile(const std::string& ifofilename, gulong &idxfilesize)
+{
+ DictInfo dict_info;
+ if (!dict_info.load_from_ifo_file(ifofilename, false))
+ return false;
+ if (dict_info.wordcount == 0)
+ return false;
+
+
+
+ ifo_file_name = dict_info.ifo_file_name;
+ wordcount = dict_info.wordcount;
+ bookname = dict_info.bookname;
+
+ idxfilesize = dict_info.index_file_size;
+
+ sametypesequence = dict_info.sametypesequence;
+
+ return true;
+}
+
+bool Dict::LookupWithRule(GPatternSpec *pspec, glong *aIndex, int iBuffLen)
+{
+ int iIndexCount = 0;
+
+ for (guint32 i = 0; i < narticles() && iIndexCount < iBuffLen - 1; i++)
+ if (g_pattern_match_string(pspec, get_key(i)))
+ aIndex[iIndexCount++] = i;
+
+ aIndex[iIndexCount] = -1; // -1 is the end.
+
+ return (iIndexCount > 0);
+}
+
+//===================================================================
+Libs::Libs(progress_func_t f)
+{
+ progress_func = f;
+ iMaxFuzzyDistance = MAX_FUZZY_DISTANCE; //need to read from cfg.
+}
+
+Libs::~Libs()
+{
+ for (std::vector<Dict *>::iterator p = oLib.begin(); p != oLib.end(); ++p)
+ delete *p;
+}
+
+void Libs::load_dict(const std::string& url)
+{
+ Dict *lib = new Dict;
+ if (lib->load(url))
+ oLib.push_back(lib);
+ else
+ delete lib;
+}
+
+class DictLoader
+{
+ public:
+ DictLoader(Libs& lib_): lib(lib_)
+ {}
+ void operator()(const std::string& url, bool disable)
+ {
+ if (!disable)
+ lib.load_dict(url);
+ }
+ private:
+ Libs& lib;
+};
+
+void Libs::load(const strlist_t& dicts_dirs,
+ const strlist_t& order_list,
+ const strlist_t& disable_list)
+{
+ for_each_file(dicts_dirs, ".ifo", order_list, disable_list,
+ DictLoader(*this));
+}
+
+class DictReLoader
+{
+ public:
+ DictReLoader(std::vector<Dict *> &p, std::vector<Dict *> &f,
+ Libs& lib_) : prev(p), future(f), lib(lib_)
+ {}
+ void operator()(const std::string& url, bool disable)
+ {
+ if (!disable)
+ {
+ Dict *dict = find(url);
+ if (dict)
+ future.push_back(dict);
+ else
+ lib.load_dict(url);
+ }
+ }
+ private:
+ std::vector<Dict *> &prev;
+ std::vector<Dict *> &future;
+ Libs& lib;
+
+ Dict *find(const std::string& url)
+ {
+ std::vector<Dict *>::iterator it;
+ for (it = prev.begin(); it != prev.end(); ++it)
+ if ((*it)->ifofilename() == url)
+ break;
+ if (it != prev.end())
+ {
+ Dict *res = *it;
+ prev.erase(it);
+ return res;
+ }
+ return NULL;
+ }
+};
+
+void Libs::reload(const strlist_t& dicts_dirs,
+ const strlist_t& order_list,
+ const strlist_t& disable_list)
+{
+ std::vector<Dict *> prev(oLib);
+ oLib.clear();
+ for_each_file(dicts_dirs, ".ifo", order_list, disable_list,
+ DictReLoader(prev, oLib, *this));
+ for (std::vector<Dict *>::iterator it = prev.begin(); it != prev.end(); ++it)
+ delete *it;
+}
+
+const gchar *Libs::poGetCurrentWord(glong * iCurrent)
+{
+ const gchar *poCurrentWord = NULL;
+ const gchar *word;
+ for (std::vector<Dict *>::size_type iLib = 0; iLib<oLib.size(); iLib++)
+ {
+ if (iCurrent[iLib] == INVALID_INDEX)
+ continue;
+ if ( iCurrent[iLib] >= narticles(iLib) || iCurrent[iLib] < 0)
+ continue;
+ if ( poCurrentWord == NULL )
+ {
+ poCurrentWord = poGetWord(iCurrent[iLib], iLib);
+ }
+ else
+ {
+ word = poGetWord(iCurrent[iLib], iLib);
+
+ if (stardict_strcmp(poCurrentWord, word) > 0 )
+ poCurrentWord = word;
+ }
+ }
+ return poCurrentWord;
+}
+
+const gchar *
+Libs::poGetNextWord(const gchar *sWord, glong *iCurrent)
+{
+ // the input can be:
+ // (word,iCurrent),read word,write iNext to iCurrent,and return next word. used by TopWin::NextCallback();
+ // (NULL,iCurrent),read iCurrent,write iNext to iCurrent,and return next word. used by AppCore::ListWords();
+ const gchar *poCurrentWord = NULL;
+ std::vector<Dict *>::size_type iCurrentLib = 0;
+ const gchar *word;
+
+ for (std::vector<Dict *>::size_type iLib = 0;iLib<oLib.size();iLib++)
+ {
+ if (sWord)
+ oLib[iLib]->Lookup(sWord, iCurrent[iLib]);
+ if (iCurrent[iLib] == INVALID_INDEX)
+ continue;
+ if (iCurrent[iLib] >= narticles(iLib) || iCurrent[iLib] < 0)
+ continue;
+ if (poCurrentWord == NULL )
+ {
+ poCurrentWord = poGetWord(iCurrent[iLib], iLib);
+ iCurrentLib = iLib;
+ }
+ else
+ {
+ word = poGetWord(iCurrent[iLib], iLib);
+
+ if (stardict_strcmp(poCurrentWord, word) > 0 )
+ {
+ poCurrentWord = word;
+ iCurrentLib = iLib;
+ }
+ }
+ }
+ if (poCurrentWord)
+ {
+ iCurrent[iCurrentLib]
+ ++;
+ for (std::vector<Dict *>::size_type iLib = 0;iLib<oLib.size();iLib++)
+ {
+ if (iLib == iCurrentLib)
+ continue;
+ if (iCurrent[iLib] == INVALID_INDEX)
+ continue;
+ if ( iCurrent[iLib] >= narticles(iLib) || iCurrent[iLib] < 0)
+ continue;
+ if (strcmp(poCurrentWord, poGetWord(iCurrent[iLib], iLib)) == 0 )
+ iCurrent[iLib]++;
+ }
+ poCurrentWord = poGetCurrentWord(iCurrent);
+ }
+ return poCurrentWord;
+}
+
+
+const gchar *
+Libs::poGetPreWord(glong * iCurrent)
+{
+ // used by TopWin::PreviousCallback(); the iCurrent is cached by AppCore::TopWinWordChange();
+ const gchar *poCurrentWord = NULL;
+ std::vector<Dict *>::size_type iCurrentLib = 0;
+ const gchar *word;
+
+ for (std::vector<Dict *>::size_type iLib = 0;iLib<oLib.size();iLib++)
+ {
+ if (iCurrent[iLib] == INVALID_INDEX)
+ iCurrent[iLib] = narticles(iLib);
+ else
+ {
+ if ( iCurrent[iLib] > narticles(iLib) || iCurrent[iLib] <= 0)
+ continue;
+ }
+ if ( poCurrentWord == NULL )
+ {
+ poCurrentWord = poGetWord(iCurrent[iLib] - 1, iLib);
+ iCurrentLib = iLib;
+ }
+ else
+ {
+ word = poGetWord(iCurrent[iLib] - 1, iLib);
+ if (stardict_strcmp(poCurrentWord, word) < 0 )
+ {
+ poCurrentWord = word;
+ iCurrentLib = iLib;
+ }
+ }
+ }
+
+ if (poCurrentWord)
+ {
+ iCurrent[iCurrentLib]
+ --;
+ for (std::vector<Dict *>::size_type iLib = 0;iLib<oLib.size();iLib++)
+ {
+ if (iLib == iCurrentLib)
+ continue;
+ if (iCurrent[iLib] > narticles(iLib) || iCurrent[iLib] <= 0)
+ continue;
+ if (strcmp(poCurrentWord, poGetWord(iCurrent[iLib] - 1, iLib)) == 0)
+ {
+ iCurrent[iLib]--;
+ }
+ else
+ {
+ if (iCurrent[iLib] == narticles(iLib))
+ iCurrent[iLib] = INVALID_INDEX;
+ }
+ }
+ }
+ return poCurrentWord;
+}
+
+bool Libs::LookupSimilarWord(const gchar* sWord, glong & iWordIndex, int iLib)
+{
+ glong iIndex;
+ bool bFound = false;
+ gchar *casestr;
+
+ if (!bFound)
+ {
+ // to lower case.
+ casestr = g_utf8_strdown(sWord, -1);
+ if (strcmp(casestr, sWord))
+ {
+ if (oLib[iLib]->Lookup(casestr, iIndex))
+ bFound = true;
+ }
+ g_free(casestr);
+ // to upper case.
+ if (!bFound)
+ {
+ casestr = g_utf8_strup(sWord, -1);
+ if (strcmp(casestr, sWord))
+ {
+ if (oLib[iLib]->Lookup(casestr, iIndex))
+ bFound = true;
+ }
+ g_free(casestr);
+ }
+ // Upper the first character and lower others.
+ if (!bFound)
+ {
+ gchar *nextchar = g_utf8_next_char(sWord);
+ gchar *firstchar = g_utf8_strup(sWord, nextchar - sWord);
+ nextchar = g_utf8_strdown(nextchar, -1);
+ casestr = g_strdup_printf("%s%s", firstchar, nextchar);
+ g_free(firstchar);
+ g_free(nextchar);
+ if (strcmp(casestr, sWord))
+ {
+ if (oLib[iLib]->Lookup(casestr, iIndex))
+ bFound = true;
+ }
+ g_free(casestr);
+ }
+ }
+
+ if (bIsPureEnglish(sWord))
+ {
+ // If not Found , try other status of sWord.
+ int iWordLen = strlen(sWord);
+ bool isupcase;
+
+ gchar *sNewWord = (gchar *)g_malloc(iWordLen + 1);
+
+ //cut one char "s" or "d"
+ if (!bFound && iWordLen > 1)
+ {
+ isupcase = sWord[iWordLen - 1] == 'S' || !strncmp(&sWord[iWordLen - 2], "ED", 2);
+ if (isupcase || sWord[iWordLen - 1] == 's' || !strncmp(&sWord[iWordLen - 2], "ed", 2))
+ {
+ strcpy(sNewWord, sWord);
+ sNewWord[iWordLen - 1] = '\0'; // cut "s" or "d"
+ if (oLib[iLib]->Lookup(sNewWord, iIndex))
+ bFound = true;
+ else if (isupcase || g_ascii_isupper(sWord[0]))
+ {
+ casestr = g_ascii_strdown(sNewWord, -1);
+ if (strcmp(casestr, sNewWord))
+ {
+ if (oLib[iLib]->Lookup(casestr, iIndex))
+ bFound = true;
+ }
+ g_free(casestr);
+ }
+ }
+ }
+
+ //cut "ly"
+ if (!bFound && iWordLen > 2)
+ {
+ isupcase = !strncmp(&sWord[iWordLen - 2], "LY", 2);
+ if (isupcase || (!strncmp(&sWord[iWordLen - 2], "ly", 2)))
+ {
+ strcpy(sNewWord, sWord);
+ sNewWord[iWordLen - 2] = '\0'; // cut "ly"
+ if (iWordLen > 5 && sNewWord[iWordLen - 3] == sNewWord[iWordLen - 4]
+ && !bIsVowel(sNewWord[iWordLen - 4]) &&
+ bIsVowel(sNewWord[iWordLen - 5]))
+ { //doubled
+
+ sNewWord[iWordLen - 3] = '\0';
+ if ( oLib[iLib]->Lookup(sNewWord, iIndex) )
+ bFound = true;
+ else
+ {
+ if (isupcase || g_ascii_isupper(sWord[0]))
+ {
+ casestr = g_ascii_strdown(sNewWord, -1);
+ if (strcmp(casestr, sNewWord))
+ {
+ if (oLib[iLib]->Lookup(casestr, iIndex))
+ bFound = true;
+ }
+ g_free(casestr);
+ }
+ if (!bFound)
+ sNewWord[iWordLen - 3] = sNewWord[iWordLen - 4]; //restore
+ }
+ }
+ if (!bFound)
+ {
+ if (oLib[iLib]->Lookup(sNewWord, iIndex))
+ bFound = true;
+ else if (isupcase || g_ascii_isupper(sWord[0]))
+ {
+ casestr = g_ascii_strdown(sNewWord, -1);
+ if (strcmp(casestr, sNewWord))
+ {
+ if (oLib[iLib]->Lookup(casestr, iIndex))
+ bFound = true;
+ }
+ g_free(casestr);
+ }
+ }
+ }
+ }
+
+ //cut "ing"
+ if (!bFound && iWordLen > 3)
+ {
+ isupcase = !strncmp(&sWord[iWordLen - 3], "ING", 3);
+ if (isupcase || !strncmp(&sWord[iWordLen - 3], "ing", 3) )
+ {
+ strcpy(sNewWord, sWord);
+ sNewWord[iWordLen - 3] = '\0';
+ if ( iWordLen > 6 && (sNewWord[iWordLen - 4] == sNewWord[iWordLen - 5])
+ && !bIsVowel(sNewWord[iWordLen - 5]) &&
+ bIsVowel(sNewWord[iWordLen - 6]))
+ { //doubled
+ sNewWord[iWordLen - 4] = '\0';
+ if (oLib[iLib]->Lookup(sNewWord, iIndex))
+ bFound = true;
+ else
+ {
+ if (isupcase || g_ascii_isupper(sWord[0]))
+ {
+ casestr = g_ascii_strdown(sNewWord, -1);
+ if (strcmp(casestr, sNewWord))
+ {
+ if (oLib[iLib]->Lookup(casestr, iIndex))
+ bFound = true;
+ }
+ g_free(casestr);
+ }
+ if (!bFound)
+ sNewWord[iWordLen - 4] = sNewWord[iWordLen - 5]; //restore
+ }
+ }
+ if ( !bFound )
+ {
+ if (oLib[iLib]->Lookup(sNewWord, iIndex))
+ bFound = true;
+ else if (isupcase || g_ascii_isupper(sWord[0]))
+ {
+ casestr = g_ascii_strdown(sNewWord, -1);
+ if (strcmp(casestr, sNewWord))
+ {
+ if (oLib[iLib]->Lookup(casestr, iIndex))
+ bFound = true;
+ }
+ g_free(casestr);
+ }
+ }
+ if (!bFound)
+ {
+ if (isupcase)
+ strcat(sNewWord, "E"); // add a char "E"
+ else
+ strcat(sNewWord, "e"); // add a char "e"
+ if (oLib[iLib]->Lookup(sNewWord, iIndex))
+ bFound = true;
+ else if (isupcase || g_ascii_isupper(sWord[0]))
+ {
+ casestr = g_ascii_strdown(sNewWord, -1);
+ if (strcmp(casestr, sNewWord))
+ {
+ if (oLib[iLib]->Lookup(casestr, iIndex))
+ bFound = true;
+ }
+ g_free(casestr);
+ }
+ }
+ }
+ }
+
+ //cut two char "es"
+ if (!bFound && iWordLen > 3)
+ {
+ isupcase = (!strncmp(&sWord[iWordLen - 2], "ES", 2) &&
+ (sWord[iWordLen - 3] == 'S' ||
+ sWord[iWordLen - 3] == 'X' ||
+ sWord[iWordLen - 3] == 'O' ||
+ (iWordLen > 4 && sWord[iWordLen - 3] == 'H' &&
+ (sWord[iWordLen - 4] == 'C' ||
+ sWord[iWordLen - 4] == 'S'))));
+ if (isupcase ||
+ (!strncmp(&sWord[iWordLen - 2], "es", 2) &&
+ (sWord[iWordLen - 3] == 's' || sWord[iWordLen - 3] == 'x' ||
+ sWord[iWordLen - 3] == 'o' ||
+ (iWordLen > 4 && sWord[iWordLen - 3] == 'h' &&
+ (sWord[iWordLen - 4] == 'c' || sWord[iWordLen - 4] == 's')))))
+ {
+ strcpy(sNewWord, sWord);
+ sNewWord[iWordLen - 2] = '\0';
+ if (oLib[iLib]->Lookup(sNewWord, iIndex))
+ bFound = true;
+ else if (isupcase || g_ascii_isupper(sWord[0]))
+ {
+ casestr = g_ascii_strdown(sNewWord, -1);
+ if (strcmp(casestr, sNewWord))
+ {
+ if (oLib[iLib]->Lookup(casestr, iIndex))
+ bFound = true;
+ }
+ g_free(casestr);
+ }
+ }
+ }
+
+ //cut "ed"
+ if (!bFound && iWordLen > 3)
+ {
+ isupcase = !strncmp(&sWord[iWordLen - 2], "ED", 2);
+ if (isupcase || !strncmp(&sWord[iWordLen - 2], "ed", 2))
+ {
+ strcpy(sNewWord, sWord);
+ sNewWord[iWordLen - 2] = '\0';
+ if (iWordLen > 5 && (sNewWord[iWordLen - 3] == sNewWord[iWordLen - 4])
+ && !bIsVowel(sNewWord[iWordLen - 4]) &&
+ bIsVowel(sNewWord[iWordLen - 5]))
+ { //doubled
+ sNewWord[iWordLen - 3] = '\0';
+ if (oLib[iLib]->Lookup(sNewWord, iIndex))
+ bFound = true;
+ else
+ {
+ if (isupcase || g_ascii_isupper(sWord[0]))
+ {
+ casestr = g_ascii_strdown(sNewWord, -1);
+ if (strcmp(casestr, sNewWord))
+ {
+ if (oLib[iLib]->Lookup(casestr, iIndex))
+ bFound = true;
+ }
+ g_free(casestr);
+ }
+ if (!bFound)
+ sNewWord[iWordLen - 3] = sNewWord[iWordLen - 4]; //restore
+ }
+ }
+ if (!bFound)
+ {
+ if (oLib[iLib]->Lookup(sNewWord, iIndex))
+ bFound = true;
+ else if (isupcase || g_ascii_isupper(sWord[0]))
+ {
+ casestr = g_ascii_strdown(sNewWord, -1);
+ if (strcmp(casestr, sNewWord))
+ {
+ if (oLib[iLib]->Lookup(casestr, iIndex))
+ bFound = true;
+ }
+ g_free(casestr);
+ }
+ }
+ }
+ }
+
+ // cut "ied" , add "y".
+ if (!bFound && iWordLen > 3)
+ {
+ isupcase = !strncmp(&sWord[iWordLen - 3], "IED", 3);
+ if (isupcase || (!strncmp(&sWord[iWordLen - 3], "ied", 3)))
+ {
+ strcpy(sNewWord, sWord);
+ sNewWord[iWordLen - 3] = '\0';
+ if (isupcase)
+ strcat(sNewWord, "Y"); // add a char "Y"
+ else
+ strcat(sNewWord, "y"); // add a char "y"
+ if (oLib[iLib]->Lookup(sNewWord, iIndex))
+ bFound = true;
+ else if (isupcase || g_ascii_isupper(sWord[0]))
+ {
+ casestr = g_ascii_strdown(sNewWord, -1);
+ if (strcmp(casestr, sNewWord))
+ {
+ if (oLib[iLib]->Lookup(casestr, iIndex))
+ bFound = true;
+ }
+ g_free(casestr);
+ }
+ }
+ }
+
+ // cut "ies" , add "y".
+ if (!bFound && iWordLen > 3)
+ {
+ isupcase = !strncmp(&sWord[iWordLen - 3], "IES", 3);
+ if (isupcase || (!strncmp(&sWord[iWordLen - 3], "ies", 3)))
+ {
+ strcpy(sNewWord, sWord);
+ sNewWord[iWordLen - 3] = '\0';
+ if (isupcase)
+ strcat(sNewWord, "Y"); // add a char "Y"
+ else
+ strcat(sNewWord, "y"); // add a char "y"
+ if (oLib[iLib]->Lookup(sNewWord, iIndex))
+ bFound = true;
+ else if (isupcase || g_ascii_isupper(sWord[0]))
+ {
+ casestr = g_ascii_strdown(sNewWord, -1);
+ if (strcmp(casestr, sNewWord))
+ {
+ if (oLib[iLib]->Lookup(casestr, iIndex))
+ bFound = true;
+ }
+ g_free(casestr);
+ }
+ }
+ }
+
+ // cut "er".
+ if (!bFound && iWordLen > 2)
+ {
+ isupcase = !strncmp(&sWord[iWordLen - 2], "ER", 2);
+ if (isupcase || (!strncmp(&sWord[iWordLen - 2], "er", 2)))
+ {
+ strcpy(sNewWord, sWord);
+ sNewWord[iWordLen - 2] = '\0';
+ if (oLib[iLib]->Lookup(sNewWord, iIndex))
+ bFound = true;
+ else if (isupcase || g_ascii_isupper(sWord[0]))
+ {
+ casestr = g_ascii_strdown(sNewWord, -1);
+ if (strcmp(casestr, sNewWord))
+ {
+ if (oLib[iLib]->Lookup(casestr, iIndex))
+ bFound = true;
+ }
+ g_free(casestr);
+ }
+ }
+ }
+
+ // cut "est".
+ if (!bFound && iWordLen > 3)
+ {
+ isupcase = !strncmp(&sWord[iWordLen - 3], "EST", 3);
+ if (isupcase || (!strncmp(&sWord[iWordLen - 3], "est", 3)))
+ {
+ strcpy(sNewWord, sWord);
+ sNewWord[iWordLen - 3] = '\0';
+ if (oLib[iLib]->Lookup(sNewWord, iIndex))
+ bFound = true;
+ else if (isupcase || g_ascii_isupper(sWord[0]))
+ {
+ casestr = g_ascii_strdown(sNewWord, -1);
+ if (strcmp(casestr, sNewWord))
+ {
+ if (oLib[iLib]->Lookup(casestr, iIndex))
+ bFound = true;
+ }
+ g_free(casestr);
+ }
+ }
+ }
+
+ g_free(sNewWord);
+ }
+
+ if (bFound)
+ iWordIndex = iIndex;
+#if 0
+
+ else
+ {
+ //don't change iWordIndex here.
+ //when LookupSimilarWord all failed too, we want to use the old LookupWord index to list words.
+ //iWordIndex = INVALID_INDEX;
+ }
+#endif
+ return bFound;
+}
+
+bool Libs::SimpleLookupWord(const gchar* sWord, glong & iWordIndex, int iLib)
+{
+ bool bFound = oLib[iLib]->Lookup(sWord, iWordIndex);
+ if (!bFound)
+ bFound = LookupSimilarWord(sWord, iWordIndex, iLib);
+ return bFound;
+}
+
+struct Fuzzystruct
+{
+ char * pMatchWord;
+ int iMatchWordDistance;
+};
+
+inline bool operator<(const Fuzzystruct & lh, const Fuzzystruct & rh)
+{
+ if (lh.iMatchWordDistance != rh.iMatchWordDistance)
+ return lh.iMatchWordDistance < rh.iMatchWordDistance;
+
+ if (lh.pMatchWord && rh.pMatchWord)
+ return stardict_strcmp(lh.pMatchWord, rh.pMatchWord) < 0;
+
+ return false;
+}
+
+static inline void unicode_strdown(gunichar *str)
+{
+ while (*str)
+ {
+ *str = g_unichar_tolower(*str);
+ ++str;
+ }
+}
+
+bool Libs::LookupWithFuzzy(const gchar *sWord, gchar *reslist[], gint reslist_size, gint iLib)
+{
+ if (sWord[0] == '\0')
+ return false;
+
+ Fuzzystruct *oFuzzystruct = new Fuzzystruct[reslist_size];
+
+ for (int i = 0; i < reslist_size; i++)
+ {
+ oFuzzystruct[i].pMatchWord = NULL;
+ oFuzzystruct[i].iMatchWordDistance = iMaxFuzzyDistance;
+ }
+ int iMaxDistance = iMaxFuzzyDistance;
+ int iDistance;
+ bool Found = false;
+ EditDistance oEditDistance;
+
+ glong iCheckWordLen;
+ const char *sCheck;
+ gunichar *ucs4_str1, *ucs4_str2;
+ glong ucs4_str2_len;
+
+ ucs4_str2 = g_utf8_to_ucs4_fast(sWord, -1, &ucs4_str2_len);
+ unicode_strdown(ucs4_str2);
+
+// for (std::vector<Dict *>::size_type iLib = 0; iLib<oLib.size(); iLib++)
+// {
+ if (progress_func)
+ progress_func();
+
+ //if (stardict_strcmp(sWord, poGetWord(0,iLib))>=0 && stardict_strcmp(sWord, poGetWord(narticles(iLib)-1,iLib))<=0) {
+ //there are Chinese dicts and English dicts...
+ if (TRUE)
+ {
+ const int iwords = narticles(iLib);
+ for (int index = 0; index < iwords; index++)
+ {
+ sCheck = poGetWord(index, iLib);
+ // tolower and skip too long or too short words
+ iCheckWordLen = g_utf8_strlen(sCheck, -1);
+ if (iCheckWordLen - ucs4_str2_len >= iMaxDistance ||
+ ucs4_str2_len - iCheckWordLen >= iMaxDistance)
+ continue;
+ ucs4_str1 = g_utf8_to_ucs4_fast(sCheck, -1, NULL);
+ if (iCheckWordLen > ucs4_str2_len)
+ ucs4_str1[ucs4_str2_len] = 0;
+ unicode_strdown(ucs4_str1);
+
+ iDistance = oEditDistance.CalEditDistance(ucs4_str1, ucs4_str2, iMaxDistance);
+ g_free(ucs4_str1);
+ if (iDistance < iMaxDistance && iDistance < ucs4_str2_len)
+ {
+ // when ucs4_str2_len=1,2 we need less fuzzy.
+ Found = true;
+ bool bAlreadyInList = false;
+ int iMaxDistanceAt = 0;
+ for (int j = 0; j < reslist_size; j++)
+ {
+ if (oFuzzystruct[j].pMatchWord &&
+ strcmp(oFuzzystruct[j].pMatchWord, sCheck) == 0 )
+ { //already in list
+ bAlreadyInList = true;
+ break;
+ }
+ //find the position,it will certainly be found (include the first time) as iMaxDistance is set by last time.
+ if (oFuzzystruct[j].iMatchWordDistance == iMaxDistance )
+ {
+ iMaxDistanceAt = j;
+ }
+ }
+ if (!bAlreadyInList)
+ {
+ if (oFuzzystruct[iMaxDistanceAt].pMatchWord)
+ g_free(oFuzzystruct[iMaxDistanceAt].pMatchWord);
+ oFuzzystruct[iMaxDistanceAt].pMatchWord = g_strdup(sCheck);
+ oFuzzystruct[iMaxDistanceAt].iMatchWordDistance = iDistance;
+ // calc new iMaxDistance
+ iMaxDistance = iDistance;
+ for (int j = 0; j < reslist_size; j++)
+ {
+ if (oFuzzystruct[j].iMatchWordDistance > iMaxDistance)
+ iMaxDistance = oFuzzystruct[j].iMatchWordDistance;
+ } // calc new iMaxDistance
+ } // add to list
+ } // find one
+ } // each word
+ } // ok for search
+// } // each lib
+ g_free(ucs4_str2);
+
+ if (Found) // sort with distance
+ std::sort(oFuzzystruct, oFuzzystruct + reslist_size);
+
+ for (gint i = 0; i < reslist_size; ++i)
+ reslist[i] = oFuzzystruct[i].pMatchWord;
+
+ delete[] oFuzzystruct;
+
+ return Found;
+}
+
+inline bool less_for_compare(const char *lh, const char *rh)
+{
+ return stardict_strcmp(lh, rh) < 0;
+}
+
+gint Libs::LookupWithRule(const gchar *word, gchar **ppMatchWord)
+{
+ glong aiIndex[MAX_MATCH_ITEM_PER_LIB + 1];
+ gint iMatchCount = 0;
+ GPatternSpec *pspec = g_pattern_spec_new(word);
+
+ for (std::vector<Dict *>::size_type iLib = 0; iLib<oLib.size(); iLib++)
+ {
+ //if(oLibs.LookdupWordsWithRule(pspec,aiIndex,MAX_MATCH_ITEM_PER_LIB+1-iMatchCount,iLib))
+ // -iMatchCount,so save time,but may got less result and the word may repeat.
+
+ if (oLib[iLib]->
+ LookupWithRule(pspec, aiIndex, MAX_MATCH_ITEM_PER_LIB + 1))
+ {
+ if (progress_func)
+ progress_func();
+ for (int i = 0; aiIndex[i] != -1; i++)
+ {
+ const gchar * sMatchWord = poGetWord(aiIndex[i], iLib);
+ bool bAlreadyInList = false;
+ for (int j = 0; j < iMatchCount; j++)
+ {
+ if (strcmp(ppMatchWord[j], sMatchWord) == 0)
+ { //already in list
+ bAlreadyInList = true;
+ break;
+ }
+ }
+ if (!bAlreadyInList)
+ ppMatchWord[iMatchCount++] = g_strdup(sMatchWord);
+ }
+ }
+ }
+ g_pattern_spec_free(pspec);
+
+ if (iMatchCount) // sort it.
+ std::sort(ppMatchWord, ppMatchWord + iMatchCount, less_for_compare);
+
+ return iMatchCount;
+}
+
+bool Libs::LookupData(const gchar *sWord, std::vector<gchar *> *reslist)
+{
+ std::vector<std::string> SearchWords;
+ std::string SearchWord;
+ const char *p = sWord;
+ while (*p)
+ {
+ if (*p == '\\')
+ {
+ p++;
+ switch (*p)
+ {
+ case ' ':
+ SearchWord += ' ';
+ break;
+ case '\\':
+ SearchWord += '\\';
+ break;
+ case 't':
+ SearchWord += '\t';
+ break;
+ case 'n':
+ SearchWord += '\n';
+ break;
+ default:
+ SearchWord += *p;
+ }
+ }
+ else if (*p == ' ')
+ {
+ if (!SearchWord.empty())
+ {
+ SearchWords.push_back(SearchWord);
+ SearchWord.clear();
+ }
+ }
+ else
+ {
+ SearchWord += *p;
+ }
+ p++;
+ }
+ if (!SearchWord.empty())
+ {
+ SearchWords.push_back(SearchWord);
+ SearchWord.clear();
+ }
+ if (SearchWords.empty())
+ return false;
+
+ guint32 max_size = 0;
+ gchar *origin_data = NULL;
+ for (std::vector<Dict *>::size_type i = 0; i<oLib.size(); ++i)
+ {
+ if (!oLib[i]->
+ containSearchData())
+ continue;
+ if (progress_func)
+ progress_func();
+ const gulong iwords = narticles(i);
+ const gchar *key;
+ guint32 offset, size;
+ for (gulong j = 0;
+ j < iwords;
+ ++j)
+ {
+ oLib[i]
+ ->get_key_and_data(j, &key, &offset, &size);
+ if (size > max_size)
+ {
+ origin_data = (gchar *)g_realloc(origin_data, size);
+ max_size = size;
+ }
+ if (oLib[i]->SearchData(SearchWords, offset, size, origin_data))
+ reslist[i].push_back(g_strdup(key));
+ }
+ }
+ g_free(origin_data);
+
+ std::vector<Dict *>::size_type i;
+ for (i = 0; i<oLib.size(); ++i)
+ if (!reslist[i].empty())
+ break;
+
+ return i != oLib.size();
+}
+
+/**************************************************/
+query_t analyze_query(const char *s, std::string& res)
+{
+ if (!s || !*s)
+ {
+ res = "";
+ return qtSIMPLE;
+ }
+ if (*s == '/')
+ {
+ res = s + 1;
+ return qtFUZZY;
+ }
+
+ if (*s == '|')
+ {
+ res = s + 1;
+ return qtDATA;
+ }
+
+ bool regexp = false;
+ const char *p = s;
+ res = "";
+ for (; *p; res += *p, ++p)
+ {
+ if (*p == '\\')
+ {
+ ++p;
+ if (!*p)
+ break;
+ continue;
+ }
+ if (*p == '*' || *p == '?')
+ regexp = true;
+ }
+ if (regexp)
+ return qtREGEXP;
+
+ return qtSIMPLE;
+}
projects / qstardict / blobdiff