git.maemo.org Git - modest/blob - src/modest-search.c

   1 /* Copyright (c) 2006, Nokia Corporation
   2  * All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions are
   6  * met:
   7  *
   8  * * Redistributions of source code must retain the above copyright
   9  *   notice, this list of conditions and the following disclaimer.
  10  * * Redistributions in binary form must reproduce the above copyright
  11  *   notice, this list of conditions and the following disclaimer in the
  12  *   documentation and/or other materials provided with the distribution.
  13  * * Neither the name of the Nokia Corporation nor the names of its
  14  *   contributors may be used to endorse or promote products derived from
  15  *   this software without specific prior written permission.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
  18  * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  19  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
  20  * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
  21  * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  22  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  23  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  24  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  25  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  26  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  27  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28  */
  29
  30 #ifndef _GNU_SOURCE
  31 #define _GNU_SOURCE
  32 #endif
  33
  34 #ifdef HAVE_CONFIG_H
  35 #include <config.h>
  36 #endif
  37
  38 #include <string.h>
  39
  40 #include <tny-shared.h>
  41 #include <tny-folder.h>
  42 #include <tny-folder-store.h>
  43 #include <tny-list.h>
  44 #include <tny-iterator.h>
  45 #include <tny-simple-list.h>
  46 #include <tny-camel-imap-store-account.h>
  47 #include <tny-camel-pop-store-account.h>
  48
  49 #include <libmodest-dbus-client/libmodest-dbus-client.h>
  50
  51 #include "modest-text-utils.h"
  52 #include "modest-account-mgr.h"
  53 #include "modest-tny-account-store.h"
  54 #include "modest-tny-account.h"
  55 #include "modest-search.h"
  56 #include "modest-runtime.h"
  57 #include "modest-platform.h"
  58
  59 static gchar *
  60 g_strdup_or_null (const gchar *str)
  61 {
  62         gchar *string = NULL;
  63
  64         if  (str != NULL) {
  65                 string = g_strdup (str);
  66         }
  67
  68         return string;
  69 }
  70
  71 static GList*
  72 add_hit (GList *list, TnyHeader *header, TnyFolder *folder)
  73 {
  74         ModestSearchHit *hit;
  75         TnyHeaderFlags   flags;
  76         char            *furl;
  77         char            *msg_url;
  78         const char      *uid;
  79         const char      *subject;
  80         const char      *sender;
  81
  82         hit = g_slice_new0 (ModestSearchHit);
  83
  84         furl = tny_folder_get_url_string (folder);
  85         printf ("DEBUG: %s: folder URL=%s\n", __FUNCTION__, furl);
  86         if (!furl) {
  87                 g_warning ("%s: tny_folder_get_url_string(): returned NULL for folder. Folder name=%s\n", __FUNCTION__, tny_folder_get_name (folder));
  88         }
  89
  90         /* Make sure that we use the short UID instead of the long UID,
  91          * and/or find out what UID form is used when finding, in camel_data_cache_get().
  92          * so we can find what we get. Philip is working on this.
  93          */
  94         uid = tny_header_get_uid (header);
  95         if (!furl) {
  96                 g_warning ("%s: tny_header_get_uid(): returned NULL for message with subject=%s\n", __FUNCTION__, tny_header_get_subject (header));
  97         }
  98
  99         msg_url = g_strdup_printf ("%s/%s", furl, uid);
 100         g_free (furl);
 101
 102         subject = tny_header_get_subject (header);
 103         sender = tny_header_get_from (header);
 104
 105         flags = tny_header_get_flags (header);
 106
 107         hit->msgid = msg_url;
 108         hit->subject = g_strdup_or_null (subject);
 109         hit->sender = g_strdup_or_null (sender);
 110         hit->folder = g_strdup_or_null (tny_folder_get_name (folder));
 111         hit->msize = tny_header_get_message_size (header);
 112         hit->has_attachment = flags & TNY_HEADER_FLAG_ATTACHMENTS;
 113         hit->is_unread = ! (flags & TNY_HEADER_FLAG_SEEN);
 114         hit->timestamp = tny_header_get_date_received (header);
 115
 116         return g_list_prepend (list, hit);
 117 }
 118
 119 /** Call this until it returns FALSE or nread is set to 0.
 120  *
 121  * @result: FALSE is something failed. */
 122 static gboolean
 123 read_chunk (TnyStream *stream, char *buffer, gsize count, gsize *nread)
 124 {
 125         gsize _nread = 0;
 126         gssize res = 0;
 127
 128         while (_nread < count) {
 129                 res = tny_stream_read (stream,
 130                                        buffer + _nread,
 131                                        count - _nread);
 132                 if (res == -1) { /* error */
 133                         *nread = _nread;
 134                         return FALSE;
 135                 }
 136
 137                 _nread += res;
 138
 139                 if (res == 0) { /* no more bytes read. */
 140                         *nread = _nread;
 141                         return TRUE;
 142                 }
 143         }
 144
 145         *nread = _nread;
 146         return TRUE;
 147 }
 148
 149 #ifdef MODEST_HAVE_OGS
 150 static gboolean
 151 search_mime_part_ogs (TnyMimePart *part, ModestSearch *search)
 152 {
 153         TnyStream *stream = NULL;
 154         char       buffer[4096];
 155         const gsize len = sizeof (buffer);
 156         gsize      nread = 0;
 157         gboolean   is_text_html = FALSE;
 158         gboolean   found = FALSE;
 159         gboolean   res = FALSE;
 160
 161         gboolean is_text = tny_mime_part_content_type_is (part, "text/*");
 162         if (!is_text) {
 163                 g_debug ("%s: tny_mime_part_content_type_is() failed to find a "
 164                         "text/* MIME part. Content type is %s",
 165                 __FUNCTION__, "Unknown (calling tny_mime_part_get_content_type(part) causes a deadlock)");
 166
 167             /* Retry with specific MIME types, because the wildcard seems to fail
 168              * in tinymail.
 169              * Actually I'm not sure anymore that it fails, so we could probalby
 170              * remove this later: murrayc */
 171             is_text = (
 172                 tny_mime_part_content_type_is (part, "text/plain") ||
 173                 tny_mime_part_content_type_is (part, "text/html") );
 174
 175                 if (is_text) {
 176                   g_debug ("%s: Retryting with text/plain or text/html succeeded",
 177                         __FUNCTION__);
 178                 }
 179         }
 180
 181         if (!is_text) {
 182             return FALSE;
 183         }
 184
 185         is_text_html = tny_mime_part_content_type_is (part, "text/html");
 186
 187         stream = tny_mime_part_get_stream (part);
 188
 189         res = read_chunk (stream, buffer, len, &nread);
 190         while (res && (nread > 0)) {
 191                 /* search->text_searcher was instantiated in modest_search_folder(). */
 192
 193                 if (is_text_html) {
 194
 195                         found = ogs_text_searcher_search_html (search->text_searcher,
 196                                                                buffer,
 197                                                                nread,
 198                                                                nread < len);
 199                 } else {
 200                         found = ogs_text_searcher_search_text (search->text_searcher,
 201                                                                buffer,
 202                                                                nread);
 203                 }
 204
 205                 if (found) {
 206                         break;
 207                 }
 208
 209                 nread = 0;
 210                 res = read_chunk (stream, buffer, len, &nread);
 211         }
 212
 213         if (!found) {
 214                 found = ogs_text_searcher_search_done (search->text_searcher);
 215         }
 216
 217         ogs_text_searcher_reset (search->text_searcher);
 218
 219         /* debug stuff:
 220         if (!found) {
 221                 buffer[len -1] = 0;
 222                 printf ("DEBUG: %s: query %s was not found in message text: %s\n",
 223                         __FUNCTION__, search->query, buffer);
 224
 225         } else {
 226                 printf ("DEBUG: %s: found.\n", __FUNCTION__);
 227         }
 228         */
 229
 230         return found;
 231 }
 232
 233 #else
 234
 235 static gboolean
 236 search_mime_part_strcmp (TnyMimePart *part, ModestSearch *search)
 237 {
 238         TnyStream *stream;
 239         char       buffer[8193];
 240         char      *chunk[2];
 241         gssize     len;
 242         gsize     nread;
 243         gboolean   found;
 244         gboolean   res;
 245
 246         if (! tny_mime_part_content_type_is (part, "text/*")) {
 247                 g_debug ("%s: No text MIME part found.\n", __FUNCTION__);
 248                 return FALSE;
 249         }
 250
 251         found = FALSE;
 252         len = (sizeof (buffer) - 1) / 2;
 253
 254         if (strlen (search->body) > len) {
 255                 g_warning ("Search term bigger then chunk."
 256                            "We might not find everything");
 257         }
 258
 259         stream = tny_mime_part_get_stream (part);
 260
 261         memset (buffer, 0, sizeof (buffer));
 262         chunk[0] = buffer;
 263         chunk[1] = buffer + len;
 264
 265         res = read_chunk (stream, chunk[0], len, &nread);
 266
 267         if (res == FALSE) {
 268                 goto done;
 269         }
 270
 271         found = !modest_text_utils_utf8_strcmp (search->body,
 272                                                 buffer,
 273                                                 TRUE);
 274         if (found) {
 275                 goto done;
 276         }
 277
 278         /* This works like this:
 279          * buffer: [ooooooooooo|xxxxxxxxxxxx|\0]
 280          *          ^chunk[0]  ^chunk[1]
 281          * we have prefilled chunk[0] now we always read into chunk[1]
 282          * and then move the content of chunk[1] to chunk[0].
 283          * The idea is to prevent not finding search terms that are
 284          * spread across 2 reads:
 285          * buffer: [ooooooooTES|Txxxxxxxxxxx|\0]
 286          * We should catch that because we always search the whole
 287          * buffer not only the chunks.
 288          *
 289          * Of course that breaks for search terms > sizeof (chunk)
 290          * but sizeof (chunk) should be big enough I guess (see
 291          * the g_warning in this function)
 292          * */
 293         while ((res = read_chunk (stream, chunk[1], len, &nread))) {
 294                 buffer[len + nread] = '\0';
 295
 296                 found = !modest_text_utils_utf8_strcmp (search->body,
 297                                                         buffer,
 298                                                         TRUE);
 299
 300                 if (found) {
 301                         break;
 302                 }
 303
 304                 /* also move the \0 */
 305                 g_memmove (chunk[0], chunk[1], len + 1);
 306         }
 307
 308 done:
 309         g_object_unref (stream);
 310         return found;
 311 }
 312 #endif /*MODEST_HAVE_OGS*/
 313
 314 static gboolean
 315 search_string (const char      *what,
 316                const char      *where,
 317                ModestSearch    *search)
 318 {
 319         gboolean found;
 320 #ifdef MODEST_HAVE_OGS
 321         if (search->flags & MODEST_SEARCH_USE_OGS) {
 322                 found = ogs_text_searcher_search_text (search->text_searcher,
 323                                                        where,
 324                                                        strlen (where));
 325
 326                 ogs_text_searcher_reset (search->text_searcher);
 327         } else {
 328 #endif
 329                 if (what == NULL || where == NULL) {
 330                         return FALSE;
 331                 }
 332
 333                 found = !modest_text_utils_utf8_strcmp (what, where, TRUE);
 334 #ifdef MODEST_HAVE_OGS
 335         }
 336 #endif
 337         return found;
 338 }
 339
 340
 341 static gboolean search_mime_part_and_child_parts (TnyMimePart *part, ModestSearch *search)
 342 {
 343         gboolean found = FALSE;
 344         #ifdef MODEST_HAVE_OGS
 345         found = search_mime_part_ogs (part, search);
 346         #else
 347         found = search_mime_part_strcmp (part, search);
 348         #endif
 349
 350         if (found) {
 351                 return found;
 352         }
 353
 354         /* Check the child part too, recursively: */
 355         TnyList *child_parts = tny_simple_list_new ();
 356         tny_mime_part_get_parts (TNY_MIME_PART (part), child_parts);
 357
 358         TnyIterator *piter = tny_list_create_iterator (child_parts);
 359         while (!found && !tny_iterator_is_done (piter)) {
 360                 TnyMimePart *pcur = (TnyMimePart *) tny_iterator_get_current (piter);
 361                 if (pcur) {
 362                         found = search_mime_part_and_child_parts (pcur, search);
 363
 364                         g_object_unref (pcur);
 365                 }
 366
 367                 tny_iterator_next (piter);
 368         }
 369
 370         g_object_unref (piter);
 371         g_object_unref (child_parts);
 372
 373         return found;
 374 }
 375
 376 /**
 377  * modest_search:
 378  * @folder: a #TnyFolder instance
 379  * @search: a #ModestSearch query
 380  *
 381  * This operation will search @folder for headers that match the query @search,
 382  * if the folder itself matches the query.
 383  * It will return a doubly linked list with URIs that point to the message.
 384  **/
 385 GList *
 386 modest_search_folder (TnyFolder *folder, ModestSearch *search)
 387 {
 388         /* Check that we should be searching this folder. */
 389         /* Note that we don't try to search sub-folders.
 390          * Maybe we should, but that should be specified. */
 391         if (search->folder && strlen (search->folder) && (strcmp (tny_folder_get_id (folder), search->folder) != 0))
 392                 return NULL;
 393
 394         GList *retval = NULL;
 395         TnyIterator *iter = NULL;
 396         TnyList *list = NULL;
 397
 398 #ifdef MODEST_HAVE_OGS
 399         if (search->flags & MODEST_SEARCH_USE_OGS) {
 400
 401                 if (search->text_searcher == NULL && search->query != NULL) {
 402                         OgsTextSearcher *text_searcher;
 403
 404                         text_searcher = ogs_text_searcher_new (FALSE);
 405                         ogs_text_searcher_parse_query (text_searcher, search->query);
 406                         search->text_searcher = text_searcher;
 407                 }
 408         }
 409 #endif
 410
 411         list = tny_simple_list_new ();
 412         GError *error = NULL;
 413         tny_folder_get_headers (folder, list, FALSE /* don't refresh */, &error);
 414         if (error) {
 415                 g_warning ("%s: tny_folder_get_headers() failed with error=%s.\n",
 416                 __FUNCTION__, error->message);
 417                 g_error_free (error);
 418                 error = NULL;
 419         }
 420
 421         iter = tny_list_create_iterator (list);
 422
 423         while (!tny_iterator_is_done (iter)) {
 424                 TnyHeader *cur = (TnyHeader *) tny_iterator_get_current (iter);
 425                 const time_t t = tny_header_get_date_sent (cur);
 426                 gboolean found = FALSE;
 427
 428                 /* Ignore deleted (not yet expunged) emails: */
 429                 if (tny_header_get_flags(cur) & TNY_HEADER_FLAG_DELETED)
 430                         goto go_next;
 431
 432                 if (search->flags & MODEST_SEARCH_BEFORE)
 433                         if (!(t <= search->end_date))
 434                                 goto go_next;
 435
 436                 if (search->flags & MODEST_SEARCH_AFTER)
 437                         if (!(t >= search->start_date))
 438                                 goto go_next;
 439
 440                 if (search->flags & MODEST_SEARCH_SIZE)
 441                         if (tny_header_get_message_size (cur) < search->minsize)
 442                                 goto go_next;
 443
 444                 if (search->flags & MODEST_SEARCH_SUBJECT) {
 445                         const char *str = tny_header_get_subject (cur);
 446
 447                         if ((found = search_string (search->subject, str, search))) {
 448                             retval = add_hit (retval, cur, folder);
 449                         }
 450                 }
 451
 452                 if (!found && search->flags & MODEST_SEARCH_SENDER) {
 453                         char *str = g_strdup (tny_header_get_from (cur));
 454
 455                         if ((found = search_string (search->from, (const gchar *) str, search))) {
 456                                 retval = add_hit (retval, cur, folder);
 457                         }
 458                         g_free (str);
 459                 }
 460
 461                 if (!found && search->flags & MODEST_SEARCH_RECIPIENT) {
 462                         const char *str = tny_header_get_to (cur);
 463
 464                         if ((found = search_string (search->recipient, str, search))) {
 465                                 retval = add_hit (retval, cur, folder);
 466                         }
 467                 }
 468
 469                 if (!found && search->flags & MODEST_SEARCH_BODY) {
 470                         TnyHeaderFlags flags;
 471                         GError      *err = NULL;
 472                         TnyMsg      *msg = NULL;
 473
 474                         flags = tny_header_get_flags (cur);
 475
 476                         if (!(flags & TNY_HEADER_FLAG_CACHED)) {
 477                                 goto go_next;
 478                         }
 479
 480                         msg = tny_folder_get_msg (folder, cur, &err);
 481
 482                         if (err != NULL || msg == NULL) {
 483                                 g_warning ("%s: Could not get message.\n", __FUNCTION__);
 484                                 g_error_free (err);
 485
 486                                 if (msg) {
 487                                         g_object_unref (msg);
 488                                 }
 489                         } else {
 490
 491                                 found = search_mime_part_and_child_parts (TNY_MIME_PART (msg),
 492                                                                           search);
 493                                 if (found) {
 494                                         retval = add_hit (retval, cur, folder);
 495                                 }
 496                         }
 497
 498                         if (msg)
 499                                 g_object_unref (msg);
 500                 }
 501
 502 go_next:
 503                 g_object_unref (cur);
 504                 tny_iterator_next (iter);
 505         }
 506
 507         g_object_unref (iter);
 508         g_object_unref (list);
 509         return retval;
 510 }
 511
 512 GList *
 513 modest_search_account (TnyAccount *account, ModestSearch *search)
 514 {
 515         TnyFolderStore      *store;
 516         TnyIterator         *iter;
 517         TnyList             *folders;
 518         GList               *hits;
 519         GError              *error;
 520
 521         error = NULL;
 522         hits = NULL;
 523
 524         store = TNY_FOLDER_STORE (account);
 525
 526         folders = tny_simple_list_new ();
 527         tny_folder_store_get_folders (store, folders, NULL, &error);
 528
 529         if (error != NULL) {
 530                 g_object_unref (folders);
 531                 return NULL;
 532         }
 533
 534         iter = tny_list_create_iterator (folders);
 535         while (!tny_iterator_is_done (iter)) {
 536                 TnyFolder *folder = NULL;
 537                 GList     *res = NULL;
 538
 539                 folder = TNY_FOLDER (tny_iterator_get_current (iter));
 540                 if (folder) {
 541                         /* g_debug ("DEBUG: %s: searching folder %s.",
 542                                 __FUNCTION__, tny_folder_get_name (folder)); */
 543
 544                         res = modest_search_folder (folder, search);
 545
 546                         if (res != NULL) {
 547                                 if (hits == NULL) {
 548                                         hits = res;
 549                                 } else {
 550                                         hits = g_list_concat (hits, res);
 551                                 }
 552                         }
 553
 554                         g_object_unref (folder);
 555                 }
 556
 557                 tny_iterator_next (iter);
 558         }
 559
 560         g_object_unref (iter);
 561         g_object_unref (folders);
 562
 563         /* printf ("DEBUG: %s: hits length = %d\n", __FUNCTION__, g_list_length (hits)); */
 564         return hits;
 565 }
 566
 567 GList *
 568 modest_search_all_accounts (ModestSearch *search)
 569 {
 570         /* printf ("DEBUG: %s: query=%s\n", __FUNCTION__, search->query); */
 571         ModestTnyAccountStore *astore;
 572         TnyList               *accounts;
 573         TnyIterator           *iter;
 574         GList                 *hits;
 575
 576         hits = NULL;
 577         astore = modest_runtime_get_account_store ();
 578
 579         accounts = tny_simple_list_new ();
 580         tny_account_store_get_accounts (TNY_ACCOUNT_STORE (astore),
 581                                         accounts,
 582                                         TNY_ACCOUNT_STORE_STORE_ACCOUNTS);
 583
 584         iter = tny_list_create_iterator (accounts);
 585         while (!tny_iterator_is_done (iter)) {
 586                 TnyAccount *account = NULL;
 587                 GList      *res = NULL;
 588
 589                 account = TNY_ACCOUNT (tny_iterator_get_current (iter));
 590                 if (account) {
 591                         /* g_debug ("DEBUG: %s: Searching account %s",
 592                          __FUNCTION__, tny_account_get_name (account)); */
 593
 594                         /* Give the account time to go online if necessary,
 595                          * for instance if this is immediately after startup,
 596                          * after D-Bus activation: */
 597                         modest_platform_check_and_wait_for_account_is_online (account);
 598
 599                         /* Search: */
 600                         res = modest_search_account (account, search);
 601
 602                         if (res != NULL) {
 603                                 if (hits == NULL) {
 604                                         hits = res;
 605                                 } else {
 606                                         hits = g_list_concat (hits, res);
 607                                 }
 608                         }
 609
 610                         g_object_unref (account);
 611                 }
 612
 613                 tny_iterator_next (iter);
 614         }
 615
 616         g_object_unref (accounts);
 617         g_object_unref (iter);
 618
 619         /* printf ("DEBUG: %s: end: hits length=%d\n", __FUNCTION__, g_list_length(hits)); */
 620         return hits;
 621 }
 622
 623