* Fixes a crash in searches when searching in a single folder, and there are more...
[modest] / src / modest-search.c
1 /* Copyright (c) 2006, Nokia Corporation
2  * All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions are
6  * met:
7  *
8  * * Redistributions of source code must retain the above copyright
9  *   notice, this list of conditions and the following disclaimer.
10  * * Redistributions in binary form must reproduce the above copyright
11  *   notice, this list of conditions and the following disclaimer in the
12  *   documentation and/or other materials provided with the distribution.
13  * * Neither the name of the Nokia Corporation nor the names of its
14  *   contributors may be used to endorse or promote products derived from
15  *   this software without specific prior written permission.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
18  * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
19  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
20  * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
21  * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
22  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
23  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
24  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
25  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
26  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28  */
29
30 #ifndef _GNU_SOURCE
31 #define _GNU_SOURCE
32 #endif
33
34 #ifdef HAVE_CONFIG_H
35 #include <config.h>
36 #endif
37
38 #include <string.h>
39
40 #include <tny-shared.h>
41 #include <tny-folder.h>
42 #include <tny-folder-store.h>
43 #include <tny-list.h>
44 #include <tny-iterator.h>
45 #include <tny-simple-list.h>
46 #include <tny-camel-imap-store-account.h>
47 #include <tny-camel-pop-store-account.h>
48
49 #include "modest-text-utils.h"
50 #include "modest-account-mgr.h"
51 #include "modest-tny-account-store.h"
52 #include "modest-tny-account.h"
53 #include "modest-tny-mime-part.h"
54 #include "modest-tny-folder.h"
55 #include "modest-search.h"
56 #include "modest-runtime.h"
57 #include "modest-platform.h"
58
59 typedef struct 
60 {
61         guint folder_count;
62         guint folder_total;
63         guint account_total;
64         GList *msg_hits;
65         ModestSearch *search;
66         ModestSearchCallback callback;
67         gpointer user_data;
68 } SearchHelper;
69
70 static SearchHelper *create_helper (ModestSearchCallback callback, 
71                                     ModestSearch *search,
72                                     gpointer user_data);
73
74 static void          check_search_finished (SearchHelper *helper);
75
76 static gchar *
77 g_strdup_or_null (const gchar *str)
78 {
79         gchar *string = NULL;
80
81         if  (str != NULL) {
82                 string = g_strdup (str);
83         }
84
85         return string;
86 }
87
88 static GList*
89 add_hit (GList *list, TnyHeader *header, TnyFolder *folder)
90 {
91         ModestSearchResultHit *hit;
92         TnyHeaderFlags   flags;
93         char            *furl;
94         char            *msg_url;
95         char      *uid;
96         char      *subject;
97         char      *sender;
98
99         hit = g_slice_new0 (ModestSearchResultHit);
100
101         furl = tny_folder_get_url_string (folder);
102         g_debug ("%s: folder URL=%s\n", __FUNCTION__, furl);
103         if (!furl) {
104                 g_warning ("%s: tny_folder_get_url_string(): returned NULL for folder. Folder name=%s\n", __FUNCTION__, tny_folder_get_name (folder));
105         }
106         
107         /* Make sure that we use the short UID instead of the long UID,
108          * and/or find out what UID form is used when finding, in camel_data_cache_get().
109          * so we can find what we get. Philip is working on this.
110          */
111         uid = tny_header_dup_uid (header);
112         if (!furl) {
113                 gchar *subject = tny_header_dup_subject (header);
114                 g_warning ("%s: tny_header_get_uid(): returned NULL for message with subject=%s\n", __FUNCTION__, subject);
115                 g_free (subject);
116         }
117         
118         msg_url = g_strdup_printf ("%s/%s", furl, uid);
119         g_free (furl);
120         g_free (uid);
121         
122         subject = tny_header_dup_subject (header);
123         sender = tny_header_dup_from (header);
124         
125         flags = tny_header_get_flags (header);
126
127         hit->msgid = msg_url;
128         hit->subject = subject;
129         hit->sender = sender;
130         hit->folder = g_strdup_or_null (tny_folder_get_name (folder));
131         hit->msize = tny_header_get_message_size (header);
132         hit->has_attachment = flags & TNY_HEADER_FLAG_ATTACHMENTS;
133         hit->is_unread = ! (flags & TNY_HEADER_FLAG_SEEN);
134         hit->timestamp = MIN (tny_header_get_date_received (header), tny_header_get_date_sent (header));
135         
136         return g_list_prepend (list, hit);
137 }
138
139 /** Call this until it returns FALSE or nread is set to 0.
140  * 
141  * @result: FALSE is something failed. */
142 static gboolean
143 read_chunk (TnyStream *stream, char *buffer, gsize count, gsize *nread)
144 {
145         gsize _nread = 0;
146         gssize res = 0;
147
148         while (_nread < count) {
149                 res = tny_stream_read (stream,
150                                        buffer + _nread, 
151                                        count - _nread);
152                 if (res == -1) { /* error */
153                         *nread = _nread;
154                         return FALSE;
155                 }
156
157                 _nread += res;
158                 
159                 if (res == 0) { /* no more bytes read. */
160                         *nread = _nread;
161                         return TRUE;
162                 }
163         }
164
165         *nread = _nread;
166         return TRUE;
167 }
168
169 #ifdef MODEST_HAVE_OGS
170 /*
171  * This function assumes that the mime part is of type "text / *"
172  */
173 static gboolean
174 search_mime_part_ogs (TnyMimePart *part, ModestSearch *search)
175 {
176         TnyStream *stream = NULL;
177         char       buffer[4096];
178         const gsize len = sizeof (buffer);
179         gsize      nread = 0;
180         gboolean   is_text_html = FALSE;
181         gboolean   found = FALSE;
182         gboolean   res = FALSE;
183         
184         is_text_html = tny_mime_part_content_type_is (part, "text/html");
185
186         stream = tny_mime_part_get_stream (part);
187
188         res = read_chunk (stream, buffer, len, &nread);
189         while (res && (nread > 0)) {
190                 /* search->text_searcher was instantiated in modest_search_folder(). */
191                 
192                 if (is_text_html) {
193                         found = ogs_text_searcher_search_html (search->text_searcher,
194                                                                buffer,
195                                                                nread,
196                                                                nread < len);
197                 } else {
198                         found = ogs_text_searcher_search_text (search->text_searcher,
199                                                                buffer,
200                                                                nread);
201                 }
202
203                 /* HACK: this helps UI refreshes because the search
204                    operations could be heavy */
205                 while (gtk_events_pending ())
206                         gtk_main_iteration ();
207
208                 if (found) {
209                         break;
210                 }
211                 
212                 nread = 0;
213                 res = read_chunk (stream, buffer, len, &nread);
214         }
215         g_object_unref (stream);
216
217         if (!found) {
218                 found = ogs_text_searcher_search_done (search->text_searcher);
219         }
220
221         ogs_text_searcher_reset (search->text_searcher);
222
223         return found;
224 }
225
226 #else
227
228 /*
229  * This function assumes that the mime part is of type "text / *"
230  */
231 static gboolean
232 search_mime_part_strcmp (TnyMimePart *part, ModestSearch *search)
233 {
234         TnyStream *stream;
235         char       buffer[8193];
236         char      *chunk[2];
237         gssize     len;
238         gsize     nread;
239         gboolean   found;
240         gboolean   res;
241
242         found = FALSE;
243         len = (sizeof (buffer) - 1) / 2;
244
245         if (strlen (search->body) > len) {
246                 g_warning ("Search term bigger then chunk."
247                            "We might not find everything");     
248         }
249
250         stream = tny_mime_part_get_stream (part);
251
252         memset (buffer, 0, sizeof (buffer));
253         chunk[0] = buffer;
254         chunk[1] = buffer + len;
255
256         res = read_chunk (stream, chunk[0], len, &nread);
257
258         if (res == FALSE) {
259                 goto done;
260         }
261
262         found = !modest_text_utils_utf8_strcmp (search->body,
263                                                 buffer,
264                                                 TRUE);
265         if (found) {
266                 goto done;
267         }
268
269         /* This works like this:
270          * buffer: [ooooooooooo|xxxxxxxxxxxx|\0] 
271          *          ^chunk[0]  ^chunk[1]
272          * we have prefilled chunk[0] now we always read into chunk[1]
273          * and then move the content of chunk[1] to chunk[0].
274          * The idea is to prevent not finding search terms that are
275          * spread across 2 reads:        
276          * buffer: [ooooooooTES|Txxxxxxxxxxx|\0] 
277          * We should catch that because we always search the whole
278          * buffer not only the chunks.
279          *
280          * Of course that breaks for search terms > sizeof (chunk)
281          * but sizeof (chunk) should be big enough I guess (see
282          * the g_warning in this function)
283          * */   
284         while ((res = read_chunk (stream, chunk[1], len, &nread))) {
285                 buffer[len + nread] = '\0';
286
287                 found = !modest_text_utils_utf8_strcmp (search->body,
288                                                         buffer,
289                                                         TRUE);
290
291                 /* HACK: this helps UI refreshes because the search
292                    operations could be heavy */
293                 while (gtk_events_pending ())
294                         gtk_main_iteration ();
295
296                 if ((found)||(nread == 0)) {
297                         break;
298                 }
299
300                 /* also move the \0 */
301                 g_memmove (chunk[0], chunk[1], len + 1);
302         }
303
304 done:
305         g_object_unref (stream);
306         return found;
307 }
308 #endif /*MODEST_HAVE_OGS*/
309
310 static gboolean
311 search_string (const char      *what,
312                const char      *where,
313                ModestSearch    *search)
314 {
315         gboolean found = FALSE;
316 #ifdef MODEST_HAVE_OGS
317         if (search->flags & MODEST_SEARCH_USE_OGS) {
318                 found = ogs_text_searcher_search_text (search->text_searcher,
319                                                        where,
320                                                        strlen (where));
321
322                 ogs_text_searcher_reset (search->text_searcher);
323         } else {
324 #endif
325                 if (what == NULL || where == NULL) {
326                         return FALSE;
327                 }
328
329                 found = !modest_text_utils_utf8_strcmp (what, where, TRUE);
330 #ifdef MODEST_HAVE_OGS
331         }
332 #endif
333
334         /* HACK: this helps UI refreshes because the search
335            operations could be heavy */
336         while (gtk_events_pending ())
337                 gtk_main_iteration ();
338
339         return found;
340 }
341
342
343 static gboolean 
344 search_mime_part_and_child_parts (TnyMimePart *part, ModestSearch *search)
345 {
346         gboolean found = FALSE;
347
348         /* Do not search into attachments */
349         if (modest_tny_mime_part_is_attachment_for_modest (part))
350                 return FALSE;
351
352         #ifdef MODEST_HAVE_OGS
353         found = search_mime_part_ogs (part, search);
354         #else
355         found = search_mime_part_strcmp (part, search);
356         #endif
357
358         if (found) {    
359                 return found;           
360         }
361         
362         /* Check the child part too, recursively: */
363         TnyList *child_parts = tny_simple_list_new ();
364         tny_mime_part_get_parts (TNY_MIME_PART (part), child_parts);
365
366         TnyIterator *piter = tny_list_create_iterator (child_parts);
367         while (!found && !tny_iterator_is_done (piter)) {
368                 TnyMimePart *pcur = (TnyMimePart *) tny_iterator_get_current (piter);
369                 if (pcur) {
370                         found = search_mime_part_and_child_parts (pcur, search);
371
372                         g_object_unref (pcur);
373                 }
374
375                 tny_iterator_next (piter);
376         }
377
378         g_object_unref (piter);
379         g_object_unref (child_parts);
380         
381         return found;
382 }
383
384 static void 
385 modest_search_folder_get_headers_cb (TnyFolder *folder, 
386                                      gboolean cancelled, 
387                                      TnyList *headers, 
388                                      GError *err, 
389                                      gpointer user_data)
390 {
391         TnyIterator *iter = NULL;
392         SearchHelper *helper;
393
394         helper = (SearchHelper *) user_data;
395
396         if (err || cancelled) {
397                 goto end;
398         }
399
400         iter = tny_list_create_iterator (headers);
401
402         while (!tny_iterator_is_done (iter)) {
403
404                 TnyHeader *cur = (TnyHeader *) tny_iterator_get_current (iter);
405                 const time_t t = tny_header_get_date_sent (cur);
406                 gboolean found = FALSE;
407                 
408                 /* Ignore deleted (not yet expunged) emails: */
409                 if (tny_header_get_flags(cur) & TNY_HEADER_FLAG_DELETED)
410                         goto go_next;
411                         
412                 if (helper->search->flags & MODEST_SEARCH_BEFORE)
413                         if (!(t <= helper->search->end_date))
414                                 goto go_next;
415
416                 if (helper->search->flags & MODEST_SEARCH_AFTER)
417                         if (!(t >= helper->search->start_date))
418                                 goto go_next;
419
420                 if (helper->search->flags & MODEST_SEARCH_SIZE)
421                         if (tny_header_get_message_size (cur) < helper->search->minsize)
422                                 goto go_next;
423
424                 if (helper->search->flags & MODEST_SEARCH_SUBJECT) {
425                         char *str = tny_header_dup_subject (cur);
426
427                         if ((found = search_string (helper->search->subject, str, helper->search))) {
428                             helper->msg_hits = add_hit (helper->msg_hits, cur, folder);
429                         }
430                         g_free (str);
431                 }
432                 
433                 if (!found && helper->search->flags & MODEST_SEARCH_SENDER) {
434                         char *str = tny_header_dup_from (cur);
435
436                         if ((found = search_string (helper->search->from, (const gchar *) str, helper->search))) {
437                                 helper->msg_hits = add_hit (helper->msg_hits, cur, folder);
438                         }
439                         g_free (str);
440                 }
441                 
442                 if (!found && helper->search->flags & MODEST_SEARCH_RECIPIENT) {
443                         char *str = tny_header_dup_to (cur);
444
445                         if ((found = search_string (helper->search->recipient, str, helper->search))) {
446                                 helper->msg_hits = add_hit (helper->msg_hits, cur, folder);
447                         }
448                         g_free (str);
449                 }
450         
451                 if (!found && helper->search->flags & MODEST_SEARCH_BODY) {
452                         TnyHeaderFlags flags;
453                         GError      *err = NULL;
454                         TnyMsg      *msg = NULL;
455
456                         flags = tny_header_get_flags (cur);
457
458                         if (!(flags & TNY_HEADER_FLAG_CACHED)) {
459                                 goto go_next;
460                         }
461
462                         msg = tny_folder_get_msg (folder, cur, &err);
463
464                         if (err != NULL || msg == NULL) {
465                                 g_warning ("%s: Could not get message.\n", __FUNCTION__);
466                                 g_error_free (err);
467
468                                 if (msg) {
469                                         g_object_unref (msg);
470                                 }
471                         } else {        
472                                 gchar *str;
473                                 str = tny_header_dup_subject (cur);
474                                 g_debug ("Searching in %s\n", str);
475                                 g_free (str);
476                         
477                                 found = search_mime_part_and_child_parts (TNY_MIME_PART (msg),
478                                                                           helper->search);
479                                 if (found) {
480                                         helper->msg_hits = add_hit (helper->msg_hits, cur, folder);
481                                 }
482                         }
483                         
484                         if (msg)
485                                 g_object_unref (msg);
486                 }
487         go_next:
488                 g_object_unref (cur);
489                 tny_iterator_next (iter);
490         }
491
492         /* Frees */
493         g_object_unref (iter);
494  end:
495         if (headers)
496                 g_object_unref (headers);
497
498         /* Check search finished */
499         helper->folder_count++;
500         check_search_finished (helper);
501 }
502
503 static void
504 _search_folder (TnyFolder *folder, 
505                 SearchHelper *helper)
506 {
507         TnyList *list = NULL;
508
509         g_debug ("%s: searching folder %s.", __FUNCTION__, tny_folder_get_name (folder));
510         
511         /* Check that we should be searching this folder. */
512         /* Note that we don't try to search sub-folders. 
513          * Maybe we should, but that should be specified. */
514         if (helper->search->folder && strlen (helper->search->folder)) {
515                 if (!strcmp (helper->search->folder, "outbox")) {
516                         if (modest_tny_folder_guess_folder_type (folder) != TNY_FOLDER_TYPE_OUTBOX) {
517                                 modest_search_folder_get_headers_cb (folder, TRUE, NULL, NULL, helper); 
518                                 return;
519                         }
520                 } else if (strcmp (tny_folder_get_id (folder), helper->search->folder) != 0) {
521                         modest_search_folder_get_headers_cb (folder, TRUE, NULL, NULL, helper); 
522                         return;
523                 }
524         }
525         
526 #ifdef MODEST_HAVE_OGS
527         if (helper->search->flags & MODEST_SEARCH_USE_OGS) {
528         
529                 if (helper->search->text_searcher == NULL && helper->search->query != NULL) {
530                         OgsTextSearcher *text_searcher; 
531
532                         text_searcher = ogs_text_searcher_new (FALSE);
533                         ogs_text_searcher_parse_query (text_searcher, helper->search->query);
534                         helper->search->text_searcher = text_searcher;
535                 }
536         }
537 #endif
538         list = tny_simple_list_new ();
539         /* Get the headers */
540         tny_folder_get_headers_async (folder, list, FALSE, 
541                                       modest_search_folder_get_headers_cb, 
542                                       NULL, helper);
543 }
544
545 void
546 modest_search_folder (TnyFolder *folder, 
547                       ModestSearch *search,
548                       ModestSearchCallback callback,
549                       gpointer user_data)
550 {
551         SearchHelper *helper;
552
553         /* Create the helper */
554         helper = create_helper (callback, search, user_data);
555
556         /* Search */
557         _search_folder (folder, helper);
558 }
559
560 static void
561 modest_search_account_get_folders_cb (TnyFolderStore *self, 
562                                       gboolean cancelled, 
563                                       TnyList *folders, 
564                                       GError *err, 
565                                       gpointer user_data)
566 {
567         TnyIterator *iter;
568         SearchHelper *helper;
569
570         helper = (SearchHelper *) user_data;
571
572         /* Remove the "account" reference */
573         helper->account_total--;
574
575         if (err || cancelled) {
576                 goto end;
577         }
578
579         iter = tny_list_create_iterator (folders);
580         while (!tny_iterator_is_done (iter)) {
581                 TnyFolder *folder = NULL;
582
583                 /* Search into folder */
584                 folder = TNY_FOLDER (tny_iterator_get_current (iter));  
585                 helper->folder_total++;
586                 _search_folder (folder, (SearchHelper *) user_data);
587                 g_object_unref (folder);
588
589                 tny_iterator_next (iter);
590         }
591         g_object_unref (iter);
592  end:
593         if (folders)
594                 g_object_unref (folders);
595
596         /* Check search finished */
597         check_search_finished (helper);
598 }
599
600 static void
601 _search_account (TnyAccount *account, 
602                  SearchHelper *helper)
603 {
604         TnyList *folders = tny_simple_list_new ();
605
606         g_debug ("%s: Searching account %s", __FUNCTION__, tny_account_get_name (account));
607
608         /* Add a "reference" to the folder total. This allows the code
609            not to finalize the helper if an account is fully refreshed
610            before we get the folders of the others */
611         helper->account_total++;
612
613         /* Get folders */
614         tny_folder_store_get_folders_async (TNY_FOLDER_STORE (account), folders, NULL, 
615                                             modest_search_account_get_folders_cb, 
616                                             NULL, helper);
617 }
618
619 void
620 modest_search_account (TnyAccount *account, 
621                        ModestSearch *search,
622                        ModestSearchCallback callback,
623                        gpointer user_data)
624 {
625         SearchHelper *helper;
626
627         /* Create the helper */
628         helper = create_helper (callback, search, user_data);
629
630         /* Search */
631         _search_account (account, helper);
632 }
633
634 void
635 modest_search_all_accounts (ModestSearch *search,
636                             ModestSearchCallback callback,
637                             gpointer user_data)
638 {
639         ModestTnyAccountStore *astore;
640         TnyList *accounts;
641         TnyIterator *iter;
642         GList *hits;
643         SearchHelper *helper;
644
645         hits = NULL;
646         astore = modest_runtime_get_account_store ();
647
648         accounts = tny_simple_list_new ();
649         tny_account_store_get_accounts (TNY_ACCOUNT_STORE (astore),
650                                         accounts,
651                                         TNY_ACCOUNT_STORE_STORE_ACCOUNTS);
652
653         /* Create the helper */
654         helper = create_helper (callback, search, user_data);
655
656         /* Search through all accounts */
657         iter = tny_list_create_iterator (accounts);
658         while (!tny_iterator_is_done (iter)) {
659                 TnyAccount *account = NULL;
660
661                 account = TNY_ACCOUNT (tny_iterator_get_current (iter));
662                 _search_account (account, helper);
663                 g_object_unref (account);
664
665                 tny_iterator_next (iter);
666         }
667         g_object_unref (iter);
668         g_object_unref (accounts);
669 }
670
671 static SearchHelper *
672 create_helper (ModestSearchCallback callback, 
673                ModestSearch *search,
674                gpointer user_data)
675 {
676         SearchHelper *helper;
677
678         helper = g_slice_new0 (SearchHelper);
679         helper->folder_count = 0;
680         helper->folder_total = 0;
681         helper->account_total = 0;
682         helper->search = search;
683         helper->callback = callback;
684         helper->user_data = user_data;
685         helper->msg_hits = NULL;
686
687         return helper;
688 }
689
690 void 
691 modest_search_free (ModestSearch *search)
692 {
693         if (search->folder)
694                 g_free (search->folder);
695         if (search->subject)
696                 g_free (search->subject);
697         if (search->from)
698                 g_free (search->from);
699         if (search->recipient)
700                 g_free (search->recipient);
701         if (search->body)
702                 g_free (search->body);
703
704 #ifdef MODEST_HAVE_OGS
705         if (search->query)
706                 g_free (search->query);
707         if (search->text_searcher)
708                 ogs_text_searcher_free (search->text_searcher); 
709 #endif
710 }
711
712 static void
713 check_search_finished (SearchHelper *helper)
714 {
715         /* If there are no more folders to check the account search has finished */
716         if (helper->folder_count == helper->folder_total && helper->account_total == 0) {
717                 /* callback */
718                 helper->callback (helper->msg_hits, helper->user_data);
719                 
720                 /* free helper */
721                 g_list_free (helper->msg_hits);
722                 g_slice_free (SearchHelper, helper);
723         }
724 }