This patch addapts modest to the new API for TnyHeader in tinymail
[modest] / src / modest-search.c
1 /* Copyright (c) 2006, Nokia Corporation
2  * All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions are
6  * met:
7  *
8  * * Redistributions of source code must retain the above copyright
9  *   notice, this list of conditions and the following disclaimer.
10  * * Redistributions in binary form must reproduce the above copyright
11  *   notice, this list of conditions and the following disclaimer in the
12  *   documentation and/or other materials provided with the distribution.
13  * * Neither the name of the Nokia Corporation nor the names of its
14  *   contributors may be used to endorse or promote products derived from
15  *   this software without specific prior written permission.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
18  * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
19  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
20  * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
21  * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
22  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
23  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
24  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
25  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
26  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28  */
29
30 #ifndef _GNU_SOURCE
31 #define _GNU_SOURCE
32 #endif
33
34 #ifdef HAVE_CONFIG_H
35 #include <config.h>
36 #endif
37
38 #include <string.h>
39
40 #include <tny-shared.h>
41 #include <tny-folder.h>
42 #include <tny-folder-store.h>
43 #include <tny-list.h>
44 #include <tny-iterator.h>
45 #include <tny-simple-list.h>
46 #include <tny-camel-imap-store-account.h>
47 #include <tny-camel-pop-store-account.h>
48
49 #include "modest-text-utils.h"
50 #include "modest-account-mgr.h"
51 #include "modest-tny-account-store.h"
52 #include "modest-tny-account.h"
53 #include "modest-tny-mime-part.h"
54 #include "modest-tny-folder.h"
55 #include "modest-search.h"
56 #include "modest-runtime.h"
57 #include "modest-platform.h"
58
59 typedef struct 
60 {
61         guint folder_count;
62         guint folder_total;
63         GList *msg_hits;
64         ModestSearch *search;
65         ModestSearchCallback callback;
66         gpointer user_data;
67 } SearchHelper;
68
69 static SearchHelper *create_helper (ModestSearchCallback callback, 
70                                     ModestSearch *search,
71                                     gpointer user_data);
72
73 static void          check_search_finished (SearchHelper *helper);
74
75 static gchar *
76 g_strdup_or_null (const gchar *str)
77 {
78         gchar *string = NULL;
79
80         if  (str != NULL) {
81                 string = g_strdup (str);
82         }
83
84         return string;
85 }
86
87 static GList*
88 add_hit (GList *list, TnyHeader *header, TnyFolder *folder)
89 {
90         ModestSearchResultHit *hit;
91         TnyHeaderFlags   flags;
92         char            *furl;
93         char            *msg_url;
94         char      *uid;
95         char      *subject;
96         char      *sender;
97
98         hit = g_slice_new0 (ModestSearchResultHit);
99
100         furl = tny_folder_get_url_string (folder);
101         g_debug ("%s: folder URL=%s\n", __FUNCTION__, furl);
102         if (!furl) {
103                 g_warning ("%s: tny_folder_get_url_string(): returned NULL for folder. Folder name=%s\n", __FUNCTION__, tny_folder_get_name (folder));
104         }
105         
106         /* Make sure that we use the short UID instead of the long UID,
107          * and/or find out what UID form is used when finding, in camel_data_cache_get().
108          * so we can find what we get. Philip is working on this.
109          */
110         uid = tny_header_dup_uid (header);
111         if (!furl) {
112                 gchar *subject = tny_header_dup_subject (header);
113                 g_warning ("%s: tny_header_get_uid(): returned NULL for message with subject=%s\n", __FUNCTION__, subject);
114                 g_free (subject);
115         }
116         
117         msg_url = g_strdup_printf ("%s/%s", furl, uid);
118         g_free (furl);
119         g_free (uid);
120         
121         subject = tny_header_dup_subject (header);
122         sender = tny_header_dup_from (header);
123         
124         flags = tny_header_get_flags (header);
125
126         hit->msgid = msg_url;
127         hit->subject = subject;
128         hit->sender = sender;
129         hit->folder = g_strdup_or_null (tny_folder_get_name (folder));
130         hit->msize = tny_header_get_message_size (header);
131         hit->has_attachment = flags & TNY_HEADER_FLAG_ATTACHMENTS;
132         hit->is_unread = ! (flags & TNY_HEADER_FLAG_SEEN);
133         hit->timestamp = MIN (tny_header_get_date_received (header), tny_header_get_date_sent (header));
134         
135         return g_list_prepend (list, hit);
136 }
137
138 /** Call this until it returns FALSE or nread is set to 0.
139  * 
140  * @result: FALSE is something failed. */
141 static gboolean
142 read_chunk (TnyStream *stream, char *buffer, gsize count, gsize *nread)
143 {
144         gsize _nread = 0;
145         gssize res = 0;
146
147         while (_nread < count) {
148                 res = tny_stream_read (stream,
149                                        buffer + _nread, 
150                                        count - _nread);
151                 if (res == -1) { /* error */
152                         *nread = _nread;
153                         return FALSE;
154                 }
155
156                 _nread += res;
157                 
158                 if (res == 0) { /* no more bytes read. */
159                         *nread = _nread;
160                         return TRUE;
161                 }
162         }
163
164         *nread = _nread;
165         return TRUE;
166 }
167
168 #ifdef MODEST_HAVE_OGS
169 /*
170  * This function assumes that the mime part is of type "text / *"
171  */
172 static gboolean
173 search_mime_part_ogs (TnyMimePart *part, ModestSearch *search)
174 {
175         TnyStream *stream = NULL;
176         char       buffer[4096];
177         const gsize len = sizeof (buffer);
178         gsize      nread = 0;
179         gboolean   is_text_html = FALSE;
180         gboolean   found = FALSE;
181         gboolean   res = FALSE;
182         
183         is_text_html = tny_mime_part_content_type_is (part, "text/html");
184
185         stream = tny_mime_part_get_stream (part);
186
187         res = read_chunk (stream, buffer, len, &nread);
188         while (res && (nread > 0)) {
189                 /* search->text_searcher was instantiated in modest_search_folder(). */
190                 
191                 if (is_text_html) {
192                         found = ogs_text_searcher_search_html (search->text_searcher,
193                                                                buffer,
194                                                                nread,
195                                                                nread < len);
196                 } else {
197                         found = ogs_text_searcher_search_text (search->text_searcher,
198                                                                buffer,
199                                                                nread);
200                 }
201
202                 /* HACK: this helps UI refreshes because the search
203                    operations could be heavy */
204                 while (gtk_events_pending ())
205                         gtk_main_iteration ();
206
207                 if (found) {
208                         break;
209                 }
210                 
211                 nread = 0;
212                 res = read_chunk (stream, buffer, len, &nread);
213         }
214         g_object_unref (stream);
215
216         if (!found) {
217                 found = ogs_text_searcher_search_done (search->text_searcher);
218         }
219
220         ogs_text_searcher_reset (search->text_searcher);
221
222         return found;
223 }
224
225 #else
226
227 /*
228  * This function assumes that the mime part is of type "text / *"
229  */
230 static gboolean
231 search_mime_part_strcmp (TnyMimePart *part, ModestSearch *search)
232 {
233         TnyStream *stream;
234         char       buffer[8193];
235         char      *chunk[2];
236         gssize     len;
237         gsize     nread;
238         gboolean   found;
239         gboolean   res;
240
241         found = FALSE;
242         len = (sizeof (buffer) - 1) / 2;
243
244         if (strlen (search->body) > len) {
245                 g_warning ("Search term bigger then chunk."
246                            "We might not find everything");     
247         }
248
249         stream = tny_mime_part_get_stream (part);
250
251         memset (buffer, 0, sizeof (buffer));
252         chunk[0] = buffer;
253         chunk[1] = buffer + len;
254
255         res = read_chunk (stream, chunk[0], len, &nread);
256
257         if (res == FALSE) {
258                 goto done;
259         }
260
261         found = !modest_text_utils_utf8_strcmp (search->body,
262                                                 buffer,
263                                                 TRUE);
264         if (found) {
265                 goto done;
266         }
267
268         /* This works like this:
269          * buffer: [ooooooooooo|xxxxxxxxxxxx|\0] 
270          *          ^chunk[0]  ^chunk[1]
271          * we have prefilled chunk[0] now we always read into chunk[1]
272          * and then move the content of chunk[1] to chunk[0].
273          * The idea is to prevent not finding search terms that are
274          * spread across 2 reads:        
275          * buffer: [ooooooooTES|Txxxxxxxxxxx|\0] 
276          * We should catch that because we always search the whole
277          * buffer not only the chunks.
278          *
279          * Of course that breaks for search terms > sizeof (chunk)
280          * but sizeof (chunk) should be big enough I guess (see
281          * the g_warning in this function)
282          * */   
283         while ((res = read_chunk (stream, chunk[1], len, &nread))) {
284                 buffer[len + nread] = '\0';
285
286                 found = !modest_text_utils_utf8_strcmp (search->body,
287                                                         buffer,
288                                                         TRUE);
289
290                 /* HACK: this helps UI refreshes because the search
291                    operations could be heavy */
292                 while (gtk_events_pending ())
293                         gtk_main_iteration ();
294
295                 if ((found)||(nread == 0)) {
296                         break;
297                 }
298
299                 /* also move the \0 */
300                 g_memmove (chunk[0], chunk[1], len + 1);
301         }
302
303 done:
304         g_object_unref (stream);
305         return found;
306 }
307 #endif /*MODEST_HAVE_OGS*/
308
309 static gboolean
310 search_string (const char      *what,
311                const char      *where,
312                ModestSearch    *search)
313 {
314         gboolean found = FALSE;
315 #ifdef MODEST_HAVE_OGS
316         if (search->flags & MODEST_SEARCH_USE_OGS) {
317                 found = ogs_text_searcher_search_text (search->text_searcher,
318                                                        where,
319                                                        strlen (where));
320
321                 ogs_text_searcher_reset (search->text_searcher);
322         } else {
323 #endif
324                 if (what == NULL || where == NULL) {
325                         return FALSE;
326                 }
327
328                 found = !modest_text_utils_utf8_strcmp (what, where, TRUE);
329 #ifdef MODEST_HAVE_OGS
330         }
331 #endif
332
333         /* HACK: this helps UI refreshes because the search
334            operations could be heavy */
335         while (gtk_events_pending ())
336                 gtk_main_iteration ();
337
338         return found;
339 }
340
341
342 static gboolean 
343 search_mime_part_and_child_parts (TnyMimePart *part, ModestSearch *search)
344 {
345         gboolean found = FALSE;
346
347         /* Do not search into attachments */
348         if (modest_tny_mime_part_is_attachment_for_modest (part))
349                 return FALSE;
350
351         #ifdef MODEST_HAVE_OGS
352         found = search_mime_part_ogs (part, search);
353         #else
354         found = search_mime_part_strcmp (part, search);
355         #endif
356
357         if (found) {    
358                 return found;           
359         }
360         
361         /* Check the child part too, recursively: */
362         TnyList *child_parts = tny_simple_list_new ();
363         tny_mime_part_get_parts (TNY_MIME_PART (part), child_parts);
364
365         TnyIterator *piter = tny_list_create_iterator (child_parts);
366         while (!found && !tny_iterator_is_done (piter)) {
367                 TnyMimePart *pcur = (TnyMimePart *) tny_iterator_get_current (piter);
368                 if (pcur) {
369                         found = search_mime_part_and_child_parts (pcur, search);
370
371                         g_object_unref (pcur);
372                 }
373
374                 tny_iterator_next (piter);
375         }
376
377         g_object_unref (piter);
378         g_object_unref (child_parts);
379         
380         return found;
381 }
382
383 static void 
384 modest_search_folder_get_headers_cb (TnyFolder *folder, 
385                                      gboolean cancelled, 
386                                      TnyList *headers, 
387                                      GError *err, 
388                                      gpointer user_data)
389 {
390         TnyIterator *iter = NULL;
391         SearchHelper *helper;
392
393         helper = (SearchHelper *) user_data;
394
395         if (err || cancelled) {
396                 goto end;
397         }
398
399         iter = tny_list_create_iterator (headers);
400
401         while (!tny_iterator_is_done (iter)) {
402
403                 TnyHeader *cur = (TnyHeader *) tny_iterator_get_current (iter);
404                 const time_t t = tny_header_get_date_sent (cur);
405                 gboolean found = FALSE;
406                 
407                 /* Ignore deleted (not yet expunged) emails: */
408                 if (tny_header_get_flags(cur) & TNY_HEADER_FLAG_DELETED)
409                         goto go_next;
410                         
411                 if (helper->search->flags & MODEST_SEARCH_BEFORE)
412                         if (!(t <= helper->search->end_date))
413                                 goto go_next;
414
415                 if (helper->search->flags & MODEST_SEARCH_AFTER)
416                         if (!(t >= helper->search->start_date))
417                                 goto go_next;
418
419                 if (helper->search->flags & MODEST_SEARCH_SIZE)
420                         if (tny_header_get_message_size (cur) < helper->search->minsize)
421                                 goto go_next;
422
423                 if (helper->search->flags & MODEST_SEARCH_SUBJECT) {
424                         char *str = tny_header_dup_subject (cur);
425
426                         if ((found = search_string (helper->search->subject, str, helper->search))) {
427                             helper->msg_hits = add_hit (helper->msg_hits, cur, folder);
428                         }
429                         g_free (str);
430                 }
431                 
432                 if (!found && helper->search->flags & MODEST_SEARCH_SENDER) {
433                         char *str = tny_header_dup_from (cur);
434
435                         if ((found = search_string (helper->search->from, (const gchar *) str, helper->search))) {
436                                 helper->msg_hits = add_hit (helper->msg_hits, cur, folder);
437                         }
438                         g_free (str);
439                 }
440                 
441                 if (!found && helper->search->flags & MODEST_SEARCH_RECIPIENT) {
442                         char *str = tny_header_dup_to (cur);
443
444                         if ((found = search_string (helper->search->recipient, str, helper->search))) {
445                                 helper->msg_hits = add_hit (helper->msg_hits, cur, folder);
446                         }
447                         g_free (str);
448                 }
449         
450                 if (!found && helper->search->flags & MODEST_SEARCH_BODY) {
451                         TnyHeaderFlags flags;
452                         GError      *err = NULL;
453                         TnyMsg      *msg = NULL;
454
455                         flags = tny_header_get_flags (cur);
456
457                         if (!(flags & TNY_HEADER_FLAG_CACHED)) {
458                                 goto go_next;
459                         }
460
461                         msg = tny_folder_get_msg (folder, cur, &err);
462
463                         if (err != NULL || msg == NULL) {
464                                 g_warning ("%s: Could not get message.\n", __FUNCTION__);
465                                 g_error_free (err);
466
467                                 if (msg) {
468                                         g_object_unref (msg);
469                                 }
470                         } else {        
471                                 gchar *str;
472                                 str = tny_header_dup_subject (cur);
473                                 g_debug ("Searching in %s\n", str);
474                                 g_free (str);
475                         
476                                 found = search_mime_part_and_child_parts (TNY_MIME_PART (msg),
477                                                                           helper->search);
478                                 if (found) {
479                                         helper->msg_hits = add_hit (helper->msg_hits, cur, folder);
480                                 }
481                         }
482                         
483                         if (msg)
484                                 g_object_unref (msg);
485                 }
486         go_next:
487                 g_object_unref (cur);
488                 tny_iterator_next (iter);
489         }
490
491         /* Frees */
492         g_object_unref (iter);
493  end:
494         if (headers)
495                 g_object_unref (headers);
496
497         /* Check search finished */
498         helper->folder_count++;
499         check_search_finished (helper);
500 }
501
502 static void
503 _search_folder (TnyFolder *folder, 
504                 SearchHelper *helper)
505 {
506         TnyList *list = NULL;
507
508         g_debug ("%s: searching folder %s.", __FUNCTION__, tny_folder_get_name (folder));
509         
510         /* Check that we should be searching this folder. */
511         /* Note that we don't try to search sub-folders. 
512          * Maybe we should, but that should be specified. */
513         if (helper->search->folder && strlen (helper->search->folder)) {
514                 if (!strcmp (helper->search->folder, "outbox")) {
515                         if (modest_tny_folder_guess_folder_type (folder) != TNY_FOLDER_TYPE_OUTBOX) {
516                                 modest_search_folder_get_headers_cb (folder, TRUE, NULL, NULL, helper); 
517                                 return;
518                         }
519                 } else if (strcmp (tny_folder_get_id (folder), helper->search->folder) != 0) {
520                         modest_search_folder_get_headers_cb (folder, TRUE, NULL, NULL, helper); 
521                         return;
522                 }
523         }
524         
525 #ifdef MODEST_HAVE_OGS
526         if (helper->search->flags & MODEST_SEARCH_USE_OGS) {
527         
528                 if (helper->search->text_searcher == NULL && helper->search->query != NULL) {
529                         OgsTextSearcher *text_searcher; 
530
531                         text_searcher = ogs_text_searcher_new (FALSE);
532                         ogs_text_searcher_parse_query (text_searcher, helper->search->query);
533                         helper->search->text_searcher = text_searcher;
534                 }
535         }
536 #endif
537         list = tny_simple_list_new ();
538         /* Get the headers */
539         tny_folder_get_headers_async (folder, list, FALSE, 
540                                       modest_search_folder_get_headers_cb, 
541                                       NULL, helper);
542 }
543
544 void
545 modest_search_folder (TnyFolder *folder, 
546                       ModestSearch *search,
547                       ModestSearchCallback callback,
548                       gpointer user_data)
549 {
550         SearchHelper *helper;
551
552         /* Create the helper */
553         helper = create_helper (callback, search, user_data);
554
555         /* Search */
556         _search_folder (folder, helper);
557 }
558
559 static void
560 modest_search_account_get_folders_cb (TnyFolderStore *self, 
561                                       gboolean cancelled, 
562                                       TnyList *folders, 
563                                       GError *err, 
564                                       gpointer user_data)
565 {
566         TnyIterator *iter;
567         SearchHelper *helper;
568
569         helper = (SearchHelper *) user_data;
570
571         if (err || cancelled) {
572                 goto end;
573         }
574
575         iter = tny_list_create_iterator (folders);
576         while (!tny_iterator_is_done (iter)) {
577                 TnyFolder *folder = NULL;
578
579                 /* Search into folder */
580                 folder = TNY_FOLDER (tny_iterator_get_current (iter));  
581                 helper->folder_total++;
582                 _search_folder (folder, (SearchHelper *) user_data);
583                 g_object_unref (folder);
584
585                 tny_iterator_next (iter);
586         }
587         g_object_unref (iter);
588  end:
589         if (folders)
590                 g_object_unref (folders);
591
592         /* Check search finished */
593         check_search_finished (helper);
594 }
595
596 static void
597 _search_account (TnyAccount *account, 
598                  SearchHelper *helper)
599 {
600         TnyList *folders = tny_simple_list_new ();
601
602         g_debug ("%s: Searching account %s", __FUNCTION__, tny_account_get_name (account));
603
604         /* Get folders */
605         tny_folder_store_get_folders_async (TNY_FOLDER_STORE (account), folders, NULL, 
606                                             modest_search_account_get_folders_cb, 
607                                             NULL, helper);
608 }
609
610 void
611 modest_search_account (TnyAccount *account, 
612                        ModestSearch *search,
613                        ModestSearchCallback callback,
614                        gpointer user_data)
615 {
616         SearchHelper *helper;
617
618         /* Create the helper */
619         helper = create_helper (callback, search, user_data);
620
621         /* Search */
622         _search_account (account, helper);
623 }
624
625 void
626 modest_search_all_accounts (ModestSearch *search,
627                             ModestSearchCallback callback,
628                             gpointer user_data)
629 {
630         ModestTnyAccountStore *astore;
631         TnyList *accounts;
632         TnyIterator *iter;
633         GList *hits;
634         SearchHelper *helper;
635
636         hits = NULL;
637         astore = modest_runtime_get_account_store ();
638
639         accounts = tny_simple_list_new ();
640         tny_account_store_get_accounts (TNY_ACCOUNT_STORE (astore),
641                                         accounts,
642                                         TNY_ACCOUNT_STORE_STORE_ACCOUNTS);
643
644         /* Create the helper */
645         helper = create_helper (callback, search, user_data);
646
647         /* Search through all accounts */
648         iter = tny_list_create_iterator (accounts);
649         while (!tny_iterator_is_done (iter)) {
650                 TnyAccount *account = NULL;
651
652                 account = TNY_ACCOUNT (tny_iterator_get_current (iter));
653                 _search_account (account, helper);
654                 g_object_unref (account);
655
656                 tny_iterator_next (iter);
657         }
658         g_object_unref (iter);
659         g_object_unref (accounts);
660 }
661
662 static SearchHelper *
663 create_helper (ModestSearchCallback callback, 
664                ModestSearch *search,
665                gpointer user_data)
666 {
667         SearchHelper *helper;
668
669         helper = g_slice_new0 (SearchHelper);
670         helper->folder_count = 0;
671         helper->folder_total = 0;
672         helper->search = search;
673         helper->callback = callback;
674         helper->user_data = user_data;
675         helper->msg_hits = NULL;
676
677         return helper;
678 }
679
680 void 
681 modest_search_free (ModestSearch *search)
682 {
683         if (search->folder)
684                 g_free (search->folder);
685         if (search->subject)
686                 g_free (search->subject);
687         if (search->from)
688                 g_free (search->from);
689         if (search->recipient)
690                 g_free (search->recipient);
691         if (search->body)
692                 g_free (search->body);
693
694 #ifdef MODEST_HAVE_OGS
695         if (search->query)
696                 g_free (search->query);
697         if (search->text_searcher)
698                 ogs_text_searcher_free (search->text_searcher); 
699 #endif
700 }
701
702 static void
703 check_search_finished (SearchHelper *helper)
704 {
705         /* If there are no more folders to check the account search has finished */
706         if (helper->folder_count == helper->folder_total) {
707                 /* callback */
708                 helper->callback (helper->msg_hits, helper->user_data);
709                 
710                 /* free helper */
711                 g_list_free (helper->msg_hits);
712                 g_slice_free (SearchHelper, helper);
713         }
714 }