Adding gmail csv support for the backend

[theonering] / src / gvoice / backend.py

diff --git a/src/gvoice/backend.py b/src/gvoice/backend.py
index 72eda38..1973676 100755 (executable)
--- a/src/gvoice/backend.py
+++ b/src/gvoice/backend.py
@@ -37,6 +37,7 @@ import itertools
 import logging
 import inspect
 
+from xml.sax import saxutils
 from xml.etree import ElementTree
 
 try:
@@ -86,7 +87,7 @@ class Message(object):
                return "%s (%s): %s" % (
                        self.whoFrom,
                        self.when,
-                       "".join(str(part) for part in self.body)
+                       "".join(unicode(part) for part in self.body)
                )
 
        def to_dict(self):
@@ -187,6 +188,7 @@ class GVoiceBackend(object):
                self._XML_ACCOUNT_URL = SECURE_URL_BASE + "contacts/"
                # HACK really this redirects to the main pge and we are grabbing some javascript
                self._XML_CONTACTS_URL = "http://www.google.com/voice/inbox/search/contact"
+               self._CSV_CONTACTS_URL = "http://mail.google.com/mail/contacts/data/export"
                self._XML_RECENT_URL = SECURE_URL_BASE + "inbox/recent/"
 
                self.XML_FEEDS = (
@@ -304,6 +306,14 @@ class GVoiceBackend(object):
                self._lastAuthed = time.time()
                return True
 
+       def persist(self):
+               self._browser.save_cookies()
+
+       def shutdown(self):
+               self._browser.save_cookies()
+               self._token = None
+               self._lastAuthed = 0.0
+
        def logout(self):
                self._browser.clear_cookies()
                self._browser.save_cookies()
@@ -387,7 +397,7 @@ class GVoiceBackend(object):
                        self._sendSmsURL,
                        {
                                'phoneNumber': flattenedPhoneNumbers,
-                               'text': message
+                               'text': unicode(message).encode("utf-8"),
                        },
                )
                self._parse_with_validation(page)
@@ -491,6 +501,15 @@ class GVoiceBackend(object):
                page = self._get_page(self._XML_CONTACTS_URL)
                return self._process_contacts(page)
 
+       def get_csv_contacts(self):
+               data = {
+                       "groupToExport": "mine",
+                       "exportType": "ALL",
+                       "out": "OUTLOOK_CSV",
+               }
+               contacts = self._get_page(self._CSV_CONTACTS_URL, data)
+               return contacts
+
        def get_voicemails(self):
                """
                @blocks
@@ -498,6 +517,8 @@ class GVoiceBackend(object):
                voicemailPage = self._get_page(self._XML_VOICEMAIL_URL)
                voicemailHtml = self._grab_html(voicemailPage)
                voicemailJson = self._grab_json(voicemailPage)
+               if voicemailJson is None:
+                       return ()
                parsedVoicemail = self._parse_voicemail(voicemailHtml)
                voicemails = self._merge_conversation_sources(parsedVoicemail, voicemailJson)
                return voicemails
@@ -509,6 +530,8 @@ class GVoiceBackend(object):
                smsPage = self._get_page(self._XML_SMS_URL)
                smsHtml = self._grab_html(smsPage)
                smsJson = self._grab_json(smsPage)
+               if smsJson is None:
+                       return ()
                parsedSms = self._parse_sms(smsHtml)
                smss = self._merge_conversation_sources(parsedSms, smsJson)
                return smss
@@ -615,12 +638,12 @@ class GVoiceBackend(object):
                        yield {
                                "id": messageId.strip(),
                                "contactId": contactId,
-                               "name": name,
+                               "name": unescape(name),
                                "time": exactTime,
                                "relTime": relativeTime,
                                "prettyNumber": prettyNumber,
                                "number": number,
-                               "location": location,
+                               "location": unescape(location),
                        }
 
        @staticmethod
@@ -649,10 +672,10 @@ class GVoiceBackend(object):
                        relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
                        conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
                        locationGroup = self._voicemailLocationRegex.search(messageHtml)
-                       conv.location = locationGroup.group(1).strip() if locationGroup else ""
+                       conv.location = unescape(locationGroup.group(1).strip() if locationGroup else "")
 
                        nameGroup = self._voicemailNameRegex.search(messageHtml)
-                       conv.name = nameGroup.group(1).strip() if nameGroup else ""
+                       conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
                        numberGroup = self._voicemailNumberRegex.search(messageHtml)
                        conv.number = numberGroup.group(1).strip() if numberGroup else ""
                        prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
@@ -701,7 +724,7 @@ class GVoiceBackend(object):
                        conv.location = ""
 
                        nameGroup = self._voicemailNameRegex.search(messageHtml)
-                       conv.name = nameGroup.group(1).strip() if nameGroup else ""
+                       conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
                        numberGroup = self._voicemailNumberRegex.search(messageHtml)
                        conv.number = numberGroup.group(1).strip() if numberGroup else ""
                        prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
@@ -762,6 +785,18 @@ class GVoiceBackend(object):
                return json
 
 
+_UNESCAPE_ENTITIES = {
+ """: '"',
+ " ": " ",
+ "'": "'",
+}
+
+
+def unescape(text):
+       plain = saxutils.unescape(text, _UNESCAPE_ENTITIES)
+       return plain
+
+
 def google_strptime(time):
        """
        Hack: Google always returns the time in the same locale.  Sadly if the
@@ -770,7 +805,7 @@ def google_strptime(time):
        """
        abbrevTime = time[:-3]
        parsedTime = datetime.datetime.strptime(abbrevTime, "%m/%d/%y %I:%M")
-       if time[-2] == "PN":
+       if time.endswith("PM"):
                parsedTime += datetime.timedelta(hours=12)
        return parsedTime
 
@@ -811,9 +846,16 @@ def itergroup(iterator, count, padValue = None):
 def safe_eval(s):
        _TRUE_REGEX = re.compile("true")
        _FALSE_REGEX = re.compile("false")
+       _COMMENT_REGEX = re.compile("^\s+//.*$", re.M)
        s = _TRUE_REGEX.sub("True", s)
        s = _FALSE_REGEX.sub("False", s)
-       return eval(s, {}, {})
+       s = _COMMENT_REGEX.sub("#", s)
+       try:
+               results = eval(s, {}, {})
+       except SyntaxError:
+               _moduleLogger.exception("Oops")
+               results = None
+       return results
 
 
 def _fake_parse_json(flattened):
@@ -848,7 +890,9 @@ def validate_response(response):
        Validates that the JSON response is A-OK
        """
        try:
-               assert 'ok' in response and response['ok']
+               assert response is not None
+               assert 'ok' in response
+               assert response['ok']
        except AssertionError:
                raise RuntimeError('There was a problem with GV: %s' % response)
 
@@ -934,6 +978,7 @@ def grab_debug_info(username, password):
                ("isdnd", backend._isDndURL),
                ("account", backend._XML_ACCOUNT_URL),
                ("contacts", backend._XML_CONTACTS_URL),
+               ("csv", backend._CSV_CONTACTS_URL),
 
                ("voicemail", backend._XML_VOICEMAIL_URL),
                ("sms", backend._XML_SMS_URL),