Adding fancier version checks to allow to the cache to be preserved longer
[theonering] / src / gvoice / backend.py
1 #!/usr/bin/python
2
3 """
4 DialCentral - Front end for Google's GoogleVoice service.
5 Copyright (C) 2008  Eric Warnke ericew AT gmail DOT com
6
7 This library is free software; you can redistribute it and/or
8 modify it under the terms of the GNU Lesser General Public
9 License as published by the Free Software Foundation; either
10 version 2.1 of the License, or (at your option) any later version.
11
12 This library is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15 Lesser General Public License for more details.
16
17 You should have received a copy of the GNU Lesser General Public
18 License along with this library; if not, write to the Free Software
19 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
20
21 Google Voice backend code
22
23 Resources
24         http://thatsmith.com/2009/03/google-voice-addon-for-firefox/
25         http://posttopic.com/topic/google-voice-add-on-development
26 """
27
28 from __future__ import with_statement
29
30 import os
31 import re
32 import urllib
33 import urllib2
34 import time
35 import datetime
36 import itertools
37 import logging
38 import inspect
39
40 from xml.etree import ElementTree
41
42 try:
43         import simplejson as _simplejson
44         simplejson = _simplejson
45 except ImportError:
46         simplejson = None
47
48 import browser_emu
49
50
51 _moduleLogger = logging.getLogger(__name__)
52
53
54 class NetworkError(RuntimeError):
55         pass
56
57
58 class MessageText(object):
59
60         ACCURACY_LOW = "med1"
61         ACCURACY_MEDIUM = "med2"
62         ACCURACY_HIGH = "high"
63
64         def __init__(self):
65                 self.accuracy = None
66                 self.text = None
67
68         def __str__(self):
69                 return self.text
70
71         def to_dict(self):
72                 return to_dict(self)
73
74         def __eq__(self, other):
75                 return self.accuracy == other.accuracy and self.text == other.text
76
77
78 class Message(object):
79
80         def __init__(self):
81                 self.whoFrom = None
82                 self.body = None
83                 self.when = None
84
85         def __str__(self):
86                 return "%s (%s): %s" % (
87                         self.whoFrom,
88                         self.when,
89                         "".join(str(part) for part in self.body)
90                 )
91
92         def to_dict(self):
93                 selfDict = to_dict(self)
94                 selfDict["body"] = [text.to_dict() for text in self.body] if self.body is not None else None
95                 return selfDict
96
97         def __eq__(self, other):
98                 return self.whoFrom == other.whoFrom and self.when == other.when and self.body == other.body
99
100
101 class Conversation(object):
102
103         TYPE_VOICEMAIL = "Voicemail"
104         TYPE_SMS = "SMS"
105
106         def __init__(self):
107                 self.type = None
108                 self.id = None
109                 self.contactId = None
110                 self.name = None
111                 self.location = None
112                 self.prettyNumber = None
113                 self.number = None
114
115                 self.time = None
116                 self.relTime = None
117                 self.messages = None
118                 self.isRead = None
119                 self.isSpam = None
120                 self.isTrash = None
121                 self.isArchived = None
122
123         def __cmp__(self, other):
124                 cmpValue = cmp(self.contactId, other.contactId)
125                 if cmpValue != 0:
126                         return cmpValue
127
128                 cmpValue = cmp(self.time, other.time)
129                 if cmpValue != 0:
130                         return cmpValue
131
132                 cmpValue = cmp(self.id, other.id)
133                 if cmpValue != 0:
134                         return cmpValue
135
136         def to_dict(self):
137                 selfDict = to_dict(self)
138                 selfDict["messages"] = [message.to_dict() for message in self.messages] if self.messages is not None else None
139                 return selfDict
140
141
142 class GVoiceBackend(object):
143         """
144         This class encapsulates all of the knowledge necessary to interact with the GoogleVoice servers
145         the functions include login, setting up a callback number, and initalting a callback
146         """
147
148         PHONE_TYPE_HOME = 1
149         PHONE_TYPE_MOBILE = 2
150         PHONE_TYPE_WORK = 3
151         PHONE_TYPE_GIZMO = 7
152
153         def __init__(self, cookieFile = None):
154                 # Important items in this function are the setup of the browser emulation and cookie file
155                 self._browser = browser_emu.MozillaEmulator(1)
156                 self._loadedFromCookies = self._browser.load_cookies(cookieFile)
157
158                 self._token = ""
159                 self._accountNum = ""
160                 self._lastAuthed = 0.0
161                 self._callbackNumber = ""
162                 self._callbackNumbers = {}
163
164                 # Suprisingly, moving all of these from class to self sped up startup time
165
166                 self._validateRe = re.compile("^\+?[0-9]{10,}$")
167
168                 self._loginURL = "https://www.google.com/accounts/ServiceLoginAuth"
169
170                 SECURE_URL_BASE = "https://www.google.com/voice/"
171                 SECURE_MOBILE_URL_BASE = SECURE_URL_BASE + "mobile/"
172                 self._forwardURL = SECURE_MOBILE_URL_BASE + "phones"
173                 self._tokenURL = SECURE_URL_BASE + "m"
174                 self._callUrl = SECURE_URL_BASE + "call/connect"
175                 self._callCancelURL = SECURE_URL_BASE + "call/cancel"
176                 self._sendSmsURL = SECURE_URL_BASE + "sms/send"
177
178                 self._isDndURL = "https://www.google.com/voice/m/donotdisturb"
179                 self._isDndRe = re.compile(r"""<input.*?id="doNotDisturb".*?checked="(.*?)"\s*/>""")
180                 self._setDndURL = "https://www.google.com/voice/m/savednd"
181
182                 self._downloadVoicemailURL = SECURE_URL_BASE + "media/send_voicemail/"
183                 self._markAsReadURL = SECURE_URL_BASE + "m/mark"
184                 self._archiveMessageURL = SECURE_URL_BASE + "m/archive"
185
186                 self._XML_SEARCH_URL = SECURE_URL_BASE + "inbox/search/"
187                 self._XML_ACCOUNT_URL = SECURE_URL_BASE + "contacts/"
188                 # HACK really this redirects to the main pge and we are grabbing some javascript
189                 self._XML_CONTACTS_URL = "http://www.google.com/voice/inbox/search/contact"
190                 self._XML_RECENT_URL = SECURE_URL_BASE + "inbox/recent/"
191
192                 self.XML_FEEDS = (
193                         'inbox', 'starred', 'all', 'spam', 'trash', 'voicemail', 'sms',
194                         'recorded', 'placed', 'received', 'missed'
195                 )
196                 self._XML_INBOX_URL = SECURE_URL_BASE + "inbox/recent/inbox"
197                 self._XML_STARRED_URL = SECURE_URL_BASE + "inbox/recent/starred"
198                 self._XML_ALL_URL = SECURE_URL_BASE + "inbox/recent/all"
199                 self._XML_SPAM_URL = SECURE_URL_BASE + "inbox/recent/spam"
200                 self._XML_TRASH_URL = SECURE_URL_BASE + "inbox/recent/trash"
201                 self._XML_VOICEMAIL_URL = SECURE_URL_BASE + "inbox/recent/voicemail/"
202                 self._XML_SMS_URL = SECURE_URL_BASE + "inbox/recent/sms/"
203                 self._XML_RECORDED_URL = SECURE_URL_BASE + "inbox/recent/recorded/"
204                 self._XML_PLACED_URL = SECURE_URL_BASE + "inbox/recent/placed/"
205                 self._XML_RECEIVED_URL = SECURE_URL_BASE + "inbox/recent/received/"
206                 self._XML_MISSED_URL = SECURE_URL_BASE + "inbox/recent/missed/"
207
208                 self._galxRe = re.compile(r"""<input.*?name="GALX".*?value="(.*?)".*?/>""", re.MULTILINE | re.DOTALL)
209                 self._tokenRe = re.compile(r"""<input.*?name="_rnr_se".*?value="(.*?)"\s*/>""")
210                 self._accountNumRe = re.compile(r"""<b class="ms\d">(.{14})</b></div>""")
211                 self._callbackRe = re.compile(r"""\s+(.*?):\s*(.*?)<br\s*/>\s*$""", re.M)
212
213                 self._contactsBodyRe = re.compile(r"""gcData\s*=\s*({.*?});""", re.MULTILINE | re.DOTALL)
214                 self._seperateVoicemailsRegex = re.compile(r"""^\s*<div id="(\w+)"\s* class=".*?gc-message.*?">""", re.MULTILINE | re.DOTALL)
215                 self._exactVoicemailTimeRegex = re.compile(r"""<span class="gc-message-time">(.*?)</span>""", re.MULTILINE)
216                 self._relativeVoicemailTimeRegex = re.compile(r"""<span class="gc-message-relative">(.*?)</span>""", re.MULTILINE)
217                 self._voicemailNameRegex = re.compile(r"""<a class=.*?gc-message-name-link.*?>(.*?)</a>""", re.MULTILINE | re.DOTALL)
218                 self._voicemailNumberRegex = re.compile(r"""<input type="hidden" class="gc-text gc-quickcall-ac" value="(.*?)"/>""", re.MULTILINE)
219                 self._prettyVoicemailNumberRegex = re.compile(r"""<span class="gc-message-type">(.*?)</span>""", re.MULTILINE)
220                 self._voicemailLocationRegex = re.compile(r"""<span class="gc-message-location">.*?<a.*?>(.*?)</a></span>""", re.MULTILINE)
221                 self._messagesContactIDRegex = re.compile(r"""<a class=".*?gc-message-name-link.*?">.*?</a>\s*?<span .*?>(.*?)</span>""", re.MULTILINE)
222                 self._voicemailMessageRegex = re.compile(r"""(<span id="\d+-\d+" class="gc-word-(.*?)">(.*?)</span>|<a .*? class="gc-message-mni">(.*?)</a>)""", re.MULTILINE)
223                 self._smsFromRegex = re.compile(r"""<span class="gc-message-sms-from">(.*?)</span>""", re.MULTILINE | re.DOTALL)
224                 self._smsTimeRegex = re.compile(r"""<span class="gc-message-sms-time">(.*?)</span>""", re.MULTILINE | re.DOTALL)
225                 self._smsTextRegex = re.compile(r"""<span class="gc-message-sms-text">(.*?)</span>""", re.MULTILINE | re.DOTALL)
226
227         def is_quick_login_possible(self):
228                 """
229                 @returns True then is_authed might be enough to login, else full login is required
230                 """
231                 return self._loadedFromCookies or 0.0 < self._lastAuthed
232
233         def is_authed(self, force = False):
234                 """
235                 Attempts to detect a current session
236                 @note Once logged in try not to reauth more than once a minute.
237                 @returns If authenticated
238                 """
239                 isRecentledAuthed = (time.time() - self._lastAuthed) < 120
240                 isPreviouslyAuthed = self._token is not None
241                 if isRecentledAuthed and isPreviouslyAuthed and not force:
242                         return True
243
244                 try:
245                         page = self._get_page(self._forwardURL)
246                         self._grab_account_info(page)
247                 except Exception, e:
248                         _moduleLogger.exception(str(e))
249                         return False
250
251                 self._browser.save_cookies()
252                 self._lastAuthed = time.time()
253                 return True
254
255         def _get_token(self):
256                 tokenPage = self._get_page(self._tokenURL)
257
258                 galxTokens = self._galxRe.search(tokenPage)
259                 if galxTokens is not None:
260                         galxToken = galxTokens.group(1)
261                 else:
262                         galxToken = ""
263                         _moduleLogger.debug("Could not grab GALX token")
264                 return galxToken
265
266         def _login(self, username, password, token):
267                 loginData = {
268                         'Email' : username,
269                         'Passwd' : password,
270                         'service': "grandcentral",
271                         "ltmpl": "mobile",
272                         "btmpl": "mobile",
273                         "PersistentCookie": "yes",
274                         "GALX": token,
275                         "continue": self._forwardURL,
276                 }
277
278                 loginSuccessOrFailurePage = self._get_page(self._loginURL, loginData)
279                 return loginSuccessOrFailurePage
280
281         def login(self, username, password):
282                 """
283                 Attempt to login to GoogleVoice
284                 @returns Whether login was successful or not
285                 """
286                 self.logout()
287                 galxToken = self._get_token()
288                 loginSuccessOrFailurePage = self._login(username, password, galxToken)
289
290                 try:
291                         self._grab_account_info(loginSuccessOrFailurePage)
292                 except Exception, e:
293                         # Retry in case the redirect failed
294                         # luckily is_authed does everything we need for a retry
295                         loggedIn = self.is_authed(True)
296                         if not loggedIn:
297                                 _moduleLogger.exception(str(e))
298                                 return False
299                         _moduleLogger.info("Redirection failed on initial login attempt, auto-corrected for this")
300
301                 self._browser.save_cookies()
302                 self._lastAuthed = time.time()
303                 return True
304
305         def logout(self):
306                 self._browser.clear_cookies()
307                 self._browser.save_cookies()
308                 self._token = None
309                 self._lastAuthed = 0.0
310
311         def is_dnd(self):
312                 isDndPage = self._get_page(self._isDndURL)
313
314                 dndGroup = self._isDndRe.search(isDndPage)
315                 if dndGroup is None:
316                         return False
317                 dndStatus = dndGroup.group(1)
318                 isDnd = True if dndStatus.strip().lower() == "true" else False
319                 return isDnd
320
321         def set_dnd(self, doNotDisturb):
322                 dndPostData = {
323                         "doNotDisturb": 1 if doNotDisturb else 0,
324                 }
325
326                 dndPage = self._get_page_with_token(self._setDndURL, dndPostData)
327
328         def call(self, outgoingNumber):
329                 """
330                 This is the main function responsible for initating the callback
331                 """
332                 outgoingNumber = self._send_validation(outgoingNumber)
333                 subscriberNumber = None
334                 phoneType = guess_phone_type(self._callbackNumber) # @todo Fix this hack
335
336                 callData = {
337                                 'outgoingNumber': outgoingNumber,
338                                 'forwardingNumber': self._callbackNumber,
339                                 'subscriberNumber': subscriberNumber or 'undefined',
340                                 'phoneType': str(phoneType),
341                                 'remember': '1',
342                 }
343                 _moduleLogger.info("%r" % callData)
344
345                 page = self._get_page_with_token(
346                         self._callUrl,
347                         callData,
348                 )
349                 self._parse_with_validation(page)
350                 return True
351
352         def cancel(self, outgoingNumber=None):
353                 """
354                 Cancels a call matching outgoing and forwarding numbers (if given). 
355                 Will raise an error if no matching call is being placed
356                 """
357                 page = self._get_page_with_token(
358                         self._callCancelURL,
359                         {
360                         'outgoingNumber': outgoingNumber or 'undefined',
361                         'forwardingNumber': self._callbackNumber or 'undefined',
362                         'cancelType': 'C2C',
363                         },
364                 )
365                 self._parse_with_validation(page)
366
367         def send_sms(self, phoneNumbers, message):
368                 validatedPhoneNumbers = [
369                         self._send_validation(phoneNumber)
370                         for phoneNumber in phoneNumbers
371                 ]
372                 flattenedPhoneNumbers = ",".join(validatedPhoneNumbers)
373                 page = self._get_page_with_token(
374                         self._sendSmsURL,
375                         {
376                                 'phoneNumber': flattenedPhoneNumbers,
377                                 'text': message
378                         },
379                 )
380                 self._parse_with_validation(page)
381
382         def search(self, query):
383                 """
384                 Search your Google Voice Account history for calls, voicemails, and sms
385                 Returns ``Folder`` instance containting matching messages
386                 """
387                 page = self._get_page(
388                         self._XML_SEARCH_URL,
389                         {"q": query},
390                 )
391                 json, html = extract_payload(page)
392                 return json
393
394         def get_feed(self, feed):
395                 actualFeed = "_XML_%s_URL" % feed.upper()
396                 feedUrl = getattr(self, actualFeed)
397
398                 page = self._get_page(feedUrl)
399                 json, html = extract_payload(page)
400
401                 return json
402
403         def download(self, messageId, adir):
404                 """
405                 Download a voicemail or recorded call MP3 matching the given ``msg``
406                 which can either be a ``Message`` instance, or a SHA1 identifier. 
407                 Saves files to ``adir`` (defaults to current directory). 
408                 Message hashes can be found in ``self.voicemail().messages`` for example. 
409                 Returns location of saved file.
410                 """
411                 page = self._get_page(self._downloadVoicemailURL, {"id": messageId})
412                 fn = os.path.join(adir, '%s.mp3' % messageId)
413                 with open(fn, 'wb') as fo:
414                         fo.write(page)
415                 return fn
416
417         def is_valid_syntax(self, number):
418                 """
419                 @returns If This number be called ( syntax validation only )
420                 """
421                 return self._validateRe.match(number) is not None
422
423         def get_account_number(self):
424                 """
425                 @returns The GoogleVoice phone number
426                 """
427                 return self._accountNum
428
429         def get_callback_numbers(self):
430                 """
431                 @returns a dictionary mapping call back numbers to descriptions
432                 @note These results are cached for 30 minutes.
433                 """
434                 if not self.is_authed():
435                         return {}
436                 return self._callbackNumbers
437
438         def set_callback_number(self, callbacknumber):
439                 """
440                 Set the number that GoogleVoice calls
441                 @param callbacknumber should be a proper 10 digit number
442                 """
443                 self._callbackNumber = callbacknumber
444                 _moduleLogger.info("Callback number changed: %r" % self._callbackNumber)
445                 return True
446
447         def get_callback_number(self):
448                 """
449                 @returns Current callback number or None
450                 """
451                 return self._callbackNumber
452
453         def get_recent(self):
454                 """
455                 @returns Iterable of (personsName, phoneNumber, exact date, relative date, action)
456                 """
457                 for action, url in (
458                         ("Received", self._XML_RECEIVED_URL),
459                         ("Missed", self._XML_MISSED_URL),
460                         ("Placed", self._XML_PLACED_URL),
461                 ):
462                         flatXml = self._get_page(url)
463
464                         allRecentHtml = self._grab_html(flatXml)
465                         allRecentData = self._parse_history(allRecentHtml)
466                         for recentCallData in allRecentData:
467                                 recentCallData["action"] = action
468                                 yield recentCallData
469
470         def get_contacts(self):
471                 """
472                 @returns Iterable of (contact id, contact name)
473                 """
474                 page = self._get_page(self._XML_CONTACTS_URL)
475                 contactsBody = self._contactsBodyRe.search(page)
476                 if contactsBody is None:
477                         raise RuntimeError("Could not extract contact information")
478                 accountData = _fake_parse_json(contactsBody.group(1))
479                 for contactId, contactDetails in accountData["contacts"].iteritems():
480                         # A zero contact id is the catch all for unknown contacts
481                         if contactId != "0":
482                                 yield contactId, contactDetails
483
484         def get_voicemails(self):
485                 voicemailPage = self._get_page(self._XML_VOICEMAIL_URL)
486                 voicemailHtml = self._grab_html(voicemailPage)
487                 voicemailJson = self._grab_json(voicemailPage)
488                 parsedVoicemail = self._parse_voicemail(voicemailHtml)
489                 voicemails = self._merge_conversation_sources(parsedVoicemail, voicemailJson)
490                 return voicemails
491
492         def get_texts(self):
493                 smsPage = self._get_page(self._XML_SMS_URL)
494                 smsHtml = self._grab_html(smsPage)
495                 smsJson = self._grab_json(smsPage)
496                 parsedSms = self._parse_sms(smsHtml)
497                 smss = self._merge_conversation_sources(parsedSms, smsJson)
498                 return smss
499
500         def mark_message(self, messageId, asRead):
501                 postData = {
502                         "read": 1 if asRead else 0,
503                         "id": messageId,
504                 }
505
506                 markPage = self._get_page(self._markAsReadURL, postData)
507
508         def archive_message(self, messageId):
509                 postData = {
510                         "id": messageId,
511                 }
512
513                 markPage = self._get_page(self._archiveMessageURL, postData)
514
515         def _grab_json(self, flatXml):
516                 xmlTree = ElementTree.fromstring(flatXml)
517                 jsonElement = xmlTree.getchildren()[0]
518                 flatJson = jsonElement.text
519                 jsonTree = parse_json(flatJson)
520                 return jsonTree
521
522         def _grab_html(self, flatXml):
523                 xmlTree = ElementTree.fromstring(flatXml)
524                 htmlElement = xmlTree.getchildren()[1]
525                 flatHtml = htmlElement.text
526                 return flatHtml
527
528         def _grab_account_info(self, page):
529                 tokenGroup = self._tokenRe.search(page)
530                 if tokenGroup is None:
531                         raise RuntimeError("Could not extract authentication token from GoogleVoice")
532                 self._token = tokenGroup.group(1)
533
534                 anGroup = self._accountNumRe.search(page)
535                 if anGroup is not None:
536                         self._accountNum = anGroup.group(1)
537                 else:
538                         _moduleLogger.debug("Could not extract account number from GoogleVoice")
539
540                 self._callbackNumbers = {}
541                 for match in self._callbackRe.finditer(page):
542                         callbackNumber = match.group(2)
543                         callbackName = match.group(1)
544                         self._callbackNumbers[callbackNumber] = callbackName
545                 if len(self._callbackNumbers) == 0:
546                         _moduleLogger.debug("Could not extract callback numbers from GoogleVoice (the troublesome page follows):\n%s" % page)
547
548         def _send_validation(self, number):
549                 if not self.is_valid_syntax(number):
550                         raise ValueError('Number is not valid: "%s"' % number)
551                 elif not self.is_authed():
552                         raise RuntimeError("Not Authenticated")
553                 return number
554
555         def _parse_history(self, historyHtml):
556                 splitVoicemail = self._seperateVoicemailsRegex.split(historyHtml)
557                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
558                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
559                         exactTime = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
560                         exactTime = google_strptime(exactTime)
561                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
562                         relativeTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
563                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
564                         location = locationGroup.group(1).strip() if locationGroup else ""
565
566                         nameGroup = self._voicemailNameRegex.search(messageHtml)
567                         name = nameGroup.group(1).strip() if nameGroup else ""
568                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
569                         number = numberGroup.group(1).strip() if numberGroup else ""
570                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
571                         prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
572                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
573                         contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
574
575                         yield {
576                                 "id": messageId.strip(),
577                                 "contactId": contactId,
578                                 "name": name,
579                                 "time": exactTime,
580                                 "relTime": relativeTime,
581                                 "prettyNumber": prettyNumber,
582                                 "number": number,
583                                 "location": location,
584                         }
585
586         @staticmethod
587         def _interpret_voicemail_regex(group):
588                 quality, content, number = group.group(2), group.group(3), group.group(4)
589                 text = MessageText()
590                 if quality is not None and content is not None:
591                         text.accuracy = quality
592                         text.text = content
593                         return text
594                 elif number is not None:
595                         text.accuracy = MessageText.ACCURACY_HIGH
596                         text.text = number
597                         return text
598
599         def _parse_voicemail(self, voicemailHtml):
600                 splitVoicemail = self._seperateVoicemailsRegex.split(voicemailHtml)
601                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
602                         conv = Conversation()
603                         conv.type = Conversation.TYPE_VOICEMAIL
604                         conv.id = messageId.strip()
605
606                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
607                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
608                         conv.time = google_strptime(exactTimeText)
609                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
610                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
611                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
612                         conv.location = locationGroup.group(1).strip() if locationGroup else ""
613
614                         nameGroup = self._voicemailNameRegex.search(messageHtml)
615                         conv.name = nameGroup.group(1).strip() if nameGroup else ""
616                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
617                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
618                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
619                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
620                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
621                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
622
623                         messageGroups = self._voicemailMessageRegex.finditer(messageHtml)
624                         messageParts = [
625                                 self._interpret_voicemail_regex(group)
626                                 for group in messageGroups
627                         ] if messageGroups else ((MessageText.ACCURACY_LOW, "No Transcription"), )
628                         message = Message()
629                         message.body = messageParts
630                         message.whoFrom = conv.name
631                         message.when = conv.time.strftime("%I:%M %p")
632                         conv.messages = (message, )
633
634                         yield conv
635
636         @staticmethod
637         def _interpret_sms_message_parts(fromPart, textPart, timePart):
638                 text = MessageText()
639                 text.accuracy = MessageText.ACCURACY_MEDIUM
640                 text.text = textPart
641
642                 message = Message()
643                 message.body = (text, )
644                 message.whoFrom = fromPart
645                 message.when = timePart
646
647                 return message
648
649         def _parse_sms(self, smsHtml):
650                 splitSms = self._seperateVoicemailsRegex.split(smsHtml)
651                 for messageId, messageHtml in itergroup(splitSms[1:], 2):
652                         conv = Conversation()
653                         conv.type = Conversation.TYPE_SMS
654                         conv.id = messageId.strip()
655
656                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
657                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
658                         conv.time = google_strptime(exactTimeText)
659                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
660                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
661                         conv.location = ""
662
663                         nameGroup = self._voicemailNameRegex.search(messageHtml)
664                         conv.name = nameGroup.group(1).strip() if nameGroup else ""
665                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
666                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
667                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
668                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
669                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
670                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
671
672                         fromGroups = self._smsFromRegex.finditer(messageHtml)
673                         fromParts = (group.group(1).strip() for group in fromGroups)
674                         textGroups = self._smsTextRegex.finditer(messageHtml)
675                         textParts = (group.group(1).strip() for group in textGroups)
676                         timeGroups = self._smsTimeRegex.finditer(messageHtml)
677                         timeParts = (group.group(1).strip() for group in timeGroups)
678
679                         messageParts = itertools.izip(fromParts, textParts, timeParts)
680                         messages = [self._interpret_sms_message_parts(*parts) for parts in messageParts]
681                         conv.messages = messages
682
683                         yield conv
684
685         @staticmethod
686         def _merge_conversation_sources(parsedMessages, json):
687                 for message in parsedMessages:
688                         jsonItem = json["messages"][message.id]
689                         message.isRead = jsonItem["isRead"]
690                         message.isSpam = jsonItem["isSpam"]
691                         message.isTrash = jsonItem["isTrash"]
692                         message.isArchived = "inbox" not in jsonItem["labels"]
693                         yield message
694
695         def _get_page(self, url, data = None, refererUrl = None):
696                 headers = {}
697                 if refererUrl is not None:
698                         headers["Referer"] = refererUrl
699
700                 encodedData = urllib.urlencode(data) if data is not None else None
701
702                 try:
703                         page = self._browser.download(url, encodedData, None, headers)
704                 except urllib2.URLError, e:
705                         _moduleLogger.error("Translating error: %s" % str(e))
706                         raise NetworkError("%s is not accesible" % url)
707
708                 return page
709
710         def _get_page_with_token(self, url, data = None, refererUrl = None):
711                 if data is None:
712                         data = {}
713                 data['_rnr_se'] = self._token
714
715                 page = self._get_page(url, data, refererUrl)
716
717                 return page
718
719         def _parse_with_validation(self, page):
720                 json = parse_json(page)
721                 validate_response(json)
722                 return json
723
724
725 def google_strptime(time):
726         """
727         Hack: Google always returns the time in the same locale.  Sadly if the
728         local system's locale is different, there isn't a way to perfectly handle
729         the time.  So instead we handle implement some time formatting
730         """
731         abbrevTime = time[:-3]
732         parsedTime = datetime.datetime.strptime(abbrevTime, "%m/%d/%y %I:%M")
733         if time[-2] == "PN":
734                 parsedTime += datetime.timedelta(hours=12)
735         return parsedTime
736
737
738 def itergroup(iterator, count, padValue = None):
739         """
740         Iterate in groups of 'count' values. If there
741         aren't enough values, the last result is padded with
742         None.
743
744         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
745         ...     print tuple(val)
746         (1, 2, 3)
747         (4, 5, 6)
748         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
749         ...     print list(val)
750         [1, 2, 3]
751         [4, 5, 6]
752         >>> for val in itergroup([1, 2, 3, 4, 5, 6, 7], 3):
753         ...     print tuple(val)
754         (1, 2, 3)
755         (4, 5, 6)
756         (7, None, None)
757         >>> for val in itergroup("123456", 3):
758         ...     print tuple(val)
759         ('1', '2', '3')
760         ('4', '5', '6')
761         >>> for val in itergroup("123456", 3):
762         ...     print repr("".join(val))
763         '123'
764         '456'
765         """
766         paddedIterator = itertools.chain(iterator, itertools.repeat(padValue, count-1))
767         nIterators = (paddedIterator, ) * count
768         return itertools.izip(*nIterators)
769
770
771 def safe_eval(s):
772         _TRUE_REGEX = re.compile("true")
773         _FALSE_REGEX = re.compile("false")
774         s = _TRUE_REGEX.sub("True", s)
775         s = _FALSE_REGEX.sub("False", s)
776         return eval(s, {}, {})
777
778
779 def _fake_parse_json(flattened):
780         return safe_eval(flattened)
781
782
783 def _actual_parse_json(flattened):
784         return simplejson.loads(flattened)
785
786
787 if simplejson is None:
788         parse_json = _fake_parse_json
789 else:
790         parse_json = _actual_parse_json
791
792
793 def extract_payload(flatXml):
794         xmlTree = ElementTree.fromstring(flatXml)
795
796         jsonElement = xmlTree.getchildren()[0]
797         flatJson = jsonElement.text
798         jsonTree = parse_json(flatJson)
799
800         htmlElement = xmlTree.getchildren()[1]
801         flatHtml = htmlElement.text
802
803         return jsonTree, flatHtml
804
805
806 def validate_response(response):
807         """
808         Validates that the JSON response is A-OK
809         """
810         try:
811                 assert 'ok' in response and response['ok']
812         except AssertionError:
813                 raise RuntimeError('There was a problem with GV: %s' % response)
814
815
816 def guess_phone_type(number):
817         if number.startswith("747") or number.startswith("1747") or number.startswith("+1747"):
818                 return GVoiceBackend.PHONE_TYPE_GIZMO
819         else:
820                 return GVoiceBackend.PHONE_TYPE_MOBILE
821
822
823 def get_sane_callback(backend):
824         """
825         Try to set a sane default callback number on these preferences
826         1) 1747 numbers ( Gizmo )
827         2) anything with gizmo in the name
828         3) anything with computer in the name
829         4) the first value
830         """
831         numbers = backend.get_callback_numbers()
832
833         priorityOrderedCriteria = [
834                 ("\+1747", None),
835                 ("1747", None),
836                 ("747", None),
837                 (None, "gizmo"),
838                 (None, "computer"),
839                 (None, "sip"),
840                 (None, None),
841         ]
842
843         for numberCriteria, descriptionCriteria in priorityOrderedCriteria:
844                 numberMatcher = None
845                 descriptionMatcher = None
846                 if numberCriteria is not None:
847                         numberMatcher = re.compile(numberCriteria)
848                 elif descriptionCriteria is not None:
849                         descriptionMatcher = re.compile(descriptionCriteria, re.I)
850
851                 for number, description in numbers.iteritems():
852                         if numberMatcher is not None and numberMatcher.match(number) is None:
853                                 continue
854                         if descriptionMatcher is not None and descriptionMatcher.match(description) is None:
855                                 continue
856                         return number
857
858
859 def set_sane_callback(backend):
860         """
861         Try to set a sane default callback number on these preferences
862         1) 1747 numbers ( Gizmo )
863         2) anything with gizmo in the name
864         3) anything with computer in the name
865         4) the first value
866         """
867         number = get_sane_callback(backend)
868         backend.set_callback_number(number)
869
870
871 def _is_not_special(name):
872         return not name.startswith("_") and name[0].lower() == name[0] and "_" not in name
873
874
875 def to_dict(obj):
876         members = inspect.getmembers(obj)
877         return dict((name, value) for (name, value) in members if _is_not_special(name))
878
879
880 def grab_debug_info(username, password):
881         cookieFile = os.path.join(".", "raw_cookies.txt")
882         try:
883                 os.remove(cookieFile)
884         except OSError:
885                 pass
886
887         backend = GVoiceBackend(cookieFile)
888         browser = backend._browser
889
890         _TEST_WEBPAGES = [
891                 ("forward", backend._forwardURL),
892                 ("token", backend._tokenURL),
893                 ("login", backend._loginURL),
894                 ("isdnd", backend._isDndURL),
895                 ("account", backend._XML_ACCOUNT_URL),
896                 ("contacts", backend._XML_CONTACTS_URL),
897
898                 ("voicemail", backend._XML_VOICEMAIL_URL),
899                 ("sms", backend._XML_SMS_URL),
900
901                 ("recent", backend._XML_RECENT_URL),
902                 ("placed", backend._XML_PLACED_URL),
903                 ("recieved", backend._XML_RECEIVED_URL),
904                 ("missed", backend._XML_MISSED_URL),
905         ]
906
907         # Get Pages
908         print "Grabbing pre-login pages"
909         for name, url in _TEST_WEBPAGES:
910                 try:
911                         page = browser.download(url)
912                 except StandardError, e:
913                         print e.message
914                         continue
915                 print "\tWriting to file"
916                 with open("not_loggedin_%s.txt" % name, "w") as f:
917                         f.write(page)
918
919         # Login
920         print "Attempting login"
921         galxToken = backend._get_token()
922         loginSuccessOrFailurePage = backend._login(username, password, galxToken)
923         with open("loggingin.txt", "w") as f:
924                 print "\tWriting to file"
925                 f.write(loginSuccessOrFailurePage)
926         try:
927                 backend._grab_account_info(loginSuccessOrFailurePage)
928         except Exception:
929                 # Retry in case the redirect failed
930                 # luckily is_authed does everything we need for a retry
931                 loggedIn = backend.is_authed(True)
932                 if not loggedIn:
933                         raise
934
935         # Get Pages
936         print "Grabbing post-login pages"
937         for name, url in _TEST_WEBPAGES:
938                 try:
939                         page = browser.download(url)
940                 except StandardError, e:
941                         print str(e)
942                         continue
943                 print "\tWriting to file"
944                 with open("loggedin_%s.txt" % name, "w") as f:
945                         f.write(page)
946
947         # Cookies
948         browser.save_cookies()
949         print "\tWriting cookies to file"
950         with open("cookies.txt", "w") as f:
951                 f.writelines(
952                         "%s: %s\n" % (c.name, c.value)
953                         for c in browser._cookies
954                 )
955
956
957 def main():
958         import sys
959         logging.basicConfig(level=logging.DEBUG)
960         args = sys.argv
961         if 3 <= len(args):
962                 username = args[1]
963                 password = args[2]
964
965         grab_debug_info(username, password)
966
967
968 if __name__ == "__main__":
969         main()