git.maemo.org Git - gc-dialer/blob - dialcentral/backends/gvoice/gvoice.py

   1 #!/usr/bin/python
   2
   3 """
   4 DialCentral - Front end for Google's GoogleVoice service.
   5 Copyright (C) 2008  Eric Warnke ericew AT gmail DOT com
   6
   7 This library is free software; you can redistribute it and/or
   8 modify it under the terms of the GNU Lesser General Public
   9 License as published by the Free Software Foundation; either
  10 version 2.1 of the License, or (at your option) any later version.
  11
  12 This library is distributed in the hope that it will be useful,
  13 but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 Lesser General Public License for more details.
  16
  17 You should have received a copy of the GNU Lesser General Public
  18 License along with this library; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 Google Voice backend code
  22
  23 Resources
  24         http://thatsmith.com/2009/03/google-voice-addon-for-firefox/
  25         http://posttopic.com/topic/google-voice-add-on-development
  26 """
  27
  28 from __future__ import with_statement
  29
  30 import os
  31 import re
  32 import urllib
  33 import urllib2
  34 import time
  35 import datetime
  36 import itertools
  37 import logging
  38 import inspect
  39
  40 from xml.sax import saxutils
  41 from xml.etree import ElementTree
  42
  43 try:
  44         import json as _json
  45         simplejson = _json
  46 except ImportError:
  47         try:
  48                 import simplejson as _simplejson
  49                 simplejson = _simplejson
  50         except ImportError:
  51                 simplejson = None
  52
  53 import browser_emu
  54
  55
  56 _moduleLogger = logging.getLogger(__name__)
  57
  58
  59 class NetworkError(RuntimeError):
  60         pass
  61
  62
  63 class MessageText(object):
  64
  65         ACCURACY_LOW = "med1"
  66         ACCURACY_MEDIUM = "med2"
  67         ACCURACY_HIGH = "high"
  68
  69         def __init__(self):
  70                 self.accuracy = None
  71                 self.text = None
  72
  73         def __str__(self):
  74                 return self.text
  75
  76         def to_dict(self):
  77                 return to_dict(self)
  78
  79         def __eq__(self, other):
  80                 return self.accuracy == other.accuracy and self.text == other.text
  81
  82
  83 class Message(object):
  84
  85         def __init__(self):
  86                 self.whoFrom = None
  87                 self.body = None
  88                 self.when = None
  89
  90         def __str__(self):
  91                 return "%s (%s): %s" % (
  92                         self.whoFrom,
  93                         self.when,
  94                         "".join(unicode(part) for part in self.body)
  95                 )
  96
  97         def to_dict(self):
  98                 selfDict = to_dict(self)
  99                 selfDict["body"] = [text.to_dict() for text in self.body] if self.body is not None else None
 100                 return selfDict
 101
 102         def __eq__(self, other):
 103                 return self.whoFrom == other.whoFrom and self.when == other.when and self.body == other.body
 104
 105
 106 class Conversation(object):
 107
 108         TYPE_VOICEMAIL = "Voicemail"
 109         TYPE_SMS = "SMS"
 110
 111         def __init__(self):
 112                 self.type = None
 113                 self.id = None
 114                 self.contactId = None
 115                 self.name = None
 116                 self.location = None
 117                 self.prettyNumber = None
 118                 self.number = None
 119
 120                 self.time = None
 121                 self.relTime = None
 122                 self.messages = None
 123                 self.isRead = None
 124                 self.isSpam = None
 125                 self.isTrash = None
 126                 self.isArchived = None
 127
 128         def __cmp__(self, other):
 129                 cmpValue = cmp(self.contactId, other.contactId)
 130                 if cmpValue != 0:
 131                         return cmpValue
 132
 133                 cmpValue = cmp(self.time, other.time)
 134                 if cmpValue != 0:
 135                         return cmpValue
 136
 137                 cmpValue = cmp(self.id, other.id)
 138                 if cmpValue != 0:
 139                         return cmpValue
 140
 141         def to_dict(self):
 142                 selfDict = to_dict(self)
 143                 selfDict["messages"] = [message.to_dict() for message in self.messages] if self.messages is not None else None
 144                 return selfDict
 145
 146
 147 class GVoiceBackend(object):
 148         """
 149         This class encapsulates all of the knowledge necessary to interact with the GoogleVoice servers
 150         the functions include login, setting up a callback number, and initalting a callback
 151         """
 152
 153         PHONE_TYPE_HOME = 1
 154         PHONE_TYPE_MOBILE = 2
 155         PHONE_TYPE_WORK = 3
 156         PHONE_TYPE_GIZMO = 7
 157
 158         def __init__(self, cookieFile = None):
 159                 # Important items in this function are the setup of the browser emulation and cookie file
 160                 self._browser = browser_emu.MozillaEmulator(1)
 161                 self._loadedFromCookies = self._browser.load_cookies(cookieFile)
 162
 163                 self._token = ""
 164                 self._accountNum = ""
 165                 self._lastAuthed = 0.0
 166                 self._callbackNumber = ""
 167                 self._callbackNumbers = {}
 168
 169                 # Suprisingly, moving all of these from class to self sped up startup time
 170
 171                 self._validateRe = re.compile("^\+?[0-9]{10,}$")
 172
 173                 self._loginURL = "https://www.google.com/accounts/ServiceLoginAuth"
 174
 175                 SECURE_URL_BASE = "https://www.google.com/voice/"
 176                 SECURE_MOBILE_URL_BASE = SECURE_URL_BASE + "mobile/"
 177                 self._tokenURL = SECURE_URL_BASE + "m"
 178                 self._callUrl = SECURE_URL_BASE + "call/connect"
 179                 self._callCancelURL = SECURE_URL_BASE + "call/cancel"
 180                 self._sendSmsURL = SECURE_URL_BASE + "sms/send"
 181
 182                 self._isDndURL = "https://www.google.com/voice/m/donotdisturb"
 183                 self._isDndRe = re.compile(r"""<input.*?id="doNotDisturb".*?checked="(.*?)"\s*/>""")
 184                 self._setDndURL = "https://www.google.com/voice/m/savednd"
 185
 186                 self._downloadVoicemailURL = SECURE_URL_BASE + "media/send_voicemail/"
 187                 self._markAsReadURL = SECURE_URL_BASE + "m/mark"
 188                 self._archiveMessageURL = SECURE_URL_BASE + "m/archive"
 189
 190                 self._XML_SEARCH_URL = SECURE_URL_BASE + "inbox/search/"
 191                 self._XML_ACCOUNT_URL = SECURE_URL_BASE + "contacts/"
 192                 # HACK really this redirects to the main pge and we are grabbing some javascript
 193                 self._XML_CONTACTS_URL = "http://www.google.com/voice/inbox/search/contact"
 194                 self._CSV_CONTACTS_URL = "http://mail.google.com/mail/contacts/data/export"
 195                 self._JSON_CONTACTS_URL = SECURE_URL_BASE + "b/0/request/user"
 196                 self._XML_RECENT_URL = SECURE_URL_BASE + "inbox/recent/"
 197
 198                 self.XML_FEEDS = (
 199                         'inbox', 'starred', 'all', 'spam', 'trash', 'voicemail', 'sms',
 200                         'recorded', 'placed', 'received', 'missed'
 201                 )
 202                 self._XML_INBOX_URL = SECURE_URL_BASE + "inbox/recent/inbox"
 203                 self._XML_STARRED_URL = SECURE_URL_BASE + "inbox/recent/starred"
 204                 self._XML_ALL_URL = SECURE_URL_BASE + "inbox/recent/all"
 205                 self._XML_SPAM_URL = SECURE_URL_BASE + "inbox/recent/spam"
 206                 self._XML_TRASH_URL = SECURE_URL_BASE + "inbox/recent/trash"
 207                 self._XML_VOICEMAIL_URL = SECURE_URL_BASE + "inbox/recent/voicemail/"
 208                 self._XML_SMS_URL = SECURE_URL_BASE + "inbox/recent/sms/"
 209                 self._JSON_SMS_URL = SECURE_URL_BASE + "b/0/request/messages/"
 210                 self._JSON_SMS_COUNT_URL = SECURE_URL_BASE + "b/0/request/unread"
 211                 self._XML_RECORDED_URL = SECURE_URL_BASE + "inbox/recent/recorded/"
 212                 self._XML_PLACED_URL = SECURE_URL_BASE + "inbox/recent/placed/"
 213                 self._XML_RECEIVED_URL = SECURE_URL_BASE + "inbox/recent/received/"
 214                 self._XML_MISSED_URL = SECURE_URL_BASE + "inbox/recent/missed/"
 215
 216                 self._galxRe = re.compile(r"""<input.*?name="GALX".*?value="(.*?)".*?/>""", re.MULTILINE | re.DOTALL)
 217
 218                 self._seperateVoicemailsRegex = re.compile(r"""^\s*<div id="(\w+)"\s* class=".*?gc-message.*?">""", re.MULTILINE | re.DOTALL)
 219                 self._exactVoicemailTimeRegex = re.compile(r"""<span class="gc-message-time">(.*?)</span>""", re.MULTILINE)
 220                 self._relativeVoicemailTimeRegex = re.compile(r"""<span class="gc-message-relative">(.*?)</span>""", re.MULTILINE)
 221                 self._voicemailNameRegex = re.compile(r"""<a class=.*?gc-message-name-link.*?>(.*?)</a>""", re.MULTILINE | re.DOTALL)
 222                 self._voicemailNumberRegex = re.compile(r"""<input type="hidden" class="gc-text gc-quickcall-ac" value="(.*?)"/>""", re.MULTILINE)
 223                 self._prettyVoicemailNumberRegex = re.compile(r"""<span class="gc-message-type">(.*?)</span>""", re.MULTILINE)
 224                 self._voicemailLocationRegex = re.compile(r"""<span class="gc-message-location">.*?<a.*?>(.*?)</a></span>""", re.MULTILINE)
 225                 self._messagesContactIDRegex = re.compile(r"""<a class=".*?gc-message-name-link.*?">.*?</a>\s*?<span .*?>(.*?)</span>""", re.MULTILINE)
 226                 self._voicemailMessageRegex = re.compile(r"""(<span id="\d+-\d+" class="gc-word-(.*?)">(.*?)</span>|<a .*? class="gc-message-mni">(.*?)</a>)""", re.MULTILINE)
 227                 self._smsFromRegex = re.compile(r"""<span class="gc-message-sms-from">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 228                 self._smsTimeRegex = re.compile(r"""<span class="gc-message-sms-time">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 229                 self._smsTextRegex = re.compile(r"""<span class="gc-message-sms-text">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 230
 231         def is_quick_login_possible(self):
 232                 """
 233                 @returns True then refresh_account_info might be enough to login, else full login is required
 234                 """
 235                 return self._loadedFromCookies or 0.0 < self._lastAuthed
 236
 237         def refresh_account_info(self):
 238                 try:
 239                         page = self._get_page(self._JSON_CONTACTS_URL)
 240                         accountData = self._grab_account_info(page)
 241                 except Exception, e:
 242                         _moduleLogger.exception(str(e))
 243                         return None
 244
 245                 self._browser.save_cookies()
 246                 self._lastAuthed = time.time()
 247                 return accountData
 248
 249         def _get_token(self):
 250                 tokenPage = self._get_page(self._tokenURL)
 251
 252                 galxTokens = self._galxRe.search(tokenPage)
 253                 if galxTokens is not None:
 254                         galxToken = galxTokens.group(1)
 255                 else:
 256                         galxToken = ""
 257                         _moduleLogger.debug("Could not grab GALX token")
 258                 return galxToken
 259
 260         def _login(self, username, password, token):
 261                 loginData = {
 262                         'Email' : username,
 263                         'Passwd' : password,
 264                         'service': "grandcentral",
 265                         "ltmpl": "mobile",
 266                         "btmpl": "mobile",
 267                         "PersistentCookie": "yes",
 268                         "GALX": token,
 269                         "continue": self._JSON_CONTACTS_URL,
 270                 }
 271
 272                 loginSuccessOrFailurePage = self._get_page(self._loginURL, loginData)
 273                 return loginSuccessOrFailurePage
 274
 275         def login(self, username, password):
 276                 """
 277                 Attempt to login to GoogleVoice
 278                 @returns Whether login was successful or not
 279                 @blocks
 280                 """
 281                 self.logout()
 282                 galxToken = self._get_token()
 283                 loginSuccessOrFailurePage = self._login(username, password, galxToken)
 284
 285                 try:
 286                         accountData = self._grab_account_info(loginSuccessOrFailurePage)
 287                 except Exception, e:
 288                         # Retry in case the redirect failed
 289                         # luckily refresh_account_info does everything we need for a retry
 290                         accountData = self.refresh_account_info()
 291                         if accountData is None:
 292                                 _moduleLogger.exception(str(e))
 293                                 return None
 294                         _moduleLogger.info("Redirection failed on initial login attempt, auto-corrected for this")
 295
 296                 self._browser.save_cookies()
 297                 self._lastAuthed = time.time()
 298                 return accountData
 299
 300         def persist(self):
 301                 self._browser.save_cookies()
 302
 303         def shutdown(self):
 304                 self._browser.save_cookies()
 305                 self._token = None
 306                 self._lastAuthed = 0.0
 307
 308         def logout(self):
 309                 self._browser.clear_cookies()
 310                 self._browser.save_cookies()
 311                 self._token = None
 312                 self._lastAuthed = 0.0
 313                 self._callbackNumbers = {}
 314
 315         def is_dnd(self):
 316                 """
 317                 @blocks
 318                 """
 319                 isDndPage = self._get_page(self._isDndURL)
 320
 321                 dndGroup = self._isDndRe.search(isDndPage)
 322                 if dndGroup is None:
 323                         return False
 324                 dndStatus = dndGroup.group(1)
 325                 isDnd = True if dndStatus.strip().lower() == "true" else False
 326                 return isDnd
 327
 328         def set_dnd(self, doNotDisturb):
 329                 """
 330                 @blocks
 331                 """
 332                 dndPostData = {
 333                         "doNotDisturb": 1 if doNotDisturb else 0,
 334                 }
 335
 336                 dndPage = self._get_page_with_token(self._setDndURL, dndPostData)
 337
 338         def call(self, outgoingNumber):
 339                 """
 340                 This is the main function responsible for initating the callback
 341                 @blocks
 342                 """
 343                 outgoingNumber = self._send_validation(outgoingNumber)
 344                 subscriberNumber = None
 345                 phoneType = guess_phone_type(self._callbackNumber) # @todo Fix this hack
 346
 347                 callData = {
 348                                 'outgoingNumber': outgoingNumber,
 349                                 'forwardingNumber': self._callbackNumber,
 350                                 'subscriberNumber': subscriberNumber or 'undefined',
 351                                 'phoneType': str(phoneType),
 352                                 'remember': '1',
 353                 }
 354                 _moduleLogger.info("%r" % callData)
 355
 356                 page = self._get_page_with_token(
 357                         self._callUrl,
 358                         callData,
 359                 )
 360                 self._parse_with_validation(page)
 361                 return True
 362
 363         def cancel(self, outgoingNumber=None):
 364                 """
 365                 Cancels a call matching outgoing and forwarding numbers (if given).
 366                 Will raise an error if no matching call is being placed
 367                 @blocks
 368                 """
 369                 page = self._get_page_with_token(
 370                         self._callCancelURL,
 371                         {
 372                         'outgoingNumber': outgoingNumber or 'undefined',
 373                         'forwardingNumber': self._callbackNumber or 'undefined',
 374                         'cancelType': 'C2C',
 375                         },
 376                 )
 377                 self._parse_with_validation(page)
 378
 379         def send_sms(self, phoneNumbers, message):
 380                 """
 381                 @blocks
 382                 """
 383                 validatedPhoneNumbers = [
 384                         self._send_validation(phoneNumber)
 385                         for phoneNumber in phoneNumbers
 386                 ]
 387                 flattenedPhoneNumbers = ",".join(validatedPhoneNumbers)
 388                 page = self._get_page_with_token(
 389                         self._sendSmsURL,
 390                         {
 391                                 'phoneNumber': flattenedPhoneNumbers,
 392                                 'text': unicode(message).encode("utf-8"),
 393                         },
 394                 )
 395                 self._parse_with_validation(page)
 396
 397         def search(self, query):
 398                 """
 399                 Search your Google Voice Account history for calls, voicemails, and sms
 400                 Returns ``Folder`` instance containting matching messages
 401                 @blocks
 402                 """
 403                 page = self._get_page(
 404                         self._XML_SEARCH_URL,
 405                         {"q": query},
 406                 )
 407                 json, html = extract_payload(page)
 408                 return json
 409
 410         def get_feed(self, feed):
 411                 """
 412                 @blocks
 413                 """
 414                 actualFeed = "_XML_%s_URL" % feed.upper()
 415                 feedUrl = getattr(self, actualFeed)
 416
 417                 page = self._get_page(feedUrl)
 418                 json, html = extract_payload(page)
 419
 420                 return json
 421
 422         def recording_url(self, messageId):
 423                 url = self._downloadVoicemailURL+messageId
 424                 return url
 425
 426         def download(self, messageId, targetPath):
 427                 """
 428                 Download a voicemail or recorded call MP3 matching the given ``msg``
 429                 which can either be a ``Message`` instance, or a SHA1 identifier.
 430                 Message hashes can be found in ``self.voicemail().messages`` for example.
 431                 @returns location of saved file.
 432                 @blocks
 433                 """
 434                 page = self._get_page(self.recording_url(messageId))
 435                 with open(targetPath, 'wb') as fo:
 436                         fo.write(page)
 437
 438         def is_valid_syntax(self, number):
 439                 """
 440                 @returns If This number be called ( syntax validation only )
 441                 """
 442                 return self._validateRe.match(number) is not None
 443
 444         def get_account_number(self):
 445                 """
 446                 @returns The GoogleVoice phone number
 447                 """
 448                 return self._accountNum
 449
 450         def get_callback_numbers(self):
 451                 """
 452                 @returns a dictionary mapping call back numbers to descriptions
 453                 @note These results are cached for 30 minutes.
 454                 """
 455                 return self._callbackNumbers
 456
 457         def set_callback_number(self, callbacknumber):
 458                 """
 459                 Set the number that GoogleVoice calls
 460                 @param callbacknumber should be a proper 10 digit number
 461                 """
 462                 self._callbackNumber = callbacknumber
 463                 _moduleLogger.info("Callback number changed: %r" % self._callbackNumber)
 464                 return True
 465
 466         def get_callback_number(self):
 467                 """
 468                 @returns Current callback number or None
 469                 """
 470                 return self._callbackNumber
 471
 472         def get_received_calls(self):
 473                 """
 474                 @returns Iterable of (personsName, phoneNumber, exact date, relative date, action)
 475                 @blocks
 476                 """
 477                 return self._parse_recent(self._get_page(self._XML_RECEIVED_URL))
 478
 479         def get_missed_calls(self):
 480                 """
 481                 @returns Iterable of (personsName, phoneNumber, exact date, relative date, action)
 482                 @blocks
 483                 """
 484                 return self._parse_recent(self._get_page(self._XML_MISSED_URL))
 485
 486         def get_placed_calls(self):
 487                 """
 488                 @returns Iterable of (personsName, phoneNumber, exact date, relative date, action)
 489                 @blocks
 490                 """
 491                 return self._parse_recent(self._get_page(self._XML_PLACED_URL))
 492
 493         def get_csv_contacts(self):
 494                 data = {
 495                         "groupToExport": "mine",
 496                         "exportType": "ALL",
 497                         "out": "OUTLOOK_CSV",
 498                 }
 499                 encodedData = urllib.urlencode(data)
 500                 contacts = self._get_page(self._CSV_CONTACTS_URL+"?"+encodedData)
 501                 return contacts
 502
 503         def get_voicemails(self):
 504                 """
 505                 @blocks
 506                 """
 507                 voicemailPage = self._get_page(self._XML_VOICEMAIL_URL)
 508                 voicemailHtml = self._grab_html(voicemailPage)
 509                 voicemailJson = self._grab_json(voicemailPage)
 510                 if voicemailJson is None:
 511                         return ()
 512                 parsedVoicemail = self._parse_voicemail(voicemailHtml)
 513                 voicemails = self._merge_conversation_sources(parsedVoicemail, voicemailJson)
 514                 return voicemails
 515
 516         def get_texts(self):
 517                 """
 518                 @blocks
 519                 """
 520                 smsPage = self._get_page(self._XML_SMS_URL)
 521                 smsHtml = self._grab_html(smsPage)
 522                 smsJson = self._grab_json(smsPage)
 523                 if smsJson is None:
 524                         return ()
 525                 parsedSms = self._parse_sms(smsHtml)
 526                 smss = self._merge_conversation_sources(parsedSms, smsJson)
 527                 return smss
 528
 529         def get_unread_counts(self):
 530                 countPage = self._get_page(self._JSON_SMS_COUNT_URL)
 531                 counts = parse_json(countPage)
 532                 counts = counts["unreadCounts"]
 533                 return counts
 534
 535         def mark_message(self, messageId, asRead):
 536                 """
 537                 @blocks
 538                 """
 539                 postData = {
 540                         "read": 1 if asRead else 0,
 541                         "id": messageId,
 542                 }
 543
 544                 markPage = self._get_page(self._markAsReadURL, postData)
 545
 546         def archive_message(self, messageId):
 547                 """
 548                 @blocks
 549                 """
 550                 postData = {
 551                         "id": messageId,
 552                 }
 553
 554                 markPage = self._get_page(self._archiveMessageURL, postData)
 555
 556         def _grab_json(self, flatXml):
 557                 xmlTree = ElementTree.fromstring(flatXml)
 558                 jsonElement = xmlTree.getchildren()[0]
 559                 flatJson = jsonElement.text
 560                 jsonTree = parse_json(flatJson)
 561                 return jsonTree
 562
 563         def _grab_html(self, flatXml):
 564                 xmlTree = ElementTree.fromstring(flatXml)
 565                 htmlElement = xmlTree.getchildren()[1]
 566                 flatHtml = htmlElement.text
 567                 return flatHtml
 568
 569         def _grab_account_info(self, page):
 570                 accountData = parse_json(page)
 571                 self._token = accountData["r"]
 572                 self._accountNum = accountData["number"]["raw"]
 573                 for callback in accountData["phones"].itervalues():
 574                         self._callbackNumbers[callback["phoneNumber"]] = callback["name"]
 575                 if len(self._callbackNumbers) == 0:
 576                         _moduleLogger.debug("Could not extract callback numbers from GoogleVoice (the troublesome page follows):\n%s" % page)
 577                 return accountData
 578
 579         def _send_validation(self, number):
 580                 if not self.is_valid_syntax(number):
 581                         raise ValueError('Number is not valid: "%s"' % number)
 582                 return number
 583
 584         def _parse_recent(self, recentPage):
 585                 allRecentHtml = self._grab_html(recentPage)
 586                 allRecentData = self._parse_history(allRecentHtml)
 587                 for recentCallData in allRecentData:
 588                         yield recentCallData
 589
 590         def _parse_history(self, historyHtml):
 591                 splitVoicemail = self._seperateVoicemailsRegex.split(historyHtml)
 592                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 593                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 594                         exactTime = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 595                         exactTime = google_strptime(exactTime)
 596                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 597                         relativeTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 598                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 599                         location = locationGroup.group(1).strip() if locationGroup else ""
 600
 601                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 602                         name = nameGroup.group(1).strip() if nameGroup else ""
 603                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 604                         number = numberGroup.group(1).strip() if numberGroup else ""
 605                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 606                         prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 607                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 608                         contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 609
 610                         yield {
 611                                 "id": messageId.strip(),
 612                                 "contactId": contactId,
 613                                 "name": unescape(name),
 614                                 "time": exactTime,
 615                                 "relTime": relativeTime,
 616                                 "prettyNumber": prettyNumber,
 617                                 "number": number,
 618                                 "location": unescape(location),
 619                         }
 620
 621         @staticmethod
 622         def _interpret_voicemail_regex(group):
 623                 quality, content, number = group.group(2), group.group(3), group.group(4)
 624                 text = MessageText()
 625                 if quality is not None and content is not None:
 626                         text.accuracy = quality
 627                         text.text = unescape(content)
 628                         return text
 629                 elif number is not None:
 630                         text.accuracy = MessageText.ACCURACY_HIGH
 631                         text.text = number
 632                         return text
 633
 634         def _parse_voicemail(self, voicemailHtml):
 635                 splitVoicemail = self._seperateVoicemailsRegex.split(voicemailHtml)
 636                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 637                         conv = Conversation()
 638                         conv.type = Conversation.TYPE_VOICEMAIL
 639                         conv.id = messageId.strip()
 640
 641                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 642                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 643                         conv.time = google_strptime(exactTimeText)
 644                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 645                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 646                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 647                         conv.location = unescape(locationGroup.group(1).strip() if locationGroup else "")
 648
 649                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 650                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 651                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 652                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 653                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 654                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 655                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 656                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 657
 658                         messageGroups = self._voicemailMessageRegex.finditer(messageHtml)
 659                         messageParts = [
 660                                 self._interpret_voicemail_regex(group)
 661                                 for group in messageGroups
 662                         ] if messageGroups else ((MessageText.ACCURACY_LOW, "No Transcription"), )
 663                         message = Message()
 664                         message.body = messageParts
 665                         message.whoFrom = conv.name
 666                         try:
 667                                 message.when = conv.time.strftime("%I:%M %p")
 668                         except ValueError:
 669                                 _moduleLogger.exception("Confusing time provided: %r" % conv.time)
 670                                 message.when = "Unknown"
 671                         conv.messages = (message, )
 672
 673                         yield conv
 674
 675         @staticmethod
 676         def _interpret_sms_message_parts(fromPart, textPart, timePart):
 677                 text = MessageText()
 678                 text.accuracy = MessageText.ACCURACY_MEDIUM
 679                 text.text = unescape(textPart)
 680
 681                 message = Message()
 682                 message.body = (text, )
 683                 message.whoFrom = fromPart
 684                 message.when = timePart
 685
 686                 return message
 687
 688         def _parse_sms(self, smsHtml):
 689                 splitSms = self._seperateVoicemailsRegex.split(smsHtml)
 690                 for messageId, messageHtml in itergroup(splitSms[1:], 2):
 691                         conv = Conversation()
 692                         conv.type = Conversation.TYPE_SMS
 693                         conv.id = messageId.strip()
 694
 695                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 696                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 697                         conv.time = google_strptime(exactTimeText)
 698                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 699                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 700                         conv.location = ""
 701
 702                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 703                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 704                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 705                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 706                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 707                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 708                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 709                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 710
 711                         fromGroups = self._smsFromRegex.finditer(messageHtml)
 712                         fromParts = (group.group(1).strip() for group in fromGroups)
 713                         textGroups = self._smsTextRegex.finditer(messageHtml)
 714                         textParts = (group.group(1).strip() for group in textGroups)
 715                         timeGroups = self._smsTimeRegex.finditer(messageHtml)
 716                         timeParts = (group.group(1).strip() for group in timeGroups)
 717
 718                         messageParts = itertools.izip(fromParts, textParts, timeParts)
 719                         messages = [self._interpret_sms_message_parts(*parts) for parts in messageParts]
 720                         conv.messages = messages
 721
 722                         yield conv
 723
 724         @staticmethod
 725         def _merge_conversation_sources(parsedMessages, json):
 726                 for message in parsedMessages:
 727                         jsonItem = json["messages"][message.id]
 728                         message.isRead = jsonItem["isRead"]
 729                         message.isSpam = jsonItem["isSpam"]
 730                         message.isTrash = jsonItem["isTrash"]
 731                         message.isArchived = "inbox" not in jsonItem["labels"]
 732                         yield message
 733
 734         def _get_page(self, url, data = None, refererUrl = None):
 735                 headers = {}
 736                 if refererUrl is not None:
 737                         headers["Referer"] = refererUrl
 738
 739                 encodedData = urllib.urlencode(data) if data is not None else None
 740
 741                 try:
 742                         page = self._browser.download(url, encodedData, None, headers)
 743                 except urllib2.URLError, e:
 744                         _moduleLogger.error("Translating error: %s" % str(e))
 745                         raise NetworkError("%s is not accesible" % url)
 746
 747                 return page
 748
 749         def _get_page_with_token(self, url, data = None, refererUrl = None):
 750                 if data is None:
 751                         data = {}
 752                 data['_rnr_se'] = self._token
 753
 754                 page = self._get_page(url, data, refererUrl)
 755
 756                 return page
 757
 758         def _parse_with_validation(self, page):
 759                 json = parse_json(page)
 760                 self._validate_response(json)
 761                 return json
 762
 763         def _validate_response(self, response):
 764                 """
 765                 Validates that the JSON response is A-OK
 766                 """
 767                 try:
 768                         assert response is not None, "Response not provided"
 769                         assert 'ok' in response, "Response lacks status"
 770                         assert response['ok'], "Response not good"
 771                 except AssertionError:
 772                         try:
 773                                 if response["data"]["code"] == 20:
 774                                         raise RuntimeError(
 775 """Ambiguous error 20 returned by Google Voice.
 776 Please verify you have configured your callback number (currently "%s").  If it is configured some other suspected causes are: non-verified callback numbers, and Gizmo5 callback numbers.""" % self._callbackNumber)
 777                         except KeyError:
 778                                 pass
 779                         raise RuntimeError('There was a problem with GV: %s' % response)
 780
 781
 782 _UNESCAPE_ENTITIES = {
 783  "&quot;": '"',
 784  "&nbsp;": " ",
 785  "&#39;": "'",
 786 }
 787
 788
 789 def unescape(text):
 790         plain = saxutils.unescape(text, _UNESCAPE_ENTITIES)
 791         return plain
 792
 793
 794 def google_strptime(time):
 795         """
 796         Hack: Google always returns the time in the same locale.  Sadly if the
 797         local system's locale is different, there isn't a way to perfectly handle
 798         the time.  So instead we handle implement some time formatting
 799         """
 800         abbrevTime = time[:-3]
 801         parsedTime = datetime.datetime.strptime(abbrevTime, "%m/%d/%y %I:%M")
 802         if time.endswith("PM"):
 803                 parsedTime += datetime.timedelta(hours=12)
 804         return parsedTime
 805
 806
 807 def itergroup(iterator, count, padValue = None):
 808         """
 809         Iterate in groups of 'count' values. If there
 810         aren't enough values, the last result is padded with
 811         None.
 812
 813         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 814         ...     print tuple(val)
 815         (1, 2, 3)
 816         (4, 5, 6)
 817         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 818         ...     print list(val)
 819         [1, 2, 3]
 820         [4, 5, 6]
 821         >>> for val in itergroup([1, 2, 3, 4, 5, 6, 7], 3):
 822         ...     print tuple(val)
 823         (1, 2, 3)
 824         (4, 5, 6)
 825         (7, None, None)
 826         >>> for val in itergroup("123456", 3):
 827         ...     print tuple(val)
 828         ('1', '2', '3')
 829         ('4', '5', '6')
 830         >>> for val in itergroup("123456", 3):
 831         ...     print repr("".join(val))
 832         '123'
 833         '456'
 834         """
 835         paddedIterator = itertools.chain(iterator, itertools.repeat(padValue, count-1))
 836         nIterators = (paddedIterator, ) * count
 837         return itertools.izip(*nIterators)
 838
 839
 840 def safe_eval(s):
 841         _TRUE_REGEX = re.compile("true")
 842         _FALSE_REGEX = re.compile("false")
 843         _COMMENT_REGEX = re.compile("^\s+//.*$", re.M)
 844         s = _TRUE_REGEX.sub("True", s)
 845         s = _FALSE_REGEX.sub("False", s)
 846         s = _COMMENT_REGEX.sub("#", s)
 847         try:
 848                 results = eval(s, {}, {})
 849         except SyntaxError:
 850                 _moduleLogger.exception("Oops")
 851                 results = None
 852         return results
 853
 854
 855 def _fake_parse_json(flattened):
 856         return safe_eval(flattened)
 857
 858
 859 def _actual_parse_json(flattened):
 860         return simplejson.loads(flattened)
 861
 862
 863 if simplejson is None:
 864         parse_json = _fake_parse_json
 865 else:
 866         parse_json = _actual_parse_json
 867
 868
 869 def extract_payload(flatXml):
 870         xmlTree = ElementTree.fromstring(flatXml)
 871
 872         jsonElement = xmlTree.getchildren()[0]
 873         flatJson = jsonElement.text
 874         jsonTree = parse_json(flatJson)
 875
 876         htmlElement = xmlTree.getchildren()[1]
 877         flatHtml = htmlElement.text
 878
 879         return jsonTree, flatHtml
 880
 881
 882 def guess_phone_type(number):
 883         if number.startswith("747") or number.startswith("1747") or number.startswith("+1747"):
 884                 return GVoiceBackend.PHONE_TYPE_GIZMO
 885         else:
 886                 return GVoiceBackend.PHONE_TYPE_MOBILE
 887
 888
 889 def get_sane_callback(backend):
 890         """
 891         Try to set a sane default callback number on these preferences
 892         1) 1747 numbers ( Gizmo )
 893         2) anything with gizmo in the name
 894         3) anything with computer in the name
 895         4) the first value
 896         """
 897         numbers = backend.get_callback_numbers()
 898
 899         priorityOrderedCriteria = [
 900                 ("\+1747", None),
 901                 ("1747", None),
 902                 ("747", None),
 903                 (None, "gizmo"),
 904                 (None, "computer"),
 905                 (None, "sip"),
 906                 (None, None),
 907         ]
 908
 909         for numberCriteria, descriptionCriteria in priorityOrderedCriteria:
 910                 numberMatcher = None
 911                 descriptionMatcher = None
 912                 if numberCriteria is not None:
 913                         numberMatcher = re.compile(numberCriteria)
 914                 elif descriptionCriteria is not None:
 915                         descriptionMatcher = re.compile(descriptionCriteria, re.I)
 916
 917                 for number, description in numbers.iteritems():
 918                         if numberMatcher is not None and numberMatcher.match(number) is None:
 919                                 continue
 920                         if descriptionMatcher is not None and descriptionMatcher.match(description) is None:
 921                                 continue
 922                         return number
 923
 924
 925 def set_sane_callback(backend):
 926         """
 927         Try to set a sane default callback number on these preferences
 928         1) 1747 numbers ( Gizmo )
 929         2) anything with gizmo in the name
 930         3) anything with computer in the name
 931         4) the first value
 932         """
 933         number = get_sane_callback(backend)
 934         backend.set_callback_number(number)
 935
 936
 937 def _is_not_special(name):
 938         return not name.startswith("_") and name[0].lower() == name[0] and "_" not in name
 939
 940
 941 def to_dict(obj):
 942         members = inspect.getmembers(obj)
 943         return dict((name, value) for (name, value) in members if _is_not_special(name))
 944
 945
 946 def grab_debug_info(username, password):
 947         cookieFile = os.path.join(".", "raw_cookies.txt")
 948         try:
 949                 os.remove(cookieFile)
 950         except OSError:
 951                 pass
 952
 953         backend = GVoiceBackend(cookieFile)
 954         browser = backend._browser
 955
 956         _TEST_WEBPAGES = [
 957                 ("token", backend._tokenURL),
 958                 ("login", backend._loginURL),
 959                 ("isdnd", backend._isDndURL),
 960                 ("account", backend._XML_ACCOUNT_URL),
 961                 ("html_contacts", backend._XML_CONTACTS_URL),
 962                 ("contacts", backend._JSON_CONTACTS_URL),
 963                 ("csv", backend._CSV_CONTACTS_URL),
 964
 965                 ("voicemail", backend._XML_VOICEMAIL_URL),
 966                 ("html_sms", backend._XML_SMS_URL),
 967                 ("sms", backend._JSON_SMS_URL),
 968                 ("count", backend._JSON_SMS_COUNT_URL),
 969
 970                 ("recent", backend._XML_RECENT_URL),
 971                 ("placed", backend._XML_PLACED_URL),
 972                 ("recieved", backend._XML_RECEIVED_URL),
 973                 ("missed", backend._XML_MISSED_URL),
 974         ]
 975
 976         # Get Pages
 977         print "Grabbing pre-login pages"
 978         for name, url in _TEST_WEBPAGES:
 979                 try:
 980                         page = browser.download(url)
 981                 except StandardError, e:
 982                         print e.message
 983                         continue
 984                 print "\tWriting to file"
 985                 with open("not_loggedin_%s.txt" % name, "w") as f:
 986                         f.write(page)
 987
 988         # Login
 989         print "Attempting login"
 990         galxToken = backend._get_token()
 991         loginSuccessOrFailurePage = backend._login(username, password, galxToken)
 992         with open("loggingin.txt", "w") as f:
 993                 print "\tWriting to file"
 994                 f.write(loginSuccessOrFailurePage)
 995         try:
 996                 backend._grab_account_info(loginSuccessOrFailurePage)
 997         except Exception:
 998                 # Retry in case the redirect failed
 999                 # luckily refresh_account_info does everything we need for a retry
1000                 loggedIn = backend.refresh_account_info() is not None
1001                 if not loggedIn:
1002                         raise
1003
1004         # Get Pages
1005         print "Grabbing post-login pages"
1006         for name, url in _TEST_WEBPAGES:
1007                 try:
1008                         page = browser.download(url)
1009                 except StandardError, e:
1010                         print str(e)
1011                         continue
1012                 print "\tWriting to file"
1013                 with open("loggedin_%s.txt" % name, "w") as f:
1014                         f.write(page)
1015
1016         # Cookies
1017         browser.save_cookies()
1018         print "\tWriting cookies to file"
1019         with open("cookies.txt", "w") as f:
1020                 f.writelines(
1021                         "%s: %s\n" % (c.name, c.value)
1022                         for c in browser._cookies
1023                 )
1024
1025
1026 def grab_voicemails(username, password):
1027         cookieFile = os.path.join(".", "raw_cookies.txt")
1028         try:
1029                 os.remove(cookieFile)
1030         except OSError:
1031                 pass
1032
1033         backend = GVoiceBackend(cookieFile)
1034         backend.login(username, password)
1035         voicemails = list(backend.get_voicemails())
1036         for voicemail in voicemails:
1037                 print voicemail.id
1038                 backend.download(voicemail.id, ".")
1039
1040
1041 def main():
1042         import sys
1043         logging.basicConfig(level=logging.DEBUG)
1044         args = sys.argv
1045         if 3 <= len(args):
1046                 username = args[1]
1047                 password = args[2]
1048
1049         grab_debug_info(username, password)
1050         grab_voicemails(username, password)
1051
1052
1053 if __name__ == "__main__":
1054         main()