git.maemo.org Git - gc-dialer/blob - src/backends/gvoice/gvoice.py

   1 #!/usr/bin/python
   2
   3 """
   4 DialCentral - Front end for Google's GoogleVoice service.
   5 Copyright (C) 2008  Eric Warnke ericew AT gmail DOT com
   6
   7 This library is free software; you can redistribute it and/or
   8 modify it under the terms of the GNU Lesser General Public
   9 License as published by the Free Software Foundation; either
  10 version 2.1 of the License, or (at your option) any later version.
  11
  12 This library is distributed in the hope that it will be useful,
  13 but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 Lesser General Public License for more details.
  16
  17 You should have received a copy of the GNU Lesser General Public
  18 License along with this library; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 Google Voice backend code
  22
  23 Resources
  24         http://thatsmith.com/2009/03/google-voice-addon-for-firefox/
  25         http://posttopic.com/topic/google-voice-add-on-development
  26 """
  27
  28 from __future__ import with_statement
  29
  30 import os
  31 import re
  32 import urllib
  33 import urllib2
  34 import time
  35 import datetime
  36 import itertools
  37 import logging
  38 import inspect
  39
  40 from xml.sax import saxutils
  41 from xml.etree import ElementTree
  42
  43 try:
  44         import simplejson as _simplejson
  45         simplejson = _simplejson
  46 except ImportError:
  47         simplejson = None
  48
  49 import browser_emu
  50
  51
  52 _moduleLogger = logging.getLogger(__name__)
  53
  54
  55 class NetworkError(RuntimeError):
  56         pass
  57
  58
  59 class MessageText(object):
  60
  61         ACCURACY_LOW = "med1"
  62         ACCURACY_MEDIUM = "med2"
  63         ACCURACY_HIGH = "high"
  64
  65         def __init__(self):
  66                 self.accuracy = None
  67                 self.text = None
  68
  69         def __str__(self):
  70                 return self.text
  71
  72         def to_dict(self):
  73                 return to_dict(self)
  74
  75         def __eq__(self, other):
  76                 return self.accuracy == other.accuracy and self.text == other.text
  77
  78
  79 class Message(object):
  80
  81         def __init__(self):
  82                 self.whoFrom = None
  83                 self.body = None
  84                 self.when = None
  85
  86         def __str__(self):
  87                 return "%s (%s): %s" % (
  88                         self.whoFrom,
  89                         self.when,
  90                         "".join(unicode(part) for part in self.body)
  91                 )
  92
  93         def to_dict(self):
  94                 selfDict = to_dict(self)
  95                 selfDict["body"] = [text.to_dict() for text in self.body] if self.body is not None else None
  96                 return selfDict
  97
  98         def __eq__(self, other):
  99                 return self.whoFrom == other.whoFrom and self.when == other.when and self.body == other.body
 100
 101
 102 class Conversation(object):
 103
 104         TYPE_VOICEMAIL = "Voicemail"
 105         TYPE_SMS = "SMS"
 106
 107         def __init__(self):
 108                 self.type = None
 109                 self.id = None
 110                 self.contactId = None
 111                 self.name = None
 112                 self.location = None
 113                 self.prettyNumber = None
 114                 self.number = None
 115
 116                 self.time = None
 117                 self.relTime = None
 118                 self.messages = None
 119                 self.isRead = None
 120                 self.isSpam = None
 121                 self.isTrash = None
 122                 self.isArchived = None
 123
 124         def __cmp__(self, other):
 125                 cmpValue = cmp(self.contactId, other.contactId)
 126                 if cmpValue != 0:
 127                         return cmpValue
 128
 129                 cmpValue = cmp(self.time, other.time)
 130                 if cmpValue != 0:
 131                         return cmpValue
 132
 133                 cmpValue = cmp(self.id, other.id)
 134                 if cmpValue != 0:
 135                         return cmpValue
 136
 137         def to_dict(self):
 138                 selfDict = to_dict(self)
 139                 selfDict["messages"] = [message.to_dict() for message in self.messages] if self.messages is not None else None
 140                 return selfDict
 141
 142
 143 class GVoiceBackend(object):
 144         """
 145         This class encapsulates all of the knowledge necessary to interact with the GoogleVoice servers
 146         the functions include login, setting up a callback number, and initalting a callback
 147         """
 148
 149         PHONE_TYPE_HOME = 1
 150         PHONE_TYPE_MOBILE = 2
 151         PHONE_TYPE_WORK = 3
 152         PHONE_TYPE_GIZMO = 7
 153
 154         def __init__(self, cookieFile = None):
 155                 # Important items in this function are the setup of the browser emulation and cookie file
 156                 self._browser = browser_emu.MozillaEmulator(1)
 157                 self._loadedFromCookies = self._browser.load_cookies(cookieFile)
 158
 159                 self._token = ""
 160                 self._accountNum = ""
 161                 self._lastAuthed = 0.0
 162                 self._callbackNumber = ""
 163                 self._callbackNumbers = {}
 164
 165                 # Suprisingly, moving all of these from class to self sped up startup time
 166
 167                 self._validateRe = re.compile("^\+?[0-9]{10,}$")
 168
 169                 self._loginURL = "https://www.google.com/accounts/ServiceLoginAuth"
 170
 171                 SECURE_URL_BASE = "https://www.google.com/voice/"
 172                 SECURE_MOBILE_URL_BASE = SECURE_URL_BASE + "mobile/"
 173                 self._tokenURL = SECURE_URL_BASE + "m"
 174                 self._callUrl = SECURE_URL_BASE + "call/connect"
 175                 self._callCancelURL = SECURE_URL_BASE + "call/cancel"
 176                 self._sendSmsURL = SECURE_URL_BASE + "sms/send"
 177
 178                 self._isDndURL = "https://www.google.com/voice/m/donotdisturb"
 179                 self._isDndRe = re.compile(r"""<input.*?id="doNotDisturb".*?checked="(.*?)"\s*/>""")
 180                 self._setDndURL = "https://www.google.com/voice/m/savednd"
 181
 182                 self._downloadVoicemailURL = SECURE_URL_BASE + "media/send_voicemail/"
 183                 self._markAsReadURL = SECURE_URL_BASE + "m/mark"
 184                 self._archiveMessageURL = SECURE_URL_BASE + "m/archive"
 185
 186                 self._XML_SEARCH_URL = SECURE_URL_BASE + "inbox/search/"
 187                 self._XML_ACCOUNT_URL = SECURE_URL_BASE + "contacts/"
 188                 # HACK really this redirects to the main pge and we are grabbing some javascript
 189                 self._XML_CONTACTS_URL = "http://www.google.com/voice/inbox/search/contact"
 190                 self._CSV_CONTACTS_URL = "http://mail.google.com/mail/contacts/data/export"
 191                 self._JSON_CONTACTS_URL = SECURE_URL_BASE + "b/0/request/user"
 192                 self._XML_RECENT_URL = SECURE_URL_BASE + "inbox/recent/"
 193
 194                 self.XML_FEEDS = (
 195                         'inbox', 'starred', 'all', 'spam', 'trash', 'voicemail', 'sms',
 196                         'recorded', 'placed', 'received', 'missed'
 197                 )
 198                 self._XML_INBOX_URL = SECURE_URL_BASE + "inbox/recent/inbox"
 199                 self._XML_STARRED_URL = SECURE_URL_BASE + "inbox/recent/starred"
 200                 self._XML_ALL_URL = SECURE_URL_BASE + "inbox/recent/all"
 201                 self._XML_SPAM_URL = SECURE_URL_BASE + "inbox/recent/spam"
 202                 self._XML_TRASH_URL = SECURE_URL_BASE + "inbox/recent/trash"
 203                 self._XML_VOICEMAIL_URL = SECURE_URL_BASE + "inbox/recent/voicemail/"
 204                 self._XML_SMS_URL = SECURE_URL_BASE + "inbox/recent/sms/"
 205                 self._JSON_SMS_URL = SECURE_URL_BASE + "b/0/request/messages/"
 206                 self._JSON_SMS_COUNT_URL = SECURE_URL_BASE + "b/0/request/unread"
 207                 self._XML_RECORDED_URL = SECURE_URL_BASE + "inbox/recent/recorded/"
 208                 self._XML_PLACED_URL = SECURE_URL_BASE + "inbox/recent/placed/"
 209                 self._XML_RECEIVED_URL = SECURE_URL_BASE + "inbox/recent/received/"
 210                 self._XML_MISSED_URL = SECURE_URL_BASE + "inbox/recent/missed/"
 211
 212                 self._galxRe = re.compile(r"""<input.*?name="GALX".*?value="(.*?)".*?/>""", re.MULTILINE | re.DOTALL)
 213
 214                 self._seperateVoicemailsRegex = re.compile(r"""^\s*<div id="(\w+)"\s* class=".*?gc-message.*?">""", re.MULTILINE | re.DOTALL)
 215                 self._exactVoicemailTimeRegex = re.compile(r"""<span class="gc-message-time">(.*?)</span>""", re.MULTILINE)
 216                 self._relativeVoicemailTimeRegex = re.compile(r"""<span class="gc-message-relative">(.*?)</span>""", re.MULTILINE)
 217                 self._voicemailNameRegex = re.compile(r"""<a class=.*?gc-message-name-link.*?>(.*?)</a>""", re.MULTILINE | re.DOTALL)
 218                 self._voicemailNumberRegex = re.compile(r"""<input type="hidden" class="gc-text gc-quickcall-ac" value="(.*?)"/>""", re.MULTILINE)
 219                 self._prettyVoicemailNumberRegex = re.compile(r"""<span class="gc-message-type">(.*?)</span>""", re.MULTILINE)
 220                 self._voicemailLocationRegex = re.compile(r"""<span class="gc-message-location">.*?<a.*?>(.*?)</a></span>""", re.MULTILINE)
 221                 self._messagesContactIDRegex = re.compile(r"""<a class=".*?gc-message-name-link.*?">.*?</a>\s*?<span .*?>(.*?)</span>""", re.MULTILINE)
 222                 self._voicemailMessageRegex = re.compile(r"""(<span id="\d+-\d+" class="gc-word-(.*?)">(.*?)</span>|<a .*? class="gc-message-mni">(.*?)</a>)""", re.MULTILINE)
 223                 self._smsFromRegex = re.compile(r"""<span class="gc-message-sms-from">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 224                 self._smsTimeRegex = re.compile(r"""<span class="gc-message-sms-time">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 225                 self._smsTextRegex = re.compile(r"""<span class="gc-message-sms-text">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 226
 227         def is_quick_login_possible(self):
 228                 """
 229                 @returns True then refresh_account_info might be enough to login, else full login is required
 230                 """
 231                 return self._loadedFromCookies or 0.0 < self._lastAuthed
 232
 233         def refresh_account_info(self):
 234                 try:
 235                         page = self._get_page(self._JSON_CONTACTS_URL)
 236                         accountData = self._grab_account_info(page)
 237                 except Exception, e:
 238                         _moduleLogger.exception(str(e))
 239                         return None
 240
 241                 self._browser.save_cookies()
 242                 self._lastAuthed = time.time()
 243                 return accountData
 244
 245         def _get_token(self):
 246                 tokenPage = self._get_page(self._tokenURL)
 247
 248                 galxTokens = self._galxRe.search(tokenPage)
 249                 if galxTokens is not None:
 250                         galxToken = galxTokens.group(1)
 251                 else:
 252                         galxToken = ""
 253                         _moduleLogger.debug("Could not grab GALX token")
 254                 return galxToken
 255
 256         def _login(self, username, password, token):
 257                 loginData = {
 258                         'Email' : username,
 259                         'Passwd' : password,
 260                         'service': "grandcentral",
 261                         "ltmpl": "mobile",
 262                         "btmpl": "mobile",
 263                         "PersistentCookie": "yes",
 264                         "GALX": token,
 265                         "continue": self._JSON_CONTACTS_URL,
 266                 }
 267
 268                 loginSuccessOrFailurePage = self._get_page(self._loginURL, loginData)
 269                 return loginSuccessOrFailurePage
 270
 271         def login(self, username, password):
 272                 """
 273                 Attempt to login to GoogleVoice
 274                 @returns Whether login was successful or not
 275                 @blocks
 276                 """
 277                 self.logout()
 278                 galxToken = self._get_token()
 279                 loginSuccessOrFailurePage = self._login(username, password, galxToken)
 280
 281                 try:
 282                         accountData = self._grab_account_info(loginSuccessOrFailurePage)
 283                 except Exception, e:
 284                         # Retry in case the redirect failed
 285                         # luckily refresh_account_info does everything we need for a retry
 286                         accountData = self.refresh_account_info()
 287                         if accountData is None:
 288                                 _moduleLogger.exception(str(e))
 289                                 return None
 290                         _moduleLogger.info("Redirection failed on initial login attempt, auto-corrected for this")
 291
 292                 self._browser.save_cookies()
 293                 self._lastAuthed = time.time()
 294                 return accountData
 295
 296         def persist(self):
 297                 self._browser.save_cookies()
 298
 299         def shutdown(self):
 300                 self._browser.save_cookies()
 301                 self._token = None
 302                 self._lastAuthed = 0.0
 303
 304         def logout(self):
 305                 self._browser.clear_cookies()
 306                 self._browser.save_cookies()
 307                 self._token = None
 308                 self._lastAuthed = 0.0
 309
 310         def is_dnd(self):
 311                 """
 312                 @blocks
 313                 """
 314                 isDndPage = self._get_page(self._isDndURL)
 315
 316                 dndGroup = self._isDndRe.search(isDndPage)
 317                 if dndGroup is None:
 318                         return False
 319                 dndStatus = dndGroup.group(1)
 320                 isDnd = True if dndStatus.strip().lower() == "true" else False
 321                 return isDnd
 322
 323         def set_dnd(self, doNotDisturb):
 324                 """
 325                 @blocks
 326                 """
 327                 dndPostData = {
 328                         "doNotDisturb": 1 if doNotDisturb else 0,
 329                 }
 330
 331                 dndPage = self._get_page_with_token(self._setDndURL, dndPostData)
 332
 333         def call(self, outgoingNumber):
 334                 """
 335                 This is the main function responsible for initating the callback
 336                 @blocks
 337                 """
 338                 outgoingNumber = self._send_validation(outgoingNumber)
 339                 subscriberNumber = None
 340                 phoneType = guess_phone_type(self._callbackNumber) # @todo Fix this hack
 341
 342                 callData = {
 343                                 'outgoingNumber': outgoingNumber,
 344                                 'forwardingNumber': self._callbackNumber,
 345                                 'subscriberNumber': subscriberNumber or 'undefined',
 346                                 'phoneType': str(phoneType),
 347                                 'remember': '1',
 348                 }
 349                 _moduleLogger.info("%r" % callData)
 350
 351                 page = self._get_page_with_token(
 352                         self._callUrl,
 353                         callData,
 354                 )
 355                 self._parse_with_validation(page)
 356                 return True
 357
 358         def cancel(self, outgoingNumber=None):
 359                 """
 360                 Cancels a call matching outgoing and forwarding numbers (if given).
 361                 Will raise an error if no matching call is being placed
 362                 @blocks
 363                 """
 364                 page = self._get_page_with_token(
 365                         self._callCancelURL,
 366                         {
 367                         'outgoingNumber': outgoingNumber or 'undefined',
 368                         'forwardingNumber': self._callbackNumber or 'undefined',
 369                         'cancelType': 'C2C',
 370                         },
 371                 )
 372                 self._parse_with_validation(page)
 373
 374         def send_sms(self, phoneNumbers, message):
 375                 """
 376                 @blocks
 377                 """
 378                 validatedPhoneNumbers = [
 379                         self._send_validation(phoneNumber)
 380                         for phoneNumber in phoneNumbers
 381                 ]
 382                 flattenedPhoneNumbers = ",".join(validatedPhoneNumbers)
 383                 page = self._get_page_with_token(
 384                         self._sendSmsURL,
 385                         {
 386                                 'phoneNumber': flattenedPhoneNumbers,
 387                                 'text': unicode(message).encode("utf-8"),
 388                         },
 389                 )
 390                 self._parse_with_validation(page)
 391
 392         def search(self, query):
 393                 """
 394                 Search your Google Voice Account history for calls, voicemails, and sms
 395                 Returns ``Folder`` instance containting matching messages
 396                 @blocks
 397                 """
 398                 page = self._get_page(
 399                         self._XML_SEARCH_URL,
 400                         {"q": query},
 401                 )
 402                 json, html = extract_payload(page)
 403                 return json
 404
 405         def get_feed(self, feed):
 406                 """
 407                 @blocks
 408                 """
 409                 actualFeed = "_XML_%s_URL" % feed.upper()
 410                 feedUrl = getattr(self, actualFeed)
 411
 412                 page = self._get_page(feedUrl)
 413                 json, html = extract_payload(page)
 414
 415                 return json
 416
 417         def recording_url(self, messageId):
 418                 url = self._downloadVoicemailURL+messageId
 419                 return url
 420
 421         def download(self, messageId, targetPath):
 422                 """
 423                 Download a voicemail or recorded call MP3 matching the given ``msg``
 424                 which can either be a ``Message`` instance, or a SHA1 identifier.
 425                 Message hashes can be found in ``self.voicemail().messages`` for example.
 426                 @returns location of saved file.
 427                 @blocks
 428                 """
 429                 page = self._get_page(self.recording_url(messageId))
 430                 with open(targetPath, 'wb') as fo:
 431                         fo.write(page)
 432
 433         def is_valid_syntax(self, number):
 434                 """
 435                 @returns If This number be called ( syntax validation only )
 436                 """
 437                 return self._validateRe.match(number) is not None
 438
 439         def get_account_number(self):
 440                 """
 441                 @returns The GoogleVoice phone number
 442                 """
 443                 return self._accountNum
 444
 445         def get_callback_numbers(self):
 446                 """
 447                 @returns a dictionary mapping call back numbers to descriptions
 448                 @note These results are cached for 30 minutes.
 449                 """
 450                 return self._callbackNumbers
 451
 452         def set_callback_number(self, callbacknumber):
 453                 """
 454                 Set the number that GoogleVoice calls
 455                 @param callbacknumber should be a proper 10 digit number
 456                 """
 457                 self._callbackNumber = callbacknumber
 458                 _moduleLogger.info("Callback number changed: %r" % self._callbackNumber)
 459                 return True
 460
 461         def get_callback_number(self):
 462                 """
 463                 @returns Current callback number or None
 464                 """
 465                 return self._callbackNumber
 466
 467         def get_recent(self):
 468                 """
 469                 @returns Iterable of (personsName, phoneNumber, exact date, relative date, action)
 470                 @blocks
 471                 """
 472                 recentPages = [
 473                         (action, self._get_page(url))
 474                         for action, url in (
 475                                 ("Received", self._XML_RECEIVED_URL),
 476                                 ("Missed", self._XML_MISSED_URL),
 477                                 ("Placed", self._XML_PLACED_URL),
 478                         )
 479                 ]
 480                 return self._parse_recent(recentPages)
 481
 482         def get_csv_contacts(self):
 483                 data = {
 484                         "groupToExport": "mine",
 485                         "exportType": "ALL",
 486                         "out": "OUTLOOK_CSV",
 487                 }
 488                 encodedData = urllib.urlencode(data)
 489                 contacts = self._get_page(self._CSV_CONTACTS_URL+"?"+encodedData)
 490                 return contacts
 491
 492         def get_voicemails(self):
 493                 """
 494                 @blocks
 495                 """
 496                 voicemailPage = self._get_page(self._XML_VOICEMAIL_URL)
 497                 voicemailHtml = self._grab_html(voicemailPage)
 498                 voicemailJson = self._grab_json(voicemailPage)
 499                 if voicemailJson is None:
 500                         return ()
 501                 parsedVoicemail = self._parse_voicemail(voicemailHtml)
 502                 voicemails = self._merge_conversation_sources(parsedVoicemail, voicemailJson)
 503                 return voicemails
 504
 505         def get_texts(self):
 506                 """
 507                 @blocks
 508                 """
 509                 smsPage = self._get_page(self._XML_SMS_URL)
 510                 smsHtml = self._grab_html(smsPage)
 511                 smsJson = self._grab_json(smsPage)
 512                 if smsJson is None:
 513                         return ()
 514                 parsedSms = self._parse_sms(smsHtml)
 515                 smss = self._merge_conversation_sources(parsedSms, smsJson)
 516                 return smss
 517
 518         def get_unread_counts(self):
 519                 countPage = self._get_page(self._JSON_SMS_COUNT_URL)
 520                 counts = parse_json(countPage)
 521                 counts = counts["unreadCounts"]
 522                 return counts
 523
 524         def mark_message(self, messageId, asRead):
 525                 """
 526                 @blocks
 527                 """
 528                 postData = {
 529                         "read": 1 if asRead else 0,
 530                         "id": messageId,
 531                 }
 532
 533                 markPage = self._get_page(self._markAsReadURL, postData)
 534
 535         def archive_message(self, messageId):
 536                 """
 537                 @blocks
 538                 """
 539                 postData = {
 540                         "id": messageId,
 541                 }
 542
 543                 markPage = self._get_page(self._archiveMessageURL, postData)
 544
 545         def _grab_json(self, flatXml):
 546                 xmlTree = ElementTree.fromstring(flatXml)
 547                 jsonElement = xmlTree.getchildren()[0]
 548                 flatJson = jsonElement.text
 549                 jsonTree = parse_json(flatJson)
 550                 return jsonTree
 551
 552         def _grab_html(self, flatXml):
 553                 xmlTree = ElementTree.fromstring(flatXml)
 554                 htmlElement = xmlTree.getchildren()[1]
 555                 flatHtml = htmlElement.text
 556                 return flatHtml
 557
 558         def _grab_account_info(self, page):
 559                 accountData = parse_json(page)
 560                 self._token = accountData["r"]
 561                 self._accountNum = accountData["number"]["raw"]
 562                 for callback in accountData["phones"].itervalues():
 563                         self._callbackNumbers[callback["phoneNumber"]] = callback["name"]
 564                 if len(self._callbackNumbers) == 0:
 565                         _moduleLogger.debug("Could not extract callback numbers from GoogleVoice (the troublesome page follows):\n%s" % page)
 566                 return accountData
 567
 568         def _send_validation(self, number):
 569                 if not self.is_valid_syntax(number):
 570                         raise ValueError('Number is not valid: "%s"' % number)
 571                 return number
 572
 573         def _parse_recent(self, recentPages):
 574                 for action, flatXml in recentPages:
 575                         allRecentHtml = self._grab_html(flatXml)
 576                         allRecentData = self._parse_history(allRecentHtml)
 577                         for recentCallData in allRecentData:
 578                                 recentCallData["action"] = action
 579                                 yield recentCallData
 580
 581         def _parse_history(self, historyHtml):
 582                 splitVoicemail = self._seperateVoicemailsRegex.split(historyHtml)
 583                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 584                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 585                         exactTime = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 586                         exactTime = google_strptime(exactTime)
 587                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 588                         relativeTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 589                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 590                         location = locationGroup.group(1).strip() if locationGroup else ""
 591
 592                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 593                         name = nameGroup.group(1).strip() if nameGroup else ""
 594                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 595                         number = numberGroup.group(1).strip() if numberGroup else ""
 596                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 597                         prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 598                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 599                         contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 600
 601                         yield {
 602                                 "id": messageId.strip(),
 603                                 "contactId": contactId,
 604                                 "name": unescape(name),
 605                                 "time": exactTime,
 606                                 "relTime": relativeTime,
 607                                 "prettyNumber": prettyNumber,
 608                                 "number": number,
 609                                 "location": unescape(location),
 610                         }
 611
 612         @staticmethod
 613         def _interpret_voicemail_regex(group):
 614                 quality, content, number = group.group(2), group.group(3), group.group(4)
 615                 text = MessageText()
 616                 if quality is not None and content is not None:
 617                         text.accuracy = quality
 618                         text.text = unescape(content)
 619                         return text
 620                 elif number is not None:
 621                         text.accuracy = MessageText.ACCURACY_HIGH
 622                         text.text = number
 623                         return text
 624
 625         def _parse_voicemail(self, voicemailHtml):
 626                 splitVoicemail = self._seperateVoicemailsRegex.split(voicemailHtml)
 627                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 628                         conv = Conversation()
 629                         conv.type = Conversation.TYPE_VOICEMAIL
 630                         conv.id = messageId.strip()
 631
 632                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 633                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 634                         conv.time = google_strptime(exactTimeText)
 635                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 636                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 637                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 638                         conv.location = unescape(locationGroup.group(1).strip() if locationGroup else "")
 639
 640                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 641                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 642                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 643                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 644                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 645                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 646                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 647                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 648
 649                         messageGroups = self._voicemailMessageRegex.finditer(messageHtml)
 650                         messageParts = [
 651                                 self._interpret_voicemail_regex(group)
 652                                 for group in messageGroups
 653                         ] if messageGroups else ((MessageText.ACCURACY_LOW, "No Transcription"), )
 654                         message = Message()
 655                         message.body = messageParts
 656                         message.whoFrom = conv.name
 657                         try:
 658                                 message.when = conv.time.strftime("%I:%M %p")
 659                         except ValueError:
 660                                 _moduleLogger.exception("Confusing time provided: %r" % conv.time)
 661                                 message.when = "Unknown"
 662                         conv.messages = (message, )
 663
 664                         yield conv
 665
 666         @staticmethod
 667         def _interpret_sms_message_parts(fromPart, textPart, timePart):
 668                 text = MessageText()
 669                 text.accuracy = MessageText.ACCURACY_MEDIUM
 670                 text.text = unescape(textPart)
 671
 672                 message = Message()
 673                 message.body = (text, )
 674                 message.whoFrom = fromPart
 675                 message.when = timePart
 676
 677                 return message
 678
 679         def _parse_sms(self, smsHtml):
 680                 splitSms = self._seperateVoicemailsRegex.split(smsHtml)
 681                 for messageId, messageHtml in itergroup(splitSms[1:], 2):
 682                         conv = Conversation()
 683                         conv.type = Conversation.TYPE_SMS
 684                         conv.id = messageId.strip()
 685
 686                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 687                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 688                         conv.time = google_strptime(exactTimeText)
 689                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 690                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 691                         conv.location = ""
 692
 693                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 694                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 695                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 696                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 697                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 698                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 699                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 700                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 701
 702                         fromGroups = self._smsFromRegex.finditer(messageHtml)
 703                         fromParts = (group.group(1).strip() for group in fromGroups)
 704                         textGroups = self._smsTextRegex.finditer(messageHtml)
 705                         textParts = (group.group(1).strip() for group in textGroups)
 706                         timeGroups = self._smsTimeRegex.finditer(messageHtml)
 707                         timeParts = (group.group(1).strip() for group in timeGroups)
 708
 709                         messageParts = itertools.izip(fromParts, textParts, timeParts)
 710                         messages = [self._interpret_sms_message_parts(*parts) for parts in messageParts]
 711                         conv.messages = messages
 712
 713                         yield conv
 714
 715         @staticmethod
 716         def _merge_conversation_sources(parsedMessages, json):
 717                 for message in parsedMessages:
 718                         jsonItem = json["messages"][message.id]
 719                         message.isRead = jsonItem["isRead"]
 720                         message.isSpam = jsonItem["isSpam"]
 721                         message.isTrash = jsonItem["isTrash"]
 722                         message.isArchived = "inbox" not in jsonItem["labels"]
 723                         yield message
 724
 725         def _get_page(self, url, data = None, refererUrl = None):
 726                 headers = {}
 727                 if refererUrl is not None:
 728                         headers["Referer"] = refererUrl
 729
 730                 encodedData = urllib.urlencode(data) if data is not None else None
 731
 732                 try:
 733                         page = self._browser.download(url, encodedData, None, headers)
 734                 except urllib2.URLError, e:
 735                         _moduleLogger.error("Translating error: %s" % str(e))
 736                         raise NetworkError("%s is not accesible" % url)
 737
 738                 return page
 739
 740         def _get_page_with_token(self, url, data = None, refererUrl = None):
 741                 if data is None:
 742                         data = {}
 743                 data['_rnr_se'] = self._token
 744
 745                 page = self._get_page(url, data, refererUrl)
 746
 747                 return page
 748
 749         def _parse_with_validation(self, page):
 750                 json = parse_json(page)
 751                 self._validate_response(json)
 752                 return json
 753
 754         def _validate_response(self, response):
 755                 """
 756                 Validates that the JSON response is A-OK
 757                 """
 758                 try:
 759                         assert response is not None, "Response not provided"
 760                         assert 'ok' in response, "Response lacks status"
 761                         assert response['ok'], "Response not good"
 762                 except AssertionError:
 763                         try:
 764                                 if response["data"]["code"] == 20:
 765                                         raise RuntimeError(
 766 """Ambiguous error 20 returned by Google Voice.
 767 Please verify you have configured your callback number (currently "%s").  If it is configured some other suspected causes are: non-verified callback numbers, and Gizmo5 callback numbers.""" % self._callbackNumber)
 768                         except KeyError:
 769                                 pass
 770                         raise RuntimeError('There was a problem with GV: %s' % response)
 771
 772
 773 _UNESCAPE_ENTITIES = {
 774  "&quot;": '"',
 775  "&nbsp;": " ",
 776  "&#39;": "'",
 777 }
 778
 779
 780 def unescape(text):
 781         plain = saxutils.unescape(text, _UNESCAPE_ENTITIES)
 782         return plain
 783
 784
 785 def google_strptime(time):
 786         """
 787         Hack: Google always returns the time in the same locale.  Sadly if the
 788         local system's locale is different, there isn't a way to perfectly handle
 789         the time.  So instead we handle implement some time formatting
 790         """
 791         abbrevTime = time[:-3]
 792         parsedTime = datetime.datetime.strptime(abbrevTime, "%m/%d/%y %I:%M")
 793         if time.endswith("PM"):
 794                 parsedTime += datetime.timedelta(hours=12)
 795         return parsedTime
 796
 797
 798 def itergroup(iterator, count, padValue = None):
 799         """
 800         Iterate in groups of 'count' values. If there
 801         aren't enough values, the last result is padded with
 802         None.
 803
 804         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 805         ...     print tuple(val)
 806         (1, 2, 3)
 807         (4, 5, 6)
 808         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 809         ...     print list(val)
 810         [1, 2, 3]
 811         [4, 5, 6]
 812         >>> for val in itergroup([1, 2, 3, 4, 5, 6, 7], 3):
 813         ...     print tuple(val)
 814         (1, 2, 3)
 815         (4, 5, 6)
 816         (7, None, None)
 817         >>> for val in itergroup("123456", 3):
 818         ...     print tuple(val)
 819         ('1', '2', '3')
 820         ('4', '5', '6')
 821         >>> for val in itergroup("123456", 3):
 822         ...     print repr("".join(val))
 823         '123'
 824         '456'
 825         """
 826         paddedIterator = itertools.chain(iterator, itertools.repeat(padValue, count-1))
 827         nIterators = (paddedIterator, ) * count
 828         return itertools.izip(*nIterators)
 829
 830
 831 def safe_eval(s):
 832         _TRUE_REGEX = re.compile("true")
 833         _FALSE_REGEX = re.compile("false")
 834         _COMMENT_REGEX = re.compile("^\s+//.*$", re.M)
 835         s = _TRUE_REGEX.sub("True", s)
 836         s = _FALSE_REGEX.sub("False", s)
 837         s = _COMMENT_REGEX.sub("#", s)
 838         try:
 839                 results = eval(s, {}, {})
 840         except SyntaxError:
 841                 _moduleLogger.exception("Oops")
 842                 results = None
 843         return results
 844
 845
 846 def _fake_parse_json(flattened):
 847         return safe_eval(flattened)
 848
 849
 850 def _actual_parse_json(flattened):
 851         return simplejson.loads(flattened)
 852
 853
 854 if simplejson is None:
 855         parse_json = _fake_parse_json
 856 else:
 857         parse_json = _actual_parse_json
 858
 859
 860 def extract_payload(flatXml):
 861         xmlTree = ElementTree.fromstring(flatXml)
 862
 863         jsonElement = xmlTree.getchildren()[0]
 864         flatJson = jsonElement.text
 865         jsonTree = parse_json(flatJson)
 866
 867         htmlElement = xmlTree.getchildren()[1]
 868         flatHtml = htmlElement.text
 869
 870         return jsonTree, flatHtml
 871
 872
 873 def guess_phone_type(number):
 874         if number.startswith("747") or number.startswith("1747") or number.startswith("+1747"):
 875                 return GVoiceBackend.PHONE_TYPE_GIZMO
 876         else:
 877                 return GVoiceBackend.PHONE_TYPE_MOBILE
 878
 879
 880 def get_sane_callback(backend):
 881         """
 882         Try to set a sane default callback number on these preferences
 883         1) 1747 numbers ( Gizmo )
 884         2) anything with gizmo in the name
 885         3) anything with computer in the name
 886         4) the first value
 887         """
 888         numbers = backend.get_callback_numbers()
 889
 890         priorityOrderedCriteria = [
 891                 ("\+1747", None),
 892                 ("1747", None),
 893                 ("747", None),
 894                 (None, "gizmo"),
 895                 (None, "computer"),
 896                 (None, "sip"),
 897                 (None, None),
 898         ]
 899
 900         for numberCriteria, descriptionCriteria in priorityOrderedCriteria:
 901                 numberMatcher = None
 902                 descriptionMatcher = None
 903                 if numberCriteria is not None:
 904                         numberMatcher = re.compile(numberCriteria)
 905                 elif descriptionCriteria is not None:
 906                         descriptionMatcher = re.compile(descriptionCriteria, re.I)
 907
 908                 for number, description in numbers.iteritems():
 909                         if numberMatcher is not None and numberMatcher.match(number) is None:
 910                                 continue
 911                         if descriptionMatcher is not None and descriptionMatcher.match(description) is None:
 912                                 continue
 913                         return number
 914
 915
 916 def set_sane_callback(backend):
 917         """
 918         Try to set a sane default callback number on these preferences
 919         1) 1747 numbers ( Gizmo )
 920         2) anything with gizmo in the name
 921         3) anything with computer in the name
 922         4) the first value
 923         """
 924         number = get_sane_callback(backend)
 925         backend.set_callback_number(number)
 926
 927
 928 def _is_not_special(name):
 929         return not name.startswith("_") and name[0].lower() == name[0] and "_" not in name
 930
 931
 932 def to_dict(obj):
 933         members = inspect.getmembers(obj)
 934         return dict((name, value) for (name, value) in members if _is_not_special(name))
 935
 936
 937 def grab_debug_info(username, password):
 938         cookieFile = os.path.join(".", "raw_cookies.txt")
 939         try:
 940                 os.remove(cookieFile)
 941         except OSError:
 942                 pass
 943
 944         backend = GVoiceBackend(cookieFile)
 945         browser = backend._browser
 946
 947         _TEST_WEBPAGES = [
 948                 ("token", backend._tokenURL),
 949                 ("login", backend._loginURL),
 950                 ("isdnd", backend._isDndURL),
 951                 ("account", backend._XML_ACCOUNT_URL),
 952                 ("html_contacts", backend._XML_CONTACTS_URL),
 953                 ("contacts", backend._JSON_CONTACTS_URL),
 954                 ("csv", backend._CSV_CONTACTS_URL),
 955
 956                 ("voicemail", backend._XML_VOICEMAIL_URL),
 957                 ("html_sms", backend._XML_SMS_URL),
 958                 ("sms", backend._JSON_SMS_URL),
 959                 ("count", backend._JSON_SMS_COUNT_URL),
 960
 961                 ("recent", backend._XML_RECENT_URL),
 962                 ("placed", backend._XML_PLACED_URL),
 963                 ("recieved", backend._XML_RECEIVED_URL),
 964                 ("missed", backend._XML_MISSED_URL),
 965         ]
 966
 967         # Get Pages
 968         print "Grabbing pre-login pages"
 969         for name, url in _TEST_WEBPAGES:
 970                 try:
 971                         page = browser.download(url)
 972                 except StandardError, e:
 973                         print e.message
 974                         continue
 975                 print "\tWriting to file"
 976                 with open("not_loggedin_%s.txt" % name, "w") as f:
 977                         f.write(page)
 978
 979         # Login
 980         print "Attempting login"
 981         galxToken = backend._get_token()
 982         loginSuccessOrFailurePage = backend._login(username, password, galxToken)
 983         with open("loggingin.txt", "w") as f:
 984                 print "\tWriting to file"
 985                 f.write(loginSuccessOrFailurePage)
 986         try:
 987                 backend._grab_account_info(loginSuccessOrFailurePage)
 988         except Exception:
 989                 # Retry in case the redirect failed
 990                 # luckily refresh_account_info does everything we need for a retry
 991                 loggedIn = backend.refresh_account_info() is not None
 992                 if not loggedIn:
 993                         raise
 994
 995         # Get Pages
 996         print "Grabbing post-login pages"
 997         for name, url in _TEST_WEBPAGES:
 998                 try:
 999                         page = browser.download(url)
1000                 except StandardError, e:
1001                         print str(e)
1002                         continue
1003                 print "\tWriting to file"
1004                 with open("loggedin_%s.txt" % name, "w") as f:
1005                         f.write(page)
1006
1007         # Cookies
1008         browser.save_cookies()
1009         print "\tWriting cookies to file"
1010         with open("cookies.txt", "w") as f:
1011                 f.writelines(
1012                         "%s: %s\n" % (c.name, c.value)
1013                         for c in browser._cookies
1014                 )
1015
1016
1017 def grab_voicemails(username, password):
1018         cookieFile = os.path.join(".", "raw_cookies.txt")
1019         try:
1020                 os.remove(cookieFile)
1021         except OSError:
1022                 pass
1023
1024         backend = GVoiceBackend(cookieFile)
1025         backend.login(username, password)
1026         voicemails = list(backend.get_voicemails())
1027         for voicemail in voicemails:
1028                 print voicemail.id
1029                 backend.download(voicemail.id, ".")
1030
1031
1032 def main():
1033         import sys
1034         logging.basicConfig(level=logging.DEBUG)
1035         args = sys.argv
1036         if 3 <= len(args):
1037                 username = args[1]
1038                 password = args[2]
1039
1040         grab_debug_info(username, password)
1041         grab_voicemails(username, password)
1042
1043
1044 if __name__ == "__main__":
1045         main()