git.maemo.org Git - gc-dialer/blob - dialcentral/backends/gvoice/gvoice.py

   1 #!/usr/bin/python
   2
   3 """
   4 DialCentral - Front end for Google's GoogleVoice service.
   5 Copyright (C) 2008  Eric Warnke ericew AT gmail DOT com
   6
   7 This library is free software; you can redistribute it and/or
   8 modify it under the terms of the GNU Lesser General Public
   9 License as published by the Free Software Foundation; either
  10 version 2.1 of the License, or (at your option) any later version.
  11
  12 This library is distributed in the hope that it will be useful,
  13 but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 Lesser General Public License for more details.
  16
  17 You should have received a copy of the GNU Lesser General Public
  18 License along with this library; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 Google Voice backend code
  22
  23 Resources
  24         http://thatsmith.com/2009/03/google-voice-addon-for-firefox/
  25         http://posttopic.com/topic/google-voice-add-on-development
  26 """
  27
  28 from __future__ import with_statement
  29
  30 import os
  31 import re
  32 import urllib
  33 import urllib2
  34 import time
  35 import datetime
  36 import itertools
  37 import logging
  38 import inspect
  39
  40 from xml.sax import saxutils
  41 from xml.etree import ElementTree
  42
  43 try:
  44         import json as _json
  45         simplejson = _json
  46 except ImportError:
  47         try:
  48                 import simplejson as _simplejson
  49                 simplejson = _simplejson
  50         except ImportError:
  51                 simplejson = None
  52
  53 import browser_emu
  54
  55
  56 _moduleLogger = logging.getLogger(__name__)
  57
  58
  59 class NetworkError(RuntimeError):
  60         pass
  61
  62
  63 class MessageText(object):
  64
  65         ACCURACY_LOW = "med1"
  66         ACCURACY_MEDIUM = "med2"
  67         ACCURACY_HIGH = "high"
  68
  69         def __init__(self):
  70                 self.accuracy = None
  71                 self.text = None
  72
  73         def __str__(self):
  74                 return self.text
  75
  76         def to_dict(self):
  77                 return to_dict(self)
  78
  79         def __eq__(self, other):
  80                 return self.accuracy == other.accuracy and self.text == other.text
  81
  82
  83 class Message(object):
  84
  85         def __init__(self):
  86                 self.whoFrom = None
  87                 self.body = None
  88                 self.when = None
  89
  90         def __str__(self):
  91                 return "%s (%s): %s" % (
  92                         self.whoFrom,
  93                         self.when,
  94                         "".join(unicode(part) for part in self.body)
  95                 )
  96
  97         def to_dict(self):
  98                 selfDict = to_dict(self)
  99                 selfDict["body"] = [text.to_dict() for text in self.body] if self.body is not None else None
 100                 return selfDict
 101
 102         def __eq__(self, other):
 103                 return self.whoFrom == other.whoFrom and self.when == other.when and self.body == other.body
 104
 105
 106 class Conversation(object):
 107
 108         TYPE_VOICEMAIL = "Voicemail"
 109         TYPE_SMS = "SMS"
 110
 111         def __init__(self):
 112                 self.type = None
 113                 self.id = None
 114                 self.contactId = None
 115                 self.name = None
 116                 self.location = None
 117                 self.prettyNumber = None
 118                 self.number = None
 119
 120                 self.time = None
 121                 self.relTime = None
 122                 self.messages = None
 123                 self.isRead = None
 124                 self.isSpam = None
 125                 self.isTrash = None
 126                 self.isArchived = None
 127
 128         def __cmp__(self, other):
 129                 cmpValue = cmp(self.contactId, other.contactId)
 130                 if cmpValue != 0:
 131                         return cmpValue
 132
 133                 cmpValue = cmp(self.time, other.time)
 134                 if cmpValue != 0:
 135                         return cmpValue
 136
 137                 cmpValue = cmp(self.id, other.id)
 138                 if cmpValue != 0:
 139                         return cmpValue
 140
 141         def to_dict(self):
 142                 selfDict = to_dict(self)
 143                 selfDict["messages"] = [message.to_dict() for message in self.messages] if self.messages is not None else None
 144                 return selfDict
 145
 146
 147 class GVoiceBackend(object):
 148         """
 149         This class encapsulates all of the knowledge necessary to interact with the GoogleVoice servers
 150         the functions include login, setting up a callback number, and initalting a callback
 151         """
 152
 153         PHONE_TYPE_HOME = 1
 154         PHONE_TYPE_MOBILE = 2
 155         PHONE_TYPE_WORK = 3
 156         PHONE_TYPE_GIZMO = 7
 157
 158         def __init__(self, cookieFile = None):
 159                 # Important items in this function are the setup of the browser emulation and cookie file
 160                 self._browser = browser_emu.MozillaEmulator(1)
 161                 self._loadedFromCookies = self._browser.load_cookies(cookieFile)
 162
 163                 self._token = ""
 164                 self._accountNum = ""
 165                 self._lastAuthed = 0.0
 166                 self._callbackNumber = ""
 167                 self._callbackNumbers = {}
 168
 169                 # Suprisingly, moving all of these from class to self sped up startup time
 170
 171                 self._validateRe = re.compile("^\+?[0-9]{10,}$")
 172
 173                 self._loginURL = "https://www.google.com/accounts/ServiceLoginAuth"
 174
 175                 SECURE_URL_BASE = "https://www.google.com/voice/"
 176                 SECURE_MOBILE_URL_BASE = SECURE_URL_BASE + "mobile/"
 177                 self._tokenURL = SECURE_URL_BASE + "m"
 178                 self._callUrl = SECURE_URL_BASE + "call/connect"
 179                 self._callCancelURL = SECURE_URL_BASE + "call/cancel"
 180                 self._sendSmsURL = SECURE_URL_BASE + "sms/send"
 181
 182                 self._isDndURL = "https://www.google.com/voice/m/donotdisturb"
 183                 self._isDndRe = re.compile(r"""<input.*?id="doNotDisturb".*?checked="(.*?)"\s*/>""")
 184                 self._setDndURL = "https://www.google.com/voice/m/savednd"
 185
 186                 self._downloadVoicemailURL = SECURE_URL_BASE + "media/send_voicemail/"
 187                 self._markAsReadURL = SECURE_URL_BASE + "m/mark"
 188                 self._archiveMessageURL = SECURE_URL_BASE + "m/archive"
 189
 190                 self._XML_SEARCH_URL = SECURE_URL_BASE + "inbox/search/"
 191                 self._XML_ACCOUNT_URL = SECURE_URL_BASE + "contacts/"
 192                 # HACK really this redirects to the main pge and we are grabbing some javascript
 193                 self._XML_CONTACTS_URL = "http://www.google.com/voice/inbox/search/contact"
 194                 self._CSV_CONTACTS_URL = "http://mail.google.com/mail/contacts/data/export"
 195                 self._JSON_CONTACTS_URL = SECURE_URL_BASE + "b/0/request/user"
 196                 self._XML_RECENT_URL = SECURE_URL_BASE + "inbox/recent/"
 197
 198                 self.XML_FEEDS = (
 199                         'inbox', 'starred', 'all', 'spam', 'trash', 'voicemail', 'sms',
 200                         'recorded', 'placed', 'received', 'missed'
 201                 )
 202                 self._XML_INBOX_URL = SECURE_URL_BASE + "inbox/recent/inbox"
 203                 self._XML_STARRED_URL = SECURE_URL_BASE + "inbox/recent/starred"
 204                 self._XML_ALL_URL = SECURE_URL_BASE + "inbox/recent/all"
 205                 self._XML_SPAM_URL = SECURE_URL_BASE + "inbox/recent/spam"
 206                 self._XML_TRASH_URL = SECURE_URL_BASE + "inbox/recent/trash"
 207                 self._XML_VOICEMAIL_URL = SECURE_URL_BASE + "inbox/recent/voicemail/"
 208                 self._XML_SMS_URL = SECURE_URL_BASE + "inbox/recent/sms/"
 209                 self._JSON_SMS_URL = SECURE_URL_BASE + "b/0/request/messages/"
 210                 self._JSON_SMS_COUNT_URL = SECURE_URL_BASE + "b/0/request/unread"
 211                 self._XML_RECORDED_URL = SECURE_URL_BASE + "inbox/recent/recorded/"
 212                 self._XML_PLACED_URL = SECURE_URL_BASE + "inbox/recent/placed/"
 213                 self._XML_RECEIVED_URL = SECURE_URL_BASE + "inbox/recent/received/"
 214                 self._XML_MISSED_URL = SECURE_URL_BASE + "inbox/recent/missed/"
 215
 216                 self._galxRe = re.compile(r"""<input.*?name="GALX".*?value="(.*?)".*?/>""", re.MULTILINE | re.DOTALL)
 217
 218                 self._seperateVoicemailsRegex = re.compile(r"""^\s*<div id="(\w+)"\s* class=".*?gc-message.*?">""", re.MULTILINE | re.DOTALL)
 219                 self._exactVoicemailTimeRegex = re.compile(r"""<span class="gc-message-time">(.*?)</span>""", re.MULTILINE)
 220                 self._relativeVoicemailTimeRegex = re.compile(r"""<span class="gc-message-relative">(.*?)</span>""", re.MULTILINE)
 221                 self._voicemailNameRegex = re.compile(r"""<a class=.*?gc-message-name-link.*?>(.*?)</a>""", re.MULTILINE | re.DOTALL)
 222                 self._voicemailNumberRegex = re.compile(r"""<input type="hidden" class="gc-text gc-quickcall-ac" value="(.*?)"/>""", re.MULTILINE)
 223                 self._prettyVoicemailNumberRegex = re.compile(r"""<span class="gc-message-type">(.*?)</span>""", re.MULTILINE)
 224                 self._voicemailLocationRegex = re.compile(r"""<span class="gc-message-location">.*?<a.*?>(.*?)</a></span>""", re.MULTILINE)
 225                 self._messagesContactIDRegex = re.compile(r"""<a class=".*?gc-message-name-link.*?">.*?</a>\s*?<span .*?>(.*?)</span>""", re.MULTILINE)
 226                 self._voicemailMessageRegex = re.compile(r"""(<span id="\d+-\d+" class="gc-word-(.*?)">(.*?)</span>|<a .*? class="gc-message-mni">(.*?)</a>)""", re.MULTILINE)
 227                 self._smsFromRegex = re.compile(r"""<span class="gc-message-sms-from">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 228                 self._smsTimeRegex = re.compile(r"""<span class="gc-message-sms-time">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 229                 self._smsTextRegex = re.compile(r"""<span class="gc-message-sms-text">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 230
 231         def is_quick_login_possible(self):
 232                 """
 233                 @returns True then refresh_account_info might be enough to login, else full login is required
 234                 """
 235                 return self._loadedFromCookies or 0.0 < self._lastAuthed
 236
 237         def refresh_account_info(self):
 238                 try:
 239                         page = self._get_page(self._JSON_CONTACTS_URL)
 240                         accountData = self._grab_account_info(page)
 241                 except Exception, e:
 242                         _moduleLogger.exception(str(e))
 243                         return None
 244
 245                 self._browser.save_cookies()
 246                 self._lastAuthed = time.time()
 247                 return accountData
 248
 249         def _get_token(self):
 250                 tokenPage = self._get_page(self._tokenURL)
 251
 252                 galxTokens = self._galxRe.search(tokenPage)
 253                 if galxTokens is not None:
 254                         galxToken = galxTokens.group(1)
 255                 else:
 256                         galxToken = ""
 257                         _moduleLogger.debug("Could not grab GALX token")
 258                 return galxToken
 259
 260         def _login(self, username, password, token):
 261                 loginData = {
 262                         'Email' : username,
 263                         'Passwd' : password,
 264                         'service': "grandcentral",
 265                         "ltmpl": "mobile",
 266                         "btmpl": "mobile",
 267                         "PersistentCookie": "yes",
 268                         "GALX": token,
 269                         "continue": self._JSON_CONTACTS_URL,
 270                 }
 271
 272                 loginSuccessOrFailurePage = self._get_page(self._loginURL, loginData)
 273                 return loginSuccessOrFailurePage
 274
 275         def login(self, username, password):
 276                 """
 277                 Attempt to login to GoogleVoice
 278                 @returns Whether login was successful or not
 279                 @blocks
 280                 """
 281                 self.logout()
 282                 galxToken = self._get_token()
 283                 loginSuccessOrFailurePage = self._login(username, password, galxToken)
 284
 285                 try:
 286                         accountData = self._grab_account_info(loginSuccessOrFailurePage)
 287                 except Exception, e:
 288                         # Retry in case the redirect failed
 289                         # luckily refresh_account_info does everything we need for a retry
 290                         accountData = self.refresh_account_info()
 291                         if accountData is None:
 292                                 _moduleLogger.exception(str(e))
 293                                 return None
 294                         _moduleLogger.info("Redirection failed on initial login attempt, auto-corrected for this")
 295
 296                 self._browser.save_cookies()
 297                 self._lastAuthed = time.time()
 298                 return accountData
 299
 300         def persist(self):
 301                 self._browser.save_cookies()
 302
 303         def shutdown(self):
 304                 self._browser.save_cookies()
 305                 self._token = None
 306                 self._lastAuthed = 0.0
 307
 308         def logout(self):
 309                 self._browser.clear_cookies()
 310                 self._browser.save_cookies()
 311                 self._token = None
 312                 self._lastAuthed = 0.0
 313                 self._callbackNumbers = {}
 314
 315         def is_dnd(self):
 316                 """
 317                 @blocks
 318                 """
 319                 isDndPage = self._get_page(self._isDndURL)
 320
 321                 dndGroup = self._isDndRe.search(isDndPage)
 322                 if dndGroup is None:
 323                         return False
 324                 dndStatus = dndGroup.group(1)
 325                 isDnd = True if dndStatus.strip().lower() == "true" else False
 326                 return isDnd
 327
 328         def set_dnd(self, doNotDisturb):
 329                 """
 330                 @blocks
 331                 """
 332                 dndPostData = {
 333                         "doNotDisturb": 1 if doNotDisturb else 0,
 334                 }
 335
 336                 dndPage = self._get_page_with_token(self._setDndURL, dndPostData)
 337
 338         def call(self, outgoingNumber):
 339                 """
 340                 This is the main function responsible for initating the callback
 341                 @blocks
 342                 """
 343                 outgoingNumber = self._send_validation(outgoingNumber)
 344                 subscriberNumber = None
 345                 phoneType = guess_phone_type(self._callbackNumber) # @todo Fix this hack
 346
 347                 callData = {
 348                                 'outgoingNumber': outgoingNumber,
 349                                 'forwardingNumber': self._callbackNumber,
 350                                 'subscriberNumber': subscriberNumber or 'undefined',
 351                                 'phoneType': str(phoneType),
 352                                 'remember': '1',
 353                 }
 354                 _moduleLogger.info("%r" % callData)
 355
 356                 page = self._get_page_with_token(
 357                         self._callUrl,
 358                         callData,
 359                 )
 360                 self._parse_with_validation(page)
 361                 return True
 362
 363         def cancel(self, outgoingNumber=None):
 364                 """
 365                 Cancels a call matching outgoing and forwarding numbers (if given).
 366                 Will raise an error if no matching call is being placed
 367                 @blocks
 368                 """
 369                 page = self._get_page_with_token(
 370                         self._callCancelURL,
 371                         {
 372                         'outgoingNumber': outgoingNumber or 'undefined',
 373                         'forwardingNumber': self._callbackNumber or 'undefined',
 374                         'cancelType': 'C2C',
 375                         },
 376                 )
 377                 self._parse_with_validation(page)
 378
 379         def send_sms(self, phoneNumbers, message):
 380                 """
 381                 @blocks
 382                 """
 383                 validatedPhoneNumbers = [
 384                         self._send_validation(phoneNumber)
 385                         for phoneNumber in phoneNumbers
 386                 ]
 387                 flattenedPhoneNumbers = ",".join(validatedPhoneNumbers)
 388                 page = self._get_page_with_token(
 389                         self._sendSmsURL,
 390                         {
 391                                 'phoneNumber': flattenedPhoneNumbers,
 392                                 'text': unicode(message).encode("utf-8"),
 393                         },
 394                 )
 395                 self._parse_with_validation(page)
 396
 397         def search(self, query):
 398                 """
 399                 Search your Google Voice Account history for calls, voicemails, and sms
 400                 Returns ``Folder`` instance containting matching messages
 401                 @blocks
 402                 """
 403                 page = self._get_page(
 404                         self._XML_SEARCH_URL,
 405                         {"q": query},
 406                 )
 407                 json, html = extract_payload(page)
 408                 return json
 409
 410         def get_feed(self, feed):
 411                 """
 412                 @blocks
 413                 """
 414                 actualFeed = "_XML_%s_URL" % feed.upper()
 415                 feedUrl = getattr(self, actualFeed)
 416
 417                 page = self._get_page(feedUrl)
 418                 json, html = extract_payload(page)
 419
 420                 return json
 421
 422         def recording_url(self, messageId):
 423                 url = self._downloadVoicemailURL+messageId
 424                 return url
 425
 426         def download(self, messageId, targetPath):
 427                 """
 428                 Download a voicemail or recorded call MP3 matching the given ``msg``
 429                 which can either be a ``Message`` instance, or a SHA1 identifier.
 430                 Message hashes can be found in ``self.voicemail().messages`` for example.
 431                 @returns location of saved file.
 432                 @blocks
 433                 """
 434                 page = self._get_page(self.recording_url(messageId))
 435                 with open(targetPath, 'wb') as fo:
 436                         fo.write(page)
 437
 438         def is_valid_syntax(self, number):
 439                 """
 440                 @returns If This number be called ( syntax validation only )
 441                 """
 442                 return self._validateRe.match(number) is not None
 443
 444         def get_account_number(self):
 445                 """
 446                 @returns The GoogleVoice phone number
 447                 """
 448                 return self._accountNum
 449
 450         def get_callback_numbers(self):
 451                 """
 452                 @returns a dictionary mapping call back numbers to descriptions
 453                 @note These results are cached for 30 minutes.
 454                 """
 455                 return self._callbackNumbers
 456
 457         def set_callback_number(self, callbacknumber):
 458                 """
 459                 Set the number that GoogleVoice calls
 460                 @param callbacknumber should be a proper 10 digit number
 461                 """
 462                 self._callbackNumber = callbacknumber
 463                 _moduleLogger.info("Callback number changed: %r" % self._callbackNumber)
 464                 return True
 465
 466         def get_callback_number(self):
 467                 """
 468                 @returns Current callback number or None
 469                 """
 470                 return self._callbackNumber
 471
 472         def get_received_calls(self):
 473                 """
 474                 @returns Iterable of (personsName, phoneNumber, exact date, relative date, action)
 475                 @blocks
 476                 """
 477                 return self._parse_recent(self._get_page(self._XML_RECEIVED_URL))
 478
 479         def get_missed_calls(self):
 480                 """
 481                 @returns Iterable of (personsName, phoneNumber, exact date, relative date, action)
 482                 @blocks
 483                 """
 484                 return self._parse_recent(self._get_page(self._XML_MISSED_URL))
 485
 486         def get_placed_calls(self):
 487                 """
 488                 @returns Iterable of (personsName, phoneNumber, exact date, relative date, action)
 489                 @blocks
 490                 """
 491                 return self._parse_recent(self._get_page(self._XML_PLACED_URL))
 492
 493         def get_csv_contacts(self):
 494                 data = {
 495                         "groupToExport": "mine",
 496                         "exportType": "ALL",
 497                         "out": "OUTLOOK_CSV",
 498                 }
 499                 encodedData = urllib.urlencode(data)
 500                 contacts = self._get_page(self._CSV_CONTACTS_URL+"?"+encodedData)
 501                 return contacts
 502
 503         def get_voicemails(self):
 504                 """
 505                 @blocks
 506                 """
 507                 voicemailPage = self._get_page(self._XML_VOICEMAIL_URL)
 508                 voicemailHtml = self._grab_html(voicemailPage)
 509                 voicemailJson = self._grab_json(voicemailPage)
 510                 if voicemailJson is None:
 511                         return ()
 512                 parsedVoicemail = self._parse_voicemail(voicemailHtml)
 513                 voicemails = self._merge_conversation_sources(parsedVoicemail, voicemailJson)
 514                 return voicemails
 515
 516         def get_texts(self):
 517                 """
 518                 @blocks
 519                 """
 520                 smsPage = self._get_page(self._XML_SMS_URL)
 521                 smsHtml = self._grab_html(smsPage)
 522                 smsJson = self._grab_json(smsPage)
 523                 if smsJson is None:
 524                         return ()
 525                 parsedSms = self._parse_sms(smsHtml)
 526                 smss = self._merge_conversation_sources(parsedSms, smsJson)
 527                 return smss
 528
 529         def get_unread_counts(self):
 530                 countPage = self._get_page(self._JSON_SMS_COUNT_URL)
 531                 counts = parse_json(countPage)
 532                 counts = counts["unreadCounts"]
 533                 return counts
 534
 535         def mark_message(self, messageId, asRead):
 536                 """
 537                 @blocks
 538                 """
 539                 postData = {
 540                         "read": 1 if asRead else 0,
 541                         "id": messageId,
 542                 }
 543
 544                 markPage = self._get_page(self._markAsReadURL, postData)
 545
 546         def archive_message(self, messageId):
 547                 """
 548                 @blocks
 549                 """
 550                 postData = {
 551                         "id": messageId,
 552                 }
 553
 554                 markPage = self._get_page(self._archiveMessageURL, postData)
 555
 556         def _grab_json(self, flatXml):
 557                 xmlTree = ElementTree.fromstring(flatXml)
 558                 jsonElement = xmlTree.getchildren()[0]
 559                 flatJson = jsonElement.text
 560                 jsonTree = parse_json(flatJson)
 561                 return jsonTree
 562
 563         def _grab_html(self, flatXml):
 564                 xmlTree = ElementTree.fromstring(flatXml)
 565                 htmlElement = xmlTree.getchildren()[1]
 566                 flatHtml = htmlElement.text
 567                 return flatHtml
 568
 569         def _grab_account_info(self, page):
 570                 accountData = parse_json(page)
 571                 self._token = accountData["r"]
 572                 self._accountNum = accountData["number"]["raw"]
 573                 for callback in accountData["phones"].itervalues():
 574                         self._callbackNumbers[callback["phoneNumber"]] = callback["name"]
 575                 if len(self._callbackNumbers) == 0:
 576                         _moduleLogger.debug("Could not extract callback numbers from GoogleVoice (the troublesome page follows):\n%s" % page)
 577                 return accountData
 578
 579         def _send_validation(self, number):
 580                 if not self.is_valid_syntax(number):
 581                         raise ValueError('Number is not valid: "%s"' % number)
 582                 return number
 583
 584         def _parse_recent(self, recentPage):
 585                 allRecentHtml = self._grab_html(recentPage)
 586                 allRecentData = self._parse_history(allRecentHtml)
 587                 for recentCallData in allRecentData:
 588                         yield recentCallData
 589
 590         def _parse_history(self, historyHtml):
 591                 splitVoicemail = self._seperateVoicemailsRegex.split(historyHtml)
 592                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 593                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 594                         exactTime = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 595                         exactTime = google_strptime(exactTime)
 596                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 597                         relativeTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 598                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 599                         location = locationGroup.group(1).strip() if locationGroup else ""
 600
 601                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 602                         name = nameGroup.group(1).strip() if nameGroup else ""
 603                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 604                         number = numberGroup.group(1).strip() if numberGroup else ""
 605                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 606                         prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 607                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 608                         contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 609
 610                         yield {
 611                                 "id": messageId.strip(),
 612                                 "contactId": contactId,
 613                                 "name": unescape(name),
 614                                 "time": exactTime,
 615                                 "relTime": relativeTime,
 616                                 "prettyNumber": prettyNumber,
 617                                 "number": number,
 618                                 "location": unescape(location),
 619                         }
 620
 621         @staticmethod
 622         def _interpret_voicemail_regex(group):
 623                 quality, content, number = group.group(2), group.group(3), group.group(4)
 624                 text = MessageText()
 625                 if quality is not None and content is not None:
 626                         text.accuracy = quality
 627                         text.text = unescape(content)
 628                         return text
 629                 elif number is not None:
 630                         text.accuracy = MessageText.ACCURACY_HIGH
 631                         text.text = number
 632                         return text
 633
 634         def _parse_voicemail(self, voicemailHtml):
 635                 splitVoicemail = self._seperateVoicemailsRegex.split(voicemailHtml)
 636                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 637                         conv = Conversation()
 638                         conv.type = Conversation.TYPE_VOICEMAIL
 639                         conv.id = messageId.strip()
 640
 641                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 642                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 643                         conv.time = google_strptime(exactTimeText)
 644                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 645                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 646                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 647                         conv.location = unescape(locationGroup.group(1).strip() if locationGroup else "")
 648
 649                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 650                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 651                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 652                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 653                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 654                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 655                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 656                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 657
 658                         messageGroups = self._voicemailMessageRegex.finditer(messageHtml)
 659                         messageParts = [
 660                                 self._interpret_voicemail_regex(group)
 661                                 for group in messageGroups
 662                         ] if messageGroups else ((MessageText.ACCURACY_LOW, "No Transcription"), )
 663                         message = Message()
 664                         message.body = messageParts
 665                         message.whoFrom = conv.name
 666                         try:
 667                                 message.when = conv.time.strftime("%I:%M %p")
 668                         except ValueError:
 669                                 _moduleLogger.exception("Confusing time provided: %r" % conv.time)
 670                                 message.when = "Unknown"
 671                         conv.messages = (message, )
 672
 673                         yield conv
 674
 675         @staticmethod
 676         def _interpret_sms_message_parts(fromPart, textPart, timePart):
 677                 text = MessageText()
 678                 text.accuracy = MessageText.ACCURACY_MEDIUM
 679                 text.text = unescape(textPart)
 680
 681                 message = Message()
 682                 message.body = (text, )
 683                 message.whoFrom = fromPart
 684                 message.when = timePart
 685
 686                 return message
 687
 688         def _parse_sms(self, smsHtml):
 689                 splitSms = self._seperateVoicemailsRegex.split(smsHtml)
 690                 for messageId, messageHtml in itergroup(splitSms[1:], 2):
 691                         conv = Conversation()
 692                         conv.type = Conversation.TYPE_SMS
 693                         conv.id = messageId.strip()
 694
 695                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 696                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 697                         conv.time = google_strptime(exactTimeText)
 698                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 699                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 700                         conv.location = ""
 701
 702                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 703                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 704                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 705                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 706                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 707                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 708                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 709                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 710
 711                         fromGroups = self._smsFromRegex.finditer(messageHtml)
 712                         fromParts = (group.group(1).strip() for group in fromGroups)
 713                         textGroups = self._smsTextRegex.finditer(messageHtml)
 714                         textParts = (group.group(1).strip() for group in textGroups)
 715                         timeGroups = self._smsTimeRegex.finditer(messageHtml)
 716                         timeParts = (group.group(1).strip() for group in timeGroups)
 717
 718                         messageParts = itertools.izip(fromParts, textParts, timeParts)
 719                         messages = [self._interpret_sms_message_parts(*parts) for parts in messageParts]
 720                         conv.messages = messages
 721
 722                         yield conv
 723
 724         @staticmethod
 725         def _merge_conversation_sources(parsedMessages, json):
 726                 for message in parsedMessages:
 727                         jsonItem = json["messages"][message.id]
 728                         message.isRead = jsonItem["isRead"]
 729                         message.isSpam = jsonItem["isSpam"]
 730                         message.isTrash = jsonItem["isTrash"]
 731                         message.isArchived = "inbox" not in jsonItem["labels"]
 732                         yield message
 733
 734         def _get_page(self, url, data = None, refererUrl = None):
 735                 headers = {}
 736                 if refererUrl is not None:
 737                         headers["Referer"] = refererUrl
 738
 739                 encodedData = urllib.urlencode(data) if data is not None else None
 740
 741                 try:
 742                         page = self._browser.download(url, encodedData, None, headers)
 743                 except urllib2.URLError, e:
 744                         _moduleLogger.error("Translating error: %s" % str(e))
 745                         raise NetworkError("%s is not accesible" % url)
 746
 747                 return page
 748
 749         def _get_page_with_token(self, url, data = None, refererUrl = None):
 750                 if data is None:
 751                         data = {}
 752                 data['_rnr_se'] = self._token
 753
 754                 page = self._get_page(url, data, refererUrl)
 755
 756                 return page
 757
 758         def _parse_with_validation(self, page):
 759                 json = parse_json(page)
 760                 self._validate_response(json)
 761                 return json
 762
 763         def _validate_response(self, response):
 764                 """
 765                 Validates that the JSON response is A-OK
 766                 """
 767                 try:
 768                         assert response is not None, "Response not provided"
 769                         assert 'ok' in response, "Response lacks status"
 770                         assert response['ok'], "Response not good"
 771                 except AssertionError:
 772                         try:
 773                                 if response["data"]["code"] == 20:
 774                                         raise RuntimeError(
 775 """Ambiguous error 20 returned by Google Voice.
 776 Please verify you have configured your callback number (currently "%s").  If it is configured some other suspected causes are: non-verified callback numbers, and Gizmo5 callback numbers.""" % self._callbackNumber)
 777                         except KeyError:
 778                                 pass
 779                         raise RuntimeError('There was a problem with GV: %s' % response)
 780
 781
 782 _UNESCAPE_ENTITIES = {
 783  "&quot;": '"',
 784  "&nbsp;": " ",
 785  "&#39;": "'",
 786 }
 787
 788
 789 def unescape(text):
 790         plain = saxutils.unescape(text, _UNESCAPE_ENTITIES)
 791         return plain
 792
 793
 794 def google_strptime(time):
 795         """
 796         Hack: Google always returns the time in the same locale.  Sadly if the
 797         local system's locale is different, there isn't a way to perfectly handle
 798         the time.  So instead we handle implement some time formatting
 799         """
 800         try:
 801                 abbrevTime = time[:-3]
 802                 parsedTime = datetime.datetime.strptime(abbrevTime, "%m/%d/%y %I:%M")
 803         except:
 804                 _moduleLogger.exception("Fooey, going with a dummy date")
 805                 parsedTime = datetime.datetime(1, 1, 1)
 806                 return parsedTime
 807         if time.endswith("PM"):
 808                 parsedTime += datetime.timedelta(hours=12)
 809         elif time.endswith("AM"):
 810                 pass
 811         else:
 812                 _moduleLogger.error("Unknown time of date for %r" % time)
 813         return parsedTime
 814
 815
 816 def itergroup(iterator, count, padValue = None):
 817         """
 818         Iterate in groups of 'count' values. If there
 819         aren't enough values, the last result is padded with
 820         None.
 821
 822         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 823         ...     print tuple(val)
 824         (1, 2, 3)
 825         (4, 5, 6)
 826         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 827         ...     print list(val)
 828         [1, 2, 3]
 829         [4, 5, 6]
 830         >>> for val in itergroup([1, 2, 3, 4, 5, 6, 7], 3):
 831         ...     print tuple(val)
 832         (1, 2, 3)
 833         (4, 5, 6)
 834         (7, None, None)
 835         >>> for val in itergroup("123456", 3):
 836         ...     print tuple(val)
 837         ('1', '2', '3')
 838         ('4', '5', '6')
 839         >>> for val in itergroup("123456", 3):
 840         ...     print repr("".join(val))
 841         '123'
 842         '456'
 843         """
 844         paddedIterator = itertools.chain(iterator, itertools.repeat(padValue, count-1))
 845         nIterators = (paddedIterator, ) * count
 846         return itertools.izip(*nIterators)
 847
 848
 849 def safe_eval(s):
 850         _TRUE_REGEX = re.compile("true")
 851         _FALSE_REGEX = re.compile("false")
 852         _COMMENT_REGEX = re.compile("^\s+//.*$", re.M)
 853         s = _TRUE_REGEX.sub("True", s)
 854         s = _FALSE_REGEX.sub("False", s)
 855         s = _COMMENT_REGEX.sub("#", s)
 856         try:
 857                 results = eval(s, {}, {})
 858         except SyntaxError:
 859                 _moduleLogger.exception("Oops")
 860                 results = None
 861         return results
 862
 863
 864 def _fake_parse_json(flattened):
 865         return safe_eval(flattened)
 866
 867
 868 def _actual_parse_json(flattened):
 869         return simplejson.loads(flattened)
 870
 871
 872 if simplejson is None:
 873         parse_json = _fake_parse_json
 874 else:
 875         parse_json = _actual_parse_json
 876
 877
 878 def extract_payload(flatXml):
 879         xmlTree = ElementTree.fromstring(flatXml)
 880
 881         jsonElement = xmlTree.getchildren()[0]
 882         flatJson = jsonElement.text
 883         jsonTree = parse_json(flatJson)
 884
 885         htmlElement = xmlTree.getchildren()[1]
 886         flatHtml = htmlElement.text
 887
 888         return jsonTree, flatHtml
 889
 890
 891 def guess_phone_type(number):
 892         if number.startswith("747") or number.startswith("1747") or number.startswith("+1747"):
 893                 return GVoiceBackend.PHONE_TYPE_GIZMO
 894         else:
 895                 return GVoiceBackend.PHONE_TYPE_MOBILE
 896
 897
 898 def get_sane_callback(backend):
 899         """
 900         Try to set a sane default callback number on these preferences
 901         1) 1747 numbers ( Gizmo )
 902         2) anything with gizmo in the name
 903         3) anything with computer in the name
 904         4) the first value
 905         """
 906         numbers = backend.get_callback_numbers()
 907
 908         priorityOrderedCriteria = [
 909                 ("\+1747", None),
 910                 ("1747", None),
 911                 ("747", None),
 912                 (None, "gizmo"),
 913                 (None, "computer"),
 914                 (None, "sip"),
 915                 (None, None),
 916         ]
 917
 918         for numberCriteria, descriptionCriteria in priorityOrderedCriteria:
 919                 numberMatcher = None
 920                 descriptionMatcher = None
 921                 if numberCriteria is not None:
 922                         numberMatcher = re.compile(numberCriteria)
 923                 elif descriptionCriteria is not None:
 924                         descriptionMatcher = re.compile(descriptionCriteria, re.I)
 925
 926                 for number, description in numbers.iteritems():
 927                         if numberMatcher is not None and numberMatcher.match(number) is None:
 928                                 continue
 929                         if descriptionMatcher is not None and descriptionMatcher.match(description) is None:
 930                                 continue
 931                         return number
 932
 933
 934 def set_sane_callback(backend):
 935         """
 936         Try to set a sane default callback number on these preferences
 937         1) 1747 numbers ( Gizmo )
 938         2) anything with gizmo in the name
 939         3) anything with computer in the name
 940         4) the first value
 941         """
 942         number = get_sane_callback(backend)
 943         backend.set_callback_number(number)
 944
 945
 946 def _is_not_special(name):
 947         return not name.startswith("_") and name[0].lower() == name[0] and "_" not in name
 948
 949
 950 def to_dict(obj):
 951         members = inspect.getmembers(obj)
 952         return dict((name, value) for (name, value) in members if _is_not_special(name))
 953
 954
 955 def grab_debug_info(username, password):
 956         cookieFile = os.path.join(".", "raw_cookies.txt")
 957         try:
 958                 os.remove(cookieFile)
 959         except OSError:
 960                 pass
 961
 962         backend = GVoiceBackend(cookieFile)
 963         browser = backend._browser
 964
 965         _TEST_WEBPAGES = [
 966                 ("token", backend._tokenURL),
 967                 ("login", backend._loginURL),
 968                 ("isdnd", backend._isDndURL),
 969                 ("account", backend._XML_ACCOUNT_URL),
 970                 ("html_contacts", backend._XML_CONTACTS_URL),
 971                 ("contacts", backend._JSON_CONTACTS_URL),
 972                 ("csv", backend._CSV_CONTACTS_URL),
 973
 974                 ("voicemail", backend._XML_VOICEMAIL_URL),
 975                 ("html_sms", backend._XML_SMS_URL),
 976                 ("sms", backend._JSON_SMS_URL),
 977                 ("count", backend._JSON_SMS_COUNT_URL),
 978
 979                 ("recent", backend._XML_RECENT_URL),
 980                 ("placed", backend._XML_PLACED_URL),
 981                 ("recieved", backend._XML_RECEIVED_URL),
 982                 ("missed", backend._XML_MISSED_URL),
 983         ]
 984
 985         # Get Pages
 986         print "Grabbing pre-login pages"
 987         for name, url in _TEST_WEBPAGES:
 988                 try:
 989                         page = browser.download(url)
 990                 except StandardError, e:
 991                         print e.message
 992                         continue
 993                 print "\tWriting to file"
 994                 with open("not_loggedin_%s.txt" % name, "w") as f:
 995                         f.write(page)
 996
 997         # Login
 998         print "Attempting login"
 999         galxToken = backend._get_token()
1000         loginSuccessOrFailurePage = backend._login(username, password, galxToken)
1001         with open("loggingin.txt", "w") as f:
1002                 print "\tWriting to file"
1003                 f.write(loginSuccessOrFailurePage)
1004         try:
1005                 backend._grab_account_info(loginSuccessOrFailurePage)
1006         except Exception:
1007                 # Retry in case the redirect failed
1008                 # luckily refresh_account_info does everything we need for a retry
1009                 loggedIn = backend.refresh_account_info() is not None
1010                 if not loggedIn:
1011                         raise
1012
1013         # Get Pages
1014         print "Grabbing post-login pages"
1015         for name, url in _TEST_WEBPAGES:
1016                 try:
1017                         page = browser.download(url)
1018                 except StandardError, e:
1019                         print str(e)
1020                         continue
1021                 print "\tWriting to file"
1022                 with open("loggedin_%s.txt" % name, "w") as f:
1023                         f.write(page)
1024
1025         # Cookies
1026         browser.save_cookies()
1027         print "\tWriting cookies to file"
1028         with open("cookies.txt", "w") as f:
1029                 f.writelines(
1030                         "%s: %s\n" % (c.name, c.value)
1031                         for c in browser._cookies
1032                 )
1033
1034
1035 def grab_voicemails(username, password):
1036         cookieFile = os.path.join(".", "raw_cookies.txt")
1037         try:
1038                 os.remove(cookieFile)
1039         except OSError:
1040                 pass
1041
1042         backend = GVoiceBackend(cookieFile)
1043         backend.login(username, password)
1044         voicemails = list(backend.get_voicemails())
1045         for voicemail in voicemails:
1046                 print voicemail.id
1047                 backend.download(voicemail.id, ".")
1048
1049
1050 def main():
1051         import sys
1052         logging.basicConfig(level=logging.DEBUG)
1053         args = sys.argv
1054         if 3 <= len(args):
1055                 username = args[1]
1056                 password = args[2]
1057
1058         grab_debug_info(username, password)
1059         grab_voicemails(username, password)
1060
1061
1062 if __name__ == "__main__":
1063         main()