git.maemo.org Git - gc-dialer/blob - src/backends/gvoice.py

   1 #!/usr/bin/python
   2
   3 """
   4 DialCentral - Front end for Google's GoogleVoice service.
   5 Copyright (C) 2008  Eric Warnke ericew AT gmail DOT com
   6
   7 This library is free software; you can redistribute it and/or
   8 modify it under the terms of the GNU Lesser General Public
   9 License as published by the Free Software Foundation; either
  10 version 2.1 of the License, or (at your option) any later version.
  11
  12 This library is distributed in the hope that it will be useful,
  13 but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 Lesser General Public License for more details.
  16
  17 You should have received a copy of the GNU Lesser General Public
  18 License along with this library; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 Google Voice backend code
  22
  23 Resources
  24         http://thatsmith.com/2009/03/google-voice-addon-for-firefox/
  25         http://posttopic.com/topic/google-voice-add-on-development
  26 """
  27
  28 from __future__ import with_statement
  29
  30 import os
  31 import re
  32 import urllib
  33 import urllib2
  34 import time
  35 import datetime
  36 import itertools
  37 import logging
  38 import inspect
  39
  40 from xml.sax import saxutils
  41 from xml.etree import ElementTree
  42
  43 try:
  44         import simplejson as _simplejson
  45         simplejson = _simplejson
  46 except ImportError:
  47         simplejson = None
  48
  49 import browser_emu
  50
  51
  52 _moduleLogger = logging.getLogger(__name__)
  53
  54
  55 class NetworkError(RuntimeError):
  56         pass
  57
  58
  59 class MessageText(object):
  60
  61         ACCURACY_LOW = "med1"
  62         ACCURACY_MEDIUM = "med2"
  63         ACCURACY_HIGH = "high"
  64
  65         def __init__(self):
  66                 self.accuracy = None
  67                 self.text = None
  68
  69         def __str__(self):
  70                 return self.text
  71
  72         def to_dict(self):
  73                 return to_dict(self)
  74
  75         def __eq__(self, other):
  76                 return self.accuracy == other.accuracy and self.text == other.text
  77
  78
  79 class Message(object):
  80
  81         def __init__(self):
  82                 self.whoFrom = None
  83                 self.body = None
  84                 self.when = None
  85
  86         def __str__(self):
  87                 return "%s (%s): %s" % (
  88                         self.whoFrom,
  89                         self.when,
  90                         "".join(str(part) for part in self.body)
  91                 )
  92
  93         def to_dict(self):
  94                 selfDict = to_dict(self)
  95                 selfDict["body"] = [text.to_dict() for text in self.body] if self.body is not None else None
  96                 return selfDict
  97
  98         def __eq__(self, other):
  99                 return self.whoFrom == other.whoFrom and self.when == other.when and self.body == other.body
 100
 101
 102 class Conversation(object):
 103
 104         TYPE_VOICEMAIL = "Voicemail"
 105         TYPE_SMS = "SMS"
 106
 107         def __init__(self):
 108                 self.type = None
 109                 self.id = None
 110                 self.contactId = None
 111                 self.name = None
 112                 self.location = None
 113                 self.prettyNumber = None
 114                 self.number = None
 115
 116                 self.time = None
 117                 self.relTime = None
 118                 self.messages = None
 119                 self.isRead = None
 120                 self.isSpam = None
 121                 self.isTrash = None
 122                 self.isArchived = None
 123
 124         def __cmp__(self, other):
 125                 cmpValue = cmp(self.contactId, other.contactId)
 126                 if cmpValue != 0:
 127                         return cmpValue
 128
 129                 cmpValue = cmp(self.time, other.time)
 130                 if cmpValue != 0:
 131                         return cmpValue
 132
 133                 cmpValue = cmp(self.id, other.id)
 134                 if cmpValue != 0:
 135                         return cmpValue
 136
 137         def to_dict(self):
 138                 selfDict = to_dict(self)
 139                 selfDict["messages"] = [message.to_dict() for message in self.messages] if self.messages is not None else None
 140                 return selfDict
 141
 142
 143 class GVoiceBackend(object):
 144         """
 145         This class encapsulates all of the knowledge necessary to interact with the GoogleVoice servers
 146         the functions include login, setting up a callback number, and initalting a callback
 147         """
 148
 149         PHONE_TYPE_HOME = 1
 150         PHONE_TYPE_MOBILE = 2
 151         PHONE_TYPE_WORK = 3
 152         PHONE_TYPE_GIZMO = 7
 153
 154         def __init__(self, cookieFile = None):
 155                 # Important items in this function are the setup of the browser emulation and cookie file
 156                 self._browser = browser_emu.MozillaEmulator(1)
 157                 self._loadedFromCookies = self._browser.load_cookies(cookieFile)
 158
 159                 self._token = ""
 160                 self._accountNum = ""
 161                 self._lastAuthed = 0.0
 162                 self._callbackNumber = ""
 163                 self._callbackNumbers = {}
 164
 165                 # Suprisingly, moving all of these from class to self sped up startup time
 166
 167                 self._validateRe = re.compile("^\+?[0-9]{10,}$")
 168
 169                 self._loginURL = "https://www.google.com/accounts/ServiceLoginAuth"
 170
 171                 SECURE_URL_BASE = "https://www.google.com/voice/"
 172                 SECURE_MOBILE_URL_BASE = SECURE_URL_BASE + "mobile/"
 173                 self._forwardURL = SECURE_MOBILE_URL_BASE + "phones"
 174                 self._tokenURL = SECURE_URL_BASE + "m"
 175                 self._callUrl = SECURE_URL_BASE + "call/connect"
 176                 self._callCancelURL = SECURE_URL_BASE + "call/cancel"
 177                 self._sendSmsURL = SECURE_URL_BASE + "sms/send"
 178
 179                 self._isDndURL = "https://www.google.com/voice/m/donotdisturb"
 180                 self._isDndRe = re.compile(r"""<input.*?id="doNotDisturb".*?checked="(.*?)"\s*/>""")
 181                 self._setDndURL = "https://www.google.com/voice/m/savednd"
 182
 183                 self._downloadVoicemailURL = SECURE_URL_BASE + "media/send_voicemail/"
 184                 self._markAsReadURL = SECURE_URL_BASE + "m/mark"
 185                 self._archiveMessageURL = SECURE_URL_BASE + "m/archive"
 186
 187                 self._XML_SEARCH_URL = SECURE_URL_BASE + "inbox/search/"
 188                 self._XML_ACCOUNT_URL = SECURE_URL_BASE + "contacts/"
 189                 # HACK really this redirects to the main pge and we are grabbing some javascript
 190                 self._XML_CONTACTS_URL = "http://www.google.com/voice/inbox/search/contact"
 191                 self._XML_RECENT_URL = SECURE_URL_BASE + "inbox/recent/"
 192
 193                 self.XML_FEEDS = (
 194                         'inbox', 'starred', 'all', 'spam', 'trash', 'voicemail', 'sms',
 195                         'recorded', 'placed', 'received', 'missed'
 196                 )
 197                 self._XML_INBOX_URL = SECURE_URL_BASE + "inbox/recent/inbox"
 198                 self._XML_STARRED_URL = SECURE_URL_BASE + "inbox/recent/starred"
 199                 self._XML_ALL_URL = SECURE_URL_BASE + "inbox/recent/all"
 200                 self._XML_SPAM_URL = SECURE_URL_BASE + "inbox/recent/spam"
 201                 self._XML_TRASH_URL = SECURE_URL_BASE + "inbox/recent/trash"
 202                 self._XML_VOICEMAIL_URL = SECURE_URL_BASE + "inbox/recent/voicemail/"
 203                 self._XML_SMS_URL = SECURE_URL_BASE + "inbox/recent/sms/"
 204                 self._XML_RECORDED_URL = SECURE_URL_BASE + "inbox/recent/recorded/"
 205                 self._XML_PLACED_URL = SECURE_URL_BASE + "inbox/recent/placed/"
 206                 self._XML_RECEIVED_URL = SECURE_URL_BASE + "inbox/recent/received/"
 207                 self._XML_MISSED_URL = SECURE_URL_BASE + "inbox/recent/missed/"
 208
 209                 self._galxRe = re.compile(r"""<input.*?name="GALX".*?value="(.*?)".*?/>""", re.MULTILINE | re.DOTALL)
 210                 self._tokenRe = re.compile(r"""<input.*?name="_rnr_se".*?value="(.*?)"\s*/>""")
 211                 self._accountNumRe = re.compile(r"""<b class="ms\d">(.{14})</b></div>""")
 212                 self._callbackRe = re.compile(r"""\s+(.*?):\s*(.*?)<br\s*/>\s*$""", re.M)
 213
 214                 self._contactsBodyRe = re.compile(r"""gcData\s*=\s*({.*?});""", re.MULTILINE | re.DOTALL)
 215                 self._seperateVoicemailsRegex = re.compile(r"""^\s*<div id="(\w+)"\s* class=".*?gc-message.*?">""", re.MULTILINE | re.DOTALL)
 216                 self._exactVoicemailTimeRegex = re.compile(r"""<span class="gc-message-time">(.*?)</span>""", re.MULTILINE)
 217                 self._relativeVoicemailTimeRegex = re.compile(r"""<span class="gc-message-relative">(.*?)</span>""", re.MULTILINE)
 218                 self._voicemailNameRegex = re.compile(r"""<a class=.*?gc-message-name-link.*?>(.*?)</a>""", re.MULTILINE | re.DOTALL)
 219                 self._voicemailNumberRegex = re.compile(r"""<input type="hidden" class="gc-text gc-quickcall-ac" value="(.*?)"/>""", re.MULTILINE)
 220                 self._prettyVoicemailNumberRegex = re.compile(r"""<span class="gc-message-type">(.*?)</span>""", re.MULTILINE)
 221                 self._voicemailLocationRegex = re.compile(r"""<span class="gc-message-location">.*?<a.*?>(.*?)</a></span>""", re.MULTILINE)
 222                 self._messagesContactIDRegex = re.compile(r"""<a class=".*?gc-message-name-link.*?">.*?</a>\s*?<span .*?>(.*?)</span>""", re.MULTILINE)
 223                 self._voicemailMessageRegex = re.compile(r"""(<span id="\d+-\d+" class="gc-word-(.*?)">(.*?)</span>|<a .*? class="gc-message-mni">(.*?)</a>)""", re.MULTILINE)
 224                 self._smsFromRegex = re.compile(r"""<span class="gc-message-sms-from">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 225                 self._smsTimeRegex = re.compile(r"""<span class="gc-message-sms-time">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 226                 self._smsTextRegex = re.compile(r"""<span class="gc-message-sms-text">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 227
 228         def is_quick_login_possible(self):
 229                 """
 230                 @returns True then is_authed might be enough to login, else full login is required
 231                 """
 232                 return self._loadedFromCookies or 0.0 < self._lastAuthed
 233
 234         def is_authed(self, force = False):
 235                 """
 236                 Attempts to detect a current session
 237                 @note Once logged in try not to reauth more than once a minute.
 238                 @returns If authenticated
 239                 """
 240                 isRecentledAuthed = (time.time() - self._lastAuthed) < 120
 241                 isPreviouslyAuthed = self._token is not None
 242                 if isRecentledAuthed and isPreviouslyAuthed and not force:
 243                         return True
 244
 245                 try:
 246                         page = self._get_page(self._forwardURL)
 247                         self._grab_account_info(page)
 248                 except Exception, e:
 249                         _moduleLogger.exception(str(e))
 250                         return False
 251
 252                 self._browser.save_cookies()
 253                 self._lastAuthed = time.time()
 254                 return True
 255
 256         def _get_token(self):
 257                 tokenPage = self._get_page(self._tokenURL)
 258
 259                 galxTokens = self._galxRe.search(tokenPage)
 260                 if galxTokens is not None:
 261                         galxToken = galxTokens.group(1)
 262                 else:
 263                         galxToken = ""
 264                         _moduleLogger.debug("Could not grab GALX token")
 265                 return galxToken
 266
 267         def _login(self, username, password, token):
 268                 loginData = {
 269                         'Email' : username,
 270                         'Passwd' : password,
 271                         'service': "grandcentral",
 272                         "ltmpl": "mobile",
 273                         "btmpl": "mobile",
 274                         "PersistentCookie": "yes",
 275                         "GALX": token,
 276                         "continue": self._forwardURL,
 277                 }
 278
 279                 loginSuccessOrFailurePage = self._get_page(self._loginURL, loginData)
 280                 return loginSuccessOrFailurePage
 281
 282         def login(self, username, password):
 283                 """
 284                 Attempt to login to GoogleVoice
 285                 @returns Whether login was successful or not
 286                 """
 287                 self.logout()
 288                 galxToken = self._get_token()
 289                 loginSuccessOrFailurePage = self._login(username, password, galxToken)
 290
 291                 try:
 292                         self._grab_account_info(loginSuccessOrFailurePage)
 293                 except Exception, e:
 294                         # Retry in case the redirect failed
 295                         # luckily is_authed does everything we need for a retry
 296                         loggedIn = self.is_authed(True)
 297                         if not loggedIn:
 298                                 _moduleLogger.exception(str(e))
 299                                 return False
 300                         _moduleLogger.info("Redirection failed on initial login attempt, auto-corrected for this")
 301
 302                 self._browser.save_cookies()
 303                 self._lastAuthed = time.time()
 304                 return True
 305
 306         def logout(self):
 307                 self._browser.clear_cookies()
 308                 self._browser.save_cookies()
 309                 self._token = None
 310                 self._lastAuthed = 0.0
 311
 312         def is_dnd(self):
 313                 isDndPage = self._get_page(self._isDndURL)
 314
 315                 dndGroup = self._isDndRe.search(isDndPage)
 316                 if dndGroup is None:
 317                         return False
 318                 dndStatus = dndGroup.group(1)
 319                 isDnd = True if dndStatus.strip().lower() == "true" else False
 320                 return isDnd
 321
 322         def set_dnd(self, doNotDisturb):
 323                 dndPostData = {
 324                         "doNotDisturb": 1 if doNotDisturb else 0,
 325                 }
 326
 327                 dndPage = self._get_page_with_token(self._setDndURL, dndPostData)
 328
 329         def call(self, outgoingNumber):
 330                 """
 331                 This is the main function responsible for initating the callback
 332                 """
 333                 outgoingNumber = self._send_validation(outgoingNumber)
 334                 subscriberNumber = None
 335                 phoneType = guess_phone_type(self._callbackNumber) # @todo Fix this hack
 336
 337                 callData = {
 338                                 'outgoingNumber': outgoingNumber,
 339                                 'forwardingNumber': self._callbackNumber,
 340                                 'subscriberNumber': subscriberNumber or 'undefined',
 341                                 'phoneType': str(phoneType),
 342                                 'remember': '1',
 343                 }
 344                 _moduleLogger.info("%r" % callData)
 345
 346                 page = self._get_page_with_token(
 347                         self._callUrl,
 348                         callData,
 349                 )
 350                 self._parse_with_validation(page)
 351                 return True
 352
 353         def cancel(self, outgoingNumber=None):
 354                 """
 355                 Cancels a call matching outgoing and forwarding numbers (if given).
 356                 Will raise an error if no matching call is being placed
 357                 """
 358                 page = self._get_page_with_token(
 359                         self._callCancelURL,
 360                         {
 361                         'outgoingNumber': outgoingNumber or 'undefined',
 362                         'forwardingNumber': self._callbackNumber or 'undefined',
 363                         'cancelType': 'C2C',
 364                         },
 365                 )
 366                 self._parse_with_validation(page)
 367
 368         def send_sms(self, phoneNumbers, message):
 369                 validatedPhoneNumbers = [
 370                         self._send_validation(phoneNumber)
 371                         for phoneNumber in phoneNumbers
 372                 ]
 373                 flattenedPhoneNumbers = ",".join(validatedPhoneNumbers)
 374                 page = self._get_page_with_token(
 375                         self._sendSmsURL,
 376                         {
 377                                 'phoneNumber': flattenedPhoneNumbers,
 378                                 'text': message
 379                         },
 380                 )
 381                 self._parse_with_validation(page)
 382
 383         def search(self, query):
 384                 """
 385                 Search your Google Voice Account history for calls, voicemails, and sms
 386                 Returns ``Folder`` instance containting matching messages
 387                 """
 388                 page = self._get_page(
 389                         self._XML_SEARCH_URL,
 390                         {"q": query},
 391                 )
 392                 json, html = extract_payload(page)
 393                 return json
 394
 395         def get_feed(self, feed):
 396                 actualFeed = "_XML_%s_URL" % feed.upper()
 397                 feedUrl = getattr(self, actualFeed)
 398
 399                 page = self._get_page(feedUrl)
 400                 json, html = extract_payload(page)
 401
 402                 return json
 403
 404         def download(self, messageId, adir):
 405                 """
 406                 Download a voicemail or recorded call MP3 matching the given ``msg``
 407                 which can either be a ``Message`` instance, or a SHA1 identifier.
 408                 Saves files to ``adir`` (defaults to current directory).
 409                 Message hashes can be found in ``self.voicemail().messages`` for example.
 410                 Returns location of saved file.
 411                 """
 412                 page = self._get_page(self._downloadVoicemailURL, {"id": messageId})
 413                 fn = os.path.join(adir, '%s.mp3' % messageId)
 414                 with open(fn, 'wb') as fo:
 415                         fo.write(page)
 416                 return fn
 417
 418         def is_valid_syntax(self, number):
 419                 """
 420                 @returns If This number be called ( syntax validation only )
 421                 """
 422                 return self._validateRe.match(number) is not None
 423
 424         def get_account_number(self):
 425                 """
 426                 @returns The GoogleVoice phone number
 427                 """
 428                 return self._accountNum
 429
 430         def get_callback_numbers(self):
 431                 """
 432                 @returns a dictionary mapping call back numbers to descriptions
 433                 @note These results are cached for 30 minutes.
 434                 """
 435                 if not self.is_authed():
 436                         return {}
 437                 return self._callbackNumbers
 438
 439         def set_callback_number(self, callbacknumber):
 440                 """
 441                 Set the number that GoogleVoice calls
 442                 @param callbacknumber should be a proper 10 digit number
 443                 """
 444                 self._callbackNumber = callbacknumber
 445                 _moduleLogger.info("Callback number changed: %r" % self._callbackNumber)
 446                 return True
 447
 448         def get_callback_number(self):
 449                 """
 450                 @returns Current callback number or None
 451                 """
 452                 return self._callbackNumber
 453
 454         def get_recent(self):
 455                 """
 456                 @returns Iterable of (personsName, phoneNumber, exact date, relative date, action)
 457                 """
 458                 for action, url in (
 459                         ("Received", self._XML_RECEIVED_URL),
 460                         ("Missed", self._XML_MISSED_URL),
 461                         ("Placed", self._XML_PLACED_URL),
 462                 ):
 463                         flatXml = self._get_page(url)
 464
 465                         allRecentHtml = self._grab_html(flatXml)
 466                         allRecentData = self._parse_history(allRecentHtml)
 467                         for recentCallData in allRecentData:
 468                                 recentCallData["action"] = action
 469                                 yield recentCallData
 470
 471         def get_contacts(self):
 472                 """
 473                 @returns Iterable of (contact id, contact name)
 474                 """
 475                 page = self._get_page(self._XML_CONTACTS_URL)
 476                 contactsBody = self._contactsBodyRe.search(page)
 477                 if contactsBody is None:
 478                         raise RuntimeError("Could not extract contact information")
 479                 accountData = _fake_parse_json(contactsBody.group(1))
 480                 if accountData is None:
 481                         return
 482                 for contactId, contactDetails in accountData["contacts"].iteritems():
 483                         # A zero contact id is the catch all for unknown contacts
 484                         if contactId != "0":
 485                                 if "name" in contactDetails:
 486                                         contactDetails["name"] = unescape(contactDetails["name"])
 487                                 yield contactId, contactDetails
 488
 489         def get_voicemails(self):
 490                 voicemailPage = self._get_page(self._XML_VOICEMAIL_URL)
 491                 voicemailHtml = self._grab_html(voicemailPage)
 492                 voicemailJson = self._grab_json(voicemailPage)
 493                 if voicemailJson is None:
 494                         return ()
 495                 parsedVoicemail = self._parse_voicemail(voicemailHtml)
 496                 voicemails = self._merge_conversation_sources(parsedVoicemail, voicemailJson)
 497                 return voicemails
 498
 499         def get_texts(self):
 500                 smsPage = self._get_page(self._XML_SMS_URL)
 501                 smsHtml = self._grab_html(smsPage)
 502                 smsJson = self._grab_json(smsPage)
 503                 if smsJson is None:
 504                         return ()
 505                 parsedSms = self._parse_sms(smsHtml)
 506                 smss = self._merge_conversation_sources(parsedSms, smsJson)
 507                 return smss
 508
 509         def mark_message(self, messageId, asRead):
 510                 postData = {
 511                         "read": 1 if asRead else 0,
 512                         "id": messageId,
 513                 }
 514
 515                 markPage = self._get_page(self._markAsReadURL, postData)
 516
 517         def archive_message(self, messageId):
 518                 postData = {
 519                         "id": messageId,
 520                 }
 521
 522                 markPage = self._get_page(self._archiveMessageURL, postData)
 523
 524         def _grab_json(self, flatXml):
 525                 xmlTree = ElementTree.fromstring(flatXml)
 526                 jsonElement = xmlTree.getchildren()[0]
 527                 flatJson = jsonElement.text
 528                 jsonTree = parse_json(flatJson)
 529                 return jsonTree
 530
 531         def _grab_html(self, flatXml):
 532                 xmlTree = ElementTree.fromstring(flatXml)
 533                 htmlElement = xmlTree.getchildren()[1]
 534                 flatHtml = htmlElement.text
 535                 return flatHtml
 536
 537         def _grab_account_info(self, page):
 538                 tokenGroup = self._tokenRe.search(page)
 539                 if tokenGroup is None:
 540                         raise RuntimeError("Could not extract authentication token from GoogleVoice")
 541                 self._token = tokenGroup.group(1)
 542
 543                 anGroup = self._accountNumRe.search(page)
 544                 if anGroup is not None:
 545                         self._accountNum = anGroup.group(1)
 546                 else:
 547                         _moduleLogger.debug("Could not extract account number from GoogleVoice")
 548
 549                 self._callbackNumbers = {}
 550                 for match in self._callbackRe.finditer(page):
 551                         callbackNumber = match.group(2)
 552                         callbackName = match.group(1)
 553                         self._callbackNumbers[callbackNumber] = callbackName
 554                 if len(self._callbackNumbers) == 0:
 555                         _moduleLogger.debug("Could not extract callback numbers from GoogleVoice (the troublesome page follows):\n%s" % page)
 556
 557         def _send_validation(self, number):
 558                 if not self.is_valid_syntax(number):
 559                         raise ValueError('Number is not valid: "%s"' % number)
 560                 elif not self.is_authed():
 561                         raise RuntimeError("Not Authenticated")
 562                 return number
 563
 564         def _parse_history(self, historyHtml):
 565                 splitVoicemail = self._seperateVoicemailsRegex.split(historyHtml)
 566                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 567                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 568                         exactTime = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 569                         exactTime = google_strptime(exactTime)
 570                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 571                         relativeTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 572                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 573                         location = locationGroup.group(1).strip() if locationGroup else ""
 574
 575                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 576                         name = nameGroup.group(1).strip() if nameGroup else ""
 577                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 578                         number = numberGroup.group(1).strip() if numberGroup else ""
 579                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 580                         prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 581                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 582                         contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 583
 584                         yield {
 585                                 "id": messageId.strip(),
 586                                 "contactId": contactId,
 587                                 "name": unescape(name),
 588                                 "time": exactTime,
 589                                 "relTime": relativeTime,
 590                                 "prettyNumber": prettyNumber,
 591                                 "number": number,
 592                                 "location": unescape(location),
 593                         }
 594
 595         @staticmethod
 596         def _interpret_voicemail_regex(group):
 597                 quality, content, number = group.group(2), group.group(3), group.group(4)
 598                 text = MessageText()
 599                 if quality is not None and content is not None:
 600                         text.accuracy = quality
 601                         text.text = unescape(content)
 602                         return text
 603                 elif number is not None:
 604                         text.accuracy = MessageText.ACCURACY_HIGH
 605                         text.text = number
 606                         return text
 607
 608         def _parse_voicemail(self, voicemailHtml):
 609                 splitVoicemail = self._seperateVoicemailsRegex.split(voicemailHtml)
 610                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 611                         conv = Conversation()
 612                         conv.type = Conversation.TYPE_VOICEMAIL
 613                         conv.id = messageId.strip()
 614
 615                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 616                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 617                         conv.time = google_strptime(exactTimeText)
 618                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 619                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 620                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 621                         conv.location = unescape(locationGroup.group(1).strip() if locationGroup else "")
 622
 623                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 624                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 625                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 626                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 627                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 628                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 629                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 630                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 631
 632                         messageGroups = self._voicemailMessageRegex.finditer(messageHtml)
 633                         messageParts = [
 634                                 self._interpret_voicemail_regex(group)
 635                                 for group in messageGroups
 636                         ] if messageGroups else ((MessageText.ACCURACY_LOW, "No Transcription"), )
 637                         message = Message()
 638                         message.body = messageParts
 639                         message.whoFrom = conv.name
 640                         message.when = conv.time.strftime("%I:%M %p")
 641                         conv.messages = (message, )
 642
 643                         yield conv
 644
 645         @staticmethod
 646         def _interpret_sms_message_parts(fromPart, textPart, timePart):
 647                 text = MessageText()
 648                 text.accuracy = MessageText.ACCURACY_MEDIUM
 649                 text.text = unescape(textPart)
 650
 651                 message = Message()
 652                 message.body = (text, )
 653                 message.whoFrom = fromPart
 654                 message.when = timePart
 655
 656                 return message
 657
 658         def _parse_sms(self, smsHtml):
 659                 splitSms = self._seperateVoicemailsRegex.split(smsHtml)
 660                 for messageId, messageHtml in itergroup(splitSms[1:], 2):
 661                         conv = Conversation()
 662                         conv.type = Conversation.TYPE_SMS
 663                         conv.id = messageId.strip()
 664
 665                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 666                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 667                         conv.time = google_strptime(exactTimeText)
 668                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 669                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 670                         conv.location = ""
 671
 672                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 673                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 674                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 675                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 676                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 677                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 678                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 679                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 680
 681                         fromGroups = self._smsFromRegex.finditer(messageHtml)
 682                         fromParts = (group.group(1).strip() for group in fromGroups)
 683                         textGroups = self._smsTextRegex.finditer(messageHtml)
 684                         textParts = (group.group(1).strip() for group in textGroups)
 685                         timeGroups = self._smsTimeRegex.finditer(messageHtml)
 686                         timeParts = (group.group(1).strip() for group in timeGroups)
 687
 688                         messageParts = itertools.izip(fromParts, textParts, timeParts)
 689                         messages = [self._interpret_sms_message_parts(*parts) for parts in messageParts]
 690                         conv.messages = messages
 691
 692                         yield conv
 693
 694         @staticmethod
 695         def _merge_conversation_sources(parsedMessages, json):
 696                 for message in parsedMessages:
 697                         jsonItem = json["messages"][message.id]
 698                         message.isRead = jsonItem["isRead"]
 699                         message.isSpam = jsonItem["isSpam"]
 700                         message.isTrash = jsonItem["isTrash"]
 701                         message.isArchived = "inbox" not in jsonItem["labels"]
 702                         yield message
 703
 704         def _get_page(self, url, data = None, refererUrl = None):
 705                 headers = {}
 706                 if refererUrl is not None:
 707                         headers["Referer"] = refererUrl
 708
 709                 encodedData = urllib.urlencode(data) if data is not None else None
 710
 711                 try:
 712                         page = self._browser.download(url, encodedData, None, headers)
 713                 except urllib2.URLError, e:
 714                         _moduleLogger.error("Translating error: %s" % str(e))
 715                         raise NetworkError("%s is not accesible" % url)
 716
 717                 return page
 718
 719         def _get_page_with_token(self, url, data = None, refererUrl = None):
 720                 if data is None:
 721                         data = {}
 722                 data['_rnr_se'] = self._token
 723
 724                 page = self._get_page(url, data, refererUrl)
 725
 726                 return page
 727
 728         def _parse_with_validation(self, page):
 729                 json = parse_json(page)
 730                 validate_response(json)
 731                 return json
 732
 733
 734 _UNESCAPE_ENTITIES = {
 735  "&quot;": '"',
 736  "&nbsp;": " ",
 737  "&#39;": "'",
 738 }
 739
 740
 741 def unescape(text):
 742         plain = saxutils.unescape(text, _UNESCAPE_ENTITIES)
 743         return plain
 744
 745
 746 def google_strptime(time):
 747         """
 748         Hack: Google always returns the time in the same locale.  Sadly if the
 749         local system's locale is different, there isn't a way to perfectly handle
 750         the time.  So instead we handle implement some time formatting
 751         """
 752         abbrevTime = time[:-3]
 753         parsedTime = datetime.datetime.strptime(abbrevTime, "%m/%d/%y %I:%M")
 754         if time[-2] == "PN":
 755                 parsedTime += datetime.timedelta(hours=12)
 756         return parsedTime
 757
 758
 759 def itergroup(iterator, count, padValue = None):
 760         """
 761         Iterate in groups of 'count' values. If there
 762         aren't enough values, the last result is padded with
 763         None.
 764
 765         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 766         ...     print tuple(val)
 767         (1, 2, 3)
 768         (4, 5, 6)
 769         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 770         ...     print list(val)
 771         [1, 2, 3]
 772         [4, 5, 6]
 773         >>> for val in itergroup([1, 2, 3, 4, 5, 6, 7], 3):
 774         ...     print tuple(val)
 775         (1, 2, 3)
 776         (4, 5, 6)
 777         (7, None, None)
 778         >>> for val in itergroup("123456", 3):
 779         ...     print tuple(val)
 780         ('1', '2', '3')
 781         ('4', '5', '6')
 782         >>> for val in itergroup("123456", 3):
 783         ...     print repr("".join(val))
 784         '123'
 785         '456'
 786         """
 787         paddedIterator = itertools.chain(iterator, itertools.repeat(padValue, count-1))
 788         nIterators = (paddedIterator, ) * count
 789         return itertools.izip(*nIterators)
 790
 791
 792 def safe_eval(s):
 793         _TRUE_REGEX = re.compile("true")
 794         _FALSE_REGEX = re.compile("false")
 795         s = _TRUE_REGEX.sub("True", s)
 796         s = _FALSE_REGEX.sub("False", s)
 797         try:
 798                 results = eval(s, {}, {})
 799         except SyntaxError:
 800                 _moduleLogger.exception("Oops")
 801                 return None
 802
 803
 804 def _fake_parse_json(flattened):
 805         return safe_eval(flattened)
 806
 807
 808 def _actual_parse_json(flattened):
 809         return simplejson.loads(flattened)
 810
 811
 812 if simplejson is None:
 813         parse_json = _fake_parse_json
 814 else:
 815         parse_json = _actual_parse_json
 816
 817
 818 def extract_payload(flatXml):
 819         xmlTree = ElementTree.fromstring(flatXml)
 820
 821         jsonElement = xmlTree.getchildren()[0]
 822         flatJson = jsonElement.text
 823         jsonTree = parse_json(flatJson)
 824
 825         htmlElement = xmlTree.getchildren()[1]
 826         flatHtml = htmlElement.text
 827
 828         return jsonTree, flatHtml
 829
 830
 831 def validate_response(response):
 832         """
 833         Validates that the JSON response is A-OK
 834         """
 835         try:
 836                 assert response is not None
 837                 assert 'ok' in response
 838                 assert response['ok']
 839         except AssertionError:
 840                 raise RuntimeError('There was a problem with GV: %s' % response)
 841
 842
 843 def guess_phone_type(number):
 844         if number.startswith("747") or number.startswith("1747") or number.startswith("+1747"):
 845                 return GVoiceBackend.PHONE_TYPE_GIZMO
 846         else:
 847                 return GVoiceBackend.PHONE_TYPE_MOBILE
 848
 849
 850 def get_sane_callback(backend):
 851         """
 852         Try to set a sane default callback number on these preferences
 853         1) 1747 numbers ( Gizmo )
 854         2) anything with gizmo in the name
 855         3) anything with computer in the name
 856         4) the first value
 857         """
 858         numbers = backend.get_callback_numbers()
 859
 860         priorityOrderedCriteria = [
 861                 ("\+1747", None),
 862                 ("1747", None),
 863                 ("747", None),
 864                 (None, "gizmo"),
 865                 (None, "computer"),
 866                 (None, "sip"),
 867                 (None, None),
 868         ]
 869
 870         for numberCriteria, descriptionCriteria in priorityOrderedCriteria:
 871                 numberMatcher = None
 872                 descriptionMatcher = None
 873                 if numberCriteria is not None:
 874                         numberMatcher = re.compile(numberCriteria)
 875                 elif descriptionCriteria is not None:
 876                         descriptionMatcher = re.compile(descriptionCriteria, re.I)
 877
 878                 for number, description in numbers.iteritems():
 879                         if numberMatcher is not None and numberMatcher.match(number) is None:
 880                                 continue
 881                         if descriptionMatcher is not None and descriptionMatcher.match(description) is None:
 882                                 continue
 883                         return number
 884
 885
 886 def set_sane_callback(backend):
 887         """
 888         Try to set a sane default callback number on these preferences
 889         1) 1747 numbers ( Gizmo )
 890         2) anything with gizmo in the name
 891         3) anything with computer in the name
 892         4) the first value
 893         """
 894         number = get_sane_callback(backend)
 895         backend.set_callback_number(number)
 896
 897
 898 def _is_not_special(name):
 899         return not name.startswith("_") and name[0].lower() == name[0] and "_" not in name
 900
 901
 902 def to_dict(obj):
 903         members = inspect.getmembers(obj)
 904         return dict((name, value) for (name, value) in members if _is_not_special(name))
 905
 906
 907 def grab_debug_info(username, password):
 908         cookieFile = os.path.join(".", "raw_cookies.txt")
 909         try:
 910                 os.remove(cookieFile)
 911         except OSError:
 912                 pass
 913
 914         backend = GVoiceBackend(cookieFile)
 915         browser = backend._browser
 916
 917         _TEST_WEBPAGES = [
 918                 ("forward", backend._forwardURL),
 919                 ("token", backend._tokenURL),
 920                 ("login", backend._loginURL),
 921                 ("isdnd", backend._isDndURL),
 922                 ("account", backend._XML_ACCOUNT_URL),
 923                 ("contacts", backend._XML_CONTACTS_URL),
 924
 925                 ("voicemail", backend._XML_VOICEMAIL_URL),
 926                 ("sms", backend._XML_SMS_URL),
 927
 928                 ("recent", backend._XML_RECENT_URL),
 929                 ("placed", backend._XML_PLACED_URL),
 930                 ("recieved", backend._XML_RECEIVED_URL),
 931                 ("missed", backend._XML_MISSED_URL),
 932         ]
 933
 934         # Get Pages
 935         print "Grabbing pre-login pages"
 936         for name, url in _TEST_WEBPAGES:
 937                 try:
 938                         page = browser.download(url)
 939                 except StandardError, e:
 940                         print e.message
 941                         continue
 942                 print "\tWriting to file"
 943                 with open("not_loggedin_%s.txt" % name, "w") as f:
 944                         f.write(page)
 945
 946         # Login
 947         print "Attempting login"
 948         galxToken = backend._get_token()
 949         loginSuccessOrFailurePage = backend._login(username, password, galxToken)
 950         with open("loggingin.txt", "w") as f:
 951                 print "\tWriting to file"
 952                 f.write(loginSuccessOrFailurePage)
 953         try:
 954                 backend._grab_account_info(loginSuccessOrFailurePage)
 955         except Exception:
 956                 # Retry in case the redirect failed
 957                 # luckily is_authed does everything we need for a retry
 958                 loggedIn = backend.is_authed(True)
 959                 if not loggedIn:
 960                         raise
 961
 962         # Get Pages
 963         print "Grabbing post-login pages"
 964         for name, url in _TEST_WEBPAGES:
 965                 try:
 966                         page = browser.download(url)
 967                 except StandardError, e:
 968                         print str(e)
 969                         continue
 970                 print "\tWriting to file"
 971                 with open("loggedin_%s.txt" % name, "w") as f:
 972                         f.write(page)
 973
 974         # Cookies
 975         browser.save_cookies()
 976         print "\tWriting cookies to file"
 977         with open("cookies.txt", "w") as f:
 978                 f.writelines(
 979                         "%s: %s\n" % (c.name, c.value)
 980                         for c in browser._cookies
 981                 )
 982
 983
 984 def main():
 985         import sys
 986         logging.basicConfig(level=logging.DEBUG)
 987         args = sys.argv
 988         if 3 <= len(args):
 989                 username = args[1]
 990                 password = args[2]
 991
 992         grab_debug_info(username, password)
 993
 994
 995 if __name__ == "__main__":
 996         main()