git.maemo.org Git - theonering/blob - src/gvoice/backend.py

   1 #!/usr/bin/python
   2
   3 """
   4 DialCentral - Front end for Google's GoogleVoice service.
   5 Copyright (C) 2008  Eric Warnke ericew AT gmail DOT com
   6
   7 This library is free software; you can redistribute it and/or
   8 modify it under the terms of the GNU Lesser General Public
   9 License as published by the Free Software Foundation; either
  10 version 2.1 of the License, or (at your option) any later version.
  11
  12 This library is distributed in the hope that it will be useful,
  13 but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 Lesser General Public License for more details.
  16
  17 You should have received a copy of the GNU Lesser General Public
  18 License along with this library; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 Google Voice backend code
  22
  23 Resources
  24         http://thatsmith.com/2009/03/google-voice-addon-for-firefox/
  25         http://posttopic.com/topic/google-voice-add-on-development
  26 """
  27
  28 from __future__ import with_statement
  29
  30 import os
  31 import re
  32 import urllib
  33 import urllib2
  34 import time
  35 import datetime
  36 import itertools
  37 import logging
  38 import inspect
  39
  40 from xml.sax import saxutils
  41 from xml.etree import ElementTree
  42
  43 try:
  44         import simplejson as _simplejson
  45         simplejson = _simplejson
  46 except ImportError:
  47         simplejson = None
  48
  49 import browser_emu
  50
  51
  52 _moduleLogger = logging.getLogger(__name__)
  53
  54
  55 class NetworkError(RuntimeError):
  56         pass
  57
  58
  59 class MessageText(object):
  60
  61         ACCURACY_LOW = "med1"
  62         ACCURACY_MEDIUM = "med2"
  63         ACCURACY_HIGH = "high"
  64
  65         def __init__(self):
  66                 self.accuracy = None
  67                 self.text = None
  68
  69         def __str__(self):
  70                 return self.text
  71
  72         def to_dict(self):
  73                 return to_dict(self)
  74
  75         def __eq__(self, other):
  76                 return self.accuracy == other.accuracy and self.text == other.text
  77
  78
  79 class Message(object):
  80
  81         def __init__(self):
  82                 self.whoFrom = None
  83                 self.body = None
  84                 self.when = None
  85
  86         def __str__(self):
  87                 return "%s (%s): %s" % (
  88                         self.whoFrom,
  89                         self.when,
  90                         "".join(unicode(part) for part in self.body)
  91                 )
  92
  93         def to_dict(self):
  94                 selfDict = to_dict(self)
  95                 selfDict["body"] = [text.to_dict() for text in self.body] if self.body is not None else None
  96                 return selfDict
  97
  98         def __eq__(self, other):
  99                 return self.whoFrom == other.whoFrom and self.when == other.when and self.body == other.body
 100
 101
 102 class Conversation(object):
 103
 104         TYPE_VOICEMAIL = "Voicemail"
 105         TYPE_SMS = "SMS"
 106
 107         def __init__(self):
 108                 self.type = None
 109                 self.id = None
 110                 self.contactId = None
 111                 self.name = None
 112                 self.location = None
 113                 self.prettyNumber = None
 114                 self.number = None
 115
 116                 self.time = None
 117                 self.relTime = None
 118                 self.messages = None
 119                 self.isRead = None
 120                 self.isSpam = None
 121                 self.isTrash = None
 122                 self.isArchived = None
 123
 124         def __cmp__(self, other):
 125                 cmpValue = cmp(self.contactId, other.contactId)
 126                 if cmpValue != 0:
 127                         return cmpValue
 128
 129                 cmpValue = cmp(self.time, other.time)
 130                 if cmpValue != 0:
 131                         return cmpValue
 132
 133                 cmpValue = cmp(self.id, other.id)
 134                 if cmpValue != 0:
 135                         return cmpValue
 136
 137         def to_dict(self):
 138                 selfDict = to_dict(self)
 139                 selfDict["messages"] = [message.to_dict() for message in self.messages] if self.messages is not None else None
 140                 return selfDict
 141
 142
 143 class GVoiceBackend(object):
 144         """
 145         This class encapsulates all of the knowledge necessary to interact with the GoogleVoice servers
 146         the functions include login, setting up a callback number, and initalting a callback
 147         """
 148
 149         PHONE_TYPE_HOME = 1
 150         PHONE_TYPE_MOBILE = 2
 151         PHONE_TYPE_WORK = 3
 152         PHONE_TYPE_GIZMO = 7
 153
 154         def __init__(self, cookieFile = None):
 155                 # Important items in this function are the setup of the browser emulation and cookie file
 156                 self._browser = browser_emu.MozillaEmulator(1)
 157                 self._loadedFromCookies = self._browser.load_cookies(cookieFile)
 158
 159                 self._token = ""
 160                 self._accountNum = ""
 161                 self._lastAuthed = 0.0
 162                 self._callbackNumber = ""
 163                 self._callbackNumbers = {}
 164
 165                 # Suprisingly, moving all of these from class to self sped up startup time
 166
 167                 self._validateRe = re.compile("^\+?[0-9]{10,}$")
 168
 169                 self._loginURL = "https://www.google.com/accounts/ServiceLoginAuth"
 170
 171                 SECURE_URL_BASE = "https://www.google.com/voice/"
 172                 SECURE_MOBILE_URL_BASE = SECURE_URL_BASE + "mobile/"
 173                 self._forwardURL = SECURE_MOBILE_URL_BASE + "phones"
 174                 self._tokenURL = SECURE_URL_BASE + "m"
 175                 self._callUrl = SECURE_URL_BASE + "call/connect"
 176                 self._callCancelURL = SECURE_URL_BASE + "call/cancel"
 177                 self._sendSmsURL = SECURE_URL_BASE + "sms/send"
 178
 179                 self._isDndURL = "https://www.google.com/voice/m/donotdisturb"
 180                 self._isDndRe = re.compile(r"""<input.*?id="doNotDisturb".*?checked="(.*?)"\s*/>""")
 181                 self._setDndURL = "https://www.google.com/voice/m/savednd"
 182
 183                 self._downloadVoicemailURL = SECURE_URL_BASE + "media/send_voicemail/"
 184                 self._markAsReadURL = SECURE_URL_BASE + "m/mark"
 185                 self._archiveMessageURL = SECURE_URL_BASE + "m/archive"
 186
 187                 self._XML_SEARCH_URL = SECURE_URL_BASE + "inbox/search/"
 188                 self._XML_ACCOUNT_URL = SECURE_URL_BASE + "contacts/"
 189                 # HACK really this redirects to the main pge and we are grabbing some javascript
 190                 self._XML_CONTACTS_URL = "http://www.google.com/voice/inbox/search/contact"
 191                 self._CSV_CONTACTS_URL = "http://mail.google.com/mail/contacts/data/export"
 192                 self._XML_RECENT_URL = SECURE_URL_BASE + "inbox/recent/"
 193
 194                 self.XML_FEEDS = (
 195                         'inbox', 'starred', 'all', 'spam', 'trash', 'voicemail', 'sms',
 196                         'recorded', 'placed', 'received', 'missed'
 197                 )
 198                 self._XML_INBOX_URL = SECURE_URL_BASE + "inbox/recent/inbox"
 199                 self._XML_STARRED_URL = SECURE_URL_BASE + "inbox/recent/starred"
 200                 self._XML_ALL_URL = SECURE_URL_BASE + "inbox/recent/all"
 201                 self._XML_SPAM_URL = SECURE_URL_BASE + "inbox/recent/spam"
 202                 self._XML_TRASH_URL = SECURE_URL_BASE + "inbox/recent/trash"
 203                 self._XML_VOICEMAIL_URL = SECURE_URL_BASE + "inbox/recent/voicemail/"
 204                 self._XML_SMS_URL = SECURE_URL_BASE + "inbox/recent/sms/"
 205                 self._XML_RECORDED_URL = SECURE_URL_BASE + "inbox/recent/recorded/"
 206                 self._XML_PLACED_URL = SECURE_URL_BASE + "inbox/recent/placed/"
 207                 self._XML_RECEIVED_URL = SECURE_URL_BASE + "inbox/recent/received/"
 208                 self._XML_MISSED_URL = SECURE_URL_BASE + "inbox/recent/missed/"
 209
 210                 self._galxRe = re.compile(r"""<input.*?name="GALX".*?value="(.*?)".*?/>""", re.MULTILINE | re.DOTALL)
 211                 self._tokenRe = re.compile(r"""<input.*?name="_rnr_se".*?value="(.*?)"\s*/>""")
 212                 self._accountNumRe = re.compile(r"""<b class="ms\d">(.{14})</b></div>""")
 213                 self._callbackRe = re.compile(r"""\s+(.*?):\s*(.*?)<br\s*/>\s*$""", re.M)
 214
 215                 self._contactsBodyRe = re.compile(r"""gcData\s*=\s*({.*?});""", re.MULTILINE | re.DOTALL)
 216                 self._seperateVoicemailsRegex = re.compile(r"""^\s*<div id="(\w+)"\s* class=".*?gc-message.*?">""", re.MULTILINE | re.DOTALL)
 217                 self._exactVoicemailTimeRegex = re.compile(r"""<span class="gc-message-time">(.*?)</span>""", re.MULTILINE)
 218                 self._relativeVoicemailTimeRegex = re.compile(r"""<span class="gc-message-relative">(.*?)</span>""", re.MULTILINE)
 219                 self._voicemailNameRegex = re.compile(r"""<a class=.*?gc-message-name-link.*?>(.*?)</a>""", re.MULTILINE | re.DOTALL)
 220                 self._voicemailNumberRegex = re.compile(r"""<input type="hidden" class="gc-text gc-quickcall-ac" value="(.*?)"/>""", re.MULTILINE)
 221                 self._prettyVoicemailNumberRegex = re.compile(r"""<span class="gc-message-type">(.*?)</span>""", re.MULTILINE)
 222                 self._voicemailLocationRegex = re.compile(r"""<span class="gc-message-location">.*?<a.*?>(.*?)</a></span>""", re.MULTILINE)
 223                 self._messagesContactIDRegex = re.compile(r"""<a class=".*?gc-message-name-link.*?">.*?</a>\s*?<span .*?>(.*?)</span>""", re.MULTILINE)
 224                 self._voicemailMessageRegex = re.compile(r"""(<span id="\d+-\d+" class="gc-word-(.*?)">(.*?)</span>|<a .*? class="gc-message-mni">(.*?)</a>)""", re.MULTILINE)
 225                 self._smsFromRegex = re.compile(r"""<span class="gc-message-sms-from">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 226                 self._smsTimeRegex = re.compile(r"""<span class="gc-message-sms-time">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 227                 self._smsTextRegex = re.compile(r"""<span class="gc-message-sms-text">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 228
 229         def is_quick_login_possible(self):
 230                 """
 231                 @returns True then is_authed might be enough to login, else full login is required
 232                 """
 233                 return self._loadedFromCookies or 0.0 < self._lastAuthed
 234
 235         def is_authed(self, force = False):
 236                 """
 237                 Attempts to detect a current session
 238                 @note Once logged in try not to reauth more than once a minute.
 239                 @returns If authenticated
 240                 @blocks
 241                 """
 242                 isRecentledAuthed = (time.time() - self._lastAuthed) < 120
 243                 isPreviouslyAuthed = self._token is not None
 244                 if isRecentledAuthed and isPreviouslyAuthed and not force:
 245                         return True
 246
 247                 try:
 248                         page = self._get_page(self._forwardURL)
 249                         self._grab_account_info(page)
 250                 except Exception, e:
 251                         _moduleLogger.exception(str(e))
 252                         return False
 253
 254                 self._browser.save_cookies()
 255                 self._lastAuthed = time.time()
 256                 return True
 257
 258         def _get_token(self):
 259                 tokenPage = self._get_page(self._tokenURL)
 260
 261                 galxTokens = self._galxRe.search(tokenPage)
 262                 if galxTokens is not None:
 263                         galxToken = galxTokens.group(1)
 264                 else:
 265                         galxToken = ""
 266                         _moduleLogger.debug("Could not grab GALX token")
 267                 return galxToken
 268
 269         def _login(self, username, password, token):
 270                 loginData = {
 271                         'Email' : username,
 272                         'Passwd' : password,
 273                         'service': "grandcentral",
 274                         "ltmpl": "mobile",
 275                         "btmpl": "mobile",
 276                         "PersistentCookie": "yes",
 277                         "GALX": token,
 278                         "continue": self._forwardURL,
 279                 }
 280
 281                 loginSuccessOrFailurePage = self._get_page(self._loginURL, loginData)
 282                 return loginSuccessOrFailurePage
 283
 284         def login(self, username, password):
 285                 """
 286                 Attempt to login to GoogleVoice
 287                 @returns Whether login was successful or not
 288                 @blocks
 289                 """
 290                 self.logout()
 291                 galxToken = self._get_token()
 292                 loginSuccessOrFailurePage = self._login(username, password, galxToken)
 293
 294                 try:
 295                         self._grab_account_info(loginSuccessOrFailurePage)
 296                 except Exception, e:
 297                         # Retry in case the redirect failed
 298                         # luckily is_authed does everything we need for a retry
 299                         loggedIn = self.is_authed(True)
 300                         if not loggedIn:
 301                                 _moduleLogger.exception(str(e))
 302                                 return False
 303                         _moduleLogger.info("Redirection failed on initial login attempt, auto-corrected for this")
 304
 305                 self._browser.save_cookies()
 306                 self._lastAuthed = time.time()
 307                 return True
 308
 309         def persist(self):
 310                 self._browser.save_cookies()
 311
 312         def shutdown(self):
 313                 self._browser.save_cookies()
 314                 self._token = None
 315                 self._lastAuthed = 0.0
 316
 317         def logout(self):
 318                 self._browser.clear_cookies()
 319                 self._browser.save_cookies()
 320                 self._token = None
 321                 self._lastAuthed = 0.0
 322
 323         def is_dnd(self):
 324                 """
 325                 @blocks
 326                 """
 327                 isDndPage = self._get_page(self._isDndURL)
 328
 329                 dndGroup = self._isDndRe.search(isDndPage)
 330                 if dndGroup is None:
 331                         return False
 332                 dndStatus = dndGroup.group(1)
 333                 isDnd = True if dndStatus.strip().lower() == "true" else False
 334                 return isDnd
 335
 336         def set_dnd(self, doNotDisturb):
 337                 """
 338                 @blocks
 339                 """
 340                 dndPostData = {
 341                         "doNotDisturb": 1 if doNotDisturb else 0,
 342                 }
 343
 344                 dndPage = self._get_page_with_token(self._setDndURL, dndPostData)
 345
 346         def call(self, outgoingNumber):
 347                 """
 348                 This is the main function responsible for initating the callback
 349                 @blocks
 350                 """
 351                 outgoingNumber = self._send_validation(outgoingNumber)
 352                 subscriberNumber = None
 353                 phoneType = guess_phone_type(self._callbackNumber) # @todo Fix this hack
 354
 355                 callData = {
 356                                 'outgoingNumber': outgoingNumber,
 357                                 'forwardingNumber': self._callbackNumber,
 358                                 'subscriberNumber': subscriberNumber or 'undefined',
 359                                 'phoneType': str(phoneType),
 360                                 'remember': '1',
 361                 }
 362                 _moduleLogger.info("%r" % callData)
 363
 364                 page = self._get_page_with_token(
 365                         self._callUrl,
 366                         callData,
 367                 )
 368                 self._parse_with_validation(page)
 369                 return True
 370
 371         def cancel(self, outgoingNumber=None):
 372                 """
 373                 Cancels a call matching outgoing and forwarding numbers (if given).
 374                 Will raise an error if no matching call is being placed
 375                 @blocks
 376                 """
 377                 page = self._get_page_with_token(
 378                         self._callCancelURL,
 379                         {
 380                         'outgoingNumber': outgoingNumber or 'undefined',
 381                         'forwardingNumber': self._callbackNumber or 'undefined',
 382                         'cancelType': 'C2C',
 383                         },
 384                 )
 385                 self._parse_with_validation(page)
 386
 387         def send_sms(self, phoneNumbers, message):
 388                 """
 389                 @blocks
 390                 """
 391                 validatedPhoneNumbers = [
 392                         self._send_validation(phoneNumber)
 393                         for phoneNumber in phoneNumbers
 394                 ]
 395                 flattenedPhoneNumbers = ",".join(validatedPhoneNumbers)
 396                 page = self._get_page_with_token(
 397                         self._sendSmsURL,
 398                         {
 399                                 'phoneNumber': flattenedPhoneNumbers,
 400                                 'text': unicode(message).encode("utf-8"),
 401                         },
 402                 )
 403                 self._parse_with_validation(page)
 404
 405         def search(self, query):
 406                 """
 407                 Search your Google Voice Account history for calls, voicemails, and sms
 408                 Returns ``Folder`` instance containting matching messages
 409                 @blocks
 410                 """
 411                 page = self._get_page(
 412                         self._XML_SEARCH_URL,
 413                         {"q": query},
 414                 )
 415                 json, html = extract_payload(page)
 416                 return json
 417
 418         def get_feed(self, feed):
 419                 """
 420                 @blocks
 421                 """
 422                 actualFeed = "_XML_%s_URL" % feed.upper()
 423                 feedUrl = getattr(self, actualFeed)
 424
 425                 page = self._get_page(feedUrl)
 426                 json, html = extract_payload(page)
 427
 428                 return json
 429
 430         def download(self, messageId, adir):
 431                 """
 432                 Download a voicemail or recorded call MP3 matching the given ``msg``
 433                 which can either be a ``Message`` instance, or a SHA1 identifier.
 434                 Saves files to ``adir`` (defaults to current directory).
 435                 Message hashes can be found in ``self.voicemail().messages`` for example.
 436                 @returns location of saved file.
 437                 @blocks
 438                 """
 439                 page = self._get_page(self._downloadVoicemailURL, {"id": messageId})
 440                 fn = os.path.join(adir, '%s.mp3' % messageId)
 441                 with open(fn, 'wb') as fo:
 442                         fo.write(page)
 443                 return fn
 444
 445         def is_valid_syntax(self, number):
 446                 """
 447                 @returns If This number be called ( syntax validation only )
 448                 """
 449                 return self._validateRe.match(number) is not None
 450
 451         def get_account_number(self):
 452                 """
 453                 @returns The GoogleVoice phone number
 454                 """
 455                 return self._accountNum
 456
 457         def get_callback_numbers(self):
 458                 """
 459                 @returns a dictionary mapping call back numbers to descriptions
 460                 @note These results are cached for 30 minutes.
 461                 """
 462                 if not self.is_authed():
 463                         return {}
 464                 return self._callbackNumbers
 465
 466         def set_callback_number(self, callbacknumber):
 467                 """
 468                 Set the number that GoogleVoice calls
 469                 @param callbacknumber should be a proper 10 digit number
 470                 """
 471                 self._callbackNumber = callbacknumber
 472                 _moduleLogger.info("Callback number changed: %r" % self._callbackNumber)
 473                 return True
 474
 475         def get_callback_number(self):
 476                 """
 477                 @returns Current callback number or None
 478                 """
 479                 return self._callbackNumber
 480
 481         def get_recent(self):
 482                 """
 483                 @returns Iterable of (personsName, phoneNumber, exact date, relative date, action)
 484                 @blocks
 485                 """
 486                 recentPages = [
 487                         (action, self._get_page(url))
 488                         for action, url in (
 489                                 ("Received", self._XML_RECEIVED_URL),
 490                                 ("Missed", self._XML_MISSED_URL),
 491                                 ("Placed", self._XML_PLACED_URL),
 492                         )
 493                 ]
 494                 return self._parse_recent(recentPages)
 495
 496         def get_contacts(self):
 497                 """
 498                 @returns Iterable of (contact id, contact name)
 499                 @blocks
 500                 """
 501                 page = self._get_page(self._XML_CONTACTS_URL)
 502                 return self._process_contacts(page)
 503
 504         def get_csv_contacts(self):
 505                 data = {
 506                         "groupToExport": "mine",
 507                         "exportType": "ALL",
 508                         "out": "OUTLOOK_CSV",
 509                 }
 510                 contacts = self._get_page(self._CSV_CONTACTS_URL, data)
 511                 return contacts
 512
 513         def get_voicemails(self):
 514                 """
 515                 @blocks
 516                 """
 517                 voicemailPage = self._get_page(self._XML_VOICEMAIL_URL)
 518                 voicemailHtml = self._grab_html(voicemailPage)
 519                 voicemailJson = self._grab_json(voicemailPage)
 520                 if voicemailJson is None:
 521                         return ()
 522                 parsedVoicemail = self._parse_voicemail(voicemailHtml)
 523                 voicemails = self._merge_conversation_sources(parsedVoicemail, voicemailJson)
 524                 return voicemails
 525
 526         def get_texts(self):
 527                 """
 528                 @blocks
 529                 """
 530                 smsPage = self._get_page(self._XML_SMS_URL)
 531                 smsHtml = self._grab_html(smsPage)
 532                 smsJson = self._grab_json(smsPage)
 533                 if smsJson is None:
 534                         return ()
 535                 parsedSms = self._parse_sms(smsHtml)
 536                 smss = self._merge_conversation_sources(parsedSms, smsJson)
 537                 return smss
 538
 539         def mark_message(self, messageId, asRead):
 540                 """
 541                 @blocks
 542                 """
 543                 postData = {
 544                         "read": 1 if asRead else 0,
 545                         "id": messageId,
 546                 }
 547
 548                 markPage = self._get_page(self._markAsReadURL, postData)
 549
 550         def archive_message(self, messageId):
 551                 """
 552                 @blocks
 553                 """
 554                 postData = {
 555                         "id": messageId,
 556                 }
 557
 558                 markPage = self._get_page(self._archiveMessageURL, postData)
 559
 560         def _grab_json(self, flatXml):
 561                 xmlTree = ElementTree.fromstring(flatXml)
 562                 jsonElement = xmlTree.getchildren()[0]
 563                 flatJson = jsonElement.text
 564                 jsonTree = parse_json(flatJson)
 565                 return jsonTree
 566
 567         def _grab_html(self, flatXml):
 568                 xmlTree = ElementTree.fromstring(flatXml)
 569                 htmlElement = xmlTree.getchildren()[1]
 570                 flatHtml = htmlElement.text
 571                 return flatHtml
 572
 573         def _grab_account_info(self, page):
 574                 tokenGroup = self._tokenRe.search(page)
 575                 if tokenGroup is None:
 576                         raise RuntimeError("Could not extract authentication token from GoogleVoice")
 577                 self._token = tokenGroup.group(1)
 578
 579                 anGroup = self._accountNumRe.search(page)
 580                 if anGroup is not None:
 581                         self._accountNum = anGroup.group(1)
 582                 else:
 583                         _moduleLogger.debug("Could not extract account number from GoogleVoice")
 584
 585                 self._callbackNumbers = {}
 586                 for match in self._callbackRe.finditer(page):
 587                         callbackNumber = match.group(2)
 588                         callbackName = match.group(1)
 589                         self._callbackNumbers[callbackNumber] = callbackName
 590                 if len(self._callbackNumbers) == 0:
 591                         _moduleLogger.debug("Could not extract callback numbers from GoogleVoice (the troublesome page follows):\n%s" % page)
 592
 593         def _send_validation(self, number):
 594                 if not self.is_valid_syntax(number):
 595                         raise ValueError('Number is not valid: "%s"' % number)
 596                 elif not self.is_authed():
 597                         raise RuntimeError("Not Authenticated")
 598                 return number
 599
 600         def _parse_recent(self, recentPages):
 601                 for action, flatXml in recentPages:
 602                         allRecentHtml = self._grab_html(flatXml)
 603                         allRecentData = self._parse_history(allRecentHtml)
 604                         for recentCallData in allRecentData:
 605                                 recentCallData["action"] = action
 606                                 yield recentCallData
 607
 608         def _process_contacts(self, page):
 609                 contactsBody = self._contactsBodyRe.search(page)
 610                 if contactsBody is None:
 611                         raise RuntimeError("Could not extract contact information")
 612                 accountData = _fake_parse_json(contactsBody.group(1))
 613                 for contactId, contactDetails in accountData["contacts"].iteritems():
 614                         # A zero contact id is the catch all for unknown contacts
 615                         if contactId != "0":
 616                                 yield contactId, contactDetails
 617
 618         def _parse_history(self, historyHtml):
 619                 splitVoicemail = self._seperateVoicemailsRegex.split(historyHtml)
 620                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 621                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 622                         exactTime = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 623                         exactTime = google_strptime(exactTime)
 624                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 625                         relativeTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 626                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 627                         location = locationGroup.group(1).strip() if locationGroup else ""
 628
 629                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 630                         name = nameGroup.group(1).strip() if nameGroup else ""
 631                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 632                         number = numberGroup.group(1).strip() if numberGroup else ""
 633                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 634                         prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 635                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 636                         contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 637
 638                         yield {
 639                                 "id": messageId.strip(),
 640                                 "contactId": contactId,
 641                                 "name": unescape(name),
 642                                 "time": exactTime,
 643                                 "relTime": relativeTime,
 644                                 "prettyNumber": prettyNumber,
 645                                 "number": number,
 646                                 "location": unescape(location),
 647                         }
 648
 649         @staticmethod
 650         def _interpret_voicemail_regex(group):
 651                 quality, content, number = group.group(2), group.group(3), group.group(4)
 652                 text = MessageText()
 653                 if quality is not None and content is not None:
 654                         text.accuracy = quality
 655                         text.text = content
 656                         return text
 657                 elif number is not None:
 658                         text.accuracy = MessageText.ACCURACY_HIGH
 659                         text.text = number
 660                         return text
 661
 662         def _parse_voicemail(self, voicemailHtml):
 663                 splitVoicemail = self._seperateVoicemailsRegex.split(voicemailHtml)
 664                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 665                         conv = Conversation()
 666                         conv.type = Conversation.TYPE_VOICEMAIL
 667                         conv.id = messageId.strip()
 668
 669                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 670                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 671                         conv.time = google_strptime(exactTimeText)
 672                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 673                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 674                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 675                         conv.location = unescape(locationGroup.group(1).strip() if locationGroup else "")
 676
 677                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 678                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 679                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 680                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 681                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 682                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 683                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 684                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 685
 686                         messageGroups = self._voicemailMessageRegex.finditer(messageHtml)
 687                         messageParts = [
 688                                 self._interpret_voicemail_regex(group)
 689                                 for group in messageGroups
 690                         ] if messageGroups else ((MessageText.ACCURACY_LOW, "No Transcription"), )
 691                         message = Message()
 692                         message.body = messageParts
 693                         message.whoFrom = conv.name
 694                         message.when = conv.time.strftime("%I:%M %p")
 695                         conv.messages = (message, )
 696
 697                         yield conv
 698
 699         @staticmethod
 700         def _interpret_sms_message_parts(fromPart, textPart, timePart):
 701                 text = MessageText()
 702                 text.accuracy = MessageText.ACCURACY_MEDIUM
 703                 text.text = textPart
 704
 705                 message = Message()
 706                 message.body = (text, )
 707                 message.whoFrom = fromPart
 708                 message.when = timePart
 709
 710                 return message
 711
 712         def _parse_sms(self, smsHtml):
 713                 splitSms = self._seperateVoicemailsRegex.split(smsHtml)
 714                 for messageId, messageHtml in itergroup(splitSms[1:], 2):
 715                         conv = Conversation()
 716                         conv.type = Conversation.TYPE_SMS
 717                         conv.id = messageId.strip()
 718
 719                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 720                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 721                         conv.time = google_strptime(exactTimeText)
 722                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 723                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 724                         conv.location = ""
 725
 726                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 727                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 728                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 729                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 730                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 731                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 732                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 733                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 734
 735                         fromGroups = self._smsFromRegex.finditer(messageHtml)
 736                         fromParts = (group.group(1).strip() for group in fromGroups)
 737                         textGroups = self._smsTextRegex.finditer(messageHtml)
 738                         textParts = (group.group(1).strip() for group in textGroups)
 739                         timeGroups = self._smsTimeRegex.finditer(messageHtml)
 740                         timeParts = (group.group(1).strip() for group in timeGroups)
 741
 742                         messageParts = itertools.izip(fromParts, textParts, timeParts)
 743                         messages = [self._interpret_sms_message_parts(*parts) for parts in messageParts]
 744                         conv.messages = messages
 745
 746                         yield conv
 747
 748         @staticmethod
 749         def _merge_conversation_sources(parsedMessages, json):
 750                 for message in parsedMessages:
 751                         jsonItem = json["messages"][message.id]
 752                         message.isRead = jsonItem["isRead"]
 753                         message.isSpam = jsonItem["isSpam"]
 754                         message.isTrash = jsonItem["isTrash"]
 755                         message.isArchived = "inbox" not in jsonItem["labels"]
 756                         yield message
 757
 758         def _get_page(self, url, data = None, refererUrl = None):
 759                 headers = {}
 760                 if refererUrl is not None:
 761                         headers["Referer"] = refererUrl
 762
 763                 encodedData = urllib.urlencode(data) if data is not None else None
 764
 765                 try:
 766                         page = self._browser.download(url, encodedData, None, headers)
 767                 except urllib2.URLError, e:
 768                         _moduleLogger.error("Translating error: %s" % str(e))
 769                         raise NetworkError("%s is not accesible" % url)
 770
 771                 return page
 772
 773         def _get_page_with_token(self, url, data = None, refererUrl = None):
 774                 if data is None:
 775                         data = {}
 776                 data['_rnr_se'] = self._token
 777
 778                 page = self._get_page(url, data, refererUrl)
 779
 780                 return page
 781
 782         def _parse_with_validation(self, page):
 783                 json = parse_json(page)
 784                 validate_response(json)
 785                 return json
 786
 787
 788 _UNESCAPE_ENTITIES = {
 789  "&quot;": '"',
 790  "&nbsp;": " ",
 791  "&#39;": "'",
 792 }
 793
 794
 795 def unescape(text):
 796         plain = saxutils.unescape(text, _UNESCAPE_ENTITIES)
 797         return plain
 798
 799
 800 def google_strptime(time):
 801         """
 802         Hack: Google always returns the time in the same locale.  Sadly if the
 803         local system's locale is different, there isn't a way to perfectly handle
 804         the time.  So instead we handle implement some time formatting
 805         """
 806         abbrevTime = time[:-3]
 807         parsedTime = datetime.datetime.strptime(abbrevTime, "%m/%d/%y %I:%M")
 808         if time.endswith("PM"):
 809                 parsedTime += datetime.timedelta(hours=12)
 810         return parsedTime
 811
 812
 813 def itergroup(iterator, count, padValue = None):
 814         """
 815         Iterate in groups of 'count' values. If there
 816         aren't enough values, the last result is padded with
 817         None.
 818
 819         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 820         ...     print tuple(val)
 821         (1, 2, 3)
 822         (4, 5, 6)
 823         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 824         ...     print list(val)
 825         [1, 2, 3]
 826         [4, 5, 6]
 827         >>> for val in itergroup([1, 2, 3, 4, 5, 6, 7], 3):
 828         ...     print tuple(val)
 829         (1, 2, 3)
 830         (4, 5, 6)
 831         (7, None, None)
 832         >>> for val in itergroup("123456", 3):
 833         ...     print tuple(val)
 834         ('1', '2', '3')
 835         ('4', '5', '6')
 836         >>> for val in itergroup("123456", 3):
 837         ...     print repr("".join(val))
 838         '123'
 839         '456'
 840         """
 841         paddedIterator = itertools.chain(iterator, itertools.repeat(padValue, count-1))
 842         nIterators = (paddedIterator, ) * count
 843         return itertools.izip(*nIterators)
 844
 845
 846 def safe_eval(s):
 847         _TRUE_REGEX = re.compile("true")
 848         _FALSE_REGEX = re.compile("false")
 849         _COMMENT_REGEX = re.compile("^\s+//.*$", re.M)
 850         s = _TRUE_REGEX.sub("True", s)
 851         s = _FALSE_REGEX.sub("False", s)
 852         s = _COMMENT_REGEX.sub("#", s)
 853         try:
 854                 results = eval(s, {}, {})
 855         except SyntaxError:
 856                 _moduleLogger.exception("Oops")
 857                 results = None
 858         return results
 859
 860
 861 def _fake_parse_json(flattened):
 862         return safe_eval(flattened)
 863
 864
 865 def _actual_parse_json(flattened):
 866         return simplejson.loads(flattened)
 867
 868
 869 if simplejson is None:
 870         parse_json = _fake_parse_json
 871 else:
 872         parse_json = _actual_parse_json
 873
 874
 875 def extract_payload(flatXml):
 876         xmlTree = ElementTree.fromstring(flatXml)
 877
 878         jsonElement = xmlTree.getchildren()[0]
 879         flatJson = jsonElement.text
 880         jsonTree = parse_json(flatJson)
 881
 882         htmlElement = xmlTree.getchildren()[1]
 883         flatHtml = htmlElement.text
 884
 885         return jsonTree, flatHtml
 886
 887
 888 def validate_response(response):
 889         """
 890         Validates that the JSON response is A-OK
 891         """
 892         try:
 893                 assert response is not None
 894                 assert 'ok' in response
 895                 assert response['ok']
 896         except AssertionError:
 897                 raise RuntimeError('There was a problem with GV: %s' % response)
 898
 899
 900 def guess_phone_type(number):
 901         if number.startswith("747") or number.startswith("1747") or number.startswith("+1747"):
 902                 return GVoiceBackend.PHONE_TYPE_GIZMO
 903         else:
 904                 return GVoiceBackend.PHONE_TYPE_MOBILE
 905
 906
 907 def get_sane_callback(backend):
 908         """
 909         Try to set a sane default callback number on these preferences
 910         1) 1747 numbers ( Gizmo )
 911         2) anything with gizmo in the name
 912         3) anything with computer in the name
 913         4) the first value
 914         """
 915         numbers = backend.get_callback_numbers()
 916
 917         priorityOrderedCriteria = [
 918                 ("\+1747", None),
 919                 ("1747", None),
 920                 ("747", None),
 921                 (None, "gizmo"),
 922                 (None, "computer"),
 923                 (None, "sip"),
 924                 (None, None),
 925         ]
 926
 927         for numberCriteria, descriptionCriteria in priorityOrderedCriteria:
 928                 numberMatcher = None
 929                 descriptionMatcher = None
 930                 if numberCriteria is not None:
 931                         numberMatcher = re.compile(numberCriteria)
 932                 elif descriptionCriteria is not None:
 933                         descriptionMatcher = re.compile(descriptionCriteria, re.I)
 934
 935                 for number, description in numbers.iteritems():
 936                         if numberMatcher is not None and numberMatcher.match(number) is None:
 937                                 continue
 938                         if descriptionMatcher is not None and descriptionMatcher.match(description) is None:
 939                                 continue
 940                         return number
 941
 942
 943 def set_sane_callback(backend):
 944         """
 945         Try to set a sane default callback number on these preferences
 946         1) 1747 numbers ( Gizmo )
 947         2) anything with gizmo in the name
 948         3) anything with computer in the name
 949         4) the first value
 950         """
 951         number = get_sane_callback(backend)
 952         backend.set_callback_number(number)
 953
 954
 955 def _is_not_special(name):
 956         return not name.startswith("_") and name[0].lower() == name[0] and "_" not in name
 957
 958
 959 def to_dict(obj):
 960         members = inspect.getmembers(obj)
 961         return dict((name, value) for (name, value) in members if _is_not_special(name))
 962
 963
 964 def grab_debug_info(username, password):
 965         cookieFile = os.path.join(".", "raw_cookies.txt")
 966         try:
 967                 os.remove(cookieFile)
 968         except OSError:
 969                 pass
 970
 971         backend = GVoiceBackend(cookieFile)
 972         browser = backend._browser
 973
 974         _TEST_WEBPAGES = [
 975                 ("forward", backend._forwardURL),
 976                 ("token", backend._tokenURL),
 977                 ("login", backend._loginURL),
 978                 ("isdnd", backend._isDndURL),
 979                 ("account", backend._XML_ACCOUNT_URL),
 980                 ("contacts", backend._XML_CONTACTS_URL),
 981                 ("csv", backend._CSV_CONTACTS_URL),
 982
 983                 ("voicemail", backend._XML_VOICEMAIL_URL),
 984                 ("sms", backend._XML_SMS_URL),
 985
 986                 ("recent", backend._XML_RECENT_URL),
 987                 ("placed", backend._XML_PLACED_URL),
 988                 ("recieved", backend._XML_RECEIVED_URL),
 989                 ("missed", backend._XML_MISSED_URL),
 990         ]
 991
 992         # Get Pages
 993         print "Grabbing pre-login pages"
 994         for name, url in _TEST_WEBPAGES:
 995                 try:
 996                         page = browser.download(url)
 997                 except StandardError, e:
 998                         print e.message
 999                         continue
1000                 print "\tWriting to file"
1001                 with open("not_loggedin_%s.txt" % name, "w") as f:
1002                         f.write(page)
1003
1004         # Login
1005         print "Attempting login"
1006         galxToken = backend._get_token()
1007         loginSuccessOrFailurePage = backend._login(username, password, galxToken)
1008         with open("loggingin.txt", "w") as f:
1009                 print "\tWriting to file"
1010                 f.write(loginSuccessOrFailurePage)
1011         try:
1012                 backend._grab_account_info(loginSuccessOrFailurePage)
1013         except Exception:
1014                 # Retry in case the redirect failed
1015                 # luckily is_authed does everything we need for a retry
1016                 loggedIn = backend.is_authed(True)
1017                 if not loggedIn:
1018                         raise
1019
1020         # Get Pages
1021         print "Grabbing post-login pages"
1022         for name, url in _TEST_WEBPAGES:
1023                 try:
1024                         page = browser.download(url)
1025                 except StandardError, e:
1026                         print str(e)
1027                         continue
1028                 print "\tWriting to file"
1029                 with open("loggedin_%s.txt" % name, "w") as f:
1030                         f.write(page)
1031
1032         # Cookies
1033         browser.save_cookies()
1034         print "\tWriting cookies to file"
1035         with open("cookies.txt", "w") as f:
1036                 f.writelines(
1037                         "%s: %s\n" % (c.name, c.value)
1038                         for c in browser._cookies
1039                 )
1040
1041
1042 def main():
1043         import sys
1044         logging.basicConfig(level=logging.DEBUG)
1045         args = sys.argv
1046         if 3 <= len(args):
1047                 username = args[1]
1048                 password = args[2]
1049
1050         grab_debug_info(username, password)
1051
1052
1053 if __name__ == "__main__":
1054         main()