git.maemo.org Git - theonering/blob - src/gvoice/backend.py

   1 #!/usr/bin/python
   2
   3 """
   4 DialCentral - Front end for Google's GoogleVoice service.
   5 Copyright (C) 2008  Eric Warnke ericew AT gmail DOT com
   6
   7 This library is free software; you can redistribute it and/or
   8 modify it under the terms of the GNU Lesser General Public
   9 License as published by the Free Software Foundation; either
  10 version 2.1 of the License, or (at your option) any later version.
  11
  12 This library is distributed in the hope that it will be useful,
  13 but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 Lesser General Public License for more details.
  16
  17 You should have received a copy of the GNU Lesser General Public
  18 License along with this library; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 Google Voice backend code
  22
  23 Resources
  24         http://thatsmith.com/2009/03/google-voice-addon-for-firefox/
  25         http://posttopic.com/topic/google-voice-add-on-development
  26 """
  27
  28 from __future__ import with_statement
  29
  30 import os
  31 import re
  32 import urllib
  33 import urllib2
  34 import time
  35 import datetime
  36 import itertools
  37 import logging
  38 import inspect
  39
  40 from xml.sax import saxutils
  41 from xml.etree import ElementTree
  42
  43 try:
  44         import simplejson as _simplejson
  45         simplejson = _simplejson
  46 except ImportError:
  47         simplejson = None
  48
  49 import browser_emu
  50
  51
  52 _moduleLogger = logging.getLogger(__name__)
  53
  54
  55 class NetworkError(RuntimeError):
  56         pass
  57
  58
  59 class MessageText(object):
  60
  61         ACCURACY_LOW = "med1"
  62         ACCURACY_MEDIUM = "med2"
  63         ACCURACY_HIGH = "high"
  64
  65         def __init__(self):
  66                 self.accuracy = None
  67                 self.text = None
  68
  69         def __str__(self):
  70                 return self.text
  71
  72         def to_dict(self):
  73                 return to_dict(self)
  74
  75         def __eq__(self, other):
  76                 return self.accuracy == other.accuracy and self.text == other.text
  77
  78
  79 class Message(object):
  80
  81         def __init__(self):
  82                 self.whoFrom = None
  83                 self.body = None
  84                 self.when = None
  85
  86         def __str__(self):
  87                 return "%s (%s): %s" % (
  88                         self.whoFrom,
  89                         self.when,
  90                         "".join(unicode(part) for part in self.body)
  91                 )
  92
  93         def to_dict(self):
  94                 selfDict = to_dict(self)
  95                 selfDict["body"] = [text.to_dict() for text in self.body] if self.body is not None else None
  96                 return selfDict
  97
  98         def __eq__(self, other):
  99                 return self.whoFrom == other.whoFrom and self.when == other.when and self.body == other.body
 100
 101
 102 class Conversation(object):
 103
 104         TYPE_VOICEMAIL = "Voicemail"
 105         TYPE_SMS = "SMS"
 106
 107         def __init__(self):
 108                 self.type = None
 109                 self.id = None
 110                 self.contactId = None
 111                 self.name = None
 112                 self.location = None
 113                 self.prettyNumber = None
 114                 self.number = None
 115
 116                 self.time = None
 117                 self.relTime = None
 118                 self.messages = None
 119                 self.isRead = None
 120                 self.isSpam = None
 121                 self.isTrash = None
 122                 self.isArchived = None
 123
 124         def __cmp__(self, other):
 125                 cmpValue = cmp(self.contactId, other.contactId)
 126                 if cmpValue != 0:
 127                         return cmpValue
 128
 129                 cmpValue = cmp(self.time, other.time)
 130                 if cmpValue != 0:
 131                         return cmpValue
 132
 133                 cmpValue = cmp(self.id, other.id)
 134                 if cmpValue != 0:
 135                         return cmpValue
 136
 137         def to_dict(self):
 138                 selfDict = to_dict(self)
 139                 selfDict["messages"] = [message.to_dict() for message in self.messages] if self.messages is not None else None
 140                 return selfDict
 141
 142
 143 class GVoiceBackend(object):
 144         """
 145         This class encapsulates all of the knowledge necessary to interact with the GoogleVoice servers
 146         the functions include login, setting up a callback number, and initalting a callback
 147         """
 148
 149         PHONE_TYPE_HOME = 1
 150         PHONE_TYPE_MOBILE = 2
 151         PHONE_TYPE_WORK = 3
 152         PHONE_TYPE_GIZMO = 7
 153
 154         def __init__(self, cookieFile = None):
 155                 # Important items in this function are the setup of the browser emulation and cookie file
 156                 self._browser = browser_emu.MozillaEmulator(1)
 157                 self._loadedFromCookies = self._browser.load_cookies(cookieFile)
 158
 159                 self._token = ""
 160                 self._accountNum = ""
 161                 self._lastAuthed = 0.0
 162                 self._callbackNumber = ""
 163                 self._callbackNumbers = {}
 164
 165                 # Suprisingly, moving all of these from class to self sped up startup time
 166
 167                 self._validateRe = re.compile("^\+?[0-9]{10,}$")
 168
 169                 self._loginURL = "https://www.google.com/accounts/ServiceLoginAuth"
 170
 171                 SECURE_URL_BASE = "https://www.google.com/voice/"
 172                 SECURE_MOBILE_URL_BASE = SECURE_URL_BASE + "mobile/"
 173                 self._forwardURL = SECURE_MOBILE_URL_BASE + "phones"
 174                 self._tokenURL = SECURE_URL_BASE + "m"
 175                 self._callUrl = SECURE_URL_BASE + "call/connect"
 176                 self._callCancelURL = SECURE_URL_BASE + "call/cancel"
 177                 self._sendSmsURL = SECURE_URL_BASE + "sms/send"
 178
 179                 self._isDndURL = "https://www.google.com/voice/m/donotdisturb"
 180                 self._isDndRe = re.compile(r"""<input.*?id="doNotDisturb".*?checked="(.*?)"\s*/>""")
 181                 self._setDndURL = "https://www.google.com/voice/m/savednd"
 182
 183                 self._downloadVoicemailURL = SECURE_URL_BASE + "media/send_voicemail/"
 184                 self._markAsReadURL = SECURE_URL_BASE + "m/mark"
 185                 self._archiveMessageURL = SECURE_URL_BASE + "m/archive"
 186
 187                 self._XML_SEARCH_URL = SECURE_URL_BASE + "inbox/search/"
 188                 self._XML_ACCOUNT_URL = SECURE_URL_BASE + "contacts/"
 189                 # HACK really this redirects to the main pge and we are grabbing some javascript
 190                 self._XML_CONTACTS_URL = "http://www.google.com/voice/inbox/search/contact"
 191                 self._CSV_CONTACTS_URL = "http://mail.google.com/mail/contacts/data/export"
 192                 self._XML_RECENT_URL = SECURE_URL_BASE + "inbox/recent/"
 193
 194                 self.XML_FEEDS = (
 195                         'inbox', 'starred', 'all', 'spam', 'trash', 'voicemail', 'sms',
 196                         'recorded', 'placed', 'received', 'missed'
 197                 )
 198                 self._XML_INBOX_URL = SECURE_URL_BASE + "inbox/recent/inbox"
 199                 self._XML_STARRED_URL = SECURE_URL_BASE + "inbox/recent/starred"
 200                 self._XML_ALL_URL = SECURE_URL_BASE + "inbox/recent/all"
 201                 self._XML_SPAM_URL = SECURE_URL_BASE + "inbox/recent/spam"
 202                 self._XML_TRASH_URL = SECURE_URL_BASE + "inbox/recent/trash"
 203                 self._XML_VOICEMAIL_URL = SECURE_URL_BASE + "inbox/recent/voicemail/"
 204                 self._XML_SMS_URL = SECURE_URL_BASE + "inbox/recent/sms/"
 205                 self._XML_RECORDED_URL = SECURE_URL_BASE + "inbox/recent/recorded/"
 206                 self._XML_PLACED_URL = SECURE_URL_BASE + "inbox/recent/placed/"
 207                 self._XML_RECEIVED_URL = SECURE_URL_BASE + "inbox/recent/received/"
 208                 self._XML_MISSED_URL = SECURE_URL_BASE + "inbox/recent/missed/"
 209
 210                 self._galxRe = re.compile(r"""<input.*?name="GALX".*?value="(.*?)".*?/>""", re.MULTILINE | re.DOTALL)
 211                 self._tokenRe = re.compile(r"""<input.*?name="_rnr_se".*?value="(.*?)"\s*/>""")
 212                 self._accountNumRe = re.compile(r"""<b class="ms\d">(.{14})</b></div>""")
 213                 self._callbackRe = re.compile(r"""\s+(.*?):\s*(.*?)<br\s*/>\s*$""", re.M)
 214
 215                 self._contactsBodyRe = re.compile(r"""gcData\s*=\s*({.*?});""", re.MULTILINE | re.DOTALL)
 216                 self._seperateVoicemailsRegex = re.compile(r"""^\s*<div id="(\w+)"\s* class=".*?gc-message.*?">""", re.MULTILINE | re.DOTALL)
 217                 self._exactVoicemailTimeRegex = re.compile(r"""<span class="gc-message-time">(.*?)</span>""", re.MULTILINE)
 218                 self._relativeVoicemailTimeRegex = re.compile(r"""<span class="gc-message-relative">(.*?)</span>""", re.MULTILINE)
 219                 self._voicemailNameRegex = re.compile(r"""<a class=.*?gc-message-name-link.*?>(.*?)</a>""", re.MULTILINE | re.DOTALL)
 220                 self._voicemailNumberRegex = re.compile(r"""<input type="hidden" class="gc-text gc-quickcall-ac" value="(.*?)"/>""", re.MULTILINE)
 221                 self._prettyVoicemailNumberRegex = re.compile(r"""<span class="gc-message-type">(.*?)</span>""", re.MULTILINE)
 222                 self._voicemailLocationRegex = re.compile(r"""<span class="gc-message-location">.*?<a.*?>(.*?)</a></span>""", re.MULTILINE)
 223                 self._messagesContactIDRegex = re.compile(r"""<a class=".*?gc-message-name-link.*?">.*?</a>\s*?<span .*?>(.*?)</span>""", re.MULTILINE)
 224                 self._voicemailMessageRegex = re.compile(r"""(<span id="\d+-\d+" class="gc-word-(.*?)">(.*?)</span>|<a .*? class="gc-message-mni">(.*?)</a>)""", re.MULTILINE)
 225                 self._smsFromRegex = re.compile(r"""<span class="gc-message-sms-from">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 226                 self._smsTimeRegex = re.compile(r"""<span class="gc-message-sms-time">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 227                 self._smsTextRegex = re.compile(r"""<span class="gc-message-sms-text">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 228
 229         def is_quick_login_possible(self):
 230                 """
 231                 @returns True then is_authed might be enough to login, else full login is required
 232                 """
 233                 return self._loadedFromCookies or 0.0 < self._lastAuthed
 234
 235         def is_authed(self, force = False):
 236                 """
 237                 Attempts to detect a current session
 238                 @note Once logged in try not to reauth more than once a minute.
 239                 @returns If authenticated
 240                 @blocks
 241                 """
 242                 isRecentledAuthed = (time.time() - self._lastAuthed) < 120
 243                 isPreviouslyAuthed = self._token is not None
 244                 if isRecentledAuthed and isPreviouslyAuthed and not force:
 245                         return True
 246
 247                 try:
 248                         page = self._get_page(self._forwardURL)
 249                         self._grab_account_info(page)
 250                 except Exception, e:
 251                         _moduleLogger.exception(str(e))
 252                         return False
 253
 254                 self._browser.save_cookies()
 255                 self._lastAuthed = time.time()
 256                 return True
 257
 258         def _get_token(self):
 259                 tokenPage = self._get_page(self._tokenURL)
 260
 261                 galxTokens = self._galxRe.search(tokenPage)
 262                 if galxTokens is not None:
 263                         galxToken = galxTokens.group(1)
 264                 else:
 265                         galxToken = ""
 266                         _moduleLogger.debug("Could not grab GALX token")
 267                 return galxToken
 268
 269         def _login(self, username, password, token):
 270                 loginData = {
 271                         'Email' : username,
 272                         'Passwd' : password,
 273                         'service': "grandcentral",
 274                         "ltmpl": "mobile",
 275                         "btmpl": "mobile",
 276                         "PersistentCookie": "yes",
 277                         "GALX": token,
 278                         "continue": self._forwardURL,
 279                 }
 280
 281                 loginSuccessOrFailurePage = self._get_page(self._loginURL, loginData)
 282                 return loginSuccessOrFailurePage
 283
 284         def login(self, username, password):
 285                 """
 286                 Attempt to login to GoogleVoice
 287                 @returns Whether login was successful or not
 288                 @blocks
 289                 """
 290                 self.logout()
 291                 galxToken = self._get_token()
 292                 loginSuccessOrFailurePage = self._login(username, password, galxToken)
 293
 294                 try:
 295                         self._grab_account_info(loginSuccessOrFailurePage)
 296                 except Exception, e:
 297                         # Retry in case the redirect failed
 298                         # luckily is_authed does everything we need for a retry
 299                         loggedIn = self.is_authed(True)
 300                         if not loggedIn:
 301                                 _moduleLogger.exception(str(e))
 302                                 return False
 303                         _moduleLogger.info("Redirection failed on initial login attempt, auto-corrected for this")
 304
 305                 self._browser.save_cookies()
 306                 self._lastAuthed = time.time()
 307                 return True
 308
 309         def persist(self):
 310                 self._browser.save_cookies()
 311
 312         def shutdown(self):
 313                 self._browser.save_cookies()
 314                 self._token = None
 315                 self._lastAuthed = 0.0
 316
 317         def logout(self):
 318                 self._browser.clear_cookies()
 319                 self._browser.save_cookies()
 320                 self._token = None
 321                 self._lastAuthed = 0.0
 322
 323         def is_dnd(self):
 324                 """
 325                 @blocks
 326                 """
 327                 isDndPage = self._get_page(self._isDndURL)
 328
 329                 dndGroup = self._isDndRe.search(isDndPage)
 330                 if dndGroup is None:
 331                         return False
 332                 dndStatus = dndGroup.group(1)
 333                 isDnd = True if dndStatus.strip().lower() == "true" else False
 334                 return isDnd
 335
 336         def set_dnd(self, doNotDisturb):
 337                 """
 338                 @blocks
 339                 """
 340                 dndPostData = {
 341                         "doNotDisturb": 1 if doNotDisturb else 0,
 342                 }
 343
 344                 dndPage = self._get_page_with_token(self._setDndURL, dndPostData)
 345
 346         def call(self, outgoingNumber):
 347                 """
 348                 This is the main function responsible for initating the callback
 349                 @blocks
 350                 """
 351                 outgoingNumber = self._send_validation(outgoingNumber)
 352                 subscriberNumber = None
 353                 phoneType = guess_phone_type(self._callbackNumber) # @todo Fix this hack
 354
 355                 callData = {
 356                                 'outgoingNumber': outgoingNumber,
 357                                 'forwardingNumber': self._callbackNumber,
 358                                 'subscriberNumber': subscriberNumber or 'undefined',
 359                                 'phoneType': str(phoneType),
 360                                 'remember': '1',
 361                 }
 362                 _moduleLogger.info("%r" % callData)
 363
 364                 page = self._get_page_with_token(
 365                         self._callUrl,
 366                         callData,
 367                 )
 368                 self._parse_with_validation(page)
 369                 return True
 370
 371         def cancel(self, outgoingNumber=None):
 372                 """
 373                 Cancels a call matching outgoing and forwarding numbers (if given).
 374                 Will raise an error if no matching call is being placed
 375                 @blocks
 376                 """
 377                 page = self._get_page_with_token(
 378                         self._callCancelURL,
 379                         {
 380                         'outgoingNumber': outgoingNumber or 'undefined',
 381                         'forwardingNumber': self._callbackNumber or 'undefined',
 382                         'cancelType': 'C2C',
 383                         },
 384                 )
 385                 self._parse_with_validation(page)
 386
 387         def send_sms(self, phoneNumbers, message):
 388                 """
 389                 @blocks
 390                 """
 391                 validatedPhoneNumbers = [
 392                         self._send_validation(phoneNumber)
 393                         for phoneNumber in phoneNumbers
 394                 ]
 395                 flattenedPhoneNumbers = ",".join(validatedPhoneNumbers)
 396                 page = self._get_page_with_token(
 397                         self._sendSmsURL,
 398                         {
 399                                 'phoneNumber': flattenedPhoneNumbers,
 400                                 'text': unicode(message).encode("utf-8"),
 401                         },
 402                 )
 403                 self._parse_with_validation(page)
 404
 405         def search(self, query):
 406                 """
 407                 Search your Google Voice Account history for calls, voicemails, and sms
 408                 Returns ``Folder`` instance containting matching messages
 409                 @blocks
 410                 """
 411                 page = self._get_page(
 412                         self._XML_SEARCH_URL,
 413                         {"q": query},
 414                 )
 415                 json, html = extract_payload(page)
 416                 return json
 417
 418         def get_feed(self, feed):
 419                 """
 420                 @blocks
 421                 """
 422                 actualFeed = "_XML_%s_URL" % feed.upper()
 423                 feedUrl = getattr(self, actualFeed)
 424
 425                 page = self._get_page(feedUrl)
 426                 json, html = extract_payload(page)
 427
 428                 return json
 429
 430         def download(self, messageId, adir):
 431                 """
 432                 Download a voicemail or recorded call MP3 matching the given ``msg``
 433                 which can either be a ``Message`` instance, or a SHA1 identifier.
 434                 Saves files to ``adir`` (defaults to current directory).
 435                 Message hashes can be found in ``self.voicemail().messages`` for example.
 436                 @returns location of saved file.
 437                 @blocks
 438                 """
 439                 page = self._get_page(self._downloadVoicemailURL, {"id": messageId})
 440                 fn = os.path.join(adir, '%s.mp3' % messageId)
 441                 with open(fn, 'wb') as fo:
 442                         fo.write(page)
 443                 return fn
 444
 445         def is_valid_syntax(self, number):
 446                 """
 447                 @returns If This number be called ( syntax validation only )
 448                 """
 449                 return self._validateRe.match(number) is not None
 450
 451         def get_account_number(self):
 452                 """
 453                 @returns The GoogleVoice phone number
 454                 """
 455                 return self._accountNum
 456
 457         def get_callback_numbers(self):
 458                 """
 459                 @returns a dictionary mapping call back numbers to descriptions
 460                 @note These results are cached for 30 minutes.
 461                 """
 462                 if not self.is_authed():
 463                         return {}
 464                 return self._callbackNumbers
 465
 466         def set_callback_number(self, callbacknumber):
 467                 """
 468                 Set the number that GoogleVoice calls
 469                 @param callbacknumber should be a proper 10 digit number
 470                 """
 471                 self._callbackNumber = callbacknumber
 472                 _moduleLogger.info("Callback number changed: %r" % self._callbackNumber)
 473                 return True
 474
 475         def get_callback_number(self):
 476                 """
 477                 @returns Current callback number or None
 478                 """
 479                 return self._callbackNumber
 480
 481         def get_recent(self):
 482                 """
 483                 @returns Iterable of (personsName, phoneNumber, exact date, relative date, action)
 484                 @blocks
 485                 """
 486                 recentPages = [
 487                         (action, self._get_page(url))
 488                         for action, url in (
 489                                 ("Received", self._XML_RECEIVED_URL),
 490                                 ("Missed", self._XML_MISSED_URL),
 491                                 ("Placed", self._XML_PLACED_URL),
 492                         )
 493                 ]
 494                 return self._parse_recent(recentPages)
 495
 496         def get_contacts(self):
 497                 """
 498                 @returns Iterable of (contact id, contact name)
 499                 @blocks
 500                 """
 501                 page = self._get_page(self._XML_CONTACTS_URL)
 502                 return self._process_contacts(page)
 503
 504         def get_csv_contacts(self):
 505                 data = {
 506                         "groupToExport": "mine",
 507                         "exportType": "ALL",
 508                         "out": "OUTLOOK_CSV",
 509                 }
 510                 encodedData = urllib.urlencode(data)
 511                 contacts = self._get_page(self._CSV_CONTACTS_URL+"?"+encodedData)
 512                 return contacts
 513
 514         def get_voicemails(self):
 515                 """
 516                 @blocks
 517                 """
 518                 voicemailPage = self._get_page(self._XML_VOICEMAIL_URL)
 519                 voicemailHtml = self._grab_html(voicemailPage)
 520                 voicemailJson = self._grab_json(voicemailPage)
 521                 if voicemailJson is None:
 522                         return ()
 523                 parsedVoicemail = self._parse_voicemail(voicemailHtml)
 524                 voicemails = self._merge_conversation_sources(parsedVoicemail, voicemailJson)
 525                 return voicemails
 526
 527         def get_texts(self):
 528                 """
 529                 @blocks
 530                 """
 531                 smsPage = self._get_page(self._XML_SMS_URL)
 532                 smsHtml = self._grab_html(smsPage)
 533                 smsJson = self._grab_json(smsPage)
 534                 if smsJson is None:
 535                         return ()
 536                 parsedSms = self._parse_sms(smsHtml)
 537                 smss = self._merge_conversation_sources(parsedSms, smsJson)
 538                 return smss
 539
 540         def mark_message(self, messageId, asRead):
 541                 """
 542                 @blocks
 543                 """
 544                 postData = {
 545                         "read": 1 if asRead else 0,
 546                         "id": messageId,
 547                 }
 548
 549                 markPage = self._get_page(self._markAsReadURL, postData)
 550
 551         def archive_message(self, messageId):
 552                 """
 553                 @blocks
 554                 """
 555                 postData = {
 556                         "id": messageId,
 557                 }
 558
 559                 markPage = self._get_page(self._archiveMessageURL, postData)
 560
 561         def _grab_json(self, flatXml):
 562                 xmlTree = ElementTree.fromstring(flatXml)
 563                 jsonElement = xmlTree.getchildren()[0]
 564                 flatJson = jsonElement.text
 565                 jsonTree = parse_json(flatJson)
 566                 return jsonTree
 567
 568         def _grab_html(self, flatXml):
 569                 xmlTree = ElementTree.fromstring(flatXml)
 570                 htmlElement = xmlTree.getchildren()[1]
 571                 flatHtml = htmlElement.text
 572                 return flatHtml
 573
 574         def _grab_account_info(self, page):
 575                 tokenGroup = self._tokenRe.search(page)
 576                 if tokenGroup is None:
 577                         raise RuntimeError("Could not extract authentication token from GoogleVoice")
 578                 self._token = tokenGroup.group(1)
 579
 580                 anGroup = self._accountNumRe.search(page)
 581                 if anGroup is not None:
 582                         self._accountNum = anGroup.group(1)
 583                 else:
 584                         _moduleLogger.debug("Could not extract account number from GoogleVoice")
 585
 586                 self._callbackNumbers = {}
 587                 for match in self._callbackRe.finditer(page):
 588                         callbackNumber = match.group(2)
 589                         callbackName = match.group(1)
 590                         self._callbackNumbers[callbackNumber] = callbackName
 591                 if len(self._callbackNumbers) == 0:
 592                         _moduleLogger.debug("Could not extract callback numbers from GoogleVoice (the troublesome page follows):\n%s" % page)
 593
 594         def _send_validation(self, number):
 595                 if not self.is_valid_syntax(number):
 596                         raise ValueError('Number is not valid: "%s"' % number)
 597                 elif not self.is_authed():
 598                         raise RuntimeError("Not Authenticated")
 599                 return number
 600
 601         def _parse_recent(self, recentPages):
 602                 for action, flatXml in recentPages:
 603                         allRecentHtml = self._grab_html(flatXml)
 604                         allRecentData = self._parse_history(allRecentHtml)
 605                         for recentCallData in allRecentData:
 606                                 recentCallData["action"] = action
 607                                 yield recentCallData
 608
 609         def _process_contacts(self, page):
 610                 contactsBody = self._contactsBodyRe.search(page)
 611                 if contactsBody is None:
 612                         raise RuntimeError("Could not extract contact information")
 613                 accountData = _fake_parse_json(contactsBody.group(1))
 614                 for contactId, contactDetails in accountData["contacts"].iteritems():
 615                         # A zero contact id is the catch all for unknown contacts
 616                         if contactId != "0":
 617                                 yield contactId, contactDetails
 618
 619         def _parse_history(self, historyHtml):
 620                 splitVoicemail = self._seperateVoicemailsRegex.split(historyHtml)
 621                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 622                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 623                         exactTime = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 624                         exactTime = google_strptime(exactTime)
 625                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 626                         relativeTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 627                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 628                         location = locationGroup.group(1).strip() if locationGroup else ""
 629
 630                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 631                         name = nameGroup.group(1).strip() if nameGroup else ""
 632                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 633                         number = numberGroup.group(1).strip() if numberGroup else ""
 634                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 635                         prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 636                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 637                         contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 638
 639                         yield {
 640                                 "id": messageId.strip(),
 641                                 "contactId": contactId,
 642                                 "name": unescape(name),
 643                                 "time": exactTime,
 644                                 "relTime": relativeTime,
 645                                 "prettyNumber": prettyNumber,
 646                                 "number": number,
 647                                 "location": unescape(location),
 648                         }
 649
 650         @staticmethod
 651         def _interpret_voicemail_regex(group):
 652                 quality, content, number = group.group(2), group.group(3), group.group(4)
 653                 text = MessageText()
 654                 if quality is not None and content is not None:
 655                         text.accuracy = quality
 656                         text.text = content
 657                         return text
 658                 elif number is not None:
 659                         text.accuracy = MessageText.ACCURACY_HIGH
 660                         text.text = number
 661                         return text
 662
 663         def _parse_voicemail(self, voicemailHtml):
 664                 splitVoicemail = self._seperateVoicemailsRegex.split(voicemailHtml)
 665                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 666                         conv = Conversation()
 667                         conv.type = Conversation.TYPE_VOICEMAIL
 668                         conv.id = messageId.strip()
 669
 670                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 671                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 672                         conv.time = google_strptime(exactTimeText)
 673                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 674                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 675                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 676                         conv.location = unescape(locationGroup.group(1).strip() if locationGroup else "")
 677
 678                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 679                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 680                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 681                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 682                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 683                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 684                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 685                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 686
 687                         messageGroups = self._voicemailMessageRegex.finditer(messageHtml)
 688                         messageParts = [
 689                                 self._interpret_voicemail_regex(group)
 690                                 for group in messageGroups
 691                         ] if messageGroups else ((MessageText.ACCURACY_LOW, "No Transcription"), )
 692                         message = Message()
 693                         message.body = messageParts
 694                         message.whoFrom = conv.name
 695                         message.when = conv.time.strftime("%I:%M %p")
 696                         conv.messages = (message, )
 697
 698                         yield conv
 699
 700         @staticmethod
 701         def _interpret_sms_message_parts(fromPart, textPart, timePart):
 702                 text = MessageText()
 703                 text.accuracy = MessageText.ACCURACY_MEDIUM
 704                 text.text = textPart
 705
 706                 message = Message()
 707                 message.body = (text, )
 708                 message.whoFrom = fromPart
 709                 message.when = timePart
 710
 711                 return message
 712
 713         def _parse_sms(self, smsHtml):
 714                 splitSms = self._seperateVoicemailsRegex.split(smsHtml)
 715                 for messageId, messageHtml in itergroup(splitSms[1:], 2):
 716                         conv = Conversation()
 717                         conv.type = Conversation.TYPE_SMS
 718                         conv.id = messageId.strip()
 719
 720                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 721                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 722                         conv.time = google_strptime(exactTimeText)
 723                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 724                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 725                         conv.location = ""
 726
 727                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 728                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 729                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 730                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 731                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 732                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 733                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 734                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 735
 736                         fromGroups = self._smsFromRegex.finditer(messageHtml)
 737                         fromParts = (group.group(1).strip() for group in fromGroups)
 738                         textGroups = self._smsTextRegex.finditer(messageHtml)
 739                         textParts = (group.group(1).strip() for group in textGroups)
 740                         timeGroups = self._smsTimeRegex.finditer(messageHtml)
 741                         timeParts = (group.group(1).strip() for group in timeGroups)
 742
 743                         messageParts = itertools.izip(fromParts, textParts, timeParts)
 744                         messages = [self._interpret_sms_message_parts(*parts) for parts in messageParts]
 745                         conv.messages = messages
 746
 747                         yield conv
 748
 749         @staticmethod
 750         def _merge_conversation_sources(parsedMessages, json):
 751                 for message in parsedMessages:
 752                         jsonItem = json["messages"][message.id]
 753                         message.isRead = jsonItem["isRead"]
 754                         message.isSpam = jsonItem["isSpam"]
 755                         message.isTrash = jsonItem["isTrash"]
 756                         message.isArchived = "inbox" not in jsonItem["labels"]
 757                         yield message
 758
 759         def _get_page(self, url, data = None, refererUrl = None):
 760                 headers = {}
 761                 if refererUrl is not None:
 762                         headers["Referer"] = refererUrl
 763
 764                 encodedData = urllib.urlencode(data) if data is not None else None
 765
 766                 try:
 767                         page = self._browser.download(url, encodedData, None, headers)
 768                 except urllib2.URLError, e:
 769                         _moduleLogger.error("Translating error: %s" % str(e))
 770                         raise NetworkError("%s is not accesible" % url)
 771
 772                 return page
 773
 774         def _get_page_with_token(self, url, data = None, refererUrl = None):
 775                 if data is None:
 776                         data = {}
 777                 data['_rnr_se'] = self._token
 778
 779                 page = self._get_page(url, data, refererUrl)
 780
 781                 return page
 782
 783         def _parse_with_validation(self, page):
 784                 json = parse_json(page)
 785                 validate_response(json)
 786                 return json
 787
 788
 789 _UNESCAPE_ENTITIES = {
 790  "&quot;": '"',
 791  "&nbsp;": " ",
 792  "&#39;": "'",
 793 }
 794
 795
 796 def unescape(text):
 797         plain = saxutils.unescape(text, _UNESCAPE_ENTITIES)
 798         return plain
 799
 800
 801 def google_strptime(time):
 802         """
 803         Hack: Google always returns the time in the same locale.  Sadly if the
 804         local system's locale is different, there isn't a way to perfectly handle
 805         the time.  So instead we handle implement some time formatting
 806         """
 807         abbrevTime = time[:-3]
 808         parsedTime = datetime.datetime.strptime(abbrevTime, "%m/%d/%y %I:%M")
 809         if time.endswith("PM"):
 810                 parsedTime += datetime.timedelta(hours=12)
 811         return parsedTime
 812
 813
 814 def itergroup(iterator, count, padValue = None):
 815         """
 816         Iterate in groups of 'count' values. If there
 817         aren't enough values, the last result is padded with
 818         None.
 819
 820         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 821         ...     print tuple(val)
 822         (1, 2, 3)
 823         (4, 5, 6)
 824         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 825         ...     print list(val)
 826         [1, 2, 3]
 827         [4, 5, 6]
 828         >>> for val in itergroup([1, 2, 3, 4, 5, 6, 7], 3):
 829         ...     print tuple(val)
 830         (1, 2, 3)
 831         (4, 5, 6)
 832         (7, None, None)
 833         >>> for val in itergroup("123456", 3):
 834         ...     print tuple(val)
 835         ('1', '2', '3')
 836         ('4', '5', '6')
 837         >>> for val in itergroup("123456", 3):
 838         ...     print repr("".join(val))
 839         '123'
 840         '456'
 841         """
 842         paddedIterator = itertools.chain(iterator, itertools.repeat(padValue, count-1))
 843         nIterators = (paddedIterator, ) * count
 844         return itertools.izip(*nIterators)
 845
 846
 847 def safe_eval(s):
 848         _TRUE_REGEX = re.compile("true")
 849         _FALSE_REGEX = re.compile("false")
 850         _COMMENT_REGEX = re.compile("^\s+//.*$", re.M)
 851         s = _TRUE_REGEX.sub("True", s)
 852         s = _FALSE_REGEX.sub("False", s)
 853         s = _COMMENT_REGEX.sub("#", s)
 854         try:
 855                 results = eval(s, {}, {})
 856         except SyntaxError:
 857                 _moduleLogger.exception("Oops")
 858                 results = None
 859         return results
 860
 861
 862 def _fake_parse_json(flattened):
 863         return safe_eval(flattened)
 864
 865
 866 def _actual_parse_json(flattened):
 867         return simplejson.loads(flattened)
 868
 869
 870 if simplejson is None:
 871         parse_json = _fake_parse_json
 872 else:
 873         parse_json = _actual_parse_json
 874
 875
 876 def extract_payload(flatXml):
 877         xmlTree = ElementTree.fromstring(flatXml)
 878
 879         jsonElement = xmlTree.getchildren()[0]
 880         flatJson = jsonElement.text
 881         jsonTree = parse_json(flatJson)
 882
 883         htmlElement = xmlTree.getchildren()[1]
 884         flatHtml = htmlElement.text
 885
 886         return jsonTree, flatHtml
 887
 888
 889 def validate_response(response):
 890         """
 891         Validates that the JSON response is A-OK
 892         """
 893         try:
 894                 assert response is not None
 895                 assert 'ok' in response
 896                 assert response['ok']
 897         except AssertionError:
 898                 raise RuntimeError('There was a problem with GV: %s' % response)
 899
 900
 901 def guess_phone_type(number):
 902         if number.startswith("747") or number.startswith("1747") or number.startswith("+1747"):
 903                 return GVoiceBackend.PHONE_TYPE_GIZMO
 904         else:
 905                 return GVoiceBackend.PHONE_TYPE_MOBILE
 906
 907
 908 def get_sane_callback(backend):
 909         """
 910         Try to set a sane default callback number on these preferences
 911         1) 1747 numbers ( Gizmo )
 912         2) anything with gizmo in the name
 913         3) anything with computer in the name
 914         4) the first value
 915         """
 916         numbers = backend.get_callback_numbers()
 917
 918         priorityOrderedCriteria = [
 919                 ("\+1747", None),
 920                 ("1747", None),
 921                 ("747", None),
 922                 (None, "gizmo"),
 923                 (None, "computer"),
 924                 (None, "sip"),
 925                 (None, None),
 926         ]
 927
 928         for numberCriteria, descriptionCriteria in priorityOrderedCriteria:
 929                 numberMatcher = None
 930                 descriptionMatcher = None
 931                 if numberCriteria is not None:
 932                         numberMatcher = re.compile(numberCriteria)
 933                 elif descriptionCriteria is not None:
 934                         descriptionMatcher = re.compile(descriptionCriteria, re.I)
 935
 936                 for number, description in numbers.iteritems():
 937                         if numberMatcher is not None and numberMatcher.match(number) is None:
 938                                 continue
 939                         if descriptionMatcher is not None and descriptionMatcher.match(description) is None:
 940                                 continue
 941                         return number
 942
 943
 944 def set_sane_callback(backend):
 945         """
 946         Try to set a sane default callback number on these preferences
 947         1) 1747 numbers ( Gizmo )
 948         2) anything with gizmo in the name
 949         3) anything with computer in the name
 950         4) the first value
 951         """
 952         number = get_sane_callback(backend)
 953         backend.set_callback_number(number)
 954
 955
 956 def _is_not_special(name):
 957         return not name.startswith("_") and name[0].lower() == name[0] and "_" not in name
 958
 959
 960 def to_dict(obj):
 961         members = inspect.getmembers(obj)
 962         return dict((name, value) for (name, value) in members if _is_not_special(name))
 963
 964
 965 def grab_debug_info(username, password):
 966         cookieFile = os.path.join(".", "raw_cookies.txt")
 967         try:
 968                 os.remove(cookieFile)
 969         except OSError:
 970                 pass
 971
 972         backend = GVoiceBackend(cookieFile)
 973         browser = backend._browser
 974
 975         _TEST_WEBPAGES = [
 976                 ("forward", backend._forwardURL),
 977                 ("token", backend._tokenURL),
 978                 ("login", backend._loginURL),
 979                 ("isdnd", backend._isDndURL),
 980                 ("account", backend._XML_ACCOUNT_URL),
 981                 ("contacts", backend._XML_CONTACTS_URL),
 982                 ("csv", backend._CSV_CONTACTS_URL),
 983
 984                 ("voicemail", backend._XML_VOICEMAIL_URL),
 985                 ("sms", backend._XML_SMS_URL),
 986
 987                 ("recent", backend._XML_RECENT_URL),
 988                 ("placed", backend._XML_PLACED_URL),
 989                 ("recieved", backend._XML_RECEIVED_URL),
 990                 ("missed", backend._XML_MISSED_URL),
 991         ]
 992
 993         # Get Pages
 994         print "Grabbing pre-login pages"
 995         for name, url in _TEST_WEBPAGES:
 996                 try:
 997                         page = browser.download(url)
 998                 except StandardError, e:
 999                         print e.message
1000                         continue
1001                 print "\tWriting to file"
1002                 with open("not_loggedin_%s.txt" % name, "w") as f:
1003                         f.write(page)
1004
1005         # Login
1006         print "Attempting login"
1007         galxToken = backend._get_token()
1008         loginSuccessOrFailurePage = backend._login(username, password, galxToken)
1009         with open("loggingin.txt", "w") as f:
1010                 print "\tWriting to file"
1011                 f.write(loginSuccessOrFailurePage)
1012         try:
1013                 backend._grab_account_info(loginSuccessOrFailurePage)
1014         except Exception:
1015                 # Retry in case the redirect failed
1016                 # luckily is_authed does everything we need for a retry
1017                 loggedIn = backend.is_authed(True)
1018                 if not loggedIn:
1019                         raise
1020
1021         # Get Pages
1022         print "Grabbing post-login pages"
1023         for name, url in _TEST_WEBPAGES:
1024                 try:
1025                         page = browser.download(url)
1026                 except StandardError, e:
1027                         print str(e)
1028                         continue
1029                 print "\tWriting to file"
1030                 with open("loggedin_%s.txt" % name, "w") as f:
1031                         f.write(page)
1032
1033         # Cookies
1034         browser.save_cookies()
1035         print "\tWriting cookies to file"
1036         with open("cookies.txt", "w") as f:
1037                 f.writelines(
1038                         "%s: %s\n" % (c.name, c.value)
1039                         for c in browser._cookies
1040                 )
1041
1042
1043 def main():
1044         import sys
1045         logging.basicConfig(level=logging.DEBUG)
1046         args = sys.argv
1047         if 3 <= len(args):
1048                 username = args[1]
1049                 password = args[2]
1050
1051         grab_debug_info(username, password)
1052
1053
1054 if __name__ == "__main__":
1055         main()