git.maemo.org Git - gc-dialer/blob - src/backends/gvoice/gvoice.py

   1 #!/usr/bin/python
   2
   3 """
   4 DialCentral - Front end for Google's GoogleVoice service.
   5 Copyright (C) 2008  Eric Warnke ericew AT gmail DOT com
   6
   7 This library is free software; you can redistribute it and/or
   8 modify it under the terms of the GNU Lesser General Public
   9 License as published by the Free Software Foundation; either
  10 version 2.1 of the License, or (at your option) any later version.
  11
  12 This library is distributed in the hope that it will be useful,
  13 but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 Lesser General Public License for more details.
  16
  17 You should have received a copy of the GNU Lesser General Public
  18 License along with this library; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 Google Voice backend code
  22
  23 Resources
  24         http://thatsmith.com/2009/03/google-voice-addon-for-firefox/
  25         http://posttopic.com/topic/google-voice-add-on-development
  26 """
  27
  28 from __future__ import with_statement
  29
  30 import os
  31 import re
  32 import urllib
  33 import urllib2
  34 import time
  35 import datetime
  36 import itertools
  37 import logging
  38 import inspect
  39
  40 from xml.sax import saxutils
  41 from xml.etree import ElementTree
  42
  43 try:
  44         import simplejson as _simplejson
  45         simplejson = _simplejson
  46 except ImportError:
  47         simplejson = None
  48
  49 import browser_emu
  50
  51
  52 _moduleLogger = logging.getLogger(__name__)
  53
  54
  55 class NetworkError(RuntimeError):
  56         pass
  57
  58
  59 class MessageText(object):
  60
  61         ACCURACY_LOW = "med1"
  62         ACCURACY_MEDIUM = "med2"
  63         ACCURACY_HIGH = "high"
  64
  65         def __init__(self):
  66                 self.accuracy = None
  67                 self.text = None
  68
  69         def __str__(self):
  70                 return self.text
  71
  72         def to_dict(self):
  73                 return to_dict(self)
  74
  75         def __eq__(self, other):
  76                 return self.accuracy == other.accuracy and self.text == other.text
  77
  78
  79 class Message(object):
  80
  81         def __init__(self):
  82                 self.whoFrom = None
  83                 self.body = None
  84                 self.when = None
  85
  86         def __str__(self):
  87                 return "%s (%s): %s" % (
  88                         self.whoFrom,
  89                         self.when,
  90                         "".join(unicode(part) for part in self.body)
  91                 )
  92
  93         def to_dict(self):
  94                 selfDict = to_dict(self)
  95                 selfDict["body"] = [text.to_dict() for text in self.body] if self.body is not None else None
  96                 return selfDict
  97
  98         def __eq__(self, other):
  99                 return self.whoFrom == other.whoFrom and self.when == other.when and self.body == other.body
 100
 101
 102 class Conversation(object):
 103
 104         TYPE_VOICEMAIL = "Voicemail"
 105         TYPE_SMS = "SMS"
 106
 107         def __init__(self):
 108                 self.type = None
 109                 self.id = None
 110                 self.contactId = None
 111                 self.name = None
 112                 self.location = None
 113                 self.prettyNumber = None
 114                 self.number = None
 115
 116                 self.time = None
 117                 self.relTime = None
 118                 self.messages = None
 119                 self.isRead = None
 120                 self.isSpam = None
 121                 self.isTrash = None
 122                 self.isArchived = None
 123
 124         def __cmp__(self, other):
 125                 cmpValue = cmp(self.contactId, other.contactId)
 126                 if cmpValue != 0:
 127                         return cmpValue
 128
 129                 cmpValue = cmp(self.time, other.time)
 130                 if cmpValue != 0:
 131                         return cmpValue
 132
 133                 cmpValue = cmp(self.id, other.id)
 134                 if cmpValue != 0:
 135                         return cmpValue
 136
 137         def to_dict(self):
 138                 selfDict = to_dict(self)
 139                 selfDict["messages"] = [message.to_dict() for message in self.messages] if self.messages is not None else None
 140                 return selfDict
 141
 142
 143 class GVoiceBackend(object):
 144         """
 145         This class encapsulates all of the knowledge necessary to interact with the GoogleVoice servers
 146         the functions include login, setting up a callback number, and initalting a callback
 147         """
 148
 149         PHONE_TYPE_HOME = 1
 150         PHONE_TYPE_MOBILE = 2
 151         PHONE_TYPE_WORK = 3
 152         PHONE_TYPE_GIZMO = 7
 153
 154         def __init__(self, cookieFile = None):
 155                 # Important items in this function are the setup of the browser emulation and cookie file
 156                 self._browser = browser_emu.MozillaEmulator(1)
 157                 self._loadedFromCookies = self._browser.load_cookies(cookieFile)
 158
 159                 self._token = ""
 160                 self._accountNum = ""
 161                 self._lastAuthed = 0.0
 162                 self._callbackNumber = ""
 163                 self._callbackNumbers = {}
 164
 165                 # Suprisingly, moving all of these from class to self sped up startup time
 166
 167                 self._validateRe = re.compile("^\+?[0-9]{10,}$")
 168
 169                 self._loginURL = "https://www.google.com/accounts/ServiceLoginAuth"
 170
 171                 SECURE_URL_BASE = "https://www.google.com/voice/"
 172                 SECURE_MOBILE_URL_BASE = SECURE_URL_BASE + "mobile/"
 173                 self._forwardURL = SECURE_MOBILE_URL_BASE + "phones"
 174                 self._tokenURL = SECURE_URL_BASE + "m"
 175                 self._callUrl = SECURE_URL_BASE + "call/connect"
 176                 self._callCancelURL = SECURE_URL_BASE + "call/cancel"
 177                 self._sendSmsURL = SECURE_URL_BASE + "sms/send"
 178
 179                 self._isDndURL = "https://www.google.com/voice/m/donotdisturb"
 180                 self._isDndRe = re.compile(r"""<input.*?id="doNotDisturb".*?checked="(.*?)"\s*/>""")
 181                 self._setDndURL = "https://www.google.com/voice/m/savednd"
 182
 183                 self._downloadVoicemailURL = SECURE_URL_BASE + "media/send_voicemail/"
 184                 self._markAsReadURL = SECURE_URL_BASE + "m/mark"
 185                 self._archiveMessageURL = SECURE_URL_BASE + "m/archive"
 186
 187                 self._XML_SEARCH_URL = SECURE_URL_BASE + "inbox/search/"
 188                 self._XML_ACCOUNT_URL = SECURE_URL_BASE + "contacts/"
 189                 # HACK really this redirects to the main pge and we are grabbing some javascript
 190                 self._XML_CONTACTS_URL = "http://www.google.com/voice/inbox/search/contact"
 191                 self._CSV_CONTACTS_URL = "http://mail.google.com/mail/contacts/data/export"
 192                 self._JSON_CONTACTS_URL = SECURE_URL_BASE + "b/0/request/user"
 193                 self._XML_RECENT_URL = SECURE_URL_BASE + "inbox/recent/"
 194
 195                 self.XML_FEEDS = (
 196                         'inbox', 'starred', 'all', 'spam', 'trash', 'voicemail', 'sms',
 197                         'recorded', 'placed', 'received', 'missed'
 198                 )
 199                 self._XML_INBOX_URL = SECURE_URL_BASE + "inbox/recent/inbox"
 200                 self._XML_STARRED_URL = SECURE_URL_BASE + "inbox/recent/starred"
 201                 self._XML_ALL_URL = SECURE_URL_BASE + "inbox/recent/all"
 202                 self._XML_SPAM_URL = SECURE_URL_BASE + "inbox/recent/spam"
 203                 self._XML_TRASH_URL = SECURE_URL_BASE + "inbox/recent/trash"
 204                 self._XML_VOICEMAIL_URL = SECURE_URL_BASE + "inbox/recent/voicemail/"
 205                 self._XML_SMS_URL = SECURE_URL_BASE + "inbox/recent/sms/"
 206                 self._JSON_SMS_URL = SECURE_URL_BASE + "b/0/request/messages/"
 207                 self._JSON_SMS_COUNT_URL = SECURE_URL_BASE + "b/0/request/unread"
 208                 self._XML_RECORDED_URL = SECURE_URL_BASE + "inbox/recent/recorded/"
 209                 self._XML_PLACED_URL = SECURE_URL_BASE + "inbox/recent/placed/"
 210                 self._XML_RECEIVED_URL = SECURE_URL_BASE + "inbox/recent/received/"
 211                 self._XML_MISSED_URL = SECURE_URL_BASE + "inbox/recent/missed/"
 212
 213                 self._galxRe = re.compile(r"""<input.*?name="GALX".*?value="(.*?)".*?/>""", re.MULTILINE | re.DOTALL)
 214                 self._tokenRe = re.compile(r"""<input.*?name="_rnr_se".*?value="(.*?)"\s*/>""")
 215                 self._accountNumRe = re.compile(r"""<b class="ms\d">(.{14})</b></div>""")
 216                 self._callbackRe = re.compile(r"""\s+(.*?):\s*(.*?)<br\s*/>\s*$""", re.M)
 217
 218                 self._seperateVoicemailsRegex = re.compile(r"""^\s*<div id="(\w+)"\s* class=".*?gc-message.*?">""", re.MULTILINE | re.DOTALL)
 219                 self._exactVoicemailTimeRegex = re.compile(r"""<span class="gc-message-time">(.*?)</span>""", re.MULTILINE)
 220                 self._relativeVoicemailTimeRegex = re.compile(r"""<span class="gc-message-relative">(.*?)</span>""", re.MULTILINE)
 221                 self._voicemailNameRegex = re.compile(r"""<a class=.*?gc-message-name-link.*?>(.*?)</a>""", re.MULTILINE | re.DOTALL)
 222                 self._voicemailNumberRegex = re.compile(r"""<input type="hidden" class="gc-text gc-quickcall-ac" value="(.*?)"/>""", re.MULTILINE)
 223                 self._prettyVoicemailNumberRegex = re.compile(r"""<span class="gc-message-type">(.*?)</span>""", re.MULTILINE)
 224                 self._voicemailLocationRegex = re.compile(r"""<span class="gc-message-location">.*?<a.*?>(.*?)</a></span>""", re.MULTILINE)
 225                 self._messagesContactIDRegex = re.compile(r"""<a class=".*?gc-message-name-link.*?">.*?</a>\s*?<span .*?>(.*?)</span>""", re.MULTILINE)
 226                 self._voicemailMessageRegex = re.compile(r"""(<span id="\d+-\d+" class="gc-word-(.*?)">(.*?)</span>|<a .*? class="gc-message-mni">(.*?)</a>)""", re.MULTILINE)
 227                 self._smsFromRegex = re.compile(r"""<span class="gc-message-sms-from">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 228                 self._smsTimeRegex = re.compile(r"""<span class="gc-message-sms-time">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 229                 self._smsTextRegex = re.compile(r"""<span class="gc-message-sms-text">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 230
 231         def is_quick_login_possible(self):
 232                 """
 233                 @returns True then is_authed might be enough to login, else full login is required
 234                 """
 235                 return self._loadedFromCookies or 0.0 < self._lastAuthed
 236
 237         def is_authed(self, force = False):
 238                 """
 239                 Attempts to detect a current session
 240                 @note Once logged in try not to reauth more than once a minute.
 241                 @returns If authenticated
 242                 @blocks
 243                 """
 244                 isRecentledAuthed = (time.time() - self._lastAuthed) < 120
 245                 isPreviouslyAuthed = self._token is not None
 246                 if isRecentledAuthed and isPreviouslyAuthed and not force:
 247                         return True
 248
 249                 try:
 250                         page = self._get_page(self._forwardURL)
 251                         self._grab_account_info(page)
 252                 except Exception, e:
 253                         _moduleLogger.exception(str(e))
 254                         return False
 255
 256                 self._browser.save_cookies()
 257                 self._lastAuthed = time.time()
 258                 return True
 259
 260         def _get_token(self):
 261                 tokenPage = self._get_page(self._tokenURL)
 262
 263                 galxTokens = self._galxRe.search(tokenPage)
 264                 if galxTokens is not None:
 265                         galxToken = galxTokens.group(1)
 266                 else:
 267                         galxToken = ""
 268                         _moduleLogger.debug("Could not grab GALX token")
 269                 return galxToken
 270
 271         def _login(self, username, password, token):
 272                 loginData = {
 273                         'Email' : username,
 274                         'Passwd' : password,
 275                         'service': "grandcentral",
 276                         "ltmpl": "mobile",
 277                         "btmpl": "mobile",
 278                         "PersistentCookie": "yes",
 279                         "GALX": token,
 280                         "continue": self._forwardURL,
 281                 }
 282
 283                 loginSuccessOrFailurePage = self._get_page(self._loginURL, loginData)
 284                 return loginSuccessOrFailurePage
 285
 286         def login(self, username, password):
 287                 """
 288                 Attempt to login to GoogleVoice
 289                 @returns Whether login was successful or not
 290                 @blocks
 291                 """
 292                 self.logout()
 293                 galxToken = self._get_token()
 294                 loginSuccessOrFailurePage = self._login(username, password, galxToken)
 295
 296                 try:
 297                         self._grab_account_info(loginSuccessOrFailurePage)
 298                 except Exception, e:
 299                         # Retry in case the redirect failed
 300                         # luckily is_authed does everything we need for a retry
 301                         loggedIn = self.is_authed(True)
 302                         if not loggedIn:
 303                                 _moduleLogger.exception(str(e))
 304                                 return False
 305                         _moduleLogger.info("Redirection failed on initial login attempt, auto-corrected for this")
 306
 307                 self._browser.save_cookies()
 308                 self._lastAuthed = time.time()
 309                 return True
 310
 311         def persist(self):
 312                 self._browser.save_cookies()
 313
 314         def shutdown(self):
 315                 self._browser.save_cookies()
 316                 self._token = None
 317                 self._lastAuthed = 0.0
 318
 319         def logout(self):
 320                 self._browser.clear_cookies()
 321                 self._browser.save_cookies()
 322                 self._token = None
 323                 self._lastAuthed = 0.0
 324
 325         def is_dnd(self):
 326                 """
 327                 @blocks
 328                 """
 329                 isDndPage = self._get_page(self._isDndURL)
 330
 331                 dndGroup = self._isDndRe.search(isDndPage)
 332                 if dndGroup is None:
 333                         return False
 334                 dndStatus = dndGroup.group(1)
 335                 isDnd = True if dndStatus.strip().lower() == "true" else False
 336                 return isDnd
 337
 338         def set_dnd(self, doNotDisturb):
 339                 """
 340                 @blocks
 341                 """
 342                 dndPostData = {
 343                         "doNotDisturb": 1 if doNotDisturb else 0,
 344                 }
 345
 346                 dndPage = self._get_page_with_token(self._setDndURL, dndPostData)
 347
 348         def call(self, outgoingNumber):
 349                 """
 350                 This is the main function responsible for initating the callback
 351                 @blocks
 352                 """
 353                 outgoingNumber = self._send_validation(outgoingNumber)
 354                 subscriberNumber = None
 355                 phoneType = guess_phone_type(self._callbackNumber) # @todo Fix this hack
 356
 357                 callData = {
 358                                 'outgoingNumber': outgoingNumber,
 359                                 'forwardingNumber': self._callbackNumber,
 360                                 'subscriberNumber': subscriberNumber or 'undefined',
 361                                 'phoneType': str(phoneType),
 362                                 'remember': '1',
 363                 }
 364                 _moduleLogger.info("%r" % callData)
 365
 366                 page = self._get_page_with_token(
 367                         self._callUrl,
 368                         callData,
 369                 )
 370                 self._parse_with_validation(page)
 371                 return True
 372
 373         def cancel(self, outgoingNumber=None):
 374                 """
 375                 Cancels a call matching outgoing and forwarding numbers (if given).
 376                 Will raise an error if no matching call is being placed
 377                 @blocks
 378                 """
 379                 page = self._get_page_with_token(
 380                         self._callCancelURL,
 381                         {
 382                         'outgoingNumber': outgoingNumber or 'undefined',
 383                         'forwardingNumber': self._callbackNumber or 'undefined',
 384                         'cancelType': 'C2C',
 385                         },
 386                 )
 387                 self._parse_with_validation(page)
 388
 389         def send_sms(self, phoneNumbers, message):
 390                 """
 391                 @blocks
 392                 """
 393                 validatedPhoneNumbers = [
 394                         self._send_validation(phoneNumber)
 395                         for phoneNumber in phoneNumbers
 396                 ]
 397                 flattenedPhoneNumbers = ",".join(validatedPhoneNumbers)
 398                 page = self._get_page_with_token(
 399                         self._sendSmsURL,
 400                         {
 401                                 'phoneNumber': flattenedPhoneNumbers,
 402                                 'text': unicode(message).encode("utf-8"),
 403                         },
 404                 )
 405                 self._parse_with_validation(page)
 406
 407         def search(self, query):
 408                 """
 409                 Search your Google Voice Account history for calls, voicemails, and sms
 410                 Returns ``Folder`` instance containting matching messages
 411                 @blocks
 412                 """
 413                 page = self._get_page(
 414                         self._XML_SEARCH_URL,
 415                         {"q": query},
 416                 )
 417                 json, html = extract_payload(page)
 418                 return json
 419
 420         def get_feed(self, feed):
 421                 """
 422                 @blocks
 423                 """
 424                 actualFeed = "_XML_%s_URL" % feed.upper()
 425                 feedUrl = getattr(self, actualFeed)
 426
 427                 page = self._get_page(feedUrl)
 428                 json, html = extract_payload(page)
 429
 430                 return json
 431
 432         def download(self, messageId, adir):
 433                 """
 434                 Download a voicemail or recorded call MP3 matching the given ``msg``
 435                 which can either be a ``Message`` instance, or a SHA1 identifier.
 436                 Saves files to ``adir`` (defaults to current directory).
 437                 Message hashes can be found in ``self.voicemail().messages`` for example.
 438                 @returns location of saved file.
 439                 @blocks
 440                 """
 441                 page = self._get_page(self._downloadVoicemailURL, {"id": messageId})
 442                 fn = os.path.join(adir, '%s.mp3' % messageId)
 443                 with open(fn, 'wb') as fo:
 444                         fo.write(page)
 445                 return fn
 446
 447         def is_valid_syntax(self, number):
 448                 """
 449                 @returns If This number be called ( syntax validation only )
 450                 """
 451                 return self._validateRe.match(number) is not None
 452
 453         def get_account_number(self):
 454                 """
 455                 @returns The GoogleVoice phone number
 456                 """
 457                 return self._accountNum
 458
 459         def get_callback_numbers(self):
 460                 """
 461                 @returns a dictionary mapping call back numbers to descriptions
 462                 @note These results are cached for 30 minutes.
 463                 """
 464                 return self._callbackNumbers
 465
 466         def set_callback_number(self, callbacknumber):
 467                 """
 468                 Set the number that GoogleVoice calls
 469                 @param callbacknumber should be a proper 10 digit number
 470                 """
 471                 self._callbackNumber = callbacknumber
 472                 _moduleLogger.info("Callback number changed: %r" % self._callbackNumber)
 473                 return True
 474
 475         def get_callback_number(self):
 476                 """
 477                 @returns Current callback number or None
 478                 """
 479                 return self._callbackNumber
 480
 481         def get_recent(self):
 482                 """
 483                 @returns Iterable of (personsName, phoneNumber, exact date, relative date, action)
 484                 @blocks
 485                 """
 486                 recentPages = [
 487                         (action, self._get_page(url))
 488                         for action, url in (
 489                                 ("Received", self._XML_RECEIVED_URL),
 490                                 ("Missed", self._XML_MISSED_URL),
 491                                 ("Placed", self._XML_PLACED_URL),
 492                         )
 493                 ]
 494                 return self._parse_recent(recentPages)
 495
 496         def get_contacts(self):
 497                 """
 498                 @returns Iterable of (contact id, contact name)
 499                 @blocks
 500                 """
 501                 page = self._get_page(self._JSON_CONTACTS_URL)
 502                 return self._process_contacts(page)
 503
 504         def get_csv_contacts(self):
 505                 data = {
 506                         "groupToExport": "mine",
 507                         "exportType": "ALL",
 508                         "out": "OUTLOOK_CSV",
 509                 }
 510                 encodedData = urllib.urlencode(data)
 511                 contacts = self._get_page(self._CSV_CONTACTS_URL+"?"+encodedData)
 512                 return contacts
 513
 514         def get_voicemails(self):
 515                 """
 516                 @blocks
 517                 """
 518                 voicemailPage = self._get_page(self._XML_VOICEMAIL_URL)
 519                 voicemailHtml = self._grab_html(voicemailPage)
 520                 voicemailJson = self._grab_json(voicemailPage)
 521                 if voicemailJson is None:
 522                         return ()
 523                 parsedVoicemail = self._parse_voicemail(voicemailHtml)
 524                 voicemails = self._merge_conversation_sources(parsedVoicemail, voicemailJson)
 525                 return voicemails
 526
 527         def get_texts(self):
 528                 """
 529                 @blocks
 530                 """
 531                 smsPage = self._get_page(self._XML_SMS_URL)
 532                 smsHtml = self._grab_html(smsPage)
 533                 smsJson = self._grab_json(smsPage)
 534                 if smsJson is None:
 535                         return ()
 536                 parsedSms = self._parse_sms(smsHtml)
 537                 smss = self._merge_conversation_sources(parsedSms, smsJson)
 538                 return smss
 539
 540         def get_unread_counts(self):
 541                 countPage = self._get_page(self._JSON_SMS_COUNT_URL)
 542                 counts = parse_json(countPage)
 543                 counts = counts["unreadCounts"]
 544                 return counts
 545
 546         def mark_message(self, messageId, asRead):
 547                 """
 548                 @blocks
 549                 """
 550                 postData = {
 551                         "read": 1 if asRead else 0,
 552                         "id": messageId,
 553                 }
 554
 555                 markPage = self._get_page(self._markAsReadURL, postData)
 556
 557         def archive_message(self, messageId):
 558                 """
 559                 @blocks
 560                 """
 561                 postData = {
 562                         "id": messageId,
 563                 }
 564
 565                 markPage = self._get_page(self._archiveMessageURL, postData)
 566
 567         def _grab_json(self, flatXml):
 568                 xmlTree = ElementTree.fromstring(flatXml)
 569                 jsonElement = xmlTree.getchildren()[0]
 570                 flatJson = jsonElement.text
 571                 jsonTree = parse_json(flatJson)
 572                 return jsonTree
 573
 574         def _grab_html(self, flatXml):
 575                 xmlTree = ElementTree.fromstring(flatXml)
 576                 htmlElement = xmlTree.getchildren()[1]
 577                 flatHtml = htmlElement.text
 578                 return flatHtml
 579
 580         def _grab_account_info(self, page):
 581                 tokenGroup = self._tokenRe.search(page)
 582                 if tokenGroup is None:
 583                         raise RuntimeError("Could not extract authentication token from GoogleVoice")
 584                 self._token = tokenGroup.group(1)
 585
 586                 anGroup = self._accountNumRe.search(page)
 587                 if anGroup is not None:
 588                         self._accountNum = anGroup.group(1)
 589                 else:
 590                         _moduleLogger.debug("Could not extract account number from GoogleVoice")
 591
 592                 self._callbackNumbers = {}
 593                 for match in self._callbackRe.finditer(page):
 594                         callbackNumber = match.group(2)
 595                         callbackName = match.group(1)
 596                         self._callbackNumbers[callbackNumber] = callbackName
 597                 if len(self._callbackNumbers) == 0:
 598                         _moduleLogger.debug("Could not extract callback numbers from GoogleVoice (the troublesome page follows):\n%s" % page)
 599
 600         def _send_validation(self, number):
 601                 if not self.is_valid_syntax(number):
 602                         raise ValueError('Number is not valid: "%s"' % number)
 603                 elif not self.is_authed():
 604                         raise RuntimeError("Not Authenticated")
 605                 return number
 606
 607         def _parse_recent(self, recentPages):
 608                 for action, flatXml in recentPages:
 609                         allRecentHtml = self._grab_html(flatXml)
 610                         allRecentData = self._parse_history(allRecentHtml)
 611                         for recentCallData in allRecentData:
 612                                 recentCallData["action"] = action
 613                                 yield recentCallData
 614
 615         def _process_contacts(self, page):
 616                 accountData = parse_json(page)
 617                 for contactId, contactDetails in accountData["contacts"].iteritems():
 618                         # A zero contact id is the catch all for unknown contacts
 619                         if contactId != "0":
 620                                 yield contactId, contactDetails
 621
 622         def _parse_history(self, historyHtml):
 623                 splitVoicemail = self._seperateVoicemailsRegex.split(historyHtml)
 624                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 625                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 626                         exactTime = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 627                         exactTime = google_strptime(exactTime)
 628                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 629                         relativeTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 630                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 631                         location = locationGroup.group(1).strip() if locationGroup else ""
 632
 633                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 634                         name = nameGroup.group(1).strip() if nameGroup else ""
 635                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 636                         number = numberGroup.group(1).strip() if numberGroup else ""
 637                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 638                         prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 639                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 640                         contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 641
 642                         yield {
 643                                 "id": messageId.strip(),
 644                                 "contactId": contactId,
 645                                 "name": unescape(name),
 646                                 "time": exactTime,
 647                                 "relTime": relativeTime,
 648                                 "prettyNumber": prettyNumber,
 649                                 "number": number,
 650                                 "location": unescape(location),
 651                         }
 652
 653         @staticmethod
 654         def _interpret_voicemail_regex(group):
 655                 quality, content, number = group.group(2), group.group(3), group.group(4)
 656                 text = MessageText()
 657                 if quality is not None and content is not None:
 658                         text.accuracy = quality
 659                         text.text = unescape(content)
 660                         return text
 661                 elif number is not None:
 662                         text.accuracy = MessageText.ACCURACY_HIGH
 663                         text.text = number
 664                         return text
 665
 666         def _parse_voicemail(self, voicemailHtml):
 667                 splitVoicemail = self._seperateVoicemailsRegex.split(voicemailHtml)
 668                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 669                         conv = Conversation()
 670                         conv.type = Conversation.TYPE_VOICEMAIL
 671                         conv.id = messageId.strip()
 672
 673                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 674                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 675                         conv.time = google_strptime(exactTimeText)
 676                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 677                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 678                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 679                         conv.location = unescape(locationGroup.group(1).strip() if locationGroup else "")
 680
 681                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 682                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 683                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 684                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 685                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 686                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 687                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 688                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 689
 690                         messageGroups = self._voicemailMessageRegex.finditer(messageHtml)
 691                         messageParts = [
 692                                 self._interpret_voicemail_regex(group)
 693                                 for group in messageGroups
 694                         ] if messageGroups else ((MessageText.ACCURACY_LOW, "No Transcription"), )
 695                         message = Message()
 696                         message.body = messageParts
 697                         message.whoFrom = conv.name
 698                         try:
 699                                 message.when = conv.time.strftime("%I:%M %p")
 700                         except ValueError:
 701                                 _moduleLogger.exception("Confusing time provided: %r" % conv.time)
 702                                 message.when = "Unknown"
 703                         conv.messages = (message, )
 704
 705                         yield conv
 706
 707         @staticmethod
 708         def _interpret_sms_message_parts(fromPart, textPart, timePart):
 709                 text = MessageText()
 710                 text.accuracy = MessageText.ACCURACY_MEDIUM
 711                 text.text = unescape(textPart)
 712
 713                 message = Message()
 714                 message.body = (text, )
 715                 message.whoFrom = fromPart
 716                 message.when = timePart
 717
 718                 return message
 719
 720         def _parse_sms(self, smsHtml):
 721                 splitSms = self._seperateVoicemailsRegex.split(smsHtml)
 722                 for messageId, messageHtml in itergroup(splitSms[1:], 2):
 723                         conv = Conversation()
 724                         conv.type = Conversation.TYPE_SMS
 725                         conv.id = messageId.strip()
 726
 727                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 728                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 729                         conv.time = google_strptime(exactTimeText)
 730                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 731                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 732                         conv.location = ""
 733
 734                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 735                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 736                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 737                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 738                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 739                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 740                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 741                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 742
 743                         fromGroups = self._smsFromRegex.finditer(messageHtml)
 744                         fromParts = (group.group(1).strip() for group in fromGroups)
 745                         textGroups = self._smsTextRegex.finditer(messageHtml)
 746                         textParts = (group.group(1).strip() for group in textGroups)
 747                         timeGroups = self._smsTimeRegex.finditer(messageHtml)
 748                         timeParts = (group.group(1).strip() for group in timeGroups)
 749
 750                         messageParts = itertools.izip(fromParts, textParts, timeParts)
 751                         messages = [self._interpret_sms_message_parts(*parts) for parts in messageParts]
 752                         conv.messages = messages
 753
 754                         yield conv
 755
 756         @staticmethod
 757         def _merge_conversation_sources(parsedMessages, json):
 758                 for message in parsedMessages:
 759                         jsonItem = json["messages"][message.id]
 760                         message.isRead = jsonItem["isRead"]
 761                         message.isSpam = jsonItem["isSpam"]
 762                         message.isTrash = jsonItem["isTrash"]
 763                         message.isArchived = "inbox" not in jsonItem["labels"]
 764                         yield message
 765
 766         def _get_page(self, url, data = None, refererUrl = None):
 767                 headers = {}
 768                 if refererUrl is not None:
 769                         headers["Referer"] = refererUrl
 770
 771                 encodedData = urllib.urlencode(data) if data is not None else None
 772
 773                 try:
 774                         page = self._browser.download(url, encodedData, None, headers)
 775                 except urllib2.URLError, e:
 776                         _moduleLogger.error("Translating error: %s" % str(e))
 777                         raise NetworkError("%s is not accesible" % url)
 778
 779                 return page
 780
 781         def _get_page_with_token(self, url, data = None, refererUrl = None):
 782                 if data is None:
 783                         data = {}
 784                 data['_rnr_se'] = self._token
 785
 786                 page = self._get_page(url, data, refererUrl)
 787
 788                 return page
 789
 790         def _parse_with_validation(self, page):
 791                 json = parse_json(page)
 792                 self._validate_response(json)
 793                 return json
 794
 795         def _validate_response(self, response):
 796                 """
 797                 Validates that the JSON response is A-OK
 798                 """
 799                 try:
 800                         assert response is not None, "Response not provided"
 801                         assert 'ok' in response, "Response lacks status"
 802                         assert response['ok'], "Response not good"
 803                 except AssertionError:
 804                         try:
 805                                 if response["data"]["code"] == 20:
 806                                         raise RuntimeError(
 807 """Ambiguous error 20 returned by Google Voice.
 808 Please verify you have configured your callback number (currently "%s").  If it is configured some other suspected causes are: non-verified callback numbers, and Gizmo5 callback numbers.""" % self._callbackNumber)
 809                         except KeyError:
 810                                 pass
 811                         raise RuntimeError('There was a problem with GV: %s' % response)
 812
 813
 814 _UNESCAPE_ENTITIES = {
 815  "&quot;": '"',
 816  "&nbsp;": " ",
 817  "&#39;": "'",
 818 }
 819
 820
 821 def unescape(text):
 822         plain = saxutils.unescape(text, _UNESCAPE_ENTITIES)
 823         return plain
 824
 825
 826 def google_strptime(time):
 827         """
 828         Hack: Google always returns the time in the same locale.  Sadly if the
 829         local system's locale is different, there isn't a way to perfectly handle
 830         the time.  So instead we handle implement some time formatting
 831         """
 832         abbrevTime = time[:-3]
 833         parsedTime = datetime.datetime.strptime(abbrevTime, "%m/%d/%y %I:%M")
 834         if time.endswith("PM"):
 835                 parsedTime += datetime.timedelta(hours=12)
 836         return parsedTime
 837
 838
 839 def itergroup(iterator, count, padValue = None):
 840         """
 841         Iterate in groups of 'count' values. If there
 842         aren't enough values, the last result is padded with
 843         None.
 844
 845         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 846         ...     print tuple(val)
 847         (1, 2, 3)
 848         (4, 5, 6)
 849         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 850         ...     print list(val)
 851         [1, 2, 3]
 852         [4, 5, 6]
 853         >>> for val in itergroup([1, 2, 3, 4, 5, 6, 7], 3):
 854         ...     print tuple(val)
 855         (1, 2, 3)
 856         (4, 5, 6)
 857         (7, None, None)
 858         >>> for val in itergroup("123456", 3):
 859         ...     print tuple(val)
 860         ('1', '2', '3')
 861         ('4', '5', '6')
 862         >>> for val in itergroup("123456", 3):
 863         ...     print repr("".join(val))
 864         '123'
 865         '456'
 866         """
 867         paddedIterator = itertools.chain(iterator, itertools.repeat(padValue, count-1))
 868         nIterators = (paddedIterator, ) * count
 869         return itertools.izip(*nIterators)
 870
 871
 872 def safe_eval(s):
 873         _TRUE_REGEX = re.compile("true")
 874         _FALSE_REGEX = re.compile("false")
 875         _COMMENT_REGEX = re.compile("^\s+//.*$", re.M)
 876         s = _TRUE_REGEX.sub("True", s)
 877         s = _FALSE_REGEX.sub("False", s)
 878         s = _COMMENT_REGEX.sub("#", s)
 879         try:
 880                 results = eval(s, {}, {})
 881         except SyntaxError:
 882                 _moduleLogger.exception("Oops")
 883                 results = None
 884         return results
 885
 886
 887 def _fake_parse_json(flattened):
 888         return safe_eval(flattened)
 889
 890
 891 def _actual_parse_json(flattened):
 892         return simplejson.loads(flattened)
 893
 894
 895 if simplejson is None:
 896         parse_json = _fake_parse_json
 897 else:
 898         parse_json = _actual_parse_json
 899
 900
 901 def extract_payload(flatXml):
 902         xmlTree = ElementTree.fromstring(flatXml)
 903
 904         jsonElement = xmlTree.getchildren()[0]
 905         flatJson = jsonElement.text
 906         jsonTree = parse_json(flatJson)
 907
 908         htmlElement = xmlTree.getchildren()[1]
 909         flatHtml = htmlElement.text
 910
 911         return jsonTree, flatHtml
 912
 913
 914 def guess_phone_type(number):
 915         if number.startswith("747") or number.startswith("1747") or number.startswith("+1747"):
 916                 return GVoiceBackend.PHONE_TYPE_GIZMO
 917         else:
 918                 return GVoiceBackend.PHONE_TYPE_MOBILE
 919
 920
 921 def get_sane_callback(backend):
 922         """
 923         Try to set a sane default callback number on these preferences
 924         1) 1747 numbers ( Gizmo )
 925         2) anything with gizmo in the name
 926         3) anything with computer in the name
 927         4) the first value
 928         """
 929         numbers = backend.get_callback_numbers()
 930
 931         priorityOrderedCriteria = [
 932                 ("\+1747", None),
 933                 ("1747", None),
 934                 ("747", None),
 935                 (None, "gizmo"),
 936                 (None, "computer"),
 937                 (None, "sip"),
 938                 (None, None),
 939         ]
 940
 941         for numberCriteria, descriptionCriteria in priorityOrderedCriteria:
 942                 numberMatcher = None
 943                 descriptionMatcher = None
 944                 if numberCriteria is not None:
 945                         numberMatcher = re.compile(numberCriteria)
 946                 elif descriptionCriteria is not None:
 947                         descriptionMatcher = re.compile(descriptionCriteria, re.I)
 948
 949                 for number, description in numbers.iteritems():
 950                         if numberMatcher is not None and numberMatcher.match(number) is None:
 951                                 continue
 952                         if descriptionMatcher is not None and descriptionMatcher.match(description) is None:
 953                                 continue
 954                         return number
 955
 956
 957 def set_sane_callback(backend):
 958         """
 959         Try to set a sane default callback number on these preferences
 960         1) 1747 numbers ( Gizmo )
 961         2) anything with gizmo in the name
 962         3) anything with computer in the name
 963         4) the first value
 964         """
 965         number = get_sane_callback(backend)
 966         backend.set_callback_number(number)
 967
 968
 969 def _is_not_special(name):
 970         return not name.startswith("_") and name[0].lower() == name[0] and "_" not in name
 971
 972
 973 def to_dict(obj):
 974         members = inspect.getmembers(obj)
 975         return dict((name, value) for (name, value) in members if _is_not_special(name))
 976
 977
 978 def grab_debug_info(username, password):
 979         cookieFile = os.path.join(".", "raw_cookies.txt")
 980         try:
 981                 os.remove(cookieFile)
 982         except OSError:
 983                 pass
 984
 985         backend = GVoiceBackend(cookieFile)
 986         browser = backend._browser
 987
 988         _TEST_WEBPAGES = [
 989                 ("forward", backend._forwardURL),
 990                 ("token", backend._tokenURL),
 991                 ("login", backend._loginURL),
 992                 ("isdnd", backend._isDndURL),
 993                 ("account", backend._XML_ACCOUNT_URL),
 994                 ("html_contacts", backend._XML_CONTACTS_URL),
 995                 ("contacts", backend._JSON_CONTACTS_URL),
 996                 ("csv", backend._CSV_CONTACTS_URL),
 997
 998                 ("voicemail", backend._XML_VOICEMAIL_URL),
 999                 ("html_sms", backend._XML_SMS_URL),
1000                 ("sms", backend._JSON_SMS_URL),
1001                 ("count", backend._JSON_SMS_COUNT_URL),
1002
1003                 ("recent", backend._XML_RECENT_URL),
1004                 ("placed", backend._XML_PLACED_URL),
1005                 ("recieved", backend._XML_RECEIVED_URL),
1006                 ("missed", backend._XML_MISSED_URL),
1007         ]
1008
1009         # Get Pages
1010         print "Grabbing pre-login pages"
1011         for name, url in _TEST_WEBPAGES:
1012                 try:
1013                         page = browser.download(url)
1014                 except StandardError, e:
1015                         print e.message
1016                         continue
1017                 print "\tWriting to file"
1018                 with open("not_loggedin_%s.txt" % name, "w") as f:
1019                         f.write(page)
1020
1021         # Login
1022         print "Attempting login"
1023         galxToken = backend._get_token()
1024         loginSuccessOrFailurePage = backend._login(username, password, galxToken)
1025         with open("loggingin.txt", "w") as f:
1026                 print "\tWriting to file"
1027                 f.write(loginSuccessOrFailurePage)
1028         try:
1029                 backend._grab_account_info(loginSuccessOrFailurePage)
1030         except Exception:
1031                 # Retry in case the redirect failed
1032                 # luckily is_authed does everything we need for a retry
1033                 loggedIn = backend.is_authed(True)
1034                 if not loggedIn:
1035                         raise
1036
1037         # Get Pages
1038         print "Grabbing post-login pages"
1039         for name, url in _TEST_WEBPAGES:
1040                 try:
1041                         page = browser.download(url)
1042                 except StandardError, e:
1043                         print str(e)
1044                         continue
1045                 print "\tWriting to file"
1046                 with open("loggedin_%s.txt" % name, "w") as f:
1047                         f.write(page)
1048
1049         # Cookies
1050         browser.save_cookies()
1051         print "\tWriting cookies to file"
1052         with open("cookies.txt", "w") as f:
1053                 f.writelines(
1054                         "%s: %s\n" % (c.name, c.value)
1055                         for c in browser._cookies
1056                 )
1057
1058
1059 def main():
1060         import sys
1061         logging.basicConfig(level=logging.DEBUG)
1062         args = sys.argv
1063         if 3 <= len(args):
1064                 username = args[1]
1065                 password = args[2]
1066
1067         grab_debug_info(username, password)
1068
1069
1070 if __name__ == "__main__":
1071         main()