git.maemo.org Git - gc-dialer/blob - src/backends/gvoice/gvoice.py

   1 #!/usr/bin/python
   2
   3 """
   4 DialCentral - Front end for Google's GoogleVoice service.
   5 Copyright (C) 2008  Eric Warnke ericew AT gmail DOT com
   6
   7 This library is free software; you can redistribute it and/or
   8 modify it under the terms of the GNU Lesser General Public
   9 License as published by the Free Software Foundation; either
  10 version 2.1 of the License, or (at your option) any later version.
  11
  12 This library is distributed in the hope that it will be useful,
  13 but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 Lesser General Public License for more details.
  16
  17 You should have received a copy of the GNU Lesser General Public
  18 License along with this library; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 Google Voice backend code
  22
  23 Resources
  24         http://thatsmith.com/2009/03/google-voice-addon-for-firefox/
  25         http://posttopic.com/topic/google-voice-add-on-development
  26 """
  27
  28 from __future__ import with_statement
  29
  30 import os
  31 import re
  32 import urllib
  33 import urllib2
  34 import time
  35 import datetime
  36 import itertools
  37 import logging
  38 import inspect
  39
  40 from xml.sax import saxutils
  41 from xml.etree import ElementTree
  42
  43 try:
  44         import simplejson as _simplejson
  45         simplejson = _simplejson
  46 except ImportError:
  47         simplejson = None
  48
  49 import browser_emu
  50
  51
  52 _moduleLogger = logging.getLogger(__name__)
  53
  54
  55 class NetworkError(RuntimeError):
  56         pass
  57
  58
  59 class MessageText(object):
  60
  61         ACCURACY_LOW = "med1"
  62         ACCURACY_MEDIUM = "med2"
  63         ACCURACY_HIGH = "high"
  64
  65         def __init__(self):
  66                 self.accuracy = None
  67                 self.text = None
  68
  69         def __str__(self):
  70                 return self.text
  71
  72         def to_dict(self):
  73                 return to_dict(self)
  74
  75         def __eq__(self, other):
  76                 return self.accuracy == other.accuracy and self.text == other.text
  77
  78
  79 class Message(object):
  80
  81         def __init__(self):
  82                 self.whoFrom = None
  83                 self.body = None
  84                 self.when = None
  85
  86         def __str__(self):
  87                 return "%s (%s): %s" % (
  88                         self.whoFrom,
  89                         self.when,
  90                         "".join(unicode(part) for part in self.body)
  91                 )
  92
  93         def to_dict(self):
  94                 selfDict = to_dict(self)
  95                 selfDict["body"] = [text.to_dict() for text in self.body] if self.body is not None else None
  96                 return selfDict
  97
  98         def __eq__(self, other):
  99                 return self.whoFrom == other.whoFrom and self.when == other.when and self.body == other.body
 100
 101
 102 class Conversation(object):
 103
 104         TYPE_VOICEMAIL = "Voicemail"
 105         TYPE_SMS = "SMS"
 106
 107         def __init__(self):
 108                 self.type = None
 109                 self.id = None
 110                 self.contactId = None
 111                 self.name = None
 112                 self.location = None
 113                 self.prettyNumber = None
 114                 self.number = None
 115
 116                 self.time = None
 117                 self.relTime = None
 118                 self.messages = None
 119                 self.isRead = None
 120                 self.isSpam = None
 121                 self.isTrash = None
 122                 self.isArchived = None
 123
 124         def __cmp__(self, other):
 125                 cmpValue = cmp(self.contactId, other.contactId)
 126                 if cmpValue != 0:
 127                         return cmpValue
 128
 129                 cmpValue = cmp(self.time, other.time)
 130                 if cmpValue != 0:
 131                         return cmpValue
 132
 133                 cmpValue = cmp(self.id, other.id)
 134                 if cmpValue != 0:
 135                         return cmpValue
 136
 137         def to_dict(self):
 138                 selfDict = to_dict(self)
 139                 selfDict["messages"] = [message.to_dict() for message in self.messages] if self.messages is not None else None
 140                 return selfDict
 141
 142
 143 class GVoiceBackend(object):
 144         """
 145         This class encapsulates all of the knowledge necessary to interact with the GoogleVoice servers
 146         the functions include login, setting up a callback number, and initalting a callback
 147         """
 148
 149         PHONE_TYPE_HOME = 1
 150         PHONE_TYPE_MOBILE = 2
 151         PHONE_TYPE_WORK = 3
 152         PHONE_TYPE_GIZMO = 7
 153
 154         def __init__(self, cookieFile = None):
 155                 # Important items in this function are the setup of the browser emulation and cookie file
 156                 self._browser = browser_emu.MozillaEmulator(1)
 157                 self._loadedFromCookies = self._browser.load_cookies(cookieFile)
 158
 159                 self._token = ""
 160                 self._accountNum = ""
 161                 self._lastAuthed = 0.0
 162                 self._callbackNumber = ""
 163                 self._callbackNumbers = {}
 164
 165                 # Suprisingly, moving all of these from class to self sped up startup time
 166
 167                 self._validateRe = re.compile("^\+?[0-9]{10,}$")
 168
 169                 self._loginURL = "https://www.google.com/accounts/ServiceLoginAuth"
 170
 171                 SECURE_URL_BASE = "https://www.google.com/voice/"
 172                 SECURE_MOBILE_URL_BASE = SECURE_URL_BASE + "mobile/"
 173                 self._tokenURL = SECURE_URL_BASE + "m"
 174                 self._callUrl = SECURE_URL_BASE + "call/connect"
 175                 self._callCancelURL = SECURE_URL_BASE + "call/cancel"
 176                 self._sendSmsURL = SECURE_URL_BASE + "sms/send"
 177
 178                 self._isDndURL = "https://www.google.com/voice/m/donotdisturb"
 179                 self._isDndRe = re.compile(r"""<input.*?id="doNotDisturb".*?checked="(.*?)"\s*/>""")
 180                 self._setDndURL = "https://www.google.com/voice/m/savednd"
 181
 182                 self._downloadVoicemailURL = SECURE_URL_BASE + "media/send_voicemail/"
 183                 self._markAsReadURL = SECURE_URL_BASE + "m/mark"
 184                 self._archiveMessageURL = SECURE_URL_BASE + "m/archive"
 185
 186                 self._XML_SEARCH_URL = SECURE_URL_BASE + "inbox/search/"
 187                 self._XML_ACCOUNT_URL = SECURE_URL_BASE + "contacts/"
 188                 # HACK really this redirects to the main pge and we are grabbing some javascript
 189                 self._XML_CONTACTS_URL = "http://www.google.com/voice/inbox/search/contact"
 190                 self._CSV_CONTACTS_URL = "http://mail.google.com/mail/contacts/data/export"
 191                 self._JSON_CONTACTS_URL = SECURE_URL_BASE + "b/0/request/user"
 192                 self._XML_RECENT_URL = SECURE_URL_BASE + "inbox/recent/"
 193
 194                 self.XML_FEEDS = (
 195                         'inbox', 'starred', 'all', 'spam', 'trash', 'voicemail', 'sms',
 196                         'recorded', 'placed', 'received', 'missed'
 197                 )
 198                 self._XML_INBOX_URL = SECURE_URL_BASE + "inbox/recent/inbox"
 199                 self._XML_STARRED_URL = SECURE_URL_BASE + "inbox/recent/starred"
 200                 self._XML_ALL_URL = SECURE_URL_BASE + "inbox/recent/all"
 201                 self._XML_SPAM_URL = SECURE_URL_BASE + "inbox/recent/spam"
 202                 self._XML_TRASH_URL = SECURE_URL_BASE + "inbox/recent/trash"
 203                 self._XML_VOICEMAIL_URL = SECURE_URL_BASE + "inbox/recent/voicemail/"
 204                 self._XML_SMS_URL = SECURE_URL_BASE + "inbox/recent/sms/"
 205                 self._JSON_SMS_URL = SECURE_URL_BASE + "b/0/request/messages/"
 206                 self._JSON_SMS_COUNT_URL = SECURE_URL_BASE + "b/0/request/unread"
 207                 self._XML_RECORDED_URL = SECURE_URL_BASE + "inbox/recent/recorded/"
 208                 self._XML_PLACED_URL = SECURE_URL_BASE + "inbox/recent/placed/"
 209                 self._XML_RECEIVED_URL = SECURE_URL_BASE + "inbox/recent/received/"
 210                 self._XML_MISSED_URL = SECURE_URL_BASE + "inbox/recent/missed/"
 211
 212                 self._galxRe = re.compile(r"""<input.*?name="GALX".*?value="(.*?)".*?/>""", re.MULTILINE | re.DOTALL)
 213
 214                 self._seperateVoicemailsRegex = re.compile(r"""^\s*<div id="(\w+)"\s* class=".*?gc-message.*?">""", re.MULTILINE | re.DOTALL)
 215                 self._exactVoicemailTimeRegex = re.compile(r"""<span class="gc-message-time">(.*?)</span>""", re.MULTILINE)
 216                 self._relativeVoicemailTimeRegex = re.compile(r"""<span class="gc-message-relative">(.*?)</span>""", re.MULTILINE)
 217                 self._voicemailNameRegex = re.compile(r"""<a class=.*?gc-message-name-link.*?>(.*?)</a>""", re.MULTILINE | re.DOTALL)
 218                 self._voicemailNumberRegex = re.compile(r"""<input type="hidden" class="gc-text gc-quickcall-ac" value="(.*?)"/>""", re.MULTILINE)
 219                 self._prettyVoicemailNumberRegex = re.compile(r"""<span class="gc-message-type">(.*?)</span>""", re.MULTILINE)
 220                 self._voicemailLocationRegex = re.compile(r"""<span class="gc-message-location">.*?<a.*?>(.*?)</a></span>""", re.MULTILINE)
 221                 self._messagesContactIDRegex = re.compile(r"""<a class=".*?gc-message-name-link.*?">.*?</a>\s*?<span .*?>(.*?)</span>""", re.MULTILINE)
 222                 self._voicemailMessageRegex = re.compile(r"""(<span id="\d+-\d+" class="gc-word-(.*?)">(.*?)</span>|<a .*? class="gc-message-mni">(.*?)</a>)""", re.MULTILINE)
 223                 self._smsFromRegex = re.compile(r"""<span class="gc-message-sms-from">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 224                 self._smsTimeRegex = re.compile(r"""<span class="gc-message-sms-time">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 225                 self._smsTextRegex = re.compile(r"""<span class="gc-message-sms-text">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 226
 227         def is_quick_login_possible(self):
 228                 """
 229                 @returns True then refresh_account_info might be enough to login, else full login is required
 230                 """
 231                 return self._loadedFromCookies or 0.0 < self._lastAuthed
 232
 233         def refresh_account_info(self):
 234                 try:
 235                         page = self._get_page(self._JSON_CONTACTS_URL)
 236                         accountData = self._grab_account_info(page)
 237                 except Exception, e:
 238                         _moduleLogger.exception(str(e))
 239                         return None
 240
 241                 self._browser.save_cookies()
 242                 self._lastAuthed = time.time()
 243                 return accountData
 244
 245         def _get_token(self):
 246                 tokenPage = self._get_page(self._tokenURL)
 247
 248                 galxTokens = self._galxRe.search(tokenPage)
 249                 if galxTokens is not None:
 250                         galxToken = galxTokens.group(1)
 251                 else:
 252                         galxToken = ""
 253                         _moduleLogger.debug("Could not grab GALX token")
 254                 return galxToken
 255
 256         def _login(self, username, password, token):
 257                 loginData = {
 258                         'Email' : username,
 259                         'Passwd' : password,
 260                         'service': "grandcentral",
 261                         "ltmpl": "mobile",
 262                         "btmpl": "mobile",
 263                         "PersistentCookie": "yes",
 264                         "GALX": token,
 265                         "continue": self._JSON_CONTACTS_URL,
 266                 }
 267
 268                 loginSuccessOrFailurePage = self._get_page(self._loginURL, loginData)
 269                 return loginSuccessOrFailurePage
 270
 271         def login(self, username, password):
 272                 """
 273                 Attempt to login to GoogleVoice
 274                 @returns Whether login was successful or not
 275                 @blocks
 276                 """
 277                 self.logout()
 278                 galxToken = self._get_token()
 279                 loginSuccessOrFailurePage = self._login(username, password, galxToken)
 280
 281                 try:
 282                         accountData = self._grab_account_info(loginSuccessOrFailurePage)
 283                 except Exception, e:
 284                         # Retry in case the redirect failed
 285                         # luckily refresh_account_info does everything we need for a retry
 286                         accountData = self.refresh_account_info()
 287                         if accountData is None:
 288                                 _moduleLogger.exception(str(e))
 289                                 return None
 290                         _moduleLogger.info("Redirection failed on initial login attempt, auto-corrected for this")
 291
 292                 self._browser.save_cookies()
 293                 self._lastAuthed = time.time()
 294                 return accountData
 295
 296         def persist(self):
 297                 self._browser.save_cookies()
 298
 299         def shutdown(self):
 300                 self._browser.save_cookies()
 301                 self._token = None
 302                 self._lastAuthed = 0.0
 303
 304         def logout(self):
 305                 self._browser.clear_cookies()
 306                 self._browser.save_cookies()
 307                 self._token = None
 308                 self._lastAuthed = 0.0
 309
 310         def is_dnd(self):
 311                 """
 312                 @blocks
 313                 """
 314                 isDndPage = self._get_page(self._isDndURL)
 315
 316                 dndGroup = self._isDndRe.search(isDndPage)
 317                 if dndGroup is None:
 318                         return False
 319                 dndStatus = dndGroup.group(1)
 320                 isDnd = True if dndStatus.strip().lower() == "true" else False
 321                 return isDnd
 322
 323         def set_dnd(self, doNotDisturb):
 324                 """
 325                 @blocks
 326                 """
 327                 dndPostData = {
 328                         "doNotDisturb": 1 if doNotDisturb else 0,
 329                 }
 330
 331                 dndPage = self._get_page_with_token(self._setDndURL, dndPostData)
 332
 333         def call(self, outgoingNumber):
 334                 """
 335                 This is the main function responsible for initating the callback
 336                 @blocks
 337                 """
 338                 outgoingNumber = self._send_validation(outgoingNumber)
 339                 subscriberNumber = None
 340                 phoneType = guess_phone_type(self._callbackNumber) # @todo Fix this hack
 341
 342                 callData = {
 343                                 'outgoingNumber': outgoingNumber,
 344                                 'forwardingNumber': self._callbackNumber,
 345                                 'subscriberNumber': subscriberNumber or 'undefined',
 346                                 'phoneType': str(phoneType),
 347                                 'remember': '1',
 348                 }
 349                 _moduleLogger.info("%r" % callData)
 350
 351                 page = self._get_page_with_token(
 352                         self._callUrl,
 353                         callData,
 354                 )
 355                 self._parse_with_validation(page)
 356                 return True
 357
 358         def cancel(self, outgoingNumber=None):
 359                 """
 360                 Cancels a call matching outgoing and forwarding numbers (if given).
 361                 Will raise an error if no matching call is being placed
 362                 @blocks
 363                 """
 364                 page = self._get_page_with_token(
 365                         self._callCancelURL,
 366                         {
 367                         'outgoingNumber': outgoingNumber or 'undefined',
 368                         'forwardingNumber': self._callbackNumber or 'undefined',
 369                         'cancelType': 'C2C',
 370                         },
 371                 )
 372                 self._parse_with_validation(page)
 373
 374         def send_sms(self, phoneNumbers, message):
 375                 """
 376                 @blocks
 377                 """
 378                 validatedPhoneNumbers = [
 379                         self._send_validation(phoneNumber)
 380                         for phoneNumber in phoneNumbers
 381                 ]
 382                 flattenedPhoneNumbers = ",".join(validatedPhoneNumbers)
 383                 page = self._get_page_with_token(
 384                         self._sendSmsURL,
 385                         {
 386                                 'phoneNumber': flattenedPhoneNumbers,
 387                                 'text': unicode(message).encode("utf-8"),
 388                         },
 389                 )
 390                 self._parse_with_validation(page)
 391
 392         def search(self, query):
 393                 """
 394                 Search your Google Voice Account history for calls, voicemails, and sms
 395                 Returns ``Folder`` instance containting matching messages
 396                 @blocks
 397                 """
 398                 page = self._get_page(
 399                         self._XML_SEARCH_URL,
 400                         {"q": query},
 401                 )
 402                 json, html = extract_payload(page)
 403                 return json
 404
 405         def get_feed(self, feed):
 406                 """
 407                 @blocks
 408                 """
 409                 actualFeed = "_XML_%s_URL" % feed.upper()
 410                 feedUrl = getattr(self, actualFeed)
 411
 412                 page = self._get_page(feedUrl)
 413                 json, html = extract_payload(page)
 414
 415                 return json
 416
 417         def download(self, messageId, adir):
 418                 """
 419                 Download a voicemail or recorded call MP3 matching the given ``msg``
 420                 which can either be a ``Message`` instance, or a SHA1 identifier.
 421                 Saves files to ``adir`` (defaults to current directory).
 422                 Message hashes can be found in ``self.voicemail().messages`` for example.
 423                 @returns location of saved file.
 424                 @blocks
 425                 """
 426                 page = self._get_page(self._downloadVoicemailURL, {"id": messageId})
 427                 fn = os.path.join(adir, '%s.mp3' % messageId)
 428                 with open(fn, 'wb') as fo:
 429                         fo.write(page)
 430                 return fn
 431
 432         def is_valid_syntax(self, number):
 433                 """
 434                 @returns If This number be called ( syntax validation only )
 435                 """
 436                 return self._validateRe.match(number) is not None
 437
 438         def get_account_number(self):
 439                 """
 440                 @returns The GoogleVoice phone number
 441                 """
 442                 return self._accountNum
 443
 444         def get_callback_numbers(self):
 445                 """
 446                 @returns a dictionary mapping call back numbers to descriptions
 447                 @note These results are cached for 30 minutes.
 448                 """
 449                 return self._callbackNumbers
 450
 451         def set_callback_number(self, callbacknumber):
 452                 """
 453                 Set the number that GoogleVoice calls
 454                 @param callbacknumber should be a proper 10 digit number
 455                 """
 456                 self._callbackNumber = callbacknumber
 457                 _moduleLogger.info("Callback number changed: %r" % self._callbackNumber)
 458                 return True
 459
 460         def get_callback_number(self):
 461                 """
 462                 @returns Current callback number or None
 463                 """
 464                 return self._callbackNumber
 465
 466         def get_recent(self):
 467                 """
 468                 @returns Iterable of (personsName, phoneNumber, exact date, relative date, action)
 469                 @blocks
 470                 """
 471                 recentPages = [
 472                         (action, self._get_page(url))
 473                         for action, url in (
 474                                 ("Received", self._XML_RECEIVED_URL),
 475                                 ("Missed", self._XML_MISSED_URL),
 476                                 ("Placed", self._XML_PLACED_URL),
 477                         )
 478                 ]
 479                 return self._parse_recent(recentPages)
 480
 481         def get_csv_contacts(self):
 482                 data = {
 483                         "groupToExport": "mine",
 484                         "exportType": "ALL",
 485                         "out": "OUTLOOK_CSV",
 486                 }
 487                 encodedData = urllib.urlencode(data)
 488                 contacts = self._get_page(self._CSV_CONTACTS_URL+"?"+encodedData)
 489                 return contacts
 490
 491         def get_voicemails(self):
 492                 """
 493                 @blocks
 494                 """
 495                 voicemailPage = self._get_page(self._XML_VOICEMAIL_URL)
 496                 voicemailHtml = self._grab_html(voicemailPage)
 497                 voicemailJson = self._grab_json(voicemailPage)
 498                 if voicemailJson is None:
 499                         return ()
 500                 parsedVoicemail = self._parse_voicemail(voicemailHtml)
 501                 voicemails = self._merge_conversation_sources(parsedVoicemail, voicemailJson)
 502                 return voicemails
 503
 504         def get_texts(self):
 505                 """
 506                 @blocks
 507                 """
 508                 smsPage = self._get_page(self._XML_SMS_URL)
 509                 smsHtml = self._grab_html(smsPage)
 510                 smsJson = self._grab_json(smsPage)
 511                 if smsJson is None:
 512                         return ()
 513                 parsedSms = self._parse_sms(smsHtml)
 514                 smss = self._merge_conversation_sources(parsedSms, smsJson)
 515                 return smss
 516
 517         def get_unread_counts(self):
 518                 countPage = self._get_page(self._JSON_SMS_COUNT_URL)
 519                 counts = parse_json(countPage)
 520                 counts = counts["unreadCounts"]
 521                 return counts
 522
 523         def mark_message(self, messageId, asRead):
 524                 """
 525                 @blocks
 526                 """
 527                 postData = {
 528                         "read": 1 if asRead else 0,
 529                         "id": messageId,
 530                 }
 531
 532                 markPage = self._get_page(self._markAsReadURL, postData)
 533
 534         def archive_message(self, messageId):
 535                 """
 536                 @blocks
 537                 """
 538                 postData = {
 539                         "id": messageId,
 540                 }
 541
 542                 markPage = self._get_page(self._archiveMessageURL, postData)
 543
 544         def _grab_json(self, flatXml):
 545                 xmlTree = ElementTree.fromstring(flatXml)
 546                 jsonElement = xmlTree.getchildren()[0]
 547                 flatJson = jsonElement.text
 548                 jsonTree = parse_json(flatJson)
 549                 return jsonTree
 550
 551         def _grab_html(self, flatXml):
 552                 xmlTree = ElementTree.fromstring(flatXml)
 553                 htmlElement = xmlTree.getchildren()[1]
 554                 flatHtml = htmlElement.text
 555                 return flatHtml
 556
 557         def _grab_account_info(self, page):
 558                 accountData = parse_json(page)
 559                 self._token = accountData["r"]
 560                 self._accountNum = accountData["number"]["raw"]
 561                 for callback in accountData["phones"].itervalues():
 562                         self._callbackNumbers[callback["phoneNumber"]] = callback["name"]
 563                 if len(self._callbackNumbers) == 0:
 564                         _moduleLogger.debug("Could not extract callback numbers from GoogleVoice (the troublesome page follows):\n%s" % page)
 565                 return accountData
 566
 567         def _send_validation(self, number):
 568                 if not self.is_valid_syntax(number):
 569                         raise ValueError('Number is not valid: "%s"' % number)
 570                 return number
 571
 572         def _parse_recent(self, recentPages):
 573                 for action, flatXml in recentPages:
 574                         allRecentHtml = self._grab_html(flatXml)
 575                         allRecentData = self._parse_history(allRecentHtml)
 576                         for recentCallData in allRecentData:
 577                                 recentCallData["action"] = action
 578                                 yield recentCallData
 579
 580         def _parse_history(self, historyHtml):
 581                 splitVoicemail = self._seperateVoicemailsRegex.split(historyHtml)
 582                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 583                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 584                         exactTime = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 585                         exactTime = google_strptime(exactTime)
 586                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 587                         relativeTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 588                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 589                         location = locationGroup.group(1).strip() if locationGroup else ""
 590
 591                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 592                         name = nameGroup.group(1).strip() if nameGroup else ""
 593                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 594                         number = numberGroup.group(1).strip() if numberGroup else ""
 595                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 596                         prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 597                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 598                         contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 599
 600                         yield {
 601                                 "id": messageId.strip(),
 602                                 "contactId": contactId,
 603                                 "name": unescape(name),
 604                                 "time": exactTime,
 605                                 "relTime": relativeTime,
 606                                 "prettyNumber": prettyNumber,
 607                                 "number": number,
 608                                 "location": unescape(location),
 609                         }
 610
 611         @staticmethod
 612         def _interpret_voicemail_regex(group):
 613                 quality, content, number = group.group(2), group.group(3), group.group(4)
 614                 text = MessageText()
 615                 if quality is not None and content is not None:
 616                         text.accuracy = quality
 617                         text.text = unescape(content)
 618                         return text
 619                 elif number is not None:
 620                         text.accuracy = MessageText.ACCURACY_HIGH
 621                         text.text = number
 622                         return text
 623
 624         def _parse_voicemail(self, voicemailHtml):
 625                 splitVoicemail = self._seperateVoicemailsRegex.split(voicemailHtml)
 626                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 627                         conv = Conversation()
 628                         conv.type = Conversation.TYPE_VOICEMAIL
 629                         conv.id = messageId.strip()
 630
 631                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 632                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 633                         conv.time = google_strptime(exactTimeText)
 634                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 635                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 636                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 637                         conv.location = unescape(locationGroup.group(1).strip() if locationGroup else "")
 638
 639                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 640                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 641                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 642                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 643                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 644                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 645                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 646                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 647
 648                         messageGroups = self._voicemailMessageRegex.finditer(messageHtml)
 649                         messageParts = [
 650                                 self._interpret_voicemail_regex(group)
 651                                 for group in messageGroups
 652                         ] if messageGroups else ((MessageText.ACCURACY_LOW, "No Transcription"), )
 653                         message = Message()
 654                         message.body = messageParts
 655                         message.whoFrom = conv.name
 656                         try:
 657                                 message.when = conv.time.strftime("%I:%M %p")
 658                         except ValueError:
 659                                 _moduleLogger.exception("Confusing time provided: %r" % conv.time)
 660                                 message.when = "Unknown"
 661                         conv.messages = (message, )
 662
 663                         yield conv
 664
 665         @staticmethod
 666         def _interpret_sms_message_parts(fromPart, textPart, timePart):
 667                 text = MessageText()
 668                 text.accuracy = MessageText.ACCURACY_MEDIUM
 669                 text.text = unescape(textPart)
 670
 671                 message = Message()
 672                 message.body = (text, )
 673                 message.whoFrom = fromPart
 674                 message.when = timePart
 675
 676                 return message
 677
 678         def _parse_sms(self, smsHtml):
 679                 splitSms = self._seperateVoicemailsRegex.split(smsHtml)
 680                 for messageId, messageHtml in itergroup(splitSms[1:], 2):
 681                         conv = Conversation()
 682                         conv.type = Conversation.TYPE_SMS
 683                         conv.id = messageId.strip()
 684
 685                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 686                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 687                         conv.time = google_strptime(exactTimeText)
 688                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 689                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 690                         conv.location = ""
 691
 692                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 693                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 694                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 695                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 696                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 697                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 698                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 699                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 700
 701                         fromGroups = self._smsFromRegex.finditer(messageHtml)
 702                         fromParts = (group.group(1).strip() for group in fromGroups)
 703                         textGroups = self._smsTextRegex.finditer(messageHtml)
 704                         textParts = (group.group(1).strip() for group in textGroups)
 705                         timeGroups = self._smsTimeRegex.finditer(messageHtml)
 706                         timeParts = (group.group(1).strip() for group in timeGroups)
 707
 708                         messageParts = itertools.izip(fromParts, textParts, timeParts)
 709                         messages = [self._interpret_sms_message_parts(*parts) for parts in messageParts]
 710                         conv.messages = messages
 711
 712                         yield conv
 713
 714         @staticmethod
 715         def _merge_conversation_sources(parsedMessages, json):
 716                 for message in parsedMessages:
 717                         jsonItem = json["messages"][message.id]
 718                         message.isRead = jsonItem["isRead"]
 719                         message.isSpam = jsonItem["isSpam"]
 720                         message.isTrash = jsonItem["isTrash"]
 721                         message.isArchived = "inbox" not in jsonItem["labels"]
 722                         yield message
 723
 724         def _get_page(self, url, data = None, refererUrl = None):
 725                 headers = {}
 726                 if refererUrl is not None:
 727                         headers["Referer"] = refererUrl
 728
 729                 encodedData = urllib.urlencode(data) if data is not None else None
 730
 731                 try:
 732                         page = self._browser.download(url, encodedData, None, headers)
 733                 except urllib2.URLError, e:
 734                         _moduleLogger.error("Translating error: %s" % str(e))
 735                         raise NetworkError("%s is not accesible" % url)
 736
 737                 return page
 738
 739         def _get_page_with_token(self, url, data = None, refererUrl = None):
 740                 if data is None:
 741                         data = {}
 742                 data['_rnr_se'] = self._token
 743
 744                 page = self._get_page(url, data, refererUrl)
 745
 746                 return page
 747
 748         def _parse_with_validation(self, page):
 749                 json = parse_json(page)
 750                 self._validate_response(json)
 751                 return json
 752
 753         def _validate_response(self, response):
 754                 """
 755                 Validates that the JSON response is A-OK
 756                 """
 757                 try:
 758                         assert response is not None, "Response not provided"
 759                         assert 'ok' in response, "Response lacks status"
 760                         assert response['ok'], "Response not good"
 761                 except AssertionError:
 762                         try:
 763                                 if response["data"]["code"] == 20:
 764                                         raise RuntimeError(
 765 """Ambiguous error 20 returned by Google Voice.
 766 Please verify you have configured your callback number (currently "%s").  If it is configured some other suspected causes are: non-verified callback numbers, and Gizmo5 callback numbers.""" % self._callbackNumber)
 767                         except KeyError:
 768                                 pass
 769                         raise RuntimeError('There was a problem with GV: %s' % response)
 770
 771
 772 _UNESCAPE_ENTITIES = {
 773  "&quot;": '"',
 774  "&nbsp;": " ",
 775  "&#39;": "'",
 776 }
 777
 778
 779 def unescape(text):
 780         plain = saxutils.unescape(text, _UNESCAPE_ENTITIES)
 781         return plain
 782
 783
 784 def google_strptime(time):
 785         """
 786         Hack: Google always returns the time in the same locale.  Sadly if the
 787         local system's locale is different, there isn't a way to perfectly handle
 788         the time.  So instead we handle implement some time formatting
 789         """
 790         abbrevTime = time[:-3]
 791         parsedTime = datetime.datetime.strptime(abbrevTime, "%m/%d/%y %I:%M")
 792         if time.endswith("PM"):
 793                 parsedTime += datetime.timedelta(hours=12)
 794         return parsedTime
 795
 796
 797 def itergroup(iterator, count, padValue = None):
 798         """
 799         Iterate in groups of 'count' values. If there
 800         aren't enough values, the last result is padded with
 801         None.
 802
 803         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 804         ...     print tuple(val)
 805         (1, 2, 3)
 806         (4, 5, 6)
 807         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 808         ...     print list(val)
 809         [1, 2, 3]
 810         [4, 5, 6]
 811         >>> for val in itergroup([1, 2, 3, 4, 5, 6, 7], 3):
 812         ...     print tuple(val)
 813         (1, 2, 3)
 814         (4, 5, 6)
 815         (7, None, None)
 816         >>> for val in itergroup("123456", 3):
 817         ...     print tuple(val)
 818         ('1', '2', '3')
 819         ('4', '5', '6')
 820         >>> for val in itergroup("123456", 3):
 821         ...     print repr("".join(val))
 822         '123'
 823         '456'
 824         """
 825         paddedIterator = itertools.chain(iterator, itertools.repeat(padValue, count-1))
 826         nIterators = (paddedIterator, ) * count
 827         return itertools.izip(*nIterators)
 828
 829
 830 def safe_eval(s):
 831         _TRUE_REGEX = re.compile("true")
 832         _FALSE_REGEX = re.compile("false")
 833         _COMMENT_REGEX = re.compile("^\s+//.*$", re.M)
 834         s = _TRUE_REGEX.sub("True", s)
 835         s = _FALSE_REGEX.sub("False", s)
 836         s = _COMMENT_REGEX.sub("#", s)
 837         try:
 838                 results = eval(s, {}, {})
 839         except SyntaxError:
 840                 _moduleLogger.exception("Oops")
 841                 results = None
 842         return results
 843
 844
 845 def _fake_parse_json(flattened):
 846         return safe_eval(flattened)
 847
 848
 849 def _actual_parse_json(flattened):
 850         return simplejson.loads(flattened)
 851
 852
 853 if simplejson is None:
 854         parse_json = _fake_parse_json
 855 else:
 856         parse_json = _actual_parse_json
 857
 858
 859 def extract_payload(flatXml):
 860         xmlTree = ElementTree.fromstring(flatXml)
 861
 862         jsonElement = xmlTree.getchildren()[0]
 863         flatJson = jsonElement.text
 864         jsonTree = parse_json(flatJson)
 865
 866         htmlElement = xmlTree.getchildren()[1]
 867         flatHtml = htmlElement.text
 868
 869         return jsonTree, flatHtml
 870
 871
 872 def guess_phone_type(number):
 873         if number.startswith("747") or number.startswith("1747") or number.startswith("+1747"):
 874                 return GVoiceBackend.PHONE_TYPE_GIZMO
 875         else:
 876                 return GVoiceBackend.PHONE_TYPE_MOBILE
 877
 878
 879 def get_sane_callback(backend):
 880         """
 881         Try to set a sane default callback number on these preferences
 882         1) 1747 numbers ( Gizmo )
 883         2) anything with gizmo in the name
 884         3) anything with computer in the name
 885         4) the first value
 886         """
 887         numbers = backend.get_callback_numbers()
 888
 889         priorityOrderedCriteria = [
 890                 ("\+1747", None),
 891                 ("1747", None),
 892                 ("747", None),
 893                 (None, "gizmo"),
 894                 (None, "computer"),
 895                 (None, "sip"),
 896                 (None, None),
 897         ]
 898
 899         for numberCriteria, descriptionCriteria in priorityOrderedCriteria:
 900                 numberMatcher = None
 901                 descriptionMatcher = None
 902                 if numberCriteria is not None:
 903                         numberMatcher = re.compile(numberCriteria)
 904                 elif descriptionCriteria is not None:
 905                         descriptionMatcher = re.compile(descriptionCriteria, re.I)
 906
 907                 for number, description in numbers.iteritems():
 908                         if numberMatcher is not None and numberMatcher.match(number) is None:
 909                                 continue
 910                         if descriptionMatcher is not None and descriptionMatcher.match(description) is None:
 911                                 continue
 912                         return number
 913
 914
 915 def set_sane_callback(backend):
 916         """
 917         Try to set a sane default callback number on these preferences
 918         1) 1747 numbers ( Gizmo )
 919         2) anything with gizmo in the name
 920         3) anything with computer in the name
 921         4) the first value
 922         """
 923         number = get_sane_callback(backend)
 924         backend.set_callback_number(number)
 925
 926
 927 def _is_not_special(name):
 928         return not name.startswith("_") and name[0].lower() == name[0] and "_" not in name
 929
 930
 931 def to_dict(obj):
 932         members = inspect.getmembers(obj)
 933         return dict((name, value) for (name, value) in members if _is_not_special(name))
 934
 935
 936 def grab_debug_info(username, password):
 937         cookieFile = os.path.join(".", "raw_cookies.txt")
 938         try:
 939                 os.remove(cookieFile)
 940         except OSError:
 941                 pass
 942
 943         backend = GVoiceBackend(cookieFile)
 944         browser = backend._browser
 945
 946         _TEST_WEBPAGES = [
 947                 ("token", backend._tokenURL),
 948                 ("login", backend._loginURL),
 949                 ("isdnd", backend._isDndURL),
 950                 ("account", backend._XML_ACCOUNT_URL),
 951                 ("html_contacts", backend._XML_CONTACTS_URL),
 952                 ("contacts", backend._JSON_CONTACTS_URL),
 953                 ("csv", backend._CSV_CONTACTS_URL),
 954
 955                 ("voicemail", backend._XML_VOICEMAIL_URL),
 956                 ("html_sms", backend._XML_SMS_URL),
 957                 ("sms", backend._JSON_SMS_URL),
 958                 ("count", backend._JSON_SMS_COUNT_URL),
 959
 960                 ("recent", backend._XML_RECENT_URL),
 961                 ("placed", backend._XML_PLACED_URL),
 962                 ("recieved", backend._XML_RECEIVED_URL),
 963                 ("missed", backend._XML_MISSED_URL),
 964         ]
 965
 966         # Get Pages
 967         print "Grabbing pre-login pages"
 968         for name, url in _TEST_WEBPAGES:
 969                 try:
 970                         page = browser.download(url)
 971                 except StandardError, e:
 972                         print e.message
 973                         continue
 974                 print "\tWriting to file"
 975                 with open("not_loggedin_%s.txt" % name, "w") as f:
 976                         f.write(page)
 977
 978         # Login
 979         print "Attempting login"
 980         galxToken = backend._get_token()
 981         loginSuccessOrFailurePage = backend._login(username, password, galxToken)
 982         with open("loggingin.txt", "w") as f:
 983                 print "\tWriting to file"
 984                 f.write(loginSuccessOrFailurePage)
 985         try:
 986                 backend._grab_account_info(loginSuccessOrFailurePage)
 987         except Exception:
 988                 # Retry in case the redirect failed
 989                 # luckily refresh_account_info does everything we need for a retry
 990                 loggedIn = backend.refresh_account_info() is not None
 991                 if not loggedIn:
 992                         raise
 993
 994         # Get Pages
 995         print "Grabbing post-login pages"
 996         for name, url in _TEST_WEBPAGES:
 997                 try:
 998                         page = browser.download(url)
 999                 except StandardError, e:
1000                         print str(e)
1001                         continue
1002                 print "\tWriting to file"
1003                 with open("loggedin_%s.txt" % name, "w") as f:
1004                         f.write(page)
1005
1006         # Cookies
1007         browser.save_cookies()
1008         print "\tWriting cookies to file"
1009         with open("cookies.txt", "w") as f:
1010                 f.writelines(
1011                         "%s: %s\n" % (c.name, c.value)
1012                         for c in browser._cookies
1013                 )
1014
1015
1016 def main():
1017         import sys
1018         logging.basicConfig(level=logging.DEBUG)
1019         args = sys.argv
1020         if 3 <= len(args):
1021                 username = args[1]
1022                 password = args[2]
1023
1024         grab_debug_info(username, password)
1025
1026
1027 if __name__ == "__main__":
1028         main()