git.maemo.org Git - gc-dialer/blob - src/backends/gvoice/gvoice.py

   1 #!/usr/bin/python
   2
   3 """
   4 DialCentral - Front end for Google's GoogleVoice service.
   5 Copyright (C) 2008  Eric Warnke ericew AT gmail DOT com
   6
   7 This library is free software; you can redistribute it and/or
   8 modify it under the terms of the GNU Lesser General Public
   9 License as published by the Free Software Foundation; either
  10 version 2.1 of the License, or (at your option) any later version.
  11
  12 This library is distributed in the hope that it will be useful,
  13 but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 Lesser General Public License for more details.
  16
  17 You should have received a copy of the GNU Lesser General Public
  18 License along with this library; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 Google Voice backend code
  22
  23 Resources
  24         http://thatsmith.com/2009/03/google-voice-addon-for-firefox/
  25         http://posttopic.com/topic/google-voice-add-on-development
  26 """
  27
  28 from __future__ import with_statement
  29
  30 import os
  31 import re
  32 import urllib
  33 import urllib2
  34 import time
  35 import datetime
  36 import itertools
  37 import logging
  38 import inspect
  39
  40 from xml.sax import saxutils
  41 from xml.etree import ElementTree
  42
  43 try:
  44         import simplejson as _simplejson
  45         simplejson = _simplejson
  46 except ImportError:
  47         simplejson = None
  48
  49 import browser_emu
  50
  51
  52 _moduleLogger = logging.getLogger(__name__)
  53
  54
  55 class NetworkError(RuntimeError):
  56         pass
  57
  58
  59 class MessageText(object):
  60
  61         ACCURACY_LOW = "med1"
  62         ACCURACY_MEDIUM = "med2"
  63         ACCURACY_HIGH = "high"
  64
  65         def __init__(self):
  66                 self.accuracy = None
  67                 self.text = None
  68
  69         def __str__(self):
  70                 return self.text
  71
  72         def to_dict(self):
  73                 return to_dict(self)
  74
  75         def __eq__(self, other):
  76                 return self.accuracy == other.accuracy and self.text == other.text
  77
  78
  79 class Message(object):
  80
  81         def __init__(self):
  82                 self.whoFrom = None
  83                 self.body = None
  84                 self.when = None
  85
  86         def __str__(self):
  87                 return "%s (%s): %s" % (
  88                         self.whoFrom,
  89                         self.when,
  90                         "".join(unicode(part) for part in self.body)
  91                 )
  92
  93         def to_dict(self):
  94                 selfDict = to_dict(self)
  95                 selfDict["body"] = [text.to_dict() for text in self.body] if self.body is not None else None
  96                 return selfDict
  97
  98         def __eq__(self, other):
  99                 return self.whoFrom == other.whoFrom and self.when == other.when and self.body == other.body
 100
 101
 102 class Conversation(object):
 103
 104         TYPE_VOICEMAIL = "Voicemail"
 105         TYPE_SMS = "SMS"
 106
 107         def __init__(self):
 108                 self.type = None
 109                 self.id = None
 110                 self.contactId = None
 111                 self.name = None
 112                 self.location = None
 113                 self.prettyNumber = None
 114                 self.number = None
 115
 116                 self.time = None
 117                 self.relTime = None
 118                 self.messages = None
 119                 self.isRead = None
 120                 self.isSpam = None
 121                 self.isTrash = None
 122                 self.isArchived = None
 123
 124         def __cmp__(self, other):
 125                 cmpValue = cmp(self.contactId, other.contactId)
 126                 if cmpValue != 0:
 127                         return cmpValue
 128
 129                 cmpValue = cmp(self.time, other.time)
 130                 if cmpValue != 0:
 131                         return cmpValue
 132
 133                 cmpValue = cmp(self.id, other.id)
 134                 if cmpValue != 0:
 135                         return cmpValue
 136
 137         def to_dict(self):
 138                 selfDict = to_dict(self)
 139                 selfDict["messages"] = [message.to_dict() for message in self.messages] if self.messages is not None else None
 140                 return selfDict
 141
 142
 143 class GVoiceBackend(object):
 144         """
 145         This class encapsulates all of the knowledge necessary to interact with the GoogleVoice servers
 146         the functions include login, setting up a callback number, and initalting a callback
 147         """
 148
 149         PHONE_TYPE_HOME = 1
 150         PHONE_TYPE_MOBILE = 2
 151         PHONE_TYPE_WORK = 3
 152         PHONE_TYPE_GIZMO = 7
 153
 154         def __init__(self, cookieFile = None):
 155                 # Important items in this function are the setup of the browser emulation and cookie file
 156                 self._browser = browser_emu.MozillaEmulator(1)
 157                 self._loadedFromCookies = self._browser.load_cookies(cookieFile)
 158
 159                 self._token = ""
 160                 self._accountNum = ""
 161                 self._lastAuthed = 0.0
 162                 self._callbackNumber = ""
 163                 self._callbackNumbers = {}
 164
 165                 # Suprisingly, moving all of these from class to self sped up startup time
 166
 167                 self._validateRe = re.compile("^\+?[0-9]{10,}$")
 168
 169                 self._loginURL = "https://www.google.com/accounts/ServiceLoginAuth"
 170
 171                 SECURE_URL_BASE = "https://www.google.com/voice/"
 172                 SECURE_MOBILE_URL_BASE = SECURE_URL_BASE + "mobile/"
 173                 self._tokenURL = SECURE_URL_BASE + "m"
 174                 self._callUrl = SECURE_URL_BASE + "call/connect"
 175                 self._callCancelURL = SECURE_URL_BASE + "call/cancel"
 176                 self._sendSmsURL = SECURE_URL_BASE + "sms/send"
 177
 178                 self._isDndURL = "https://www.google.com/voice/m/donotdisturb"
 179                 self._isDndRe = re.compile(r"""<input.*?id="doNotDisturb".*?checked="(.*?)"\s*/>""")
 180                 self._setDndURL = "https://www.google.com/voice/m/savednd"
 181
 182                 self._downloadVoicemailURL = SECURE_URL_BASE + "media/send_voicemail/"
 183                 self._markAsReadURL = SECURE_URL_BASE + "m/mark"
 184                 self._archiveMessageURL = SECURE_URL_BASE + "m/archive"
 185
 186                 self._XML_SEARCH_URL = SECURE_URL_BASE + "inbox/search/"
 187                 self._XML_ACCOUNT_URL = SECURE_URL_BASE + "contacts/"
 188                 # HACK really this redirects to the main pge and we are grabbing some javascript
 189                 self._XML_CONTACTS_URL = "http://www.google.com/voice/inbox/search/contact"
 190                 self._CSV_CONTACTS_URL = "http://mail.google.com/mail/contacts/data/export"
 191                 self._JSON_CONTACTS_URL = SECURE_URL_BASE + "b/0/request/user"
 192                 self._XML_RECENT_URL = SECURE_URL_BASE + "inbox/recent/"
 193
 194                 self.XML_FEEDS = (
 195                         'inbox', 'starred', 'all', 'spam', 'trash', 'voicemail', 'sms',
 196                         'recorded', 'placed', 'received', 'missed'
 197                 )
 198                 self._XML_INBOX_URL = SECURE_URL_BASE + "inbox/recent/inbox"
 199                 self._XML_STARRED_URL = SECURE_URL_BASE + "inbox/recent/starred"
 200                 self._XML_ALL_URL = SECURE_URL_BASE + "inbox/recent/all"
 201                 self._XML_SPAM_URL = SECURE_URL_BASE + "inbox/recent/spam"
 202                 self._XML_TRASH_URL = SECURE_URL_BASE + "inbox/recent/trash"
 203                 self._XML_VOICEMAIL_URL = SECURE_URL_BASE + "inbox/recent/voicemail/"
 204                 self._XML_SMS_URL = SECURE_URL_BASE + "inbox/recent/sms/"
 205                 self._JSON_SMS_URL = SECURE_URL_BASE + "b/0/request/messages/"
 206                 self._JSON_SMS_COUNT_URL = SECURE_URL_BASE + "b/0/request/unread"
 207                 self._XML_RECORDED_URL = SECURE_URL_BASE + "inbox/recent/recorded/"
 208                 self._XML_PLACED_URL = SECURE_URL_BASE + "inbox/recent/placed/"
 209                 self._XML_RECEIVED_URL = SECURE_URL_BASE + "inbox/recent/received/"
 210                 self._XML_MISSED_URL = SECURE_URL_BASE + "inbox/recent/missed/"
 211
 212                 self._galxRe = re.compile(r"""<input.*?name="GALX".*?value="(.*?)".*?/>""", re.MULTILINE | re.DOTALL)
 213
 214                 self._seperateVoicemailsRegex = re.compile(r"""^\s*<div id="(\w+)"\s* class=".*?gc-message.*?">""", re.MULTILINE | re.DOTALL)
 215                 self._exactVoicemailTimeRegex = re.compile(r"""<span class="gc-message-time">(.*?)</span>""", re.MULTILINE)
 216                 self._relativeVoicemailTimeRegex = re.compile(r"""<span class="gc-message-relative">(.*?)</span>""", re.MULTILINE)
 217                 self._voicemailNameRegex = re.compile(r"""<a class=.*?gc-message-name-link.*?>(.*?)</a>""", re.MULTILINE | re.DOTALL)
 218                 self._voicemailNumberRegex = re.compile(r"""<input type="hidden" class="gc-text gc-quickcall-ac" value="(.*?)"/>""", re.MULTILINE)
 219                 self._prettyVoicemailNumberRegex = re.compile(r"""<span class="gc-message-type">(.*?)</span>""", re.MULTILINE)
 220                 self._voicemailLocationRegex = re.compile(r"""<span class="gc-message-location">.*?<a.*?>(.*?)</a></span>""", re.MULTILINE)
 221                 self._messagesContactIDRegex = re.compile(r"""<a class=".*?gc-message-name-link.*?">.*?</a>\s*?<span .*?>(.*?)</span>""", re.MULTILINE)
 222                 self._voicemailMessageRegex = re.compile(r"""(<span id="\d+-\d+" class="gc-word-(.*?)">(.*?)</span>|<a .*? class="gc-message-mni">(.*?)</a>)""", re.MULTILINE)
 223                 self._smsFromRegex = re.compile(r"""<span class="gc-message-sms-from">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 224                 self._smsTimeRegex = re.compile(r"""<span class="gc-message-sms-time">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 225                 self._smsTextRegex = re.compile(r"""<span class="gc-message-sms-text">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 226
 227         def is_quick_login_possible(self):
 228                 """
 229                 @returns True then refresh_account_info might be enough to login, else full login is required
 230                 """
 231                 return self._loadedFromCookies or 0.0 < self._lastAuthed
 232
 233         def refresh_account_info(self):
 234                 try:
 235                         page = self._get_page(self._JSON_CONTACTS_URL)
 236                         accountData = self._grab_account_info(page)
 237                 except Exception, e:
 238                         _moduleLogger.exception(str(e))
 239                         return None
 240
 241                 self._browser.save_cookies()
 242                 self._lastAuthed = time.time()
 243                 return accountData
 244
 245         def _get_token(self):
 246                 tokenPage = self._get_page(self._tokenURL)
 247
 248                 galxTokens = self._galxRe.search(tokenPage)
 249                 if galxTokens is not None:
 250                         galxToken = galxTokens.group(1)
 251                 else:
 252                         galxToken = ""
 253                         _moduleLogger.debug("Could not grab GALX token")
 254                 return galxToken
 255
 256         def _login(self, username, password, token):
 257                 loginData = {
 258                         'Email' : username,
 259                         'Passwd' : password,
 260                         'service': "grandcentral",
 261                         "ltmpl": "mobile",
 262                         "btmpl": "mobile",
 263                         "PersistentCookie": "yes",
 264                         "GALX": token,
 265                         "continue": self._JSON_CONTACTS_URL,
 266                 }
 267
 268                 loginSuccessOrFailurePage = self._get_page(self._loginURL, loginData)
 269                 return loginSuccessOrFailurePage
 270
 271         def login(self, username, password):
 272                 """
 273                 Attempt to login to GoogleVoice
 274                 @returns Whether login was successful or not
 275                 @blocks
 276                 """
 277                 self.logout()
 278                 galxToken = self._get_token()
 279                 loginSuccessOrFailurePage = self._login(username, password, galxToken)
 280
 281                 try:
 282                         accountData = self._grab_account_info(loginSuccessOrFailurePage)
 283                 except Exception, e:
 284                         # Retry in case the redirect failed
 285                         # luckily refresh_account_info does everything we need for a retry
 286                         accountData = self.refresh_account_info()
 287                         if accountData is None:
 288                                 _moduleLogger.exception(str(e))
 289                                 return None
 290                         _moduleLogger.info("Redirection failed on initial login attempt, auto-corrected for this")
 291
 292                 self._browser.save_cookies()
 293                 self._lastAuthed = time.time()
 294                 return accountData
 295
 296         def persist(self):
 297                 self._browser.save_cookies()
 298
 299         def shutdown(self):
 300                 self._browser.save_cookies()
 301                 self._token = None
 302                 self._lastAuthed = 0.0
 303
 304         def logout(self):
 305                 self._browser.clear_cookies()
 306                 self._browser.save_cookies()
 307                 self._token = None
 308                 self._lastAuthed = 0.0
 309
 310         def is_dnd(self):
 311                 """
 312                 @blocks
 313                 """
 314                 isDndPage = self._get_page(self._isDndURL)
 315
 316                 dndGroup = self._isDndRe.search(isDndPage)
 317                 if dndGroup is None:
 318                         return False
 319                 dndStatus = dndGroup.group(1)
 320                 isDnd = True if dndStatus.strip().lower() == "true" else False
 321                 return isDnd
 322
 323         def set_dnd(self, doNotDisturb):
 324                 """
 325                 @blocks
 326                 """
 327                 dndPostData = {
 328                         "doNotDisturb": 1 if doNotDisturb else 0,
 329                 }
 330
 331                 dndPage = self._get_page_with_token(self._setDndURL, dndPostData)
 332
 333         def call(self, outgoingNumber):
 334                 """
 335                 This is the main function responsible for initating the callback
 336                 @blocks
 337                 """
 338                 outgoingNumber = self._send_validation(outgoingNumber)
 339                 subscriberNumber = None
 340                 phoneType = guess_phone_type(self._callbackNumber) # @todo Fix this hack
 341
 342                 callData = {
 343                                 'outgoingNumber': outgoingNumber,
 344                                 'forwardingNumber': self._callbackNumber,
 345                                 'subscriberNumber': subscriberNumber or 'undefined',
 346                                 'phoneType': str(phoneType),
 347                                 'remember': '1',
 348                 }
 349                 _moduleLogger.info("%r" % callData)
 350
 351                 page = self._get_page_with_token(
 352                         self._callUrl,
 353                         callData,
 354                 )
 355                 self._parse_with_validation(page)
 356                 return True
 357
 358         def cancel(self, outgoingNumber=None):
 359                 """
 360                 Cancels a call matching outgoing and forwarding numbers (if given).
 361                 Will raise an error if no matching call is being placed
 362                 @blocks
 363                 """
 364                 page = self._get_page_with_token(
 365                         self._callCancelURL,
 366                         {
 367                         'outgoingNumber': outgoingNumber or 'undefined',
 368                         'forwardingNumber': self._callbackNumber or 'undefined',
 369                         'cancelType': 'C2C',
 370                         },
 371                 )
 372                 self._parse_with_validation(page)
 373
 374         def send_sms(self, phoneNumbers, message):
 375                 """
 376                 @blocks
 377                 """
 378                 validatedPhoneNumbers = [
 379                         self._send_validation(phoneNumber)
 380                         for phoneNumber in phoneNumbers
 381                 ]
 382                 flattenedPhoneNumbers = ",".join(validatedPhoneNumbers)
 383                 page = self._get_page_with_token(
 384                         self._sendSmsURL,
 385                         {
 386                                 'phoneNumber': flattenedPhoneNumbers,
 387                                 'text': unicode(message).encode("utf-8"),
 388                         },
 389                 )
 390                 self._parse_with_validation(page)
 391
 392         def search(self, query):
 393                 """
 394                 Search your Google Voice Account history for calls, voicemails, and sms
 395                 Returns ``Folder`` instance containting matching messages
 396                 @blocks
 397                 """
 398                 page = self._get_page(
 399                         self._XML_SEARCH_URL,
 400                         {"q": query},
 401                 )
 402                 json, html = extract_payload(page)
 403                 return json
 404
 405         def get_feed(self, feed):
 406                 """
 407                 @blocks
 408                 """
 409                 actualFeed = "_XML_%s_URL" % feed.upper()
 410                 feedUrl = getattr(self, actualFeed)
 411
 412                 page = self._get_page(feedUrl)
 413                 json, html = extract_payload(page)
 414
 415                 return json
 416
 417         def download(self, messageId, adir):
 418                 """
 419                 Download a voicemail or recorded call MP3 matching the given ``msg``
 420                 which can either be a ``Message`` instance, or a SHA1 identifier.
 421                 Saves files to ``adir`` (defaults to current directory).
 422                 Message hashes can be found in ``self.voicemail().messages`` for example.
 423                 @returns location of saved file.
 424                 @blocks
 425                 """
 426                 page = self._get_page(self._downloadVoicemailURL, {"id": messageId})
 427                 fn = os.path.join(adir, '%s.mp3' % messageId)
 428                 with open(fn, 'wb') as fo:
 429                         fo.write(page)
 430                 return fn
 431
 432         def is_valid_syntax(self, number):
 433                 """
 434                 @returns If This number be called ( syntax validation only )
 435                 """
 436                 return self._validateRe.match(number) is not None
 437
 438         def get_account_number(self):
 439                 """
 440                 @returns The GoogleVoice phone number
 441                 """
 442                 return self._accountNum
 443
 444         def get_callback_numbers(self):
 445                 """
 446                 @returns a dictionary mapping call back numbers to descriptions
 447                 @note These results are cached for 30 minutes.
 448                 """
 449                 return self._callbackNumbers
 450
 451         def set_callback_number(self, callbacknumber):
 452                 """
 453                 Set the number that GoogleVoice calls
 454                 @param callbacknumber should be a proper 10 digit number
 455                 """
 456                 self._callbackNumber = callbacknumber
 457                 _moduleLogger.info("Callback number changed: %r" % self._callbackNumber)
 458                 return True
 459
 460         def get_callback_number(self):
 461                 """
 462                 @returns Current callback number or None
 463                 """
 464                 return self._callbackNumber
 465
 466         def get_recent(self):
 467                 """
 468                 @returns Iterable of (personsName, phoneNumber, exact date, relative date, action)
 469                 @blocks
 470                 """
 471                 recentPages = [
 472                         (action, self._get_page(url))
 473                         for action, url in (
 474                                 ("Received", self._XML_RECEIVED_URL),
 475                                 ("Missed", self._XML_MISSED_URL),
 476                                 ("Placed", self._XML_PLACED_URL),
 477                         )
 478                 ]
 479                 return self._parse_recent(recentPages)
 480
 481         def get_contacts(self):
 482                 """
 483                 @returns Iterable of (contact id, contact name)
 484                 @blocks
 485                 """
 486                 page = self._get_page(self._JSON_CONTACTS_URL)
 487                 return self._process_contacts(page)
 488
 489         def get_csv_contacts(self):
 490                 data = {
 491                         "groupToExport": "mine",
 492                         "exportType": "ALL",
 493                         "out": "OUTLOOK_CSV",
 494                 }
 495                 encodedData = urllib.urlencode(data)
 496                 contacts = self._get_page(self._CSV_CONTACTS_URL+"?"+encodedData)
 497                 return contacts
 498
 499         def get_voicemails(self):
 500                 """
 501                 @blocks
 502                 """
 503                 voicemailPage = self._get_page(self._XML_VOICEMAIL_URL)
 504                 voicemailHtml = self._grab_html(voicemailPage)
 505                 voicemailJson = self._grab_json(voicemailPage)
 506                 if voicemailJson is None:
 507                         return ()
 508                 parsedVoicemail = self._parse_voicemail(voicemailHtml)
 509                 voicemails = self._merge_conversation_sources(parsedVoicemail, voicemailJson)
 510                 return voicemails
 511
 512         def get_texts(self):
 513                 """
 514                 @blocks
 515                 """
 516                 smsPage = self._get_page(self._XML_SMS_URL)
 517                 smsHtml = self._grab_html(smsPage)
 518                 smsJson = self._grab_json(smsPage)
 519                 if smsJson is None:
 520                         return ()
 521                 parsedSms = self._parse_sms(smsHtml)
 522                 smss = self._merge_conversation_sources(parsedSms, smsJson)
 523                 return smss
 524
 525         def get_unread_counts(self):
 526                 countPage = self._get_page(self._JSON_SMS_COUNT_URL)
 527                 counts = parse_json(countPage)
 528                 counts = counts["unreadCounts"]
 529                 return counts
 530
 531         def mark_message(self, messageId, asRead):
 532                 """
 533                 @blocks
 534                 """
 535                 postData = {
 536                         "read": 1 if asRead else 0,
 537                         "id": messageId,
 538                 }
 539
 540                 markPage = self._get_page(self._markAsReadURL, postData)
 541
 542         def archive_message(self, messageId):
 543                 """
 544                 @blocks
 545                 """
 546                 postData = {
 547                         "id": messageId,
 548                 }
 549
 550                 markPage = self._get_page(self._archiveMessageURL, postData)
 551
 552         def _grab_json(self, flatXml):
 553                 xmlTree = ElementTree.fromstring(flatXml)
 554                 jsonElement = xmlTree.getchildren()[0]
 555                 flatJson = jsonElement.text
 556                 jsonTree = parse_json(flatJson)
 557                 return jsonTree
 558
 559         def _grab_html(self, flatXml):
 560                 xmlTree = ElementTree.fromstring(flatXml)
 561                 htmlElement = xmlTree.getchildren()[1]
 562                 flatHtml = htmlElement.text
 563                 return flatHtml
 564
 565         def _grab_account_info(self, page):
 566                 accountData = parse_json(page)
 567                 self._token = accountData["r"]
 568                 self._accountNum = accountData["number"]["raw"]
 569                 for callback in accountData["phones"].itervalues():
 570                         self._callbackNumbers[callback["phoneNumber"]] = callback["name"]
 571                 if len(self._callbackNumbers) == 0:
 572                         _moduleLogger.debug("Could not extract callback numbers from GoogleVoice (the troublesome page follows):\n%s" % page)
 573                 return accountData
 574
 575         def _send_validation(self, number):
 576                 if not self.is_valid_syntax(number):
 577                         raise ValueError('Number is not valid: "%s"' % number)
 578                 return number
 579
 580         def _parse_recent(self, recentPages):
 581                 for action, flatXml in recentPages:
 582                         allRecentHtml = self._grab_html(flatXml)
 583                         allRecentData = self._parse_history(allRecentHtml)
 584                         for recentCallData in allRecentData:
 585                                 recentCallData["action"] = action
 586                                 yield recentCallData
 587
 588         def _process_contacts(self, page):
 589                 accountData = parse_json(page)
 590                 for contactId, contactDetails in accountData["contacts"].iteritems():
 591                         # A zero contact id is the catch all for unknown contacts
 592                         if contactId != "0":
 593                                 yield contactId, contactDetails
 594
 595         def _parse_history(self, historyHtml):
 596                 splitVoicemail = self._seperateVoicemailsRegex.split(historyHtml)
 597                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 598                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 599                         exactTime = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 600                         exactTime = google_strptime(exactTime)
 601                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 602                         relativeTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 603                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 604                         location = locationGroup.group(1).strip() if locationGroup else ""
 605
 606                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 607                         name = nameGroup.group(1).strip() if nameGroup else ""
 608                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 609                         number = numberGroup.group(1).strip() if numberGroup else ""
 610                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 611                         prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 612                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 613                         contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 614
 615                         yield {
 616                                 "id": messageId.strip(),
 617                                 "contactId": contactId,
 618                                 "name": unescape(name),
 619                                 "time": exactTime,
 620                                 "relTime": relativeTime,
 621                                 "prettyNumber": prettyNumber,
 622                                 "number": number,
 623                                 "location": unescape(location),
 624                         }
 625
 626         @staticmethod
 627         def _interpret_voicemail_regex(group):
 628                 quality, content, number = group.group(2), group.group(3), group.group(4)
 629                 text = MessageText()
 630                 if quality is not None and content is not None:
 631                         text.accuracy = quality
 632                         text.text = unescape(content)
 633                         return text
 634                 elif number is not None:
 635                         text.accuracy = MessageText.ACCURACY_HIGH
 636                         text.text = number
 637                         return text
 638
 639         def _parse_voicemail(self, voicemailHtml):
 640                 splitVoicemail = self._seperateVoicemailsRegex.split(voicemailHtml)
 641                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 642                         conv = Conversation()
 643                         conv.type = Conversation.TYPE_VOICEMAIL
 644                         conv.id = messageId.strip()
 645
 646                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 647                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 648                         conv.time = google_strptime(exactTimeText)
 649                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 650                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 651                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 652                         conv.location = unescape(locationGroup.group(1).strip() if locationGroup else "")
 653
 654                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 655                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 656                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 657                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 658                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 659                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 660                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 661                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 662
 663                         messageGroups = self._voicemailMessageRegex.finditer(messageHtml)
 664                         messageParts = [
 665                                 self._interpret_voicemail_regex(group)
 666                                 for group in messageGroups
 667                         ] if messageGroups else ((MessageText.ACCURACY_LOW, "No Transcription"), )
 668                         message = Message()
 669                         message.body = messageParts
 670                         message.whoFrom = conv.name
 671                         try:
 672                                 message.when = conv.time.strftime("%I:%M %p")
 673                         except ValueError:
 674                                 _moduleLogger.exception("Confusing time provided: %r" % conv.time)
 675                                 message.when = "Unknown"
 676                         conv.messages = (message, )
 677
 678                         yield conv
 679
 680         @staticmethod
 681         def _interpret_sms_message_parts(fromPart, textPart, timePart):
 682                 text = MessageText()
 683                 text.accuracy = MessageText.ACCURACY_MEDIUM
 684                 text.text = unescape(textPart)
 685
 686                 message = Message()
 687                 message.body = (text, )
 688                 message.whoFrom = fromPart
 689                 message.when = timePart
 690
 691                 return message
 692
 693         def _parse_sms(self, smsHtml):
 694                 splitSms = self._seperateVoicemailsRegex.split(smsHtml)
 695                 for messageId, messageHtml in itergroup(splitSms[1:], 2):
 696                         conv = Conversation()
 697                         conv.type = Conversation.TYPE_SMS
 698                         conv.id = messageId.strip()
 699
 700                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 701                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 702                         conv.time = google_strptime(exactTimeText)
 703                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 704                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 705                         conv.location = ""
 706
 707                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 708                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 709                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 710                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 711                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 712                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 713                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 714                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 715
 716                         fromGroups = self._smsFromRegex.finditer(messageHtml)
 717                         fromParts = (group.group(1).strip() for group in fromGroups)
 718                         textGroups = self._smsTextRegex.finditer(messageHtml)
 719                         textParts = (group.group(1).strip() for group in textGroups)
 720                         timeGroups = self._smsTimeRegex.finditer(messageHtml)
 721                         timeParts = (group.group(1).strip() for group in timeGroups)
 722
 723                         messageParts = itertools.izip(fromParts, textParts, timeParts)
 724                         messages = [self._interpret_sms_message_parts(*parts) for parts in messageParts]
 725                         conv.messages = messages
 726
 727                         yield conv
 728
 729         @staticmethod
 730         def _merge_conversation_sources(parsedMessages, json):
 731                 for message in parsedMessages:
 732                         jsonItem = json["messages"][message.id]
 733                         message.isRead = jsonItem["isRead"]
 734                         message.isSpam = jsonItem["isSpam"]
 735                         message.isTrash = jsonItem["isTrash"]
 736                         message.isArchived = "inbox" not in jsonItem["labels"]
 737                         yield message
 738
 739         def _get_page(self, url, data = None, refererUrl = None):
 740                 headers = {}
 741                 if refererUrl is not None:
 742                         headers["Referer"] = refererUrl
 743
 744                 encodedData = urllib.urlencode(data) if data is not None else None
 745
 746                 try:
 747                         page = self._browser.download(url, encodedData, None, headers)
 748                 except urllib2.URLError, e:
 749                         _moduleLogger.error("Translating error: %s" % str(e))
 750                         raise NetworkError("%s is not accesible" % url)
 751
 752                 return page
 753
 754         def _get_page_with_token(self, url, data = None, refererUrl = None):
 755                 if data is None:
 756                         data = {}
 757                 data['_rnr_se'] = self._token
 758
 759                 page = self._get_page(url, data, refererUrl)
 760
 761                 return page
 762
 763         def _parse_with_validation(self, page):
 764                 json = parse_json(page)
 765                 self._validate_response(json)
 766                 return json
 767
 768         def _validate_response(self, response):
 769                 """
 770                 Validates that the JSON response is A-OK
 771                 """
 772                 try:
 773                         assert response is not None, "Response not provided"
 774                         assert 'ok' in response, "Response lacks status"
 775                         assert response['ok'], "Response not good"
 776                 except AssertionError:
 777                         try:
 778                                 if response["data"]["code"] == 20:
 779                                         raise RuntimeError(
 780 """Ambiguous error 20 returned by Google Voice.
 781 Please verify you have configured your callback number (currently "%s").  If it is configured some other suspected causes are: non-verified callback numbers, and Gizmo5 callback numbers.""" % self._callbackNumber)
 782                         except KeyError:
 783                                 pass
 784                         raise RuntimeError('There was a problem with GV: %s' % response)
 785
 786
 787 _UNESCAPE_ENTITIES = {
 788  "&quot;": '"',
 789  "&nbsp;": " ",
 790  "&#39;": "'",
 791 }
 792
 793
 794 def unescape(text):
 795         plain = saxutils.unescape(text, _UNESCAPE_ENTITIES)
 796         return plain
 797
 798
 799 def google_strptime(time):
 800         """
 801         Hack: Google always returns the time in the same locale.  Sadly if the
 802         local system's locale is different, there isn't a way to perfectly handle
 803         the time.  So instead we handle implement some time formatting
 804         """
 805         abbrevTime = time[:-3]
 806         parsedTime = datetime.datetime.strptime(abbrevTime, "%m/%d/%y %I:%M")
 807         if time.endswith("PM"):
 808                 parsedTime += datetime.timedelta(hours=12)
 809         return parsedTime
 810
 811
 812 def itergroup(iterator, count, padValue = None):
 813         """
 814         Iterate in groups of 'count' values. If there
 815         aren't enough values, the last result is padded with
 816         None.
 817
 818         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 819         ...     print tuple(val)
 820         (1, 2, 3)
 821         (4, 5, 6)
 822         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 823         ...     print list(val)
 824         [1, 2, 3]
 825         [4, 5, 6]
 826         >>> for val in itergroup([1, 2, 3, 4, 5, 6, 7], 3):
 827         ...     print tuple(val)
 828         (1, 2, 3)
 829         (4, 5, 6)
 830         (7, None, None)
 831         >>> for val in itergroup("123456", 3):
 832         ...     print tuple(val)
 833         ('1', '2', '3')
 834         ('4', '5', '6')
 835         >>> for val in itergroup("123456", 3):
 836         ...     print repr("".join(val))
 837         '123'
 838         '456'
 839         """
 840         paddedIterator = itertools.chain(iterator, itertools.repeat(padValue, count-1))
 841         nIterators = (paddedIterator, ) * count
 842         return itertools.izip(*nIterators)
 843
 844
 845 def safe_eval(s):
 846         _TRUE_REGEX = re.compile("true")
 847         _FALSE_REGEX = re.compile("false")
 848         _COMMENT_REGEX = re.compile("^\s+//.*$", re.M)
 849         s = _TRUE_REGEX.sub("True", s)
 850         s = _FALSE_REGEX.sub("False", s)
 851         s = _COMMENT_REGEX.sub("#", s)
 852         try:
 853                 results = eval(s, {}, {})
 854         except SyntaxError:
 855                 _moduleLogger.exception("Oops")
 856                 results = None
 857         return results
 858
 859
 860 def _fake_parse_json(flattened):
 861         return safe_eval(flattened)
 862
 863
 864 def _actual_parse_json(flattened):
 865         return simplejson.loads(flattened)
 866
 867
 868 if simplejson is None:
 869         parse_json = _fake_parse_json
 870 else:
 871         parse_json = _actual_parse_json
 872
 873
 874 def extract_payload(flatXml):
 875         xmlTree = ElementTree.fromstring(flatXml)
 876
 877         jsonElement = xmlTree.getchildren()[0]
 878         flatJson = jsonElement.text
 879         jsonTree = parse_json(flatJson)
 880
 881         htmlElement = xmlTree.getchildren()[1]
 882         flatHtml = htmlElement.text
 883
 884         return jsonTree, flatHtml
 885
 886
 887 def guess_phone_type(number):
 888         if number.startswith("747") or number.startswith("1747") or number.startswith("+1747"):
 889                 return GVoiceBackend.PHONE_TYPE_GIZMO
 890         else:
 891                 return GVoiceBackend.PHONE_TYPE_MOBILE
 892
 893
 894 def get_sane_callback(backend):
 895         """
 896         Try to set a sane default callback number on these preferences
 897         1) 1747 numbers ( Gizmo )
 898         2) anything with gizmo in the name
 899         3) anything with computer in the name
 900         4) the first value
 901         """
 902         numbers = backend.get_callback_numbers()
 903
 904         priorityOrderedCriteria = [
 905                 ("\+1747", None),
 906                 ("1747", None),
 907                 ("747", None),
 908                 (None, "gizmo"),
 909                 (None, "computer"),
 910                 (None, "sip"),
 911                 (None, None),
 912         ]
 913
 914         for numberCriteria, descriptionCriteria in priorityOrderedCriteria:
 915                 numberMatcher = None
 916                 descriptionMatcher = None
 917                 if numberCriteria is not None:
 918                         numberMatcher = re.compile(numberCriteria)
 919                 elif descriptionCriteria is not None:
 920                         descriptionMatcher = re.compile(descriptionCriteria, re.I)
 921
 922                 for number, description in numbers.iteritems():
 923                         if numberMatcher is not None and numberMatcher.match(number) is None:
 924                                 continue
 925                         if descriptionMatcher is not None and descriptionMatcher.match(description) is None:
 926                                 continue
 927                         return number
 928
 929
 930 def set_sane_callback(backend):
 931         """
 932         Try to set a sane default callback number on these preferences
 933         1) 1747 numbers ( Gizmo )
 934         2) anything with gizmo in the name
 935         3) anything with computer in the name
 936         4) the first value
 937         """
 938         number = get_sane_callback(backend)
 939         backend.set_callback_number(number)
 940
 941
 942 def _is_not_special(name):
 943         return not name.startswith("_") and name[0].lower() == name[0] and "_" not in name
 944
 945
 946 def to_dict(obj):
 947         members = inspect.getmembers(obj)
 948         return dict((name, value) for (name, value) in members if _is_not_special(name))
 949
 950
 951 def grab_debug_info(username, password):
 952         cookieFile = os.path.join(".", "raw_cookies.txt")
 953         try:
 954                 os.remove(cookieFile)
 955         except OSError:
 956                 pass
 957
 958         backend = GVoiceBackend(cookieFile)
 959         browser = backend._browser
 960
 961         _TEST_WEBPAGES = [
 962                 ("token", backend._tokenURL),
 963                 ("login", backend._loginURL),
 964                 ("isdnd", backend._isDndURL),
 965                 ("account", backend._XML_ACCOUNT_URL),
 966                 ("html_contacts", backend._XML_CONTACTS_URL),
 967                 ("contacts", backend._JSON_CONTACTS_URL),
 968                 ("csv", backend._CSV_CONTACTS_URL),
 969
 970                 ("voicemail", backend._XML_VOICEMAIL_URL),
 971                 ("html_sms", backend._XML_SMS_URL),
 972                 ("sms", backend._JSON_SMS_URL),
 973                 ("count", backend._JSON_SMS_COUNT_URL),
 974
 975                 ("recent", backend._XML_RECENT_URL),
 976                 ("placed", backend._XML_PLACED_URL),
 977                 ("recieved", backend._XML_RECEIVED_URL),
 978                 ("missed", backend._XML_MISSED_URL),
 979         ]
 980
 981         # Get Pages
 982         print "Grabbing pre-login pages"
 983         for name, url in _TEST_WEBPAGES:
 984                 try:
 985                         page = browser.download(url)
 986                 except StandardError, e:
 987                         print e.message
 988                         continue
 989                 print "\tWriting to file"
 990                 with open("not_loggedin_%s.txt" % name, "w") as f:
 991                         f.write(page)
 992
 993         # Login
 994         print "Attempting login"
 995         galxToken = backend._get_token()
 996         loginSuccessOrFailurePage = backend._login(username, password, galxToken)
 997         with open("loggingin.txt", "w") as f:
 998                 print "\tWriting to file"
 999                 f.write(loginSuccessOrFailurePage)
1000         try:
1001                 backend._grab_account_info(loginSuccessOrFailurePage)
1002         except Exception:
1003                 # Retry in case the redirect failed
1004                 # luckily refresh_account_info does everything we need for a retry
1005                 loggedIn = backend.refresh_account_info() is not None
1006                 if not loggedIn:
1007                         raise
1008
1009         # Get Pages
1010         print "Grabbing post-login pages"
1011         for name, url in _TEST_WEBPAGES:
1012                 try:
1013                         page = browser.download(url)
1014                 except StandardError, e:
1015                         print str(e)
1016                         continue
1017                 print "\tWriting to file"
1018                 with open("loggedin_%s.txt" % name, "w") as f:
1019                         f.write(page)
1020
1021         # Cookies
1022         browser.save_cookies()
1023         print "\tWriting cookies to file"
1024         with open("cookies.txt", "w") as f:
1025                 f.writelines(
1026                         "%s: %s\n" % (c.name, c.value)
1027                         for c in browser._cookies
1028                 )
1029
1030
1031 def main():
1032         import sys
1033         logging.basicConfig(level=logging.DEBUG)
1034         args = sys.argv
1035         if 3 <= len(args):
1036                 username = args[1]
1037                 password = args[2]
1038
1039         grab_debug_info(username, password)
1040
1041
1042 if __name__ == "__main__":
1043         main()