git.maemo.org Git - gc-dialer/blob - src/backends/gvoice/gvoice.py

   1 #!/usr/bin/python
   2
   3 """
   4 DialCentral - Front end for Google's GoogleVoice service.
   5 Copyright (C) 2008  Eric Warnke ericew AT gmail DOT com
   6
   7 This library is free software; you can redistribute it and/or
   8 modify it under the terms of the GNU Lesser General Public
   9 License as published by the Free Software Foundation; either
  10 version 2.1 of the License, or (at your option) any later version.
  11
  12 This library is distributed in the hope that it will be useful,
  13 but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 Lesser General Public License for more details.
  16
  17 You should have received a copy of the GNU Lesser General Public
  18 License along with this library; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 Google Voice backend code
  22
  23 Resources
  24         http://thatsmith.com/2009/03/google-voice-addon-for-firefox/
  25         http://posttopic.com/topic/google-voice-add-on-development
  26 """
  27
  28 from __future__ import with_statement
  29
  30 import os
  31 import re
  32 import urllib
  33 import urllib2
  34 import time
  35 import datetime
  36 import itertools
  37 import logging
  38 import inspect
  39
  40 from xml.sax import saxutils
  41 from xml.etree import ElementTree
  42
  43 try:
  44         import simplejson as _simplejson
  45         simplejson = _simplejson
  46 except ImportError:
  47         simplejson = None
  48
  49 import browser_emu
  50
  51
  52 _moduleLogger = logging.getLogger(__name__)
  53
  54
  55 class NetworkError(RuntimeError):
  56         pass
  57
  58
  59 class MessageText(object):
  60
  61         ACCURACY_LOW = "med1"
  62         ACCURACY_MEDIUM = "med2"
  63         ACCURACY_HIGH = "high"
  64
  65         def __init__(self):
  66                 self.accuracy = None
  67                 self.text = None
  68
  69         def __str__(self):
  70                 return self.text
  71
  72         def to_dict(self):
  73                 return to_dict(self)
  74
  75         def __eq__(self, other):
  76                 return self.accuracy == other.accuracy and self.text == other.text
  77
  78
  79 class Message(object):
  80
  81         def __init__(self):
  82                 self.whoFrom = None
  83                 self.body = None
  84                 self.when = None
  85
  86         def __str__(self):
  87                 return "%s (%s): %s" % (
  88                         self.whoFrom,
  89                         self.when,
  90                         "".join(unicode(part) for part in self.body)
  91                 )
  92
  93         def to_dict(self):
  94                 selfDict = to_dict(self)
  95                 selfDict["body"] = [text.to_dict() for text in self.body] if self.body is not None else None
  96                 return selfDict
  97
  98         def __eq__(self, other):
  99                 return self.whoFrom == other.whoFrom and self.when == other.when and self.body == other.body
 100
 101
 102 class Conversation(object):
 103
 104         TYPE_VOICEMAIL = "Voicemail"
 105         TYPE_SMS = "SMS"
 106
 107         def __init__(self):
 108                 self.type = None
 109                 self.id = None
 110                 self.contactId = None
 111                 self.name = None
 112                 self.location = None
 113                 self.prettyNumber = None
 114                 self.number = None
 115
 116                 self.time = None
 117                 self.relTime = None
 118                 self.messages = None
 119                 self.isRead = None
 120                 self.isSpam = None
 121                 self.isTrash = None
 122                 self.isArchived = None
 123
 124         def __cmp__(self, other):
 125                 cmpValue = cmp(self.contactId, other.contactId)
 126                 if cmpValue != 0:
 127                         return cmpValue
 128
 129                 cmpValue = cmp(self.time, other.time)
 130                 if cmpValue != 0:
 131                         return cmpValue
 132
 133                 cmpValue = cmp(self.id, other.id)
 134                 if cmpValue != 0:
 135                         return cmpValue
 136
 137         def to_dict(self):
 138                 selfDict = to_dict(self)
 139                 selfDict["messages"] = [message.to_dict() for message in self.messages] if self.messages is not None else None
 140                 return selfDict
 141
 142
 143 class GVoiceBackend(object):
 144         """
 145         This class encapsulates all of the knowledge necessary to interact with the GoogleVoice servers
 146         the functions include login, setting up a callback number, and initalting a callback
 147         """
 148
 149         PHONE_TYPE_HOME = 1
 150         PHONE_TYPE_MOBILE = 2
 151         PHONE_TYPE_WORK = 3
 152         PHONE_TYPE_GIZMO = 7
 153
 154         def __init__(self, cookieFile = None):
 155                 # Important items in this function are the setup of the browser emulation and cookie file
 156                 self._browser = browser_emu.MozillaEmulator(1)
 157                 self._loadedFromCookies = self._browser.load_cookies(cookieFile)
 158
 159                 self._token = ""
 160                 self._accountNum = ""
 161                 self._lastAuthed = 0.0
 162                 self._callbackNumber = ""
 163                 self._callbackNumbers = {}
 164
 165                 # Suprisingly, moving all of these from class to self sped up startup time
 166
 167                 self._validateRe = re.compile("^\+?[0-9]{10,}$")
 168
 169                 self._loginURL = "https://www.google.com/accounts/ServiceLoginAuth"
 170
 171                 SECURE_URL_BASE = "https://www.google.com/voice/"
 172                 SECURE_MOBILE_URL_BASE = SECURE_URL_BASE + "mobile/"
 173                 self._forwardURL = SECURE_MOBILE_URL_BASE + "phones"
 174                 self._tokenURL = SECURE_URL_BASE + "m"
 175                 self._callUrl = SECURE_URL_BASE + "call/connect"
 176                 self._callCancelURL = SECURE_URL_BASE + "call/cancel"
 177                 self._sendSmsURL = SECURE_URL_BASE + "sms/send"
 178
 179                 self._isDndURL = "https://www.google.com/voice/m/donotdisturb"
 180                 self._isDndRe = re.compile(r"""<input.*?id="doNotDisturb".*?checked="(.*?)"\s*/>""")
 181                 self._setDndURL = "https://www.google.com/voice/m/savednd"
 182
 183                 self._downloadVoicemailURL = SECURE_URL_BASE + "media/send_voicemail/"
 184                 self._markAsReadURL = SECURE_URL_BASE + "m/mark"
 185                 self._archiveMessageURL = SECURE_URL_BASE + "m/archive"
 186
 187                 self._XML_SEARCH_URL = SECURE_URL_BASE + "inbox/search/"
 188                 self._XML_ACCOUNT_URL = SECURE_URL_BASE + "contacts/"
 189                 # HACK really this redirects to the main pge and we are grabbing some javascript
 190                 self._XML_CONTACTS_URL = "http://www.google.com/voice/inbox/search/contact"
 191                 self._CSV_CONTACTS_URL = "http://mail.google.com/mail/contacts/data/export"
 192                 self._JSON_CONTACTS_URL = SECURE_URL_BASE + "b/0/request/user"
 193                 self._XML_RECENT_URL = SECURE_URL_BASE + "inbox/recent/"
 194
 195                 self.XML_FEEDS = (
 196                         'inbox', 'starred', 'all', 'spam', 'trash', 'voicemail', 'sms',
 197                         'recorded', 'placed', 'received', 'missed'
 198                 )
 199                 self._XML_INBOX_URL = SECURE_URL_BASE + "inbox/recent/inbox"
 200                 self._XML_STARRED_URL = SECURE_URL_BASE + "inbox/recent/starred"
 201                 self._XML_ALL_URL = SECURE_URL_BASE + "inbox/recent/all"
 202                 self._XML_SPAM_URL = SECURE_URL_BASE + "inbox/recent/spam"
 203                 self._XML_TRASH_URL = SECURE_URL_BASE + "inbox/recent/trash"
 204                 self._XML_VOICEMAIL_URL = SECURE_URL_BASE + "inbox/recent/voicemail/"
 205                 self._XML_SMS_URL = SECURE_URL_BASE + "inbox/recent/sms/"
 206                 self._XML_RECORDED_URL = SECURE_URL_BASE + "inbox/recent/recorded/"
 207                 self._XML_PLACED_URL = SECURE_URL_BASE + "inbox/recent/placed/"
 208                 self._XML_RECEIVED_URL = SECURE_URL_BASE + "inbox/recent/received/"
 209                 self._XML_MISSED_URL = SECURE_URL_BASE + "inbox/recent/missed/"
 210
 211                 self._galxRe = re.compile(r"""<input.*?name="GALX".*?value="(.*?)".*?/>""", re.MULTILINE | re.DOTALL)
 212                 self._tokenRe = re.compile(r"""<input.*?name="_rnr_se".*?value="(.*?)"\s*/>""")
 213                 self._accountNumRe = re.compile(r"""<b class="ms\d">(.{14})</b></div>""")
 214                 self._callbackRe = re.compile(r"""\s+(.*?):\s*(.*?)<br\s*/>\s*$""", re.M)
 215
 216                 self._seperateVoicemailsRegex = re.compile(r"""^\s*<div id="(\w+)"\s* class=".*?gc-message.*?">""", re.MULTILINE | re.DOTALL)
 217                 self._exactVoicemailTimeRegex = re.compile(r"""<span class="gc-message-time">(.*?)</span>""", re.MULTILINE)
 218                 self._relativeVoicemailTimeRegex = re.compile(r"""<span class="gc-message-relative">(.*?)</span>""", re.MULTILINE)
 219                 self._voicemailNameRegex = re.compile(r"""<a class=.*?gc-message-name-link.*?>(.*?)</a>""", re.MULTILINE | re.DOTALL)
 220                 self._voicemailNumberRegex = re.compile(r"""<input type="hidden" class="gc-text gc-quickcall-ac" value="(.*?)"/>""", re.MULTILINE)
 221                 self._prettyVoicemailNumberRegex = re.compile(r"""<span class="gc-message-type">(.*?)</span>""", re.MULTILINE)
 222                 self._voicemailLocationRegex = re.compile(r"""<span class="gc-message-location">.*?<a.*?>(.*?)</a></span>""", re.MULTILINE)
 223                 self._messagesContactIDRegex = re.compile(r"""<a class=".*?gc-message-name-link.*?">.*?</a>\s*?<span .*?>(.*?)</span>""", re.MULTILINE)
 224                 self._voicemailMessageRegex = re.compile(r"""(<span id="\d+-\d+" class="gc-word-(.*?)">(.*?)</span>|<a .*? class="gc-message-mni">(.*?)</a>)""", re.MULTILINE)
 225                 self._smsFromRegex = re.compile(r"""<span class="gc-message-sms-from">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 226                 self._smsTimeRegex = re.compile(r"""<span class="gc-message-sms-time">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 227                 self._smsTextRegex = re.compile(r"""<span class="gc-message-sms-text">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 228
 229         def is_quick_login_possible(self):
 230                 """
 231                 @returns True then is_authed might be enough to login, else full login is required
 232                 """
 233                 return self._loadedFromCookies or 0.0 < self._lastAuthed
 234
 235         def is_authed(self, force = False):
 236                 """
 237                 Attempts to detect a current session
 238                 @note Once logged in try not to reauth more than once a minute.
 239                 @returns If authenticated
 240                 @blocks
 241                 """
 242                 isRecentledAuthed = (time.time() - self._lastAuthed) < 120
 243                 isPreviouslyAuthed = self._token is not None
 244                 if isRecentledAuthed and isPreviouslyAuthed and not force:
 245                         return True
 246
 247                 try:
 248                         page = self._get_page(self._forwardURL)
 249                         self._grab_account_info(page)
 250                 except Exception, e:
 251                         _moduleLogger.exception(str(e))
 252                         return False
 253
 254                 self._browser.save_cookies()
 255                 self._lastAuthed = time.time()
 256                 return True
 257
 258         def _get_token(self):
 259                 tokenPage = self._get_page(self._tokenURL)
 260
 261                 galxTokens = self._galxRe.search(tokenPage)
 262                 if galxTokens is not None:
 263                         galxToken = galxTokens.group(1)
 264                 else:
 265                         galxToken = ""
 266                         _moduleLogger.debug("Could not grab GALX token")
 267                 return galxToken
 268
 269         def _login(self, username, password, token):
 270                 loginData = {
 271                         'Email' : username,
 272                         'Passwd' : password,
 273                         'service': "grandcentral",
 274                         "ltmpl": "mobile",
 275                         "btmpl": "mobile",
 276                         "PersistentCookie": "yes",
 277                         "GALX": token,
 278                         "continue": self._forwardURL,
 279                 }
 280
 281                 loginSuccessOrFailurePage = self._get_page(self._loginURL, loginData)
 282                 return loginSuccessOrFailurePage
 283
 284         def login(self, username, password):
 285                 """
 286                 Attempt to login to GoogleVoice
 287                 @returns Whether login was successful or not
 288                 @blocks
 289                 """
 290                 self.logout()
 291                 galxToken = self._get_token()
 292                 loginSuccessOrFailurePage = self._login(username, password, galxToken)
 293
 294                 try:
 295                         self._grab_account_info(loginSuccessOrFailurePage)
 296                 except Exception, e:
 297                         # Retry in case the redirect failed
 298                         # luckily is_authed does everything we need for a retry
 299                         loggedIn = self.is_authed(True)
 300                         if not loggedIn:
 301                                 _moduleLogger.exception(str(e))
 302                                 return False
 303                         _moduleLogger.info("Redirection failed on initial login attempt, auto-corrected for this")
 304
 305                 self._browser.save_cookies()
 306                 self._lastAuthed = time.time()
 307                 return True
 308
 309         def persist(self):
 310                 self._browser.save_cookies()
 311
 312         def shutdown(self):
 313                 self._browser.save_cookies()
 314                 self._token = None
 315                 self._lastAuthed = 0.0
 316
 317         def logout(self):
 318                 self._browser.clear_cookies()
 319                 self._browser.save_cookies()
 320                 self._token = None
 321                 self._lastAuthed = 0.0
 322
 323         def is_dnd(self):
 324                 """
 325                 @blocks
 326                 """
 327                 isDndPage = self._get_page(self._isDndURL)
 328
 329                 dndGroup = self._isDndRe.search(isDndPage)
 330                 if dndGroup is None:
 331                         return False
 332                 dndStatus = dndGroup.group(1)
 333                 isDnd = True if dndStatus.strip().lower() == "true" else False
 334                 return isDnd
 335
 336         def set_dnd(self, doNotDisturb):
 337                 """
 338                 @blocks
 339                 """
 340                 dndPostData = {
 341                         "doNotDisturb": 1 if doNotDisturb else 0,
 342                 }
 343
 344                 dndPage = self._get_page_with_token(self._setDndURL, dndPostData)
 345
 346         def call(self, outgoingNumber):
 347                 """
 348                 This is the main function responsible for initating the callback
 349                 @blocks
 350                 """
 351                 outgoingNumber = self._send_validation(outgoingNumber)
 352                 subscriberNumber = None
 353                 phoneType = guess_phone_type(self._callbackNumber) # @todo Fix this hack
 354
 355                 callData = {
 356                                 'outgoingNumber': outgoingNumber,
 357                                 'forwardingNumber': self._callbackNumber,
 358                                 'subscriberNumber': subscriberNumber or 'undefined',
 359                                 'phoneType': str(phoneType),
 360                                 'remember': '1',
 361                 }
 362                 _moduleLogger.info("%r" % callData)
 363
 364                 page = self._get_page_with_token(
 365                         self._callUrl,
 366                         callData,
 367                 )
 368                 self._parse_with_validation(page)
 369                 return True
 370
 371         def cancel(self, outgoingNumber=None):
 372                 """
 373                 Cancels a call matching outgoing and forwarding numbers (if given).
 374                 Will raise an error if no matching call is being placed
 375                 @blocks
 376                 """
 377                 page = self._get_page_with_token(
 378                         self._callCancelURL,
 379                         {
 380                         'outgoingNumber': outgoingNumber or 'undefined',
 381                         'forwardingNumber': self._callbackNumber or 'undefined',
 382                         'cancelType': 'C2C',
 383                         },
 384                 )
 385                 self._parse_with_validation(page)
 386
 387         def send_sms(self, phoneNumbers, message):
 388                 """
 389                 @blocks
 390                 """
 391                 validatedPhoneNumbers = [
 392                         self._send_validation(phoneNumber)
 393                         for phoneNumber in phoneNumbers
 394                 ]
 395                 flattenedPhoneNumbers = ",".join(validatedPhoneNumbers)
 396                 page = self._get_page_with_token(
 397                         self._sendSmsURL,
 398                         {
 399                                 'phoneNumber': flattenedPhoneNumbers,
 400                                 'text': unicode(message).encode("utf-8"),
 401                         },
 402                 )
 403                 self._parse_with_validation(page)
 404
 405         def search(self, query):
 406                 """
 407                 Search your Google Voice Account history for calls, voicemails, and sms
 408                 Returns ``Folder`` instance containting matching messages
 409                 @blocks
 410                 """
 411                 page = self._get_page(
 412                         self._XML_SEARCH_URL,
 413                         {"q": query},
 414                 )
 415                 json, html = extract_payload(page)
 416                 return json
 417
 418         def get_feed(self, feed):
 419                 """
 420                 @blocks
 421                 """
 422                 actualFeed = "_XML_%s_URL" % feed.upper()
 423                 feedUrl = getattr(self, actualFeed)
 424
 425                 page = self._get_page(feedUrl)
 426                 json, html = extract_payload(page)
 427
 428                 return json
 429
 430         def download(self, messageId, adir):
 431                 """
 432                 Download a voicemail or recorded call MP3 matching the given ``msg``
 433                 which can either be a ``Message`` instance, or a SHA1 identifier.
 434                 Saves files to ``adir`` (defaults to current directory).
 435                 Message hashes can be found in ``self.voicemail().messages`` for example.
 436                 @returns location of saved file.
 437                 @blocks
 438                 """
 439                 page = self._get_page(self._downloadVoicemailURL, {"id": messageId})
 440                 fn = os.path.join(adir, '%s.mp3' % messageId)
 441                 with open(fn, 'wb') as fo:
 442                         fo.write(page)
 443                 return fn
 444
 445         def is_valid_syntax(self, number):
 446                 """
 447                 @returns If This number be called ( syntax validation only )
 448                 """
 449                 return self._validateRe.match(number) is not None
 450
 451         def get_account_number(self):
 452                 """
 453                 @returns The GoogleVoice phone number
 454                 """
 455                 return self._accountNum
 456
 457         def get_callback_numbers(self):
 458                 """
 459                 @returns a dictionary mapping call back numbers to descriptions
 460                 @note These results are cached for 30 minutes.
 461                 """
 462                 if not self.is_authed():
 463                         return {}
 464                 return self._callbackNumbers
 465
 466         def set_callback_number(self, callbacknumber):
 467                 """
 468                 Set the number that GoogleVoice calls
 469                 @param callbacknumber should be a proper 10 digit number
 470                 """
 471                 self._callbackNumber = callbacknumber
 472                 _moduleLogger.info("Callback number changed: %r" % self._callbackNumber)
 473                 return True
 474
 475         def get_callback_number(self):
 476                 """
 477                 @returns Current callback number or None
 478                 """
 479                 return self._callbackNumber
 480
 481         def get_recent(self):
 482                 """
 483                 @returns Iterable of (personsName, phoneNumber, exact date, relative date, action)
 484                 @blocks
 485                 """
 486                 recentPages = [
 487                         (action, self._get_page(url))
 488                         for action, url in (
 489                                 ("Received", self._XML_RECEIVED_URL),
 490                                 ("Missed", self._XML_MISSED_URL),
 491                                 ("Placed", self._XML_PLACED_URL),
 492                         )
 493                 ]
 494                 return self._parse_recent(recentPages)
 495
 496         def get_contacts(self):
 497                 """
 498                 @returns Iterable of (contact id, contact name)
 499                 @blocks
 500                 """
 501                 page = self._get_page(self._JSON_CONTACTS_URL)
 502                 return self._process_contacts(page)
 503
 504         def get_csv_contacts(self):
 505                 data = {
 506                         "groupToExport": "mine",
 507                         "exportType": "ALL",
 508                         "out": "OUTLOOK_CSV",
 509                 }
 510                 encodedData = urllib.urlencode(data)
 511                 contacts = self._get_page(self._CSV_CONTACTS_URL+"?"+encodedData)
 512                 return contacts
 513
 514         def get_voicemails(self):
 515                 """
 516                 @blocks
 517                 """
 518                 voicemailPage = self._get_page(self._XML_VOICEMAIL_URL)
 519                 voicemailHtml = self._grab_html(voicemailPage)
 520                 voicemailJson = self._grab_json(voicemailPage)
 521                 if voicemailJson is None:
 522                         return ()
 523                 parsedVoicemail = self._parse_voicemail(voicemailHtml)
 524                 voicemails = self._merge_conversation_sources(parsedVoicemail, voicemailJson)
 525                 return voicemails
 526
 527         def get_texts(self):
 528                 """
 529                 @blocks
 530                 """
 531                 smsPage = self._get_page(self._XML_SMS_URL)
 532                 smsHtml = self._grab_html(smsPage)
 533                 smsJson = self._grab_json(smsPage)
 534                 if smsJson is None:
 535                         return ()
 536                 parsedSms = self._parse_sms(smsHtml)
 537                 smss = self._merge_conversation_sources(parsedSms, smsJson)
 538                 return smss
 539
 540         def mark_message(self, messageId, asRead):
 541                 """
 542                 @blocks
 543                 """
 544                 postData = {
 545                         "read": 1 if asRead else 0,
 546                         "id": messageId,
 547                 }
 548
 549                 markPage = self._get_page(self._markAsReadURL, postData)
 550
 551         def archive_message(self, messageId):
 552                 """
 553                 @blocks
 554                 """
 555                 postData = {
 556                         "id": messageId,
 557                 }
 558
 559                 markPage = self._get_page(self._archiveMessageURL, postData)
 560
 561         def _grab_json(self, flatXml):
 562                 xmlTree = ElementTree.fromstring(flatXml)
 563                 jsonElement = xmlTree.getchildren()[0]
 564                 flatJson = jsonElement.text
 565                 jsonTree = parse_json(flatJson)
 566                 return jsonTree
 567
 568         def _grab_html(self, flatXml):
 569                 xmlTree = ElementTree.fromstring(flatXml)
 570                 htmlElement = xmlTree.getchildren()[1]
 571                 flatHtml = htmlElement.text
 572                 return flatHtml
 573
 574         def _grab_account_info(self, page):
 575                 tokenGroup = self._tokenRe.search(page)
 576                 if tokenGroup is None:
 577                         raise RuntimeError("Could not extract authentication token from GoogleVoice")
 578                 self._token = tokenGroup.group(1)
 579
 580                 anGroup = self._accountNumRe.search(page)
 581                 if anGroup is not None:
 582                         self._accountNum = anGroup.group(1)
 583                 else:
 584                         _moduleLogger.debug("Could not extract account number from GoogleVoice")
 585
 586                 self._callbackNumbers = {}
 587                 for match in self._callbackRe.finditer(page):
 588                         callbackNumber = match.group(2)
 589                         callbackName = match.group(1)
 590                         self._callbackNumbers[callbackNumber] = callbackName
 591                 if len(self._callbackNumbers) == 0:
 592                         _moduleLogger.debug("Could not extract callback numbers from GoogleVoice (the troublesome page follows):\n%s" % page)
 593
 594         def _send_validation(self, number):
 595                 if not self.is_valid_syntax(number):
 596                         raise ValueError('Number is not valid: "%s"' % number)
 597                 elif not self.is_authed():
 598                         raise RuntimeError("Not Authenticated")
 599                 return number
 600
 601         def _parse_recent(self, recentPages):
 602                 for action, flatXml in recentPages:
 603                         allRecentHtml = self._grab_html(flatXml)
 604                         allRecentData = self._parse_history(allRecentHtml)
 605                         for recentCallData in allRecentData:
 606                                 recentCallData["action"] = action
 607                                 yield recentCallData
 608
 609         def _process_contacts(self, page):
 610                 accountData = parse_json(page)
 611                 for contactId, contactDetails in accountData["contacts"].iteritems():
 612                         # A zero contact id is the catch all for unknown contacts
 613                         if contactId != "0":
 614                                 yield contactId, contactDetails
 615
 616         def _parse_history(self, historyHtml):
 617                 splitVoicemail = self._seperateVoicemailsRegex.split(historyHtml)
 618                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 619                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 620                         exactTime = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 621                         exactTime = google_strptime(exactTime)
 622                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 623                         relativeTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 624                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 625                         location = locationGroup.group(1).strip() if locationGroup else ""
 626
 627                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 628                         name = nameGroup.group(1).strip() if nameGroup else ""
 629                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 630                         number = numberGroup.group(1).strip() if numberGroup else ""
 631                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 632                         prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 633                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 634                         contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 635
 636                         yield {
 637                                 "id": messageId.strip(),
 638                                 "contactId": contactId,
 639                                 "name": unescape(name),
 640                                 "time": exactTime,
 641                                 "relTime": relativeTime,
 642                                 "prettyNumber": prettyNumber,
 643                                 "number": number,
 644                                 "location": unescape(location),
 645                         }
 646
 647         @staticmethod
 648         def _interpret_voicemail_regex(group):
 649                 quality, content, number = group.group(2), group.group(3), group.group(4)
 650                 text = MessageText()
 651                 if quality is not None and content is not None:
 652                         text.accuracy = quality
 653                         text.text = unescape(content)
 654                         return text
 655                 elif number is not None:
 656                         text.accuracy = MessageText.ACCURACY_HIGH
 657                         text.text = number
 658                         return text
 659
 660         def _parse_voicemail(self, voicemailHtml):
 661                 splitVoicemail = self._seperateVoicemailsRegex.split(voicemailHtml)
 662                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 663                         conv = Conversation()
 664                         conv.type = Conversation.TYPE_VOICEMAIL
 665                         conv.id = messageId.strip()
 666
 667                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 668                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 669                         conv.time = google_strptime(exactTimeText)
 670                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 671                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 672                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 673                         conv.location = unescape(locationGroup.group(1).strip() if locationGroup else "")
 674
 675                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 676                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 677                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 678                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 679                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 680                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 681                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 682                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 683
 684                         messageGroups = self._voicemailMessageRegex.finditer(messageHtml)
 685                         messageParts = [
 686                                 self._interpret_voicemail_regex(group)
 687                                 for group in messageGroups
 688                         ] if messageGroups else ((MessageText.ACCURACY_LOW, "No Transcription"), )
 689                         message = Message()
 690                         message.body = messageParts
 691                         message.whoFrom = conv.name
 692                         message.when = conv.time.strftime("%I:%M %p")
 693                         conv.messages = (message, )
 694
 695                         yield conv
 696
 697         @staticmethod
 698         def _interpret_sms_message_parts(fromPart, textPart, timePart):
 699                 text = MessageText()
 700                 text.accuracy = MessageText.ACCURACY_MEDIUM
 701                 text.text = unescape(textPart)
 702
 703                 message = Message()
 704                 message.body = (text, )
 705                 message.whoFrom = fromPart
 706                 message.when = timePart
 707
 708                 return message
 709
 710         def _parse_sms(self, smsHtml):
 711                 splitSms = self._seperateVoicemailsRegex.split(smsHtml)
 712                 for messageId, messageHtml in itergroup(splitSms[1:], 2):
 713                         conv = Conversation()
 714                         conv.type = Conversation.TYPE_SMS
 715                         conv.id = messageId.strip()
 716
 717                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 718                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 719                         conv.time = google_strptime(exactTimeText)
 720                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 721                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 722                         conv.location = ""
 723
 724                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 725                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 726                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 727                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 728                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 729                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 730                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 731                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 732
 733                         fromGroups = self._smsFromRegex.finditer(messageHtml)
 734                         fromParts = (group.group(1).strip() for group in fromGroups)
 735                         textGroups = self._smsTextRegex.finditer(messageHtml)
 736                         textParts = (group.group(1).strip() for group in textGroups)
 737                         timeGroups = self._smsTimeRegex.finditer(messageHtml)
 738                         timeParts = (group.group(1).strip() for group in timeGroups)
 739
 740                         messageParts = itertools.izip(fromParts, textParts, timeParts)
 741                         messages = [self._interpret_sms_message_parts(*parts) for parts in messageParts]
 742                         conv.messages = messages
 743
 744                         yield conv
 745
 746         @staticmethod
 747         def _merge_conversation_sources(parsedMessages, json):
 748                 for message in parsedMessages:
 749                         jsonItem = json["messages"][message.id]
 750                         message.isRead = jsonItem["isRead"]
 751                         message.isSpam = jsonItem["isSpam"]
 752                         message.isTrash = jsonItem["isTrash"]
 753                         message.isArchived = "inbox" not in jsonItem["labels"]
 754                         yield message
 755
 756         def _get_page(self, url, data = None, refererUrl = None):
 757                 headers = {}
 758                 if refererUrl is not None:
 759                         headers["Referer"] = refererUrl
 760
 761                 encodedData = urllib.urlencode(data) if data is not None else None
 762
 763                 try:
 764                         page = self._browser.download(url, encodedData, None, headers)
 765                 except urllib2.URLError, e:
 766                         _moduleLogger.error("Translating error: %s" % str(e))
 767                         raise NetworkError("%s is not accesible" % url)
 768
 769                 return page
 770
 771         def _get_page_with_token(self, url, data = None, refererUrl = None):
 772                 if data is None:
 773                         data = {}
 774                 data['_rnr_se'] = self._token
 775
 776                 page = self._get_page(url, data, refererUrl)
 777
 778                 return page
 779
 780         def _parse_with_validation(self, page):
 781                 json = parse_json(page)
 782                 validate_response(json)
 783                 return json
 784
 785
 786 _UNESCAPE_ENTITIES = {
 787  "&quot;": '"',
 788  "&nbsp;": " ",
 789  "&#39;": "'",
 790 }
 791
 792
 793 def unescape(text):
 794         plain = saxutils.unescape(text, _UNESCAPE_ENTITIES)
 795         return plain
 796
 797
 798 def google_strptime(time):
 799         """
 800         Hack: Google always returns the time in the same locale.  Sadly if the
 801         local system's locale is different, there isn't a way to perfectly handle
 802         the time.  So instead we handle implement some time formatting
 803         """
 804         abbrevTime = time[:-3]
 805         parsedTime = datetime.datetime.strptime(abbrevTime, "%m/%d/%y %I:%M")
 806         if time.endswith("PM"):
 807                 parsedTime += datetime.timedelta(hours=12)
 808         return parsedTime
 809
 810
 811 def itergroup(iterator, count, padValue = None):
 812         """
 813         Iterate in groups of 'count' values. If there
 814         aren't enough values, the last result is padded with
 815         None.
 816
 817         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 818         ...     print tuple(val)
 819         (1, 2, 3)
 820         (4, 5, 6)
 821         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 822         ...     print list(val)
 823         [1, 2, 3]
 824         [4, 5, 6]
 825         >>> for val in itergroup([1, 2, 3, 4, 5, 6, 7], 3):
 826         ...     print tuple(val)
 827         (1, 2, 3)
 828         (4, 5, 6)
 829         (7, None, None)
 830         >>> for val in itergroup("123456", 3):
 831         ...     print tuple(val)
 832         ('1', '2', '3')
 833         ('4', '5', '6')
 834         >>> for val in itergroup("123456", 3):
 835         ...     print repr("".join(val))
 836         '123'
 837         '456'
 838         """
 839         paddedIterator = itertools.chain(iterator, itertools.repeat(padValue, count-1))
 840         nIterators = (paddedIterator, ) * count
 841         return itertools.izip(*nIterators)
 842
 843
 844 def safe_eval(s):
 845         _TRUE_REGEX = re.compile("true")
 846         _FALSE_REGEX = re.compile("false")
 847         _COMMENT_REGEX = re.compile("^\s+//.*$", re.M)
 848         s = _TRUE_REGEX.sub("True", s)
 849         s = _FALSE_REGEX.sub("False", s)
 850         s = _COMMENT_REGEX.sub("#", s)
 851         try:
 852                 results = eval(s, {}, {})
 853         except SyntaxError:
 854                 _moduleLogger.exception("Oops")
 855                 results = None
 856         return results
 857
 858
 859 def _fake_parse_json(flattened):
 860         return safe_eval(flattened)
 861
 862
 863 def _actual_parse_json(flattened):
 864         return simplejson.loads(flattened)
 865
 866
 867 if simplejson is None:
 868         parse_json = _fake_parse_json
 869 else:
 870         parse_json = _actual_parse_json
 871
 872
 873 def extract_payload(flatXml):
 874         xmlTree = ElementTree.fromstring(flatXml)
 875
 876         jsonElement = xmlTree.getchildren()[0]
 877         flatJson = jsonElement.text
 878         jsonTree = parse_json(flatJson)
 879
 880         htmlElement = xmlTree.getchildren()[1]
 881         flatHtml = htmlElement.text
 882
 883         return jsonTree, flatHtml
 884
 885
 886 def validate_response(response):
 887         """
 888         Validates that the JSON response is A-OK
 889         """
 890         try:
 891                 assert response is not None
 892                 assert 'ok' in response
 893                 assert response['ok']
 894         except AssertionError:
 895                 raise RuntimeError('There was a problem with GV: %s' % response)
 896
 897
 898 def guess_phone_type(number):
 899         if number.startswith("747") or number.startswith("1747") or number.startswith("+1747"):
 900                 return GVoiceBackend.PHONE_TYPE_GIZMO
 901         else:
 902                 return GVoiceBackend.PHONE_TYPE_MOBILE
 903
 904
 905 def get_sane_callback(backend):
 906         """
 907         Try to set a sane default callback number on these preferences
 908         1) 1747 numbers ( Gizmo )
 909         2) anything with gizmo in the name
 910         3) anything with computer in the name
 911         4) the first value
 912         """
 913         numbers = backend.get_callback_numbers()
 914
 915         priorityOrderedCriteria = [
 916                 ("\+1747", None),
 917                 ("1747", None),
 918                 ("747", None),
 919                 (None, "gizmo"),
 920                 (None, "computer"),
 921                 (None, "sip"),
 922                 (None, None),
 923         ]
 924
 925         for numberCriteria, descriptionCriteria in priorityOrderedCriteria:
 926                 numberMatcher = None
 927                 descriptionMatcher = None
 928                 if numberCriteria is not None:
 929                         numberMatcher = re.compile(numberCriteria)
 930                 elif descriptionCriteria is not None:
 931                         descriptionMatcher = re.compile(descriptionCriteria, re.I)
 932
 933                 for number, description in numbers.iteritems():
 934                         if numberMatcher is not None and numberMatcher.match(number) is None:
 935                                 continue
 936                         if descriptionMatcher is not None and descriptionMatcher.match(description) is None:
 937                                 continue
 938                         return number
 939
 940
 941 def set_sane_callback(backend):
 942         """
 943         Try to set a sane default callback number on these preferences
 944         1) 1747 numbers ( Gizmo )
 945         2) anything with gizmo in the name
 946         3) anything with computer in the name
 947         4) the first value
 948         """
 949         number = get_sane_callback(backend)
 950         backend.set_callback_number(number)
 951
 952
 953 def _is_not_special(name):
 954         return not name.startswith("_") and name[0].lower() == name[0] and "_" not in name
 955
 956
 957 def to_dict(obj):
 958         members = inspect.getmembers(obj)
 959         return dict((name, value) for (name, value) in members if _is_not_special(name))
 960
 961
 962 def grab_debug_info(username, password):
 963         cookieFile = os.path.join(".", "raw_cookies.txt")
 964         try:
 965                 os.remove(cookieFile)
 966         except OSError:
 967                 pass
 968
 969         backend = GVoiceBackend(cookieFile)
 970         browser = backend._browser
 971
 972         _TEST_WEBPAGES = [
 973                 ("forward", backend._forwardURL),
 974                 ("token", backend._tokenURL),
 975                 ("login", backend._loginURL),
 976                 ("isdnd", backend._isDndURL),
 977                 ("account", backend._XML_ACCOUNT_URL),
 978                 ("contacts", backend._XML_CONTACTS_URL),
 979                 ("csv", backend._CSV_CONTACTS_URL),
 980
 981                 ("voicemail", backend._XML_VOICEMAIL_URL),
 982                 ("sms", backend._XML_SMS_URL),
 983
 984                 ("recent", backend._XML_RECENT_URL),
 985                 ("placed", backend._XML_PLACED_URL),
 986                 ("recieved", backend._XML_RECEIVED_URL),
 987                 ("missed", backend._XML_MISSED_URL),
 988         ]
 989
 990         # Get Pages
 991         print "Grabbing pre-login pages"
 992         for name, url in _TEST_WEBPAGES:
 993                 try:
 994                         page = browser.download(url)
 995                 except StandardError, e:
 996                         print e.message
 997                         continue
 998                 print "\tWriting to file"
 999                 with open("not_loggedin_%s.txt" % name, "w") as f:
1000                         f.write(page)
1001
1002         # Login
1003         print "Attempting login"
1004         galxToken = backend._get_token()
1005         loginSuccessOrFailurePage = backend._login(username, password, galxToken)
1006         with open("loggingin.txt", "w") as f:
1007                 print "\tWriting to file"
1008                 f.write(loginSuccessOrFailurePage)
1009         try:
1010                 backend._grab_account_info(loginSuccessOrFailurePage)
1011         except Exception:
1012                 # Retry in case the redirect failed
1013                 # luckily is_authed does everything we need for a retry
1014                 loggedIn = backend.is_authed(True)
1015                 if not loggedIn:
1016                         raise
1017
1018         # Get Pages
1019         print "Grabbing post-login pages"
1020         for name, url in _TEST_WEBPAGES:
1021                 try:
1022                         page = browser.download(url)
1023                 except StandardError, e:
1024                         print str(e)
1025                         continue
1026                 print "\tWriting to file"
1027                 with open("loggedin_%s.txt" % name, "w") as f:
1028                         f.write(page)
1029
1030         # Cookies
1031         browser.save_cookies()
1032         print "\tWriting cookies to file"
1033         with open("cookies.txt", "w") as f:
1034                 f.writelines(
1035                         "%s: %s\n" % (c.name, c.value)
1036                         for c in browser._cookies
1037                 )
1038
1039
1040 def main():
1041         import sys
1042         logging.basicConfig(level=logging.DEBUG)
1043         args = sys.argv
1044         if 3 <= len(args):
1045                 username = args[1]
1046                 password = args[2]
1047
1048         grab_debug_info(username, password)
1049
1050
1051 if __name__ == "__main__":
1052         main()