git.maemo.org Git - gc-dialer/blob - src/backends/gvoice/gvoice.py

   1 #!/usr/bin/python
   2
   3 """
   4 DialCentral - Front end for Google's GoogleVoice service.
   5 Copyright (C) 2008  Eric Warnke ericew AT gmail DOT com
   6
   7 This library is free software; you can redistribute it and/or
   8 modify it under the terms of the GNU Lesser General Public
   9 License as published by the Free Software Foundation; either
  10 version 2.1 of the License, or (at your option) any later version.
  11
  12 This library is distributed in the hope that it will be useful,
  13 but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 Lesser General Public License for more details.
  16
  17 You should have received a copy of the GNU Lesser General Public
  18 License along with this library; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 Google Voice backend code
  22
  23 Resources
  24         http://thatsmith.com/2009/03/google-voice-addon-for-firefox/
  25         http://posttopic.com/topic/google-voice-add-on-development
  26 """
  27
  28 from __future__ import with_statement
  29
  30 import os
  31 import re
  32 import urllib
  33 import urllib2
  34 import time
  35 import datetime
  36 import itertools
  37 import logging
  38 import inspect
  39
  40 from xml.sax import saxutils
  41 from xml.etree import ElementTree
  42
  43 try:
  44         import simplejson as _simplejson
  45         simplejson = _simplejson
  46 except ImportError:
  47         simplejson = None
  48
  49 import browser_emu
  50
  51
  52 _moduleLogger = logging.getLogger(__name__)
  53
  54
  55 class NetworkError(RuntimeError):
  56         pass
  57
  58
  59 class MessageText(object):
  60
  61         ACCURACY_LOW = "med1"
  62         ACCURACY_MEDIUM = "med2"
  63         ACCURACY_HIGH = "high"
  64
  65         def __init__(self):
  66                 self.accuracy = None
  67                 self.text = None
  68
  69         def __str__(self):
  70                 return self.text
  71
  72         def to_dict(self):
  73                 return to_dict(self)
  74
  75         def __eq__(self, other):
  76                 return self.accuracy == other.accuracy and self.text == other.text
  77
  78
  79 class Message(object):
  80
  81         def __init__(self):
  82                 self.whoFrom = None
  83                 self.body = None
  84                 self.when = None
  85
  86         def __str__(self):
  87                 return "%s (%s): %s" % (
  88                         self.whoFrom,
  89                         self.when,
  90                         "".join(unicode(part) for part in self.body)
  91                 )
  92
  93         def to_dict(self):
  94                 selfDict = to_dict(self)
  95                 selfDict["body"] = [text.to_dict() for text in self.body] if self.body is not None else None
  96                 return selfDict
  97
  98         def __eq__(self, other):
  99                 return self.whoFrom == other.whoFrom and self.when == other.when and self.body == other.body
 100
 101
 102 class Conversation(object):
 103
 104         TYPE_VOICEMAIL = "Voicemail"
 105         TYPE_SMS = "SMS"
 106
 107         def __init__(self):
 108                 self.type = None
 109                 self.id = None
 110                 self.contactId = None
 111                 self.name = None
 112                 self.location = None
 113                 self.prettyNumber = None
 114                 self.number = None
 115
 116                 self.time = None
 117                 self.relTime = None
 118                 self.messages = None
 119                 self.isRead = None
 120                 self.isSpam = None
 121                 self.isTrash = None
 122                 self.isArchived = None
 123
 124         def __cmp__(self, other):
 125                 cmpValue = cmp(self.contactId, other.contactId)
 126                 if cmpValue != 0:
 127                         return cmpValue
 128
 129                 cmpValue = cmp(self.time, other.time)
 130                 if cmpValue != 0:
 131                         return cmpValue
 132
 133                 cmpValue = cmp(self.id, other.id)
 134                 if cmpValue != 0:
 135                         return cmpValue
 136
 137         def to_dict(self):
 138                 selfDict = to_dict(self)
 139                 selfDict["messages"] = [message.to_dict() for message in self.messages] if self.messages is not None else None
 140                 return selfDict
 141
 142
 143 class GVoiceBackend(object):
 144         """
 145         This class encapsulates all of the knowledge necessary to interact with the GoogleVoice servers
 146         the functions include login, setting up a callback number, and initalting a callback
 147         """
 148
 149         PHONE_TYPE_HOME = 1
 150         PHONE_TYPE_MOBILE = 2
 151         PHONE_TYPE_WORK = 3
 152         PHONE_TYPE_GIZMO = 7
 153
 154         def __init__(self, cookieFile = None):
 155                 # Important items in this function are the setup of the browser emulation and cookie file
 156                 self._browser = browser_emu.MozillaEmulator(1)
 157                 self._loadedFromCookies = self._browser.load_cookies(cookieFile)
 158
 159                 self._token = ""
 160                 self._accountNum = ""
 161                 self._lastAuthed = 0.0
 162                 self._callbackNumber = ""
 163                 self._callbackNumbers = {}
 164
 165                 # Suprisingly, moving all of these from class to self sped up startup time
 166
 167                 self._validateRe = re.compile("^\+?[0-9]{10,}$")
 168
 169                 self._loginURL = "https://www.google.com/accounts/ServiceLoginAuth"
 170
 171                 SECURE_URL_BASE = "https://www.google.com/voice/"
 172                 SECURE_MOBILE_URL_BASE = SECURE_URL_BASE + "mobile/"
 173                 self._forwardURL = SECURE_MOBILE_URL_BASE + "phones"
 174                 self._tokenURL = SECURE_URL_BASE + "m"
 175                 self._callUrl = SECURE_URL_BASE + "call/connect"
 176                 self._callCancelURL = SECURE_URL_BASE + "call/cancel"
 177                 self._sendSmsURL = SECURE_URL_BASE + "sms/send"
 178
 179                 self._isDndURL = "https://www.google.com/voice/m/donotdisturb"
 180                 self._isDndRe = re.compile(r"""<input.*?id="doNotDisturb".*?checked="(.*?)"\s*/>""")
 181                 self._setDndURL = "https://www.google.com/voice/m/savednd"
 182
 183                 self._downloadVoicemailURL = SECURE_URL_BASE + "media/send_voicemail/"
 184                 self._markAsReadURL = SECURE_URL_BASE + "m/mark"
 185                 self._archiveMessageURL = SECURE_URL_BASE + "m/archive"
 186
 187                 self._XML_SEARCH_URL = SECURE_URL_BASE + "inbox/search/"
 188                 self._XML_ACCOUNT_URL = SECURE_URL_BASE + "contacts/"
 189                 # HACK really this redirects to the main pge and we are grabbing some javascript
 190                 self._XML_CONTACTS_URL = "http://www.google.com/voice/inbox/search/contact"
 191                 self._CSV_CONTACTS_URL = "http://mail.google.com/mail/contacts/data/export"
 192                 self._JSON_CONTACTS_URL = SECURE_URL_BASE + "b/0/request/user"
 193                 self._XML_RECENT_URL = SECURE_URL_BASE + "inbox/recent/"
 194
 195                 self.XML_FEEDS = (
 196                         'inbox', 'starred', 'all', 'spam', 'trash', 'voicemail', 'sms',
 197                         'recorded', 'placed', 'received', 'missed'
 198                 )
 199                 self._XML_INBOX_URL = SECURE_URL_BASE + "inbox/recent/inbox"
 200                 self._XML_STARRED_URL = SECURE_URL_BASE + "inbox/recent/starred"
 201                 self._XML_ALL_URL = SECURE_URL_BASE + "inbox/recent/all"
 202                 self._XML_SPAM_URL = SECURE_URL_BASE + "inbox/recent/spam"
 203                 self._XML_TRASH_URL = SECURE_URL_BASE + "inbox/recent/trash"
 204                 self._XML_VOICEMAIL_URL = SECURE_URL_BASE + "inbox/recent/voicemail/"
 205                 self._XML_SMS_URL = SECURE_URL_BASE + "inbox/recent/sms/"
 206                 self._JSON_SMS_URL = SECURE_URL_BASE + "b/0/request/messages/"
 207                 self._JSON_SMS_COUNT_URL = SECURE_URL_BASE + "b/0/request/unread"
 208                 self._XML_RECORDED_URL = SECURE_URL_BASE + "inbox/recent/recorded/"
 209                 self._XML_PLACED_URL = SECURE_URL_BASE + "inbox/recent/placed/"
 210                 self._XML_RECEIVED_URL = SECURE_URL_BASE + "inbox/recent/received/"
 211                 self._XML_MISSED_URL = SECURE_URL_BASE + "inbox/recent/missed/"
 212
 213                 self._galxRe = re.compile(r"""<input.*?name="GALX".*?value="(.*?)".*?/>""", re.MULTILINE | re.DOTALL)
 214                 self._tokenRe = re.compile(r"""<input.*?name="_rnr_se".*?value="(.*?)"\s*/>""")
 215                 self._accountNumRe = re.compile(r"""<b class="ms\d">(.{14})</b></div>""")
 216                 self._callbackRe = re.compile(r"""\s+(.*?):\s*(.*?)<br\s*/>\s*$""", re.M)
 217
 218                 self._seperateVoicemailsRegex = re.compile(r"""^\s*<div id="(\w+)"\s* class=".*?gc-message.*?">""", re.MULTILINE | re.DOTALL)
 219                 self._exactVoicemailTimeRegex = re.compile(r"""<span class="gc-message-time">(.*?)</span>""", re.MULTILINE)
 220                 self._relativeVoicemailTimeRegex = re.compile(r"""<span class="gc-message-relative">(.*?)</span>""", re.MULTILINE)
 221                 self._voicemailNameRegex = re.compile(r"""<a class=.*?gc-message-name-link.*?>(.*?)</a>""", re.MULTILINE | re.DOTALL)
 222                 self._voicemailNumberRegex = re.compile(r"""<input type="hidden" class="gc-text gc-quickcall-ac" value="(.*?)"/>""", re.MULTILINE)
 223                 self._prettyVoicemailNumberRegex = re.compile(r"""<span class="gc-message-type">(.*?)</span>""", re.MULTILINE)
 224                 self._voicemailLocationRegex = re.compile(r"""<span class="gc-message-location">.*?<a.*?>(.*?)</a></span>""", re.MULTILINE)
 225                 self._messagesContactIDRegex = re.compile(r"""<a class=".*?gc-message-name-link.*?">.*?</a>\s*?<span .*?>(.*?)</span>""", re.MULTILINE)
 226                 self._voicemailMessageRegex = re.compile(r"""(<span id="\d+-\d+" class="gc-word-(.*?)">(.*?)</span>|<a .*? class="gc-message-mni">(.*?)</a>)""", re.MULTILINE)
 227                 self._smsFromRegex = re.compile(r"""<span class="gc-message-sms-from">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 228                 self._smsTimeRegex = re.compile(r"""<span class="gc-message-sms-time">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 229                 self._smsTextRegex = re.compile(r"""<span class="gc-message-sms-text">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 230
 231         def is_quick_login_possible(self):
 232                 """
 233                 @returns True then is_authed might be enough to login, else full login is required
 234                 """
 235                 return self._loadedFromCookies or 0.0 < self._lastAuthed
 236
 237         def is_authed(self, force = False):
 238                 """
 239                 Attempts to detect a current session
 240                 @note Once logged in try not to reauth more than once a minute.
 241                 @returns If authenticated
 242                 @blocks
 243                 """
 244                 isRecentledAuthed = (time.time() - self._lastAuthed) < 120
 245                 isPreviouslyAuthed = self._token is not None
 246                 if isRecentledAuthed and isPreviouslyAuthed and not force:
 247                         return True
 248
 249                 try:
 250                         page = self._get_page(self._forwardURL)
 251                         self._grab_account_info(page)
 252                 except Exception, e:
 253                         _moduleLogger.exception(str(e))
 254                         return False
 255
 256                 self._browser.save_cookies()
 257                 self._lastAuthed = time.time()
 258                 return True
 259
 260         def _get_token(self):
 261                 tokenPage = self._get_page(self._tokenURL)
 262
 263                 galxTokens = self._galxRe.search(tokenPage)
 264                 if galxTokens is not None:
 265                         galxToken = galxTokens.group(1)
 266                 else:
 267                         galxToken = ""
 268                         _moduleLogger.debug("Could not grab GALX token")
 269                 return galxToken
 270
 271         def _login(self, username, password, token):
 272                 loginData = {
 273                         'Email' : username,
 274                         'Passwd' : password,
 275                         'service': "grandcentral",
 276                         "ltmpl": "mobile",
 277                         "btmpl": "mobile",
 278                         "PersistentCookie": "yes",
 279                         "GALX": token,
 280                         "continue": self._forwardURL,
 281                 }
 282
 283                 loginSuccessOrFailurePage = self._get_page(self._loginURL, loginData)
 284                 return loginSuccessOrFailurePage
 285
 286         def login(self, username, password):
 287                 """
 288                 Attempt to login to GoogleVoice
 289                 @returns Whether login was successful or not
 290                 @blocks
 291                 """
 292                 self.logout()
 293                 galxToken = self._get_token()
 294                 loginSuccessOrFailurePage = self._login(username, password, galxToken)
 295
 296                 try:
 297                         self._grab_account_info(loginSuccessOrFailurePage)
 298                 except Exception, e:
 299                         # Retry in case the redirect failed
 300                         # luckily is_authed does everything we need for a retry
 301                         loggedIn = self.is_authed(True)
 302                         if not loggedIn:
 303                                 _moduleLogger.exception(str(e))
 304                                 return False
 305                         _moduleLogger.info("Redirection failed on initial login attempt, auto-corrected for this")
 306
 307                 self._browser.save_cookies()
 308                 self._lastAuthed = time.time()
 309                 return True
 310
 311         def persist(self):
 312                 self._browser.save_cookies()
 313
 314         def shutdown(self):
 315                 self._browser.save_cookies()
 316                 self._token = None
 317                 self._lastAuthed = 0.0
 318
 319         def logout(self):
 320                 self._browser.clear_cookies()
 321                 self._browser.save_cookies()
 322                 self._token = None
 323                 self._lastAuthed = 0.0
 324
 325         def is_dnd(self):
 326                 """
 327                 @blocks
 328                 """
 329                 isDndPage = self._get_page(self._isDndURL)
 330
 331                 dndGroup = self._isDndRe.search(isDndPage)
 332                 if dndGroup is None:
 333                         return False
 334                 dndStatus = dndGroup.group(1)
 335                 isDnd = True if dndStatus.strip().lower() == "true" else False
 336                 return isDnd
 337
 338         def set_dnd(self, doNotDisturb):
 339                 """
 340                 @blocks
 341                 """
 342                 dndPostData = {
 343                         "doNotDisturb": 1 if doNotDisturb else 0,
 344                 }
 345
 346                 dndPage = self._get_page_with_token(self._setDndURL, dndPostData)
 347
 348         def call(self, outgoingNumber):
 349                 """
 350                 This is the main function responsible for initating the callback
 351                 @blocks
 352                 """
 353                 outgoingNumber = self._send_validation(outgoingNumber)
 354                 subscriberNumber = None
 355                 phoneType = guess_phone_type(self._callbackNumber) # @todo Fix this hack
 356
 357                 callData = {
 358                                 'outgoingNumber': outgoingNumber,
 359                                 'forwardingNumber': self._callbackNumber,
 360                                 'subscriberNumber': subscriberNumber or 'undefined',
 361                                 'phoneType': str(phoneType),
 362                                 'remember': '1',
 363                 }
 364                 _moduleLogger.info("%r" % callData)
 365
 366                 page = self._get_page_with_token(
 367                         self._callUrl,
 368                         callData,
 369                 )
 370                 self._parse_with_validation(page)
 371                 return True
 372
 373         def cancel(self, outgoingNumber=None):
 374                 """
 375                 Cancels a call matching outgoing and forwarding numbers (if given).
 376                 Will raise an error if no matching call is being placed
 377                 @blocks
 378                 """
 379                 page = self._get_page_with_token(
 380                         self._callCancelURL,
 381                         {
 382                         'outgoingNumber': outgoingNumber or 'undefined',
 383                         'forwardingNumber': self._callbackNumber or 'undefined',
 384                         'cancelType': 'C2C',
 385                         },
 386                 )
 387                 self._parse_with_validation(page)
 388
 389         def send_sms(self, phoneNumbers, message):
 390                 """
 391                 @blocks
 392                 """
 393                 validatedPhoneNumbers = [
 394                         self._send_validation(phoneNumber)
 395                         for phoneNumber in phoneNumbers
 396                 ]
 397                 flattenedPhoneNumbers = ",".join(validatedPhoneNumbers)
 398                 page = self._get_page_with_token(
 399                         self._sendSmsURL,
 400                         {
 401                                 'phoneNumber': flattenedPhoneNumbers,
 402                                 'text': unicode(message).encode("utf-8"),
 403                         },
 404                 )
 405                 self._parse_with_validation(page)
 406
 407         def search(self, query):
 408                 """
 409                 Search your Google Voice Account history for calls, voicemails, and sms
 410                 Returns ``Folder`` instance containting matching messages
 411                 @blocks
 412                 """
 413                 page = self._get_page(
 414                         self._XML_SEARCH_URL,
 415                         {"q": query},
 416                 )
 417                 json, html = extract_payload(page)
 418                 return json
 419
 420         def get_feed(self, feed):
 421                 """
 422                 @blocks
 423                 """
 424                 actualFeed = "_XML_%s_URL" % feed.upper()
 425                 feedUrl = getattr(self, actualFeed)
 426
 427                 page = self._get_page(feedUrl)
 428                 json, html = extract_payload(page)
 429
 430                 return json
 431
 432         def download(self, messageId, adir):
 433                 """
 434                 Download a voicemail or recorded call MP3 matching the given ``msg``
 435                 which can either be a ``Message`` instance, or a SHA1 identifier.
 436                 Saves files to ``adir`` (defaults to current directory).
 437                 Message hashes can be found in ``self.voicemail().messages`` for example.
 438                 @returns location of saved file.
 439                 @blocks
 440                 """
 441                 page = self._get_page(self._downloadVoicemailURL, {"id": messageId})
 442                 fn = os.path.join(adir, '%s.mp3' % messageId)
 443                 with open(fn, 'wb') as fo:
 444                         fo.write(page)
 445                 return fn
 446
 447         def is_valid_syntax(self, number):
 448                 """
 449                 @returns If This number be called ( syntax validation only )
 450                 """
 451                 return self._validateRe.match(number) is not None
 452
 453         def get_account_number(self):
 454                 """
 455                 @returns The GoogleVoice phone number
 456                 """
 457                 return self._accountNum
 458
 459         def get_callback_numbers(self):
 460                 """
 461                 @returns a dictionary mapping call back numbers to descriptions
 462                 @note These results are cached for 30 minutes.
 463                 """
 464                 if not self.is_authed():
 465                         return {}
 466                 return self._callbackNumbers
 467
 468         def set_callback_number(self, callbacknumber):
 469                 """
 470                 Set the number that GoogleVoice calls
 471                 @param callbacknumber should be a proper 10 digit number
 472                 """
 473                 self._callbackNumber = callbacknumber
 474                 _moduleLogger.info("Callback number changed: %r" % self._callbackNumber)
 475                 return True
 476
 477         def get_callback_number(self):
 478                 """
 479                 @returns Current callback number or None
 480                 """
 481                 return self._callbackNumber
 482
 483         def get_recent(self):
 484                 """
 485                 @returns Iterable of (personsName, phoneNumber, exact date, relative date, action)
 486                 @blocks
 487                 """
 488                 recentPages = [
 489                         (action, self._get_page(url))
 490                         for action, url in (
 491                                 ("Received", self._XML_RECEIVED_URL),
 492                                 ("Missed", self._XML_MISSED_URL),
 493                                 ("Placed", self._XML_PLACED_URL),
 494                         )
 495                 ]
 496                 return self._parse_recent(recentPages)
 497
 498         def get_contacts(self):
 499                 """
 500                 @returns Iterable of (contact id, contact name)
 501                 @blocks
 502                 """
 503                 page = self._get_page(self._JSON_CONTACTS_URL)
 504                 return self._process_contacts(page)
 505
 506         def get_csv_contacts(self):
 507                 data = {
 508                         "groupToExport": "mine",
 509                         "exportType": "ALL",
 510                         "out": "OUTLOOK_CSV",
 511                 }
 512                 encodedData = urllib.urlencode(data)
 513                 contacts = self._get_page(self._CSV_CONTACTS_URL+"?"+encodedData)
 514                 return contacts
 515
 516         def get_voicemails(self):
 517                 """
 518                 @blocks
 519                 """
 520                 voicemailPage = self._get_page(self._XML_VOICEMAIL_URL)
 521                 voicemailHtml = self._grab_html(voicemailPage)
 522                 voicemailJson = self._grab_json(voicemailPage)
 523                 if voicemailJson is None:
 524                         return ()
 525                 parsedVoicemail = self._parse_voicemail(voicemailHtml)
 526                 voicemails = self._merge_conversation_sources(parsedVoicemail, voicemailJson)
 527                 return voicemails
 528
 529         def get_texts(self):
 530                 """
 531                 @blocks
 532                 """
 533                 smsPage = self._get_page(self._XML_SMS_URL)
 534                 smsHtml = self._grab_html(smsPage)
 535                 smsJson = self._grab_json(smsPage)
 536                 if smsJson is None:
 537                         return ()
 538                 parsedSms = self._parse_sms(smsHtml)
 539                 smss = self._merge_conversation_sources(parsedSms, smsJson)
 540                 return smss
 541
 542         def get_unread_counts(self):
 543                 countPage = self._get_page(self._JSON_SMS_COUNT_URL)
 544                 counts = parse_json(countPage)
 545                 counts = counts["unreadCounts"]
 546                 return counts
 547
 548         def mark_message(self, messageId, asRead):
 549                 """
 550                 @blocks
 551                 """
 552                 postData = {
 553                         "read": 1 if asRead else 0,
 554                         "id": messageId,
 555                 }
 556
 557                 markPage = self._get_page(self._markAsReadURL, postData)
 558
 559         def archive_message(self, messageId):
 560                 """
 561                 @blocks
 562                 """
 563                 postData = {
 564                         "id": messageId,
 565                 }
 566
 567                 markPage = self._get_page(self._archiveMessageURL, postData)
 568
 569         def _grab_json(self, flatXml):
 570                 xmlTree = ElementTree.fromstring(flatXml)
 571                 jsonElement = xmlTree.getchildren()[0]
 572                 flatJson = jsonElement.text
 573                 jsonTree = parse_json(flatJson)
 574                 return jsonTree
 575
 576         def _grab_html(self, flatXml):
 577                 xmlTree = ElementTree.fromstring(flatXml)
 578                 htmlElement = xmlTree.getchildren()[1]
 579                 flatHtml = htmlElement.text
 580                 return flatHtml
 581
 582         def _grab_account_info(self, page):
 583                 tokenGroup = self._tokenRe.search(page)
 584                 if tokenGroup is None:
 585                         raise RuntimeError("Could not extract authentication token from GoogleVoice")
 586                 self._token = tokenGroup.group(1)
 587
 588                 anGroup = self._accountNumRe.search(page)
 589                 if anGroup is not None:
 590                         self._accountNum = anGroup.group(1)
 591                 else:
 592                         _moduleLogger.debug("Could not extract account number from GoogleVoice")
 593
 594                 self._callbackNumbers = {}
 595                 for match in self._callbackRe.finditer(page):
 596                         callbackNumber = match.group(2)
 597                         callbackName = match.group(1)
 598                         self._callbackNumbers[callbackNumber] = callbackName
 599                 if len(self._callbackNumbers) == 0:
 600                         _moduleLogger.debug("Could not extract callback numbers from GoogleVoice (the troublesome page follows):\n%s" % page)
 601
 602         def _send_validation(self, number):
 603                 if not self.is_valid_syntax(number):
 604                         raise ValueError('Number is not valid: "%s"' % number)
 605                 elif not self.is_authed():
 606                         raise RuntimeError("Not Authenticated")
 607                 return number
 608
 609         def _parse_recent(self, recentPages):
 610                 for action, flatXml in recentPages:
 611                         allRecentHtml = self._grab_html(flatXml)
 612                         allRecentData = self._parse_history(allRecentHtml)
 613                         for recentCallData in allRecentData:
 614                                 recentCallData["action"] = action
 615                                 yield recentCallData
 616
 617         def _process_contacts(self, page):
 618                 accountData = parse_json(page)
 619                 for contactId, contactDetails in accountData["contacts"].iteritems():
 620                         # A zero contact id is the catch all for unknown contacts
 621                         if contactId != "0":
 622                                 yield contactId, contactDetails
 623
 624         def _parse_history(self, historyHtml):
 625                 splitVoicemail = self._seperateVoicemailsRegex.split(historyHtml)
 626                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 627                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 628                         exactTime = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 629                         exactTime = google_strptime(exactTime)
 630                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 631                         relativeTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 632                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 633                         location = locationGroup.group(1).strip() if locationGroup else ""
 634
 635                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 636                         name = nameGroup.group(1).strip() if nameGroup else ""
 637                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 638                         number = numberGroup.group(1).strip() if numberGroup else ""
 639                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 640                         prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 641                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 642                         contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 643
 644                         yield {
 645                                 "id": messageId.strip(),
 646                                 "contactId": contactId,
 647                                 "name": unescape(name),
 648                                 "time": exactTime,
 649                                 "relTime": relativeTime,
 650                                 "prettyNumber": prettyNumber,
 651                                 "number": number,
 652                                 "location": unescape(location),
 653                         }
 654
 655         @staticmethod
 656         def _interpret_voicemail_regex(group):
 657                 quality, content, number = group.group(2), group.group(3), group.group(4)
 658                 text = MessageText()
 659                 if quality is not None and content is not None:
 660                         text.accuracy = quality
 661                         text.text = unescape(content)
 662                         return text
 663                 elif number is not None:
 664                         text.accuracy = MessageText.ACCURACY_HIGH
 665                         text.text = number
 666                         return text
 667
 668         def _parse_voicemail(self, voicemailHtml):
 669                 splitVoicemail = self._seperateVoicemailsRegex.split(voicemailHtml)
 670                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 671                         conv = Conversation()
 672                         conv.type = Conversation.TYPE_VOICEMAIL
 673                         conv.id = messageId.strip()
 674
 675                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 676                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 677                         conv.time = google_strptime(exactTimeText)
 678                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 679                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 680                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 681                         conv.location = unescape(locationGroup.group(1).strip() if locationGroup else "")
 682
 683                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 684                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 685                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 686                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 687                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 688                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 689                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 690                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 691
 692                         messageGroups = self._voicemailMessageRegex.finditer(messageHtml)
 693                         messageParts = [
 694                                 self._interpret_voicemail_regex(group)
 695                                 for group in messageGroups
 696                         ] if messageGroups else ((MessageText.ACCURACY_LOW, "No Transcription"), )
 697                         message = Message()
 698                         message.body = messageParts
 699                         message.whoFrom = conv.name
 700                         try:
 701                                 message.when = conv.time.strftime("%I:%M %p")
 702                         except ValueError:
 703                                 _moduleLogger.exception("Confusing time provided: %r" % conv.time)
 704                                 message.when = "Unknown"
 705                         conv.messages = (message, )
 706
 707                         yield conv
 708
 709         @staticmethod
 710         def _interpret_sms_message_parts(fromPart, textPart, timePart):
 711                 text = MessageText()
 712                 text.accuracy = MessageText.ACCURACY_MEDIUM
 713                 text.text = unescape(textPart)
 714
 715                 message = Message()
 716                 message.body = (text, )
 717                 message.whoFrom = fromPart
 718                 message.when = timePart
 719
 720                 return message
 721
 722         def _parse_sms(self, smsHtml):
 723                 splitSms = self._seperateVoicemailsRegex.split(smsHtml)
 724                 for messageId, messageHtml in itergroup(splitSms[1:], 2):
 725                         conv = Conversation()
 726                         conv.type = Conversation.TYPE_SMS
 727                         conv.id = messageId.strip()
 728
 729                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 730                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 731                         conv.time = google_strptime(exactTimeText)
 732                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 733                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 734                         conv.location = ""
 735
 736                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 737                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 738                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 739                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 740                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 741                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 742                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 743                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 744
 745                         fromGroups = self._smsFromRegex.finditer(messageHtml)
 746                         fromParts = (group.group(1).strip() for group in fromGroups)
 747                         textGroups = self._smsTextRegex.finditer(messageHtml)
 748                         textParts = (group.group(1).strip() for group in textGroups)
 749                         timeGroups = self._smsTimeRegex.finditer(messageHtml)
 750                         timeParts = (group.group(1).strip() for group in timeGroups)
 751
 752                         messageParts = itertools.izip(fromParts, textParts, timeParts)
 753                         messages = [self._interpret_sms_message_parts(*parts) for parts in messageParts]
 754                         conv.messages = messages
 755
 756                         yield conv
 757
 758         @staticmethod
 759         def _merge_conversation_sources(parsedMessages, json):
 760                 for message in parsedMessages:
 761                         jsonItem = json["messages"][message.id]
 762                         message.isRead = jsonItem["isRead"]
 763                         message.isSpam = jsonItem["isSpam"]
 764                         message.isTrash = jsonItem["isTrash"]
 765                         message.isArchived = "inbox" not in jsonItem["labels"]
 766                         yield message
 767
 768         def _get_page(self, url, data = None, refererUrl = None):
 769                 headers = {}
 770                 if refererUrl is not None:
 771                         headers["Referer"] = refererUrl
 772
 773                 encodedData = urllib.urlencode(data) if data is not None else None
 774
 775                 try:
 776                         page = self._browser.download(url, encodedData, None, headers)
 777                 except urllib2.URLError, e:
 778                         _moduleLogger.error("Translating error: %s" % str(e))
 779                         raise NetworkError("%s is not accesible" % url)
 780
 781                 return page
 782
 783         def _get_page_with_token(self, url, data = None, refererUrl = None):
 784                 if data is None:
 785                         data = {}
 786                 data['_rnr_se'] = self._token
 787
 788                 page = self._get_page(url, data, refererUrl)
 789
 790                 return page
 791
 792         def _parse_with_validation(self, page):
 793                 json = parse_json(page)
 794                 validate_response(json)
 795                 return json
 796
 797
 798 _UNESCAPE_ENTITIES = {
 799  "&quot;": '"',
 800  "&nbsp;": " ",
 801  "&#39;": "'",
 802 }
 803
 804
 805 def unescape(text):
 806         plain = saxutils.unescape(text, _UNESCAPE_ENTITIES)
 807         return plain
 808
 809
 810 def google_strptime(time):
 811         """
 812         Hack: Google always returns the time in the same locale.  Sadly if the
 813         local system's locale is different, there isn't a way to perfectly handle
 814         the time.  So instead we handle implement some time formatting
 815         """
 816         abbrevTime = time[:-3]
 817         parsedTime = datetime.datetime.strptime(abbrevTime, "%m/%d/%y %I:%M")
 818         if time.endswith("PM"):
 819                 parsedTime += datetime.timedelta(hours=12)
 820         return parsedTime
 821
 822
 823 def itergroup(iterator, count, padValue = None):
 824         """
 825         Iterate in groups of 'count' values. If there
 826         aren't enough values, the last result is padded with
 827         None.
 828
 829         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 830         ...     print tuple(val)
 831         (1, 2, 3)
 832         (4, 5, 6)
 833         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 834         ...     print list(val)
 835         [1, 2, 3]
 836         [4, 5, 6]
 837         >>> for val in itergroup([1, 2, 3, 4, 5, 6, 7], 3):
 838         ...     print tuple(val)
 839         (1, 2, 3)
 840         (4, 5, 6)
 841         (7, None, None)
 842         >>> for val in itergroup("123456", 3):
 843         ...     print tuple(val)
 844         ('1', '2', '3')
 845         ('4', '5', '6')
 846         >>> for val in itergroup("123456", 3):
 847         ...     print repr("".join(val))
 848         '123'
 849         '456'
 850         """
 851         paddedIterator = itertools.chain(iterator, itertools.repeat(padValue, count-1))
 852         nIterators = (paddedIterator, ) * count
 853         return itertools.izip(*nIterators)
 854
 855
 856 def safe_eval(s):
 857         _TRUE_REGEX = re.compile("true")
 858         _FALSE_REGEX = re.compile("false")
 859         _COMMENT_REGEX = re.compile("^\s+//.*$", re.M)
 860         s = _TRUE_REGEX.sub("True", s)
 861         s = _FALSE_REGEX.sub("False", s)
 862         s = _COMMENT_REGEX.sub("#", s)
 863         try:
 864                 results = eval(s, {}, {})
 865         except SyntaxError:
 866                 _moduleLogger.exception("Oops")
 867                 results = None
 868         return results
 869
 870
 871 def _fake_parse_json(flattened):
 872         return safe_eval(flattened)
 873
 874
 875 def _actual_parse_json(flattened):
 876         return simplejson.loads(flattened)
 877
 878
 879 if simplejson is None:
 880         parse_json = _fake_parse_json
 881 else:
 882         parse_json = _actual_parse_json
 883
 884
 885 def extract_payload(flatXml):
 886         xmlTree = ElementTree.fromstring(flatXml)
 887
 888         jsonElement = xmlTree.getchildren()[0]
 889         flatJson = jsonElement.text
 890         jsonTree = parse_json(flatJson)
 891
 892         htmlElement = xmlTree.getchildren()[1]
 893         flatHtml = htmlElement.text
 894
 895         return jsonTree, flatHtml
 896
 897
 898 def validate_response(response):
 899         """
 900         Validates that the JSON response is A-OK
 901         """
 902         try:
 903                 assert response is not None, "Response not provided"
 904                 assert 'ok' in response, "Response lacks status"
 905                 assert response['ok'], "Response not good"
 906         except AssertionError:
 907                 raise RuntimeError('There was a problem with GV: %s' % response)
 908
 909
 910 def guess_phone_type(number):
 911         if number.startswith("747") or number.startswith("1747") or number.startswith("+1747"):
 912                 return GVoiceBackend.PHONE_TYPE_GIZMO
 913         else:
 914                 return GVoiceBackend.PHONE_TYPE_MOBILE
 915
 916
 917 def get_sane_callback(backend):
 918         """
 919         Try to set a sane default callback number on these preferences
 920         1) 1747 numbers ( Gizmo )
 921         2) anything with gizmo in the name
 922         3) anything with computer in the name
 923         4) the first value
 924         """
 925         numbers = backend.get_callback_numbers()
 926
 927         priorityOrderedCriteria = [
 928                 ("\+1747", None),
 929                 ("1747", None),
 930                 ("747", None),
 931                 (None, "gizmo"),
 932                 (None, "computer"),
 933                 (None, "sip"),
 934                 (None, None),
 935         ]
 936
 937         for numberCriteria, descriptionCriteria in priorityOrderedCriteria:
 938                 numberMatcher = None
 939                 descriptionMatcher = None
 940                 if numberCriteria is not None:
 941                         numberMatcher = re.compile(numberCriteria)
 942                 elif descriptionCriteria is not None:
 943                         descriptionMatcher = re.compile(descriptionCriteria, re.I)
 944
 945                 for number, description in numbers.iteritems():
 946                         if numberMatcher is not None and numberMatcher.match(number) is None:
 947                                 continue
 948                         if descriptionMatcher is not None and descriptionMatcher.match(description) is None:
 949                                 continue
 950                         return number
 951
 952
 953 def set_sane_callback(backend):
 954         """
 955         Try to set a sane default callback number on these preferences
 956         1) 1747 numbers ( Gizmo )
 957         2) anything with gizmo in the name
 958         3) anything with computer in the name
 959         4) the first value
 960         """
 961         number = get_sane_callback(backend)
 962         backend.set_callback_number(number)
 963
 964
 965 def _is_not_special(name):
 966         return not name.startswith("_") and name[0].lower() == name[0] and "_" not in name
 967
 968
 969 def to_dict(obj):
 970         members = inspect.getmembers(obj)
 971         return dict((name, value) for (name, value) in members if _is_not_special(name))
 972
 973
 974 def grab_debug_info(username, password):
 975         cookieFile = os.path.join(".", "raw_cookies.txt")
 976         try:
 977                 os.remove(cookieFile)
 978         except OSError:
 979                 pass
 980
 981         backend = GVoiceBackend(cookieFile)
 982         browser = backend._browser
 983
 984         _TEST_WEBPAGES = [
 985                 ("forward", backend._forwardURL),
 986                 ("token", backend._tokenURL),
 987                 ("login", backend._loginURL),
 988                 ("isdnd", backend._isDndURL),
 989                 ("account", backend._XML_ACCOUNT_URL),
 990                 ("html_contacts", backend._XML_CONTACTS_URL),
 991                 ("contacts", backend._JSON_CONTACTS_URL),
 992                 ("csv", backend._CSV_CONTACTS_URL),
 993
 994                 ("voicemail", backend._XML_VOICEMAIL_URL),
 995                 ("html_sms", backend._XML_SMS_URL),
 996                 ("sms", backend._JSON_SMS_URL),
 997                 ("count", backend._JSON_SMS_COUNT_URL),
 998
 999                 ("recent", backend._XML_RECENT_URL),
1000                 ("placed", backend._XML_PLACED_URL),
1001                 ("recieved", backend._XML_RECEIVED_URL),
1002                 ("missed", backend._XML_MISSED_URL),
1003         ]
1004
1005         # Get Pages
1006         print "Grabbing pre-login pages"
1007         for name, url in _TEST_WEBPAGES:
1008                 try:
1009                         page = browser.download(url)
1010                 except StandardError, e:
1011                         print e.message
1012                         continue
1013                 print "\tWriting to file"
1014                 with open("not_loggedin_%s.txt" % name, "w") as f:
1015                         f.write(page)
1016
1017         # Login
1018         print "Attempting login"
1019         galxToken = backend._get_token()
1020         loginSuccessOrFailurePage = backend._login(username, password, galxToken)
1021         with open("loggingin.txt", "w") as f:
1022                 print "\tWriting to file"
1023                 f.write(loginSuccessOrFailurePage)
1024         try:
1025                 backend._grab_account_info(loginSuccessOrFailurePage)
1026         except Exception:
1027                 # Retry in case the redirect failed
1028                 # luckily is_authed does everything we need for a retry
1029                 loggedIn = backend.is_authed(True)
1030                 if not loggedIn:
1031                         raise
1032
1033         # Get Pages
1034         print "Grabbing post-login pages"
1035         for name, url in _TEST_WEBPAGES:
1036                 try:
1037                         page = browser.download(url)
1038                 except StandardError, e:
1039                         print str(e)
1040                         continue
1041                 print "\tWriting to file"
1042                 with open("loggedin_%s.txt" % name, "w") as f:
1043                         f.write(page)
1044
1045         # Cookies
1046         browser.save_cookies()
1047         print "\tWriting cookies to file"
1048         with open("cookies.txt", "w") as f:
1049                 f.writelines(
1050                         "%s: %s\n" % (c.name, c.value)
1051                         for c in browser._cookies
1052                 )
1053
1054
1055 def main():
1056         import sys
1057         logging.basicConfig(level=logging.DEBUG)
1058         args = sys.argv
1059         if 3 <= len(args):
1060                 username = args[1]
1061                 password = args[2]
1062
1063         grab_debug_info(username, password)
1064
1065
1066 if __name__ == "__main__":
1067         main()