git.maemo.org Git - theonering/blob - src/gvoice/backend.py

   1 #!/usr/bin/python
   2
   3 """
   4 DialCentral - Front end for Google's GoogleVoice service.
   5 Copyright (C) 2008  Eric Warnke ericew AT gmail DOT com
   6
   7 This library is free software; you can redistribute it and/or
   8 modify it under the terms of the GNU Lesser General Public
   9 License as published by the Free Software Foundation; either
  10 version 2.1 of the License, or (at your option) any later version.
  11
  12 This library is distributed in the hope that it will be useful,
  13 but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 Lesser General Public License for more details.
  16
  17 You should have received a copy of the GNU Lesser General Public
  18 License along with this library; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 Google Voice backend code
  22
  23 Resources
  24         http://thatsmith.com/2009/03/google-voice-addon-for-firefox/
  25         http://posttopic.com/topic/google-voice-add-on-development
  26 """
  27
  28 from __future__ import with_statement
  29
  30 import os
  31 import re
  32 import urllib
  33 import urllib2
  34 import time
  35 import datetime
  36 import itertools
  37 import logging
  38 import inspect
  39
  40 from xml.sax import saxutils
  41 from xml.etree import ElementTree
  42
  43 try:
  44         import simplejson as _simplejson
  45         simplejson = _simplejson
  46 except ImportError:
  47         simplejson = None
  48
  49 import browser_emu
  50
  51
  52 _moduleLogger = logging.getLogger(__name__)
  53
  54
  55 class NetworkError(RuntimeError):
  56         pass
  57
  58
  59 class MessageText(object):
  60
  61         ACCURACY_LOW = "med1"
  62         ACCURACY_MEDIUM = "med2"
  63         ACCURACY_HIGH = "high"
  64
  65         def __init__(self):
  66                 self.accuracy = None
  67                 self.text = None
  68
  69         def __str__(self):
  70                 return self.text
  71
  72         def to_dict(self):
  73                 return to_dict(self)
  74
  75         def __eq__(self, other):
  76                 return self.accuracy == other.accuracy and self.text == other.text
  77
  78
  79 class Message(object):
  80
  81         def __init__(self):
  82                 self.whoFrom = None
  83                 self.body = None
  84                 self.when = None
  85
  86         def __str__(self):
  87                 return "%s (%s): %s" % (
  88                         self.whoFrom,
  89                         self.when,
  90                         "".join(unicode(part) for part in self.body)
  91                 )
  92
  93         def to_dict(self):
  94                 selfDict = to_dict(self)
  95                 selfDict["body"] = [text.to_dict() for text in self.body] if self.body is not None else None
  96                 return selfDict
  97
  98         def __eq__(self, other):
  99                 return self.whoFrom == other.whoFrom and self.when == other.when and self.body == other.body
 100
 101
 102 class Conversation(object):
 103
 104         TYPE_VOICEMAIL = "Voicemail"
 105         TYPE_SMS = "SMS"
 106
 107         def __init__(self):
 108                 self.type = None
 109                 self.id = None
 110                 self.contactId = None
 111                 self.name = None
 112                 self.location = None
 113                 self.prettyNumber = None
 114                 self.number = None
 115
 116                 self.time = None
 117                 self.relTime = None
 118                 self.messages = None
 119                 self.isRead = None
 120                 self.isSpam = None
 121                 self.isTrash = None
 122                 self.isArchived = None
 123
 124         def __cmp__(self, other):
 125                 cmpValue = cmp(self.contactId, other.contactId)
 126                 if cmpValue != 0:
 127                         return cmpValue
 128
 129                 cmpValue = cmp(self.time, other.time)
 130                 if cmpValue != 0:
 131                         return cmpValue
 132
 133                 cmpValue = cmp(self.id, other.id)
 134                 if cmpValue != 0:
 135                         return cmpValue
 136
 137         def to_dict(self):
 138                 selfDict = to_dict(self)
 139                 selfDict["messages"] = [message.to_dict() for message in self.messages] if self.messages is not None else None
 140                 return selfDict
 141
 142
 143 class GVoiceBackend(object):
 144         """
 145         This class encapsulates all of the knowledge necessary to interact with the GoogleVoice servers
 146         the functions include login, setting up a callback number, and initalting a callback
 147         """
 148
 149         PHONE_TYPE_HOME = 1
 150         PHONE_TYPE_MOBILE = 2
 151         PHONE_TYPE_WORK = 3
 152         PHONE_TYPE_GIZMO = 7
 153
 154         def __init__(self, cookieFile = None):
 155                 # Important items in this function are the setup of the browser emulation and cookie file
 156                 self._browser = browser_emu.MozillaEmulator(1)
 157                 self._loadedFromCookies = self._browser.load_cookies(cookieFile)
 158
 159                 self._token = ""
 160                 self._accountNum = ""
 161                 self._lastAuthed = 0.0
 162                 self._callbackNumber = ""
 163                 self._callbackNumbers = {}
 164
 165                 # Suprisingly, moving all of these from class to self sped up startup time
 166
 167                 self._validateRe = re.compile("^\+?[0-9]{10,}$")
 168
 169                 self._loginURL = "https://www.google.com/accounts/ServiceLoginAuth"
 170
 171                 SECURE_URL_BASE = "https://www.google.com/voice/"
 172                 SECURE_MOBILE_URL_BASE = SECURE_URL_BASE + "mobile/"
 173                 self._forwardURL = SECURE_MOBILE_URL_BASE + "phones"
 174                 self._tokenURL = SECURE_URL_BASE + "m"
 175                 self._callUrl = SECURE_URL_BASE + "call/connect"
 176                 self._callCancelURL = SECURE_URL_BASE + "call/cancel"
 177                 self._sendSmsURL = SECURE_URL_BASE + "sms/send"
 178
 179                 self._isDndURL = "https://www.google.com/voice/m/donotdisturb"
 180                 self._isDndRe = re.compile(r"""<input.*?id="doNotDisturb".*?checked="(.*?)"\s*/>""")
 181                 self._setDndURL = "https://www.google.com/voice/m/savednd"
 182
 183                 self._downloadVoicemailURL = SECURE_URL_BASE + "media/send_voicemail/"
 184                 self._markAsReadURL = SECURE_URL_BASE + "m/mark"
 185                 self._archiveMessageURL = SECURE_URL_BASE + "m/archive"
 186
 187                 self._XML_SEARCH_URL = SECURE_URL_BASE + "inbox/search/"
 188                 self._XML_ACCOUNT_URL = SECURE_URL_BASE + "contacts/"
 189                 # HACK really this redirects to the main pge and we are grabbing some javascript
 190                 self._XML_CONTACTS_URL = "http://www.google.com/voice/inbox/search/contact"
 191                 self._CSV_CONTACTS_URL = "http://mail.google.com/mail/contacts/data/export"
 192                 self._JSON_CONTACTS_URL = SECURE_URL_BASE + "b/0/request/user"
 193                 self._XML_RECENT_URL = SECURE_URL_BASE + "inbox/recent/"
 194
 195                 self.XML_FEEDS = (
 196                         'inbox', 'starred', 'all', 'spam', 'trash', 'voicemail', 'sms',
 197                         'recorded', 'placed', 'received', 'missed'
 198                 )
 199                 self._XML_INBOX_URL = SECURE_URL_BASE + "inbox/recent/inbox"
 200                 self._XML_STARRED_URL = SECURE_URL_BASE + "inbox/recent/starred"
 201                 self._XML_ALL_URL = SECURE_URL_BASE + "inbox/recent/all"
 202                 self._XML_SPAM_URL = SECURE_URL_BASE + "inbox/recent/spam"
 203                 self._XML_TRASH_URL = SECURE_URL_BASE + "inbox/recent/trash"
 204                 self._XML_VOICEMAIL_URL = SECURE_URL_BASE + "inbox/recent/voicemail/"
 205                 self._XML_SMS_URL = SECURE_URL_BASE + "inbox/recent/sms/"
 206                 self._JSON_SMS_URL = SECURE_URL_BASE + "b/0/request/messages/"
 207                 self._JSON_SMS_COUNT_URL = SECURE_URL_BASE + "b/0/request/unread"
 208                 self._XML_RECORDED_URL = SECURE_URL_BASE + "inbox/recent/recorded/"
 209                 self._XML_PLACED_URL = SECURE_URL_BASE + "inbox/recent/placed/"
 210                 self._XML_RECEIVED_URL = SECURE_URL_BASE + "inbox/recent/received/"
 211                 self._XML_MISSED_URL = SECURE_URL_BASE + "inbox/recent/missed/"
 212
 213                 self._galxRe = re.compile(r"""<input.*?name="GALX".*?value="(.*?)".*?/>""", re.MULTILINE | re.DOTALL)
 214                 self._tokenRe = re.compile(r"""<input.*?name="_rnr_se".*?value="(.*?)"\s*/>""")
 215                 self._accountNumRe = re.compile(r"""<b class="ms\d">(.{14})</b></div>""")
 216                 self._callbackRe = re.compile(r"""\s+(.*?):\s*(.*?)<br\s*/>\s*$""", re.M)
 217
 218                 self._seperateVoicemailsRegex = re.compile(r"""^\s*<div id="(\w+)"\s* class=".*?gc-message.*?">""", re.MULTILINE | re.DOTALL)
 219                 self._exactVoicemailTimeRegex = re.compile(r"""<span class="gc-message-time">(.*?)</span>""", re.MULTILINE)
 220                 self._relativeVoicemailTimeRegex = re.compile(r"""<span class="gc-message-relative">(.*?)</span>""", re.MULTILINE)
 221                 self._voicemailNameRegex = re.compile(r"""<a class=.*?gc-message-name-link.*?>(.*?)</a>""", re.MULTILINE | re.DOTALL)
 222                 self._voicemailNumberRegex = re.compile(r"""<input type="hidden" class="gc-text gc-quickcall-ac" value="(.*?)"/>""", re.MULTILINE)
 223                 self._prettyVoicemailNumberRegex = re.compile(r"""<span class="gc-message-type">(.*?)</span>""", re.MULTILINE)
 224                 self._voicemailLocationRegex = re.compile(r"""<span class="gc-message-location">.*?<a.*?>(.*?)</a></span>""", re.MULTILINE)
 225                 self._messagesContactIDRegex = re.compile(r"""<a class=".*?gc-message-name-link.*?">.*?</a>\s*?<span .*?>(.*?)</span>""", re.MULTILINE)
 226                 self._voicemailMessageRegex = re.compile(r"""(<span id="\d+-\d+" class="gc-word-(.*?)">(.*?)</span>|<a .*? class="gc-message-mni">(.*?)</a>)""", re.MULTILINE)
 227                 self._smsFromRegex = re.compile(r"""<span class="gc-message-sms-from">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 228                 self._smsTimeRegex = re.compile(r"""<span class="gc-message-sms-time">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 229                 self._smsTextRegex = re.compile(r"""<span class="gc-message-sms-text">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 230
 231         def is_quick_login_possible(self):
 232                 """
 233                 @returns True then is_authed might be enough to login, else full login is required
 234                 """
 235                 return self._loadedFromCookies or 0.0 < self._lastAuthed
 236
 237         def is_authed(self, force = False):
 238                 """
 239                 Attempts to detect a current session
 240                 @note Once logged in try not to reauth more than once a minute.
 241                 @returns If authenticated
 242                 @blocks
 243                 """
 244                 isRecentledAuthed = (time.time() - self._lastAuthed) < 120
 245                 isPreviouslyAuthed = self._token is not None
 246                 if isRecentledAuthed and isPreviouslyAuthed and not force:
 247                         return True
 248
 249                 try:
 250                         page = self._get_page(self._forwardURL)
 251                         self._grab_account_info(page)
 252                 except Exception, e:
 253                         _moduleLogger.exception(str(e))
 254                         return False
 255
 256                 self._browser.save_cookies()
 257                 self._lastAuthed = time.time()
 258                 return True
 259
 260         def _get_token(self):
 261                 tokenPage = self._get_page(self._tokenURL)
 262
 263                 galxTokens = self._galxRe.search(tokenPage)
 264                 if galxTokens is not None:
 265                         galxToken = galxTokens.group(1)
 266                 else:
 267                         galxToken = ""
 268                         _moduleLogger.debug("Could not grab GALX token")
 269                 return galxToken
 270
 271         def _login(self, username, password, token):
 272                 loginData = {
 273                         'Email' : username,
 274                         'Passwd' : password,
 275                         'service': "grandcentral",
 276                         "ltmpl": "mobile",
 277                         "btmpl": "mobile",
 278                         "PersistentCookie": "yes",
 279                         "GALX": token,
 280                         "continue": self._forwardURL,
 281                 }
 282
 283                 loginSuccessOrFailurePage = self._get_page(self._loginURL, loginData)
 284                 return loginSuccessOrFailurePage
 285
 286         def login(self, username, password):
 287                 """
 288                 Attempt to login to GoogleVoice
 289                 @returns Whether login was successful or not
 290                 @blocks
 291                 """
 292                 self.logout()
 293                 galxToken = self._get_token()
 294                 loginSuccessOrFailurePage = self._login(username, password, galxToken)
 295
 296                 try:
 297                         self._grab_account_info(loginSuccessOrFailurePage)
 298                 except Exception, e:
 299                         # Retry in case the redirect failed
 300                         # luckily is_authed does everything we need for a retry
 301                         loggedIn = self.is_authed(True)
 302                         if not loggedIn:
 303                                 _moduleLogger.exception(str(e))
 304                                 return False
 305                         _moduleLogger.info("Redirection failed on initial login attempt, auto-corrected for this")
 306
 307                 self._browser.save_cookies()
 308                 self._lastAuthed = time.time()
 309                 return True
 310
 311         def persist(self):
 312                 self._browser.save_cookies()
 313
 314         def shutdown(self):
 315                 self._browser.save_cookies()
 316                 self._token = None
 317                 self._lastAuthed = 0.0
 318
 319         def logout(self):
 320                 self._browser.clear_cookies()
 321                 self._browser.save_cookies()
 322                 self._token = None
 323                 self._lastAuthed = 0.0
 324
 325         def is_dnd(self):
 326                 """
 327                 @blocks
 328                 """
 329                 isDndPage = self._get_page(self._isDndURL)
 330
 331                 dndGroup = self._isDndRe.search(isDndPage)
 332                 if dndGroup is None:
 333                         return False
 334                 dndStatus = dndGroup.group(1)
 335                 isDnd = True if dndStatus.strip().lower() == "true" else False
 336                 return isDnd
 337
 338         def set_dnd(self, doNotDisturb):
 339                 """
 340                 @blocks
 341                 """
 342                 dndPostData = {
 343                         "doNotDisturb": 1 if doNotDisturb else 0,
 344                 }
 345
 346                 dndPage = self._get_page_with_token(self._setDndURL, dndPostData)
 347
 348         def call(self, outgoingNumber):
 349                 """
 350                 This is the main function responsible for initating the callback
 351                 @blocks
 352                 """
 353                 outgoingNumber = self._send_validation(outgoingNumber)
 354                 subscriberNumber = None
 355                 phoneType = guess_phone_type(self._callbackNumber) # @todo Fix this hack
 356
 357                 callData = {
 358                                 'outgoingNumber': outgoingNumber,
 359                                 'forwardingNumber': self._callbackNumber,
 360                                 'subscriberNumber': subscriberNumber or 'undefined',
 361                                 'phoneType': str(phoneType),
 362                                 'remember': '1',
 363                 }
 364                 _moduleLogger.info("%r" % callData)
 365
 366                 page = self._get_page_with_token(
 367                         self._callUrl,
 368                         callData,
 369                 )
 370                 self._parse_with_validation(page)
 371                 return True
 372
 373         def cancel(self, outgoingNumber=None):
 374                 """
 375                 Cancels a call matching outgoing and forwarding numbers (if given).
 376                 Will raise an error if no matching call is being placed
 377                 @blocks
 378                 """
 379                 page = self._get_page_with_token(
 380                         self._callCancelURL,
 381                         {
 382                         'outgoingNumber': outgoingNumber or 'undefined',
 383                         'forwardingNumber': self._callbackNumber or 'undefined',
 384                         'cancelType': 'C2C',
 385                         },
 386                 )
 387                 self._parse_with_validation(page)
 388
 389         def send_sms(self, phoneNumbers, message):
 390                 """
 391                 @blocks
 392                 """
 393                 validatedPhoneNumbers = [
 394                         self._send_validation(phoneNumber)
 395                         for phoneNumber in phoneNumbers
 396                 ]
 397                 flattenedPhoneNumbers = ",".join(validatedPhoneNumbers)
 398                 page = self._get_page_with_token(
 399                         self._sendSmsURL,
 400                         {
 401                                 'phoneNumber': flattenedPhoneNumbers,
 402                                 'text': unicode(message).encode("utf-8"),
 403                         },
 404                 )
 405                 self._parse_with_validation(page)
 406
 407         def search(self, query):
 408                 """
 409                 Search your Google Voice Account history for calls, voicemails, and sms
 410                 Returns ``Folder`` instance containting matching messages
 411                 @blocks
 412                 """
 413                 page = self._get_page(
 414                         self._XML_SEARCH_URL,
 415                         {"q": query},
 416                 )
 417                 json, html = extract_payload(page)
 418                 return json
 419
 420         def get_feed(self, feed):
 421                 """
 422                 @blocks
 423                 """
 424                 actualFeed = "_XML_%s_URL" % feed.upper()
 425                 feedUrl = getattr(self, actualFeed)
 426
 427                 page = self._get_page(feedUrl)
 428                 json, html = extract_payload(page)
 429
 430                 return json
 431
 432         def download(self, messageId, adir):
 433                 """
 434                 Download a voicemail or recorded call MP3 matching the given ``msg``
 435                 which can either be a ``Message`` instance, or a SHA1 identifier.
 436                 Saves files to ``adir`` (defaults to current directory).
 437                 Message hashes can be found in ``self.voicemail().messages`` for example.
 438                 @returns location of saved file.
 439                 @blocks
 440                 """
 441                 page = self._get_page(self._downloadVoicemailURL, {"id": messageId})
 442                 fn = os.path.join(adir, '%s.mp3' % messageId)
 443                 with open(fn, 'wb') as fo:
 444                         fo.write(page)
 445                 return fn
 446
 447         def is_valid_syntax(self, number):
 448                 """
 449                 @returns If This number be called ( syntax validation only )
 450                 """
 451                 return self._validateRe.match(number) is not None
 452
 453         def get_account_number(self):
 454                 """
 455                 @returns The GoogleVoice phone number
 456                 """
 457                 return self._accountNum
 458
 459         def get_callback_numbers(self):
 460                 """
 461                 @returns a dictionary mapping call back numbers to descriptions
 462                 @note These results are cached for 30 minutes.
 463                 """
 464                 if not self.is_authed():
 465                         return {}
 466                 return self._callbackNumbers
 467
 468         def set_callback_number(self, callbacknumber):
 469                 """
 470                 Set the number that GoogleVoice calls
 471                 @param callbacknumber should be a proper 10 digit number
 472                 """
 473                 self._callbackNumber = callbacknumber
 474                 _moduleLogger.info("Callback number changed: %r" % self._callbackNumber)
 475                 return True
 476
 477         def get_callback_number(self):
 478                 """
 479                 @returns Current callback number or None
 480                 """
 481                 return self._callbackNumber
 482
 483         def get_recent(self):
 484                 """
 485                 @returns Iterable of (personsName, phoneNumber, exact date, relative date, action)
 486                 @blocks
 487                 """
 488                 recentPages = [
 489                         (action, self._get_page(url))
 490                         for action, url in (
 491                                 ("Received", self._XML_RECEIVED_URL),
 492                                 ("Missed", self._XML_MISSED_URL),
 493                                 ("Placed", self._XML_PLACED_URL),
 494                         )
 495                 ]
 496                 return self._parse_recent(recentPages)
 497
 498         def get_contacts(self):
 499                 """
 500                 @returns Iterable of (contact id, contact name)
 501                 @blocks
 502                 """
 503                 page = self._get_page(self._JSON_CONTACTS_URL)
 504                 return self._process_contacts(page)
 505
 506         def get_csv_contacts(self):
 507                 data = {
 508                         "groupToExport": "mine",
 509                         "exportType": "ALL",
 510                         "out": "OUTLOOK_CSV",
 511                 }
 512                 encodedData = urllib.urlencode(data)
 513                 contacts = self._get_page(self._CSV_CONTACTS_URL+"?"+encodedData)
 514                 return contacts
 515
 516         def get_voicemails(self):
 517                 """
 518                 @blocks
 519                 """
 520                 voicemailPage = self._get_page(self._XML_VOICEMAIL_URL)
 521                 voicemailHtml = self._grab_html(voicemailPage)
 522                 voicemailJson = self._grab_json(voicemailPage)
 523                 if voicemailJson is None:
 524                         return ()
 525                 parsedVoicemail = self._parse_voicemail(voicemailHtml)
 526                 voicemails = self._merge_conversation_sources(parsedVoicemail, voicemailJson)
 527                 return voicemails
 528
 529         def get_texts(self):
 530                 """
 531                 @blocks
 532                 """
 533                 smsPage = self._get_page(self._XML_SMS_URL)
 534                 smsHtml = self._grab_html(smsPage)
 535                 smsJson = self._grab_json(smsPage)
 536                 if smsJson is None:
 537                         return ()
 538                 parsedSms = self._parse_sms(smsHtml)
 539                 smss = self._merge_conversation_sources(parsedSms, smsJson)
 540                 return smss
 541
 542         def get_unread_counts(self):
 543                 countPage = self._get_page(self._JSON_SMS_COUNT_URL)
 544                 counts = parse_json(countPage)
 545                 counts = counts["unreadCounts"]
 546                 return counts
 547
 548         def mark_message(self, messageId, asRead):
 549                 """
 550                 @blocks
 551                 """
 552                 postData = {
 553                         "read": 1 if asRead else 0,
 554                         "id": messageId,
 555                 }
 556
 557                 markPage = self._get_page(self._markAsReadURL, postData)
 558
 559         def archive_message(self, messageId):
 560                 """
 561                 @blocks
 562                 """
 563                 postData = {
 564                         "id": messageId,
 565                 }
 566
 567                 markPage = self._get_page(self._archiveMessageURL, postData)
 568
 569         def _grab_json(self, flatXml):
 570                 xmlTree = ElementTree.fromstring(flatXml)
 571                 jsonElement = xmlTree.getchildren()[0]
 572                 flatJson = jsonElement.text
 573                 jsonTree = parse_json(flatJson)
 574                 return jsonTree
 575
 576         def _grab_html(self, flatXml):
 577                 xmlTree = ElementTree.fromstring(flatXml)
 578                 htmlElement = xmlTree.getchildren()[1]
 579                 flatHtml = htmlElement.text
 580                 return flatHtml
 581
 582         def _grab_account_info(self, page):
 583                 tokenGroup = self._tokenRe.search(page)
 584                 if tokenGroup is None:
 585                         raise RuntimeError("Could not extract authentication token from GoogleVoice")
 586                 self._token = tokenGroup.group(1)
 587
 588                 anGroup = self._accountNumRe.search(page)
 589                 if anGroup is not None:
 590                         self._accountNum = anGroup.group(1)
 591                 else:
 592                         _moduleLogger.debug("Could not extract account number from GoogleVoice")
 593
 594                 self._callbackNumbers = {}
 595                 for match in self._callbackRe.finditer(page):
 596                         callbackNumber = match.group(2)
 597                         callbackName = match.group(1)
 598                         self._callbackNumbers[callbackNumber] = callbackName
 599                 if len(self._callbackNumbers) == 0:
 600                         _moduleLogger.debug("Could not extract callback numbers from GoogleVoice (the troublesome page follows):\n%s" % page)
 601
 602         def _send_validation(self, number):
 603                 if not self.is_valid_syntax(number):
 604                         raise ValueError('Number is not valid: "%s"' % number)
 605                 elif not self.is_authed():
 606                         raise RuntimeError("Not Authenticated")
 607                 return number
 608
 609         def _parse_recent(self, recentPages):
 610                 for action, flatXml in recentPages:
 611                         allRecentHtml = self._grab_html(flatXml)
 612                         allRecentData = self._parse_history(allRecentHtml)
 613                         for recentCallData in allRecentData:
 614                                 recentCallData["action"] = action
 615                                 yield recentCallData
 616
 617         def _process_contacts(self, page):
 618                 accountData = parse_json(page)
 619                 for contactId, contactDetails in accountData["contacts"].iteritems():
 620                         # A zero contact id is the catch all for unknown contacts
 621                         if contactId != "0":
 622                                 yield contactId, contactDetails
 623
 624         def _parse_history(self, historyHtml):
 625                 splitVoicemail = self._seperateVoicemailsRegex.split(historyHtml)
 626                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 627                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 628                         exactTime = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 629                         exactTime = google_strptime(exactTime)
 630                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 631                         relativeTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 632                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 633                         location = locationGroup.group(1).strip() if locationGroup else ""
 634
 635                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 636                         name = nameGroup.group(1).strip() if nameGroup else ""
 637                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 638                         number = numberGroup.group(1).strip() if numberGroup else ""
 639                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 640                         prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 641                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 642                         contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 643
 644                         yield {
 645                                 "id": messageId.strip(),
 646                                 "contactId": contactId,
 647                                 "name": unescape(name),
 648                                 "time": exactTime,
 649                                 "relTime": relativeTime,
 650                                 "prettyNumber": prettyNumber,
 651                                 "number": number,
 652                                 "location": unescape(location),
 653                         }
 654
 655         @staticmethod
 656         def _interpret_voicemail_regex(group):
 657                 quality, content, number = group.group(2), group.group(3), group.group(4)
 658                 text = MessageText()
 659                 if quality is not None and content is not None:
 660                         text.accuracy = quality
 661                         text.text = content
 662                         return text
 663                 elif number is not None:
 664                         text.accuracy = MessageText.ACCURACY_HIGH
 665                         text.text = number
 666                         return text
 667
 668         def _parse_voicemail(self, voicemailHtml):
 669                 splitVoicemail = self._seperateVoicemailsRegex.split(voicemailHtml)
 670                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 671                         conv = Conversation()
 672                         conv.type = Conversation.TYPE_VOICEMAIL
 673                         conv.id = messageId.strip()
 674
 675                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 676                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 677                         conv.time = google_strptime(exactTimeText)
 678                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 679                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 680                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 681                         conv.location = unescape(locationGroup.group(1).strip() if locationGroup else "")
 682
 683                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 684                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 685                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 686                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 687                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 688                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 689                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 690                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 691
 692                         messageGroups = self._voicemailMessageRegex.finditer(messageHtml)
 693                         messageParts = [
 694                                 self._interpret_voicemail_regex(group)
 695                                 for group in messageGroups
 696                         ] if messageGroups else ((MessageText.ACCURACY_LOW, "No Transcription"), )
 697                         message = Message()
 698                         message.body = messageParts
 699                         message.whoFrom = conv.name
 700                         message.when = conv.time.strftime("%I:%M %p")
 701                         conv.messages = (message, )
 702
 703                         yield conv
 704
 705         @staticmethod
 706         def _interpret_sms_message_parts(fromPart, textPart, timePart):
 707                 text = MessageText()
 708                 text.accuracy = MessageText.ACCURACY_MEDIUM
 709                 text.text = textPart
 710
 711                 message = Message()
 712                 message.body = (text, )
 713                 message.whoFrom = fromPart
 714                 message.when = timePart
 715
 716                 return message
 717
 718         def _parse_sms(self, smsHtml):
 719                 splitSms = self._seperateVoicemailsRegex.split(smsHtml)
 720                 for messageId, messageHtml in itergroup(splitSms[1:], 2):
 721                         conv = Conversation()
 722                         conv.type = Conversation.TYPE_SMS
 723                         conv.id = messageId.strip()
 724
 725                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 726                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 727                         conv.time = google_strptime(exactTimeText)
 728                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 729                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 730                         conv.location = ""
 731
 732                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 733                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 734                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 735                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 736                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 737                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 738                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 739                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 740
 741                         fromGroups = self._smsFromRegex.finditer(messageHtml)
 742                         fromParts = (group.group(1).strip() for group in fromGroups)
 743                         textGroups = self._smsTextRegex.finditer(messageHtml)
 744                         textParts = (group.group(1).strip() for group in textGroups)
 745                         timeGroups = self._smsTimeRegex.finditer(messageHtml)
 746                         timeParts = (group.group(1).strip() for group in timeGroups)
 747
 748                         messageParts = itertools.izip(fromParts, textParts, timeParts)
 749                         messages = [self._interpret_sms_message_parts(*parts) for parts in messageParts]
 750                         conv.messages = messages
 751
 752                         yield conv
 753
 754         @staticmethod
 755         def _merge_conversation_sources(parsedMessages, json):
 756                 for message in parsedMessages:
 757                         jsonItem = json["messages"][message.id]
 758                         message.isRead = jsonItem["isRead"]
 759                         message.isSpam = jsonItem["isSpam"]
 760                         message.isTrash = jsonItem["isTrash"]
 761                         message.isArchived = "inbox" not in jsonItem["labels"]
 762                         yield message
 763
 764         def _get_page(self, url, data = None, refererUrl = None):
 765                 headers = {}
 766                 if refererUrl is not None:
 767                         headers["Referer"] = refererUrl
 768
 769                 encodedData = urllib.urlencode(data) if data is not None else None
 770
 771                 try:
 772                         page = self._browser.download(url, encodedData, None, headers)
 773                 except urllib2.URLError, e:
 774                         _moduleLogger.error("Translating error: %s" % str(e))
 775                         raise NetworkError("%s is not accesible" % url)
 776
 777                 return page
 778
 779         def _get_page_with_token(self, url, data = None, refererUrl = None):
 780                 if data is None:
 781                         data = {}
 782                 data['_rnr_se'] = self._token
 783
 784                 page = self._get_page(url, data, refererUrl)
 785
 786                 return page
 787
 788         def _parse_with_validation(self, page):
 789                 json = parse_json(page)
 790                 validate_response(json)
 791                 return json
 792
 793
 794 _UNESCAPE_ENTITIES = {
 795  "&quot;": '"',
 796  "&nbsp;": " ",
 797  "&#39;": "'",
 798 }
 799
 800
 801 def unescape(text):
 802         plain = saxutils.unescape(text, _UNESCAPE_ENTITIES)
 803         return plain
 804
 805
 806 def google_strptime(time):
 807         """
 808         Hack: Google always returns the time in the same locale.  Sadly if the
 809         local system's locale is different, there isn't a way to perfectly handle
 810         the time.  So instead we handle implement some time formatting
 811         """
 812         abbrevTime = time[:-3]
 813         parsedTime = datetime.datetime.strptime(abbrevTime, "%m/%d/%y %I:%M")
 814         if time.endswith("PM"):
 815                 parsedTime += datetime.timedelta(hours=12)
 816         return parsedTime
 817
 818
 819 def itergroup(iterator, count, padValue = None):
 820         """
 821         Iterate in groups of 'count' values. If there
 822         aren't enough values, the last result is padded with
 823         None.
 824
 825         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 826         ...     print tuple(val)
 827         (1, 2, 3)
 828         (4, 5, 6)
 829         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 830         ...     print list(val)
 831         [1, 2, 3]
 832         [4, 5, 6]
 833         >>> for val in itergroup([1, 2, 3, 4, 5, 6, 7], 3):
 834         ...     print tuple(val)
 835         (1, 2, 3)
 836         (4, 5, 6)
 837         (7, None, None)
 838         >>> for val in itergroup("123456", 3):
 839         ...     print tuple(val)
 840         ('1', '2', '3')
 841         ('4', '5', '6')
 842         >>> for val in itergroup("123456", 3):
 843         ...     print repr("".join(val))
 844         '123'
 845         '456'
 846         """
 847         paddedIterator = itertools.chain(iterator, itertools.repeat(padValue, count-1))
 848         nIterators = (paddedIterator, ) * count
 849         return itertools.izip(*nIterators)
 850
 851
 852 def safe_eval(s):
 853         _TRUE_REGEX = re.compile("true")
 854         _FALSE_REGEX = re.compile("false")
 855         _COMMENT_REGEX = re.compile("^\s+//.*$", re.M)
 856         s = _TRUE_REGEX.sub("True", s)
 857         s = _FALSE_REGEX.sub("False", s)
 858         s = _COMMENT_REGEX.sub("#", s)
 859         try:
 860                 results = eval(s, {}, {})
 861         except SyntaxError:
 862                 _moduleLogger.exception("Oops")
 863                 results = None
 864         return results
 865
 866
 867 def _fake_parse_json(flattened):
 868         return safe_eval(flattened)
 869
 870
 871 def _actual_parse_json(flattened):
 872         return simplejson.loads(flattened)
 873
 874
 875 if simplejson is None:
 876         parse_json = _fake_parse_json
 877 else:
 878         parse_json = _actual_parse_json
 879
 880
 881 def extract_payload(flatXml):
 882         xmlTree = ElementTree.fromstring(flatXml)
 883
 884         jsonElement = xmlTree.getchildren()[0]
 885         flatJson = jsonElement.text
 886         jsonTree = parse_json(flatJson)
 887
 888         htmlElement = xmlTree.getchildren()[1]
 889         flatHtml = htmlElement.text
 890
 891         return jsonTree, flatHtml
 892
 893
 894 def validate_response(response):
 895         """
 896         Validates that the JSON response is A-OK
 897         """
 898         try:
 899                 assert response is not None
 900                 assert 'ok' in response
 901                 assert response['ok']
 902         except AssertionError:
 903                 raise RuntimeError('There was a problem with GV: %s' % response)
 904
 905
 906 def guess_phone_type(number):
 907         if number.startswith("747") or number.startswith("1747") or number.startswith("+1747"):
 908                 return GVoiceBackend.PHONE_TYPE_GIZMO
 909         else:
 910                 return GVoiceBackend.PHONE_TYPE_MOBILE
 911
 912
 913 def get_sane_callback(backend):
 914         """
 915         Try to set a sane default callback number on these preferences
 916         1) 1747 numbers ( Gizmo )
 917         2) anything with gizmo in the name
 918         3) anything with computer in the name
 919         4) the first value
 920         """
 921         numbers = backend.get_callback_numbers()
 922
 923         priorityOrderedCriteria = [
 924                 ("\+1747", None),
 925                 ("1747", None),
 926                 ("747", None),
 927                 (None, "gizmo"),
 928                 (None, "computer"),
 929                 (None, "sip"),
 930                 (None, None),
 931         ]
 932
 933         for numberCriteria, descriptionCriteria in priorityOrderedCriteria:
 934                 numberMatcher = None
 935                 descriptionMatcher = None
 936                 if numberCriteria is not None:
 937                         numberMatcher = re.compile(numberCriteria)
 938                 elif descriptionCriteria is not None:
 939                         descriptionMatcher = re.compile(descriptionCriteria, re.I)
 940
 941                 for number, description in numbers.iteritems():
 942                         if numberMatcher is not None and numberMatcher.match(number) is None:
 943                                 continue
 944                         if descriptionMatcher is not None and descriptionMatcher.match(description) is None:
 945                                 continue
 946                         return number
 947
 948
 949 def set_sane_callback(backend):
 950         """
 951         Try to set a sane default callback number on these preferences
 952         1) 1747 numbers ( Gizmo )
 953         2) anything with gizmo in the name
 954         3) anything with computer in the name
 955         4) the first value
 956         """
 957         number = get_sane_callback(backend)
 958         backend.set_callback_number(number)
 959
 960
 961 def _is_not_special(name):
 962         return not name.startswith("_") and name[0].lower() == name[0] and "_" not in name
 963
 964
 965 def to_dict(obj):
 966         members = inspect.getmembers(obj)
 967         return dict((name, value) for (name, value) in members if _is_not_special(name))
 968
 969
 970 def grab_debug_info(username, password):
 971         cookieFile = os.path.join(".", "raw_cookies.txt")
 972         try:
 973                 os.remove(cookieFile)
 974         except OSError:
 975                 pass
 976
 977         backend = GVoiceBackend(cookieFile)
 978         browser = backend._browser
 979
 980         _TEST_WEBPAGES = [
 981                 ("forward", backend._forwardURL),
 982                 ("token", backend._tokenURL),
 983                 ("login", backend._loginURL),
 984                 ("isdnd", backend._isDndURL),
 985                 ("account", backend._XML_ACCOUNT_URL),
 986                 ("html_contacts", backend._XML_CONTACTS_URL),
 987                 ("contacts", backend._JSON_CONTACTS_URL),
 988                 ("csv", backend._CSV_CONTACTS_URL),
 989
 990                 ("voicemail", backend._XML_VOICEMAIL_URL),
 991                 ("html_sms", backend._XML_SMS_URL),
 992                 ("sms", backend._JSON_SMS_URL),
 993                 ("count", backend._JSON_SMS_COUNT_URL),
 994
 995                 ("recent", backend._XML_RECENT_URL),
 996                 ("placed", backend._XML_PLACED_URL),
 997                 ("recieved", backend._XML_RECEIVED_URL),
 998                 ("missed", backend._XML_MISSED_URL),
 999         ]
1000
1001         # Get Pages
1002         print "Grabbing pre-login pages"
1003         for name, url in _TEST_WEBPAGES:
1004                 try:
1005                         page = browser.download(url)
1006                 except StandardError, e:
1007                         print e.message
1008                         continue
1009                 print "\tWriting to file"
1010                 with open("not_loggedin_%s.txt" % name, "w") as f:
1011                         f.write(page)
1012
1013         # Login
1014         print "Attempting login"
1015         galxToken = backend._get_token()
1016         loginSuccessOrFailurePage = backend._login(username, password, galxToken)
1017         with open("loggingin.txt", "w") as f:
1018                 print "\tWriting to file"
1019                 f.write(loginSuccessOrFailurePage)
1020         try:
1021                 backend._grab_account_info(loginSuccessOrFailurePage)
1022         except Exception:
1023                 # Retry in case the redirect failed
1024                 # luckily is_authed does everything we need for a retry
1025                 loggedIn = backend.is_authed(True)
1026                 if not loggedIn:
1027                         raise
1028
1029         # Get Pages
1030         print "Grabbing post-login pages"
1031         for name, url in _TEST_WEBPAGES:
1032                 try:
1033                         page = browser.download(url)
1034                 except StandardError, e:
1035                         print str(e)
1036                         continue
1037                 print "\tWriting to file"
1038                 with open("loggedin_%s.txt" % name, "w") as f:
1039                         f.write(page)
1040
1041         # Cookies
1042         browser.save_cookies()
1043         print "\tWriting cookies to file"
1044         with open("cookies.txt", "w") as f:
1045                 f.writelines(
1046                         "%s: %s\n" % (c.name, c.value)
1047                         for c in browser._cookies
1048                 )
1049
1050
1051 def main():
1052         import sys
1053         logging.basicConfig(level=logging.DEBUG)
1054         args = sys.argv
1055         if 3 <= len(args):
1056                 username = args[1]
1057                 password = args[2]
1058
1059         grab_debug_info(username, password)
1060
1061
1062 if __name__ == "__main__":
1063         main()