git.maemo.org Git - theonering/blob - src/gvoice/backend.py

   1 #!/usr/bin/python
   2
   3 """
   4 DialCentral - Front end for Google's GoogleVoice service.
   5 Copyright (C) 2008  Eric Warnke ericew AT gmail DOT com
   6
   7 This library is free software; you can redistribute it and/or
   8 modify it under the terms of the GNU Lesser General Public
   9 License as published by the Free Software Foundation; either
  10 version 2.1 of the License, or (at your option) any later version.
  11
  12 This library is distributed in the hope that it will be useful,
  13 but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 Lesser General Public License for more details.
  16
  17 You should have received a copy of the GNU Lesser General Public
  18 License along with this library; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 Google Voice backend code
  22
  23 Resources
  24         http://thatsmith.com/2009/03/google-voice-addon-for-firefox/
  25         http://posttopic.com/topic/google-voice-add-on-development
  26 """
  27
  28 from __future__ import with_statement
  29
  30 import os
  31 import re
  32 import urllib
  33 import urllib2
  34 import time
  35 import datetime
  36 import itertools
  37 import logging
  38 import inspect
  39
  40 from xml.sax import saxutils
  41 from xml.etree import ElementTree
  42
  43 try:
  44         import simplejson as _simplejson
  45         simplejson = _simplejson
  46 except ImportError:
  47         simplejson = None
  48
  49 import browser_emu
  50
  51
  52 _moduleLogger = logging.getLogger(__name__)
  53
  54
  55 class NetworkError(RuntimeError):
  56         pass
  57
  58
  59 class MessageText(object):
  60
  61         ACCURACY_LOW = "med1"
  62         ACCURACY_MEDIUM = "med2"
  63         ACCURACY_HIGH = "high"
  64
  65         def __init__(self):
  66                 self.accuracy = None
  67                 self.text = None
  68
  69         def __str__(self):
  70                 return self.text
  71
  72         def to_dict(self):
  73                 return to_dict(self)
  74
  75         def __eq__(self, other):
  76                 return self.accuracy == other.accuracy and self.text == other.text
  77
  78
  79 class Message(object):
  80
  81         def __init__(self):
  82                 self.whoFrom = None
  83                 self.body = None
  84                 self.when = None
  85
  86         def __str__(self):
  87                 return "%s (%s): %s" % (
  88                         self.whoFrom,
  89                         self.when,
  90                         "".join(unicode(part) for part in self.body)
  91                 )
  92
  93         def to_dict(self):
  94                 selfDict = to_dict(self)
  95                 selfDict["body"] = [text.to_dict() for text in self.body] if self.body is not None else None
  96                 return selfDict
  97
  98         def __eq__(self, other):
  99                 return self.whoFrom == other.whoFrom and self.when == other.when and self.body == other.body
 100
 101
 102 class Conversation(object):
 103
 104         TYPE_VOICEMAIL = "Voicemail"
 105         TYPE_SMS = "SMS"
 106
 107         def __init__(self):
 108                 self.type = None
 109                 self.id = None
 110                 self.contactId = None
 111                 self.name = None
 112                 self.location = None
 113                 self.prettyNumber = None
 114                 self.number = None
 115
 116                 self.time = None
 117                 self.relTime = None
 118                 self.messages = None
 119                 self.isRead = None
 120                 self.isSpam = None
 121                 self.isTrash = None
 122                 self.isArchived = None
 123
 124         def __cmp__(self, other):
 125                 cmpValue = cmp(self.contactId, other.contactId)
 126                 if cmpValue != 0:
 127                         return cmpValue
 128
 129                 cmpValue = cmp(self.time, other.time)
 130                 if cmpValue != 0:
 131                         return cmpValue
 132
 133                 cmpValue = cmp(self.id, other.id)
 134                 if cmpValue != 0:
 135                         return cmpValue
 136
 137         def to_dict(self):
 138                 selfDict = to_dict(self)
 139                 selfDict["messages"] = [message.to_dict() for message in self.messages] if self.messages is not None else None
 140                 return selfDict
 141
 142
 143 class GVoiceBackend(object):
 144         """
 145         This class encapsulates all of the knowledge necessary to interact with the GoogleVoice servers
 146         the functions include login, setting up a callback number, and initalting a callback
 147         """
 148
 149         PHONE_TYPE_HOME = 1
 150         PHONE_TYPE_MOBILE = 2
 151         PHONE_TYPE_WORK = 3
 152         PHONE_TYPE_GIZMO = 7
 153
 154         def __init__(self, cookieFile = None):
 155                 # Important items in this function are the setup of the browser emulation and cookie file
 156                 self._browser = browser_emu.MozillaEmulator(1)
 157                 self._loadedFromCookies = self._browser.load_cookies(cookieFile)
 158
 159                 self._token = ""
 160                 self._accountNum = ""
 161                 self._lastAuthed = 0.0
 162                 self._callbackNumber = ""
 163                 self._callbackNumbers = {}
 164
 165                 # Suprisingly, moving all of these from class to self sped up startup time
 166
 167                 self._validateRe = re.compile("^\+?[0-9]{10,}$")
 168
 169                 self._loginURL = "https://www.google.com/accounts/ServiceLoginAuth"
 170
 171                 SECURE_URL_BASE = "https://www.google.com/voice/"
 172                 SECURE_MOBILE_URL_BASE = SECURE_URL_BASE + "mobile/"
 173                 self._forwardURL = SECURE_MOBILE_URL_BASE + "phones"
 174                 self._tokenURL = SECURE_URL_BASE + "m"
 175                 self._callUrl = SECURE_URL_BASE + "call/connect"
 176                 self._callCancelURL = SECURE_URL_BASE + "call/cancel"
 177                 self._sendSmsURL = SECURE_URL_BASE + "sms/send"
 178
 179                 self._isDndURL = "https://www.google.com/voice/m/donotdisturb"
 180                 self._isDndRe = re.compile(r"""<input.*?id="doNotDisturb".*?checked="(.*?)"\s*/>""")
 181                 self._setDndURL = "https://www.google.com/voice/m/savednd"
 182
 183                 self._downloadVoicemailURL = SECURE_URL_BASE + "media/send_voicemail/"
 184                 self._markAsReadURL = SECURE_URL_BASE + "m/mark"
 185                 self._archiveMessageURL = SECURE_URL_BASE + "m/archive"
 186
 187                 self._XML_SEARCH_URL = SECURE_URL_BASE + "inbox/search/"
 188                 self._XML_ACCOUNT_URL = SECURE_URL_BASE + "contacts/"
 189                 # HACK really this redirects to the main pge and we are grabbing some javascript
 190                 self._XML_CONTACTS_URL = "http://www.google.com/voice/inbox/search/contact"
 191                 self._CSV_CONTACTS_URL = "http://mail.google.com/mail/contacts/data/export"
 192                 self._JSON_CONTACTS_URL = SECURE_URL_BASE + "b/0/request/user"
 193                 self._XML_RECENT_URL = SECURE_URL_BASE + "inbox/recent/"
 194
 195                 self.XML_FEEDS = (
 196                         'inbox', 'starred', 'all', 'spam', 'trash', 'voicemail', 'sms',
 197                         'recorded', 'placed', 'received', 'missed'
 198                 )
 199                 self._XML_INBOX_URL = SECURE_URL_BASE + "inbox/recent/inbox"
 200                 self._XML_STARRED_URL = SECURE_URL_BASE + "inbox/recent/starred"
 201                 self._XML_ALL_URL = SECURE_URL_BASE + "inbox/recent/all"
 202                 self._XML_SPAM_URL = SECURE_URL_BASE + "inbox/recent/spam"
 203                 self._XML_TRASH_URL = SECURE_URL_BASE + "inbox/recent/trash"
 204                 self._XML_VOICEMAIL_URL = SECURE_URL_BASE + "inbox/recent/voicemail/"
 205                 self._XML_SMS_URL = SECURE_URL_BASE + "inbox/recent/sms/"
 206                 self._JSON_SMS_URL = SECURE_URL_BASE + "b/0/request/messages/"
 207                 self._JSON_SMS_COUNT_URL = SECURE_URL_BASE + "b/0/request/unread"
 208                 self._XML_RECORDED_URL = SECURE_URL_BASE + "inbox/recent/recorded/"
 209                 self._XML_PLACED_URL = SECURE_URL_BASE + "inbox/recent/placed/"
 210                 self._XML_RECEIVED_URL = SECURE_URL_BASE + "inbox/recent/received/"
 211                 self._XML_MISSED_URL = SECURE_URL_BASE + "inbox/recent/missed/"
 212
 213                 self._galxRe = re.compile(r"""<input.*?name="GALX".*?value="(.*?)".*?/>""", re.MULTILINE | re.DOTALL)
 214                 self._tokenRe = re.compile(r"""<input.*?name="_rnr_se".*?value="(.*?)"\s*/>""")
 215                 self._accountNumRe = re.compile(r"""<b class="ms\d">(.{14})</b></div>""")
 216                 self._callbackRe = re.compile(r"""\s+(.*?):\s*(.*?)<br\s*/>\s*$""", re.M)
 217
 218                 self._seperateVoicemailsRegex = re.compile(r"""^\s*<div id="(\w+)"\s* class=".*?gc-message.*?">""", re.MULTILINE | re.DOTALL)
 219                 self._exactVoicemailTimeRegex = re.compile(r"""<span class="gc-message-time">(.*?)</span>""", re.MULTILINE)
 220                 self._relativeVoicemailTimeRegex = re.compile(r"""<span class="gc-message-relative">(.*?)</span>""", re.MULTILINE)
 221                 self._voicemailNameRegex = re.compile(r"""<a class=.*?gc-message-name-link.*?>(.*?)</a>""", re.MULTILINE | re.DOTALL)
 222                 self._voicemailNumberRegex = re.compile(r"""<input type="hidden" class="gc-text gc-quickcall-ac" value="(.*?)"/>""", re.MULTILINE)
 223                 self._prettyVoicemailNumberRegex = re.compile(r"""<span class="gc-message-type">(.*?)</span>""", re.MULTILINE)
 224                 self._voicemailLocationRegex = re.compile(r"""<span class="gc-message-location">.*?<a.*?>(.*?)</a></span>""", re.MULTILINE)
 225                 self._messagesContactIDRegex = re.compile(r"""<a class=".*?gc-message-name-link.*?">.*?</a>\s*?<span .*?>(.*?)</span>""", re.MULTILINE)
 226                 self._voicemailMessageRegex = re.compile(r"""(<span id="\d+-\d+" class="gc-word-(.*?)">(.*?)</span>|<a .*? class="gc-message-mni">(.*?)</a>)""", re.MULTILINE)
 227                 self._smsFromRegex = re.compile(r"""<span class="gc-message-sms-from">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 228                 self._smsTimeRegex = re.compile(r"""<span class="gc-message-sms-time">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 229                 self._smsTextRegex = re.compile(r"""<span class="gc-message-sms-text">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 230
 231         def is_quick_login_possible(self):
 232                 """
 233                 @returns True then is_authed might be enough to login, else full login is required
 234                 """
 235                 return self._loadedFromCookies or 0.0 < self._lastAuthed
 236
 237         def is_authed(self, force = False):
 238                 """
 239                 Attempts to detect a current session
 240                 @note Once logged in try not to reauth more than once a minute.
 241                 @returns If authenticated
 242                 @blocks
 243                 """
 244                 isRecentledAuthed = (time.time() - self._lastAuthed) < 120
 245                 isPreviouslyAuthed = self._token is not None
 246                 if isRecentledAuthed and isPreviouslyAuthed and not force:
 247                         return True
 248
 249                 try:
 250                         page = self._get_page(self._forwardURL)
 251                         self._grab_account_info(page)
 252                 except Exception, e:
 253                         _moduleLogger.exception(str(e))
 254                         return False
 255
 256                 self._browser.save_cookies()
 257                 self._lastAuthed = time.time()
 258                 return True
 259
 260         def _get_token(self):
 261                 tokenPage = self._get_page(self._tokenURL)
 262
 263                 galxTokens = self._galxRe.search(tokenPage)
 264                 if galxTokens is not None:
 265                         galxToken = galxTokens.group(1)
 266                 else:
 267                         galxToken = ""
 268                         _moduleLogger.debug("Could not grab GALX token")
 269                 return galxToken
 270
 271         def _login(self, username, password, token):
 272                 loginData = {
 273                         'Email' : username,
 274                         'Passwd' : password,
 275                         'service': "grandcentral",
 276                         "ltmpl": "mobile",
 277                         "btmpl": "mobile",
 278                         "PersistentCookie": "yes",
 279                         "GALX": token,
 280                         "continue": self._forwardURL,
 281                 }
 282
 283                 loginSuccessOrFailurePage = self._get_page(self._loginURL, loginData)
 284                 return loginSuccessOrFailurePage
 285
 286         def login(self, username, password):
 287                 """
 288                 Attempt to login to GoogleVoice
 289                 @returns Whether login was successful or not
 290                 @blocks
 291                 """
 292                 self.logout()
 293                 galxToken = self._get_token()
 294                 loginSuccessOrFailurePage = self._login(username, password, galxToken)
 295
 296                 try:
 297                         self._grab_account_info(loginSuccessOrFailurePage)
 298                 except Exception, e:
 299                         # Retry in case the redirect failed
 300                         # luckily is_authed does everything we need for a retry
 301                         loggedIn = self.is_authed(True)
 302                         if not loggedIn:
 303                                 _moduleLogger.exception(str(e))
 304                                 return False
 305                         _moduleLogger.info("Redirection failed on initial login attempt, auto-corrected for this")
 306
 307                 self._browser.save_cookies()
 308                 self._lastAuthed = time.time()
 309                 return True
 310
 311         def persist(self):
 312                 self._browser.save_cookies()
 313
 314         def shutdown(self):
 315                 self._browser.save_cookies()
 316                 self._token = None
 317                 self._lastAuthed = 0.0
 318
 319         def logout(self):
 320                 self._browser.clear_cookies()
 321                 self._browser.save_cookies()
 322                 self._token = None
 323                 self._lastAuthed = 0.0
 324
 325         def is_dnd(self):
 326                 """
 327                 @blocks
 328                 """
 329                 isDndPage = self._get_page(self._isDndURL)
 330
 331                 dndGroup = self._isDndRe.search(isDndPage)
 332                 if dndGroup is None:
 333                         return False
 334                 dndStatus = dndGroup.group(1)
 335                 isDnd = True if dndStatus.strip().lower() == "true" else False
 336                 return isDnd
 337
 338         def set_dnd(self, doNotDisturb):
 339                 """
 340                 @blocks
 341                 """
 342                 dndPostData = {
 343                         "doNotDisturb": 1 if doNotDisturb else 0,
 344                 }
 345
 346                 dndPage = self._get_page_with_token(self._setDndURL, dndPostData)
 347
 348         def call(self, outgoingNumber):
 349                 """
 350                 This is the main function responsible for initating the callback
 351                 @blocks
 352                 """
 353                 outgoingNumber = self._send_validation(outgoingNumber)
 354                 subscriberNumber = None
 355                 phoneType = guess_phone_type(self._callbackNumber) # @todo Fix this hack
 356
 357                 callData = {
 358                                 'outgoingNumber': outgoingNumber,
 359                                 'forwardingNumber': self._callbackNumber,
 360                                 'subscriberNumber': subscriberNumber or 'undefined',
 361                                 'phoneType': str(phoneType),
 362                                 'remember': '1',
 363                 }
 364                 _moduleLogger.info("%r" % callData)
 365
 366                 page = self._get_page_with_token(
 367                         self._callUrl,
 368                         callData,
 369                 )
 370                 self._parse_with_validation(page)
 371                 return True
 372
 373         def cancel(self, outgoingNumber=None):
 374                 """
 375                 Cancels a call matching outgoing and forwarding numbers (if given).
 376                 Will raise an error if no matching call is being placed
 377                 @blocks
 378                 """
 379                 page = self._get_page_with_token(
 380                         self._callCancelURL,
 381                         {
 382                         'outgoingNumber': outgoingNumber or 'undefined',
 383                         'forwardingNumber': self._callbackNumber or 'undefined',
 384                         'cancelType': 'C2C',
 385                         },
 386                 )
 387                 self._parse_with_validation(page)
 388
 389         def send_sms(self, phoneNumbers, message):
 390                 """
 391                 @blocks
 392                 """
 393                 validatedPhoneNumbers = [
 394                         self._send_validation(phoneNumber)
 395                         for phoneNumber in phoneNumbers
 396                 ]
 397                 flattenedPhoneNumbers = ",".join(validatedPhoneNumbers)
 398                 page = self._get_page_with_token(
 399                         self._sendSmsURL,
 400                         {
 401                                 'phoneNumber': flattenedPhoneNumbers,
 402                                 'text': unicode(message).encode("utf-8"),
 403                         },
 404                 )
 405                 self._parse_with_validation(page)
 406
 407         def search(self, query):
 408                 """
 409                 Search your Google Voice Account history for calls, voicemails, and sms
 410                 Returns ``Folder`` instance containting matching messages
 411                 @blocks
 412                 """
 413                 page = self._get_page(
 414                         self._XML_SEARCH_URL,
 415                         {"q": query},
 416                 )
 417                 json, html = extract_payload(page)
 418                 return json
 419
 420         def get_feed(self, feed):
 421                 """
 422                 @blocks
 423                 """
 424                 actualFeed = "_XML_%s_URL" % feed.upper()
 425                 feedUrl = getattr(self, actualFeed)
 426
 427                 page = self._get_page(feedUrl)
 428                 json, html = extract_payload(page)
 429
 430                 return json
 431
 432         def download(self, messageId, adir):
 433                 """
 434                 Download a voicemail or recorded call MP3 matching the given ``msg``
 435                 which can either be a ``Message`` instance, or a SHA1 identifier.
 436                 Saves files to ``adir`` (defaults to current directory).
 437                 Message hashes can be found in ``self.voicemail().messages`` for example.
 438                 @returns location of saved file.
 439                 @blocks
 440                 """
 441                 page = self._get_page(self._downloadVoicemailURL, {"id": messageId})
 442                 fn = os.path.join(adir, '%s.mp3' % messageId)
 443                 with open(fn, 'wb') as fo:
 444                         fo.write(page)
 445                 return fn
 446
 447         def is_valid_syntax(self, number):
 448                 """
 449                 @returns If This number be called ( syntax validation only )
 450                 """
 451                 return self._validateRe.match(number) is not None
 452
 453         def get_account_number(self):
 454                 """
 455                 @returns The GoogleVoice phone number
 456                 """
 457                 return self._accountNum
 458
 459         def get_callback_numbers(self):
 460                 """
 461                 @returns a dictionary mapping call back numbers to descriptions
 462                 @note These results are cached for 30 minutes.
 463                 """
 464                 if not self.is_authed():
 465                         return {}
 466                 return self._callbackNumbers
 467
 468         def set_callback_number(self, callbacknumber):
 469                 """
 470                 Set the number that GoogleVoice calls
 471                 @param callbacknumber should be a proper 10 digit number
 472                 """
 473                 self._callbackNumber = callbacknumber
 474                 _moduleLogger.info("Callback number changed: %r" % self._callbackNumber)
 475                 return True
 476
 477         def get_callback_number(self):
 478                 """
 479                 @returns Current callback number or None
 480                 """
 481                 return self._callbackNumber
 482
 483         def get_recent(self):
 484                 """
 485                 @returns Iterable of (personsName, phoneNumber, exact date, relative date, action)
 486                 @blocks
 487                 """
 488                 recentPages = [
 489                         (action, self._get_page(url))
 490                         for action, url in (
 491                                 ("Received", self._XML_RECEIVED_URL),
 492                                 ("Missed", self._XML_MISSED_URL),
 493                                 ("Placed", self._XML_PLACED_URL),
 494                         )
 495                 ]
 496                 return self._parse_recent(recentPages)
 497
 498         def get_contacts(self):
 499                 """
 500                 @returns Iterable of (contact id, contact name)
 501                 @blocks
 502                 """
 503                 page = self._get_page(self._JSON_CONTACTS_URL)
 504                 return self._process_contacts(page)
 505
 506         def get_csv_contacts(self):
 507                 data = {
 508                         "groupToExport": "mine",
 509                         "exportType": "ALL",
 510                         "out": "OUTLOOK_CSV",
 511                 }
 512                 encodedData = urllib.urlencode(data)
 513                 contacts = self._get_page(self._CSV_CONTACTS_URL+"?"+encodedData)
 514                 return contacts
 515
 516         def get_voicemails(self):
 517                 """
 518                 @blocks
 519                 """
 520                 voicemailPage = self._get_page(self._XML_VOICEMAIL_URL)
 521                 voicemailHtml = self._grab_html(voicemailPage)
 522                 voicemailJson = self._grab_json(voicemailPage)
 523                 if voicemailJson is None:
 524                         return ()
 525                 parsedVoicemail = self._parse_voicemail(voicemailHtml)
 526                 voicemails = self._merge_conversation_sources(parsedVoicemail, voicemailJson)
 527                 return voicemails
 528
 529         def get_texts(self):
 530                 """
 531                 @blocks
 532                 """
 533                 smsPage = self._get_page(self._XML_SMS_URL)
 534                 smsHtml = self._grab_html(smsPage)
 535                 smsJson = self._grab_json(smsPage)
 536                 if smsJson is None:
 537                         return ()
 538                 parsedSms = self._parse_sms(smsHtml)
 539                 smss = self._merge_conversation_sources(parsedSms, smsJson)
 540                 return smss
 541
 542         def get_unread_counts(self):
 543                 countPage = self._get_page(self._JSON_SMS_COUNT_URL)
 544                 counts = parse_json(countPage)
 545                 counts = counts["unreadCounts"]
 546                 return counts
 547
 548         def mark_message(self, messageId, asRead):
 549                 """
 550                 @blocks
 551                 """
 552                 postData = {
 553                         "read": 1 if asRead else 0,
 554                         "id": messageId,
 555                 }
 556
 557                 markPage = self._get_page(self._markAsReadURL, postData)
 558
 559         def archive_message(self, messageId):
 560                 """
 561                 @blocks
 562                 """
 563                 postData = {
 564                         "id": messageId,
 565                 }
 566
 567                 markPage = self._get_page(self._archiveMessageURL, postData)
 568
 569         def _grab_json(self, flatXml):
 570                 xmlTree = ElementTree.fromstring(flatXml)
 571                 jsonElement = xmlTree.getchildren()[0]
 572                 flatJson = jsonElement.text
 573                 jsonTree = parse_json(flatJson)
 574                 return jsonTree
 575
 576         def _grab_html(self, flatXml):
 577                 xmlTree = ElementTree.fromstring(flatXml)
 578                 htmlElement = xmlTree.getchildren()[1]
 579                 flatHtml = htmlElement.text
 580                 return flatHtml
 581
 582         def _grab_account_info(self, page):
 583                 tokenGroup = self._tokenRe.search(page)
 584                 if tokenGroup is None:
 585                         raise RuntimeError("Could not extract authentication token from GoogleVoice")
 586                 self._token = tokenGroup.group(1)
 587
 588                 anGroup = self._accountNumRe.search(page)
 589                 if anGroup is not None:
 590                         self._accountNum = anGroup.group(1)
 591                 else:
 592                         _moduleLogger.debug("Could not extract account number from GoogleVoice")
 593
 594                 self._callbackNumbers = {}
 595                 for match in self._callbackRe.finditer(page):
 596                         callbackNumber = match.group(2)
 597                         callbackName = match.group(1)
 598                         self._callbackNumbers[callbackNumber] = callbackName
 599                 if len(self._callbackNumbers) == 0:
 600                         _moduleLogger.debug("Could not extract callback numbers from GoogleVoice (the troublesome page follows):\n%s" % page)
 601
 602         def _send_validation(self, number):
 603                 if not self.is_valid_syntax(number):
 604                         raise ValueError('Number is not valid: "%s"' % number)
 605                 elif not self.is_authed():
 606                         raise RuntimeError("Not Authenticated")
 607                 return number
 608
 609         def _parse_recent(self, recentPages):
 610                 for action, flatXml in recentPages:
 611                         allRecentHtml = self._grab_html(flatXml)
 612                         allRecentData = self._parse_history(allRecentHtml)
 613                         for recentCallData in allRecentData:
 614                                 recentCallData["action"] = action
 615                                 yield recentCallData
 616
 617         def _process_contacts(self, page):
 618                 accountData = parse_json(page)
 619                 for contactId, contactDetails in accountData["contacts"].iteritems():
 620                         # A zero contact id is the catch all for unknown contacts
 621                         if contactId != "0":
 622                                 yield contactId, contactDetails
 623
 624         def _parse_history(self, historyHtml):
 625                 splitVoicemail = self._seperateVoicemailsRegex.split(historyHtml)
 626                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 627                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 628                         exactTime = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 629                         exactTime = google_strptime(exactTime)
 630                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 631                         relativeTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 632                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 633                         location = locationGroup.group(1).strip() if locationGroup else ""
 634
 635                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 636                         name = nameGroup.group(1).strip() if nameGroup else ""
 637                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 638                         number = numberGroup.group(1).strip() if numberGroup else ""
 639                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 640                         prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 641                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 642                         contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 643
 644                         yield {
 645                                 "id": messageId.strip(),
 646                                 "contactId": contactId,
 647                                 "name": unescape(name),
 648                                 "time": exactTime,
 649                                 "relTime": relativeTime,
 650                                 "prettyNumber": prettyNumber,
 651                                 "number": number,
 652                                 "location": unescape(location),
 653                         }
 654
 655         @staticmethod
 656         def _interpret_voicemail_regex(group):
 657                 quality, content, number = group.group(2), group.group(3), group.group(4)
 658                 text = MessageText()
 659                 if quality is not None and content is not None:
 660                         text.accuracy = quality
 661                         text.text = content
 662                         return text
 663                 elif number is not None:
 664                         text.accuracy = MessageText.ACCURACY_HIGH
 665                         text.text = number
 666                         return text
 667
 668         def _parse_voicemail(self, voicemailHtml):
 669                 splitVoicemail = self._seperateVoicemailsRegex.split(voicemailHtml)
 670                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 671                         conv = Conversation()
 672                         conv.type = Conversation.TYPE_VOICEMAIL
 673                         conv.id = messageId.strip()
 674
 675                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 676                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 677                         conv.time = google_strptime(exactTimeText)
 678                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 679                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 680                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 681                         conv.location = unescape(locationGroup.group(1).strip() if locationGroup else "")
 682
 683                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 684                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 685                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 686                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 687                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 688                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 689                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 690                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 691
 692                         messageGroups = self._voicemailMessageRegex.finditer(messageHtml)
 693                         messageParts = [
 694                                 self._interpret_voicemail_regex(group)
 695                                 for group in messageGroups
 696                         ] if messageGroups else ((MessageText.ACCURACY_LOW, "No Transcription"), )
 697                         message = Message()
 698                         message.body = messageParts
 699                         message.whoFrom = conv.name
 700                         try:
 701                                 message.when = conv.time.strftime("%I:%M %p")
 702                         except ValueError:
 703                                 _moduleLogger.exception("Confusing time provided: %r" % conv.time)
 704                                 message.when = "Unknown"
 705                         conv.messages = (message, )
 706
 707                         yield conv
 708
 709         @staticmethod
 710         def _interpret_sms_message_parts(fromPart, textPart, timePart):
 711                 text = MessageText()
 712                 text.accuracy = MessageText.ACCURACY_MEDIUM
 713                 text.text = textPart
 714
 715                 message = Message()
 716                 message.body = (text, )
 717                 message.whoFrom = fromPart
 718                 message.when = timePart
 719
 720                 return message
 721
 722         def _parse_sms(self, smsHtml):
 723                 splitSms = self._seperateVoicemailsRegex.split(smsHtml)
 724                 for messageId, messageHtml in itergroup(splitSms[1:], 2):
 725                         conv = Conversation()
 726                         conv.type = Conversation.TYPE_SMS
 727                         conv.id = messageId.strip()
 728
 729                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 730                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 731                         conv.time = google_strptime(exactTimeText)
 732                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 733                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 734                         conv.location = ""
 735
 736                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 737                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 738                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 739                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 740                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 741                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 742                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 743                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 744
 745                         fromGroups = self._smsFromRegex.finditer(messageHtml)
 746                         fromParts = (group.group(1).strip() for group in fromGroups)
 747                         textGroups = self._smsTextRegex.finditer(messageHtml)
 748                         textParts = (group.group(1).strip() for group in textGroups)
 749                         timeGroups = self._smsTimeRegex.finditer(messageHtml)
 750                         timeParts = (group.group(1).strip() for group in timeGroups)
 751
 752                         messageParts = itertools.izip(fromParts, textParts, timeParts)
 753                         messages = [self._interpret_sms_message_parts(*parts) for parts in messageParts]
 754                         conv.messages = messages
 755
 756                         yield conv
 757
 758         @staticmethod
 759         def _merge_conversation_sources(parsedMessages, json):
 760                 for message in parsedMessages:
 761                         jsonItem = json["messages"][message.id]
 762                         message.isRead = jsonItem["isRead"]
 763                         message.isSpam = jsonItem["isSpam"]
 764                         message.isTrash = jsonItem["isTrash"]
 765                         message.isArchived = "inbox" not in jsonItem["labels"]
 766                         yield message
 767
 768         def _get_page(self, url, data = None, refererUrl = None):
 769                 headers = {}
 770                 if refererUrl is not None:
 771                         headers["Referer"] = refererUrl
 772
 773                 encodedData = urllib.urlencode(data) if data is not None else None
 774
 775                 try:
 776                         page = self._browser.download(url, encodedData, None, headers)
 777                 except urllib2.URLError, e:
 778                         _moduleLogger.error("Translating error: %s" % str(e))
 779                         raise NetworkError("%s is not accesible" % url)
 780
 781                 return page
 782
 783         def _get_page_with_token(self, url, data = None, refererUrl = None):
 784                 if data is None:
 785                         data = {}
 786                 data['_rnr_se'] = self._token
 787
 788                 page = self._get_page(url, data, refererUrl)
 789
 790                 return page
 791
 792         def _parse_with_validation(self, page):
 793                 json = parse_json(page)
 794                 self._validate_response(json)
 795                 return json
 796
 797         def _validate_response(self, response):
 798                 """
 799                 Validates that the JSON response is A-OK
 800                 """
 801                 try:
 802                         assert response is not None, "Response not provided"
 803                         assert 'ok' in response, "Response lacks status"
 804                         assert response['ok'], "Response not good"
 805                 except AssertionError:
 806                         try:
 807                                 if response["data"]["code"] == 20:
 808                                         raise RuntimeError(
 809 """Ambiguous error 20 returned by Google Voice.
 810 Please verify you have configured your callback number (currently "%s").  If it is configured some other suspected causes are: non-verified callback numbers, and Gizmo5 callback numbers.""" % self._callbackNumber)
 811                         except KeyError:
 812                                 pass
 813                         raise RuntimeError('There was a problem with GV: %s' % response)
 814
 815
 816 _UNESCAPE_ENTITIES = {
 817  "&quot;": '"',
 818  "&nbsp;": " ",
 819  "&#39;": "'",
 820 }
 821
 822
 823 def unescape(text):
 824         plain = saxutils.unescape(text, _UNESCAPE_ENTITIES)
 825         return plain
 826
 827
 828 def google_strptime(time):
 829         """
 830         Hack: Google always returns the time in the same locale.  Sadly if the
 831         local system's locale is different, there isn't a way to perfectly handle
 832         the time.  So instead we handle implement some time formatting
 833         """
 834         abbrevTime = time[:-3]
 835         parsedTime = datetime.datetime.strptime(abbrevTime, "%m/%d/%y %I:%M")
 836         if time.endswith("PM"):
 837                 parsedTime += datetime.timedelta(hours=12)
 838         return parsedTime
 839
 840
 841 def itergroup(iterator, count, padValue = None):
 842         """
 843         Iterate in groups of 'count' values. If there
 844         aren't enough values, the last result is padded with
 845         None.
 846
 847         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 848         ...     print tuple(val)
 849         (1, 2, 3)
 850         (4, 5, 6)
 851         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 852         ...     print list(val)
 853         [1, 2, 3]
 854         [4, 5, 6]
 855         >>> for val in itergroup([1, 2, 3, 4, 5, 6, 7], 3):
 856         ...     print tuple(val)
 857         (1, 2, 3)
 858         (4, 5, 6)
 859         (7, None, None)
 860         >>> for val in itergroup("123456", 3):
 861         ...     print tuple(val)
 862         ('1', '2', '3')
 863         ('4', '5', '6')
 864         >>> for val in itergroup("123456", 3):
 865         ...     print repr("".join(val))
 866         '123'
 867         '456'
 868         """
 869         paddedIterator = itertools.chain(iterator, itertools.repeat(padValue, count-1))
 870         nIterators = (paddedIterator, ) * count
 871         return itertools.izip(*nIterators)
 872
 873
 874 def safe_eval(s):
 875         _TRUE_REGEX = re.compile("true")
 876         _FALSE_REGEX = re.compile("false")
 877         _COMMENT_REGEX = re.compile("^\s+//.*$", re.M)
 878         s = _TRUE_REGEX.sub("True", s)
 879         s = _FALSE_REGEX.sub("False", s)
 880         s = _COMMENT_REGEX.sub("#", s)
 881         try:
 882                 results = eval(s, {}, {})
 883         except SyntaxError:
 884                 _moduleLogger.exception("Oops")
 885                 results = None
 886         return results
 887
 888
 889 def _fake_parse_json(flattened):
 890         return safe_eval(flattened)
 891
 892
 893 def _actual_parse_json(flattened):
 894         return simplejson.loads(flattened)
 895
 896
 897 if simplejson is None:
 898         parse_json = _fake_parse_json
 899 else:
 900         parse_json = _actual_parse_json
 901
 902
 903 def extract_payload(flatXml):
 904         xmlTree = ElementTree.fromstring(flatXml)
 905
 906         jsonElement = xmlTree.getchildren()[0]
 907         flatJson = jsonElement.text
 908         jsonTree = parse_json(flatJson)
 909
 910         htmlElement = xmlTree.getchildren()[1]
 911         flatHtml = htmlElement.text
 912
 913         return jsonTree, flatHtml
 914
 915
 916 def guess_phone_type(number):
 917         if number.startswith("747") or number.startswith("1747") or number.startswith("+1747"):
 918                 return GVoiceBackend.PHONE_TYPE_GIZMO
 919         else:
 920                 return GVoiceBackend.PHONE_TYPE_MOBILE
 921
 922
 923 def get_sane_callback(backend):
 924         """
 925         Try to set a sane default callback number on these preferences
 926         1) 1747 numbers ( Gizmo )
 927         2) anything with gizmo in the name
 928         3) anything with computer in the name
 929         4) the first value
 930         """
 931         numbers = backend.get_callback_numbers()
 932
 933         priorityOrderedCriteria = [
 934                 ("\+1747", None),
 935                 ("1747", None),
 936                 ("747", None),
 937                 (None, "gizmo"),
 938                 (None, "computer"),
 939                 (None, "sip"),
 940                 (None, None),
 941         ]
 942
 943         for numberCriteria, descriptionCriteria in priorityOrderedCriteria:
 944                 numberMatcher = None
 945                 descriptionMatcher = None
 946                 if numberCriteria is not None:
 947                         numberMatcher = re.compile(numberCriteria)
 948                 elif descriptionCriteria is not None:
 949                         descriptionMatcher = re.compile(descriptionCriteria, re.I)
 950
 951                 for number, description in numbers.iteritems():
 952                         if numberMatcher is not None and numberMatcher.match(number) is None:
 953                                 continue
 954                         if descriptionMatcher is not None and descriptionMatcher.match(description) is None:
 955                                 continue
 956                         return number
 957
 958
 959 def set_sane_callback(backend):
 960         """
 961         Try to set a sane default callback number on these preferences
 962         1) 1747 numbers ( Gizmo )
 963         2) anything with gizmo in the name
 964         3) anything with computer in the name
 965         4) the first value
 966         """
 967         number = get_sane_callback(backend)
 968         backend.set_callback_number(number)
 969
 970
 971 def _is_not_special(name):
 972         return not name.startswith("_") and name[0].lower() == name[0] and "_" not in name
 973
 974
 975 def to_dict(obj):
 976         members = inspect.getmembers(obj)
 977         return dict((name, value) for (name, value) in members if _is_not_special(name))
 978
 979
 980 def grab_debug_info(username, password):
 981         cookieFile = os.path.join(".", "raw_cookies.txt")
 982         try:
 983                 os.remove(cookieFile)
 984         except OSError:
 985                 pass
 986
 987         backend = GVoiceBackend(cookieFile)
 988         browser = backend._browser
 989
 990         _TEST_WEBPAGES = [
 991                 ("forward", backend._forwardURL),
 992                 ("token", backend._tokenURL),
 993                 ("login", backend._loginURL),
 994                 ("isdnd", backend._isDndURL),
 995                 ("account", backend._XML_ACCOUNT_URL),
 996                 ("html_contacts", backend._XML_CONTACTS_URL),
 997                 ("contacts", backend._JSON_CONTACTS_URL),
 998                 ("csv", backend._CSV_CONTACTS_URL),
 999
1000                 ("voicemail", backend._XML_VOICEMAIL_URL),
1001                 ("html_sms", backend._XML_SMS_URL),
1002                 ("sms", backend._JSON_SMS_URL),
1003                 ("count", backend._JSON_SMS_COUNT_URL),
1004
1005                 ("recent", backend._XML_RECENT_URL),
1006                 ("placed", backend._XML_PLACED_URL),
1007                 ("recieved", backend._XML_RECEIVED_URL),
1008                 ("missed", backend._XML_MISSED_URL),
1009         ]
1010
1011         # Get Pages
1012         print "Grabbing pre-login pages"
1013         for name, url in _TEST_WEBPAGES:
1014                 try:
1015                         page = browser.download(url)
1016                 except StandardError, e:
1017                         print e.message
1018                         continue
1019                 print "\tWriting to file"
1020                 with open("not_loggedin_%s.txt" % name, "w") as f:
1021                         f.write(page)
1022
1023         # Login
1024         print "Attempting login"
1025         galxToken = backend._get_token()
1026         loginSuccessOrFailurePage = backend._login(username, password, galxToken)
1027         with open("loggingin.txt", "w") as f:
1028                 print "\tWriting to file"
1029                 f.write(loginSuccessOrFailurePage)
1030         try:
1031                 backend._grab_account_info(loginSuccessOrFailurePage)
1032         except Exception:
1033                 # Retry in case the redirect failed
1034                 # luckily is_authed does everything we need for a retry
1035                 loggedIn = backend.is_authed(True)
1036                 if not loggedIn:
1037                         raise
1038
1039         # Get Pages
1040         print "Grabbing post-login pages"
1041         for name, url in _TEST_WEBPAGES:
1042                 try:
1043                         page = browser.download(url)
1044                 except StandardError, e:
1045                         print str(e)
1046                         continue
1047                 print "\tWriting to file"
1048                 with open("loggedin_%s.txt" % name, "w") as f:
1049                         f.write(page)
1050
1051         # Cookies
1052         browser.save_cookies()
1053         print "\tWriting cookies to file"
1054         with open("cookies.txt", "w") as f:
1055                 f.writelines(
1056                         "%s: %s\n" % (c.name, c.value)
1057                         for c in browser._cookies
1058                 )
1059
1060
1061 def main():
1062         import sys
1063         logging.basicConfig(level=logging.DEBUG)
1064         args = sys.argv
1065         if 3 <= len(args):
1066                 username = args[1]
1067                 password = args[2]
1068
1069         grab_debug_info(username, password)
1070
1071
1072 if __name__ == "__main__":
1073         main()