git.maemo.org Git - gc-dialer/blob - src/backends/gvoice/gvoice.py

   1 #!/usr/bin/python
   2
   3 """
   4 DialCentral - Front end for Google's GoogleVoice service.
   5 Copyright (C) 2008  Eric Warnke ericew AT gmail DOT com
   6
   7 This library is free software; you can redistribute it and/or
   8 modify it under the terms of the GNU Lesser General Public
   9 License as published by the Free Software Foundation; either
  10 version 2.1 of the License, or (at your option) any later version.
  11
  12 This library is distributed in the hope that it will be useful,
  13 but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 Lesser General Public License for more details.
  16
  17 You should have received a copy of the GNU Lesser General Public
  18 License along with this library; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 Google Voice backend code
  22
  23 Resources
  24         http://thatsmith.com/2009/03/google-voice-addon-for-firefox/
  25         http://posttopic.com/topic/google-voice-add-on-development
  26 """
  27
  28 from __future__ import with_statement
  29
  30 import os
  31 import re
  32 import urllib
  33 import urllib2
  34 import time
  35 import datetime
  36 import itertools
  37 import logging
  38 import inspect
  39
  40 from xml.sax import saxutils
  41 from xml.etree import ElementTree
  42
  43 try:
  44         import simplejson as _simplejson
  45         simplejson = _simplejson
  46 except ImportError:
  47         simplejson = None
  48
  49 import browser_emu
  50
  51
  52 _moduleLogger = logging.getLogger(__name__)
  53
  54
  55 class NetworkError(RuntimeError):
  56         pass
  57
  58
  59 class MessageText(object):
  60
  61         ACCURACY_LOW = "med1"
  62         ACCURACY_MEDIUM = "med2"
  63         ACCURACY_HIGH = "high"
  64
  65         def __init__(self):
  66                 self.accuracy = None
  67                 self.text = None
  68
  69         def __str__(self):
  70                 return self.text
  71
  72         def to_dict(self):
  73                 return to_dict(self)
  74
  75         def __eq__(self, other):
  76                 return self.accuracy == other.accuracy and self.text == other.text
  77
  78
  79 class Message(object):
  80
  81         def __init__(self):
  82                 self.whoFrom = None
  83                 self.body = None
  84                 self.when = None
  85
  86         def __str__(self):
  87                 return "%s (%s): %s" % (
  88                         self.whoFrom,
  89                         self.when,
  90                         "".join(unicode(part) for part in self.body)
  91                 )
  92
  93         def to_dict(self):
  94                 selfDict = to_dict(self)
  95                 selfDict["body"] = [text.to_dict() for text in self.body] if self.body is not None else None
  96                 return selfDict
  97
  98         def __eq__(self, other):
  99                 return self.whoFrom == other.whoFrom and self.when == other.when and self.body == other.body
 100
 101
 102 class Conversation(object):
 103
 104         TYPE_VOICEMAIL = "Voicemail"
 105         TYPE_SMS = "SMS"
 106
 107         def __init__(self):
 108                 self.type = None
 109                 self.id = None
 110                 self.contactId = None
 111                 self.name = None
 112                 self.location = None
 113                 self.prettyNumber = None
 114                 self.number = None
 115
 116                 self.time = None
 117                 self.relTime = None
 118                 self.messages = None
 119                 self.isRead = None
 120                 self.isSpam = None
 121                 self.isTrash = None
 122                 self.isArchived = None
 123
 124         def __cmp__(self, other):
 125                 cmpValue = cmp(self.contactId, other.contactId)
 126                 if cmpValue != 0:
 127                         return cmpValue
 128
 129                 cmpValue = cmp(self.time, other.time)
 130                 if cmpValue != 0:
 131                         return cmpValue
 132
 133                 cmpValue = cmp(self.id, other.id)
 134                 if cmpValue != 0:
 135                         return cmpValue
 136
 137         def to_dict(self):
 138                 selfDict = to_dict(self)
 139                 selfDict["messages"] = [message.to_dict() for message in self.messages] if self.messages is not None else None
 140                 return selfDict
 141
 142
 143 class GVoiceBackend(object):
 144         """
 145         This class encapsulates all of the knowledge necessary to interact with the GoogleVoice servers
 146         the functions include login, setting up a callback number, and initalting a callback
 147         """
 148
 149         PHONE_TYPE_HOME = 1
 150         PHONE_TYPE_MOBILE = 2
 151         PHONE_TYPE_WORK = 3
 152         PHONE_TYPE_GIZMO = 7
 153
 154         def __init__(self, cookieFile = None):
 155                 # Important items in this function are the setup of the browser emulation and cookie file
 156                 self._browser = browser_emu.MozillaEmulator(1)
 157                 self._loadedFromCookies = self._browser.load_cookies(cookieFile)
 158
 159                 self._token = ""
 160                 self._accountNum = ""
 161                 self._lastAuthed = 0.0
 162                 self._callbackNumber = ""
 163                 self._callbackNumbers = {}
 164
 165                 # Suprisingly, moving all of these from class to self sped up startup time
 166
 167                 self._validateRe = re.compile("^\+?[0-9]{10,}$")
 168
 169                 self._loginURL = "https://www.google.com/accounts/ServiceLoginAuth"
 170
 171                 SECURE_URL_BASE = "https://www.google.com/voice/"
 172                 SECURE_MOBILE_URL_BASE = SECURE_URL_BASE + "mobile/"
 173                 self._tokenURL = SECURE_URL_BASE + "m"
 174                 self._callUrl = SECURE_URL_BASE + "call/connect"
 175                 self._callCancelURL = SECURE_URL_BASE + "call/cancel"
 176                 self._sendSmsURL = SECURE_URL_BASE + "sms/send"
 177
 178                 self._isDndURL = "https://www.google.com/voice/m/donotdisturb"
 179                 self._isDndRe = re.compile(r"""<input.*?id="doNotDisturb".*?checked="(.*?)"\s*/>""")
 180                 self._setDndURL = "https://www.google.com/voice/m/savednd"
 181
 182                 self._downloadVoicemailURL = SECURE_URL_BASE + "media/send_voicemail/"
 183                 self._markAsReadURL = SECURE_URL_BASE + "m/mark"
 184                 self._archiveMessageURL = SECURE_URL_BASE + "m/archive"
 185
 186                 self._XML_SEARCH_URL = SECURE_URL_BASE + "inbox/search/"
 187                 self._XML_ACCOUNT_URL = SECURE_URL_BASE + "contacts/"
 188                 # HACK really this redirects to the main pge and we are grabbing some javascript
 189                 self._XML_CONTACTS_URL = "http://www.google.com/voice/inbox/search/contact"
 190                 self._CSV_CONTACTS_URL = "http://mail.google.com/mail/contacts/data/export"
 191                 self._JSON_CONTACTS_URL = SECURE_URL_BASE + "b/0/request/user"
 192                 self._XML_RECENT_URL = SECURE_URL_BASE + "inbox/recent/"
 193
 194                 self.XML_FEEDS = (
 195                         'inbox', 'starred', 'all', 'spam', 'trash', 'voicemail', 'sms',
 196                         'recorded', 'placed', 'received', 'missed'
 197                 )
 198                 self._XML_INBOX_URL = SECURE_URL_BASE + "inbox/recent/inbox"
 199                 self._XML_STARRED_URL = SECURE_URL_BASE + "inbox/recent/starred"
 200                 self._XML_ALL_URL = SECURE_URL_BASE + "inbox/recent/all"
 201                 self._XML_SPAM_URL = SECURE_URL_BASE + "inbox/recent/spam"
 202                 self._XML_TRASH_URL = SECURE_URL_BASE + "inbox/recent/trash"
 203                 self._XML_VOICEMAIL_URL = SECURE_URL_BASE + "inbox/recent/voicemail/"
 204                 self._XML_SMS_URL = SECURE_URL_BASE + "inbox/recent/sms/"
 205                 self._JSON_SMS_URL = SECURE_URL_BASE + "b/0/request/messages/"
 206                 self._JSON_SMS_COUNT_URL = SECURE_URL_BASE + "b/0/request/unread"
 207                 self._XML_RECORDED_URL = SECURE_URL_BASE + "inbox/recent/recorded/"
 208                 self._XML_PLACED_URL = SECURE_URL_BASE + "inbox/recent/placed/"
 209                 self._XML_RECEIVED_URL = SECURE_URL_BASE + "inbox/recent/received/"
 210                 self._XML_MISSED_URL = SECURE_URL_BASE + "inbox/recent/missed/"
 211
 212                 self._galxRe = re.compile(r"""<input.*?name="GALX".*?value="(.*?)".*?/>""", re.MULTILINE | re.DOTALL)
 213
 214                 self._seperateVoicemailsRegex = re.compile(r"""^\s*<div id="(\w+)"\s* class=".*?gc-message.*?">""", re.MULTILINE | re.DOTALL)
 215                 self._exactVoicemailTimeRegex = re.compile(r"""<span class="gc-message-time">(.*?)</span>""", re.MULTILINE)
 216                 self._relativeVoicemailTimeRegex = re.compile(r"""<span class="gc-message-relative">(.*?)</span>""", re.MULTILINE)
 217                 self._voicemailNameRegex = re.compile(r"""<a class=.*?gc-message-name-link.*?>(.*?)</a>""", re.MULTILINE | re.DOTALL)
 218                 self._voicemailNumberRegex = re.compile(r"""<input type="hidden" class="gc-text gc-quickcall-ac" value="(.*?)"/>""", re.MULTILINE)
 219                 self._prettyVoicemailNumberRegex = re.compile(r"""<span class="gc-message-type">(.*?)</span>""", re.MULTILINE)
 220                 self._voicemailLocationRegex = re.compile(r"""<span class="gc-message-location">.*?<a.*?>(.*?)</a></span>""", re.MULTILINE)
 221                 self._messagesContactIDRegex = re.compile(r"""<a class=".*?gc-message-name-link.*?">.*?</a>\s*?<span .*?>(.*?)</span>""", re.MULTILINE)
 222                 self._voicemailMessageRegex = re.compile(r"""(<span id="\d+-\d+" class="gc-word-(.*?)">(.*?)</span>|<a .*? class="gc-message-mni">(.*?)</a>)""", re.MULTILINE)
 223                 self._smsFromRegex = re.compile(r"""<span class="gc-message-sms-from">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 224                 self._smsTimeRegex = re.compile(r"""<span class="gc-message-sms-time">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 225                 self._smsTextRegex = re.compile(r"""<span class="gc-message-sms-text">(.*?)</span>""", re.MULTILINE | re.DOTALL)
 226
 227         def is_quick_login_possible(self):
 228                 """
 229                 @returns True then refresh_account_info might be enough to login, else full login is required
 230                 """
 231                 return self._loadedFromCookies or 0.0 < self._lastAuthed
 232
 233         def refresh_account_info(self):
 234                 try:
 235                         page = self._get_page(self._JSON_CONTACTS_URL)
 236                         accountData = self._grab_account_info(page)
 237                 except Exception, e:
 238                         _moduleLogger.exception(str(e))
 239                         return None
 240
 241                 self._browser.save_cookies()
 242                 self._lastAuthed = time.time()
 243                 return accountData
 244
 245         def _get_token(self):
 246                 tokenPage = self._get_page(self._tokenURL)
 247
 248                 galxTokens = self._galxRe.search(tokenPage)
 249                 if galxTokens is not None:
 250                         galxToken = galxTokens.group(1)
 251                 else:
 252                         galxToken = ""
 253                         _moduleLogger.debug("Could not grab GALX token")
 254                 return galxToken
 255
 256         def _login(self, username, password, token):
 257                 loginData = {
 258                         'Email' : username,
 259                         'Passwd' : password,
 260                         'service': "grandcentral",
 261                         "ltmpl": "mobile",
 262                         "btmpl": "mobile",
 263                         "PersistentCookie": "yes",
 264                         "GALX": token,
 265                         "continue": self._JSON_CONTACTS_URL,
 266                 }
 267
 268                 loginSuccessOrFailurePage = self._get_page(self._loginURL, loginData)
 269                 return loginSuccessOrFailurePage
 270
 271         def login(self, username, password):
 272                 """
 273                 Attempt to login to GoogleVoice
 274                 @returns Whether login was successful or not
 275                 @blocks
 276                 """
 277                 self.logout()
 278                 galxToken = self._get_token()
 279                 loginSuccessOrFailurePage = self._login(username, password, galxToken)
 280
 281                 try:
 282                         accountData = self._grab_account_info(loginSuccessOrFailurePage)
 283                 except Exception, e:
 284                         # Retry in case the redirect failed
 285                         # luckily refresh_account_info does everything we need for a retry
 286                         accountData = self.refresh_account_info()
 287                         if accountData is None:
 288                                 _moduleLogger.exception(str(e))
 289                                 return None
 290                         _moduleLogger.info("Redirection failed on initial login attempt, auto-corrected for this")
 291
 292                 self._browser.save_cookies()
 293                 self._lastAuthed = time.time()
 294                 return accountData
 295
 296         def persist(self):
 297                 self._browser.save_cookies()
 298
 299         def shutdown(self):
 300                 self._browser.save_cookies()
 301                 self._token = None
 302                 self._lastAuthed = 0.0
 303
 304         def logout(self):
 305                 self._browser.clear_cookies()
 306                 self._browser.save_cookies()
 307                 self._token = None
 308                 self._lastAuthed = 0.0
 309
 310         def is_dnd(self):
 311                 """
 312                 @blocks
 313                 """
 314                 isDndPage = self._get_page(self._isDndURL)
 315
 316                 dndGroup = self._isDndRe.search(isDndPage)
 317                 if dndGroup is None:
 318                         return False
 319                 dndStatus = dndGroup.group(1)
 320                 isDnd = True if dndStatus.strip().lower() == "true" else False
 321                 return isDnd
 322
 323         def set_dnd(self, doNotDisturb):
 324                 """
 325                 @blocks
 326                 """
 327                 dndPostData = {
 328                         "doNotDisturb": 1 if doNotDisturb else 0,
 329                 }
 330
 331                 dndPage = self._get_page_with_token(self._setDndURL, dndPostData)
 332
 333         def call(self, outgoingNumber):
 334                 """
 335                 This is the main function responsible for initating the callback
 336                 @blocks
 337                 """
 338                 outgoingNumber = self._send_validation(outgoingNumber)
 339                 subscriberNumber = None
 340                 phoneType = guess_phone_type(self._callbackNumber) # @todo Fix this hack
 341
 342                 callData = {
 343                                 'outgoingNumber': outgoingNumber,
 344                                 'forwardingNumber': self._callbackNumber,
 345                                 'subscriberNumber': subscriberNumber or 'undefined',
 346                                 'phoneType': str(phoneType),
 347                                 'remember': '1',
 348                 }
 349                 _moduleLogger.info("%r" % callData)
 350
 351                 page = self._get_page_with_token(
 352                         self._callUrl,
 353                         callData,
 354                 )
 355                 self._parse_with_validation(page)
 356                 return True
 357
 358         def cancel(self, outgoingNumber=None):
 359                 """
 360                 Cancels a call matching outgoing and forwarding numbers (if given).
 361                 Will raise an error if no matching call is being placed
 362                 @blocks
 363                 """
 364                 page = self._get_page_with_token(
 365                         self._callCancelURL,
 366                         {
 367                         'outgoingNumber': outgoingNumber or 'undefined',
 368                         'forwardingNumber': self._callbackNumber or 'undefined',
 369                         'cancelType': 'C2C',
 370                         },
 371                 )
 372                 self._parse_with_validation(page)
 373
 374         def send_sms(self, phoneNumbers, message):
 375                 """
 376                 @blocks
 377                 """
 378                 validatedPhoneNumbers = [
 379                         self._send_validation(phoneNumber)
 380                         for phoneNumber in phoneNumbers
 381                 ]
 382                 flattenedPhoneNumbers = ",".join(validatedPhoneNumbers)
 383                 page = self._get_page_with_token(
 384                         self._sendSmsURL,
 385                         {
 386                                 'phoneNumber': flattenedPhoneNumbers,
 387                                 'text': unicode(message).encode("utf-8"),
 388                         },
 389                 )
 390                 self._parse_with_validation(page)
 391
 392         def search(self, query):
 393                 """
 394                 Search your Google Voice Account history for calls, voicemails, and sms
 395                 Returns ``Folder`` instance containting matching messages
 396                 @blocks
 397                 """
 398                 page = self._get_page(
 399                         self._XML_SEARCH_URL,
 400                         {"q": query},
 401                 )
 402                 json, html = extract_payload(page)
 403                 return json
 404
 405         def get_feed(self, feed):
 406                 """
 407                 @blocks
 408                 """
 409                 actualFeed = "_XML_%s_URL" % feed.upper()
 410                 feedUrl = getattr(self, actualFeed)
 411
 412                 page = self._get_page(feedUrl)
 413                 json, html = extract_payload(page)
 414
 415                 return json
 416
 417         def recording_url(self, messageId):
 418                 url = self._downloadVoicemailURL+messageId
 419                 return url
 420
 421         def download(self, messageId, adir):
 422                 """
 423                 Download a voicemail or recorded call MP3 matching the given ``msg``
 424                 which can either be a ``Message`` instance, or a SHA1 identifier.
 425                 Saves files to ``adir`` (defaults to current directory).
 426                 Message hashes can be found in ``self.voicemail().messages`` for example.
 427                 @returns location of saved file.
 428                 @blocks
 429                 """
 430                 page = self._get_page(self.recording_url(messageId))
 431                 fn = os.path.join(adir, '%s.mp3' % messageId)
 432                 with open(fn, 'wb') as fo:
 433                         fo.write(page)
 434                 return fn
 435
 436         def is_valid_syntax(self, number):
 437                 """
 438                 @returns If This number be called ( syntax validation only )
 439                 """
 440                 return self._validateRe.match(number) is not None
 441
 442         def get_account_number(self):
 443                 """
 444                 @returns The GoogleVoice phone number
 445                 """
 446                 return self._accountNum
 447
 448         def get_callback_numbers(self):
 449                 """
 450                 @returns a dictionary mapping call back numbers to descriptions
 451                 @note These results are cached for 30 minutes.
 452                 """
 453                 return self._callbackNumbers
 454
 455         def set_callback_number(self, callbacknumber):
 456                 """
 457                 Set the number that GoogleVoice calls
 458                 @param callbacknumber should be a proper 10 digit number
 459                 """
 460                 self._callbackNumber = callbacknumber
 461                 _moduleLogger.info("Callback number changed: %r" % self._callbackNumber)
 462                 return True
 463
 464         def get_callback_number(self):
 465                 """
 466                 @returns Current callback number or None
 467                 """
 468                 return self._callbackNumber
 469
 470         def get_recent(self):
 471                 """
 472                 @returns Iterable of (personsName, phoneNumber, exact date, relative date, action)
 473                 @blocks
 474                 """
 475                 recentPages = [
 476                         (action, self._get_page(url))
 477                         for action, url in (
 478                                 ("Received", self._XML_RECEIVED_URL),
 479                                 ("Missed", self._XML_MISSED_URL),
 480                                 ("Placed", self._XML_PLACED_URL),
 481                         )
 482                 ]
 483                 return self._parse_recent(recentPages)
 484
 485         def get_csv_contacts(self):
 486                 data = {
 487                         "groupToExport": "mine",
 488                         "exportType": "ALL",
 489                         "out": "OUTLOOK_CSV",
 490                 }
 491                 encodedData = urllib.urlencode(data)
 492                 contacts = self._get_page(self._CSV_CONTACTS_URL+"?"+encodedData)
 493                 return contacts
 494
 495         def get_voicemails(self):
 496                 """
 497                 @blocks
 498                 """
 499                 voicemailPage = self._get_page(self._XML_VOICEMAIL_URL)
 500                 voicemailHtml = self._grab_html(voicemailPage)
 501                 voicemailJson = self._grab_json(voicemailPage)
 502                 if voicemailJson is None:
 503                         return ()
 504                 parsedVoicemail = self._parse_voicemail(voicemailHtml)
 505                 voicemails = self._merge_conversation_sources(parsedVoicemail, voicemailJson)
 506                 return voicemails
 507
 508         def get_texts(self):
 509                 """
 510                 @blocks
 511                 """
 512                 smsPage = self._get_page(self._XML_SMS_URL)
 513                 smsHtml = self._grab_html(smsPage)
 514                 smsJson = self._grab_json(smsPage)
 515                 if smsJson is None:
 516                         return ()
 517                 parsedSms = self._parse_sms(smsHtml)
 518                 smss = self._merge_conversation_sources(parsedSms, smsJson)
 519                 return smss
 520
 521         def get_unread_counts(self):
 522                 countPage = self._get_page(self._JSON_SMS_COUNT_URL)
 523                 counts = parse_json(countPage)
 524                 counts = counts["unreadCounts"]
 525                 return counts
 526
 527         def mark_message(self, messageId, asRead):
 528                 """
 529                 @blocks
 530                 """
 531                 postData = {
 532                         "read": 1 if asRead else 0,
 533                         "id": messageId,
 534                 }
 535
 536                 markPage = self._get_page(self._markAsReadURL, postData)
 537
 538         def archive_message(self, messageId):
 539                 """
 540                 @blocks
 541                 """
 542                 postData = {
 543                         "id": messageId,
 544                 }
 545
 546                 markPage = self._get_page(self._archiveMessageURL, postData)
 547
 548         def _grab_json(self, flatXml):
 549                 xmlTree = ElementTree.fromstring(flatXml)
 550                 jsonElement = xmlTree.getchildren()[0]
 551                 flatJson = jsonElement.text
 552                 jsonTree = parse_json(flatJson)
 553                 return jsonTree
 554
 555         def _grab_html(self, flatXml):
 556                 xmlTree = ElementTree.fromstring(flatXml)
 557                 htmlElement = xmlTree.getchildren()[1]
 558                 flatHtml = htmlElement.text
 559                 return flatHtml
 560
 561         def _grab_account_info(self, page):
 562                 accountData = parse_json(page)
 563                 self._token = accountData["r"]
 564                 self._accountNum = accountData["number"]["raw"]
 565                 for callback in accountData["phones"].itervalues():
 566                         self._callbackNumbers[callback["phoneNumber"]] = callback["name"]
 567                 if len(self._callbackNumbers) == 0:
 568                         _moduleLogger.debug("Could not extract callback numbers from GoogleVoice (the troublesome page follows):\n%s" % page)
 569                 return accountData
 570
 571         def _send_validation(self, number):
 572                 if not self.is_valid_syntax(number):
 573                         raise ValueError('Number is not valid: "%s"' % number)
 574                 return number
 575
 576         def _parse_recent(self, recentPages):
 577                 for action, flatXml in recentPages:
 578                         allRecentHtml = self._grab_html(flatXml)
 579                         allRecentData = self._parse_history(allRecentHtml)
 580                         for recentCallData in allRecentData:
 581                                 recentCallData["action"] = action
 582                                 yield recentCallData
 583
 584         def _parse_history(self, historyHtml):
 585                 splitVoicemail = self._seperateVoicemailsRegex.split(historyHtml)
 586                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 587                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 588                         exactTime = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 589                         exactTime = google_strptime(exactTime)
 590                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 591                         relativeTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 592                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 593                         location = locationGroup.group(1).strip() if locationGroup else ""
 594
 595                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 596                         name = nameGroup.group(1).strip() if nameGroup else ""
 597                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 598                         number = numberGroup.group(1).strip() if numberGroup else ""
 599                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 600                         prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 601                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 602                         contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 603
 604                         yield {
 605                                 "id": messageId.strip(),
 606                                 "contactId": contactId,
 607                                 "name": unescape(name),
 608                                 "time": exactTime,
 609                                 "relTime": relativeTime,
 610                                 "prettyNumber": prettyNumber,
 611                                 "number": number,
 612                                 "location": unescape(location),
 613                         }
 614
 615         @staticmethod
 616         def _interpret_voicemail_regex(group):
 617                 quality, content, number = group.group(2), group.group(3), group.group(4)
 618                 text = MessageText()
 619                 if quality is not None and content is not None:
 620                         text.accuracy = quality
 621                         text.text = unescape(content)
 622                         return text
 623                 elif number is not None:
 624                         text.accuracy = MessageText.ACCURACY_HIGH
 625                         text.text = number
 626                         return text
 627
 628         def _parse_voicemail(self, voicemailHtml):
 629                 splitVoicemail = self._seperateVoicemailsRegex.split(voicemailHtml)
 630                 for messageId, messageHtml in itergroup(splitVoicemail[1:], 2):
 631                         conv = Conversation()
 632                         conv.type = Conversation.TYPE_VOICEMAIL
 633                         conv.id = messageId.strip()
 634
 635                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 636                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 637                         conv.time = google_strptime(exactTimeText)
 638                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 639                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 640                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
 641                         conv.location = unescape(locationGroup.group(1).strip() if locationGroup else "")
 642
 643                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 644                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 645                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 646                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 647                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 648                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 649                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 650                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 651
 652                         messageGroups = self._voicemailMessageRegex.finditer(messageHtml)
 653                         messageParts = [
 654                                 self._interpret_voicemail_regex(group)
 655                                 for group in messageGroups
 656                         ] if messageGroups else ((MessageText.ACCURACY_LOW, "No Transcription"), )
 657                         message = Message()
 658                         message.body = messageParts
 659                         message.whoFrom = conv.name
 660                         try:
 661                                 message.when = conv.time.strftime("%I:%M %p")
 662                         except ValueError:
 663                                 _moduleLogger.exception("Confusing time provided: %r" % conv.time)
 664                                 message.when = "Unknown"
 665                         conv.messages = (message, )
 666
 667                         yield conv
 668
 669         @staticmethod
 670         def _interpret_sms_message_parts(fromPart, textPart, timePart):
 671                 text = MessageText()
 672                 text.accuracy = MessageText.ACCURACY_MEDIUM
 673                 text.text = unescape(textPart)
 674
 675                 message = Message()
 676                 message.body = (text, )
 677                 message.whoFrom = fromPart
 678                 message.when = timePart
 679
 680                 return message
 681
 682         def _parse_sms(self, smsHtml):
 683                 splitSms = self._seperateVoicemailsRegex.split(smsHtml)
 684                 for messageId, messageHtml in itergroup(splitSms[1:], 2):
 685                         conv = Conversation()
 686                         conv.type = Conversation.TYPE_SMS
 687                         conv.id = messageId.strip()
 688
 689                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
 690                         exactTimeText = exactTimeGroup.group(1).strip() if exactTimeGroup else ""
 691                         conv.time = google_strptime(exactTimeText)
 692                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
 693                         conv.relTime = relativeTimeGroup.group(1).strip() if relativeTimeGroup else ""
 694                         conv.location = ""
 695
 696                         nameGroup = self._voicemailNameRegex.search(messageHtml)
 697                         conv.name = unescape(nameGroup.group(1).strip() if nameGroup else "")
 698                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
 699                         conv.number = numberGroup.group(1).strip() if numberGroup else ""
 700                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
 701                         conv.prettyNumber = prettyNumberGroup.group(1).strip() if prettyNumberGroup else ""
 702                         contactIdGroup = self._messagesContactIDRegex.search(messageHtml)
 703                         conv.contactId = contactIdGroup.group(1).strip() if contactIdGroup else ""
 704
 705                         fromGroups = self._smsFromRegex.finditer(messageHtml)
 706                         fromParts = (group.group(1).strip() for group in fromGroups)
 707                         textGroups = self._smsTextRegex.finditer(messageHtml)
 708                         textParts = (group.group(1).strip() for group in textGroups)
 709                         timeGroups = self._smsTimeRegex.finditer(messageHtml)
 710                         timeParts = (group.group(1).strip() for group in timeGroups)
 711
 712                         messageParts = itertools.izip(fromParts, textParts, timeParts)
 713                         messages = [self._interpret_sms_message_parts(*parts) for parts in messageParts]
 714                         conv.messages = messages
 715
 716                         yield conv
 717
 718         @staticmethod
 719         def _merge_conversation_sources(parsedMessages, json):
 720                 for message in parsedMessages:
 721                         jsonItem = json["messages"][message.id]
 722                         message.isRead = jsonItem["isRead"]
 723                         message.isSpam = jsonItem["isSpam"]
 724                         message.isTrash = jsonItem["isTrash"]
 725                         message.isArchived = "inbox" not in jsonItem["labels"]
 726                         yield message
 727
 728         def _get_page(self, url, data = None, refererUrl = None):
 729                 headers = {}
 730                 if refererUrl is not None:
 731                         headers["Referer"] = refererUrl
 732
 733                 encodedData = urllib.urlencode(data) if data is not None else None
 734
 735                 try:
 736                         page = self._browser.download(url, encodedData, None, headers)
 737                 except urllib2.URLError, e:
 738                         _moduleLogger.error("Translating error: %s" % str(e))
 739                         raise NetworkError("%s is not accesible" % url)
 740
 741                 return page
 742
 743         def _get_page_with_token(self, url, data = None, refererUrl = None):
 744                 if data is None:
 745                         data = {}
 746                 data['_rnr_se'] = self._token
 747
 748                 page = self._get_page(url, data, refererUrl)
 749
 750                 return page
 751
 752         def _parse_with_validation(self, page):
 753                 json = parse_json(page)
 754                 self._validate_response(json)
 755                 return json
 756
 757         def _validate_response(self, response):
 758                 """
 759                 Validates that the JSON response is A-OK
 760                 """
 761                 try:
 762                         assert response is not None, "Response not provided"
 763                         assert 'ok' in response, "Response lacks status"
 764                         assert response['ok'], "Response not good"
 765                 except AssertionError:
 766                         try:
 767                                 if response["data"]["code"] == 20:
 768                                         raise RuntimeError(
 769 """Ambiguous error 20 returned by Google Voice.
 770 Please verify you have configured your callback number (currently "%s").  If it is configured some other suspected causes are: non-verified callback numbers, and Gizmo5 callback numbers.""" % self._callbackNumber)
 771                         except KeyError:
 772                                 pass
 773                         raise RuntimeError('There was a problem with GV: %s' % response)
 774
 775
 776 _UNESCAPE_ENTITIES = {
 777  "&quot;": '"',
 778  "&nbsp;": " ",
 779  "&#39;": "'",
 780 }
 781
 782
 783 def unescape(text):
 784         plain = saxutils.unescape(text, _UNESCAPE_ENTITIES)
 785         return plain
 786
 787
 788 def google_strptime(time):
 789         """
 790         Hack: Google always returns the time in the same locale.  Sadly if the
 791         local system's locale is different, there isn't a way to perfectly handle
 792         the time.  So instead we handle implement some time formatting
 793         """
 794         abbrevTime = time[:-3]
 795         parsedTime = datetime.datetime.strptime(abbrevTime, "%m/%d/%y %I:%M")
 796         if time.endswith("PM"):
 797                 parsedTime += datetime.timedelta(hours=12)
 798         return parsedTime
 799
 800
 801 def itergroup(iterator, count, padValue = None):
 802         """
 803         Iterate in groups of 'count' values. If there
 804         aren't enough values, the last result is padded with
 805         None.
 806
 807         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 808         ...     print tuple(val)
 809         (1, 2, 3)
 810         (4, 5, 6)
 811         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
 812         ...     print list(val)
 813         [1, 2, 3]
 814         [4, 5, 6]
 815         >>> for val in itergroup([1, 2, 3, 4, 5, 6, 7], 3):
 816         ...     print tuple(val)
 817         (1, 2, 3)
 818         (4, 5, 6)
 819         (7, None, None)
 820         >>> for val in itergroup("123456", 3):
 821         ...     print tuple(val)
 822         ('1', '2', '3')
 823         ('4', '5', '6')
 824         >>> for val in itergroup("123456", 3):
 825         ...     print repr("".join(val))
 826         '123'
 827         '456'
 828         """
 829         paddedIterator = itertools.chain(iterator, itertools.repeat(padValue, count-1))
 830         nIterators = (paddedIterator, ) * count
 831         return itertools.izip(*nIterators)
 832
 833
 834 def safe_eval(s):
 835         _TRUE_REGEX = re.compile("true")
 836         _FALSE_REGEX = re.compile("false")
 837         _COMMENT_REGEX = re.compile("^\s+//.*$", re.M)
 838         s = _TRUE_REGEX.sub("True", s)
 839         s = _FALSE_REGEX.sub("False", s)
 840         s = _COMMENT_REGEX.sub("#", s)
 841         try:
 842                 results = eval(s, {}, {})
 843         except SyntaxError:
 844                 _moduleLogger.exception("Oops")
 845                 results = None
 846         return results
 847
 848
 849 def _fake_parse_json(flattened):
 850         return safe_eval(flattened)
 851
 852
 853 def _actual_parse_json(flattened):
 854         return simplejson.loads(flattened)
 855
 856
 857 if simplejson is None:
 858         parse_json = _fake_parse_json
 859 else:
 860         parse_json = _actual_parse_json
 861
 862
 863 def extract_payload(flatXml):
 864         xmlTree = ElementTree.fromstring(flatXml)
 865
 866         jsonElement = xmlTree.getchildren()[0]
 867         flatJson = jsonElement.text
 868         jsonTree = parse_json(flatJson)
 869
 870         htmlElement = xmlTree.getchildren()[1]
 871         flatHtml = htmlElement.text
 872
 873         return jsonTree, flatHtml
 874
 875
 876 def guess_phone_type(number):
 877         if number.startswith("747") or number.startswith("1747") or number.startswith("+1747"):
 878                 return GVoiceBackend.PHONE_TYPE_GIZMO
 879         else:
 880                 return GVoiceBackend.PHONE_TYPE_MOBILE
 881
 882
 883 def get_sane_callback(backend):
 884         """
 885         Try to set a sane default callback number on these preferences
 886         1) 1747 numbers ( Gizmo )
 887         2) anything with gizmo in the name
 888         3) anything with computer in the name
 889         4) the first value
 890         """
 891         numbers = backend.get_callback_numbers()
 892
 893         priorityOrderedCriteria = [
 894                 ("\+1747", None),
 895                 ("1747", None),
 896                 ("747", None),
 897                 (None, "gizmo"),
 898                 (None, "computer"),
 899                 (None, "sip"),
 900                 (None, None),
 901         ]
 902
 903         for numberCriteria, descriptionCriteria in priorityOrderedCriteria:
 904                 numberMatcher = None
 905                 descriptionMatcher = None
 906                 if numberCriteria is not None:
 907                         numberMatcher = re.compile(numberCriteria)
 908                 elif descriptionCriteria is not None:
 909                         descriptionMatcher = re.compile(descriptionCriteria, re.I)
 910
 911                 for number, description in numbers.iteritems():
 912                         if numberMatcher is not None and numberMatcher.match(number) is None:
 913                                 continue
 914                         if descriptionMatcher is not None and descriptionMatcher.match(description) is None:
 915                                 continue
 916                         return number
 917
 918
 919 def set_sane_callback(backend):
 920         """
 921         Try to set a sane default callback number on these preferences
 922         1) 1747 numbers ( Gizmo )
 923         2) anything with gizmo in the name
 924         3) anything with computer in the name
 925         4) the first value
 926         """
 927         number = get_sane_callback(backend)
 928         backend.set_callback_number(number)
 929
 930
 931 def _is_not_special(name):
 932         return not name.startswith("_") and name[0].lower() == name[0] and "_" not in name
 933
 934
 935 def to_dict(obj):
 936         members = inspect.getmembers(obj)
 937         return dict((name, value) for (name, value) in members if _is_not_special(name))
 938
 939
 940 def grab_debug_info(username, password):
 941         cookieFile = os.path.join(".", "raw_cookies.txt")
 942         try:
 943                 os.remove(cookieFile)
 944         except OSError:
 945                 pass
 946
 947         backend = GVoiceBackend(cookieFile)
 948         browser = backend._browser
 949
 950         _TEST_WEBPAGES = [
 951                 ("token", backend._tokenURL),
 952                 ("login", backend._loginURL),
 953                 ("isdnd", backend._isDndURL),
 954                 ("account", backend._XML_ACCOUNT_URL),
 955                 ("html_contacts", backend._XML_CONTACTS_URL),
 956                 ("contacts", backend._JSON_CONTACTS_URL),
 957                 ("csv", backend._CSV_CONTACTS_URL),
 958
 959                 ("voicemail", backend._XML_VOICEMAIL_URL),
 960                 ("html_sms", backend._XML_SMS_URL),
 961                 ("sms", backend._JSON_SMS_URL),
 962                 ("count", backend._JSON_SMS_COUNT_URL),
 963
 964                 ("recent", backend._XML_RECENT_URL),
 965                 ("placed", backend._XML_PLACED_URL),
 966                 ("recieved", backend._XML_RECEIVED_URL),
 967                 ("missed", backend._XML_MISSED_URL),
 968         ]
 969
 970         # Get Pages
 971         print "Grabbing pre-login pages"
 972         for name, url in _TEST_WEBPAGES:
 973                 try:
 974                         page = browser.download(url)
 975                 except StandardError, e:
 976                         print e.message
 977                         continue
 978                 print "\tWriting to file"
 979                 with open("not_loggedin_%s.txt" % name, "w") as f:
 980                         f.write(page)
 981
 982         # Login
 983         print "Attempting login"
 984         galxToken = backend._get_token()
 985         loginSuccessOrFailurePage = backend._login(username, password, galxToken)
 986         with open("loggingin.txt", "w") as f:
 987                 print "\tWriting to file"
 988                 f.write(loginSuccessOrFailurePage)
 989         try:
 990                 backend._grab_account_info(loginSuccessOrFailurePage)
 991         except Exception:
 992                 # Retry in case the redirect failed
 993                 # luckily refresh_account_info does everything we need for a retry
 994                 loggedIn = backend.refresh_account_info() is not None
 995                 if not loggedIn:
 996                         raise
 997
 998         # Get Pages
 999         print "Grabbing post-login pages"
1000         for name, url in _TEST_WEBPAGES:
1001                 try:
1002                         page = browser.download(url)
1003                 except StandardError, e:
1004                         print str(e)
1005                         continue
1006                 print "\tWriting to file"
1007                 with open("loggedin_%s.txt" % name, "w") as f:
1008                         f.write(page)
1009
1010         # Cookies
1011         browser.save_cookies()
1012         print "\tWriting cookies to file"
1013         with open("cookies.txt", "w") as f:
1014                 f.writelines(
1015                         "%s: %s\n" % (c.name, c.value)
1016                         for c in browser._cookies
1017                 )
1018
1019
1020 def grab_voicemails(username, password):
1021         cookieFile = os.path.join(".", "raw_cookies.txt")
1022         try:
1023                 os.remove(cookieFile)
1024         except OSError:
1025                 pass
1026
1027         backend = GVoiceBackend(cookieFile)
1028         backend.login(username, password)
1029         voicemails = list(backend.get_voicemails())
1030         for voicemail in voicemails:
1031                 print voicemail.id
1032                 backend.download(voicemail.id, ".")
1033
1034
1035 def main():
1036         import sys
1037         logging.basicConfig(level=logging.DEBUG)
1038         args = sys.argv
1039         if 3 <= len(args):
1040                 username = args[1]
1041                 password = args[2]
1042
1043         grab_debug_info(username, password)
1044         grab_voicemails(username, password)
1045
1046
1047 if __name__ == "__main__":
1048         main()