6609042657641f3c1f3978729266f63236273858
[gc-dialer] / src / gv_backend.py
1 #!/usr/bin/python
2
3 """
4 DialCentral - Front end for Google's Grand Central service.
5 Copyright (C) 2008  Eric Warnke ericew AT gmail DOT com
6
7 This library is free software; you can redistribute it and/or
8 modify it under the terms of the GNU Lesser General Public
9 License as published by the Free Software Foundation; either
10 version 2.1 of the License, or (at your option) any later version.
11
12 This library is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15 Lesser General Public License for more details.
16
17 You should have received a copy of the GNU Lesser General Public
18 License along with this library; if not, write to the Free Software
19 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
20
21 Google Voice backend code
22
23 Resources
24         http://thatsmith.com/2009/03/google-voice-addon-for-firefox/
25         http://posttopic.com/topic/google-voice-add-on-development
26 """
27
28
29 import os
30 import re
31 import urllib
32 import urllib2
33 import time
34 import datetime
35 import itertools
36 import warnings
37 import traceback
38 from xml.sax import saxutils
39
40 from xml.etree import ElementTree
41
42 import browser_emu
43
44 try:
45         import simplejson
46 except ImportError:
47         simplejson = None
48
49
50 _TRUE_REGEX = re.compile("true")
51 _FALSE_REGEX = re.compile("false")
52
53
54 def safe_eval(s):
55         s = _TRUE_REGEX.sub("True", s)
56         s = _FALSE_REGEX.sub("False", s)
57         return eval(s, {}, {})
58
59
60 if simplejson is None:
61         def parse_json(flattened):
62                 return safe_eval(flattened)
63 else:
64         def parse_json(flattened):
65                 return simplejson.loads(flattened)
66
67
68 def itergroup(iterator, count, padValue = None):
69         """
70         Iterate in groups of 'count' values. If there
71         aren't enough values, the last result is padded with
72         None.
73
74         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
75         ...     print tuple(val)
76         (1, 2, 3)
77         (4, 5, 6)
78         >>> for val in itergroup([1, 2, 3, 4, 5, 6], 3):
79         ...     print list(val)
80         [1, 2, 3]
81         [4, 5, 6]
82         >>> for val in itergroup([1, 2, 3, 4, 5, 6, 7], 3):
83         ...     print tuple(val)
84         (1, 2, 3)
85         (4, 5, 6)
86         (7, None, None)
87         >>> for val in itergroup("123456", 3):
88         ...     print tuple(val)
89         ('1', '2', '3')
90         ('4', '5', '6')
91         >>> for val in itergroup("123456", 3):
92         ...     print repr("".join(val))
93         '123'
94         '456'
95         """
96         paddedIterator = itertools.chain(iterator, itertools.repeat(padValue, count-1))
97         nIterators = (paddedIterator, ) * count
98         return itertools.izip(*nIterators)
99
100
101 class GVDialer(object):
102         """
103         This class encapsulates all of the knowledge necessary to interace with the grandcentral servers
104         the functions include login, setting up a callback number, and initalting a callback
105         """
106
107         def __init__(self, cookieFile = None):
108                 # Important items in this function are the setup of the browser emulation and cookie file
109                 self._browser = browser_emu.MozillaEmulator(1)
110                 if cookieFile is None:
111                         cookieFile = os.path.join(os.path.expanduser("~"), ".gv_cookies.txt")
112                 self._browser.cookies.filename = cookieFile
113                 if os.path.isfile(cookieFile):
114                         self._browser.cookies.load()
115
116                 self._token = ""
117                 self._accountNum = None
118                 self._lastAuthed = 0.0
119                 self._callbackNumber = ""
120                 self._callbackNumbers = {}
121
122                 self.__contacts = None
123
124         def is_authed(self, force = False):
125                 """
126                 Attempts to detect a current session
127                 @note Once logged in try not to reauth more than once a minute.
128                 @returns If authenticated
129                 """
130
131                 if (time.time() - self._lastAuthed) < 60 and not force:
132                         return True
133
134                 try:
135                         self._grab_account_info()
136                 except StandardError, e:
137                         warnings.warn(traceback.format_exc())
138                         return False
139
140                 self._browser.cookies.save()
141                 self._lastAuthed = time.time()
142                 return True
143
144         _loginURL = "https://www.google.com/accounts/ServiceLoginAuth"
145
146         def login(self, username, password):
147                 """
148                 Attempt to login to grandcentral
149                 @returns Whether login was successful or not
150                 """
151                 if self.is_authed():
152                         return True
153
154                 loginPostData = urllib.urlencode({
155                         'Email' : username,
156                         'Passwd' : password,
157                         'service': "grandcentral",
158                         "ltmpl": "mobile",
159                         "btmpl": "mobile",
160                         "PersistentCookie": "yes",
161                 })
162
163                 try:
164                         loginSuccessOrFailurePage = self._browser.download(self._loginURL, loginPostData)
165                 except urllib2.URLError, e:
166                         warnings.warn(traceback.format_exc())
167                         raise RuntimeError("%s is not accesible" % self._loginURL)
168
169                 return self.is_authed()
170
171         def logout(self):
172                 self._lastAuthed = 0.0
173                 self._browser.cookies.clear()
174                 self._browser.cookies.save()
175
176                 self.clear_caches()
177
178         _gvDialingStrRe = re.compile("This may take a few seconds", re.M)
179         _clicktocallURL = "https://www.google.com/voice/m/sendcall"
180
181         def dial(self, number):
182                 """
183                 This is the main function responsible for initating the callback
184                 """
185                 number = self._send_validation(number)
186                 try:
187                         clickToCallData = urllib.urlencode({
188                                 "number": number,
189                                 "phone": self._callbackNumber,
190                                 "_rnr_se": self._token,
191                         })
192                         otherData = {
193                                 'Referer' : 'https://google.com/voice/m/callsms',
194                         }
195                         callSuccessPage = self._browser.download(self._clicktocallURL, clickToCallData, None, otherData)
196                 except urllib2.URLError, e:
197                         warnings.warn(traceback.format_exc())
198                         raise RuntimeError("%s is not accesible" % self._clicktocallURL)
199
200                 if self._gvDialingStrRe.search(callSuccessPage) is None:
201                         raise RuntimeError("Google Voice returned an error")
202
203                 return True
204
205         _sendSmsURL = "https://www.google.com/voice/m/sendsms"
206
207         def send_sms(self, number, message):
208                 number = self._send_validation(number)
209                 try:
210                         smsData = urllib.urlencode({
211                                 "number": number,
212                                 "smstext": message,
213                                 "_rnr_se": self._token,
214                                 "id": "undefined",
215                                 "c": "undefined",
216                         })
217                         otherData = {
218                                 'Referer' : 'https://google.com/voice/m/sms',
219                         }
220                         smsSuccessPage = self._browser.download(self._sendSmsURL, smsData, None, otherData)
221                 except urllib2.URLError, e:
222                         warnings.warn(traceback.format_exc())
223                         raise RuntimeError("%s is not accesible" % self._sendSmsURL)
224
225                 return True
226
227         def clear_caches(self):
228                 self.__contacts = None
229
230         _validateRe = re.compile("^[0-9]{10,}$")
231
232         def is_valid_syntax(self, number):
233                 """
234                 @returns If This number be called ( syntax validation only )
235                 """
236                 return self._validateRe.match(number) is not None
237
238         def get_account_number(self):
239                 """
240                 @returns The grand central phone number
241                 """
242                 return self._accountNum
243
244         def set_sane_callback(self):
245                 """
246                 Try to set a sane default callback number on these preferences
247                 1) 1747 numbers ( Gizmo )
248                 2) anything with gizmo in the name
249                 3) anything with computer in the name
250                 4) the first value
251                 """
252                 numbers = self.get_callback_numbers()
253
254                 for number, description in numbers.iteritems():
255                         if re.compile(r"""1747""").match(number) is not None:
256                                 self.set_callback_number(number)
257                                 return
258
259                 for number, description in numbers.iteritems():
260                         if re.compile(r"""gizmo""", re.I).search(description) is not None:
261                                 self.set_callback_number(number)
262                                 return
263
264                 for number, description in numbers.iteritems():
265                         if re.compile(r"""computer""", re.I).search(description) is not None:
266                                 self.set_callback_number(number)
267                                 return
268
269                 for number, description in numbers.iteritems():
270                         self.set_callback_number(number)
271                         return
272
273         def get_callback_numbers(self):
274                 """
275                 @returns a dictionary mapping call back numbers to descriptions
276                 @note These results are cached for 30 minutes.
277                 """
278                 if time.time() - self._lastAuthed < 1800 or self.is_authed():
279                         return self._callbackNumbers
280
281                 return {}
282
283         _setforwardURL = "https://www.google.com//voice/m/setphone"
284
285         def set_callback_number(self, callbacknumber):
286                 """
287                 Set the number that grandcental calls
288                 @param callbacknumber should be a proper 10 digit number
289                 """
290                 self._callbackNumber = callbacknumber
291                 callbackPostData = urllib.urlencode({
292                         '_rnr_se': self._token,
293                         'phone': callbacknumber
294                 })
295                 try:
296                         callbackSetPage = self._browser.download(self._setforwardURL, callbackPostData)
297                 except urllib2.URLError, e:
298                         warnings.warn(traceback.format_exc())
299                         raise RuntimeError("%s is not accesible" % self._setforwardURL)
300
301                 # @bug This does not seem to be keeping on my tablet (but works on the
302                 # desktop), or the reading isn't working too well
303                 self._browser.cookies.save()
304                 return True
305
306         def get_callback_number(self):
307                 """
308                 @returns Current callback number or None
309                 """
310                 for c in self._browser.cookies:
311                         if c.name == "gv-ph":
312                                 return c.value
313                 return self._callbackNumber
314
315         def get_recent(self):
316                 """
317                 @todo Sort this stuff
318                 @returns Iterable of (personsName, phoneNumber, date, action)
319                 """
320                 sortedRecent = [
321                         (exactDate, name, number, relativeDate, action)
322                         for (name, number, exactDate, relativeDate, action) in self._get_recent()
323                 ]
324                 sortedRecent.sort(reverse = True)
325                 for exactDate, name, number, relativeDate, action in sortedRecent:
326                         yield name, number, relativeDate, action
327
328         def get_addressbooks(self):
329                 """
330                 @returns Iterable of (Address Book Factory, Book Id, Book Name)
331                 """
332                 yield self, "", ""
333
334         def open_addressbook(self, bookId):
335                 return self
336
337         @staticmethod
338         def contact_source_short_name(contactId):
339                 return "GV"
340
341         @staticmethod
342         def factory_name():
343                 return "Google Voice"
344
345         _contactsRe = re.compile(r"""<a href="/voice/m/contact/(\d+)">(.*?)</a>""", re.S)
346         _contactsNextRe = re.compile(r""".*<a href="/voice/m/contacts(\?p=\d+)">Next.*?</a>""", re.S)
347         _contactsURL = "https://www.google.com/voice/mobile/contacts"
348
349         def get_contacts(self):
350                 """
351                 @returns Iterable of (contact id, contact name)
352                 """
353                 if self.__contacts is None:
354                         self.__contacts = []
355
356                         contactsPagesUrls = [self._contactsURL]
357                         for contactsPageUrl in contactsPagesUrls:
358                                 try:
359                                         contactsPage = self._browser.download(contactsPageUrl)
360                                 except urllib2.URLError, e:
361                                         warnings.warn(traceback.format_exc())
362                                         raise RuntimeError("%s is not accesible" % contactsPageUrl)
363                                 for contact_match in self._contactsRe.finditer(contactsPage):
364                                         contactId = contact_match.group(1)
365                                         contactName = saxutils.unescape(contact_match.group(2))
366                                         contact = contactId, contactName
367                                         self.__contacts.append(contact)
368                                         yield contact
369
370                                 next_match = self._contactsNextRe.match(contactsPage)
371                                 if next_match is not None:
372                                         newContactsPageUrl = self._contactsURL + next_match.group(1)
373                                         contactsPagesUrls.append(newContactsPageUrl)
374                 else:
375                         for contact in self.__contacts:
376                                 yield contact
377
378         _contactDetailPhoneRe = re.compile(r"""<div.*?>([0-9\-\(\) \t]+?)<span.*?>\((\w+)\)</span>""", re.S)
379         _contactDetailURL = "https://www.google.com/voice/mobile/contact"
380
381         def get_contact_details(self, contactId):
382                 """
383                 @returns Iterable of (Phone Type, Phone Number)
384                 """
385                 try:
386                         detailPage = self._browser.download(self._contactDetailURL + '/' + contactId)
387                 except urllib2.URLError, e:
388                         warnings.warn(traceback.format_exc())
389                         raise RuntimeError("%s is not accesible" % self._contactDetailURL)
390
391                 for detail_match in self._contactDetailPhoneRe.finditer(detailPage):
392                         phoneNumber = detail_match.group(1)
393                         phoneType = saxutils.unescape(detail_match.group(2))
394                         yield (phoneType, phoneNumber)
395
396         _voicemailURL = "https://www.google.com/voice/inbox/recent/voicemail/"
397         _smsURL = "https://www.google.com/voice/inbox/recent/sms/"
398
399         def get_messages(self):
400                 try:
401                         voicemailPage = self._browser.download(self._voicemailURL)
402                 except urllib2.URLError, e:
403                         warnings.warn(traceback.format_exc())
404                         raise RuntimeError("%s is not accesible" % self._voicemailURL)
405                 voicemailHtml = self._grab_html(voicemailPage)
406                 parsedVoicemail = self._parse_voicemail(voicemailHtml)
407                 decoratedVoicemails = self._decorate_voicemail(parsedVoicemail)
408
409                 try:
410                         smsPage = self._browser.download(self._smsURL)
411                 except urllib2.URLError, e:
412                         warnings.warn(traceback.format_exc())
413                         raise RuntimeError("%s is not accesible" % self._smsURL)
414                 smsHtml = self._grab_html(smsPage)
415                 parsedSms = self._parse_sms(smsHtml)
416                 decoratedSms = self._decorate_sms(parsedSms)
417
418                 allMessages = itertools.chain(decoratedVoicemails, decoratedSms)
419                 sortedMessages = list(allMessages)
420                 for exactDate, header, number, relativeDate, message in sortedMessages:
421                         yield header, number, relativeDate, message
422
423         def _grab_json(self, flatXml):
424                 xmlTree = ElementTree.fromstring(flatXml)
425                 jsonElement = xmlTree.getchildren()[0]
426                 flatJson = jsonElement.text
427                 jsonTree = parse_json(flatJson)
428                 return jsonTree
429
430         def _grab_html(self, flatXml):
431                 xmlTree = ElementTree.fromstring(flatXml)
432                 htmlElement = xmlTree.getchildren()[1]
433                 flatHtml = htmlElement.text
434                 return flatHtml
435
436         _tokenRe = re.compile(r"""<input.*?name="_rnr_se".*?value="(.*?)"\s*/>""")
437         _accountNumRe = re.compile(r"""<b class="ms\d">(.{14})</b></div>""")
438         _callbackRe = re.compile(r"""\s+(.*?):\s*(.*?)<br\s*/>\s*$""", re.M)
439         _forwardURL = "https://www.google.com/voice/mobile/phones"
440
441         def _grab_account_info(self):
442                 page = self._browser.download(self._forwardURL)
443
444                 tokenGroup = self._tokenRe.search(page)
445                 if tokenGroup is None:
446                         raise RuntimeError("Could not extract authentication token from GoogleVoice")
447                 self._token = tokenGroup.group(1)
448
449                 anGroup = self._accountNumRe.search(page)
450                 if anGroup is None:
451                         raise RuntimeError("Could not extract account number from GoogleVoice")
452                 self._accountNum = anGroup.group(1)
453
454                 self._callbackNumbers = {}
455                 for match in self._callbackRe.finditer(page):
456                         callbackNumber = match.group(2)
457                         callbackName = match.group(1)
458                         self._callbackNumbers[callbackNumber] = callbackName
459
460         def _send_validation(self, number):
461                 if not self.is_valid_syntax(number):
462                         raise ValueError('Number is not valid: "%s"' % number)
463                 elif not self.is_authed():
464                         raise RuntimeError("Not Authenticated")
465
466                 if len(number) == 11 and number[0] == 1:
467                         # Strip leading 1 from 11 digit dialing
468                         number = number[1:]
469                 return number
470
471         _recentCallsURL = "https://www.google.com/voice/inbox/recent/"
472         _placedCallsURL = "https://www.google.com/voice/inbox/recent/placed/"
473         _receivedCallsURL = "https://www.google.com/voice/inbox/recent/received/"
474         _missedCallsURL = "https://www.google.com/voice/inbox/recent/missed/"
475
476         def _get_recent(self):
477                 """
478                 @returns Iterable of (personsName, phoneNumber, exact date, relative date, action)
479                 """
480                 for url in (
481                         self._receivedCallsURL,
482                         self._missedCallsURL,
483                         self._placedCallsURL,
484                 ):
485                         try:
486                                 flatXml = self._browser.download(url)
487                         except urllib2.URLError, e:
488                                 warnings.warn(traceback.format_exc())
489                                 raise RuntimeError("%s is not accesible" % url)
490
491                         allRecentData = self._grab_json(flatXml)
492                         for recentCallData in allRecentData["messages"].itervalues():
493                                 number = recentCallData["displayNumber"]
494                                 exactDate = recentCallData["displayStartDateTime"]
495                                 relativeDate = recentCallData["relativeStartTime"]
496                                 action = ", ".join((
497                                         label.title()
498                                         for label in recentCallData["labels"]
499                                                 if label.lower() != "all" and label.lower() != "inbox"
500                                 ))
501                                 number = saxutils.unescape(number)
502                                 exactDate = saxutils.unescape(exactDate)
503                                 exactDate = datetime.datetime.strptime(exactDate, "%m/%d/%y %I:%M %p")
504                                 relativeDate = saxutils.unescape(relativeDate)
505                                 action = saxutils.unescape(action)
506                                 yield "", number, exactDate, relativeDate, action
507
508         _seperateVoicemailsRegex = re.compile(r"""^\s*<div id="(\w+)"\s* class="gc-message.*?">""", re.MULTILINE | re.DOTALL)
509         _exactVoicemailTimeRegex = re.compile(r"""<span class="gc-message-time">(.*?)</span>""", re.MULTILINE)
510         _relativeVoicemailTimeRegex = re.compile(r"""<span class="gc-message-relative">(.*?)</span>""", re.MULTILINE)
511         _voicemailNumberRegex = re.compile(r"""<input type="hidden" class="gc-text gc-quickcall-ac" value="(.*?)"/>""", re.MULTILINE)
512         _prettyVoicemailNumberRegex = re.compile(r"""<span class="gc-message-type">(.*?)</span>""", re.MULTILINE)
513         _voicemailLocationRegex = re.compile(r"""<span class="gc-message-location">(.*?)</span>""", re.MULTILINE)
514         _voicemailMessageRegex = re.compile(r"""<span class="gc-word-(.*?)">(.*?)</span>""", re.MULTILINE)
515
516         def _parse_voicemail(self, voicemailHtml):
517                 splitVoicemail = self._seperateVoicemailsRegex.split(voicemailHtml)
518                 for id, messageHtml in itergroup(splitVoicemail[1:], 2):
519                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
520                         exactTime = exactTimeGroup.group(1) if exactTimeGroup else ""
521                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
522                         relativeTime = relativeTimeGroup.group(1) if relativeTimeGroup else ""
523                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
524                         location = locationGroup.group(1) if locationGroup else ""
525
526                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
527                         number = numberGroup.group(1) if numberGroup else ""
528                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
529                         prettyNumber = prettyNumberGroup.group(1) if prettyNumberGroup else ""
530
531                         messageGroups = self._voicemailMessageRegex.finditer(messageHtml)
532                         messageParts = (
533                                 (group.group(1), group.group(2))
534                                 for group in messageGroups
535                         ) if messageGroups else ()
536
537                         yield {
538                                 "id": id,
539                                 "time": exactTime,
540                                 "relTime": relativeTime,
541                                 "prettyNumber": prettyNumber,
542                                 "number": number,
543                                 "location": location,
544                                 "messageParts": messageParts,
545                         }
546
547         def _decorate_voicemail(self, parsedVoicemail):
548                 messagePartFormat = {
549                         "med1": "<i>%s</i>",
550                         "med2": "%s",
551                         "high": "<b>%s</b>",
552                 }
553                 for voicemailData in parsedVoicemail:
554                         exactTime = voicemailData["time"] # @todo Parse This
555                         header = "%s %s" % (voicemailData["prettyNumber"], voicemailData["location"])
556                         message = " ".join((
557                                 messagePartFormat[quality] % part
558                                 for (quality, part) in voicemailData["messageParts"]
559                         )).strip()
560                         if not message:
561                                 message = "No Transcription"
562                         yield exactTime, header, voicemailData["number"], voicemailData["relTime"], message
563
564         _smsFromRegex = re.compile(r"""<span class="gc-message-sms-from">(.*?)</span>""", re.MULTILINE)
565         _smsTextRegex = re.compile(r"""<span class="gc-message-sms-time">(.*?)</span>""", re.MULTILINE)
566         _smsTimeRegex = re.compile(r"""<span class="gc-message-sms-text">(.*?)</span>""", re.MULTILINE)
567
568         def _parse_sms(self, smsHtml):
569                 print "="*60
570                 print smsHtml
571                 print "="*60
572                 splitSms = self._seperateVoicemailsRegex.split(smsHtml)
573                 for id, messageHtml in itergroup(splitSms[1:], 2):
574                         exactTimeGroup = self._exactVoicemailTimeRegex.search(messageHtml)
575                         exactTime = exactTimeGroup.group(1) if exactTimeGroup else ""
576                         relativeTimeGroup = self._relativeVoicemailTimeRegex.search(messageHtml)
577                         relativeTime = relativeTimeGroup.group(1) if relativeTimeGroup else ""
578
579                         locationGroup = self._voicemailLocationRegex.search(messageHtml)
580                         location = locationGroup.group(1) if locationGroup else ""
581
582                         numberGroup = self._voicemailNumberRegex.search(messageHtml)
583                         number = numberGroup.group(1) if numberGroup else ""
584                         prettyNumberGroup = self._prettyVoicemailNumberRegex.search(messageHtml)
585                         prettyNumber = prettyNumberGroup.group(1) if prettyNumberGroup else ""
586
587                         fromGroups = self._smsFromRegex.finditer(messageHtml)
588                         fromParts = (group.group(1) for group in fromGroups)
589                         textGroups = self._smsTextRegex.finditer(messageHtml)
590                         textParts = (group.group(1) for group in textGroups)
591                         timeGroups = self._smsTimeRegex.finditer(messageHtml)
592                         timeParts = (group.group(1) for group in timeGroups)
593
594                         # @todo Switch from chain to izip once debugged the parts
595                         #messageParts = itertools.izip(fromParts, textParts, timeParts)
596                         messageParts = itertools.chain(fromParts, textParts, timeParts)
597
598                         # @todo Switch pprint to yield and remove list() call once debugged parts
599                         import pprint
600                         pprint.pprint({
601                                 "id": id,
602                                 "time": exactTime,
603                                 "relTime": relativeTime,
604                                 "prettyNumber": prettyNumber,
605                                 "number": number,
606                                 "location": location,
607                                 "messageParts": list(messageParts),
608                         })
609                 return ()
610
611         def _decorate_sms(self, parsedSms):
612                 return ()
613
614
615 def test_backend(username, password):
616         import pprint
617         backend = GVDialer()
618         print "Authenticated: ", backend.is_authed()
619         print "Login?: ", backend.login(username, password)
620         print "Authenticated: ", backend.is_authed()
621         print "Token: ", backend._token
622         print "Account: ", backend.get_account_number()
623         print "Callback: ", backend.get_callback_number()
624         # print "All Callback: ",
625         # pprint.pprint(backend.get_callback_numbers())
626         # print "Recent: ",
627         # pprint.pprint(list(backend.get_recent()))
628         # print "Contacts: ",
629         # for contact in backend.get_contacts():
630         #       print contact
631         #       pprint.pprint(list(backend.get_contact_details(contact[0])))
632
633         return backend