3 # This file is part of Jamaendo.
4 # Copyright (c) 2010, Kristoffer Gronlund
7 # Redistribution and use in source and binary forms, with or without
8 # modification, are permitted provided that the following conditions are met:
9 # * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
12 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
13 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15 # DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
16 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
17 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
18 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
19 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
20 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
21 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23 # An improved, structured jamendo API wrapper for the N900 with cacheing
24 # Image / cover downloads.. and more?
25 from __future__ import with_statement
26 import urllib, threading, os, time, simplejson, re
27 import logging, hashlib
28 import pycurl, StringIO
32 _GET2 = '''http://api.jamendo.com/get2/'''
33 _MP3URL = _GET2+'stream/track/redirect/?id=%d&streamencoding=mp31'
34 _OGGURL = _GET2+'stream/track/redirect/?id=%d&streamencoding=ogg2'
35 _TORRENTURL = _GET2+'bittorrent/file/redirect/?album_id=%d&type=archive&class=mp32'
38 log = logging.getLogger(__name__)
40 class StdoutLogger(object):
41 def info(self, s, *args):
43 def debug(self, s, *args):
47 # These classes can be partially constructed,
48 # and if asked for a property they don't know,
49 # makes a query internally to get the full story
51 _ARTIST_FIELDS = ['id', 'name', 'image']
52 _ALBUM_FIELDS = ['id', 'name', 'image', 'artist_name', 'artist_id', 'license_url']
53 _TRACK_FIELDS = ['id', 'name', 'album_image', 'artist_id', 'artist_name', 'album_name', 'album_id', 'numalbum', 'duration']
54 _RADIO_FIELDS = ['id', 'name', 'idstr', 'image']
55 _TAG_FIELDS = ['id', 'name']
57 _APILOCK = threading.Lock()
61 s = StringIO.StringIO()
62 c.setopt(pycurl.FOLLOWLOCATION, 1)
63 c.setopt(pycurl.URL, url)
64 c.setopt(pycurl.WRITEFUNCTION, s.write)
72 class LazyQuery(object):
73 def set_from_json(self, json):
74 for key, value in json.iteritems():
76 assert(self.ID == int(value))
78 if key.endswith('_id'):
80 setattr(self, key, value)
83 """Not automatic for now,
84 will have to do artist.load()
86 This is filled in further down
91 def _needs_load(self):
94 def _set_from(self, other):
97 def _needs_load_impl(self, *attrs):
99 if getattr(self, attr) is None:
103 def _set_from_impl(self, other, *attrs):
105 self._set_if(other, attr)
107 def _set_if(self, other, attrname):
108 if getattr(self, attrname) is None and getattr(other, attrname) is not None:
109 setattr(self, attrname, getattr(other, attrname))
113 return u"%s(%s)"%(self.__class__.__name__,
114 u", ".join(("%s:%s"%(k,repr(v))) for k,v in self.__dict__.iteritems() if not k.startswith('_')))
115 except UnicodeEncodeError:
117 #traceback.print_exc()
118 return u"%s(?)"%(self.__class__.__name__)
120 class Artist(LazyQuery):
121 def __init__(self, ID, json=None):
125 self.albums = None # None means not downloaded
127 self.set_from_json(json)
129 def _needs_load(self):
130 return self._needs_load_impl('name', 'albums')
132 def _set_from(self, other):
133 return self._set_from_impl(other, 'name', 'image', 'albums')
136 return {'name':self.name, 'image':self.image}
138 class Album(LazyQuery):
139 def __init__(self, ID, json=None):
143 self.artist_name = None
144 self.artist_id = None
145 self.license_url = None
146 self.tracks = None # None means not downloaded
148 self.set_from_json(json)
150 def torrent_url(self):
151 return _TORRENTURL%(self.ID)
154 def _needs_load(self):
155 return self._needs_load_impl('name', 'image', 'artist_name', 'artist_id', 'license_url', 'tracks')
157 def _set_from(self, other):
158 return self._set_from_impl(other, 'name', 'image', 'artist_name', 'artist_id', 'license_url', 'tracks')
161 return {'name':self.name, 'image':self.image,
162 'artist_name':self.artist_name,
163 'artist_id':self.artist_id,
164 'license_url':self.license_url}
166 class Track(LazyQuery):
167 def __init__(self, ID, json=None):
170 self.artist_id = None
171 self.artist_name = None
172 self.album_image = None
173 self.album_name = None
178 self.set_from_json(json)
181 return _MP3URL%(self.ID)
184 return _OGGURL%(self.ID)
187 return {'name':self.name,
188 'artist_id':self.artist_id,
189 'artist_name':self.artist_name,
190 'album_image':self.album_image,
191 'album_name':self.album_name,
192 'album_id':self.album_id,
193 'numalbum':self.numalbum,
194 'duration':self.duration}
196 def _needs_load(self):
197 return self._needs_load_impl('name', 'artist_name', 'artist_id', 'album_name', 'album_id', 'numalbum', 'duration')
199 def _set_from(self, other):
200 return self._set_from_impl(other, 'name', 'album_image', 'artist_name', 'artist_id', 'album_name', 'album_id', 'numalbum', 'duration')
202 class Radio(LazyQuery):
203 def __init__(self, ID, json=None):
209 self.set_from_json(json)
211 def _needs_load(self):
212 return self._needs_load_impl('name', 'idstr', 'image')
214 def _set_from(self, other):
215 return self._set_from_impl(other, 'name', 'idstr', 'image')
217 class Tag(LazyQuery):
218 def __init__(self, ID, json=None):
222 self.set_from_json(json)
224 def _needs_load(self):
225 return self._needs_load_impl('name')
227 def _set_from(self, other):
228 return self._set_from_impl(other, 'name')
230 _artists = {} # id -> Artist()
231 _albums = {} # id -> Album()
232 _tracks = {} # id -> Track()
233 _radios = {} # id -> Radio()
236 # cache sizes per session (TODO)
237 _CACHED_ARTISTS = 100
241 # cache sizes, persistant
242 _CACHED_COVERS = 2048
244 # TODO: cache queries?
246 class Ratelimit(object):
247 rate_limit = 1.0 # seconds between queries
248 last_query = time.time() - 1.5
253 if (now - cls.last_query) < cls.rate_limit:
254 time.sleep(cls.rate_limit - (now - cls.last_query))
255 cls.last_query = time.time()
257 _ratelimit = Ratelimit.ratelimit
263 def _geturl(self, url):
267 ret = simplejson.loads(curlGET(url))
273 return "#%s" % (self.__class__.__name__)
278 class CoverFetcher(threading.Thread):
280 threading.Thread.__init__(self)
282 self.cond = threading.Condition()
285 def _retrieve(self, url, fname):
286 BROKEN = 'http://imgjam.com/radios/default/default.100.png'
289 f = open(fname, 'wb')
291 c.setopt(pycurl.FOLLOWLOCATION, 1)
292 c.setopt(pycurl.URL, str(url))
293 c.setopt(pycurl.WRITEFUNCTION, f.write)
301 log.debug("Coverfetch: %s -> %s", url, fname)
304 def _fetch_cover(self, albumid, size):
306 coverdir = _COVERDIR if _COVERDIR else '/tmp'
307 to = os.path.join(coverdir, '%d-%d.jpg'%(albumid, size))
308 if not os.path.isfile(to):
309 url = _GET2+'image/album/redirect/?id=%d&imagesize=%d'%(albumid, size)
310 to = self._retrieve(url, to)
315 def _fetch_image(self, url):
317 h = hashlib.md5(url).hexdigest()
318 coverdir = _COVERDIR if _COVERDIR else '/tmp'
319 to = os.path.join(coverdir, h+'.jpg')
320 if not os.path.isfile(to):
321 to = self._retrieve(url, to)
326 def request_cover(self, albumid, size, cb):
328 self.work.insert(0, (albumid, size, cb))
332 def request_images(self, urls, cb):
333 """cb([(url, image)])"""
335 self.work = [('image', url, cb) for url in urls] + self.work
351 multi = len(work) > 1
353 if job[0] == 'image':
354 self.process_image(job[1], job[2])
356 self.process_cover(*job)
358 def process_cover(self, albumid, size, cb):
359 cover = self._fetch_cover(albumid, size)
361 cb(albumid, size, cover)
363 def process_image(self, url, cb):
364 image = self._fetch_image(url)
368 class CoverCache(object):
370 cache and fetch covers
371 TODO: background thread that
372 fetches and returns covers,
376 self._covers = {} # (albumid, size) -> file
378 self._fetcher = CoverFetcher()
379 self._fetcher.start()
380 if _COVERDIR and os.path.isdir(_COVERDIR):
383 def prime_cache(self):
385 covermatch = re.compile(r'(\d+)\-(\d+)\.jpg')
387 prev_covers = os.listdir(coverdir)
389 if len(prev_covers) > _CACHED_COVERS:
391 dropn = len(prev_covers) - _CACHED_COVERS
392 todrop = random.sample(prev_covers, dropn)
393 log.warning("Deleting from cache: %s", todrop)
395 m = covermatch.match(d)
398 os.unlink(os.path.join(coverdir, d))
400 log.exception('unlinking failed')
402 for fil in os.listdir(coverdir):
403 fl = os.path.join(coverdir, fil)
404 m = covermatch.match(fil)
405 if m and os.path.isfile(fl):
406 self._covers[(int(m.group(1)), int(m.group(2)))] = fl
408 def fetch_cover(self, albumid, size):
411 to = os.path.join(coverdir, '%d-%d.jpg'%(albumid, size))
412 if not os.path.isfile(to):
413 url = _GET2+'image/album/redirect/?id=%d&imagesize=%d'%(albumid, size)
414 urllib.urlretrieve(url, to)
415 self._covers[(albumid, size)] = to
419 def get_cover(self, albumid, size):
420 cover = self._covers.get((albumid, size), None)
422 cover = self.fetch_cover(albumid, size)
425 def get_async(self, albumid, size, cb):
426 cover = self._covers.get((albumid, size), None)
428 cb(albumid, size, cover)
430 def cb2(albumid, size, cover):
431 self._covers[(albumid, size)] = cover
432 cb(albumid, size, cover)
433 self._fetcher.request_cover(albumid, size, cb2)
435 def get_images_async(self, url_list, cb):
439 image = self._images.get(url, None)
441 found.append((url, image))
449 for url, image in results:
450 self._images[url] = image
452 self._fetcher.request_images(lookup, cb2)
454 _cover_cache = CoverCache()
456 def set_cache_dir(cachedir):
460 _COVERDIR = os.path.join(_CACHEDIR, 'covers')
463 os.makedirs(_CACHEDIR)
468 os.makedirs(_COVERDIR)
472 _cover_cache.prime_cache()
474 def get_album_cover(albumid, size=100):
476 return _cover_cache.get_cover(albumid, size)
478 def get_album_cover_async(cb, albumid, size=100):
480 _cover_cache.get_async(albumid, size, cb)
482 def get_images_async(cb, url_list):
484 _cover_cache.get_images_async(url_list, cb)
486 class CustomQuery(Query):
487 def __init__(self, url):
492 return self._geturl(self.url)
497 class GetQuery(Query):
500 'url' : _GET2+'+'.join(_ARTIST_FIELDS)+'/artist/json/?',
501 'params' : 'artist_id=%d',
502 'constructor' : Artist
505 'url' : _GET2+'+'.join(_ALBUM_FIELDS)+'/artist/json/?',
506 'params' : 'artist_id=%s',
507 'constructor' : Album
510 'url' : _GET2+'+'.join(_ALBUM_FIELDS)+'/album/json/?',
511 'params' : 'album_id=%d',
512 'constructor' : Album
515 'url' : _GET2+'+'.join(_ALBUM_FIELDS)+'/album/json/?',
516 'params' : 'album_id=%s',
517 'constructor' : Album
520 'url' : _GET2+'+'.join(_ALBUM_FIELDS)+'/album/json/?',
521 'params' : 'artist_id=%d',
522 'constructor' : [Album]
525 'url' : _GET2+'+'.join(_TRACK_FIELDS)+'/track/json/track_album+album_artist?',
527 'constructor' : Track
530 'url' : _GET2+'+'.join(_TRACK_FIELDS)+'/track/json/track_album+album_artist?',
532 'constructor' : Track
535 'url' : _GET2+'+'.join(_TRACK_FIELDS)+'/track/json/track_album+album_artist?',
536 'params' : 'order=numalbum_asc&album_id=%d',
537 'constructor' : [Track]
540 'url' : _GET2+'+'.join(_TRACK_FIELDS)+'/track/json/radio_track_inradioplaylist+track_album+album_artist/?',
541 'params' : 'order=random_asc&radio_id=%d&n=16',
542 'constructor' : [Track]
544 'favorite_albums' : {
545 'url' : _GET2+'+'.join(_ALBUM_FIELDS)+'/album/json/album_user_starred/?',
546 'params' : 'user_idstr=%s',
547 'constructor' : [Album]
550 'url' : _GET2+'+'.join(_TRACK_FIELDS)+'/track/json/track_album+album_artist?',
551 'params' : 'tag_id=%d&n=50&order=rating_desc',
552 'constructor' : [Track]
556 def __init__(self, what, ID):
559 info = GetQuery.queries[what]
560 self.url = info['url']
561 self.params = info['params']
562 self.constructor = info['constructor']
564 def construct(self, data):
565 constructor = self.constructor
566 if isinstance(constructor, list):
567 constructor = constructor[0]
568 if isinstance(data, list):
569 return [constructor(int(x['id']), json=x) for x in data]
571 return constructor(int(data['id']), json=data)
574 js = self._geturl(self.url + self.params % (self.ID))
577 return self.construct(js)
580 return self.url + self.params % (self.ID)
582 class SearchQuery(GetQuery):
583 def __init__(self, what, query=None, order=None, user=None, count=20):
584 GetQuery.__init__(self, what, None)
593 params['searchquery'] = self.query
595 params['order'] = self.order
597 params['n'] = self.count
599 params['user_idstr'] = self.user
600 js = self._geturl(self.url + urllib.urlencode(params))
603 return self.construct(js)
606 params = {'searchquery':self.query, 'order':self.order, 'n':self.count}
607 return self.url + urllib.urlencode(params)
609 class JamendoAPIException(Exception):
610 def __init__(self, url):
611 Exception.__init__(self, url)
613 def _update_cache(cache, new_items):
614 if not isinstance(new_items, list):
615 new_items = [new_items]
616 for item in new_items:
617 old = cache.get(item.ID)
621 cache[item.ID] = item
622 if isinstance(item, Artist) and item.albums:
623 for album in item.albums:
624 _update_cache(_albums, album)
625 elif isinstance(item, Album) and item.tracks:
626 for track in item.tracks:
627 _update_cache(_tracks, track)
628 # enforce cache limits here!
629 # also, TODO: save/load cache between sessions
630 # that will require storing a timestamp with
631 # each item, though..
633 # artists: 1 day - changes often
634 # albums: 2-5 days - changes less often (?)
635 # tracks: 1 week - changes rarely, queried often
637 def get_artist(artist_id):
638 """Returns: Artist"""
640 a = _artists.get(artist_id, None)
642 q = GetQuery('artist', artist_id)
645 raise JamendoAPIException(str(q))
646 _update_cache(_artists, a)
647 if isinstance(a, list):
651 def get_artists(artist_ids):
652 """Returns: [Artist]"""
654 assert(isinstance(artist_ids, list))
657 for artist_id in artist_ids:
658 a = _artists.get(artist_id, None)
660 lookup.append(artist_id)
664 q = GetQuery('artist_list', '+'.join(str(x) for x in lookup))
667 raise JamendoAPIException(str(q))
668 _update_cache(_artists, a)
670 return found + lookup
672 def get_album_list(album_ids):
673 """Returns: [Album]"""
675 assert(isinstance(album_ids, list))
678 for album_id in album_ids:
679 a = _albums.get(album_id, None)
681 lookup.append(album_id)
685 q = GetQuery('album_list', '+'.join(str(x) for x in lookup))
688 raise JamendoAPIException(str(q))
689 _update_cache(_albums, a)
691 return found + lookup
693 def get_albums(artist_id):
695 Parameter can either be an artist_id or a list of album ids.
697 if isinstance(artist_id, list):
698 return get_album_list(artist_id)
700 a = _artists.get(artist_id, None)
704 q = GetQuery('albums', artist_id)
707 raise JamendoAPIException(str(q))
708 _update_cache(_albums, a)
711 def get_album(album_id):
714 a = _albums.get(album_id, None)
716 q = GetQuery('album', album_id)
719 raise JamendoAPIException(str(q))
720 _update_cache(_albums, a)
721 if isinstance(a, list):
725 def get_track_list(track_ids):
726 """Returns: [Track]"""
728 assert(isinstance(track_ids, list))
731 for track_id in track_ids:
732 a = _tracks.get(track_id, None)
734 lookup.append(track_id)
738 q = GetQuery('track_list', '+'.join(str(x) for x in lookup))
741 raise JamendoAPIException(str(q))
742 _update_cache(_tracks, a)
744 return found + lookup
746 def get_tracks(album_id):
748 Parameter can either be an album_id or a list of track ids.
750 if isinstance(album_id, list):
751 return get_track_list(album_id)
753 a = _albums.get(album_id, None)
757 q = GetQuery('tracks', album_id)
760 raise JamendoAPIException(str(q))
761 _update_cache(_tracks, a)
764 def get_track(track_id):
767 a = _tracks.get(track_id, None)
769 q = GetQuery('track', track_id)
772 raise JamendoAPIException(str(q))
773 _update_cache(_tracks, a)
774 if isinstance(a, list):
778 def get_radio_tracks(radio_id):
779 """Returns: [Track]"""
781 q = GetQuery('radio', radio_id)
784 raise JamendoAPIException(str(q))
785 _update_cache(_tracks, a)
788 #http://api.jamendo.com/get2/id+name/track/plain/?tag_id=327&n=50&order=rating_desc
789 def get_tag_tracks(tag_id):
790 """Returns: [Track]"""
792 q = GetQuery('tag', tag_id)
795 raise JamendoAPIException(str(q))
796 _update_cache(_tracks, a)
799 def search_artists(query):
800 """Returns: [Artist]"""
802 q = SearchQuery('artist', query, 'searchweight_desc')
805 raise JamendoAPIException(str(q))
806 _update_cache(_artists, a)
809 def search_albums(query):
810 """Returns: [Album]"""
812 q = SearchQuery('album', query, 'searchweight_desc')
815 raise JamendoAPIException(str(q))
816 _update_cache(_albums, a)
819 def search_tracks(query):
820 """Returns: [Track]"""
822 q = SearchQuery('track', query=query, order='searchweight_desc')
825 raise JamendoAPIException(str(q))
826 _update_cache(_tracks, a)
829 def albums_of_the_week():
830 """Returns: [Album]"""
832 q = SearchQuery('album', order='ratingweek_desc')
835 raise JamendoAPIException(str(q))
836 _update_cache(_albums, a)
840 """Returns: [Track] (playlist)"""
842 q = SearchQuery('track', order='releasedate_desc')
845 raise JamendoAPIException(str(q))
846 _update_cache(_tracks, a)
849 def tracks_of_the_week():
850 """Returns: [Track] (playlist)"""
852 q = SearchQuery('track', order='ratingweek_desc')
855 raise JamendoAPIException(str(q))
856 _update_cache(_tracks, a)
859 def top_artists(order='rating_desc', count=20):
860 """Returns: [Artist]"""
862 q = SearchQuery('artist', order=order, count=count)
865 raise JamendoAPIException(str(q))
866 _update_cache(_artists, a)
869 def top_albums(order='rating_desc', count=20):
870 """Returns: [Album]"""
872 q = SearchQuery('album', order=order, count=count)
875 raise JamendoAPIException(str(q))
876 _update_cache(_albums, a)
879 def top_tracks(order='rating_desc', count=20):
880 """Returns: [Track]"""
882 q = SearchQuery('track', order=order, count=count)
885 raise JamendoAPIException(str(q))
886 _update_cache(_tracks, a)
889 def get_radio(radio_id):
892 q = CustomQuery(_GET2+"id+name+idstr+image/radio/json?id=%d"%(radio_id))
895 raise JamendoAPIException(str(q))
896 if isinstance(js, list):
898 return Radio(radio_id, json=js)
900 def starred_radios():
901 """Returns: [Radio]"""
903 q = CustomQuery(_GET2+"id+name+idstr+image/radio/json?order=starred_desc")
906 raise JamendoAPIException(str(q))
907 return [Radio(int(radio['id']), json=radio) for radio in js]
909 def top_tags(count=50, order='rating_desc'):
912 q = CustomQuery(_GET2+"id+name/tag/json?n=%d&order=%s"%(count, order))
915 raise JamendoAPIException(str(q))
916 return [Tag(int(tag['id']), json=tag) for tag in js]
918 def favorite_albums(user):
919 """Returns: [Album]"""
921 q = SearchQuery('favorite_albums', user=user, count=20)
924 raise JamendoAPIException(str(q))
925 _update_cache(_albums, a)
928 ### Set loader functions for classes
930 def _artist_loader(self):
931 if self._needs_load():
932 artist = get_artist(self.ID)
933 artist.albums = get_albums(self.ID)
934 self._set_from(artist)
935 Artist.load = _artist_loader
937 def _album_loader(self):
938 if self._needs_load():
939 album = get_album(self.ID)
940 album.tracks = get_tracks(self.ID)
941 self._set_from(album)
942 Album.load = _album_loader
944 def _track_loader(self):
945 track = get_track(self.ID)
946 self._set_from(track)
947 Track.load = _track_loader
949 def _radio_loader(self):
950 radio = get_radio(self.ID)
951 self._set_from(radio)
952 Radio.load = _radio_loader