X-Git-Url: http://git.maemo.org/git/?p=jamaendo;a=blobdiff_plain;f=jamaendo%2Fapi.py;h=ca4e51f8845154c44976e8c963c4dd4b4f0d0bd4;hp=0d293bc04a56eeefea0544c9c372520779f778ed;hb=ae451be237b4622abd934a611f5e2dd4d8aec883;hpb=e5275c7149e6f138488f60cf1114bc58c2c8b5f6 diff --git a/jamaendo/api.py b/jamaendo/api.py index 0d293bc..ca4e51f 100644 --- a/jamaendo/api.py +++ b/jamaendo/api.py @@ -22,8 +22,10 @@ # An improved, structured jamendo API wrapper for the N900 with cacheing # Image / cover downloads.. and more? +from __future__ import with_statement import urllib, threading, os, time, simplejson, re import logging, hashlib +import pycurl, StringIO _CACHEDIR = None _COVERDIR = None @@ -48,8 +50,24 @@ except: _ARTIST_FIELDS = ['id', 'name', 'image'] _ALBUM_FIELDS = ['id', 'name', 'image', 'artist_name', 'artist_id', 'license_url'] -_TRACK_FIELDS = ['id', 'name', 'image', 'artist_id', 'artist_name', 'album_name', 'album_id', 'numalbum', 'duration'] +_TRACK_FIELDS = ['id', 'name', 'album_image', 'artist_id', 'artist_name', 'album_name', 'album_id', 'numalbum', 'duration'] _RADIO_FIELDS = ['id', 'name', 'idstr', 'image'] +_TAG_FIELDS = ['id', 'name'] + +_APILOCK = threading.Lock() + +def curlGET(url): + c = pycurl.Curl() + s = StringIO.StringIO() + c.setopt(pycurl.FOLLOWLOCATION, 1) + c.setopt(pycurl.URL, url) + c.setopt(pycurl.WRITEFUNCTION, s.write) + try: + c.perform() + finally: + c.close() + s.seek(0) + return s.read() class LazyQuery(object): def set_from_json(self, json): @@ -114,6 +132,9 @@ class Artist(LazyQuery): def _set_from(self, other): return self._set_from_impl(other, 'name', 'image', 'albums') + def get_data(self): + return {'name':self.name, 'image':self.image} + class Album(LazyQuery): def __init__(self, ID, json=None): self.ID = int(ID) @@ -136,13 +157,19 @@ class Album(LazyQuery): def _set_from(self, other): return self._set_from_impl(other, 'name', 'image', 'artist_name', 'artist_id', 'license_url', 'tracks') + def get_data(self): + return {'name':self.name, 'image':self.image, + 'artist_name':self.artist_name, + 'artist_id':self.artist_id, + 'license_url':self.license_url} + class Track(LazyQuery): def __init__(self, ID, json=None): self.ID = int(ID) self.name = None - self.image = None self.artist_id = None self.artist_name = None + self.album_image = None self.album_name = None self.album_id = None self.numalbum = None @@ -156,11 +183,21 @@ class Track(LazyQuery): def ogg_url(self): return _OGGURL%(self.ID) + def get_data(self): + return {'name':self.name, + 'artist_id':self.artist_id, + 'artist_name':self.artist_name, + 'album_image':self.album_image, + 'album_name':self.album_name, + 'album_id':self.album_id, + 'numalbum':self.numalbum, + 'duration':self.duration} + def _needs_load(self): return self._needs_load_impl('name', 'artist_name', 'artist_id', 'album_name', 'album_id', 'numalbum', 'duration') def _set_from(self, other): - return self._set_from_impl(other, 'name', 'image', 'artist_name', 'artist_id', 'album_name', 'album_id', 'numalbum', 'duration') + return self._set_from_impl(other, 'name', 'album_image', 'artist_name', 'artist_id', 'album_name', 'album_id', 'numalbum', 'duration') class Radio(LazyQuery): def __init__(self, ID, json=None): @@ -177,6 +214,18 @@ class Radio(LazyQuery): def _set_from(self, other): return self._set_from_impl(other, 'name', 'idstr', 'image') +class Tag(LazyQuery): + def __init__(self, ID, json=None): + self.ID = int(ID) + self.name = None + if json: + self.set_from_json(json) + + def _needs_load(self): + return self._needs_load_impl('name') + + def _set_from(self, other): + return self._set_from_impl(other, 'name') _artists = {} # id -> Artist() _albums = {} # id -> Album() @@ -194,27 +243,28 @@ _CACHED_COVERS = 2048 # TODO: cache queries? -class Query(object): - rate_limit = 1.1 # seconds between queries +class Ratelimit(object): + rate_limit = 1.0 # seconds between queries last_query = time.time() - 1.5 @classmethod - def _ratelimit(cls): + def ratelimit(cls): now = time.time() - if now - cls.last_query < cls.rate_limit: + if (now - cls.last_query) < cls.rate_limit: time.sleep(cls.rate_limit - (now - cls.last_query)) - cls.last_query = now + cls.last_query = time.time() + +_ratelimit = Ratelimit.ratelimit +class Query(object): def __init__(self): pass def _geturl(self, url): + _ratelimit() log.info("%s", url) - Query._ratelimit() try: - f = urllib.urlopen(url) - ret = simplejson.load(f) - f.close() + ret = simplejson.loads(curlGET(url)) except Exception, e: return None return ret @@ -232,13 +282,32 @@ class CoverFetcher(threading.Thread): self.cond = threading.Condition() self.work = [] + def _retrieve(self, url, fname): + BROKEN = 'http://imgjam.com/radios/default/default.100.png' + if url == BROKEN: + return None + f = open(fname, 'wb') + c = pycurl.Curl() + c.setopt(pycurl.FOLLOWLOCATION, 1) + c.setopt(pycurl.URL, str(url)) + c.setopt(pycurl.WRITEFUNCTION, f.write) + try: + c.perform() + except: + fname = None + finally: + c.close() + f.close() + log.debug("Coverfetch: %s -> %s", url, fname) + return fname + def _fetch_cover(self, albumid, size): try: coverdir = _COVERDIR if _COVERDIR else '/tmp' to = os.path.join(coverdir, '%d-%d.jpg'%(albumid, size)) if not os.path.isfile(to): url = _GET2+'image/album/redirect/?id=%d&imagesize=%d'%(albumid, size) - urllib.urlretrieve(url, to) + to = self._retrieve(url, to) return to except Exception, e: return None @@ -249,7 +318,7 @@ class CoverFetcher(threading.Thread): coverdir = _COVERDIR if _COVERDIR else '/tmp' to = os.path.join(coverdir, h+'.jpg') if not os.path.isfile(to): - urllib.urlretrieve(url, to) + to = self._retrieve(url, to) return to except Exception, e: return None @@ -263,7 +332,7 @@ class CoverFetcher(threading.Thread): def request_images(self, urls, cb): """cb([(url, image)])""" self.cond.acquire() - self.work.insert(0, ('images', urls, cb)) + self.work = [('image', url, cb) for url in urls] + self.work self.cond.notify() self.cond.release() @@ -281,23 +350,20 @@ class CoverFetcher(threading.Thread): multi = len(work) > 1 for job in work: - if job[0] == 'images': - self.process_images(job[1], job[2]) + if job[0] == 'image': + self.process_image(job[1], job[2]) else: self.process_cover(*job) - if multi: - time.sleep(1.0) def process_cover(self, albumid, size, cb): - albumid, size, cb = job cover = self._fetch_cover(albumid, size) if cover: cb(albumid, size, cover) - def process_images(self, urls, cb): - results = [(url, image) for url, image in ((url, self._fetch_image(url)) for url in urls) if image is not None] - if results: - cb(results) + def process_image(self, url, cb): + image = self._fetch_image(url) + if image: + cb([(url, image)]) class CoverCache(object): """ @@ -406,13 +472,16 @@ def set_cache_dir(cachedir): _cover_cache.prime_cache() def get_album_cover(albumid, size=100): - return _cover_cache.get_cover(albumid, size) + with _APILOCK: + return _cover_cache.get_cover(albumid, size) def get_album_cover_async(cb, albumid, size=100): - _cover_cache.get_async(albumid, size, cb) + with _APILOCK: + _cover_cache.get_async(albumid, size, cb) def get_images_async(cb, url_list): - _cover_cache.get_images_async(url_list, cb) + with _APILOCK: + _cover_cache.get_images_async(url_list, cb) class CustomQuery(Query): def __init__(self, url): @@ -469,7 +538,7 @@ class GetQuery(Query): }, 'radio' : { 'url' : _GET2+'+'.join(_TRACK_FIELDS)+'/track/json/radio_track_inradioplaylist+track_album+album_artist/?', - 'params' : 'order=random_asc&radio_id=%d', + 'params' : 'order=random_asc&radio_id=%d&n=16', 'constructor' : [Track] }, 'favorite_albums' : { @@ -477,6 +546,11 @@ class GetQuery(Query): 'params' : 'user_idstr=%s', 'constructor' : [Album] }, + 'tag' : { + 'url' : _GET2+'+'.join(_TRACK_FIELDS)+'/track/json/track_album+album_artist?', + 'params' : 'tag_id=%d&n=50&order=rating_desc', + 'constructor' : [Track] + }, } def __init__(self, what, ID): @@ -506,7 +580,7 @@ class GetQuery(Query): return self.url + self.params % (self.ID) class SearchQuery(GetQuery): - def __init__(self, what, query=None, order=None, user=None, count=10): + def __init__(self, what, query=None, order=None, user=None, count=20): GetQuery.__init__(self, what, None) self.query = query self.order = order @@ -562,226 +636,294 @@ def _update_cache(cache, new_items): def get_artist(artist_id): """Returns: Artist""" - a = _artists.get(artist_id, None) - if not a: - q = GetQuery('artist', artist_id) - a = q.execute() + with _APILOCK: + a = _artists.get(artist_id, None) if not a: - raise JamendoAPIException(str(q)) - _update_cache(_artists, a) - if isinstance(a, list): - a = a[0] - return a + q = GetQuery('artist', artist_id) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_artists, a) + if isinstance(a, list): + a = a[0] + return a def get_artists(artist_ids): """Returns: [Artist]""" - assert(isinstance(artist_ids, list)) - found = [] - lookup = [] - for artist_id in artist_ids: - a = _artists.get(artist_id, None) - if not a: - lookup.append(artist_id) - else: - found.append(a) - if lookup: - q = GetQuery('artist_list', '+'.join(str(x) for x in lookup)) - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_artists, a) - lookup = a - return found + lookup + with _APILOCK: + assert(isinstance(artist_ids, list)) + found = [] + lookup = [] + for artist_id in artist_ids: + a = _artists.get(artist_id, None) + if not a: + lookup.append(artist_id) + else: + found.append(a) + if lookup: + q = GetQuery('artist_list', '+'.join(str(x) for x in lookup)) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_artists, a) + lookup = a + return found + lookup def get_album_list(album_ids): """Returns: [Album]""" - assert(isinstance(album_ids, list)) - found = [] - lookup = [] - for album_id in album_ids: - a = _albums.get(album_id, None) - if not a: - lookup.append(album_id) - else: - found.append(a) - if lookup: - q = GetQuery('album_list', '+'.join(str(x) for x in lookup)) - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_albums, a) - lookup = a - return found + lookup + with _APILOCK: + assert(isinstance(album_ids, list)) + found = [] + lookup = [] + for album_id in album_ids: + a = _albums.get(album_id, None) + if not a: + lookup.append(album_id) + else: + found.append(a) + if lookup: + q = GetQuery('album_list', '+'.join(str(x) for x in lookup)) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_albums, a) + lookup = a + return found + lookup def get_albums(artist_id): """Returns: [Album] Parameter can either be an artist_id or a list of album ids. """ - if isinstance(artist_id, list): - return get_album_list(artist_id) - a = _artists.get(artist_id, None) - if a and a.albums: - return a.albums - - q = GetQuery('albums', artist_id) - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_artists, a) - return a + with _APILOCK: + if isinstance(artist_id, list): + return get_album_list(artist_id) + a = _artists.get(artist_id, None) + if a and a.albums: + return a.albums -def get_album(album_id): - """Returns: Album""" - a = _albums.get(album_id, None) - if not a: - q = GetQuery('album', album_id) + q = GetQuery('albums', artist_id) a = q.execute() if not a: raise JamendoAPIException(str(q)) _update_cache(_albums, a) - if isinstance(a, list): - a = a[0] - return a + return a + +def get_album(album_id): + """Returns: Album""" + with _APILOCK: + a = _albums.get(album_id, None) + if not a: + q = GetQuery('album', album_id) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_albums, a) + if isinstance(a, list): + a = a[0] + return a def get_track_list(track_ids): """Returns: [Track]""" - assert(isinstance(track_ids, list)) - found = [] - lookup = [] - for track_id in track_ids: - a = _tracks.get(track_id, None) - if not a: - lookup.append(track_id) - else: - found.append(a) - if lookup: - q = GetQuery('track_list', '+'.join(str(x) for x in lookup)) - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_tracks, a) - lookup = a - return found + lookup + with _APILOCK: + assert(isinstance(track_ids, list)) + found = [] + lookup = [] + for track_id in track_ids: + a = _tracks.get(track_id, None) + if not a: + lookup.append(track_id) + else: + found.append(a) + if lookup: + q = GetQuery('track_list', '+'.join(str(x) for x in lookup)) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_tracks, a) + lookup = a + return found + lookup def get_tracks(album_id): """Returns: [Track] Parameter can either be an album_id or a list of track ids. """ - if isinstance(album_id, list): - return get_track_list(album_id) - a = _albums.get(album_id, None) - if a and a.tracks: - return a.tracks - - q = GetQuery('tracks', album_id) - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_tracks, a) - return a + with _APILOCK: + if isinstance(album_id, list): + return get_track_list(album_id) + a = _albums.get(album_id, None) + if a and a.tracks: + return a.tracks + + q = GetQuery('tracks', album_id) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_tracks, a) + return a def get_track(track_id): """Returns: Track""" - a = _tracks.get(track_id, None) - if not a: - q = GetQuery('track', track_id) + with _APILOCK: + a = _tracks.get(track_id, None) + if not a: + q = GetQuery('track', track_id) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_tracks, a) + if isinstance(a, list): + a = a[0] + return a + +def get_radio_tracks(radio_id): + """Returns: [Track]""" + with _APILOCK: + q = GetQuery('radio', radio_id) a = q.execute() if not a: raise JamendoAPIException(str(q)) _update_cache(_tracks, a) - if isinstance(a, list): - a = a[0] - return a + return a -def get_radio_tracks(radio_id): +#http://api.jamendo.com/get2/id+name/track/plain/?tag_id=327&n=50&order=rating_desc +def get_tag_tracks(tag_id): """Returns: [Track]""" - q = GetQuery('radio', radio_id) - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_tracks, a) - return a + with _APILOCK: + q = GetQuery('tag', tag_id) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_tracks, a) + return a def search_artists(query): """Returns: [Artist]""" - q = SearchQuery('artist', query, 'searchweight_desc') - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_artists, a) - return a + with _APILOCK: + q = SearchQuery('artist', query, 'searchweight_desc') + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_artists, a) + return a def search_albums(query): """Returns: [Album]""" - q = SearchQuery('album', query, 'searchweight_desc') - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_albums, a) - return a + with _APILOCK: + q = SearchQuery('album', query, 'searchweight_desc') + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_albums, a) + return a def search_tracks(query): """Returns: [Track]""" - q = SearchQuery('track', query=query, order='searchweight_desc') - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_tracks, a) - return a + with _APILOCK: + q = SearchQuery('track', query=query, order='searchweight_desc') + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_tracks, a) + return a def albums_of_the_week(): """Returns: [Album]""" - q = SearchQuery('album', order='ratingweek_desc') - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_albums, a) - return a + with _APILOCK: + q = SearchQuery('album', order='ratingweek_desc') + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_albums, a) + return a def new_releases(): """Returns: [Track] (playlist)""" - q = SearchQuery('track', order='releasedate_desc') - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_tracks, a) - return a + with _APILOCK: + q = SearchQuery('track', order='releasedate_desc') + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_tracks, a) + return a def tracks_of_the_week(): """Returns: [Track] (playlist)""" - q = SearchQuery('track', order='ratingweek_desc') - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_tracks, a) - return a + with _APILOCK: + q = SearchQuery('track', order='ratingweek_desc') + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_tracks, a) + return a + +def top_artists(order='rating_desc', count=20): + """Returns: [Artist]""" + with _APILOCK: + q = SearchQuery('artist', order=order, count=count) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_artists, a) + return a + +def top_albums(order='rating_desc', count=20): + """Returns: [Album]""" + with _APILOCK: + q = SearchQuery('album', order=order, count=count) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_albums, a) + return a + +def top_tracks(order='rating_desc', count=20): + """Returns: [Track]""" + with _APILOCK: + q = SearchQuery('track', order=order, count=count) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_tracks, a) + return a def get_radio(radio_id): """Returns: Radio""" - q = CustomQuery(_GET2+"id+name+idstr+image/radio/json?id=%d"%(radio_id)) - js = q.execute() - if not js: - raise JamendoAPIException(str(q)) - if isinstance(js, list): - ks = js[0] - return Radio(radio_id, json=js) + with _APILOCK: + q = CustomQuery(_GET2+"id+name+idstr+image/radio/json?id=%d"%(radio_id)) + js = q.execute() + if not js: + raise JamendoAPIException(str(q)) + if isinstance(js, list): + ks = js[0] + return Radio(radio_id, json=js) def starred_radios(): """Returns: [Radio]""" - q = CustomQuery(_GET2+"id+name+idstr+image/radio/json?order=starred_desc") - js = q.execute() - if not js: - raise JamendoAPIException(str(q)) - return [Radio(int(radio['id']), json=radio) for radio in js] + with _APILOCK: + q = CustomQuery(_GET2+"id+name+idstr+image/radio/json?order=starred_desc") + js = q.execute() + if not js: + raise JamendoAPIException(str(q)) + return [Radio(int(radio['id']), json=radio) for radio in js] + +def top_tags(count=50, order='rating_desc'): + """Returns: [Tag]""" + with _APILOCK: + q = CustomQuery(_GET2+"id+name/tag/json?n=%d&order=%s"%(count, order)) + js = q.execute() + if not js: + raise JamendoAPIException(str(q)) + return [Tag(int(tag['id']), json=tag) for tag in js] def favorite_albums(user): """Returns: [Album]""" - q = SearchQuery('favorite_albums', user=user, count=20) - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_albums, a) - return a + with _APILOCK: + q = SearchQuery('favorite_albums', user=user, count=20) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_albums, a) + return a ### Set loader functions for classes