X-Git-Url: http://git.maemo.org/git/?p=jamaendo;a=blobdiff_plain;f=jamaendo%2Fapi.py;h=a376b7d9a5ca3623d25136a30349608c9f22c2cc;hp=9913fcc6f2262055a1cd9703cc1d2398c265ee6a;hb=75215e5b54a5357384db5166fbecaa65164d8b94;hpb=91f0f639cd4244e40171959a16112026a8c9db87 diff --git a/jamaendo/api.py b/jamaendo/api.py index 9913fcc..a376b7d 100644 --- a/jamaendo/api.py +++ b/jamaendo/api.py @@ -1,204 +1,547 @@ -import urllib, threading, os, gzip, time, json, re -_DUMP_URL = '''http://img.jamendo.com/data/dbdump_artistalbumtrack.xml.gz''' -_DUMP = os.path.expanduser('''~/.cache/jamaendo/dbdump.xml.gz''') - -try: - os.makedirs(os.path.dirname(_DUMP)) -except OSError: - pass - -def has_dump(): - return os.path.isfile(_DUMP) - -def _file_is_old(fil, old_age): - return os.path.getmtime(fil) < (time.time() - old_age) - -def _dump_is_old(): - return not has_dump() or _file_is_old(_DUMP, 60*60*24) # 1 day - -def refresh_dump(complete_callback, progress_callback=None, force=False): - if force or _dump_is_old(): - downloader = Downloader(complete_callback, progress_callback) - downloader.start() - else: - complete_callback() - -class Downloader(threading.Thread): - def __init__(self, complete_callback, progress_callback): - threading.Thread.__init__(self) - self.complete_callback = complete_callback - self.progress_callback = progress_callback - - def actual_callback(self, numblocks, blocksize, filesize): - if self.progress_callback: - try: - percent = min((numblocks*blocksize*100)/filesize, 100) - except: - percent = 100 - self.progress_callback(percent) - - def run(self): - urllib.urlretrieve(_DUMP_URL, _DUMP, self.actual_callback) - self.complete_callback() - -def fast_iter(context, func): - for event, elem in context: - func(elem) - elem.clear() - while elem.getprevious() is not None: - del elem.getparent()[0] - del context - -from lxml import etree - -class Obj(object): - def __repr__(self): - def printable(v): - if isinstance(v, basestring): - return v.encode('utf-8') +# An improved, structured jamendo API for the N900 with cacheing +# Image / cover downloads.. and more? +import urllib, threading, os, gzip, time, simplejson, re +#import util +#if util.platform == 'maemo': +# _CACHEDIR = os.path.expanduser('''~/MyDocs/.jamaendo''') +#else: +# _CACHEDIR = os.path.expanduser('''~/.cache/jamaendo''') + +_CACHEDIR = None#'/tmp/jamaendo' +_COVERDIR = None#os.path.join(_CACHEDIR, 'covers') +_GET2 = '''http://api.jamendo.com/get2/''' +_MP3URL = _GET2+'stream/track/redirect/?id=%d&streamencoding=mp31' +_OGGURL = _GET2+'stream/track/redirect/?id=%d&streamencoding=ogg2' + + +def set_cache_dir(cachedir): + global _CACHEDIR + global _COVERDIR + _CACHEDIR = cachedir + _COVERDIR = os.path.join(_CACHEDIR, 'covers') + + try: + os.makedirs(_CACHEDIR) + except OSError: + pass + + try: + os.makedirs(_COVERDIR) + except OSError: + pass + +# These classes can be partially constructed, +# and if asked for a property they don't know, +# makes a query internally to get the full story + +_ARTIST_FIELDS = ['id', 'name', 'image'] +_ALBUM_FIELDS = ['id', 'name', 'image', 'artist_name', 'artist_id', 'license_url'] +_TRACK_FIELDS = ['id', 'name', 'image', 'artist_name', 'album_name', 'album_id', 'numalbum', 'duration'] +_RADIO_FIELDS = ['id', 'name', 'idstr', 'image'] + +class LazyQuery(object): + def set_from_json(self, json): + for key, value in json.iteritems(): + if key == 'id': + assert(self.ID == int(value)) else: - return str(v) - return "{%s}" % (", ".join("%s=%s"%(k.encode('utf-8'), printable(v)) \ - for k,v in self.__dict__.iteritems() if not k.startswith('_'))) + if key.endswith('_id'): + value = int(value) + setattr(self, key, value) -class DB(object): - def __init__(self): - self.fil = None + def load(self): + """Not automatic for now, + will have to do artist.load() - def connect(self): - self.fil = gzip.open(_DUMP) + This is filled in further down + in the file + """ + raise NotImplemented - def close(self): - self.fil.close() + def _needs_load(self): + return True - def make_artist_obj(self, element): - if element.text is not None and element.text != "": - return element.text - else: - ret = {} - for child in element: - if child.tag in ['name', 'id', 'image']: - ret[child.tag] = child.text - return ret - - def make_album_obj(self, element): - if element.text is not None and element.text != "": - return element.text - else: - ret = {} - for child in element: - if child.tag in ['name', 'id', 'image']: - if child.text: - ret[child.tag] = child.text - else: - ret[child.tag] = "" - if child.tag == 'Tracks': - tracks = [] - for track in child: - trackd = {} - for trackinfo in track: - if trackinfo.tag in ['name', 'id', 'numalbum']: - trackd[trackinfo.tag] = trackinfo.text - tracks.append(trackd) - ret['tracks'] = tracks - return ret - - def artist_walker(self, name_match): - for event, element in etree.iterparse(self.fil, tag="artist"): - name = element.xpath('./name')[0].text.lower() - if name and name.find(name_match) > -1: - yield self.make_artist_obj(element) - element.clear() - while element.getprevious() is not None: - del element.getparent()[0] - raise StopIteration - - def album_walker(self, name_match): - for event, element in etree.iterparse(self.fil, tag="album"): - name = element.xpath('./name')[0].text - if name and name.lower().find(name_match) > -1: - yield self.make_album_obj(element) - element.clear() - while element.getprevious() is not None: - del element.getparent()[0] - raise StopIteration - - def artistid_walker(self, artistids): - for event, element in etree.iterparse(self.fil, tag="artist"): - _id = element.xpath('./id')[0].text - if _id and int(_id) in artistids: - yield self.make_artist_obj(element) - element.clear() - while element.getprevious() is not None: - del element.getparent()[0] - raise StopIteration - - def albumid_walker(self, albumids): - for event, element in etree.iterparse(self.fil, tag="album"): - _id = element.xpath('./id')[0].text - if _id and int(_id) in albumids: - yield self.make_album_obj(element) - element.clear() - while element.getprevious() is not None: - del element.getparent()[0] - raise StopIteration - - def search_artists(self, substr): - substr = substr.lower() - return (artist for artist in self.artist_walker(substr)) - - def search_albums(self, substr): - substr = substr.lower() - return (album for album in self.album_walker(substr)) - - def get_album(self, artistid): - return (artist for artist in self.artistid_walker(artistid)) - - def get_album(self, albumid): - return (album for album in self.albumid_walker(albumid)) + def _set_from(self, other): + raise NotImplemented -_GET2 = '''http://api.jamendo.com/get2/''' + def _needs_load_impl(self, *attrs): + for attr in attrs: + if getattr(self, attr) is None: + return True + return False -class Query(object): - last_query = time.time() + def _set_from_impl(self, other, *attrs): + for attr in attrs: + self._set_if(other, attr) - def __init__(self, order, select=['id', 'name', 'image', 'artist_name'], request='album', track=None, n=8): - if request == 'track': - self.url = "%s%s/%s/json/%s?n=%s&order=%s" % (_GET2, '+'.join(select), request, '+'.join(track), n, order) - else: - self.url = "%s%s/%s/json/?n=%s&order=%s" % (_GET2, '+'.join(select), request, n, order) + def _set_if(self, other, attrname): + if getattr(self, attrname) is None and getattr(other, attrname) is not None: + setattr(self, attrname, getattr(other, attrname)) - def emit(self): - """ratelimited query""" + def __repr__(self): + try: + return u"%s(%s)"%(self.__class__.__name__, + u", ".join(repr(v) for k,v in self.__dict__.iteritems() if not k.startswith('_'))) + except UnicodeEncodeError: + import traceback + traceback.print_exc() + return u"%s(?)"%(self.__class__.__name__) + +class Artist(LazyQuery): + def __init__(self, ID, json=None): + self.ID = int(ID) + self.name = None + self.image = None + self.albums = None # None means not downloaded + if json: + self.set_from_json(json) + + def _needs_load(self): + return self._needs_load_impl('name', 'albums') + + def _set_from(self, other): + return self._set_from_impl(other, 'name', 'image', 'albums') + +class Album(LazyQuery): + def __init__(self, ID, json=None): + self.ID = int(ID) + self.name = None + self.image = None + self.artist_name = None + self.artist_id = None + self.license_url = None + self.tracks = None # None means not downloaded + if json: + self.set_from_json(json) + + def _needs_load(self): + return self._needs_load_impl('name', 'image', 'artist_name', 'artist_id', 'license_url', 'tracks') + + def _set_from(self, other): + return self._set_from_impl(other, 'name', 'image', 'artist_name', 'artist_id', 'license_url', 'tracks') + +class Track(LazyQuery): + def __init__(self, ID, json=None): + self.ID = int(ID) + self.name = None + self.image = None + self.artist_name = None + self.album_name = None + self.album_id = None + self.numalbum = None + self.duration = None + if json: + self.set_from_json(json) + + def mp3_url(self): + return _MP3URL%(self.ID) + + def ogg_url(self): + return _OGGURL%(self.ID) + + def _needs_load(self): + return self._needs_load_impl('name', 'artist_name', 'album_name', 'album_id', 'numalbum', 'duration') + + def _set_from(self, other): + return self._set_from_impl(other, 'name', 'image', 'artist_name', 'album_name', 'album_id', 'numalbum', 'duration') + +class Radio(LazyQuery): + def __init__(self, ID, json=None): + self.ID = int(ID) + self.name = None + self.idstr = None + self.image = None + if json: + self.set_from_json(json) + + def _needs_load(self): + return self._needs_load_impl('name', 'idstr', 'image') + + def _set_from(self, other): + return self._set_from_impl(other, 'name', 'idstr', 'image') + + +_artists = {} # id -> Artist() +_albums = {} # id -> Album() +_tracks = {} # id -> Track() +_radios = {} # id -> Radio() + + +# cache sizes per session (TODO) +_CACHED_ARTISTS = 100 +_CACHED_ALBUMS = 200 +_CACHED_TRACKS = 500 +_CACHED_RADIOS = 10 + +# TODO: cache queries? + +class Query(object): + rate_limit = 1.1 # seconds between queries + last_query = time.time() - 1.5 + + @classmethod + def _ratelimit(cls): now = time.time() - if now - self.last_query < 1.0: - time.sleep(1.0 - (now - self.last_query)) - self.last_query = now - f = urllib.urlopen(self.url) - ret = json.load(f) + if now - cls.last_query < cls.rate_limit: + time.sleep(cls.rate_limit - (now - cls.last_query)) + cls.last_query = now + + def __init__(self): + pass + + def _geturl(self, url): + print "*** %s" % (url) + Query._ratelimit() + f = urllib.urlopen(url) + ret = simplejson.load(f) f.close() return ret -class Queries(object): - albums_this_week = Query(order='ratingweek_desc') - albums_all_time = Query(order='ratingtotal_desc') - albums_this_month = Query(order='ratingmonth_desc') - albums_today = Query(order='ratingday_desc') - playlists_all_time = Query(select=['id','name', 'user_idstr'], request='playlist', order='ratingtotal_desc') - tracks_this_month = Query(select=['id', 'name', 'url', 'stream', 'album_name', 'album_url', 'album_id', 'artist_id', 'artist_name'], - request='track', - track=['track_album', 'album_artist'], - order='ratingmonth_desc') - -def get_cover(albumid, size=200): - to = '~/.cache/jamaendo/cover-%d-%d.jpg'%(albumid, size) - if not os.path.isfile(to): - url = _GET2+'image/album/redirect/?id=%d&imagesize=%d'%(albumid, size) - urllib.urlretrieve(url, to) - return to - -def get_ogg_url(trackid): - return _GET2+ 'stream/track/redirect/?id=%d&streamencoding=ogg2'%(trackid) - -def get_mp3_url(trackid): - return _GET2+ 'stream/track/redirect/?id=%d&streamencoding=mp31'%(trackid) + def __str__(self): + return "#%s" % (self.__class__.__name__) + + def execute(self): + raise NotImplemented + +class CoverCache(object): + """ + cache and fetch covers + TODO: background thread that + fetches and returns covers, + asynchronously, LIFO + """ + def __init__(self): + self._covers = {} # (albumid, size) -> file + coverdir = _COVERDIR if _COVERDIR else '/tmp' + if os.path.isdir(coverdir): + covermatch = re.compile(r'(\d+)\-(\d+)\.jpg') + for fil in os.listdir(coverdir): + fl = os.path.join(coverdir, fil) + m = covermatch.match(fil) + if m and os.path.isfile(fl): + self._covers[(int(m.group(1)), int(m.group(2)))] = fl + + def fetch_cover(self, albumid, size): + coverdir = _COVERDIR if _COVERDIR else '/tmp' + to = os.path.join(coverdir, '%d-%d.jpg'%(albumid, size)) + if not os.path.isfile(to): + url = _GET2+'image/album/redirect/?id=%d&imagesize=%d'%(albumid, size) + urllib.urlretrieve(url, to) + self._covers[(albumid, size)] = to + return to + + def get_cover(self, albumid, size): + cover = self._covers.get((albumid, size), None) + if not cover: + cover = self.fetch_cover(albumid, size) + return cover + + def get_async(self, albumid, size, cb): + cover = self._covers.get((albumid, size), None) + if cover: + cb(cover) + else: + # TODO + cover = self.fetch_cover(albumid, size) + cb(cover) + +_cover_cache = CoverCache() + +def get_album_cover(albumid, size=200): + return _cover_cache.get_cover(albumid, size) + +def get_album_cover_async(cb, albumid, size=200): + _cover_cache.get_async(albumid, size, cb) + +class CustomQuery(Query): + def __init__(self, url): + Query.__init__(self) + self.url = url + + def execute(self): + return self._geturl(self.url) + + def __str__(self): + return self.url + +class GetQuery(Query): + queries = { + 'artist' : { + 'url' : _GET2+'+'.join(_ARTIST_FIELDS)+'/artist/json/?', + 'params' : 'artist_id=%d', + 'constructor' : Artist + }, + 'album' : { + 'url' : _GET2+'+'.join(_ALBUM_FIELDS)+'/album/json/?', + 'params' : 'album_id=%d', + 'constructor' : Album + }, + 'albums' : { + 'url' : _GET2+'+'.join(_ALBUM_FIELDS)+'/album/json/?', + 'params' : 'artist_id=%d', + 'constructor' : [Album] + }, + 'track' : { + 'url' : _GET2+'+'.join(_TRACK_FIELDS)+'/track/json/track_album+album_artist?', + 'params' : 'id=%d', + 'constructor' : Track + }, + 'tracks' : { + 'url' : _GET2+'+'.join(_TRACK_FIELDS)+'/track/json/track_album+album_artist?', + 'params' : 'order=numalbum_asc&album_id=%d', + 'constructor' : [Track] + }, + 'radio' : { + 'url' : _GET2+'+'.join(_TRACK_FIELDS)+'/track/json/radio_track_inradioplaylist+track_album+album_artist/?', + 'params' : 'order=random_asc&radio_id=%d', + 'constructor' : [Track] + }, + 'favorite_albums' : { + 'url' : _GET2+'+'.join(_ALBUM_FIELDS)+'/album/json/album_user_starred/?', + 'params' : 'user_idstr=%s', + 'constructor' : [Album] + }, + #http://api.jamendo.com/get2/id+name+url+image+artist_name/album/jsonpretty/album_user_starred/?user_idstr=sylvinus&n=all + #q = SearchQuery('album', user_idstr=user) + + } +#http://api.jamendo.com/get2/id+name+image+artist_name+album_name+album_id+numalbum+duration/track/json/radio_track_inradioplaylist+track_album+album_artist/?order=numradio_asc&radio_id=283 + + def __init__(self, what, ID): + Query.__init__(self) + self.ID = ID + info = GetQuery.queries[what] + self.url = info['url'] + self.params = info['params'] + self.constructor = info['constructor'] + + def construct(self, data): + constructor = self.constructor + if isinstance(constructor, list): + constructor = constructor[0] + if isinstance(data, list): + return [constructor(int(x['id']), json=x) for x in data] + else: + return constructor(int(data['id']), json=data) + + def execute(self): + js = self._geturl(self.url + self.params % (self.ID)) + if not js: + return None + return self.construct(js) + + def __str__(self): + return self.url + self.params % (self.ID) + +class SearchQuery(GetQuery): + def __init__(self, what, query=None, order=None, user=None, count=10): + GetQuery.__init__(self, what, None) + self.query = query + self.order = order + self.count = count + self.user = user + + def execute(self): + params = {} + if self.query: + params['searchquery'] = self.query + if self.order: + params['order'] = self.order + if self.count: + params['n'] = self.count + if self.user: + params['user_idstr'] = self.user + js = self._geturl(self.url + urllib.urlencode(params)) + if not js: + return None + return self.construct(js) + + def __str__(self): + params = {'searchquery':self.query, 'order':self.order, 'n':self.count} + return self.url + urllib.urlencode(params) + +class JamendoAPIException(Exception): + def __init__(self, url): + Exception.__init__(self, url) + +def _update_cache(cache, new_items): + if not isinstance(new_items, list): + new_items = [new_items] + for item in new_items: + old = cache.get(item.ID) + if old: + old._set_from(item) + else: + cache[item.ID] = item + +def get_artist(artist_id): + """Returns: Artist""" + a = _artists.get(artist_id, None) + if not a: + q = GetQuery('artist', artist_id) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_artists, a) + if isinstance(a, list): + a = a[0] + return a + +def get_albums(artist_id): + """Returns: [Album]""" + q = GetQuery('albums', artist_id) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_artists, a) + return a + +def get_album(album_id): + """Returns: Album""" + a = _albums.get(album_id, None) + if not a: + q = GetQuery('album', album_id) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_albums, a) + if isinstance(a, list): + a = a[0] + return a + +def get_tracks(album_id): + """Returns: [Track]""" + q = GetQuery('tracks', album_id) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_tracks, a) + return a + +def get_track(track_id): + """Returns: Track""" + a = _tracks.get(track_id, None) + if not a: + q = GetQuery('track', track_id) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_tracks, a) + if isinstance(a, list): + a = a[0] + return a + +def get_radio_tracks(radio_id): + """Returns: [Track]""" + q = GetQuery('radio', radio_id) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_tracks, a) + return a + +def search_artists(query): + """Returns: [Artist]""" + q = SearchQuery('artist', query, 'searchweight_desc') + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_artists, a) + return a + +def search_albums(query): + """Returns: [Album]""" + q = SearchQuery('album', query, 'searchweight_desc') + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_albums, a) + return a + +def search_tracks(query): + """Returns: [Track]""" + q = SearchQuery('track', query=query, order='searchweight_desc') + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_tracks, a) + return a + +def albums_of_the_week(): + """Returns: [Album]""" + q = SearchQuery('album', order='ratingweek_desc') + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_albums, a) + return a + +def new_releases(): + """Returns: [Track] (playlist)""" + q = SearchQuery('track', order='releasedate_desc') + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_tracks, a) + return a + +def tracks_of_the_week(): + """Returns: [Track] (playlist)""" + q = SearchQuery('track', order='ratingweek_desc') + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_tracks, a) + return a + +def get_radio(radio_id): + """Returns: Radio""" + q = CustomQuery(_GET2+"id+name+idstr+image/radio/json?id=%d"%(radio_id)) + js = q.execute() + if not js: + raise JamendoAPIException(str(q)) + if isinstance(js, list): + ks = js[0] + return Radio(radio_id, json=js) + +def starred_radios(): + """Returns: [Radio]""" + q = CustomQuery(_GET2+"id+name+idstr+image/radio/json?order=starred_desc") + js = q.execute() + if not js: + raise JamendoAPIException(str(q)) + return [Radio(int(radio['id']), json=radio) for radio in js] + +def favorite_albums(user): + """Returns: [Album]""" + q = SearchQuery('favorite_albums', user=user, count=20) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_albums, a) + return a + +### Set loader functions for classes + +def _artist_loader(self): + if self._needs_load(): + artist = get_artist(self.ID) + self._set_from(artist) +Artist.load = _artist_loader + +def _album_loader(self): + if self._needs_load(): + album = get_album(self.ID) + album.tracks = get_tracks(self.ID) + self._set_from(album) +Album.load = _album_loader + +def _track_loader(self): + track = get_track(self.ID) + self._set_from(track) +Track.load = _track_loader + +def _radio_loader(self): + radio = get_radio(self.ID) + self._set_from(radio) +Radio.load = _radio_loader