From: Kristoffer Grönlund Date: Thu, 7 Jan 2010 23:37:55 +0000 (+0100) Subject: Background loading, initially implemented for search window X-Git-Url: http://git.maemo.org/git/?p=jamaendo;a=commitdiff_plain;h=ae451be237b4622abd934a611f5e2dd4d8aec883 Background loading, initially implemented for search window --- diff --git a/jamaendo/api.py b/jamaendo/api.py index 44cec73..ca4e51f 100644 --- a/jamaendo/api.py +++ b/jamaendo/api.py @@ -22,6 +22,7 @@ # An improved, structured jamendo API wrapper for the N900 with cacheing # Image / cover downloads.. and more? +from __future__ import with_statement import urllib, threading, os, time, simplejson, re import logging, hashlib import pycurl, StringIO @@ -53,6 +54,8 @@ _TRACK_FIELDS = ['id', 'name', 'album_image', 'artist_id', 'artist_name', 'album _RADIO_FIELDS = ['id', 'name', 'idstr', 'image'] _TAG_FIELDS = ['id', 'name'] +_APILOCK = threading.Lock() + def curlGET(url): c = pycurl.Curl() s = StringIO.StringIO() @@ -469,13 +472,16 @@ def set_cache_dir(cachedir): _cover_cache.prime_cache() def get_album_cover(albumid, size=100): - return _cover_cache.get_cover(albumid, size) + with _APILOCK: + return _cover_cache.get_cover(albumid, size) def get_album_cover_async(cb, albumid, size=100): - _cover_cache.get_async(albumid, size, cb) + with _APILOCK: + _cover_cache.get_async(albumid, size, cb) def get_images_async(cb, url_list): - _cover_cache.get_images_async(url_list, cb) + with _APILOCK: + _cover_cache.get_images_async(url_list, cb) class CustomQuery(Query): def __init__(self, url): @@ -630,271 +636,294 @@ def _update_cache(cache, new_items): def get_artist(artist_id): """Returns: Artist""" - a = _artists.get(artist_id, None) - if not a: - q = GetQuery('artist', artist_id) - a = q.execute() + with _APILOCK: + a = _artists.get(artist_id, None) if not a: - raise JamendoAPIException(str(q)) - _update_cache(_artists, a) - if isinstance(a, list): - a = a[0] - return a + q = GetQuery('artist', artist_id) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_artists, a) + if isinstance(a, list): + a = a[0] + return a def get_artists(artist_ids): """Returns: [Artist]""" - assert(isinstance(artist_ids, list)) - found = [] - lookup = [] - for artist_id in artist_ids: - a = _artists.get(artist_id, None) - if not a: - lookup.append(artist_id) - else: - found.append(a) - if lookup: - q = GetQuery('artist_list', '+'.join(str(x) for x in lookup)) - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_artists, a) - lookup = a - return found + lookup + with _APILOCK: + assert(isinstance(artist_ids, list)) + found = [] + lookup = [] + for artist_id in artist_ids: + a = _artists.get(artist_id, None) + if not a: + lookup.append(artist_id) + else: + found.append(a) + if lookup: + q = GetQuery('artist_list', '+'.join(str(x) for x in lookup)) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_artists, a) + lookup = a + return found + lookup def get_album_list(album_ids): """Returns: [Album]""" - assert(isinstance(album_ids, list)) - found = [] - lookup = [] - for album_id in album_ids: - a = _albums.get(album_id, None) - if not a: - lookup.append(album_id) - else: - found.append(a) - if lookup: - q = GetQuery('album_list', '+'.join(str(x) for x in lookup)) - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_albums, a) - lookup = a - return found + lookup + with _APILOCK: + assert(isinstance(album_ids, list)) + found = [] + lookup = [] + for album_id in album_ids: + a = _albums.get(album_id, None) + if not a: + lookup.append(album_id) + else: + found.append(a) + if lookup: + q = GetQuery('album_list', '+'.join(str(x) for x in lookup)) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_albums, a) + lookup = a + return found + lookup def get_albums(artist_id): """Returns: [Album] Parameter can either be an artist_id or a list of album ids. """ - if isinstance(artist_id, list): - return get_album_list(artist_id) - a = _artists.get(artist_id, None) - if a and a.albums: - return a.albums - - q = GetQuery('albums', artist_id) - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_albums, a) - return a + with _APILOCK: + if isinstance(artist_id, list): + return get_album_list(artist_id) + a = _artists.get(artist_id, None) + if a and a.albums: + return a.albums -def get_album(album_id): - """Returns: Album""" - a = _albums.get(album_id, None) - if not a: - q = GetQuery('album', album_id) + q = GetQuery('albums', artist_id) a = q.execute() if not a: raise JamendoAPIException(str(q)) _update_cache(_albums, a) - if isinstance(a, list): - a = a[0] - return a + return a + +def get_album(album_id): + """Returns: Album""" + with _APILOCK: + a = _albums.get(album_id, None) + if not a: + q = GetQuery('album', album_id) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_albums, a) + if isinstance(a, list): + a = a[0] + return a def get_track_list(track_ids): """Returns: [Track]""" - assert(isinstance(track_ids, list)) - found = [] - lookup = [] - for track_id in track_ids: - a = _tracks.get(track_id, None) - if not a: - lookup.append(track_id) - else: - found.append(a) - if lookup: - q = GetQuery('track_list', '+'.join(str(x) for x in lookup)) - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_tracks, a) - lookup = a - return found + lookup + with _APILOCK: + assert(isinstance(track_ids, list)) + found = [] + lookup = [] + for track_id in track_ids: + a = _tracks.get(track_id, None) + if not a: + lookup.append(track_id) + else: + found.append(a) + if lookup: + q = GetQuery('track_list', '+'.join(str(x) for x in lookup)) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_tracks, a) + lookup = a + return found + lookup def get_tracks(album_id): """Returns: [Track] Parameter can either be an album_id or a list of track ids. """ - if isinstance(album_id, list): - return get_track_list(album_id) - a = _albums.get(album_id, None) - if a and a.tracks: - return a.tracks - - q = GetQuery('tracks', album_id) - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_tracks, a) - return a + with _APILOCK: + if isinstance(album_id, list): + return get_track_list(album_id) + a = _albums.get(album_id, None) + if a and a.tracks: + return a.tracks -def get_track(track_id): - """Returns: Track""" - a = _tracks.get(track_id, None) - if not a: - q = GetQuery('track', track_id) + q = GetQuery('tracks', album_id) a = q.execute() if not a: raise JamendoAPIException(str(q)) _update_cache(_tracks, a) - if isinstance(a, list): - a = a[0] - return a + return a + +def get_track(track_id): + """Returns: Track""" + with _APILOCK: + a = _tracks.get(track_id, None) + if not a: + q = GetQuery('track', track_id) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_tracks, a) + if isinstance(a, list): + a = a[0] + return a def get_radio_tracks(radio_id): """Returns: [Track]""" - q = GetQuery('radio', radio_id) - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_tracks, a) - return a + with _APILOCK: + q = GetQuery('radio', radio_id) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_tracks, a) + return a #http://api.jamendo.com/get2/id+name/track/plain/?tag_id=327&n=50&order=rating_desc def get_tag_tracks(tag_id): """Returns: [Track]""" - q = GetQuery('tag', tag_id) - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_tracks, a) - return a + with _APILOCK: + q = GetQuery('tag', tag_id) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_tracks, a) + return a def search_artists(query): """Returns: [Artist]""" - q = SearchQuery('artist', query, 'searchweight_desc') - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_artists, a) - return a + with _APILOCK: + q = SearchQuery('artist', query, 'searchweight_desc') + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_artists, a) + return a def search_albums(query): """Returns: [Album]""" - q = SearchQuery('album', query, 'searchweight_desc') - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_albums, a) - return a + with _APILOCK: + q = SearchQuery('album', query, 'searchweight_desc') + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_albums, a) + return a def search_tracks(query): """Returns: [Track]""" - q = SearchQuery('track', query=query, order='searchweight_desc') - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_tracks, a) - return a + with _APILOCK: + q = SearchQuery('track', query=query, order='searchweight_desc') + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_tracks, a) + return a def albums_of_the_week(): """Returns: [Album]""" - q = SearchQuery('album', order='ratingweek_desc') - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_albums, a) - return a + with _APILOCK: + q = SearchQuery('album', order='ratingweek_desc') + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_albums, a) + return a def new_releases(): """Returns: [Track] (playlist)""" - q = SearchQuery('track', order='releasedate_desc') - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_tracks, a) - return a + with _APILOCK: + q = SearchQuery('track', order='releasedate_desc') + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_tracks, a) + return a def tracks_of_the_week(): """Returns: [Track] (playlist)""" - q = SearchQuery('track', order='ratingweek_desc') - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_tracks, a) - return a + with _APILOCK: + q = SearchQuery('track', order='ratingweek_desc') + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_tracks, a) + return a def top_artists(order='rating_desc', count=20): """Returns: [Artist]""" - q = SearchQuery('artist', order=order, count=count) - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_artists, a) - return a + with _APILOCK: + q = SearchQuery('artist', order=order, count=count) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_artists, a) + return a def top_albums(order='rating_desc', count=20): """Returns: [Album]""" - q = SearchQuery('album', order=order, count=count) - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_albums, a) - return a + with _APILOCK: + q = SearchQuery('album', order=order, count=count) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_albums, a) + return a def top_tracks(order='rating_desc', count=20): """Returns: [Track]""" - q = SearchQuery('track', order=order, count=count) - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_tracks, a) - return a + with _APILOCK: + q = SearchQuery('track', order=order, count=count) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_tracks, a) + return a def get_radio(radio_id): """Returns: Radio""" - q = CustomQuery(_GET2+"id+name+idstr+image/radio/json?id=%d"%(radio_id)) - js = q.execute() - if not js: - raise JamendoAPIException(str(q)) - if isinstance(js, list): - ks = js[0] - return Radio(radio_id, json=js) + with _APILOCK: + q = CustomQuery(_GET2+"id+name+idstr+image/radio/json?id=%d"%(radio_id)) + js = q.execute() + if not js: + raise JamendoAPIException(str(q)) + if isinstance(js, list): + ks = js[0] + return Radio(radio_id, json=js) def starred_radios(): """Returns: [Radio]""" - q = CustomQuery(_GET2+"id+name+idstr+image/radio/json?order=starred_desc") - js = q.execute() - if not js: - raise JamendoAPIException(str(q)) - return [Radio(int(radio['id']), json=radio) for radio in js] + with _APILOCK: + q = CustomQuery(_GET2+"id+name+idstr+image/radio/json?order=starred_desc") + js = q.execute() + if not js: + raise JamendoAPIException(str(q)) + return [Radio(int(radio['id']), json=radio) for radio in js] def top_tags(count=50, order='rating_desc'): """Returns: [Tag]""" - q = CustomQuery(_GET2+"id+name/tag/json?n=%d&order=%s"%(count, order)) - js = q.execute() - if not js: - raise JamendoAPIException(str(q)) - return [Tag(int(tag['id']), json=tag) for tag in js] + with _APILOCK: + q = CustomQuery(_GET2+"id+name/tag/json?n=%d&order=%s"%(count, order)) + js = q.execute() + if not js: + raise JamendoAPIException(str(q)) + return [Tag(int(tag['id']), json=tag) for tag in js] def favorite_albums(user): """Returns: [Album]""" - q = SearchQuery('favorite_albums', user=user, count=20) - a = q.execute() - if not a: - raise JamendoAPIException(str(q)) - _update_cache(_albums, a) - return a + with _APILOCK: + q = SearchQuery('favorite_albums', user=user, count=20) + a = q.execute() + if not a: + raise JamendoAPIException(str(q)) + _update_cache(_albums, a) + return a ### Set loader functions for classes diff --git a/jamaui/__init__.py b/jamaui/__init__.py index 5bf0b3e..483bcd4 100644 --- a/jamaui/__init__.py +++ b/jamaui/__init__.py @@ -30,7 +30,7 @@ LOG_LEVEL = logging.INFO LOG_FORMAT = "%(asctime)s %(name)-10s: [%(lineno)4d] %(levelname)-5s %(message)s" _rootlogger = logging.getLogger() -_fhandler = logging.FileHandler(LOG_FILENAME) +_fhandler = logging.FileHandler(LOG_FILENAME, mode='w') _shandler = logging.StreamHandler() _formatter = logging.Formatter(LOG_FORMAT) _fhandler.setFormatter(_formatter) diff --git a/jamaui/fetcher.py b/jamaui/fetcher.py new file mode 100644 index 0000000..d6f5ff0 --- /dev/null +++ b/jamaui/fetcher.py @@ -0,0 +1,71 @@ +# Background fetcher: +# Takes a generator and a notification identifier as parameter, +# starts a thread, and post a notification whenever data arrives. +# Posts a completion notification when done. +# Terminates if generator fails or encounters an error. + +import threading +from postoffice import postoffice +import jamaendo +import logging + +import hildon + +log = logging.getLogger(__name__) + +class _Worker(threading.Thread): + def __init__(self, generator, owner): + threading.Thread.__init__(self) + self.setDaemon(True) + self.generator = generator + self.owner = owner + + def _post(self, item): + postoffice.notify("fetch", self.owner, item) + + def _post_ok(self): + postoffice.notify("fetch-ok", self.owner) + + def _post_fail(self, e): + postoffice.notify("fetch-fail", self.owner, e) + + def run(self): + try: + for item in self.generator(): + self._post(item) + self._post_ok() + except jamaendo.JamendoAPIError, e: + log.exception("Failed to fetch, id %s" % (self.owner)) + self._post_fail(e) + +class Fetcher(object): + def __init__(self, generator, owner, on_item = None, on_ok = None, on_fail = None): + self.generator = generator + self.owner = owner + self.worker = None + + self.on_item = on_item + self.on_ok = on_ok + self.on_fail = on_fail + + def _on_item_cb(self, i, x): + self.on_item(i, x) + + def _on_ok_cb(self, i): + self.on_ok(i) + hildon.hildon_gtk_window_set_progress_indicator(self.owner, 0) + + def _on_fail_cb(self, i, e): + self.on_fail(i, e) + hildon.hildon_gtk_window_set_progress_indicator(self.owner, 0) + + def start(self): + postoffice.connect('fetch', self, self._on_item_cb) + postoffice.connect('fetch-ok', self, self._on_ok_cb) + postoffice.connect('fetch-fail', self, self._on_fail_cb) + hildon.hildon_gtk_window_set_progress_indicator(self.owner, 1) + self.worker = _Worker(self.generator, self.owner) + self.worker.start() + + def stop(self): + postoffice.disconnect(['fetch', 'fetch-ok', 'fetch-fail'], self) diff --git a/jamaui/postoffice.py b/jamaui/postoffice.py index 078316c..a84158b 100644 --- a/jamaui/postoffice.py +++ b/jamaui/postoffice.py @@ -23,38 +23,45 @@ # # message central +from __future__ import with_statement import logging +import threading log = logging.getLogger(__name__) class PostOffice(object): def __init__(self): + self.lock = threading.RLock() self.tags = {} # tag -> [callback] def notify(self, tag, *data): - clients = self.tags.get(tag) - if clients: - for ref, client in clients: - client(*data) + with self.lock: + log.info("(%s %s)", tag, " ".join(str(x) for x in data)) + clients = self.tags.get(tag) + if clients: + for ref, client in clients: + client(*data) def connect(self, tag, ref, callback): - if not isinstance(tag, list): - tag = [tag] - for t in tag: - if t not in self.tags: - self.tags[t] = [] - clients = self.tags[t] - if callback not in clients: - clients.append((ref, callback)) + with self.lock: + if not isinstance(tag, list): + tag = [tag] + for t in tag: + if t not in self.tags: + self.tags[t] = [] + clients = self.tags[t] + if callback not in clients: + clients.append((ref, callback)) def disconnect(self, tag, ref): - if not isinstance(tag, list): - tag = [tag] - for t in tag: - if t not in self.tags: - self.tags[t] = [] - self.tags[t] = [(_ref, cb) for _ref, cb in self.tags[t] if _ref != ref] + with self.lock: + if not isinstance(tag, list): + tag = [tag] + for t in tag: + if t not in self.tags: + self.tags[t] = [] + self.tags[t] = [(_ref, cb) for _ref, cb in self.tags[t] if _ref != ref] postoffice = PostOffice() diff --git a/jamaui/search.py b/jamaui/search.py index 52d74f7..ec3f366 100644 --- a/jamaui/search.py +++ b/jamaui/search.py @@ -31,6 +31,7 @@ from playerwindow import open_playerwindow from showartist import ShowArtist from showalbum import ShowAlbum from albumlist import MusicList +from fetcher import Fetcher class SearchWindow(hildon.StackableWindow): def __init__(self): @@ -40,6 +41,8 @@ class SearchWindow(hildon.StackableWindow): vbox = gtk.VBox(False, 0) + self.fetcher = None + self.connect('destroy', self.on_destroy) # Results list self.panarea = hildon.PannableArea() @@ -94,6 +97,11 @@ class SearchWindow(hildon.StackableWindow): self.menu.show_all() self.set_app_menu(self.menu) + def on_destroy(self, wnd): + if self.fetcher: + self.fetcher.stop() + self.fetcher = None + def mode_changed(self, selector, user_data): pass #current_selection = selector.get_current_text() @@ -104,6 +112,26 @@ class SearchWindow(hildon.StackableWindow): self.musiclist.set_loading(False) self.musiclist.get_model().clear() + if self.fetcher: + self.fetcher.stop() + self.fetcher = None + + itemgen = None + if mode == 0: + itemgen = lambda: jamaendo.search_artists(query=txt) + elif mode == 1: + itemgen = lambda: jamaendo.search_albums(query=txt) + elif mode == 2: + itemgen = lambda: jamaendo.search_tracks(query=txt) + else: + return + + self.fetcher = Fetcher(itemgen, self, + on_item = self.on_add_result, + on_ok = self.on_add_ok, + on_fail = self.on_add_fail) + self.fetcher.start() + ''' try: if mode == 0: items = jamaendo.search_artists(query=txt) @@ -119,6 +147,23 @@ class SearchWindow(hildon.StackableWindow): except jamaendo.JamaendoAPIException: # nothing found, force redraw self.musiclist.queue_draw() + ''' + + def on_add_result(self, wnd, item): + if wnd is self: + self.musiclist.add_items([item]) + self.idmap[item.ID] = item + + def on_add_ok(self, wnd): + if wnd is self: + self.fetcher.stop() + self.fetcher = None + + def on_add_fail(self, wnd, error): + if wnd is self: + self.musiclist.queue_draw() + self.fetcher.stop() + self.fetcher = None def row_activated(self, treeview, path, view_column): _id = self.musiclist.get_item_id(path) diff --git a/jamaui/ui.py b/jamaui/ui.py index 5427aab..100be8d 100644 --- a/jamaui/ui.py +++ b/jamaui/ui.py @@ -207,7 +207,8 @@ ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """) dialog.set_authors(("Kristoffer Gronlund ", - "Based on Panucci, written by Thomas Perl ")) + "Based on Panucci, written by Thomas Perl ", + "Icons by Joseph Wain ")) dialog.set_comments("""Jamaendo plays music from the music catalog of JAMENDO. JAMENDO is an online platform that distributes musical works under Creative Commons licenses.""")