Using lambdas to remove the use of _requests so children can be requested in parallel
[watersofshiloah] / src / stream_index.py
index 8e83769..dc4d654 100644 (file)
@@ -33,7 +33,6 @@ class Connection(object):
                if kwds is None:
                        kwds = {}
 
-               self._indexing.clear_tasks()
                self._indexing.add_task(
                        getattr(self._backend, func),
                        args,
@@ -79,6 +78,12 @@ class AudioIndex(object):
                        elif source == SOURCE_CONFERENCES:
                                assert langId is not None
                                node = ConferencesNode(self._connection, langId)
+                       elif source == SOURCE_MAGAZINES:
+                               assert langId is not None
+                               node = MagazinesNode(self._connection, langId)
+                       elif source == SOURCE_SCRIPTURES:
+                               assert langId is not None
+                               node = ScripturesNode(self._connection, langId)
                        else:
                                raise NotImplementedError(source)
                        self._sources[key] = node
@@ -146,22 +151,19 @@ class ParentNode(Node):
 
        def __init__(self, connection, parent, data, id):
                Node.__init__(self, connection, parent, data, id)
-               self._request = None
 
        def is_leaf(self):
                return False
 
        def _get_children(self, on_success, on_error):
-               assert self._request is None
                assert self._children is None
-               self._request = on_success, on_error
 
                func, args, kwds = self._get_func()
 
                self._connection.download(
                        func,
-                       self._on_success,
-                       self._on_error,
+                       lambda data: self._on_success(data, on_success, on_error),
+                       on_error,
                        args,
                        kwds,
                )
@@ -173,9 +175,7 @@ class ParentNode(Node):
                raise NotImplementedError()
 
        @misc_utils.log_exception(_moduleLogger)
-       def _on_success(self, data):
-               r = self._request
-               self._request = None
+       def _on_success(self, data, on_success, on_error):
                try:
                        self._children = [
                                self._create_child(child, i)
@@ -184,15 +184,9 @@ class ParentNode(Node):
                except Exception, e:
                        _moduleLogger.exception("Translating error")
                        self._children = None
-                       r[1](e)
+                       on_error(e)
                else:
-                       r[0](self._children)
-
-       @misc_utils.log_exception(_moduleLogger)
-       def _on_error(self, error):
-               r = self._request
-               self._request = None
-               r[1](error)
+                       on_success(self._children)
 
 
 class LeafNode(Node):
@@ -368,7 +362,157 @@ class TalkNode(LeafNode):
 
        @property
        def subtitle(self):
-               return self._data["speaker"]
+               speaker = self._data["speaker"]
+               if speaker is not None:
+                       return speaker
+               else:
+                       return ""
+
+       @property
+       def uri(self):
+               return self._data["url"]
+
+
+class MagazinesNode(ParentNode):
+
+       def __init__(self, connection, langId):
+               ParentNode.__init__(self, connection, None, {}, SOURCE_MAGAZINES)
+               self._langId = langId
+
+       @property
+       def title(self):
+               return "Magazines"
+
+       def _get_func(self):
+               return "get_magazines", (self._langId, ), {}
+
+       def _create_child(self, data, id):
+               return MagazineNode(self._connection, self, data, id)
+
+
+class MagazineNode(ParentNode):
+
+       def __init__(self, connection, parent, data, id):
+               ParentNode.__init__(self, connection, parent, data, id)
+
+       @property
+       def title(self):
+               return self._data["title"]
+
+       def _get_func(self):
+               return "get_magazine_issues", (self._data["id"], ), {}
+
+       def _create_child(self, data, id):
+               return IssueNode(self._connection, self, data, id)
+
+
+class IssueNode(ParentNode):
+
+       def __init__(self, connection, parent, data, id):
+               ParentNode.__init__(self, connection, parent, data, id)
+
+       @property
+       def title(self):
+               return self._data["title"]
+
+       def _get_func(self):
+               return "get_magazine_articles", (self._data["id"], ), {}
+
+       def _create_child(self, data, id):
+               return ArticleNode(self._connection, self, data, id)
+
+
+class ArticleNode(LeafNode):
+
+       def __init__(self, connection, parent, data, id):
+               LeafNode.__init__(self, connection, parent, data, id)
+
+       @property
+       def can_navigate(self):
+               return True
+
+       @property
+       def title(self):
+               return self._data["title"]
+
+       @property
+       def subtitle(self):
+               speaker = self._data["author"]
+               if speaker is not None:
+                       return speaker
+               else:
+                       return ""
+
+       @property
+       def uri(self):
+               return self._data["url"]
+
+
+class ScripturesNode(ParentNode):
+
+       def __init__(self, connection, langId):
+               ParentNode.__init__(self, connection, None, {}, SOURCE_SCRIPTURES)
+               self._langId = langId
+
+       @property
+       def title(self):
+               return "Scriptures"
+
+       def _get_func(self):
+               return "get_scriptures", (self._langId, ), {}
+
+       def _create_child(self, data, id):
+               return ScriptureNode(self._connection, self, data, id)
+
+
+class ScriptureNode(ParentNode):
+
+       def __init__(self, connection, parent, data, id):
+               ParentNode.__init__(self, connection, parent, data, id)
+
+       @property
+       def title(self):
+               return self._data["title"]
+
+       def _get_func(self):
+               return "get_scripture_books", (self._data["id"], ), {}
+
+       def _create_child(self, data, id):
+               return BookNode(self._connection, self, data, id)
+
+
+class BookNode(ParentNode):
+
+       def __init__(self, connection, parent, data, id):
+               ParentNode.__init__(self, connection, parent, data, id)
+
+       @property
+       def title(self):
+               return self._data["title"]
+
+       def _get_func(self):
+               return "get_scripture_chapters", (self._data["id"], ), {}
+
+       def _create_child(self, data, id):
+               return ChapterNode(self._connection, self, data, id)
+
+
+class ChapterNode(LeafNode):
+
+       def __init__(self, connection, parent, data, id):
+               LeafNode.__init__(self, connection, parent, data, id)
+
+       @property
+       def can_navigate(self):
+               return True
+
+       @property
+       def title(self):
+               return self._data["title"]
+
+       @property
+       def subtitle(self):
+               return ""
 
        @property
        def uri(self):
@@ -404,3 +548,102 @@ def common_paths(targetNode, currentNode):
        )
 
        return ancestors, currentNode, descendants
+
+
+class AsyncWalker(object):
+
+       def __init__(self, func):
+               self._func = func
+               self._run = None
+
+       def start(self, *args, **kwds):
+               assert self._run is None
+               self._run = self._func(*args, **kwds)
+               node = self._run.send(None) # priming the function
+               node.get_children(self.on_success, self.on_error)
+
+       @misc_utils.log_exception(_moduleLogger)
+       def on_success(self, children):
+               _moduleLogger.debug("Processing success for: %r", self._func)
+               try:
+                       node = self._run.send(children)
+               except StopIteration, e:
+                       pass
+               else:
+                       node.get_children(self.on_success, self.on_error)
+
+       @misc_utils.log_exception(_moduleLogger)
+       def on_error(self, error):
+               _moduleLogger.debug("Processing error for: %r", self._func)
+               try:
+                       node = self._run.throw(error)
+               except StopIteration, e:
+                       pass
+               else:
+                       node.get_children(self.on_success, self.on_error)
+
+
+def get_next(node, on_success, on_error):
+       try:
+               assert node.is_leaf(), node
+
+               # Find next branch
+               childNode = node
+               while True:
+                       parent = childNode.get_parent()
+                       siblings = yield parent
+                       for i, sibling in enumerate(siblings):
+                               if sibling is childNode:
+                                       break
+                       i += 1
+                       if i < len(siblings):
+                               sibling = siblings[i]
+                               break
+                       else:
+                               childNode = parent
+
+               # dig into that branch to find the first leaf
+               nodes = [sibling]
+               while nodes:
+                       child = nodes.pop(0)
+                       if child.is_leaf():
+                               on_success(child)
+                               return
+                       children = yield child
+                       nodes[0:0] = children
+               raise RuntimeError("Ran out of nodes when hunting for first leaf of %s" % node)
+       except Exception, e:
+               on_error(e)
+
+
+def get_previous(node, on_success, on_error):
+       try:
+               assert node.is_leaf(), node
+
+               # Find next branch
+               childNode = node
+               while True:
+                       parent = childNode.get_parent()
+                       siblings = yield parent
+                       for i, sibling in enumerate(siblings):
+                               if sibling is childNode:
+                                       break
+                       i -= 1
+                       if 0 <= i:
+                               sibling = siblings[i]
+                               break
+                       else:
+                               childNode = parent
+
+               # dig into that branch to find the first leaf
+               nodes = [sibling]
+               while nodes:
+                       child = nodes.pop(-1)
+                       if child.is_leaf():
+                               on_success(child)
+                               return
+                       children = yield child
+                       nodes[0:0] = children
+               raise RuntimeError("Ran out of nodes when hunting for first leaf of %s" % node)
+       except Exception, e:
+               on_error(e)