#
# Copyright (c) 2007-2008 INdT.
+# Copyright (c) 2011 Neal H. Walfield.
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
import logging
logger = logging.getLogger(__name__)
-VERSION = "52"
+VERSION = "120"
section = "FeedingIt"
-ranges = { "updateInterval":[0.5, 1, 2, 4, 12, 24], "expiry":[24, 48, 72, 144, 288], "fontSize":range(12,24), "orientation":["Automatic", "Landscape", "Portrait"], "artFontSize":[10, 12, 14, 16, 18, 20], "feedsort":["Manual", "Most unread", "Least unread", "Most recent", "Least recent"] }
+ranges = { "updateInterval":[0.5, 1, 2, 4, 8, 12, 24], "expiry":[24, 48, 72, 144, 288], "fontSize":range(12,24), "orientation":["Automatic", "Landscape", "Portrait"], "artFontSize":[10, 12, 14, 16, 18, 20], "feedsort":["Manual", "Most unread", "Least unread", "Most recent", "Least recent"] }
titles = {"updateInterval":"Auto-update interval", "expiry":"Delete articles", "fontSize":"List font size", "orientation":"Display orientation", "artFontSize":"Article font size","feedsort":"Feed sort order"}
subtitles = {"updateInterval":"Every %s hours", "expiry":"After %s hours", "fontSize":"%s pixels", "orientation":"%s", "artFontSize":"%s pixels", "feedsort":"%s"}
heading('Updating')
- button = hildon.CheckButton(gtk.HILDON_SIZE_FINGER_HEIGHT)
- button.set_label("Automatically update feeds")
- button.set_active(self.config["autoupdate"])
- button.connect("toggled", self.button_toggled, "autoupdate")
- vbox.pack_start(button, expand=False)
- add_setting('updateInterval')
- add_setting('expiry')
+ label = gtk.Label(gtk.HILDON_SIZE_FINGER_HEIGHT)
+ label.set_label("Use Woodchuck network daemon, or the home-screen widget for automatic updates.")
+ label.set_line_wrap(True)
+ vbox.pack_start(label, expand=False)
+
+ try:
+ import woodchuck
+ woodchuck_installed = True
+ except ImportError:
+ woodchuck_installed = False
+
+ if not woodchuck_installed:
+ def install_woodchuck_clicked(button):
+ from FeedingIt import open_in_browser
+ open_in_browser("http://maemo.org/downloads/product/raw/Maemo5/murmeltier?get_installfile")
+
+ button = hildon.Button(gtk.HILDON_SIZE_FINGER_HEIGHT, hildon.BUTTON_ARRANGEMENT_VERTICAL)
+ button.set_label("Install Woodchuck")
+ button.connect("clicked", install_woodchuck_clicked)
+ button.set_alignment(0,0,1,1)
+ vbox.pack_start(button, expand=False)
+ else:
+ button = hildon.CheckButton(gtk.HILDON_SIZE_FINGER_HEIGHT)
+ button.set_label("Woodchuck-Based Automatic Update")
+ button.set_active(self.config["woodchuck"])
+ button.connect("toggled", self.button_toggled, "woodchuck")
+ vbox.pack_start(button, expand=False)
+ add_setting('updateInterval')
+ add_setting('expiry')
heading('Network')
button = hildon.CheckButton(gtk.HILDON_SIZE_FINGER_HEIGHT)
else:
self.config[configName] = False
#print "autoup", self.autoupdate
+
+ if configName == 'woodchuck':
+ try:
+ from wc import wc_disable_set
+ wc_disable_set(not self.config['woodchuck'])
+ except Exception:
+ logger.exception("Disabling Woodchuck")
+
self.saveConfig()
def selection_changed(self, selector, button, setting):
def loadConfig(self):
self.config = {}
+
+ configParser = RawConfigParser()
try:
- configParser = RawConfigParser()
configParser.read(self.configFilename)
- self.config["fontSize"] = configParser.getint(section, "fontSize")
- self.config["artFontSize"] = configParser.getint(section, "artFontSize")
- self.config["expiry"] = configParser.getint(section, "expiry")
- self.config["autoupdate"] = configParser.getboolean(section, "autoupdate")
- self.config["updateInterval"] = configParser.getfloat(section, "updateInterval")
- self.config["orientation"] = configParser.get(section, "orientation")
- self.config["imageCache"] = configParser.getboolean(section, "imageCache")
- except:
- self.config["fontSize"] = 17
- self.config["artFontSize"] = 14
- self.config["expiry"] = 24
- self.config["autoupdate"] = False
- self.config["updateInterval"] = 4
- self.config["orientation"] = "Automatic"
- self.config["imageCache"] = False
- try:
- self.config["proxy"] = configParser.getboolean(section, "proxy")
- except:
- self.config["proxy"] = True
- try:
- self.config["hidereadfeeds"] = configParser.getboolean(section, "hidereadfeeds")
- self.config["hidereadarticles"] = configParser.getboolean(section, "hidereadarticles")
- except:
- self.config["hidereadfeeds"] = False
- self.config["hidereadarticles"] = False
- try:
- self.config["extBrowser"] = configParser.getboolean(section, "extBrowser")
- except:
- self.config["extBrowser"] = False
- try:
- self.config["feedsort"] = configParser.get(section, "feedsort")
- except:
- self.config["feedsort"] = "Manual"
- try:
- self.config["theme"] = configParser.get(section, "theme")
- except:
- self.config["theme"] = True
-
+ except Exception:
+ logger.exception("Reading %s", self.configFilename)
+
+ # The function to use to fetch the parameter, the parameter's
+ # name and the default value.
+ values = ((configParser.getint, "fontSize", 17),
+ (configParser.getint, "artFontSize", 14),
+ (configParser.getint, "expiry", 24),
+ (configParser.getboolean, "autoupdate", False),
+ (configParser.getboolean, "woodchuck", True),
+ (configParser.getboolean, "askedAboutWoodchuck", False),
+ (configParser.getint, "updateInterval", 4),
+ (configParser.get, "orientation", "Automatic"),
+ (configParser.getboolean, "imageCache", False),
+ (configParser.getboolean, "proxy", True),
+ (configParser.getboolean, "hidereadfeeds", False),
+ (configParser.getboolean, "hidereadarticles", False),
+ (configParser.getboolean, "extBrowser", False),
+ (configParser.getboolean, "theme", True),
+ (configParser.get, "feedsort", "Manual"))
+
+ for fetcher, name, default in values:
+ try:
+ v = fetcher(section, name)
+ except Exception:
+ logger.exception("Reading config variable %s", name)
+ v = default
+ self.config[name] = v
+
def saveConfig(self):
configParser = RawConfigParser()
configParser.add_section(section)
configParser.set(section, 'expiry', str(self.config["expiry"]))
configParser.set(section, 'autoupdate', str(self.config["autoupdate"]))
configParser.set(section, 'updateInterval', str(self.config["updateInterval"]))
+ configParser.set(section, 'woodchuck', str(self.config["woodchuck"]))
+ configParser.set(section, 'askedAboutWoodchuck', str(self.config["askedAboutWoodchuck"]))
configParser.set(section, 'orientation', str(self.config["orientation"]))
configParser.set(section, 'imageCache', str(self.config["imageCache"]))
configParser.set(section, 'proxy', str(self.config["proxy"]))
return self.config["autoupdate"]
def setAutoUpdateEnabled(self, value):
self.config["autoupdate"] = value
+ def getWoodchuckEnabled(self):
+ return self.config["woodchuck"]
+ def getAskedAboutWoodchuck(self):
+ return self.config["askedAboutWoodchuck"]
+ def setAskedAboutWoodchuck(self, value):
+ self.config["askedAboutWoodchuck"] = value
+ self.saveConfig()
def getUpdateInterval(self):
return float(self.config["updateInterval"])
def getReadFont(self):
return "sans italic %s" % self.config["fontSize"]
def getUnreadFont(self):
return "sans %s" % self.config["fontSize"]
- def getOrientation(self, index):
+ def getOrientation(self):
return ranges["orientation"].index(self.config["orientation"])
def getOrientationChoices(self):
return ranges["orientation"]
logger = logging.getLogger(__name__)
def getId(string):
+ if issubclass(string.__class__, unicode):
+ string = string.encode('utf8', 'replace')
+
return md5.new(string).hexdigest()
def download_callback(connection):
self.key = key
self.configdir = configdir
self.dir = "%s/%s.d" %(self.configdir, self.key)
- self.tls = threading.local ()
+ self.tls = threading.local()
if not isdir(self.dir):
mkdir(self.dir)
- if not isfile("%s/%s.db" %(self.dir, self.key)):
- self.db.execute("CREATE TABLE feed (id text, title text, contentLink text, date float, updated float, link text, read int);")
+ filename = "%s/%s.db" % (self.dir, self.key)
+ if not isfile(filename):
+ self.db.execute("CREATE TABLE feed (id text, title text, contentLink text, contentHash text, date float, updated float, link text, read int);")
self.db.execute("CREATE TABLE images (id text, imagePath text);")
self.db.commit()
+ else:
+ try:
+ self.db.execute("ALTER TABLE feed ADD COLUMN contentHash text")
+ self.db.commit()
+ except sqlite3.OperationalError, e:
+ if 'duplicate column name' in str(e):
+ pass
+ else:
+ logger.exception("Add column contentHash to %s", filename)
def addImage(self, configdir, key, baseurl, url, proxy=None, opener=None):
filename = configdir+key+".d/"+getId(url)
if(not(entry.has_key("id"))):
entry["id"] = None
content = self.extractContent(entry)
+ contentHash = getId(content)
object_size = len (content)
tmpEntry = {"title":entry["title"], "content":content,
"date":date, "link":entry["link"], "author":entry["author"], "id":entry["id"]}
id = self.generateUniqueId(tmpEntry)
current_version = self.db.execute(
- 'select date, ROWID from feed where id=?',
+ 'select date, ROWID, contentHash from feed where id=?',
(id,)).fetchone()
if (current_version is not None
- and current_version[0] == date):
+ # To detect updates, don't compare by date:
+ # compare by content.
+ #
+ # - If an article update is just a date change
+ # and the content remains the same, we don't
+ # want to register an update.
+ #
+ # - If an article's content changes but not the
+ # date, we want to recognize an update.
+ and current_version[2] == contentHash):
logger.debug("ALREADY DOWNLOADED %s (%s)"
% (entry["title"], entry["link"]))
- ## This article is already present in the feed listing. Update the "updated" time, so it doesn't expire
- self.db.execute("UPDATE feed SET updated=? WHERE id=?;",(currentTime,id))
- try:
- logger.debug("Updating already downloaded files for %s" %(id))
- filename = configdir+self.key+".d/"+id+".html"
- file = open(filename,"a")
- utime(filename, None)
- file.close()
- images = self.db.execute("SELECT imagePath FROM images where id=?;", (id, )).fetchall()
- for image in images:
- file = open(image[0],"a")
- utime(image[0], None)
- file.close()
- except:
- logger.debug("Error in refreshing images for %s" % (id))
+ ## This article is already present in the feed listing. Update the "updated" time, so it doesn't expire
+ self.db.execute("UPDATE feed SET updated=? WHERE id=?;",(currentTime,id))
+ try:
+ logger.debug("Updating already downloaded files for %s" %(id))
+ filename = configdir+self.key+".d/"+id+".html"
+ file = open(filename,"a")
+ utime(filename, None)
+ file.close()
+ images = self.db.execute("SELECT imagePath FROM images where id=?;", (id, )).fetchall()
+ for image in images:
+ file = open(image[0],"a")
+ utime(image[0], None)
+ file.close()
+ except:
+ logger.debug("Error in refreshing images for %s" % (id))
self.db.commit()
continue
# The version was updated. Mark it as unread.
logger.debug("UPDATED: %s (%s)"
% (entry["title"], entry["link"]))
- self.setEntryUnread(id)
updated_objects += 1
else:
logger.debug("NEW: %s (%s)"
soup = BeautifulSoup(self.getArticle(tmpEntry)) #tmpEntry["content"])
images = soup('img')
baseurl = tmpEntry["link"]
- #if not id in ids:
if imageCache and len(images) > 0:
self.serial_execution_lock.release ()
have_serial_execution_lock = False
values = {'id': id,
'title': tmpEntry["title"],
'contentLink': tmpEntry["contentLink"],
+ 'contentHash': contentHash,
'date': tmpEntry["date"],
'updated': currentTime,
'link': tmpEntry["link"],
def getContentLink(self, id):
return self.db.execute("SELECT contentLink FROM feed WHERE id=?;", (id,) ).fetchone()[0]
+ def getContentHash(self, id):
+ return self.db.execute("SELECT contentHash FROM feed WHERE id=?;", (id,) ).fetchone()[0]
+
def getExternalLink(self, id):
return self.db.execute("SELECT link FROM feed WHERE id=?;", (id,) ).fetchone()[0]
return text
def getContent(self, id):
- contentLink = self.db.execute("SELECT contentLink FROM feed WHERE id=?;", (id,)).fetchone()[0]
+ """
+ Return the content of the article with the specified ID. If
+ the content is not available, returns None.
+ """
+ contentLink = self.getContentLink(id)
try:
- file = open(self.entries[id]["contentLink"])
- content = file.read()
- file.close()
- except:
- content = "Content unavailable"
+ with open(contentLink, 'r') as file:
+ content = file.read()
+ except Exception:
+ logger.exception("Failed get content for %s: reading %s failed",
+ id, contentLink)
+ content = None
return content
def extractDate(self, entry):
self.db.execute("INSERT INTO feed (id, title, contentLink, date, updated, link, read) VALUES (?, ?, ?, ?, ?, ?, ?);", values)
self.db.commit()
- def updateFeed(self, configdir, url, etag, modified, expiryTime=24, proxy=None, imageCache=False):
+ # Feed.UpdateFeed calls this function.
+ def _updateFeed(self, configdir, url, etag, modified, expiryTime=24, proxy=None, imageCache=False, priority=0, postFeedUpdateFunc=None, *postFeedUpdateFuncArgs):
currentTime = 0
rows = self.db.execute("SELECT id, link FROM feed WHERE updated=0;")
for row in rows:
- currentTime = time.time()
- id = row[0]
- link = row[1]
- f = urllib2.urlopen(link)
- #entry["content"] = f.read()
- html = f.read()
- f.close()
- soup = BeautifulSoup(html)
- images = soup('img')
- baseurl = link
- for img in images:
- filename = self.addImage(configdir, self.key, baseurl, img['src'], proxy=proxy)
- img['src']=filename
- self.db.execute("INSERT INTO images (id, imagePath) VALUES (?, ?);", (id, filename) )
+ try:
+ currentTime = time.time()
+ id = row[0]
+ link = row[1]
+ f = urllib2.urlopen(link)
+ #entry["content"] = f.read()
+ html = f.read()
+ f.close()
+ soup = BeautifulSoup(html)
+ images = soup('img')
+ baseurl = link
+ for img in images:
+ filename = self.addImage(configdir, self.key, baseurl, img['src'], proxy=proxy)
+ img['src']=filename
+ self.db.execute("INSERT INTO images (id, imagePath) VALUES (?, ?);", (id, filename) )
+ self.db.commit()
+ contentLink = configdir+self.key+".d/"+id+".html"
+ file = open(contentLink, "w")
+ file.write(soup.prettify())
+ file.close()
+
+ self.db.execute("UPDATE feed SET read=0, contentLink=?, updated=? WHERE id=?;", (contentLink, time.time(), id) )
self.db.commit()
- contentLink = configdir+self.key+".d/"+id+".html"
- file = open(contentLink, "w")
- file.write(soup.prettify())
- file.close()
-
- self.db.execute("UPDATE feed SET read=0, contentLink=?, updated=? WHERE id=?;", (contentLink, time.time(), id) )
- self.db.commit()
- return (currentTime, None, None)
+ except:
+ logger.error("Error updating Archived Article: %s %s"
+ % (link,traceback.format_exc(),))
+
+ if postFeedUpdateFunc is not None:
+ postFeedUpdateFunc (self.key, currentTime, None, None, None,
+ *postFeedUpdateFuncArgs)
def purgeReadArticles(self):
rows = self.db.execute("SELECT id FROM feed WHERE read=1;")
# state.
try:
updater = os.path.basename(sys.argv[0]) == 'update_feeds.py'
- wc_init (self, True if updater else False)
+ wc_init(config, self, True if updater else False)
if wc().available() and updater:
# The list of known streams.
streams = wc().streams_list ()
logger.debug(
"Registering previously unknown channel: %s (%s)"
% (key, title,))
- # Use a default refresh interval of 6 hours.
- wc().stream_register (key, title, 6 * 60 * 60)
+ wc().stream_register(
+ key, title,
+ self.config.getUpdateInterval() * 60 * 60)
else:
# Make sure the human readable name is up to date.
if wc()[key].human_readable_name != title:
wc()[key].human_readable_name = title
stream_ids.remove (key)
+ wc()[key].freshness \
+ = self.config.getUpdateInterval() * 60 * 60
# Unregister any streams that are no longer subscribed to.
def getCategoryTitle(self, id):
return self.lookup('categories', 'title', id)
-
+
def getCategoryUnread(self, id):
count = 0
for key in self.getListOfFeeds(category=id):
human_readable_name=title,
freshness=6*60*60)
+ self.cache_invalidate('feeds')
return True
else:
return False
if wc().available ():
try:
del wc()[key]
- except KeyError:
+ except KeyError, woodchuck.Error:
logger.debug("Removing unregistered feed %s failed" % (key,))
rank = self.db.execute("SELECT rank FROM feeds WHERE id=?;", (key,) ).fetchone()[0]
% traceback.format_exc ())
woodchuck_imported = False
class PyWoodchuck (object):
- def available(self):
+ def available(self, *args, **kwargs):
return False
woodchuck = None
refresh_interval = 6 * 60 * 60
class mywoodchuck (PyWoodchuck):
- def __init__(self, listing, human_readable_name, identifier,
+ def __init__(self, config, listing, human_readable_name, identifier,
request_feedback):
try:
PyWoodchuck.__init__ (self, human_readable_name, identifier,
self.available = self.not_available
return
+ self.config = config
self.listing = listing
- def not_available(self):
+ try:
+ self.enabled = config.getWoodchuckEnabled()
+ except Exception:
+ logging.exception("Setting enabled")
+
+ def available(self, check_config=True):
+ if not PyWoodchuck.available(self):
+ return False
+ if check_config:
+ return self.config.getWoodchuckEnabled()
+ return True
+
+ def not_available(self, *args, **kwargs):
return False
# Woodchuck upcalls.
str(e)))
_w = None
-def wc_init(listing, request_feedback=False):
+def wc_init(config, listing, request_feedback=False):
"""Connect to the woodchuck server and initialize any state."""
global _w
assert _w is None
- _w = mywoodchuck (listing, "FeedingIt", "org.marcoz.feedingit",
+ _w = mywoodchuck (config, listing, "FeedingIt", "org.marcoz.feedingit",
request_feedback)
if not woodchuck_imported or not _w.available ():
else:
logger.debug("Woodchuck appears to be available.")
+def wc_disable_set(disable=True):
+ """Disable Woodchuck."""
+ if disable:
+ logger.info("Disabling Woodchuck")
+ else:
+ logger.info("Enabling Woodchuck")
+
+ global _w
+ if _w is None:
+ logging.info("Woodchuck not loaded. Not doing anything.")
+ return
+
+ if not _w.available(check_config=False):
+ logging.info("Woodchuck not available. Not doing anything.")
+ return
+
+ try:
+ _w.enabled = not disable
+ except Exception:
+ logger.exception("Disabling Woodchuck")
+
def wc():
"""Return the Woodchuck singleton."""
global _w
(configParser.getboolean, "hidereadfeeds", False),
(configParser.getboolean, "hidereadarticles", False),
(configParser.getboolean, "extBrowser", False),
+ (configParser.getboolean, "theme", True),
(configParser.get, "feedsort", "Manual"))
for fetcher, name, default in values:
configParser.set(section, 'hidereadarticles', str(self.config["hidereadarticles"]))
configParser.set(section, 'extBrowser', str(self.config["extBrowser"]))
configParser.set(section, 'feedsort', str(self.config["feedsort"]))
+ configParser.set(section, 'theme', str(self.config["theme"]))
# Writing our configuration file
file = open(self.configFilename, 'wb')
return self.config["artFontSize"]
def getExpiry(self):
return self.config["expiry"]
+ def setExpiry(self, expiry):
+ self.config["expiry"] = expiry
def isAutoUpdateEnabled(self):
return self.config["autoupdate"]
+ def setAutoUpdateEnabled(self, value):
+ self.config["autoupdate"] = value
def getWoodchuckEnabled(self):
return self.config["woodchuck"]
def getAskedAboutWoodchuck(self):
return "sans %s" % self.config["fontSize"]
def getOrientation(self):
return ranges["orientation"].index(self.config["orientation"])
+ def getOrientationChoices(self):
+ return ranges["orientation"]
+ def setOrientation(self, choice):
+ self.config["orientation"] = index
def getImageCache(self):
return self.config["imageCache"]
+ def setImageCache(self, cache):
+ self.config["imageCache"] = bool(cache)
@mainthread
def getProxy(self):
if self.config["proxy"] == False:
return (False, None)
def getHideReadFeeds(self):
return self.config["hidereadfeeds"]
+ def setHideReadFeeds(self, setting):
+ self.config["hidereadfeeds"] = bool(setting)
def getHideReadArticles(self):
return self.config["hidereadarticles"]
+ def setHideReadArticles(self, setting):
+ self.config["hidereadarticles"] = bool(setting)
def getOpenInExternalBrowser(self):
return self.config["extBrowser"]
def getFeedSortOrder(self):
return self.config["feedsort"]
+ def getFeedSortOrderChoices(self):
+ return ranges["feedsort"]
+ def setFeedSortOrder(self, setting):
+ self.config["feedsort"] = setting
+ def getTheme(self):
+ return self.config["theme"]
+ def setTheme(self, theme):
+ self.config["theme"] = bool(theme)
def getCategoryTitle(self, id):
return self.lookup('categories', 'title', id)
+ def getCategoryUnread(self, id):
+ count = 0
+ for key in self.getListOfFeeds(category=id):
+ try:
+ count = count + self.getFeedNumberOfUnreadItems(key)
+ except:
+ pass
+ return count
+
def getSortedListOfKeys(self, order, onlyUnread=False, category=1):
if order == "Most unread":
tmp = "ORDER BY unread DESC"
human_readable_name=title,
freshness=6*60*60)
+ self.cache_invalidate('feeds')
return True
else:
return False
id=1
self.db.execute("INSERT INTO categories (id, title, unread, rank) VALUES (?, ?, 0, ?)", (id, title, rank))
self.db.commit()
+ self.cache_invalidate('categories')
def removeFeed(self, key):
if wc().available ():
if isdir(self.configdir+key+".d/"):
rmtree(self.configdir+key+".d/")
+ self.cache_invalidate('feeds')
def removeCategory(self, key):
if self.db.execute("SELECT count(*) FROM categories;").fetchone()[0] > 1:
self.db.execute("UPDATE categories SET rank=rank-1 WHERE rank>?;", (rank,) )
self.db.execute("UPDATE feeds SET category=1 WHERE category=?;", (key,) )
self.db.commit()
+ self.cache_invalidate('categories')
#def saveConfig(self):
# self.listOfFeeds["feedingit-order"] = self.sortedKeys
from updatedbus import update_server_object
-CONFIGDIR="/home/user/.feedingit/"
+CONFIGDIR = os.environ.get("HOME", "/home/user") + "/.feedingit/"
#DESKTOP_FILE = "/usr/share/applications/hildon-status-menu/feedingit_status.desktop"
from socket import setdefaulttimeout