IMDb: Use SQLite indices instead of (half) MD5 hashes for lookup

The storage need increases to about 54MB for the full database, but
this should reduce CPU usage during download a bit and more importantly
it makes lookups faster. It also allows to pre-sort by number of votes.

diff --git a/src/imdb/imdb-plaintext-downloader.vala b/src/imdb/imdb-plaintext-downloader.vala
index 84a48b9..bddfff0 100644 (file)
--- a/src/imdb/imdb-plaintext-downloader.vala
+++ b/src/imdb/imdb-plaintext-downloader.vala
@@ -116,6 +116,8 @@ class IMDbDownloadServer : Object, IMDbDownloader {
                        if (MOVIES in flags) {
                                description_changed ("Downloading movie list ...");
                                yield download_async(movies, movie_parser, io_priority);
+                               description_changed ("Creating title index ...");
+                               sqlite.create_title_index ();
                        }
                        if (GENRES in flags) {
                                description_changed ("Downloading genre data ...");
@@ -132,11 +134,6 @@ class IMDbDownloadServer : Object, IMDbDownloader {
                                warning ("Failed to open/read stream: %s\n", e2.message);
                }
 
-               if (!cancellable.is_cancelled ()) {
-                       stdout.printf ("Download complete.\n");
-                       progress (100);
-               }
-
                try {
                        bool unmounted = yield m.unmount(MountUnmountFlags.NONE, null);
                        if (!unmounted) {
@@ -146,6 +143,14 @@ class IMDbDownloadServer : Object, IMDbDownloader {
                        warning ("Failed to unmount: %s\n", e4.message);
                }
 
+               description_changed ("Creating indices ...");
+               sqlite.create_votes_index ();
+
+               if (!cancellable.is_cancelled ()) {
+                       stdout.printf ("Download complete.\n");
+                       progress (100);
+               }
+
                sqlite = null;
                running = false;
 
@@ -182,7 +187,8 @@ class IMDbDownloadServer : Object, IMDbDownloader {
                        int p = (int) (100 * (sofar + stream.total_in ()) / total);
                        if (p > percent) {
                                percent = p;
-                               progress (p);
+                               if (p < 100)
+                                       progress (p);
                        }
                } while (line != null);
 
@@ -320,7 +326,7 @@ class RatingLineParser : LineParser {
        public RatingLineParser (IMDbSqlite _sqlite) {
                base (_sqlite);
                try {
-                       re_rating = new Regex ("^      .+ +[0-9]+ +([0-9.]+) +(.+)$");
+                       re_rating = new Regex ("^      .+ +([0-9]+) +([0-9.]+) +(.+)$");
                } catch (RegexError e) {
                        critical ("Failed to initialize regex: %s\n", e.message);
                }
@@ -337,9 +343,10 @@ class RatingLineParser : LineParser {
                        return;
 
                string title;
-               string rating = matchinfo.fetch (1);
+               string votes = matchinfo.fetch (1);
+               string rating = matchinfo.fetch (2);
                try {
-                       title = convert(matchinfo.fetch (2), -1, "utf-8", "latin1");
+                       title = convert(matchinfo.fetch (3), -1, "utf-8", "latin1");
                } catch (ConvertError e) {
                        return;
                }
@@ -351,6 +358,6 @@ class RatingLineParser : LineParser {
                if (skip_title (title))
                        return;
 
-               sqlite.movie_set_rating (title, (int) (rating.to_double () * 10));
+               sqlite.movie_set_rating (title, (int) (rating.to_double () * 10), votes.to_int ());
        }
 }

diff --git a/src/imdb/imdb-sqlite.vala b/src/imdb/imdb-sqlite.vala
index af9c317..a48a3f5 100644 (file)
--- a/src/imdb/imdb-sqlite.vala
+++ b/src/imdb/imdb-sqlite.vala
@@ -63,13 +63,8 @@ class IMDbSqlite : Object {
                return 0;
        }
 
-       public int add_movie (string _title, int year) {
-               string md5 = Checksum.compute_for_string (ChecksumType.MD5, _title);
-               uint64 half_md5 = (md5.ndup (16)).to_uint64 (null, 16);
-
-               string title = strip_year (_title, year);
-
-               string sql = "INSERT INTO Movies(ID, Title, Year) VALUES (%lld, \"%s\", %d);".printf ((int64) half_md5, title, year);
+       public int add_movie (string title, int year) {
+               string sql = "INSERT INTO Movies(Title, Year) VALUES (\"%s\", %d);".printf (title, year);
                int rc;
 
                rc = db.exec (sql, callback, null);
@@ -81,20 +76,8 @@ class IMDbSqlite : Object {
                return 0;
        }
 
-       private string strip_year (string title, int year) {
-               string year_suffix = " (%d)".printf (year);
-               if (title.has_suffix (year_suffix)) {
-                       return title.substring (0, title.length - year_suffix.length);
-               } else {
-                       return title.dup ();
-               }
-       }
-
-       public int movie_set_rating (string title, int rating) {
-               string md5 = Checksum.compute_for_string (ChecksumType.MD5, title);
-               uint64 half_md5 = (md5.ndup (16)).to_uint64 (null, 16);
-
-               var sql = "UPDATE Movies SET Rating=%d WHERE ID=%lld;".printf (rating, (int64) half_md5);
+       public int movie_set_rating (string title, int rating, int votes) {
+               var sql = "UPDATE Movies SET Rating=%d, Votes=%d WHERE Title=\"%s\";".printf (rating, votes, title);
                int rc;
 
                rc = db.exec (sql, callback, null);
@@ -107,8 +90,6 @@ class IMDbSqlite : Object {
        }
 
        public int movie_add_genre (string title, string genre) {
-               string md5 = Checksum.compute_for_string (ChecksumType.MD5, title);
-               uint64 half_md5 = (md5.ndup (16)).to_uint64 (null, 16);
                string sql;
                int bit;
                int rc;
@@ -127,7 +108,7 @@ class IMDbSqlite : Object {
                        }
                }
 
-               sql = "UPDATE Movies SET Genres=Genres|%d WHERE ID=%lld;".printf (bit, (int64) half_md5);
+               sql = "UPDATE Movies SET Genres=Genres|%d WHERE Title=\"%s\";".printf (bit, title);
                rc = db.exec (sql, callback, null);
                if (rc != Sqlite.OK) {
                        stderr.printf ("SQL error: %d, %s\n", rc, db.errmsg ());
@@ -148,7 +129,31 @@ class IMDbSqlite : Object {
        public int clear () {
                int rc;
 
-               rc = db.exec ("DROP TABLE IF EXISTS Movies; CREATE TABLE Movies (ID INTEGER PRIMARY KEY, Title TEXT NOT NULL, Year INTEGER, Rating INTEGER, Genres INTEGER NOT NULL DEFAULT 0); DROP TABLE IF EXISTS Genres; CREATE TABLE Genres (Bit INTEGER PRIMARY KEY, Genre TEXT NOT NULL);", callback, null);
+               rc = db.exec ("DROP TABLE IF EXISTS Movies; CREATE TABLE Movies (Title TEXT NOT NULL, Year INTEGER, Rating INTEGER, Votes INTEGER, Genres INTEGER NOT NULL DEFAULT 0); DROP TABLE IF EXISTS Genres; CREATE TABLE Genres (Bit INTEGER PRIMARY KEY, Genre TEXT NOT NULL);", callback, null);
+               if (rc != Sqlite.OK) {
+                       stderr.printf ("SQL error: %d, %s\n", rc, db.errmsg ());
+                       return 1;
+               }
+
+               return 0;
+       }
+
+       public int create_title_index () {
+               int rc;
+
+               rc = db.exec ("CREATE INDEX MovieTitles ON Movies(Title);", callback, null);
+               if (rc != Sqlite.OK) {
+                       stderr.printf ("SQL error: %d, %s\n", rc, db.errmsg ());
+                       return 1;
+               }
+
+               return 0;
+       }
+
+       public int create_votes_index () {
+               int rc;
+
+               rc = db.exec ("CREATE INDEX MovieVotes ON Movies(Votes);", callback, null);
                if (rc != Sqlite.OK) {
                        stderr.printf ("SQL error: %d, %s\n", rc, db.errmsg ());
                        return 1;
@@ -165,7 +170,7 @@ class IMDbSqlite : Object {
 
                if (filter.title != null && filter.title != "") {
                        if ("*" in filter.title)
-                               sql += sep + "Title GLOB \"%s\"".printf (filter.title);
+                               sql += sep + "Title GLOB \"%s (*)\"".printf (filter.title);
                        else
                                sql += sep + "Title LIKE \"%s%%\"".printf (filter.title);
                        sep = " AND ";
@@ -185,7 +190,7 @@ class IMDbSqlite : Object {
                if (filter.genres.field != 0) {
                        sql += sep + "Genres&%d = %d".printf (filter.genres.field, filter.genres.field);
                }
-               sql += " LIMIT %d;".printf (100);
+               sql += " ORDER BY Votes DESC LIMIT %d;".printf (100);
 
                stdout.printf("SQL: \"%s\"\n", sql);
 
@@ -198,14 +203,27 @@ class IMDbSqlite : Object {
                do {
                        rc = stmt.step ();
                        if (rc == Sqlite.ROW) {
-                               string title = stmt.column_text (0);
                                int year = stmt.column_int (1);
+                               string title = stmt.column_text (0);
                                int rating = stmt.column_int (2);
                                int genres = stmt.column_int (3);
-                               receive_movie (title, year, rating, genres);
+                               receive_movie (strip_year (title, year), year, rating, genres);
                        }
                } while (rc == Sqlite.ROW);
 
                return 0;
        }
+
+       private string strip_year (string title, int year) {
+               string year_suffix = " (%d)".printf (year);
+               if (title.has_suffix (year_suffix))
+                       return title.substring (0, title.length - year_suffix.length);
+               year_suffix = " (%d/I)".printf (year);
+               if (title.has_suffix (year_suffix))
+                       return title.substring (0, title.length - year_suffix.length);
+               year_suffix = " (%d/II)".printf (year);
+               if (title.has_suffix (year_suffix))
+                       return title.substring (0, title.length - year_suffix.length);
+               return title.dup ();
+       }
 }