IMDb: Split downloading from parsing into SQLite database

[cinaest] / src / imdb / imdb-downloader-cli.vala

diff --git a/src/imdb/imdb-downloader-cli.vala b/src/imdb/imdb-downloader-cli.vala
index 07da9ea..393a21f 100644 (file)
--- a/src/imdb/imdb-downloader-cli.vala
+++ b/src/imdb/imdb-downloader-cli.vala
@@ -55,7 +55,11 @@ class IMDbDownloaderCLI : Object, IMDbDownloader {
        // Private methods
 
        private bool do_download () {
-               download ("ftp.fu-berlin.de/pub/misc/movies/database/", MOVIES | GENRES | RATINGS | AKAS | PLOTS);
+               try {
+                       download ("ftp.fu-berlin.de/pub/misc/movies/database/", MOVIES | GENRES | RATINGS | AKAS | PLOTS);
+               } catch (Error e) {
+                       print ("Error: %s\n", e.message);
+               }
 
                return false;
        }
@@ -79,32 +83,60 @@ class IMDbDownloaderCLI : Object, IMDbDownloader {
                        var aka_parser = new AkaLineParser (sqlite);
                        var plot_parser = new PlotLineParser (sqlite);
 
-                       var downloader = new IMDbFtpDownloader (cancellable);
-                       downloader.progress_changed.connect (on_progress_changed);
+                       var downloader = new FtpDownloader (cancellable);
+
+                       var parser = new IMDbGzipParser (cancellable);
+
+                       downloader.progress.connect (on_progress);
 
                        if (MOVIES in flags) {
                                description_changed ("Downloading movie list ...");
-                               downloader.download (url + "movies.list.gz", movie_parser);
+                               downloader.download (url + "movies.list.gz", Path.build_filename (cache_dir, "movies.list.gz"));
                        }
                        percent_finished = 20;
                        if (GENRES in flags) {
                                description_changed ("Downloading genre data ...");
-                               downloader.download (url + "genres.list.gz", genre_parser);
+                               downloader.download (url + "genres.list.gz", Path.build_filename (cache_dir, "genres.list.gz"));
                        }
                        percent_finished = 40;
                        if (RATINGS in flags) {
                                description_changed ("Downloading rating data ...");
-                               downloader.download (url + "ratings.list.gz", rating_parser);
+                               downloader.download (url + "ratings.list.gz", Path.build_filename (cache_dir, "ratings.list.gz"));
                        }
                        percent_finished = 60;
                        if (AKAS in flags) {
                                description_changed ("Downloading alternative titles ...");
-                               downloader.download (url + "aka-titles.list.gz", aka_parser);
+                               downloader.download (url + "aka-titles.list.gz", Path.build_filename (cache_dir, "aka-titles.list.gz"));
                        }
                        percent_finished = 80;
                        if (PLOTS in flags) {
                                description_changed ("Downloading plots ...");
-                               downloader.download (url + "plot.list.gz", plot_parser);
+                               downloader.download (url + "plot.list.gz", Path.build_filename (cache_dir, "plot.list.gz"));
+                       }
+
+                       if (MOVIES in flags) {
+                               description_changed ("Parsing movie list ...");
+                               parser.parse (Path.build_filename (cache_dir, "movies.list.gz"), movie_parser);
+                       }
+                       percent_finished = 20;
+                       if (GENRES in flags) {
+                               description_changed ("Parsing genre data ...");
+                               parser.parse (Path.build_filename (cache_dir, "genres.list.gz"), genre_parser);
+                       }
+                       percent_finished = 40;
+                       if (RATINGS in flags) {
+                               description_changed ("Parsing rating data ...");
+                               parser.parse (Path.build_filename (cache_dir, "ratings.list.gz"), rating_parser);
+                       }
+                       percent_finished = 60;
+                       if (AKAS in flags) {
+                               description_changed ("Parsing alternative titles ...");
+                               parser.parse (Path.build_filename (cache_dir, "aka-titles.list.gz"), aka_parser);
+                       }
+                       percent_finished = 80;
+                       if (PLOTS in flags) {
+                               description_changed ("Parsing plots ...");
+                               parser.parse (Path.build_filename (cache_dir, "plot.list.gz"), plot_parser);
                        }
                } catch (Error e2) {
                        if (e2 is IOError.CANCELLED)
@@ -132,9 +164,12 @@ class IMDbDownloaderCLI : Object, IMDbDownloader {
                return null;
        }
 
-       private void on_progress_changed (int percent) {
+       private void on_progress (int dltotal, int dlnow) {
+       /*
                progress (percent_finished + percent / 5);
                stdout.printf ("%d %%\r", percent_finished + percent / 5);
+       */
+               stdout.printf ("%d / %d\r", dlnow, dltotal);
                stdout.flush ();
        }
 
@@ -153,11 +188,16 @@ class IMDbDownloaderCLI : Object, IMDbDownloader {
                loop.run ();
        }
 
+       public void show_desc (string desc) {
+               print ("DESC: \"%s\"\n", desc);
+       }
+
        public static void main () {
                Curl.global_init (Curl.GLOBAL_DEFAULT);
 
                // Start server
                var downloader = new IMDbDownloaderCLI ();
+               downloader.description_changed.connect (downloader.show_desc);
 
                Idle.add (downloader.do_download);
                downloader.run ();