git.maemo.org Git - cinaest/blob - src/imdb/imdb-plaintext-downloader.vala

   1 using GLib;
   2
   3 class IMDbDownloadServer : Object, IMDbDownloader {
   4         MainLoop loop;
   5         Cancellable cancellable;
   6         int64 sofar;
   7         int64 total;
   8         bool running;
   9         unowned IMDbSqlite sqlite;
  10         string[] mirrors = {
  11                 "ftp.fu-berlin.de/pub/misc/movies/database/",
  12                 "ftp.funet.fi/pub/mirrors/ftp.imdb.com/pub/",
  13                 "ftp.sunet.se/pub/tv+movies/imdb/"
  14         };
  15
  16         delegate void ParseLineFunction (string line);
  17
  18         construct {
  19                 loop = new MainLoop (null, false);
  20                 cancellable = new Cancellable ();
  21         }
  22
  23         // IMDbDownloader implementation
  24
  25         public void download (string mirror, int flags) throws DBus.Error {
  26                 if (running) {
  27                         message ("Download in progress. Abort.\n");
  28                         return;
  29                 }
  30                 running = true;
  31                 message ("Download started (%x).", flags);
  32                 progress (0);
  33                 download_imdb_async.begin ("ftp://anonymous@" + mirror, flags, Priority.DEFAULT_IDLE);
  34                 message ("Download finished.");
  35         }
  36
  37         public void cancel () throws DBus.Error {
  38                 cancellable.cancel ();
  39         }
  40
  41         public string[] get_mirrors () throws DBus.Error {
  42                 return mirrors;
  43         }
  44
  45         // Private methods
  46
  47         async void download_imdb_async (string mirror, int flags, int io_priority) {
  48                 Mount m;
  49                 File movies = File.new_for_uri (mirror + "/movies.list.gz");
  50                 File genres = File.new_for_uri (mirror + "/genres.list.gz");
  51                 File ratings = File.new_for_uri (mirror + "/ratings.list.gz");
  52
  53                 description_changed ("Connecting to FTP ...");
  54                 progress (0);
  55
  56                 try {
  57                         m = yield movies.find_enclosing_mount_async(io_priority, cancellable);
  58                 } catch (Error e0) {
  59                         try {
  60                                 bool mounted = yield movies.mount_enclosing_volume (MountMountFlags.NONE, null, cancellable);
  61                                 if (mounted) {
  62                                         m = yield movies.find_enclosing_mount_async(io_priority, cancellable);
  63                                 } else {
  64                                         running = false;
  65                                         return;
  66                                 }
  67                         } catch (Error e1) {
  68                                 critical ("Failed to mount: %s\n", e1.message);
  69                                 running = false;
  70                                 return;
  71                         }
  72                 }
  73                 stdout.printf ("Mounted: %s\n", m.get_name ());
  74
  75                 description_changed ("Querying file sizes ...");
  76                 try {
  77                         FileInfo info;
  78
  79                         if (MOVIES in flags) {
  80                                 info = yield movies.query_info_async ("", FileQueryInfoFlags.NONE, io_priority, cancellable);
  81                                 total += info.get_size ();
  82                         }
  83                         if (GENRES in flags) {
  84                                 info = yield genres.query_info_async ("", FileQueryInfoFlags.NONE, io_priority, cancellable);
  85                                 total += info.get_size ();
  86                         }
  87                         if (RATINGS in flags) {
  88                                 info = yield ratings.query_info_async ("", FileQueryInfoFlags.NONE, io_priority, cancellable);
  89                                 total += info.get_size ();
  90                         }
  91                 } catch (Error e3) {
  92                         warning ("Failed to get size: %s\n", e3.message);
  93                         total = 0;
  94                 }
  95
  96                 var cache_dir = Path.build_filename (Environment.get_user_cache_dir (), "cinaest");
  97                 DirUtils.create_with_parents (cache_dir, 0770);
  98
  99                 var _sqlite = new IMDbSqlite (Path.build_filename (cache_dir, "imdb.db"));
 100                 sqlite = _sqlite;
 101                 _sqlite.clear ();
 102
 103                 try {
 104                         var movie_parser = new MovieLineParser (sqlite);
 105                         var genre_parser = new GenreLineParser (sqlite);
 106                         var rating_parser = new RatingLineParser (sqlite);
 107                         sofar = 0;
 108
 109                         if (MOVIES in flags) {
 110                                 description_changed ("Downloading movie list ...");
 111                                 yield download_async(movies, movie_parser, io_priority);
 112                         }
 113                         if (GENRES in flags) {
 114                                 description_changed ("Downloading genre data ...");
 115                                 yield download_async(genres, genre_parser, io_priority);
 116                         }
 117                         if (RATINGS in flags) {
 118                                 description_changed ("Downloading rating data ...");
 119                                 yield download_async(ratings, rating_parser, io_priority);
 120                         }
 121                 } catch (Error e2) {
 122                         if (e2 is IOError.CANCELLED)
 123                                 message ("Download cancelled.\n");
 124                         else
 125                                 warning ("Failed to open/read stream: %s\n", e2.message);
 126                 }
 127
 128                 if (!cancellable.is_cancelled ()) {
 129                         stdout.printf ("Download complete.\n");
 130                         progress (100);
 131                 }
 132
 133                 try {
 134                         bool unmounted = yield m.unmount(MountUnmountFlags.NONE, null);
 135                         if (!unmounted) {
 136                                 warning ("Failed to unmount.\n");
 137                         }
 138                 } catch (Error e4) {
 139                         warning ("Failed to unmount: %s\n", e4.message);
 140                 }
 141
 142                 sqlite = null;
 143                 running = false;
 144
 145                 // FIXME - use a timeout?
 146                 loop.quit ();
 147         }
 148
 149         async void download_async (File f, LineParser parser, int io_priority) throws Error {
 150                 int percent = 0;
 151                 unowned string line = null;
 152
 153                 var stream = new GzipInputStream (yield f.read_async (io_priority, cancellable));
 154                 var data = new DataInputStream(stream);
 155
 156                 do {
 157                         size_t l;
 158
 159                         line = yield data.read_line_async (io_priority, cancellable, out l);
 160                         if (line != null)
 161                                 parser.parse_line (line);
 162
 163                         if (total == 0)
 164                                 continue;
 165                         int p = (int) (100 * (sofar + stream.total_in ()) / total);
 166                         if (p > percent) {
 167                                 percent = p;
 168                                 progress (p);
 169                         }
 170                 } while (line != null);
 171
 172                 sofar += stream.total_in ();
 173
 174                 yield stream.close_async (io_priority, cancellable);
 175         }
 176
 177         public void run () {
 178                 loop.run ();
 179         }
 180
 181         public static void main () {
 182                 try {
 183                         var conn = DBus.Bus.get (DBus.BusType.SESSION);
 184                         dynamic DBus.Object bus = conn.get_object ("org.freedesktop.DBus",
 185                                                                    "/org/freedesktop/DBus",
 186                                                                    "org.freedesktop.DBus");
 187
 188                         // Try to register service in session bus
 189                         uint request_name_result = bus.request_name (DBUS_SERVICE, (uint) 0);
 190
 191                         if (request_name_result == DBus.RequestNameReply.PRIMARY_OWNER) {
 192                                 // Start server
 193                                 var server = new IMDbDownloadServer ();
 194                                 conn.register_object (DBUS_OBJECT, server);
 195
 196                                 server.run ();
 197                         } else {
 198                                 critical ("Service \"org.maemo.cinaest.IMDb\" already registered. Abort.\n");
 199                         }
 200                 } catch (Error e) {
 201                         critical ("Oops: %s\n", e.message);
 202                 }
 203         }
 204 }
 205
 206 abstract class LineParser {
 207         internal unowned IMDbSqlite sqlite;
 208
 209         public LineParser (IMDbSqlite _sqlite) {
 210                 sqlite = _sqlite;
 211         }
 212
 213         public abstract void parse_line (string line);
 214
 215         internal bool skip_title (string title) {
 216                 if (title.has_suffix ("(TV)")) {
 217                         return true;
 218                 }
 219                 if (title.has_suffix ("(V)")) {
 220                         return true;
 221                 }
 222                 if (title.has_suffix ("(VG)")) {
 223                         return true;
 224                 }
 225                 return false;
 226         }
 227 }
 228
 229 class MovieLineParser : LineParser {
 230         Regex re_movie;
 231
 232         public MovieLineParser (IMDbSqlite _sqlite) {
 233                 base (_sqlite);
 234                 try {
 235                         re_movie = new Regex ("^([^\t]+)\t+([0-9]+)$");
 236                 } catch (RegexError e) {
 237                         critical ("Failed to initialize regex: %s\n", e.message);
 238                 }
 239         }
 240
 241         public override void parse_line (string line) {
 242                 MatchInfo matchinfo;
 243
 244                 // Skip series episodes
 245                 if (line[0] == '"')
 246                         return;
 247
 248                 if (!re_movie.match(line, 0, out matchinfo))
 249                         return;
 250
 251                 string title;
 252                 string year = matchinfo.fetch (2);
 253                 try {
 254                         title = convert(matchinfo.fetch (1), -1, "utf-8", "latin1");
 255                 } catch (ConvertError e) {
 256                         return;
 257                 }
 258
 259                 if (skip_title (title))
 260                         return;
 261
 262                 sqlite.add_movie (title, year.to_int ());
 263         }
 264 }
 265
 266 class GenreLineParser : LineParser {
 267         Regex re_genre;
 268
 269         public GenreLineParser (IMDbSqlite _sqlite) {
 270                 base (_sqlite);
 271                 try {
 272                         re_genre = new Regex ("^([^\t]+)\t+([A-Za-z-]+)$");
 273                 } catch (RegexError e) {
 274                         critical ("Failed to initialize regex: %s\n", e.message);
 275                 }
 276         }
 277
 278         public override void parse_line (string line) {
 279                 MatchInfo matchinfo;
 280
 281                 // Skip series episodes
 282                 if (line[0] == '"')
 283                         return;
 284
 285                 if (!re_genre.match(line, 0, out matchinfo))
 286                         return;
 287
 288                 string title;
 289                 string genre = matchinfo.fetch (2);
 290                 try {
 291                         title = convert(matchinfo.fetch (1), -1, "utf-8", "latin1");
 292                 } catch (ConvertError e) {
 293                         return;
 294                 }
 295
 296                 sqlite.movie_add_genre (title, genre);
 297         }
 298 }
 299
 300 class RatingLineParser : LineParser {
 301         Regex re_rating;
 302
 303         public RatingLineParser (IMDbSqlite _sqlite) {
 304                 base (_sqlite);
 305                 try {
 306                         re_rating = new Regex ("^      .+ +[0-9]+ +([0-9.]+) +(.+)$");
 307                 } catch (RegexError e) {
 308                         critical ("Failed to initialize regex: %s\n", e.message);
 309                 }
 310         }
 311
 312         public override void parse_line (string line) {
 313                 MatchInfo matchinfo;
 314
 315                 // Skip series episodes
 316                 if (line[0] == '"')
 317                         return;
 318
 319                 if (!re_rating.match(line, 0, out matchinfo))
 320                         return;
 321
 322                 string title;
 323                 string rating = matchinfo.fetch (1);
 324                 try {
 325                         title = convert(matchinfo.fetch (2), -1, "utf-8", "latin1");
 326                 } catch (ConvertError e) {
 327                         return;
 328                 }
 329
 330                 // Skip series episodes
 331                 if (title[0] == '"')
 332                         return;
 333
 334                 if (skip_title (title))
 335                         return;
 336
 337                 sqlite.movie_set_rating (title, (int) (rating.to_double () * 10));
 338         }
 339 }