00001
00021 #ifdef HAVE_CONFIG_H
00022 #include "config.h"
00023 #endif
00024
00025 #define _GNU_SOURCE
00026 #include <sys/wait.h>
00027 #include <sys/stat.h>
00028 #include <unistd.h>
00029 #include <dirent.h>
00030 #include <signal.h>
00031 #include <time.h>
00032
00033 #include <stdio.h>
00034 #include <stdlib.h>
00035 #include <string.h>
00036
00037 #include "lightmediascanner.h"
00038 #include "lightmediascanner_private.h"
00039 #include "lightmediascanner_db_private.h"
00040
00041 struct master_db {
00042 sqlite3 *handle;
00043 sqlite3_stmt *get_files;
00044 };
00045
00046 struct slave_db {
00047 sqlite3 *handle;
00048 sqlite3_stmt *transaction_begin;
00049 sqlite3_stmt *transaction_commit;
00050 sqlite3_stmt *delete_file_info;
00051 sqlite3_stmt *update_file_info;
00052 };
00053
00054
00055
00056
00057
00058
00059 struct comm_finfo {
00060 int path_len;
00061 int base;
00062 int64_t id;
00063 time_t mtime;
00064 time_t dtime;
00065 size_t size;
00066 unsigned int flags;
00067 #define COMM_FINFO_FLAG_OUTDATED 1
00068 };
00069
00070 static int
00071 _master_send_file(const struct fds *master, const struct lms_file_info finfo, unsigned int flags)
00072 {
00073 struct comm_finfo ci;
00074
00075 ci.path_len = finfo.path_len;
00076 ci.base = finfo.base;
00077 ci.id = finfo.id;
00078 ci.mtime = finfo.mtime;
00079 ci.dtime = finfo.dtime;
00080 ci.size = finfo.size;
00081 ci.flags = flags;
00082
00083 if (write(master->w, &ci, sizeof(ci)) < 0) {
00084 perror("write");
00085 return -1;
00086 }
00087
00088 if (write(master->w, finfo.path, finfo.path_len) < 0) {
00089 perror("write");
00090 return -1;
00091 }
00092
00093 return 0;
00094 }
00095
00096 static int
00097 _master_send_finish(const struct fds *master)
00098 {
00099 struct comm_finfo ci = {-1, -1, -1, -1, -1, -1, 0};
00100
00101 if (write(master->w, &ci, sizeof(ci)) < 0) {
00102 perror("write");
00103 return -1;
00104 }
00105
00106 return 0;
00107 }
00108
00109 static int
00110 _master_recv_reply(const struct fds *master, struct pollfd *pfd, int *reply, int timeout)
00111 {
00112 int r;
00113
00114 r = poll(pfd, 1, timeout);
00115 if (r < 0) {
00116 perror("poll");
00117 return -1;
00118 }
00119
00120 if (r == 0)
00121 return 1;
00122
00123 if (read(master->r, reply, sizeof(*reply)) != sizeof(*reply)) {
00124 perror("read");
00125 return -2;
00126 }
00127
00128 return 0;
00129 }
00130
00131 static int
00132 _slave_send_reply(const struct fds *slave, int reply)
00133 {
00134 if (write(slave->w, &reply, sizeof(reply)) == 0) {
00135 perror("write");
00136 return -1;
00137 }
00138 return 0;
00139 }
00140
00141 static int
00142 _slave_recv_file(const struct fds *slave, struct lms_file_info *finfo, unsigned int *flags)
00143 {
00144 struct comm_finfo ci;
00145 static char path[PATH_SIZE + 1];
00146 int r;
00147
00148 r = read(slave->r, &ci, sizeof(ci));
00149 if (r != sizeof(ci)) {
00150 perror("read");
00151 return -1;
00152 }
00153
00154 finfo->path_len = ci.path_len;
00155 finfo->base = ci.base;
00156 finfo->id = ci.id;
00157 finfo->mtime = ci.mtime;
00158 finfo->dtime = ci.dtime;
00159 finfo->size = ci.size;
00160 finfo->path = NULL;
00161 *flags = ci.flags;
00162
00163 if (ci.path_len == -1)
00164 return 0;
00165
00166 if (ci.path_len > PATH_SIZE) {
00167 fprintf(stderr, "ERROR: path too long (%d/%d)\n",
00168 ci.path_len, PATH_SIZE);
00169 return -2;
00170 }
00171
00172 r = read(slave->r, path, ci.path_len);
00173 if (r != ci.path_len) {
00174 fprintf(stderr, "ERROR: could not read whole path %d/%d\n",
00175 r, ci.path_len);
00176 return -3;
00177 }
00178
00179 path[ci.path_len] = 0;
00180 finfo->path = path;
00181 return 0;
00182 }
00183
00184
00185
00186
00187
00188
00189 static int
00190 _slave_db_compile_all_stmts(struct slave_db *db)
00191 {
00192 sqlite3 *handle;
00193
00194 handle = db->handle;
00195
00196 db->transaction_begin = lms_db_compile_stmt_begin_transaction(handle);
00197 if (!db->transaction_begin)
00198 return -1;
00199
00200 db->transaction_commit = lms_db_compile_stmt_end_transaction(handle);
00201 if (!db->transaction_commit)
00202 return -2;
00203
00204 db->delete_file_info = lms_db_compile_stmt_delete_file_info(handle);
00205 if (!db->delete_file_info)
00206 return -3;
00207
00208 db->update_file_info = lms_db_compile_stmt_update_file_info(handle);
00209 if (!db->update_file_info)
00210 return -4;
00211
00212 return 0;
00213 }
00214
00215 static struct slave_db *
00216 _slave_db_open(const char *db_path)
00217 {
00218 struct slave_db *db;
00219
00220 db = calloc(1, sizeof(*db));
00221 if (!db) {
00222 perror("calloc");
00223 return NULL;
00224 }
00225
00226 if (sqlite3_open(db_path, &db->handle) != SQLITE_OK) {
00227 fprintf(stderr, "ERROR: could not open DB \"%s\": %s\n",
00228 db_path, sqlite3_errmsg(db->handle));
00229 goto error;
00230 }
00231
00232 return db;
00233
00234 error:
00235 sqlite3_close(db->handle);
00236 free(db);
00237 return NULL;
00238 }
00239
00240 static int
00241 _slave_db_close(struct slave_db *db)
00242 {
00243 if (db->transaction_begin)
00244 lms_db_finalize_stmt(db->transaction_begin, "transaction_begin");
00245
00246 if (db->transaction_commit)
00247 lms_db_finalize_stmt(db->transaction_commit, "transaction_commit");
00248
00249 if (db->delete_file_info)
00250 lms_db_finalize_stmt(db->delete_file_info, "delete_file_info");
00251
00252 if (db->update_file_info)
00253 lms_db_finalize_stmt(db->update_file_info, "update_file_info");
00254
00255 if (sqlite3_close(db->handle) != SQLITE_OK) {
00256 fprintf(stderr, "ERROR: clould not close DB (slave): %s\n",
00257 sqlite3_errmsg(db->handle));
00258 return -1;
00259 }
00260 free(db);
00261
00262 return 0;
00263 }
00264
00265 static int
00266 _init_sync_send(struct fds *fds)
00267 {
00268 return _slave_send_reply(fds, 0);
00269 }
00270
00271 static int
00272 _slave_work_int(lms_t *lms, struct fds *fds, struct slave_db *db)
00273 {
00274 struct lms_file_info finfo;
00275 void **parser_match;
00276 unsigned int counter, flags;
00277 int r;
00278
00279 parser_match = malloc(lms->n_parsers * sizeof(*parser_match));
00280 if (!parser_match) {
00281 perror("malloc");
00282 return -6;
00283 }
00284
00285 _init_sync_send(fds);
00286
00287 counter = 0;
00288 lms_db_begin_transaction(db->transaction_begin);
00289
00290 while (((r = _slave_recv_file(fds, &finfo, &flags)) == 0) &&
00291 finfo.path_len > 0) {
00292 r = lms_db_update_file_info(db->update_file_info, &finfo);
00293 if (r < 0)
00294 fprintf(stderr, "ERROR: could not update path in DB\n");
00295 else if (flags & COMM_FINFO_FLAG_OUTDATED) {
00296 int used;
00297
00298 used = lms_parsers_check_using(lms, parser_match, &finfo);
00299 if (!used)
00300 r = 0;
00301 else {
00302 r = lms_parsers_run(lms, db->handle, parser_match, &finfo);
00303 if (r < 0) {
00304 fprintf(stderr, "ERROR: pid=%d failed to parse \"%s\".\n",
00305 getpid(), finfo.path);
00306 lms_db_delete_file_info(db->delete_file_info, &finfo);
00307 }
00308 }
00309 }
00310
00311 _slave_send_reply(fds, r);
00312 counter++;
00313 if (counter > lms->commit_interval) {
00314 lms_db_end_transaction(db->transaction_commit);
00315 lms_db_begin_transaction(db->transaction_begin);
00316 counter = 0;
00317 }
00318 }
00319
00320 free(parser_match);
00321 lms_db_end_transaction(db->transaction_commit);
00322
00323 return r;
00324 }
00325
00326 static int
00327 _slave_work(lms_t *lms, struct fds *fds)
00328 {
00329 struct slave_db *db;
00330 int r;
00331
00332 db = _slave_db_open(lms->db_path);
00333 if (!db)
00334 return -1;
00335
00336 if (lms_parsers_setup(lms, db->handle) != 0) {
00337 fprintf(stderr, "ERROR: could not setup parsers.\n");
00338 r = -2;
00339 goto end;
00340 }
00341
00342 if (_slave_db_compile_all_stmts(db) != 0) {
00343 fprintf(stderr, "ERROR: could not compile statements.\n");
00344 r = -3;
00345 goto end;
00346 }
00347
00348 if (lms_parsers_start(lms, db->handle) != 0) {
00349 fprintf(stderr, "ERROR: could not start parsers.\n");
00350 r = -4;
00351 goto end;
00352 }
00353 if (lms->n_parsers < 1) {
00354 fprintf(stderr, "ERROR: no parser could be started, exit.\n");
00355 r = -5;
00356 goto end;
00357 }
00358
00359 r = _slave_work_int(lms, fds, db);
00360
00361 end:
00362 lms_parsers_finish(lms, db->handle);
00363 _slave_db_close(db);
00364 _init_sync_send(fds);
00365
00366 return r;
00367 }
00368
00369
00370
00371
00372
00373
00374 static int
00375 _master_db_compile_all_stmts(struct master_db *db)
00376 {
00377 sqlite3 *handle;
00378
00379 handle = db->handle;
00380
00381 db->get_files = lms_db_compile_stmt_get_files(handle);
00382 if (!db->get_files)
00383 return -1;
00384
00385 return 0;
00386 }
00387
00388 static struct master_db *
00389 _master_db_open(const char *db_path)
00390 {
00391 struct master_db *db;
00392
00393 db = calloc(1, sizeof(*db));
00394 if (!db) {
00395 perror("calloc");
00396 return NULL;
00397 }
00398
00399 if (sqlite3_open(db_path, &db->handle) != SQLITE_OK) {
00400 fprintf(stderr, "ERROR: could not open DB \"%s\": %s\n",
00401 db_path, sqlite3_errmsg(db->handle));
00402 goto error;
00403 }
00404
00405 if (lms_db_create_core_tables_if_required(db->handle) != 0) {
00406 fprintf(stderr, "ERROR: could not setup tables and indexes.\n");
00407 goto error;
00408 }
00409
00410 if (_master_db_compile_all_stmts(db) != 0) {
00411 fprintf(stderr, "ERROR: could not compile statements.\n");
00412 goto error;
00413 }
00414
00415 return db;
00416
00417 error:
00418 sqlite3_close(db->handle);
00419 free(db);
00420 return NULL;
00421 }
00422
00423 static int
00424 _master_db_close(struct master_db *db)
00425 {
00426 if (db->get_files)
00427 lms_db_finalize_stmt(db->get_files, "get_files");
00428
00429 if (sqlite3_close(db->handle) != SQLITE_OK) {
00430 fprintf(stderr, "ERROR: clould not close DB (master): %s\n",
00431 sqlite3_errmsg(db->handle));
00432 return -1;
00433 }
00434 free(db);
00435
00436 return 0;
00437 }
00438
00439 static void
00440 _calc_base(struct lms_file_info *finfo)
00441 {
00442 int i;
00443
00444 for (i = finfo->path_len - 1; i >= 0; i--)
00445 if (finfo->path[i] == '/') {
00446 finfo->base = i;
00447 return;
00448 }
00449 }
00450
00451 static inline void
00452 _update_finfo_from_stmt(struct lms_file_info *finfo, sqlite3_stmt *stmt)
00453 {
00454 finfo->id = sqlite3_column_int64(stmt, 0);
00455 finfo->path = sqlite3_column_blob(stmt, 1);
00456 finfo->path_len = sqlite3_column_bytes(stmt, 1);
00457 finfo->base = 0;
00458 finfo->mtime = sqlite3_column_int(stmt, 2);
00459 finfo->dtime = sqlite3_column_int(stmt, 3);
00460 finfo->size = sqlite3_column_int(stmt, 4);
00461 }
00462
00463 static inline void
00464 _update_finfo_from_stat(struct lms_file_info *finfo, const struct stat *st)
00465 {
00466 finfo->mtime = st->st_mtime;
00467 finfo->size = st->st_size;
00468 finfo->dtime = 0;
00469 }
00470
00471 static int
00472 _check_row(struct master_db *db, struct pinfo *pinfo)
00473 {
00474 struct lms_file_info finfo;
00475 struct stat st;
00476 unsigned int flags;
00477 int r, reply;
00478
00479 _update_finfo_from_stmt(&finfo, db->get_files);
00480
00481 flags = 0;
00482 if (stat(finfo.path, &st) == 0) {
00483 if (st.st_mtime == finfo.mtime && st.st_size == finfo.size) {
00484 if (finfo.dtime == 0)
00485 return 0;
00486 else
00487 finfo.dtime = 0;
00488 } else {
00489 _update_finfo_from_stat(&finfo, &st);
00490 flags |= COMM_FINFO_FLAG_OUTDATED;
00491 }
00492 } else {
00493 if (finfo.dtime)
00494 return 0;
00495 else
00496 finfo.dtime = time(NULL);
00497 }
00498
00499 _calc_base(&finfo);
00500
00501 if (_master_send_file(&pinfo->master, finfo, flags) != 0)
00502 return -1;
00503
00504 r = _master_recv_reply(&pinfo->master, &pinfo->poll, &reply,
00505 pinfo->lms->slave_timeout);
00506 if (r < 0)
00507 return -2;
00508 else if (r == 1) {
00509 fprintf(stderr, "ERROR: slave took too long, restart %d\n",
00510 pinfo->child);
00511 if (lms_restart_slave(pinfo, _slave_work) != 0)
00512 return -3;
00513 return 1;
00514 } else {
00515 if (reply < 0) {
00516
00517 fprintf(stderr, "ERROR: pid=%d failed to parse \"%s\".\n",
00518 getpid(), finfo.path);
00519 return (-reply) << 8;
00520 } else
00521 return reply;
00522 }
00523 }
00524
00525 static int
00526 _init_sync_wait(struct pinfo *pinfo, int restart)
00527 {
00528 int r, reply;
00529
00530 do {
00531 r = _master_recv_reply(&pinfo->master, &pinfo->poll, &reply,
00532 pinfo->lms->slave_timeout);
00533 if (r < 0)
00534 return -1;
00535 else if (r == 1 && restart) {
00536 fprintf(stderr, "ERROR: slave took too long, restart %d\n",
00537 pinfo->child);
00538 if (lms_restart_slave(pinfo, _slave_work) != 0)
00539 return -2;
00540 }
00541 } while (r != 0 && restart);
00542
00543 return r;
00544 }
00545
00546 static int
00547 _master_dummy_send_finish(const struct fds *master)
00548 {
00549 return 0;
00550 }
00551
00552 static int
00553 _check(struct pinfo *pinfo, int len, char *path)
00554 {
00555 char query[PATH_SIZE + 2];
00556 struct master_db *db;
00557 int r, ret;
00558
00559 db = _master_db_open(pinfo->lms->db_path);
00560 if (!db)
00561 return -1;
00562
00563 memcpy(query, path, len);
00564 query[len] = '%';
00565 query[len + 1] = '\0';
00566 ret = lms_db_get_files(db->get_files, query, len + 1);
00567 if (ret != 0)
00568 goto end;
00569
00570 if (lms_create_slave(pinfo, _slave_work) != 0) {
00571 ret = -2;
00572 goto end;
00573 }
00574 _init_sync_wait(pinfo, 1);
00575
00576 do {
00577 r = sqlite3_step(db->get_files);
00578 if (r == SQLITE_ROW) {
00579 if (_check_row(db, pinfo) < 0) {
00580 fprintf(stderr, "ERROR: could not check row.\n");
00581 ret = -1;
00582 goto finish_slave;
00583 }
00584 } else if (r != SQLITE_DONE) {
00585 fprintf(stderr, "ERROR: could not begin transaction: %s\n",
00586 sqlite3_errmsg(db->handle));
00587 ret = -2;
00588 goto finish_slave;
00589 }
00590 } while (r != SQLITE_DONE);
00591 ret = 0;
00592
00593 finish_slave:
00594 _master_send_finish(&pinfo->master);
00595 _init_sync_wait(pinfo, 0);
00596 lms_finish_slave(pinfo, _master_dummy_send_finish);
00597
00598 end:
00599 lms_db_reset_stmt(db->get_files);
00600 _master_db_close(db);
00601
00602 return ret;
00603 }
00604
00617 int
00618 lms_check(lms_t *lms, const char *top_path)
00619 {
00620 struct pinfo pinfo;
00621 int r;
00622 char path[PATH_SIZE];
00623
00624 if (!lms) {
00625 r = -1;
00626 goto end;
00627 }
00628
00629 if (!top_path) {
00630 r = -2;
00631 goto end;
00632 }
00633
00634 if (lms->is_processing) {
00635 fprintf(stderr, "ERROR: is already processing.\n");
00636 r = -3;
00637 goto end;
00638 }
00639
00640 if (!lms->parsers) {
00641 fprintf(stderr, "ERROR: no plugins registered.\n");
00642 r = -4;
00643 goto end;
00644 }
00645
00646 pinfo.lms = lms;
00647
00648 if (lms_create_pipes(&pinfo) != 0) {
00649 r = -5;
00650 goto end;
00651 }
00652
00653 if (realpath(top_path, path) == NULL) {
00654 perror("realpath");
00655 r = -6;
00656 goto close_pipes;
00657 }
00658
00659 lms->is_processing = 1;
00660 r = _check(&pinfo, strlen(path), path);
00661 lms->is_processing = 0;
00662
00663 close_pipes:
00664 lms_close_pipes(&pinfo);
00665 end:
00666 return r;
00667 }