00001
00021 #ifdef HAVE_CONFIG_H
00022 #include "config.h"
00023 #endif
00024
00025 #define _GNU_SOURCE
00026 #include <sys/wait.h>
00027 #include <sys/stat.h>
00028 #include <unistd.h>
00029 #include <dirent.h>
00030 #include <signal.h>
00031
00032 #include <stdio.h>
00033 #include <stdlib.h>
00034 #include <string.h>
00035
00036 #include "lightmediascanner.h"
00037 #include "lightmediascanner_private.h"
00038 #include "lightmediascanner_db_private.h"
00039
00040 struct db {
00041 sqlite3 *handle;
00042 sqlite3_stmt *transaction_begin;
00043 sqlite3_stmt *transaction_commit;
00044 sqlite3_stmt *get_file_info;
00045 sqlite3_stmt *insert_file_info;
00046 sqlite3_stmt *update_file_info;
00047 sqlite3_stmt *delete_file_info;
00048 sqlite3_stmt *set_file_dtime;
00049 };
00050
00051
00052
00053
00054
00055 static int
00056 _master_send_path(const struct fds *master, int plen, int dlen, const char *p)
00057 {
00058 int lengths[2];
00059
00060 lengths[0] = plen;
00061 lengths[1] = dlen;
00062
00063 if (write(master->w, lengths, sizeof(lengths)) < 0) {
00064 perror("write");
00065 return -1;
00066 }
00067
00068 if (write(master->w, p, plen) < 0) {
00069 perror("write");
00070 return -1;
00071 }
00072
00073 return 0;
00074 }
00075
00076 static int
00077 _master_send_finish(const struct fds *master)
00078 {
00079 const int lengths[2] = {-1, -1};
00080
00081 if (write(master->w, lengths, sizeof(lengths)) < 0) {
00082 perror("write");
00083 return -1;
00084 }
00085 return 0;
00086 }
00087
00088 static int
00089 _master_recv_reply(const struct fds *master, struct pollfd *pfd, int *reply, int timeout)
00090 {
00091 int r;
00092
00093 r = poll(pfd, 1, timeout);
00094 if (r < 0) {
00095 perror("poll");
00096 return -1;
00097 }
00098
00099 if (r == 0)
00100 return 1;
00101
00102 if (read(master->r, reply, sizeof(*reply)) != sizeof(*reply)) {
00103 perror("read");
00104 return -2;
00105 }
00106
00107 return 0;
00108 }
00109
00110 static int
00111 _slave_send_reply(const struct fds *slave, int reply)
00112 {
00113 if (write(slave->w, &reply, sizeof(reply)) == 0) {
00114 perror("write");
00115 return -1;
00116 }
00117 return 0;
00118 }
00119
00120 static int
00121 _slave_recv_path(const struct fds *slave, int *plen, int *dlen, char *path)
00122 {
00123 int lengths[2], r;
00124
00125 r = read(slave->r, lengths, sizeof(lengths));
00126 if (r != sizeof(lengths)) {
00127 perror("read");
00128 return -1;
00129 }
00130 *plen = lengths[0];
00131 *dlen = lengths[1];
00132
00133 if (*plen == -1)
00134 return 0;
00135
00136 if (*plen > PATH_SIZE) {
00137 fprintf(stderr, "ERROR: path too long (%d/%d)\n", *plen, PATH_SIZE);
00138 return -2;
00139 }
00140
00141 r = read(slave->r, path, *plen);
00142 if (r != *plen) {
00143 fprintf(stderr, "ERROR: could not read whole path %d/%d\n", r, *plen);
00144 return -3;
00145 }
00146
00147 path[*plen] = 0;
00148 return 0;
00149 }
00150
00151
00152
00153
00154
00155
00156 static int
00157 _db_compile_all_stmts(struct db *db)
00158 {
00159 sqlite3 *handle;
00160
00161 handle = db->handle;
00162 db->transaction_begin = lms_db_compile_stmt_begin_transaction(handle);
00163 if (!db->transaction_begin)
00164 return -1;
00165
00166 db->transaction_commit = lms_db_compile_stmt_end_transaction(handle);
00167 if (!db->transaction_commit)
00168 return -2;
00169
00170 db->get_file_info = lms_db_compile_stmt_get_file_info(handle);
00171 if (!db->get_file_info)
00172 return -4;
00173
00174 db->insert_file_info = lms_db_compile_stmt_insert_file_info(handle);
00175 if (!db->insert_file_info)
00176 return -5;
00177
00178 db->update_file_info = lms_db_compile_stmt_update_file_info(handle);
00179 if (!db->update_file_info)
00180 return -6;
00181
00182 db->delete_file_info = lms_db_compile_stmt_delete_file_info(handle);
00183 if (!db->delete_file_info)
00184 return -6;
00185
00186 db->set_file_dtime = lms_db_compile_stmt_set_file_dtime(handle);
00187 if (!db->set_file_dtime)
00188 return -7;
00189
00190 return 0;
00191 }
00192
00193 static struct db *
00194 _db_open(const char *db_path)
00195 {
00196 struct db *db;
00197
00198 db = calloc(1, sizeof(*db));
00199 if (!db) {
00200 perror("calloc");
00201 return NULL;
00202 }
00203
00204 if (sqlite3_open(db_path, &db->handle) != SQLITE_OK) {
00205 fprintf(stderr, "ERROR: could not open DB \"%s\": %s\n",
00206 db_path, sqlite3_errmsg(db->handle));
00207 goto error;
00208 }
00209
00210 if (lms_db_create_core_tables_if_required(db->handle) != 0) {
00211 fprintf(stderr, "ERROR: could not setup tables and indexes.\n");
00212 goto error;
00213 }
00214
00215 return db;
00216
00217 error:
00218 sqlite3_close(db->handle);
00219 free(db);
00220 return NULL;
00221 }
00222
00223 static int
00224 _db_close(struct db *db)
00225 {
00226 if (db->transaction_begin)
00227 lms_db_finalize_stmt(db->transaction_begin, "transaction_begin");
00228
00229 if (db->transaction_commit)
00230 lms_db_finalize_stmt(db->transaction_commit, "transaction_commit");
00231
00232 if (db->get_file_info)
00233 lms_db_finalize_stmt(db->get_file_info, "get_file_info");
00234
00235 if (db->insert_file_info)
00236 lms_db_finalize_stmt(db->insert_file_info, "insert_file_info");
00237
00238 if (db->update_file_info)
00239 lms_db_finalize_stmt(db->update_file_info, "update_file_info");
00240
00241 if (db->delete_file_info)
00242 lms_db_finalize_stmt(db->delete_file_info, "delete_file_info");
00243
00244 if (db->set_file_dtime)
00245 lms_db_finalize_stmt(db->set_file_dtime, "set_file_dtime");
00246
00247 if (sqlite3_close(db->handle) != SQLITE_OK) {
00248 fprintf(stderr, "ERROR: clould not close DB: %s\n",
00249 sqlite3_errmsg(db->handle));
00250 return -1;
00251 }
00252 free(db);
00253
00254 return 0;
00255 }
00256
00257 static int
00258 _retrieve_file_status(struct db *db, struct lms_file_info *finfo)
00259 {
00260 struct stat st;
00261 int r;
00262
00263 if (stat(finfo->path, &st) != 0) {
00264 perror("stat");
00265 return -1;
00266 }
00267
00268 r = lms_db_get_file_info(db->get_file_info, finfo);
00269 if (r == 0) {
00270 if (st.st_mtime <= finfo->mtime && finfo->size == st.st_size)
00271 return 0;
00272 else {
00273 finfo->mtime = st.st_mtime;
00274 finfo->size = st.st_size;
00275 return 1;
00276 }
00277 } else if (r == 1) {
00278 finfo->mtime = st.st_mtime;
00279 finfo->size = st.st_size;
00280 return 1;
00281 } else
00282 return -2;
00283 }
00284
00285 static void
00286 _ctxt_init(struct lms_context *ctxt, const lms_t *lms, sqlite3 *db)
00287 {
00288 ctxt->cs_conv = lms->cs_conv;
00289 ctxt->db = db;
00290 }
00291
00292 int
00293 lms_parsers_setup(lms_t *lms, sqlite3 *db)
00294 {
00295 struct lms_context ctxt;
00296 int i;
00297
00298 _ctxt_init(&ctxt, lms, db);
00299
00300 for (i = 0; i < lms->n_parsers; i++) {
00301 lms_plugin_t *plugin;
00302 int r;
00303
00304 plugin = lms->parsers[i].plugin;
00305 r = plugin->setup(plugin, &ctxt);
00306 if (r != 0) {
00307 fprintf(stderr, "ERROR: parser \"%s\" failed to setup: %d.\n",
00308 plugin->name, r);
00309 plugin->finish(plugin, &ctxt);
00310 lms_parser_del_int(lms, i);
00311 i--;
00312 }
00313 }
00314
00315 return 0;
00316 }
00317
00318 int
00319 lms_parsers_start(lms_t *lms, sqlite3 *db)
00320 {
00321 struct lms_context ctxt;
00322 int i;
00323
00324 _ctxt_init(&ctxt, lms, db);
00325
00326 for (i = 0; i < lms->n_parsers; i++) {
00327 lms_plugin_t *plugin;
00328 int r;
00329
00330 plugin = lms->parsers[i].plugin;
00331 r = plugin->start(plugin, &ctxt);
00332 if (r != 0) {
00333 fprintf(stderr, "ERROR: parser \"%s\" failed to start: %d.\n",
00334 plugin->name, r);
00335 plugin->finish(plugin, &ctxt);
00336 lms_parser_del_int(lms, i);
00337 i--;
00338 }
00339 }
00340
00341 return 0;
00342 }
00343
00344 int
00345 lms_parsers_finish(lms_t *lms, sqlite3 *db)
00346 {
00347 struct lms_context ctxt;
00348 int i;
00349
00350 _ctxt_init(&ctxt, lms, db);
00351
00352 for (i = 0; i < lms->n_parsers; i++) {
00353 lms_plugin_t *plugin;
00354 int r;
00355
00356 plugin = lms->parsers[i].plugin;
00357 r = plugin->finish(plugin, &ctxt);
00358 if (r != 0)
00359 fprintf(stderr, "ERROR: parser \"%s\" failed to finish: %d.\n",
00360 plugin->name, r);
00361 }
00362
00363 return 0;
00364 }
00365
00366 int
00367 lms_parsers_check_using(lms_t *lms, void **parser_match, struct lms_file_info *finfo)
00368 {
00369 int used, i;
00370
00371 used = 0;
00372 for (i = 0; i < lms->n_parsers; i++) {
00373 lms_plugin_t *plugin;
00374 void *r;
00375
00376 plugin = lms->parsers[i].plugin;
00377 r = plugin->match(plugin, finfo->path, finfo->path_len, finfo->base);
00378 parser_match[i] = r;
00379 if (r)
00380 used = 1;
00381 }
00382
00383 return used;
00384 }
00385
00386 int
00387 lms_parsers_run(lms_t *lms, sqlite3 *db, void **parser_match, struct lms_file_info *finfo)
00388 {
00389 struct lms_context ctxt;
00390 int i, failed, available;
00391
00392 _ctxt_init(&ctxt, lms, db);
00393
00394 failed = 0;
00395 available = 0;
00396 for (i = 0; i < lms->n_parsers; i++) {
00397 lms_plugin_t *plugin;
00398
00399 plugin = lms->parsers[i].plugin;
00400 if (parser_match[i]) {
00401 int r;
00402
00403 available++;
00404 r = plugin->parse(plugin, &ctxt, finfo, parser_match[i]);
00405 if (r != 0)
00406 failed++;
00407 }
00408 }
00409
00410 if (!failed)
00411 return 0;
00412 else if (failed == available)
00413 return -1;
00414 else
00415 return 1;
00416 }
00417
00418 static int
00419 _slave_work(lms_t *lms, struct fds *fds)
00420 {
00421 int r, len, base, counter;
00422 char path[PATH_SIZE];
00423 void **parser_match;
00424 struct db *db;
00425
00426 db = _db_open(lms->db_path);
00427 if (!db)
00428 return -1;
00429
00430 if (lms_parsers_setup(lms, db->handle) != 0) {
00431 fprintf(stderr, "ERROR: could not setup parsers.\n");
00432 r = -2;
00433 goto end;
00434 }
00435
00436 if (_db_compile_all_stmts(db) != 0) {
00437 fprintf(stderr, "ERROR: could not compile statements.\n");
00438 r = -3;
00439 goto end;
00440 }
00441
00442 if (lms_parsers_start(lms, db->handle) != 0) {
00443 fprintf(stderr, "ERROR: could not start parsers.\n");
00444 r = -4;
00445 goto end;
00446 }
00447 if (lms->n_parsers < 1) {
00448 fprintf(stderr, "ERROR: no parser could be started, exit.\n");
00449 r = -5;
00450 goto end;
00451 }
00452
00453 parser_match = malloc(lms->n_parsers * sizeof(*parser_match));
00454 if (!parser_match) {
00455 perror("malloc");
00456 r = -6;
00457 goto end;
00458 }
00459
00460 counter = 0;
00461 lms_db_begin_transaction(db->transaction_begin);
00462
00463 while (((r = _slave_recv_path(fds, &len, &base, path)) == 0) && len > 0) {
00464 struct lms_file_info finfo;
00465 int used, r;
00466
00467 finfo.path = path;
00468 finfo.path_len = len;
00469 finfo.base = base;
00470
00471 r = _retrieve_file_status(db, &finfo);
00472 if (r == 0) {
00473 if (finfo.dtime) {
00474 finfo.dtime = 0;
00475 lms_db_set_file_dtime(db->set_file_dtime, &finfo);
00476 }
00477 goto inform_end;
00478 } else if (r < 0) {
00479 fprintf(stderr, "ERROR: could not detect file status.\n");
00480 goto inform_end;
00481 }
00482
00483 used = lms_parsers_check_using(lms, parser_match, &finfo);
00484 if (!used)
00485 goto inform_end;
00486
00487 finfo.dtime = 0;
00488 if (finfo.id > 0)
00489 r = lms_db_update_file_info(db->update_file_info, &finfo);
00490 else
00491 r = lms_db_insert_file_info(db->insert_file_info, &finfo);
00492 if (r < 0) {
00493 fprintf(stderr, "ERROR: could not register path in DB\n");
00494 goto inform_end;
00495 }
00496
00497 r = lms_parsers_run(lms, db->handle, parser_match, &finfo);
00498 if (r < 0) {
00499 fprintf(stderr, "ERROR: pid=%d failed to parse \"%s\".\n",
00500 getpid(), finfo.path);
00501 lms_db_delete_file_info(db->delete_file_info, &finfo);
00502 }
00503
00504 inform_end:
00505 _slave_send_reply(fds, r);
00506 counter++;
00507 if (counter > lms->commit_interval) {
00508 lms_db_end_transaction(db->transaction_commit);
00509 lms_db_begin_transaction(db->transaction_begin);
00510 counter = 0;
00511 }
00512 }
00513
00514 free(parser_match);
00515 lms_db_end_transaction(db->transaction_commit);
00516 end:
00517 lms_parsers_finish(lms, db->handle);
00518 _db_close(db);
00519
00520 return r;
00521 }
00522
00523
00524
00525
00526
00527
00528 static int
00529 _consume_garbage(struct pollfd *pfd)
00530 {
00531 int r;
00532
00533 while ((r = poll(pfd, 1, 0)) > 0) {
00534 if (pfd->revents & (POLLERR | POLLHUP | POLLNVAL))
00535 return 0;
00536 else if (pfd->revents & POLLIN) {
00537 char c;
00538
00539 read(pfd->fd, &c, sizeof(c));
00540 }
00541 }
00542
00543 return r;
00544 }
00545
00546 static int
00547 _close_fds(struct fds *fds)
00548 {
00549 int r;
00550
00551 r = 0;
00552 if (close(fds->r) != 0) {
00553 r--;
00554 perror("close");
00555 }
00556
00557 if (close(fds->w) != 0) {
00558 r--;
00559 perror("close");
00560 }
00561
00562 return r;
00563 }
00564
00565 int
00566 lms_close_pipes(struct pinfo *pinfo)
00567 {
00568 int r;
00569
00570 r = _close_fds(&pinfo->master);
00571 r += _close_fds(&pinfo->slave);
00572
00573 return r;
00574 }
00575
00576 int
00577 lms_create_pipes(struct pinfo *pinfo)
00578 {
00579 int fds[2];
00580
00581 if (pipe(fds) != 0) {
00582 perror("pipe");
00583 return -1;
00584 }
00585 pinfo->master.r = fds[0];
00586 pinfo->slave.w = fds[1];
00587
00588 if (pipe(fds) != 0) {
00589 perror("pipe");
00590 close(pinfo->master.r);
00591 close(pinfo->slave.w);
00592 return -1;
00593 }
00594 pinfo->slave.r = fds[0];
00595 pinfo->master.w = fds[1];
00596
00597 pinfo->poll.fd = pinfo->master.r;
00598 pinfo->poll.events = POLLIN;
00599
00600 return 0;
00601 }
00602
00603 int
00604 lms_create_slave(struct pinfo *pinfo, int (*work)(lms_t *lms, struct fds *fds))
00605 {
00606 int r;
00607
00608 pinfo->child = fork();
00609 if (pinfo->child == -1) {
00610 perror("fork");
00611 return -1;
00612 }
00613
00614 if (pinfo->child > 0)
00615 return 0;
00616
00617 _close_fds(&pinfo->master);
00618 nice(19);
00619 r = work(pinfo->lms, &pinfo->slave);
00620 lms_free(pinfo->lms);
00621 _exit(r);
00622 return r;
00623 }
00624
00625 static int
00626 _waitpid(pid_t pid)
00627 {
00628 int status;
00629 pid_t r;
00630
00631 r = waitpid(pid, &status, 0);
00632 if (r > -1)
00633 return 0;
00634 else
00635 perror("waitpid");
00636
00637 return r;
00638 }
00639
00640 int
00641 lms_finish_slave(struct pinfo *pinfo, int (*finish)(const struct fds *fds))
00642 {
00643 int r;
00644
00645 if (pinfo->child <= 0)
00646 return 0;
00647
00648 r = finish(&pinfo->master);
00649 if (r == 0)
00650 r = _waitpid(pinfo->child);
00651 else {
00652 r = kill(pinfo->child, SIGKILL);
00653 if (r < 0)
00654 perror("kill");
00655 else
00656 r =_waitpid(pinfo->child);
00657 }
00658 pinfo->child = 0;
00659
00660 return r;
00661 }
00662
00663 int
00664 lms_restart_slave(struct pinfo *pinfo, int (*work)(lms_t *lms, struct fds *fds))
00665 {
00666 int status;
00667
00668 if (waitpid(pinfo->child, &status, WNOHANG) > 0) {
00669 if (WIFEXITED(status)) {
00670 int code;
00671
00672 code = WEXITSTATUS(status);
00673 if (code != 0) {
00674 fprintf(stderr, "ERROR: slave returned %d, exit.\n", code);
00675 pinfo->child = 0;
00676 return -1;
00677 }
00678 } else {
00679 if (WIFSIGNALED(status)) {
00680 int code;
00681
00682 code = WTERMSIG(status);
00683 fprintf(stderr, "ERROR: slave was terminated by signal %d.\n",
00684 code);
00685 }
00686 pinfo->child = 0;
00687 return -1;
00688 }
00689 }
00690
00691 if (kill(pinfo->child, SIGKILL))
00692 perror("kill");
00693
00694 if (waitpid(pinfo->child, &status, 0) < 0)
00695 perror("waitpid");
00696
00697 _consume_garbage(&pinfo->poll);
00698 return lms_create_slave(pinfo, work);
00699 }
00700
00701 static int
00702 _strcat(int base, char *path, const char *name)
00703 {
00704 int new_len, name_len;
00705
00706 name_len = strlen(name);
00707 new_len = base + name_len;
00708
00709 if (new_len >= PATH_SIZE) {
00710 path[base] = '\0';
00711 fprintf(stderr,
00712 "ERROR: path concatenation too long %d of %d "
00713 "available: \"%s\" + \"%s\"\n", new_len, PATH_SIZE,
00714 path, name);
00715 return -1;
00716 }
00717
00718 memcpy(path + base, name, name_len + 1);
00719
00720 return new_len;
00721 }
00722
00723 static int
00724 _process_file(struct pinfo *pinfo, int base, char *path, const char *name)
00725 {
00726 int new_len, reply, r;
00727
00728 new_len = _strcat(base, path, name);
00729 if (new_len < 0)
00730 return -1;
00731
00732 if (_master_send_path(&pinfo->master, new_len, base, path) != 0)
00733 return -2;
00734
00735 r = _master_recv_reply(&pinfo->master, &pinfo->poll, &reply,
00736 pinfo->lms->slave_timeout);
00737 if (r < 0)
00738 return -3;
00739 else if (r == 1) {
00740 fprintf(stderr, "ERROR: slave took too long, restart %d\n",
00741 pinfo->child);
00742 if (lms_restart_slave(pinfo, _slave_work) != 0)
00743 return -4;
00744 return 1;
00745 } else {
00746 if (reply < 0) {
00747
00748 fprintf(stderr, "ERROR: pid=%d failed to parse \"%s\".\n",
00749 getpid(), path);
00750 return (-reply) << 8;
00751 } else
00752 return reply;
00753 }
00754 }
00755
00756 static int
00757 _process_dir(struct pinfo *pinfo, int base, char *path, const char *name)
00758 {
00759 DIR *dir;
00760 struct dirent *de;
00761 int new_len, r;
00762
00763 new_len = _strcat(base, path, name);
00764 if (new_len < 0)
00765 return -1;
00766 else if (new_len + 1 >= PATH_SIZE) {
00767 fprintf(stderr, "ERROR: path too long\n");
00768 return 2;
00769 }
00770
00771 dir = opendir(path);
00772 if (dir == NULL) {
00773 perror("opendir");
00774 return 3;
00775 }
00776
00777 path[new_len] = '/';
00778 new_len++;
00779
00780 r = 0;
00781 while ((de = readdir(dir)) != NULL) {
00782 if (de->d_name[0] == '.')
00783 continue;
00784 if (de->d_type == DT_REG) {
00785 if (_process_file(pinfo, new_len, path, de->d_name) < 0) {
00786 path[new_len - 1] = '\0';
00787 fprintf(stderr,
00788 "ERROR: unrecoverable error parsing file, "
00789 "exit \"%s\".\n", path);
00790 r = -4;
00791 goto end;
00792 }
00793 } else if (de->d_type == DT_DIR || de->d_type == DT_UNKNOWN) {
00794 if (_process_dir(pinfo, new_len, path, de->d_name) < 0) {
00795 path[new_len - 1] = '\0';
00796 fprintf(stderr,
00797 "ERROR: unrecoverable error parsing dir, "
00798 "exit \"%s\".\n", path);
00799 r = -5;
00800 goto end;
00801 }
00802 }
00803 }
00804
00805 end:
00806 closedir(dir);
00807 return r;
00808 }
00809
00820 int
00821 lms_process(lms_t *lms, const char *top_path)
00822 {
00823 struct pinfo pinfo;
00824 int r, len;
00825 char path[PATH_SIZE], *bname;
00826
00827 if (!lms) {
00828 r = -1;
00829 goto end;
00830 }
00831
00832 if (!top_path) {
00833 r = -2;
00834 goto end;
00835 }
00836
00837 if (lms->is_processing) {
00838 fprintf(stderr, "ERROR: is already processing.\n");
00839 r = -3;
00840 goto end;
00841 }
00842
00843 if (!lms->parsers) {
00844 fprintf(stderr, "ERROR: no plugins registered.\n");
00845 r = -4;
00846 goto end;
00847 }
00848
00849 pinfo.lms = lms;
00850
00851 if (lms_create_pipes(&pinfo) != 0) {
00852 r = -5;
00853 goto end;
00854 }
00855
00856 if (lms_create_slave(&pinfo, _slave_work) != 0) {
00857 r = -6;
00858 goto close_pipes;
00859 }
00860
00861 if (realpath(top_path, path) == NULL) {
00862 perror("realpath");
00863 r = -7;
00864 goto finish_slave;
00865 }
00866
00867
00868 len = strlen(path);
00869 for (; len >= 0 && path[len] != '/'; len--);
00870 len++;
00871 bname = strdup(path + len);
00872 if (bname == NULL) {
00873 perror("strdup");
00874 r = -8;
00875 goto finish_slave;
00876 }
00877
00878 lms->is_processing = 1;
00879 r = _process_dir(&pinfo, len, path, bname);
00880 lms->is_processing = 0;
00881 free(bname);
00882
00883 finish_slave:
00884 lms_finish_slave(&pinfo, _master_send_finish);
00885 close_pipes:
00886 lms_close_pipes(&pinfo);
00887 end:
00888 return r;
00889 }