git.maemo.org Git - qwerkisync/blob - SyncerThread.cpp

   1 /*
   2  * Copyright (C) 2011, Jamie Thompson
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public
   6  * License as published by the Free Software Foundation; either
   7  * version 3 of the License, or (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public
  15  * License along with this program; If not, see
  16  * <http://www.gnu.org/licenses/>.
  17  */
  18
  19 #include "SyncerThread.h"
  20
  21 #include "DBBackends/AllBackends.h"
  22 #include "EventProcessors/Hasher.h"
  23 #include "EventProcessors/Writer.h"
  24
  25 #include "Attachment.h"
  26 #include "EventPreventer.h"
  27 #include "Settings.h"
  28 #include "EventTypes/EventFromFileList.h"
  29 #include "EventTypes/iEvent.h"
  30 #include "EventLogBackupManager.h"
  31 #include "EventParsers/Factory.h"
  32 #include "EventParsers/iEventParser.h"
  33
  34 #include <QDebug>
  35 #include <QDir>
  36 #include <QFile>
  37 #include <QFileInfo>
  38 #include <QProcess>
  39 #include <QSharedPointer>
  40
  41 #include <stdexcept>
  42
  43 typedef QPair<QFileInfo, uint> EventFileInfo;
  44
  45 QDebug operator<<(QDebug, QList<Attachment*> &);
  46 QFileInfoList FindEvents(QFileInfoList);
  47
  48 SyncerThread::SyncerThread(Settings &settings) :
  49         QThread(), m_Settings(settings)
  50 {
  51 //      m_Restart = false;
  52         m_Abort = false;
  53 }
  54
  55 SyncerThread::~SyncerThread()
  56 {
  57         m_Mutex.lock();
  58         m_Abort = true;
  59         m_Condition.wakeOne();
  60         m_Mutex.unlock();
  61
  62         wait();
  63 }
  64
  65 void SyncerThread::Sync()
  66 {
  67         if (!isRunning())
  68         {
  69                 start(LowPriority);
  70         }
  71         else
  72         {
  73                 m_Restart = true;
  74                 m_Condition.wakeOne();
  75         }
  76 }
  77
  78 #include "NumberToNameLookup.h"
  79
  80 void SyncerThread::run()
  81 {
  82         try
  83         {
  84                 EventPreventer preventEvents(CurrentSettings());
  85                 EventLogBackupManager backupManager(CurrentSettings());
  86
  87                 if(CurrentSettings().Mode() == Settings::MODE_EXPORT)
  88                 {
  89                         qDebug() << "Exporting events";
  90
  91                         // Temp - Remove directory first so it's always just this export for now
  92                         QDir().rmpath(CurrentSettings().Directory());
  93
  94                         DBBackends::AllBackends allBackends(CurrentSettings());
  95                         EventProcessors::Writer eventWriter(CurrentSettings());
  96                         QObject::connect(&eventWriter, SIGNAL(EventProcessed(int,int)), this, SIGNAL(EventProcessed(int,int)));
  97                         allBackends.Process(eventWriter);
  98                 }
  99                 else
 100                 {
 101                         qDebug() << "Importing events";
 102
 103                         backupManager.CreateBackup();
 104
 105                         qDebug() << "Scanning filesystem";
 106
 107                         // Open chosen directory and grab *all* files within (we filter at
 108                         // parsing stage)
 109                         QFileInfoList candidates(FindEvents(
 110                                 QDir(CurrentSettings().Directory()).entryInfoList(QDir::AllEntries | QDir::NoDotAndDotDot)));
 111
 112                         // We're going to be storing hashes to match up events on disk to
 113                         // those in the DB. Assuming 1 event per file is a good starting point.
 114                         // Each file can provide multiple events, so sotre the hash of the
 115                         // event, the file information, and the record index.
 116                         uint totalEvents(candidates.count());
 117                         QHash<QPair<QString, uint>, iHashable::Hash> hashesByPath;
 118                         hashesByPath.reserve(totalEvents);
 119                         QHash<iHashable::Hash, QPair<QString, uint> > pathsByHashes;
 120                         pathsByHashes.reserve(totalEvents);
 121
 122                         qDebug() << "Hashing events";
 123                         {
 124                                 // Work our way through the candidates...
 125                                 int idx = 0;
 126                                 foreach(QFileInfo fileInfo, candidates)
 127                                 {
 128                                         ++idx;
 129                                         try
 130                                         {
 131                                                 EventTypes::EventFromFileList fileEvents(ProcessFile(fileInfo.absoluteFilePath()));
 132
 133                                                 foreach(EventTypes::EventFromFile eventFromFile, fileEvents)
 134                                                 {
 135                                                         const uint hash(eventFromFile.first->HashCode());
 136                                                         pathsByHashes.insert(hash, QPair<QString, iHashable::Hash>(fileInfo.absoluteFilePath(), eventFromFile.second));
 137                                                         hashesByPath.insert(QPair<QString, iHashable::Hash>(fileInfo.absoluteFilePath(), eventFromFile.second), hash);
 138
 139                                                         qDebug() << hash;
 140                                                 }
 141                                         }
 142                                         catch(std::runtime_error& exception)
 143                                         {
 144                                                 qDebug() << exception.what() << endl;
 145                                         }
 146
 147                                         emit EventProcessed(idx, candidates.count());
 148                                 }
 149                         }
 150
 151                         qDebug() << "Determining new events";
 152
 153                         // Calculate the new events by removing the hashes of events already
 154                         // found in the DB
 155                         QSet<iHashable::Hash> newHashes(QSet<iHashable::Hash>::fromList( pathsByHashes.keys()));
 156
 157                         //EventHasher eventHasher;
 158                         //ProcessDBEvents(eventHasher);
 159                         DBBackends::AllBackends allBackends(CurrentSettings());
 160                         EventProcessors::Hasher eventHasher;
 161                         allBackends.Process(eventHasher);
 162
 163                         foreach(iHashable::Hash hash, eventHasher.m_Hashes)
 164                                 newHashes.remove(hash);
 165
 166                         qDebug() << QString("%1 new hashes").arg(newHashes.size()) << endl;
 167                         foreach(iHashable::Hash hash, newHashes)
 168                                 qDebug() << hash << endl;
 169
 170                         // Now an optimisation: group the new hashes by the files they come
 171                         // from. This enables each file to only be parsed once and return
 172                         // all the required events from it.
 173                         QHash<QString, QList<iHashable::Hash> > newHashesByPath;
 174                         foreach(iHashable::Hash newHash, newHashes)
 175                         {
 176                                 const QString &currentPath(pathsByHashes.value(newHash).first);
 177                                 if(newHashesByPath.contains(currentPath))
 178                                         newHashesByPath[currentPath].append(newHash);
 179                                 else
 180                                         newHashesByPath[currentPath] = QList<iHashable::Hash>() << newHash;
 181                         }
 182
 183                         qDebug() << "Scanning addressbook";
 184
 185                         // Prepare the telephone-address book ID lookup.
 186                         NumberToNameLookup lookup;
 187
 188                         qDebug() << "Importing new events";
 189
 190                         // Re-parse the new events and insert them
 191                         allBackends.PreInsert();
 192                         {
 193                                 int idx = 0;
 194                                 foreach(QString filename, newHashesByPath.keys())
 195                                 {
 196                                         QList<uint> recordsToReturn;
 197                                         foreach(iHashable::Hash newHash, newHashesByPath.value(filename))
 198                                                 recordsToReturn.append(pathsByHashes.value(newHash).second);
 199
 200                                         // Repeating an action that caused an exception last time
 201                                         // shouldn't happen again, but just in case...
 202                                         try
 203                                         {
 204                                                 foreach(EventTypes::EventFromFile newEventFromFile, ProcessFile(filename, recordsToReturn))
 205                                                 {
 206                                                         // ...and insert it into the DB
 207                                                         try
 208                                                         {
 209                                                                 allBackends.Insert(*newEventFromFile.first, lookup);
 210                                                         }
 211                                                         catch(const std::runtime_error &exception)
 212                                                         {
 213                                                                 qDebug() << "Unable to insert event: " << exception.what();
 214                                                         }
 215
 216                                                         emit EventProcessed(++idx, newHashes.count());
 217                                                 }
 218                                         }
 219                                         catch(const std::runtime_error &exception)
 220                                         {
 221                                                 qDebug() << exception.what() << endl;
 222                                         }
 223
 224                                         // Just to make sure the listeners are synced even if the
 225                                         // earlier call is skipped due to errors...
 226                                         emit EventProcessed(idx, newHashes.count());
 227                                 }
 228                         }
 229                         allBackends.PostInsert(); // Perform any post-insert cleanup (i.e. reindexing)
 230
 231                         // Need to find a better way of refreshing the conversations view...
 232                         QProcess::execute("pkill rtcom");
 233
 234                         // Signal we completed successfully.
 235                         backupManager.UnlockCurrentBackup();
 236                 }
 237
 238                 // May as well call this explicitly despite it being called by the
 239                 // destructor - it's harmless.
 240                 preventEvents.RestoreAccounts();
 241         }
 242         catch(std::runtime_error exception)
 243         {
 244                 qDebug() << exception.what();
 245         }
 246 }
 247
 248 EventTypes::EventFromFileList SyncerThread::ProcessFile(const QString &path, const QList<uint> &recordsToReturn) const
 249 {
 250         qDebug() << path << endl;
 251         QFile eventFile(path);
 252
 253         // If the file's ok, process it...
 254         if (eventFile.open(QFile::ReadOnly))
 255         {
 256                 // Identify type of file...
 257                 EventParsers::iEventParser * parser(EventParsers::Factory::CreateParser(CurrentSettings(), path));
 258
 259                 // ...and grab the events from it (if it's a supported format)
 260                 if(NULL != parser)
 261                         return parser->ParseFile(eventFile, recordsToReturn);
 262                 else
 263                         return EventTypes::EventFromFileList();
 264         }
 265         else
 266                 throw std::runtime_error(QString("Unable to open: %1").arg(path).toStdString());
 267 }
 268
 269 QFileInfoList FindEvents(QFileInfoList currentCandidate)
 270 {
 271         QFileInfoList foundEvents;
 272         foreach(QFileInfo fileInfo, currentCandidate)
 273         {
 274                 if(fileInfo.isDir())
 275                         foundEvents.append(FindEvents(QDir(fileInfo.absoluteFilePath()).entryInfoList(QDir::AllEntries | QDir::NoDotAndDotDot)));
 276                 else
 277                         foundEvents.append(fileInfo);
 278         }
 279
 280         return foundEvents;
 281 }
 282
 283 QDebug operator<<(QDebug dbg, QList<Attachment*> &attachments)
 284 {
 285         dbg.nospace() << "Attachments" << "\n";
 286
 287         foreach(Attachment* attachment, attachments)
 288                 dbg.nospace() << *attachment << "\n";
 289
 290         dbg.nospace() << "\n";
 291
 292         return dbg;
 293 }