git.maemo.org Git - qwerkisync/blob - SyncerThread.cpp

   1 /*
   2  * Copyright (C) 2011, Jamie Thompson
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public
   6  * License as published by the Free Software Foundation; either
   7  * version 3 of the License, or (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public
  15  * License along with this program; If not, see
  16  * <http://www.gnu.org/licenses/>.
  17  */
  18
  19 #include "SyncerThread.h"
  20
  21 #include "DBBackends/AllBackends.h"
  22 #include "EventProcessors/Hasher.h"
  23 #include "EventProcessors/Writer.h"
  24
  25 #include "Attachment.h"
  26 #include "EventPreventer.h"
  27 #include "Settings.h"
  28 #include "EventTypes/EventFromFileList.h"
  29 #include "EventTypes/iEvent.h"
  30 #include "EventLogBackupManager.h"
  31 #include "EventParsers/Factory.h"
  32 #include "EventParsers/iEventParser.h"
  33
  34 #include <QDebug>
  35 #include <QDir>
  36 #include <QFile>
  37 #include <QFileInfo>
  38 #include <QProcess>
  39 #include <QSharedPointer>
  40
  41 #include <stdexcept>
  42
  43 typedef QPair<QFileInfo, uint> EventFileInfo;
  44
  45 QDebug operator<<(QDebug, QList<Attachment*> &);
  46 QFileInfoList FindEvents(QFileInfoList);
  47
  48 SyncerThread::SyncerThread(Settings &settings) :
  49         QThread(), m_Settings(settings)
  50 {
  51 //      m_Restart = false;
  52         m_Abort = false;
  53 }
  54
  55 SyncerThread::~SyncerThread()
  56 {
  57         m_Mutex.lock();
  58         m_Abort = true;
  59         m_Condition.wakeOne();
  60         m_Mutex.unlock();
  61
  62         wait();
  63 }
  64
  65 void SyncerThread::Sync()
  66 {
  67         if (!isRunning())
  68         {
  69                 start(LowPriority);
  70         }
  71         else
  72         {
  73                 m_Restart = true;
  74                 m_Condition.wakeOne();
  75         }
  76 }
  77
  78 #include "NumberToNameLookup.h"
  79
  80 void SyncerThread::run()
  81 {
  82         try
  83         {
  84                 EventPreventer preventEvents(CurrentSettings());
  85                 EventLogBackupManager backupManager(CurrentSettings());
  86
  87                 if(CurrentSettings().Mode() == Settings::MODE_EXPORT)
  88                 {
  89                         qDebug() << "Exporting events";
  90
  91                         // Temp - Remove directory first so it's always just this export for now
  92                         QDir().rmpath(CurrentSettings().Directory());
  93
  94                         DBBackends::AllBackends allBackends(CurrentSettings());
  95                         NumberToNameLookup lookup; // Prepare the telephone-address book ID lookup.
  96                         EventProcessors::Writer eventWriter(CurrentSettings(), lookup);
  97                         QObject::connect(&eventWriter, SIGNAL(EventProcessed(int,int)), this, SIGNAL(EventProcessed(int,int)));
  98                         allBackends.Process(eventWriter);
  99                 }
 100                 else
 101                 {
 102                         qDebug() << "Importing events";
 103
 104                         backupManager.CreateBackup();
 105
 106                         qDebug() << "Scanning filesystem";
 107
 108                         // Open chosen directory and grab *all* files within (we filter at
 109                         // parsing stage)
 110                         QFileInfoList candidates(FindEvents(
 111                                 QDir(CurrentSettings().Directory()).entryInfoList(QDir::AllEntries | QDir::NoDotAndDotDot)));
 112
 113                         // We're going to be storing hashes to match up events on disk to
 114                         // those in the DB. Assuming 1 event per file is a good starting point.
 115                         // Each file can provide multiple events, so sotre the hash of the
 116                         // event, the file information, and the record index.
 117                         uint totalEvents(candidates.count());
 118                         QHash<QPair<QString, uint>, iHashable::Hash> hashesByPath;
 119                         hashesByPath.reserve(totalEvents);
 120                         QHash<iHashable::Hash, QPair<QString, uint> > pathsByHashes;
 121                         pathsByHashes.reserve(totalEvents);
 122
 123                         qDebug() << "Hashing events";
 124                         {
 125                                 // Work our way through the candidates...
 126                                 int idx = 0;
 127                                 foreach(QFileInfo fileInfo, candidates)
 128                                 {
 129                                         ++idx;
 130                                         try
 131                                         {
 132                                                 EventTypes::EventFromFileList fileEvents(ProcessFile(fileInfo.absoluteFilePath()));
 133
 134                                                 foreach(EventTypes::EventFromFile eventFromFile, fileEvents)
 135                                                 {
 136                                                         const uint hash(eventFromFile.first->HashCode());
 137                                                         pathsByHashes.insert(hash, QPair<QString, iHashable::Hash>(fileInfo.absoluteFilePath(), eventFromFile.second));
 138                                                         hashesByPath.insert(QPair<QString, iHashable::Hash>(fileInfo.absoluteFilePath(), eventFromFile.second), hash);
 139
 140                                                         qDebug() << hash;
 141                                                 }
 142                                         }
 143                                         catch(std::runtime_error& exception)
 144                                         {
 145                                                 qDebug() << exception.what() << endl;
 146                                         }
 147
 148                                         emit EventProcessed(idx, candidates.count());
 149                                 }
 150                         }
 151
 152                         qDebug() << "Determining new events";
 153
 154                         // Calculate the new events by removing the hashes of events already
 155                         // found in the DB
 156                         QSet<iHashable::Hash> newHashes(QSet<iHashable::Hash>::fromList( pathsByHashes.keys()));
 157
 158                         //EventHasher eventHasher;
 159                         //ProcessDBEvents(eventHasher);
 160                         DBBackends::AllBackends allBackends(CurrentSettings());
 161                         EventProcessors::Hasher eventHasher;
 162                         allBackends.Process(eventHasher);
 163
 164                         foreach(iHashable::Hash hash, eventHasher.m_Hashes)
 165                                 newHashes.remove(hash);
 166
 167                         qDebug() << QString("%1 new hashes").arg(newHashes.size()) << endl;
 168                         foreach(iHashable::Hash hash, newHashes)
 169                                 qDebug() << hash << endl;
 170
 171                         // Now an optimisation: group the new hashes by the files they come
 172                         // from. This enables each file to only be parsed once and return
 173                         // all the required events from it.
 174                         QHash<QString, QList<iHashable::Hash> > newHashesByPath;
 175                         foreach(iHashable::Hash newHash, newHashes)
 176                         {
 177                                 const QString &currentPath(pathsByHashes.value(newHash).first);
 178                                 if(newHashesByPath.contains(currentPath))
 179                                         newHashesByPath[currentPath].append(newHash);
 180                                 else
 181                                         newHashesByPath[currentPath] = QList<iHashable::Hash>() << newHash;
 182                         }
 183
 184                         qDebug() << "Scanning addressbook";
 185
 186                         // Prepare the telephone-address book ID lookup.
 187                         NumberToNameLookup lookup;
 188
 189                         qDebug() << "Importing new events";
 190
 191                         // Re-parse the new events and insert them
 192                         allBackends.PreInsert();
 193                         {
 194                                 int idx = 0;
 195                                 foreach(QString filename, newHashesByPath.keys())
 196                                 {
 197                                         QList<uint> recordsToReturn;
 198                                         foreach(iHashable::Hash newHash, newHashesByPath.value(filename))
 199                                                 recordsToReturn.append(pathsByHashes.value(newHash).second);
 200
 201                                         // Repeating an action that caused an exception last time
 202                                         // shouldn't happen again, but just in case...
 203                                         try
 204                                         {
 205                                                 foreach(EventTypes::EventFromFile newEventFromFile, ProcessFile(filename, recordsToReturn))
 206                                                 {
 207                                                         // ...and insert it into the DB
 208                                                         try
 209                                                         {
 210                                                                 allBackends.Insert(*newEventFromFile.first, lookup);
 211                                                         }
 212                                                         catch(const std::runtime_error &exception)
 213                                                         {
 214                                                                 qDebug() << "Unable to insert event: " << exception.what();
 215                                                         }
 216
 217                                                         emit EventProcessed(++idx, newHashes.count());
 218                                                 }
 219                                         }
 220                                         catch(const std::runtime_error &exception)
 221                                         {
 222                                                 qDebug() << exception.what() << endl;
 223                                         }
 224
 225                                         // Just to make sure the listeners are synced even if the
 226                                         // earlier call is skipped due to errors...
 227                                         emit EventProcessed(idx, newHashes.count());
 228                                 }
 229                         }
 230                         allBackends.PostInsert(); // Perform any post-insert cleanup (i.e. reindexing)
 231
 232                         // Need to find a better way of refreshing the conversations view...
 233                         QProcess::execute("pkill rtcom");
 234
 235                         // Signal we completed successfully.
 236                         backupManager.UnlockCurrentBackup();
 237                 }
 238
 239                 // May as well call this explicitly despite it being called by the
 240                 // destructor - it's harmless.
 241                 preventEvents.RestoreAccounts();
 242         }
 243         catch(std::runtime_error exception)
 244         {
 245                 qDebug() << exception.what();
 246         }
 247 }
 248
 249 EventTypes::EventFromFileList SyncerThread::ProcessFile(const QString &path, const QList<uint> &recordsToReturn) const
 250 {
 251         qDebug() << path << endl;
 252         QFile eventFile(path);
 253
 254         // If the file's ok, process it...
 255         if (eventFile.open(QFile::ReadOnly))
 256         {
 257                 // Identify type of file...
 258                 EventParsers::iEventParser * parser(EventParsers::Factory::CreateParser(CurrentSettings(), path));
 259
 260                 // ...and grab the events from it (if it's a supported format)
 261                 if(NULL != parser)
 262                         return parser->ParseFile(eventFile, recordsToReturn);
 263                 else
 264                         return EventTypes::EventFromFileList();
 265         }
 266         else
 267                 throw std::runtime_error(QString("Unable to open: %1").arg(path).toStdString());
 268 }
 269
 270 QFileInfoList FindEvents(QFileInfoList currentCandidate)
 271 {
 272         QFileInfoList foundEvents;
 273         foreach(QFileInfo fileInfo, currentCandidate)
 274         {
 275                 if(fileInfo.isDir())
 276                         foundEvents.append(FindEvents(QDir(fileInfo.absoluteFilePath()).entryInfoList(QDir::AllEntries | QDir::NoDotAndDotDot)));
 277                 else
 278                         foundEvents.append(fileInfo);
 279         }
 280
 281         return foundEvents;
 282 }
 283
 284 QDebug operator<<(QDebug dbg, QList<Attachment*> &attachments)
 285 {
 286         dbg.nospace() << "Attachments" << "\n";
 287
 288         foreach(Attachment* attachment, attachments)
 289                 dbg.nospace() << *attachment << "\n";
 290
 291         dbg.nospace() << "\n";
 292
 293         return dbg;
 294 }