From c612397b9d42504a7b2e691107b8348606e95767 Mon Sep 17 00:00:00 2001 From: Leonardo Zide Date: Sun, 26 Nov 2017 19:21:54 -0800 Subject: [PATCH] Cache part descriptions when using loose files. --- common/lc_file.h | 17 ++ common/lc_library.cpp | 451 ++++++++++++++++++++++++------------------ common/lc_library.h | 16 +- common/pieceinf.h | 3 +- 4 files changed, 295 insertions(+), 192 deletions(-) diff --git a/common/lc_file.h b/common/lc_file.h index 62955878..97f52a17 100644 --- a/common/lc_file.h +++ b/common/lc_file.h @@ -160,6 +160,16 @@ public: return Read64(Buffer, Count); } + QString ReadQString() + { + uint32_t Size = ReadU32(); + char* Buffer = new char[Size]; + ReadBuffer(Buffer, Size); + QString String = QString::fromUtf8(Buffer, Size); + delete[] Buffer; + return String; + } + void WriteU8(const lcuint8& Value) { Write8(&Value, 1); @@ -265,6 +275,13 @@ public: WriteFloats(Vector, 3); } + void WriteQString(const QString& String) + { + QByteArray Data = String.toUtf8(); + WriteU32(Data.size()); + WriteBuffer(Data, Data.size()); + } + protected: size_t Read8(void* Buffer, size_t Count) { diff --git a/common/lc_library.cpp b/common/lc_library.cpp index 46c57239..d6dd0cf5 100644 --- a/common/lc_library.cpp +++ b/common/lc_library.cpp @@ -464,199 +464,30 @@ void lcPiecesLibrary::ReadArchiveDescriptions(const QString& OfficialFileName, c } } - SaveCacheIndex(IndexFileName); + SaveArchiveCacheIndex(IndexFileName); } } bool lcPiecesLibrary::OpenDirectory(const QDir& LibraryDir, bool ShowProgress) { - lcDiskFile PartsList(LibraryDir.absoluteFilePath(QLatin1String("parts.lst"))); - - if (PartsList.Open(QIODevice::ReadOnly)) - { - char Line[1024]; - - while (PartsList.ReadLine(Line, sizeof(Line))) - { - char OriginalLine[1024]; - strcpy(OriginalLine, Line); - - char* Chr = Line; - char* Ext = nullptr; - - while (*Chr) - { - if (*Chr >= 'a' && *Chr <= 'z') - *Chr = *Chr + 'A' - 'a'; - else if (*Chr == '.') - Ext = Chr; - else if (isspace(*Chr)) - { - *Chr++ = 0; - break; - } - - Chr++; - } - - if (Ext && !strcmp(Ext, ".DAT")) - OriginalLine[Ext - Line + 4] = 0; - else - continue; - - while (*Chr && isspace(*Chr)) - Chr++; - - char* Description = Chr; - - while (*Chr) - { - if (*Chr == '\r' || *Chr == '\n') - { - *Chr = 0; - break; - } - - Chr++; - } - - if (!*Line || !*Description) - continue; - - PieceInfo* Info = new PieceInfo(); - - strncpy(Info->mFileName, OriginalLine, sizeof(Info->mFileName)); - Info->mFileName[sizeof(Info->mFileName) - 1] = 0; - - strncpy(Info->m_strDescription, Description, sizeof(Info->m_strDescription)); - Info->m_strDescription[sizeof(Info->m_strDescription) - 1] = 0; - - mPieces[Line] = Info; - } - } - - const QLatin1String BaseFolders[] = { QLatin1String("unofficial/"), QLatin1String("") }; + const QLatin1String BaseFolders[LC_NUM_FOLDERTYPES] = { QLatin1String("unofficial/"), QLatin1String("") }; const int NumBaseFolders = sizeof(BaseFolders) / sizeof(BaseFolders[0]); - if (mPieces.empty()) + QFileInfoList FileLists[NumBaseFolders]; + + for (unsigned int BaseFolderIdx = 0; BaseFolderIdx < NumBaseFolders; BaseFolderIdx++) { - QDir BaseDirs[NumBaseFolders]; - - for (unsigned int BaseFolderIdx = 0; BaseFolderIdx < NumBaseFolders; BaseFolderIdx++) - { - BaseDirs[BaseFolderIdx] = QDir(QDir(LibraryDir.absoluteFilePath(BaseFolders[BaseFolderIdx])).absoluteFilePath(QLatin1String("parts/")), QLatin1String("*.dat"), QDir::SortFlags(QDir::Name | QDir::IgnoreCase), QDir::Files | QDir::Hidden | QDir::Readable); - QDir& Dir = BaseDirs[BaseFolderIdx]; - QStringList FileList = Dir.entryList(); - - for (int FileIdx = 0; FileIdx < FileList.size(); FileIdx++) - { - char Name[LC_PIECE_NAME_LEN]; - QByteArray FileString = FileList[FileIdx].toLatin1(); - const char* Src = FileString; - char* Dst = Name; - - while (*Src && Dst - Name < (int)sizeof(Name)) - { - if (*Src >= 'a' && *Src <= 'z') - *Dst = *Src + 'A' - 'a'; - else if (*Src == '\\') - *Dst = '/'; - else - *Dst = *Src; - - Src++; - Dst++; - } - *Dst = 0; - - if (Dst - Name <= 4) - continue; - - Dst -= 4; - if (memcmp(Dst, ".DAT", 4)) - continue; - - if (mHasUnofficial && mPieces.find(Name) != mPieces.end()) - continue; - - PieceInfo* Info = new PieceInfo(); - - if (BaseFolderIdx == 0) - mHasUnofficial = true; - - strncpy(Info->mFileName, FileString, sizeof(Info->mFileName)); - Info->mFileName[sizeof(Info->mFileName) - 1] = 0; - Info->mUnofficial = (BaseFolderIdx == 0); - - mPieces[Name] = Info; - } - } - - if (!mPieces.empty()) - { - QAtomicInt FilesLoaded; - - auto ReadDescriptions = [&BaseDirs, &FilesLoaded](const std::pair& Entry) - { - PieceInfo* Info = Entry.second; - FilesLoaded.ref(); - - QDir& Dir = BaseDirs[Info->mUnofficial ? 0 : 1]; - lcDiskFile PieceFile(Dir.absoluteFilePath(Info->mFileName)); - char Line[1024]; - - if (!PieceFile.Open(QIODevice::ReadOnly) || !PieceFile.ReadLine(Line, sizeof(Line))) - { - strcpy(Info->m_strDescription, "Unknown"); - return; - } - - const char* Src = Line + 2; - char* Dst = Info->m_strDescription; - - for (;;) - { - if (*Src != '\r' && *Src != '\n' && *Src && Dst - Info->m_strDescription < (int)sizeof(Info->m_strDescription) - 1) - { - *Dst++ = *Src++; - continue; - } - - *Dst = 0; - break; - } - }; - - QProgressDialog* ProgressDialog = new QProgressDialog(nullptr); - ProgressDialog->setWindowFlags(ProgressDialog->windowFlags() & ~Qt::WindowCloseButtonHint); - ProgressDialog->setWindowTitle(tr("Initializing")); - ProgressDialog->setLabelText(tr("Loading Parts Library")); - ProgressDialog->setMaximum(mPieces.size()); - ProgressDialog->setMinimum(0); - ProgressDialog->setValue(0); - ProgressDialog->setCancelButton(nullptr); - ProgressDialog->setAutoReset(false); - if (ShowProgress) - ProgressDialog->show(); - - QFuture LoadFuture = QtConcurrent::map(mPieces, ReadDescriptions); - - while (!LoadFuture.isFinished()) - { - ProgressDialog->setValue(FilesLoaded); - QApplication::processEvents(QEventLoop::ExcludeUserInputEvents); - } - - ProgressDialog->setValue(FilesLoaded); - QApplication::processEvents(); - - ProgressDialog->deleteLater(); - } + QString ParstPath = QDir(LibraryDir.absoluteFilePath(BaseFolders[BaseFolderIdx])).absoluteFilePath(QLatin1String("parts/")); + QDir Dir = QDir(ParstPath, QLatin1String("*.dat"), QDir::SortFlags(QDir::Name | QDir::IgnoreCase), QDir::Files | QDir::Hidden | QDir::Readable); + FileLists[BaseFolderIdx] = Dir.entryInfoList(); } - if (mPieces.empty()) + if (FileLists[LC_FOLDER_OFFICIAL].isEmpty()) return false; + mHasUnofficial = !FileLists[LC_FOLDER_UNOFFICIAL].isEmpty(); + ReadDirectoryDescriptions(FileLists, ShowProgress); + for (unsigned int BaseFolderIdx = 0; BaseFolderIdx < sizeof(BaseFolders) / sizeof(BaseFolders[0]); BaseFolderIdx++) { const char* PrimitiveDirectories[] = { "p/", "p/48/", "parts/s/" }; @@ -754,7 +585,195 @@ bool lcPiecesLibrary::OpenDirectory(const QDir& LibraryDir, bool ShowProgress) return true; } -bool lcPiecesLibrary::ReadCacheFile(const QString& FileName, lcMemFile& CacheFile) +void lcPiecesLibrary::ReadDirectoryDescriptions(const QFileInfoList (&FileLists)[LC_NUM_FOLDERTYPES], bool ShowProgress) +{ + QString IndexFileName = QFileInfo(QDir(mCachePath), QLatin1String("index")).absoluteFilePath(); + lcMemFile IndexFile; + std::vector CachedDescriptions; + + if (ReadDirectoryCacheFile(IndexFileName, IndexFile)) + { + QString LibraryPath = IndexFile.ReadQString(); + + if (LibraryPath == mLibraryDir.absolutePath()) + { + int NumDescriptions = IndexFile.ReadU32(); + CachedDescriptions.reserve(NumDescriptions); + + while (NumDescriptions--) + { + const char* FileName = (const char*)IndexFile.mBuffer + IndexFile.GetPosition(); + CachedDescriptions.push_back(FileName); + IndexFile.Seek(strlen(FileName) + 1, SEEK_CUR); + const char* Description = (const char*)IndexFile.mBuffer + IndexFile.GetPosition(); + IndexFile.Seek(strlen(Description) + 1, SEEK_CUR); + IndexFile.Seek(4 + 1 + 8, SEEK_CUR); + } + } + } + + for (int FolderIdx = 0; FolderIdx < LC_NUM_FOLDERTYPES; FolderIdx++) + { + const QFileInfoList& FileList = FileLists[FolderIdx]; + + for (int FileIdx = 0; FileIdx < FileList.size(); FileIdx++) + { + char Name[LC_PIECE_NAME_LEN]; + QByteArray FileString = FileList[FileIdx].fileName().toLatin1(); + const char* Src = FileString; + char* Dst = Name; + + while (*Src && Dst - Name < (int)sizeof(Name)) + { + if (*Src >= 'a' && *Src <= 'z') + *Dst = *Src + 'A' - 'a'; + else if (*Src == '\\') + *Dst = '/'; + else + *Dst = *Src; + + Src++; + Dst++; + } + *Dst = 0; + + if (FolderIdx == LC_FOLDER_OFFICIAL && mHasUnofficial && mPieces.find(Name) != mPieces.end()) + continue; + + PieceInfo* Info = new PieceInfo(); + + strncpy(Info->mFileName, FileString, sizeof(Info->mFileName)); + Info->mFileName[sizeof(Info->mFileName) - 1] = 0; + Info->mFolderType = FolderIdx; + Info->mFolderIndex = FileIdx; + + mPieces[Name] = Info; + } + } + + QAtomicInt FilesLoaded; + bool Modified = false; + + auto ReadDescriptions = [&FileLists, &CachedDescriptions, &FilesLoaded, &Modified](const std::pair& Entry) + { + PieceInfo* Info = Entry.second; + FilesLoaded.ref(); + + lcDiskFile PieceFile(FileLists[Info->mFolderType][Info->mFolderIndex].absoluteFilePath()); + char Line[1024]; + + if (!CachedDescriptions.empty()) + { + auto DescriptionCompare = [](const void* Key, const void* Element) + { + return strcmp((const char*)Key, *(const char**)Element); + }; + + void* CachedDescription = bsearch(Info->mFileName, &CachedDescriptions.front(), CachedDescriptions.size(), sizeof(char*), DescriptionCompare); + + if (CachedDescription) + { + const char* FileName = *(const char**)CachedDescription; + const char* Description = FileName + strlen(FileName) + 1; + strcpy(Info->m_strDescription, Description); + return; + } + } + + if (!PieceFile.Open(QIODevice::ReadOnly) || !PieceFile.ReadLine(Line, sizeof(Line))) + { + strcpy(Info->m_strDescription, "Unknown"); + return; + } + + const char* Src = Line + 2; + char* Dst = Info->m_strDescription; + + for (;;) + { + if (*Src != '\r' && *Src != '\n' && *Src && Dst - Info->m_strDescription < (int)sizeof(Info->m_strDescription) - 1) + { + *Dst++ = *Src++; + continue; + } + + *Dst = 0; + break; + } + + Modified = true; + }; + + QProgressDialog* ProgressDialog = new QProgressDialog(nullptr); + ProgressDialog->setWindowFlags(ProgressDialog->windowFlags() & ~Qt::WindowCloseButtonHint); + ProgressDialog->setWindowTitle(tr("Initializing")); + ProgressDialog->setLabelText(tr("Loading Parts Library")); + ProgressDialog->setMaximum(mPieces.size()); + ProgressDialog->setMinimum(0); + ProgressDialog->setValue(0); + ProgressDialog->setCancelButton(nullptr); + ProgressDialog->setAutoReset(false); + if (ShowProgress) + ProgressDialog->show(); + + QFuture LoadFuture = QtConcurrent::map(mPieces, ReadDescriptions); + + while (!LoadFuture.isFinished()) + { + ProgressDialog->setValue(FilesLoaded); + QApplication::processEvents(QEventLoop::ExcludeUserInputEvents); + } + + ProgressDialog->setValue(FilesLoaded); + QApplication::processEvents(QEventLoop::ExcludeUserInputEvents); + + ProgressDialog->deleteLater(); + + if (Modified) + { + lcMemFile IndexFile; + + IndexFile.WriteQString(mLibraryDir.absolutePath()); + + IndexFile.WriteU32(mPieces.size()); + + std::vector SortedPieces; + SortedPieces.reserve(mPieces.size()); + for (const auto PieceIt : mPieces) + SortedPieces.push_back(PieceIt.second); + + auto PieceInfoCompare = [](PieceInfo* Info1, PieceInfo* Info2) + { + return strcmp(Info1->mFileName, Info2->mFileName) < 0; + }; + + std::sort(SortedPieces.begin(), SortedPieces.end(), PieceInfoCompare); + + for (const PieceInfo* Info : SortedPieces) + { + if (IndexFile.WriteBuffer(Info->mFileName, strlen(Info->mFileName) + 1) == 0) + return; + + if (IndexFile.WriteBuffer(Info->m_strDescription, strlen(Info->m_strDescription) + 1) == 0) + return; + + IndexFile.WriteU32(Info->mFlags); + IndexFile.WriteU8(Info->mFolderType); + +#if (QT_VERSION >= QT_VERSION_CHECK(4, 7, 0)) + uint64_t FileTime = FileLists[Info->mFolderType][Info->mFolderIndex].lastModified().toMSecsSinceEpoch(); +#else + uint64_t FileTime = FileLists[Info->mFolderType][Info->mFolderIndex].lastModified().toTime_t(); +#endif + + IndexFile.WriteU64(FileTime); + } + + WriteDirectoryCacheFile(IndexFileName, IndexFile); + } +} + +bool lcPiecesLibrary::ReadArchiveCacheFile(const QString& FileName, lcMemFile& CacheFile) { QFile File(FileName); @@ -842,7 +861,7 @@ bool lcPiecesLibrary::ReadCacheFile(const QString& FileName, lcMemFile& CacheFil return ret == Z_STREAM_END; } -bool lcPiecesLibrary::WriteCacheFile(const QString& FileName, lcMemFile& CacheFile) +bool lcPiecesLibrary::WriteArchiveCacheFile(const QString& FileName, lcMemFile& CacheFile) { QFile File(FileName); @@ -907,11 +926,67 @@ bool lcPiecesLibrary::WriteCacheFile(const QString& FileName, lcMemFile& CacheFi return true; } +bool lcPiecesLibrary::ReadDirectoryCacheFile(const QString& FileName, lcMemFile& CacheFile) +{ + QFile File(FileName); + + if (!File.open(QIODevice::ReadOnly)) + return false; + + quint32 CacheVersion, CacheFlags; + + if (File.read((char*)&CacheVersion, sizeof(CacheVersion)) == -1 || CacheVersion != LC_LIBRARY_CACHE_VERSION) + return false; + + if (File.read((char*)&CacheFlags, sizeof(CacheFlags)) == -1 || CacheFlags != LC_LIBRARY_CACHE_DIRECTORY) + return false; + + quint32 UncompressedSize; + + if (File.read((char*)&UncompressedSize, sizeof(UncompressedSize)) == -1) + return false; + + QByteArray Data = qUncompress(File.readAll()); + if (Data.isEmpty()) + return false; + + CacheFile.SetLength(Data.size()); + CacheFile.Seek(0, SEEK_SET); + CacheFile.WriteBuffer(Data.constData(), Data.size()); + CacheFile.Seek(0, SEEK_SET); + + return true; +} + +bool lcPiecesLibrary::WriteDirectoryCacheFile(const QString& FileName, lcMemFile& CacheFile) +{ + QFile File(FileName); + + if (!File.open(QIODevice::WriteOnly)) + return false; + + quint32 CacheVersion = LC_LIBRARY_CACHE_VERSION; + if (File.write((char*)&CacheVersion, sizeof(CacheVersion)) == -1) + return false; + + quint32 CacheFlags = LC_LIBRARY_CACHE_DIRECTORY; + if (File.write((char*)&CacheFlags, sizeof(CacheFlags)) == -1) + return false; + + quint32 UncompressedSize = (quint32)CacheFile.GetLength(); + if (File.write((char*)&UncompressedSize, sizeof(UncompressedSize)) == -1) + return false; + + File.write(qCompress(CacheFile.mBuffer, CacheFile.GetLength())); + + return true; +} + bool lcPiecesLibrary::LoadCacheIndex(const QString& FileName) { lcMemFile IndexFile; - if (!ReadCacheFile(FileName, IndexFile)) + if (!ReadArchiveCacheFile(FileName, IndexFile)) return false; quint32 NumFiles; @@ -936,7 +1011,7 @@ bool lcPiecesLibrary::LoadCacheIndex(const QString& FileName) return true; } -bool lcPiecesLibrary::SaveCacheIndex(const QString& FileName) +bool lcPiecesLibrary::SaveArchiveCacheIndex(const QString& FileName) { lcMemFile IndexFile; @@ -957,7 +1032,7 @@ bool lcPiecesLibrary::SaveCacheIndex(const QString& FileName) return false; } - return WriteCacheFile(FileName, IndexFile); + return WriteArchiveCacheFile(FileName, IndexFile); } bool lcPiecesLibrary::LoadCachePiece(PieceInfo* Info) @@ -965,7 +1040,7 @@ bool lcPiecesLibrary::LoadCachePiece(PieceInfo* Info) QString FileName = QFileInfo(QDir(mCachePath), QString::fromLatin1(Info->mFileName)).absoluteFilePath(); lcMemFile MeshData; - if (!ReadCacheFile(FileName, MeshData)) + if (!ReadArchiveCacheFile(FileName, MeshData)) return false; quint32 Flags; @@ -1000,7 +1075,7 @@ bool lcPiecesLibrary::SaveCachePiece(PieceInfo* Info) QString FileName = QFileInfo(QDir(mCachePath), QString::fromLatin1(Info->mFileName)).absoluteFilePath(); - return WriteCacheFile(FileName, MeshData); + return WriteArchiveCacheFile(FileName, MeshData); } static void lcLoadPieceFuture(lcPiecesLibrary* Library) diff --git a/common/lc_library.h b/common/lc_library.h index e800d9af..fb64eee1 100644 --- a/common/lc_library.h +++ b/common/lc_library.h @@ -15,6 +15,13 @@ enum lcZipFileType LC_NUM_ZIPFILES }; +enum lcLibraryFolderType +{ + LC_FOLDER_UNOFFICIAL, + LC_FOLDER_OFFICIAL, + LC_NUM_FOLDERTYPES +}; + struct lcLibraryMeshVertex { lcVector3 Position; @@ -216,13 +223,16 @@ protected: bool OpenArchive(lcFile* File, const QString& FileName, lcZipFileType ZipFileType); bool OpenDirectory(const QDir& LibraryDir, bool ShowProgress); void ReadArchiveDescriptions(const QString& OfficialFileName, const QString& UnofficialFileName); + void ReadDirectoryDescriptions(const QFileInfoList (&FileLists)[LC_NUM_FOLDERTYPES], bool ShowProgress); - bool ReadCacheFile(const QString& FileName, lcMemFile& CacheFile); - bool WriteCacheFile(const QString& FileName, lcMemFile& CacheFile); + bool ReadArchiveCacheFile(const QString& FileName, lcMemFile& CacheFile); + bool WriteArchiveCacheFile(const QString& FileName, lcMemFile& CacheFile); bool LoadCacheIndex(const QString& FileName); - bool SaveCacheIndex(const QString& FileName); + bool SaveArchiveCacheIndex(const QString& FileName); bool LoadCachePiece(PieceInfo* Info); bool SaveCachePiece(PieceInfo* Info); + bool ReadDirectoryCacheFile(const QString& FileName, lcMemFile& CacheFile); + bool WriteDirectoryCacheFile(const QString& FileName, lcMemFile& CacheFile); lcLibraryPrimitive* FindPrimitive(const char* Name) const { diff --git a/common/pieceinf.h b/common/pieceinf.h index 41c2c715..3dfd8911 100644 --- a/common/pieceinf.h +++ b/common/pieceinf.h @@ -155,7 +155,8 @@ public: int mZipFileIndex; lcuint32 mFlags; lcPieceInfoState mState; - bool mUnofficial; + int mFolderType; + int mFolderIndex; protected: void ReleaseMesh();