merge hash index and overflow file into data.kz

Author: ray6080

src/include/storage/db_file_id.h

@@ -67,7 +67,7 @@ class OnDiskHashIndex {
 template<typename T>
 class HashIndex final : public OnDiskHashIndex {
 public:
-    HashIndex(MemoryManager& memoryManager, DBFileIDAndName dbFileIDAndName, FileHandle* fileHandle,
+    HashIndex(MemoryManager& memoryManager, FileHandle* fileHandle,
         OverflowFileHandle* overflowFileHandle, DiskArrayCollection& diskArrays, uint64_t indexPos,
         ShadowFile* shadowFile, const HashIndexHeader& indexHeaderForReadTrx,
         HashIndexHeader& indexHeaderForWriteTrx);
@@ -278,7 +278,6 @@ class HashIndex final : public OnDiskHashIndex {
         const transaction::Transaction* transaction, slot_id_t pSlotId);
 
 private:
-    DBFileIDAndName dbFileIDAndName;
     ShadowFile* shadowFile;
     uint64_t headerPageIdx;
     FileHandle* fileHandle;
@@ -307,9 +306,9 @@ inline bool HashIndex<common::ku_string_t>::equals(const transaction::Transactio
 
 class PrimaryKeyIndex {
 public:
-    PrimaryKeyIndex(const DBFileIDAndName& dbFileIDAndName, bool readOnly, bool inMemMode,
-        common::PhysicalTypeID keyDataType, MemoryManager& memoryManager, ShadowFile* shadowFile,
-        common::VirtualFileSystem* vfs, main::ClientContext* context);
+    PrimaryKeyIndex(FileHandle* dataFH, bool inMemMode, common::PhysicalTypeID keyDataType,
+        MemoryManager& memoryManager, ShadowFile* shadowFile, common::page_idx_t firstHeaderPage,
+        common::page_idx_t overflowHeaderPage);
 
     ~PrimaryKeyIndex();
 
@@ -392,17 +391,20 @@ class PrimaryKeyIndex {
 
     void writeHeaders();
 
+    void serialize(common::Serializer& serializer) const;
+
 private:
     common::PhysicalTypeID keyDataTypeID;
     FileHandle* fileHandle;
     std::unique_ptr<OverflowFile> overflowFile;
     std::vector<std::unique_ptr<OnDiskHashIndex>> hashIndices;
     std::vector<HashIndexHeader> hashIndexHeadersForReadTrx;
     std::vector<HashIndexHeader> hashIndexHeadersForWriteTrx;
-    DBFileIDAndName dbFileIDAndName;
     ShadowFile& shadowFile;
     // Stores both primary and overflow slots
     std::unique_ptr<DiskArrayCollection> hashIndexDiskArrays;
+    common::page_idx_t firstHeaderPage;
+    common::page_idx_t overflowHeaderPage;
 };
 
 } // namespace storage

src/include/storage/index/hash_index.h

@@ -36,7 +36,7 @@ class ShadowUtils {
     // Where possible, updatePage/insertNewPage should be used instead
     static ShadowPageAndFrame createShadowVersionIfNecessaryAndPinPage(
         common::page_idx_t originalPage, bool insertingNewPage, FileHandle& fileHandle,
-        DBFileID dbFileID, ShadowFile& shadowFile);
+        ShadowFile& shadowFile);
 
     static std::pair<FileHandle*, common::page_idx_t> getFileHandleAndPhysicalPageIdxToPin(
         FileHandle& fileHandle, common::page_idx_t pageIdx, const ShadowFile& shadowFile,
@@ -47,16 +47,16 @@ class ShadowUtils {
         const std::function<void(uint8_t*)>& readOp);
 
     static common::page_idx_t insertNewPage(
-        FileHandle& fileHandle, DBFileID dbFileID, ShadowFile& shadowFile,
+        FileHandle& fileHandle, ShadowFile& shadowFile,
         const std::function<void(uint8_t*)>& insertOp = [](uint8_t*) -> void {
             // DO NOTHING.
         });
 
     // Note: This function updates a page "transactionally", i.e., creates the WAL version of the
     // page if it doesn't exist. For the original page to be updated, the current WRITE trx needs to
     // commit and checkpoint.
-    static void updatePage(FileHandle& fileHandle, DBFileID dbFileID,
-        common::page_idx_t originalPageIdx, bool isInsertingNewPage, ShadowFile& shadowFile,
+    static void updatePage(FileHandle& fileHandle, common::page_idx_t originalPageIdx,
+        bool isInsertingNewPage, ShadowFile& shadowFile,
         const std::function<void(uint8_t*)>& updateOp);
 };
 } // namespace storage

src/include/storage/shadow_utils.h

@@ -29,11 +29,11 @@ class KUZU_API StorageManager {
 
     static void recover(main::ClientContext& clientContext);
 
-    void createTable(catalog::CatalogEntry* entry, main::ClientContext* context);
+    void createTable(catalog::CatalogEntry* entry, const main::ClientContext* context);
 
     void checkpoint(main::ClientContext& clientContext);
     void finalizeCheckpoint(main::ClientContext& clientContext);
-    void rollbackCheckpoint(main::ClientContext& clientContext);
+    void rollbackCheckpoint(const main::ClientContext& clientContext);
 
     Table* getTable(common::table_id_t tableID) {
         std::lock_guard lck{mtx};
@@ -54,9 +54,10 @@ class KUZU_API StorageManager {
 
     void loadTables(const catalog::Catalog& catalog, common::VirtualFileSystem* vfs,
         main::ClientContext* context);
-    void createNodeTable(catalog::NodeTableCatalogEntry* entry, main::ClientContext* context);
+    void createNodeTable(catalog::NodeTableCatalogEntry* entry);
     void createRelTable(catalog::RelTableCatalogEntry* entry);
-    void createRelTableGroup(catalog::RelGroupCatalogEntry* entry, main::ClientContext* context);
+    void createRelTableGroup(const catalog::RelGroupCatalogEntry* entry,
+        const main::ClientContext* context);
 
     void reclaimDroppedTables(const main::ClientContext& clientContext);
 

src/include/storage/storage_manager.h

@@ -101,9 +101,9 @@ struct PIPUpdates {
 class DiskArrayInternal {
 public:
     // Used when loading from file
-    DiskArrayInternal(FileHandle& fileHandle, DBFileID dbFileID,
-        const DiskArrayHeader& headerForReadTrx, DiskArrayHeader& headerForWriteTrx,
-        ShadowFile* shadowFile, uint64_t elementSize, bool bypassShadowing = false);
+    DiskArrayInternal(FileHandle& fileHandle, const DiskArrayHeader& headerForReadTrx,
+        DiskArrayHeader& headerForWriteTrx, ShadowFile* shadowFile, uint64_t elementSize,
+        bool bypassShadowing = false);
 
     virtual ~DiskArrayInternal() = default;
 
@@ -241,7 +241,6 @@ class DiskArrayInternal {
 protected:
     PageStorageInfo storageInfo;
     FileHandle& fileHandle;
-    DBFileID dbFileID;
     const DiskArrayHeader& header;
     DiskArrayHeader& headerForWriteTrx;
     bool hasTransactionalUpdates;
@@ -267,10 +266,10 @@ class DiskArray {
     // If bypassWAL is set, the buffer manager is used to pages new to this transaction to the
     // original file, but does not handle flushing them. BufferManager::flushAllDirtyPagesInFrames
     // should be called on this file handle exactly once during prepare commit.
-    DiskArray(FileHandle& fileHandle, DBFileID dbFileID, const DiskArrayHeader& headerForReadTrx,
+    DiskArray(FileHandle& fileHandle, const DiskArrayHeader& headerForReadTrx,
         DiskArrayHeader& headerForWriteTrx, ShadowFile* shadowFile, bool bypassWAL = false)
-        : diskArray(fileHandle, dbFileID, headerForReadTrx, headerForWriteTrx, shadowFile,
-              sizeof(U), bypassWAL) {}
+        : diskArray(fileHandle, headerForReadTrx, headerForWriteTrx, shadowFile, sizeof(U),
+              bypassWAL) {}
 
     // Note: This function is to be used only by the WRITE trx.
     // The return value is the idx of val in array.

src/include/storage/storage_structure/disk_array.h

@@ -27,7 +27,7 @@ class DiskArrayCollection {
     static_assert(std::has_unique_object_representations_v<HeaderPage>);
 
 public:
-    DiskArrayCollection(FileHandle& fileHandle, DBFileID dbFileID, ShadowFile& shadowFile,
+    DiskArrayCollection(FileHandle& fileHandle, ShadowFile& shadowFile,
         common::page_idx_t firstHeaderPage = 0, bool bypassShadowing = false);
 
     void checkpoint();
@@ -52,15 +52,14 @@ class DiskArrayCollection {
                                ->headers[idx % HeaderPage::NUM_HEADERS_PER_PAGE];
         auto& writeHeader = headersForWriteTrx[idx / HeaderPage::NUM_HEADERS_PER_PAGE]
                                 ->headers[idx % HeaderPage::NUM_HEADERS_PER_PAGE];
-        return std::make_unique<DiskArray<T>>(fileHandle, dbFileID, readHeader, writeHeader,
-            &shadowFile, bypassShadowing);
+        return std::make_unique<DiskArray<T>>(fileHandle, readHeader, writeHeader, &shadowFile,
+            bypassShadowing);
     }
 
     size_t addDiskArray();
 
 private:
     FileHandle& fileHandle;
-    DBFileID dbFileID;
     ShadowFile& shadowFile;
     bool bypassShadowing;
     common::page_idx_t headerPagesOnDisk;

src/include/storage/storage_structure/disk_array_collection.h

@@ -12,7 +12,6 @@
 #include "storage/storage_utils.h"
 #include "storage/wal/shadow_file.h"
 #include "storage/wal/wal.h"
-#include "transaction/transaction.h"
 
 namespace kuzu {
 namespace storage {
@@ -28,7 +27,8 @@ class OverflowFileHandle {
     // Moving the handle would invalidate those pointers
     OverflowFileHandle(OverflowFileHandle&& other) = delete;
 
-    std::string readString(transaction::TransactionType trxType, const common::ku_string_t& str);
+    std::string readString(transaction::TransactionType trxType,
+        const common::ku_string_t& str) const;
 
     bool equals(transaction::TransactionType trxType, std::string_view keyToLookup,
         const common::ku_string_t& keyInEntry) const;
@@ -81,16 +81,11 @@ class OverflowFile {
 
 public:
     // For reading an existing overflow file
-    OverflowFile(const DBFileIDAndName& dbFileIdAndName, MemoryManager& memoryManager,
-        ShadowFile* shadowFile, bool readOnly, common::VirtualFileSystem* vfs,
-        main::ClientContext* context);
+    OverflowFile(FileHandle* dataFH, MemoryManager& memoryManager, ShadowFile* shadowFile,
+        common::page_idx_t headerPageIdx);
 
     virtual ~OverflowFile() = default;
 
-    // For creating an overflow file from scratch
-    static void createEmptyFiles(const std::string& fName, common::VirtualFileSystem* vfs,
-        main::ClientContext* context);
-
     // Handles contain a reference to the overflow file
     OverflowFile(OverflowFile&& other) = delete;
 
@@ -111,13 +106,16 @@ class OverflowFile {
     }
 
 protected:
-    explicit OverflowFile(const DBFileIDAndName& dbFileIdAndName, MemoryManager& memoryManager);
+    explicit OverflowFile(FileHandle* dataFH, MemoryManager& memoryManager);
 
     common::page_idx_t getNewPageIdx() {
         // If this isn't the first call reserving the page header, then the header flag must be set
         // prior to this
-        KU_ASSERT(pageCounter == HEADER_PAGE_IDX || headerChanged);
-        return pageCounter.fetch_add(1);
+        if (fileHandle) {
+            return fileHandle->addNewPage();
+        } else {
+            return pageCounter.fetch_add(1);
+        }
     }
 
 private:
@@ -133,18 +131,18 @@ class OverflowFile {
     std::vector<std::unique_ptr<OverflowFileHandle>> handles;
     StringOverflowFileHeader header;
     common::page_idx_t numPagesOnDisk;
-    DBFileID dbFileID;
     FileHandle* fileHandle;
     ShadowFile* shadowFile;
     MemoryManager& memoryManager;
     std::atomic<common::page_idx_t> pageCounter;
     std::atomic<bool> headerChanged;
+    common::page_idx_t headerPageIdx;
 };
 
 class InMemOverflowFile final : public OverflowFile {
 public:
-    explicit InMemOverflowFile(const DBFileIDAndName& dbFileIDAndName, MemoryManager& memoryManager)
-        : OverflowFile{dbFileIDAndName, memoryManager} {}
+    explicit InMemOverflowFile(MemoryManager& memoryManager)
+        : OverflowFile{nullptr, memoryManager} {}
 };
 
 } // namespace storage

src/include/storage/storage_structure/overflow_file.h

@@ -11,21 +11,10 @@
 #include "main/client_context.h"
 #include "main/db_config.h"
 #include "main/settings.h"
-#include "storage/db_file_id.h"
 
 namespace kuzu {
 namespace storage {
 
-class StorageManager;
-
-struct DBFileIDAndName {
-    DBFileID dbFileID;
-    std::string fName;
-
-    DBFileIDAndName(DBFileID dbFileID, std::string fName)
-        : dbFileID{dbFileID}, fName{std::move(fName)} {};
-};
-
 struct PageCursor {
     PageCursor(common::page_idx_t pageIdx, uint32_t posInPage)
         : pageIdx{pageIdx}, elemPosInPage{posInPage} {};
@@ -49,10 +38,11 @@ struct PageUtils {
         auto numBytesPerNullEntry = common::NullMask::NUM_BITS_PER_NULL_ENTRY >> 3;
         auto numNullEntries =
             hasNull ?
-                (uint32_t)ceil((double)common::KUZU_PAGE_SIZE /
-                               (double)(((uint64_t)elementSize
+                static_cast<uint32_t>(ceil(
+                    static_cast<double>(common::KUZU_PAGE_SIZE) /
+                    static_cast<double>((static_cast<uint64_t>(elementSize)
                                             << common::NullMask::NUM_BITS_PER_NULL_ENTRY_LOG2) +
-                                        numBytesPerNullEntry)) :
+                                        numBytesPerNullEntry))) :
                 0;
         return (common::KUZU_PAGE_SIZE - (numNullEntries * numBytesPerNullEntry)) / elementSize;
     }
@@ -82,7 +72,7 @@ class StorageUtils {
     // TODO: Constrain T1 and T2 to numerics.
     template<typename T1, typename T2>
     static uint64_t divideAndRoundUpTo(T1 v1, T2 v2) {
-        return std::ceil((double)v1 / (double)v2);
+        return std::ceil(static_cast<double>(v1) / static_cast<double>(v2));
     }
 
     static std::string getColumnName(const std::string& propertyName, ColumnType type,
@@ -101,10 +91,6 @@ class StorageUtils {
         return std::make_pair(nodeGroupIdx, offsetInChunk);
     }
 
-    static std::string getNodeIndexFName(const common::VirtualFileSystem* vfs,
-        const std::string& directory, const common::table_id_t& tableID,
-        common::FileVersionType dbFileType);
-
     static std::string getDataFName(common::VirtualFileSystem* vfs, const std::string& directory) {
         return vfs->joinPath(directory, common::StorageConstants::DATA_FILE_NAME);
     }
@@ -116,17 +102,6 @@ class StorageUtils {
                                             common::StorageConstants::METADATA_FILE_NAME_FOR_WAL);
     }
 
-    static DBFileIDAndName getNodeIndexIDAndFName(common::VirtualFileSystem* vfs,
-        const std::string& directory, common::table_id_t tableID) {
-        auto fName = getNodeIndexFName(vfs, directory, tableID, common::FileVersionType::ORIGINAL);
-        return {DBFileID::newPKIndexFileID(tableID), fName};
-    }
-
-    static std::string getOverflowFileName(const std::string& fName) {
-        return appendSuffixOrInsertBeforeWALSuffix(fName,
-            common::StorageConstants::OVERFLOW_FILE_SUFFIX);
-    }
-
     static std::string getCatalogFilePath(common::VirtualFileSystem* vfs,
         const std::string& directory, common::FileVersionType dbFileType) {
         return vfs->joinPath(directory, dbFileType == common::FileVersionType::ORIGINAL ?

src/include/storage/storage_utils.h

@@ -3,7 +3,6 @@
 #include "catalog/catalog.h"
 #include "common/null_mask.h"
 #include "common/types/types.h"
-#include "storage/db_file_id.h"
 #include "storage/store/column_chunk_data.h"
 #include "storage/store/column_reader_writer.h"
 
@@ -137,7 +136,6 @@ class Column {
 
 protected:
     std::string name;
-    DBFileID dbFileID;
     common::LogicalType dataType;
     FileHandle* dataFH;
     MemoryManager* mm;