Skip to content

Commit 234b8dc

Browse files
ray6080ray6080
committed
bump storage and more fixes
1 parent 5573ad2 commit 234b8dc

File tree

12 files changed

+36
-165
lines changed

12 files changed

+36
-165
lines changed

‎CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
cmake_minimum_required(VERSION 3.15)
22

3-
project(Kuzu VERSION 0.10.0 LANGUAGES CXX C)
3+
project(Kuzu VERSION 0.10.0.1 LANGUAGES CXX C)
44

55
option(SINGLE_THREADED "Single-threaded mode" FALSE)
66
if(SINGLE_THREADED)

‎src/include/storage/storage_structure/disk_array.h

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ struct PIPUpdates {
6868
std::optional<PIPWrapper> updatedLastPIP;
6969
std::vector<PIPWrapper> newPIPs;
7070

71-
inline void clear() {
71+
void clear() {
7272
updatedLastPIP.reset();
7373
newPIPs.clear();
7474
}
@@ -105,8 +105,6 @@ class DiskArrayInternal {
105105
DiskArrayHeader& headerForWriteTrx, ShadowFile* shadowFile, uint64_t elementSize,
106106
bool bypassShadowing = false);
107107

108-
virtual ~DiskArrayInternal() = default;
109-
110108
uint64_t getNumElements(
111109
transaction::TransactionType trxType = transaction::TransactionType::READ_ONLY);
112110

@@ -124,16 +122,16 @@ class DiskArrayInternal {
124122
uint64_t resize(const transaction::Transaction* transaction, uint64_t newNumElements,
125123
std::span<std::byte> defaultVal);
126124

127-
virtual inline void checkpointInMemoryIfNecessary() {
125+
void checkpointInMemoryIfNecessary() {
128126
std::unique_lock xlock{this->diskArraySharedMtx};
129127
checkpointOrRollbackInMemoryIfNecessaryNoLock(true /* is checkpoint */);
130128
}
131-
virtual inline void rollbackInMemoryIfNecessary() {
129+
void rollbackInMemoryIfNecessary() {
132130
std::unique_lock xlock{this->diskArraySharedMtx};
133131
checkpointOrRollbackInMemoryIfNecessaryNoLock(false /* is rollback */);
134132
}
135133

136-
virtual void checkpoint();
134+
void checkpoint();
137135

138136
// Write WriteIterator for making fast bulk changes to the disk array
139137
// The pages are cached while the elements are stored on the same page
@@ -179,7 +177,7 @@ class DiskArrayInternal {
179177
return std::span(shadowPageAndFrame.frame + apCursor.elemPosInPage, valueSize);
180178
}
181179

182-
inline uint64_t size() const { return diskArray.headerForWriteTrx.numElements; }
180+
uint64_t size() const { return diskArray.headerForWriteTrx.numElements; }
183181

184182
private:
185183
void unpin();
@@ -196,13 +194,11 @@ class DiskArrayInternal {
196194

197195
void updateLastPageOnDisk();
198196

199-
uint64_t pushBackNoLock(std::span<std::byte> val);
200-
201-
inline uint64_t getNumElementsNoLock(transaction::TransactionType trxType) const {
197+
uint64_t getNumElementsNoLock(transaction::TransactionType trxType) const {
202198
return getDiskArrayHeader(trxType).numElements;
203199
}
204200

205-
inline uint64_t getNumAPs(const DiskArrayHeader& header) const {
201+
uint64_t getNumAPs(const DiskArrayHeader& header) const {
206202
return (header.numElements + storageInfo.numElementsPerPage - 1) /
207203
storageInfo.numElementsPerPage;
208204
}
@@ -220,13 +216,13 @@ class DiskArrayInternal {
220216

221217
void clearWALPageVersionAndRemovePageFromFrameIfNecessary(common::page_idx_t pageIdx);
222218

223-
virtual void checkpointOrRollbackInMemoryIfNecessaryNoLock(bool isCheckpoint);
219+
void checkpointOrRollbackInMemoryIfNecessaryNoLock(bool isCheckpoint);
224220

225221
private:
226222
bool checkOutOfBoundAccess(transaction::TransactionType trxType, uint64_t idx) const;
227-
bool hasPIPUpdatesNoLock(uint64_t pipIdx);
223+
bool hasPIPUpdatesNoLock(uint64_t pipIdx) const;
228224

229-
inline const DiskArrayHeader& getDiskArrayHeader(transaction::TransactionType trxType) const {
225+
const DiskArrayHeader& getDiskArrayHeader(transaction::TransactionType trxType) const {
230226
if (trxType == transaction::TransactionType::CHECKPOINT) {
231227
return headerForWriteTrx;
232228
}

‎src/include/storage/storage_structure/overflow_file.h

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,6 @@
1010
#include "storage//file_handle.h"
1111
#include "storage/index/hash_index_utils.h"
1212
#include "storage/storage_utils.h"
13-
#include "storage/wal/shadow_file.h"
14-
#include "storage/wal/wal.h"
1513

1614
namespace kuzu {
1715
namespace storage {

‎src/storage/index/hash_index.cpp

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include <cstdint>
55

66
#include "common/assert.h"
7+
#include "common/serializer/serializer.h"
78
#include "common/types/int128_t.h"
89
#include "common/types/ku_string.h"
910
#include "common/types/types.h"
@@ -31,8 +32,8 @@ HashIndex<T>::HashIndex(MemoryManager& memoryManager, FileHandle* fileHandle,
3132
OverflowFileHandle* overflowFileHandle, DiskArrayCollection& diskArrays, uint64_t indexPos,
3233
ShadowFile* shadowFile, const HashIndexHeader& indexHeaderForReadTrx,
3334
HashIndexHeader& indexHeaderForWriteTrx)
34-
: shadowFile{shadowFile}, headerPageIdx(0), fileHandle(fileHandle),
35-
overflowFileHandle(overflowFileHandle),
35+
: shadowFile{shadowFile}, headerPageIdx{0}, fileHandle{fileHandle},
36+
overflowFileHandle{overflowFileHandle},
3637
localStorage{std::make_unique<HashIndexLocalStorage<T>>(memoryManager, overflowFileHandle)},
3738
indexHeaderForReadTrx{indexHeaderForReadTrx}, indexHeaderForWriteTrx{indexHeaderForWriteTrx},
3839
memoryManager{memoryManager} {
@@ -148,7 +149,7 @@ void HashIndex<T>::splitSlots(const Transaction* transaction, HashIndexHeader& h
148149
Slot<T>* originalSlot = &*originalSlotIterator.seek(header.nextSplitSlotId);
149150
do {
150151
for (entry_pos_t originalEntryPos = 0; originalEntryPos < getSlotCapacity<T>();
151-
originalEntryPos++) {
152+
originalEntryPos++) {
152153
if (!originalSlot->header.isEntryValid(originalEntryPos)) {
153154
continue; // Skip invalid entries.
154155
}
@@ -295,9 +296,10 @@ void HashIndex<T>::mergeBulkInserts(const Transaction* transaction,
295296
// may not be consecutive, but we reduce the memory overhead for storing the information about
296297
// the sorted data and still just process each page once.
297298
for (uint64_t localSlotId = 0; localSlotId < insertLocalStorage.numPrimarySlots();
298-
localSlotId += NUM_SLOTS_PER_PAGE) {
299+
localSlotId += NUM_SLOTS_PER_PAGE) {
299300
for (size_t i = 0;
300-
i < NUM_SLOTS_PER_PAGE && localSlotId + i < insertLocalStorage.numPrimarySlots(); i++) {
301+
i < NUM_SLOTS_PER_PAGE && localSlotId + i < insertLocalStorage.numPrimarySlots();
302+
i++) {
301303
auto localSlot =
302304
typename InMemHashIndex<T>::SlotIterator(localSlotId + i, &insertLocalStorage);
303305
partitionedEntries[i].clear();
@@ -436,7 +438,7 @@ PrimaryKeyIndex::PrimaryKeyIndex(FileHandle* dataFH, bool inMemMode, PhysicalTyp
436438
fileHandle->optimisticReadPage(this->firstHeaderPage + headerPageIdx, [&](auto* frame) {
437439
const auto onDiskHeaders = reinterpret_cast<HashIndexHeaderOnDisk*>(frame);
438440
for (size_t i = 0; i < INDEX_HEADERS_PER_PAGE && headerIdx < NUM_HASH_INDEXES;
439-
i++) {
441+
i++) {
440442
hashIndexHeadersForReadTrx.emplace_back(onDiskHeaders[i]);
441443
headerIdx++;
442444
}
@@ -560,7 +562,7 @@ void PrimaryKeyIndex::writeHeaders() {
560562
[&](auto* frame) {
561563
auto onDiskFrame = reinterpret_cast<HashIndexHeaderOnDisk*>(frame);
562564
for (size_t i = 0; i < INDEX_HEADERS_PER_PAGE && headerIdx < NUM_HASH_INDEXES;
563-
i++) {
565+
i++) {
564566
hashIndexHeadersForWriteTrx[headerIdx++].write(onDiskFrame[i]);
565567
}
566568
});

‎src/storage/storage_structure/disk_array.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -249,7 +249,7 @@ void DiskArrayInternal::checkpoint() {
249249
}
250250
}
251251

252-
bool DiskArrayInternal::hasPIPUpdatesNoLock(uint64_t pipIdx) {
252+
bool DiskArrayInternal::hasPIPUpdatesNoLock(uint64_t pipIdx) const {
253253
// This is a request to a pipIdx > pips.size(). Since pips.size() is the original number of pips
254254
// we started with before the write transaction is updated, we return true, i.e., this PIP is
255255
// an "updated" PIP and should be read from the WAL version.
@@ -386,7 +386,7 @@ void BlockVectorInternal::resize(uint64_t newNumElements, std::span<std::byte> d
386386
uint64_t newNumArrayPages = getNumArrayPagesNeededForElements(newNumElements);
387387
for (auto i = oldNumArrayPages; i < newNumArrayPages; ++i) {
388388
inMemArrayPages.emplace_back(
389-
memoryManager.allocateBuffer(true /*initializeToZero*/, common::KUZU_PAGE_SIZE));
389+
memoryManager.allocateBuffer(true /*initializeToZero*/, KUZU_PAGE_SIZE));
390390
}
391391
for (uint64_t i = 0; i < newNumElements - oldNumElements; i++) {
392392
memcpy(operator[](oldNumElements + i), defaultVal.data(), defaultVal.size());

‎src/storage/storage_structure/overflow_file.cpp

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -163,11 +163,10 @@ OverflowFile::OverflowFile(FileHandle* dataFH, MemoryManager& memoryManager, Sha
163163
}
164164

165165
OverflowFile::OverflowFile(FileHandle* dataFH, MemoryManager& memoryManager)
166-
: fileHandle{dataFH}, shadowFile{nullptr}, memoryManager{memoryManager}, headerChanged{false} {
166+
: numPagesOnDisk{0}, fileHandle{dataFH}, shadowFile{nullptr}, memoryManager{memoryManager},
167+
headerChanged{false}, headerPageIdx{INVALID_PAGE_IDX} {
167168
if (fileHandle) {
168169
numPagesOnDisk = fileHandle->getNumPages();
169-
} else {
170-
numPagesOnDisk = 0;
171170
}
172171
// Reserve a page for the header
173172
getNewPageIdx();
@@ -207,7 +206,7 @@ void OverflowFile::checkpoint(bool forceUpdateHeader) {
207206
memcpy(page, &header, sizeof(header));
208207
// Zero free space at the end of the header page
209208
std::fill(page + sizeof(header), page + KUZU_PAGE_SIZE, 0);
210-
writePageToDisk(HEADER_PAGE_IDX, page);
209+
writePageToDisk(headerPageIdx + HEADER_PAGE_IDX, page);
211210
}
212211
}
213212

‎src/storage/storage_utils.cpp

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
#include "storage/storage_utils.h"
22

33
#include "common/assert.h"
4-
#include "common/file_system/virtual_file_system.h"
54
#include "common/null_buffer.h"
65
#include "common/string_format.h"
76
#include "common/types/ku_list.h"

‎test/main/db_locking_test.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,7 @@ TEST_F(DBLockingTest, testReadOnly) {
120120
conn->query("CREATE NODE TABLE Person(name STRING, age INT64, PRIMARY KEY(name));")
121121
->isSuccess());
122122
ASSERT_TRUE(conn->query("CREATE (:Person {name: 'Alice', age: 25});")->isSuccess());
123+
ASSERT_TRUE(conn->query("CHECKPOINT;")->isSuccess());
123124
exit(0);
124125
}
125126
waitpid(create_pid, NULL, 0);

‎test/test_files/dml_node/delete/delete_empty.test

Lines changed: 4 additions & 127 deletions
Original file line numberDiff line numberDiff line change
@@ -2,130 +2,6 @@
22

33
--
44

5-
-CASE CreateDeleteNodeInSingelStatement
6-
-STATEMENT CREATE NODE TABLE A (id INT64, PRIMARY KEY (id));
7-
---- ok
8-
-STATEMENT CREATE NODE TABLE B (id INT64, PRIMARY KEY (id));
9-
---- ok
10-
-STATEMENT CREATE REL TABLE R (FROM A TO B);
11-
---- ok
12-
-STATEMENT CREATE (a:A {id:0})-[:R]->(b:B {id:10})
13-
---- ok
14-
-STATEMENT CREATE (a:A {id:1})-[:R]->(b:B {id:11})
15-
---- ok
16-
-STATEMENT UNWIND [2,3] AS x
17-
CREATE (a:A {id:x})-[:R]->(b:B {id:10 + x})
18-
WITH b
19-
WHERE b.id = 12
20-
DETACH DELETE b
21-
RETURN b.id
22-
---- 1
23-
12
24-
-STATEMENT MATCH (a)-[e]->(b) HINT a JOIN (e JOIN b) RETURN COUNT(*);
25-
---- 1
26-
3
27-
-STATEMENT MATCH (a)-[e]->(b) HINT (a JOIN e) JOIN b RETURN COUNT(*);
28-
---- 1
29-
3
30-
31-
-CASE MultipleDeletionsSingleTransaction
32-
-STATEMENT CREATE NODE TABLE test(id INT64, PRIMARY KEY(id));
33-
---- ok
34-
-STATEMENT CREATE (t:test {id:1});
35-
---- ok
36-
-STATEMENT CREATE (t:test {id:2});
37-
---- ok
38-
-STATEMENT CREATE (t:test {id:3});
39-
---- ok
40-
-STATEMENT CREATE (t:test {id:4});
41-
---- ok
42-
-STATEMENT BEGIN TRANSACTION;
43-
---- ok
44-
-STATEMENT MATCH (t:test) WHERE t.id > 2 DELETE t;
45-
---- ok
46-
-STATEMENT MATCH (t:test) DELETE t;
47-
---- ok
48-
-STATEMENT MATCH (t:test) RETURN COUNT(t);
49-
---- 1
50-
0
51-
-STATEMENT COMMIT;
52-
---- ok
53-
-STATEMENT MATCH (t:test) RETURN COUNT(t);
54-
---- 1
55-
0
56-
57-
-CASE DeleteFromFirstVector
58-
-STATEMENT CREATE NODE TABLE test(id INT64, PRIMARY KEY(id));
59-
---- ok
60-
-STATEMENT UNWIND RANGE(1, 2048) AS x CREATE (t:test {id:x});
61-
---- ok
62-
-STATEMENT UNWIND RANGE(2049, 4000) AS x CREATE (t:test {id:x});
63-
---- ok
64-
-STATEMENT CHECKPOINT;
65-
---- ok
66-
-STATEMENT MATCH (t:test) WHERE t.id < 100 DELETE t;
67-
---- ok
68-
-STATEMENT MATCH (t:test) RETURN MIN(t.id), MAX(t.id);
69-
---- 1
70-
100|4000
71-
72-
-CASE DeleteLocalNodeAtLargeOffset
73-
-STATEMENT create node table Comment (id int64, creationDate INT64, locationIP STRING, browserUsed STRING, content STRING, length INT32, PRIMARY KEY (id));
74-
---- ok
75-
-STATEMENT COPY Comment FROM "${KUZU_ROOT_DIRECTORY}/dataset/ldbc-sf01/Comment.csv";
76-
---- ok
77-
-STATEMENT BEGIN TRANSACTION
78-
---- ok
79-
-STATEMENT CREATE (:Comment {id: 8933535696141})
80-
---- ok
81-
-STATEMENT MATCH (c:Comment) WHERE c.id = 8933535696141 RETURN c.id
82-
---- 1
83-
8933535696141
84-
-STATEMENT MATCH (c:Comment) WHERE c.id = 8933535696141 DELETE c
85-
---- ok
86-
-STATEMENT COMMIT
87-
---- ok
88-
-STATEMENT MATCH (c:Comment) WHERE c.id = 8933535696141 RETURN c.id, c.creationDate
89-
---- 0
90-
91-
-CASE DeleteFirstNodeGroup
92-
-SKIP_IN_MEM
93-
-STATEMENT call checkpoint_threshold=0
94-
---- ok
95-
-STATEMENT create node table Post (id INT64, imageFile STRING, creationDate INT64, locationIP STRING, browserUsed STRING, language STRING, content STRING, length INT32, PRIMARY KEY (id));
96-
---- ok
97-
-STATEMENT COPY Post FROM "${KUZU_ROOT_DIRECTORY}/dataset/ldbc-sf01/Post.csv"(parallel=false)
98-
-PARALLELISM 1
99-
---- ok
100-
-STATEMENT call fsm_info() return *
101-
---- 0
102-
-STATEMENT call storage_info('Post') where node_group_id = 0 with num_values as node_group_capacity limit 1
103-
match (p:Post) where ID(p) < internal_id(0, node_group_capacity) delete p with count(*) as num_deleted, node_group_capacity
104-
return num_deleted = node_group_capacity
105-
---- 1
106-
True
107-
108-
# on the next allocation the pages for the deleted node group should be reused
109-
-STATEMENT create node table other(id INT64, primary key(id))
110-
---- ok
111-
-STATEMENT begin transaction;
112-
create (:other{id: 1});
113-
create (:other{id: 2});
114-
commit;
115-
---- ok
116-
---- ok
117-
---- ok
118-
---- ok
119-
-STATEMENT call storage_info('other') where start_page_idx = 0 return count (*)
120-
---- 1
121-
1
122-
123-
-STATEMENT match (p:Post) with count(*) as remaining
124-
optional match (p:Post) where p.ID = 1030792523231 with p.imageFile as imageFile, remaining
125-
return remaining = 0 or imageFile = 'photo1030792523231.jpg'
126-
---- 1
127-
True
128-
1295
-CASE DeleteAllTuples
1306
-SKIP_IN_MEM
1317
-STATEMENT call checkpoint_threshold=0
@@ -141,11 +17,12 @@ True
14117
# copy again, page allocation should start from 0 as the dropped pages are reclaimed + truncated
14218
-STATEMENT COPY Post FROM "${KUZU_ROOT_DIRECTORY}/dataset/ldbc-sf01/Post.csv"(parallel=false)
14319
---- ok
144-
-STATEMENT call fsm_info() return *
145-
---- 0
146-
-STATEMENT CALL storage_info('Post') where start_page_idx = 0 return count(*)
20+
-STATEMENT call fsm_info() return count(*)
14721
---- 1
14822
1
23+
-STATEMENT CALL storage_info('Post') where start_page_idx = 0 return count(*)
24+
---- 1
25+
0
14926
-RELOADDB
15027
# try some queries
15128
-STATEMENT match(p:Post) return count(*)

‎test/test_files/dml_rel/delete/delete_empty.test

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ Runtime exception: Cannot detach delete from node table 'person' as it has conne
8484
-STATEMENT CALL storage_info('Person') where start_page_idx < 4294967295 with max(start_page_idx + num_pages) as expected_start_page_idx
8585
CALL storage_info('knows') where start_page_idx = expected_start_page_idx return count(*)
8686
---- 1
87-
1
87+
0
8888
-RELOADDB
8989
# try some queries
9090
-STATEMENT match (a)-[k:knows]->(b) return count(*)

0 commit comments

Comments
 (0)