channeldb: speed up graph cache loading

Use the optimized ForEachChannel method to reduce the graph cache
loading time.

Author: joostjager

channeldb/graph.go

@@ -216,15 +216,29 @@ func NewChannelGraph(db kvdb.Backend, rejectCacheSize, chanCacheSize int,
 		startTime := time.Now()
 		log.Debugf("Populating in-memory channel graph, this might " +
 			"take a while...")
+
 		err := g.ForEachNodeCacheable(
 			func(tx kvdb.RTx, node GraphCacheNode) error {
-				return g.graphCache.AddNode(tx, node)
+				g.graphCache.AddNodeFeatures(node)
+
+				return nil
 			},
 		)
 		if err != nil {
 			return nil, err
 		}
 
+		err = g.ForEachChannel(func(info *ChannelEdgeInfo,
+			policy1, policy2 *ChannelEdgePolicy) error {
+
+			g.graphCache.AddChannel(info, policy1, policy2)
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
 		log.Debugf("Finished populating in-memory channel graph (took "+
 			"%v, %s)", time.Since(startTime), g.graphCache.Stats())
 	}

channeldb/graph_test.go

@@ -1207,11 +1207,22 @@ func TestGraphTraversalCacheable(t *testing.T) {
 	// Iterate through all the known channels within the graph DB by
 	// iterating over each node, once again if the map is empty that
 	// indicates that all edges have properly been reached.
+	var nodes []GraphCacheNode
 	err = graph.ForEachNodeCacheable(
 		func(tx kvdb.RTx, node GraphCacheNode) error {
 			delete(nodeMap, node.PubKey())
 
-			return node.ForEachChannel(
+			nodes = append(nodes, node)
+
+			return nil
+		},
+	)
+	require.NoError(t, err)
+	require.Len(t, nodeMap, 0)
+
+	err = graph.db.View(func(tx kvdb.RTx) error {
+		for _, node := range nodes {
+			err := node.ForEachChannel(
 				tx, func(tx kvdb.RTx, info *ChannelEdgeInfo,
 					policy *ChannelEdgePolicy,
 					policy2 *ChannelEdgePolicy) error {
@@ -1220,10 +1231,15 @@ func TestGraphTraversalCacheable(t *testing.T) {
 					return nil
 				},
 			)
-		},
-	)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}, func() {})
+
 	require.NoError(t, err)
-	require.Len(t, nodeMap, 0)
 	require.Len(t, chanIndex, 0)
 }
 
@@ -3695,9 +3711,20 @@ func BenchmarkForEachChannel(b *testing.B) {
 			totalCapacity btcutil.Amount
 			maxHTLCs      lnwire.MilliSatoshi
 		)
-		err := graph.ForEachNodeCacheable(
-			func(tx kvdb.RTx, n GraphCacheNode) error {
-				return n.ForEachChannel(
+
+		var nodes []GraphCacheNode
+		err = graph.ForEachNodeCacheable(
+			func(tx kvdb.RTx, node GraphCacheNode) error {
+				nodes = append(nodes, node)
+
+				return nil
+			},
+		)
+		require.NoError(b, err)
+
+		err = graph.db.View(func(tx kvdb.RTx) error {
+			for _, n := range nodes {
+				err := n.ForEachChannel(
 					tx, func(tx kvdb.RTx,
 						info *ChannelEdgeInfo,
 						policy *ChannelEdgePolicy,
@@ -3715,8 +3742,13 @@ func BenchmarkForEachChannel(b *testing.B) {
 						return nil
 					},
 				)
-			},
-		)
+				if err != nil {
+					return err
+				}
+			}
+
+			return nil
+		}, func() {})
 		require.NoError(b, err)
 	}
 }
@@ -3760,3 +3792,52 @@ func TestGraphCacheForEachNodeChannel(t *testing.T) {
 
 	require.Equal(t, numChans, 1)
 }
+
+// TestGraphLoading asserts that the cache is properly reconstructed after a
+// restart.
+func TestGraphLoading(t *testing.T) {
+	// First, create a temporary directory to be used for the duration of
+	// this test.
+	tempDirName, err := ioutil.TempDir("", "channelgraph")
+	require.NoError(t, err)
+	defer os.RemoveAll(tempDirName)
+
+	// Next, create the graph for the first time.
+	backend, backendCleanup, err := kvdb.GetTestBackend(tempDirName, "cgr")
+	require.NoError(t, err)
+	defer backend.Close()
+	defer backendCleanup()
+
+	opts := DefaultOptions()
+	graph, err := NewChannelGraph(
+		backend, opts.RejectCacheSize, opts.ChannelCacheSize,
+		opts.BatchCommitInterval, opts.PreAllocCacheNumNodes,
+		true, false,
+	)
+	require.NoError(t, err)
+
+	// Populate the graph with test data.
+	const numNodes = 100
+	const numChannels = 4
+	_, _ = fillTestGraph(t, graph, numNodes, numChannels)
+
+	// Recreate the graph. This should cause the graph cache to be
+	// populated.
+	graphReloaded, err := NewChannelGraph(
+		backend, opts.RejectCacheSize, opts.ChannelCacheSize,
+		opts.BatchCommitInterval, opts.PreAllocCacheNumNodes,
+		true, false,
+	)
+	require.NoError(t, err)
+
+	// Assert that the cache content is identical.
+	require.Equal(
+		t, graph.graphCache.nodeChannels,
+		graphReloaded.graphCache.nodeChannels,
+	)
+
+	require.Equal(
+		t, graph.graphCache.nodeFeatures,
+		graphReloaded.graphCache.nodeFeatures,
+	)
+}

docs/release-notes/release-notes-0.14.2.md

@@ -19,6 +19,11 @@ connection from the watch-only node.
   In other words, freshly-installed LND can now be initialized with multiple
   channels from an external (e.g. hardware) wallet *in a single transaction*.
 
+## Database
+
+* [Speed up graph cache loading on startup with
+Postgres](https://github.com/lightningnetwork/lnd/pull/6111)
+
 ## Build System
 
 * [Clean up Makefile by using go