Benchmark ext mem connect

cmake/Utils.cmake

@@ -120,6 +120,7 @@ function(xgboost_set_cuda_flags target)
     # need to link with CCCL and CUDA runtime.
     target_link_libraries(${target} PRIVATE CCCL::CCCL CUDA::cudart_static)
   endif()
+  target_link_libraries(${target} PRIVATE CUDA::nvml)
   target_compile_definitions(${target} PRIVATE -DXGBOOST_USE_CUDA=1)
   target_include_directories(
     ${target} PRIVATE

jvm-packages/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <groupId>ml.dmlc</groupId>
-    <artifactId>xgboost-jvm_2.12</artifactId>
+    <artifactId>xgboost-jvm_2.13</artifactId>
     <version>3.1.0-SNAPSHOT</version>
     <packaging>pom</packaging>
     <name>XGBoost JVM Package</name>
@@ -43,15 +43,15 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-        <maven.compiler.source>1.8</maven.compiler.source>
-        <maven.compiler.target>1.8</maven.compiler.target>
+        <maven.compiler.source>17</maven.compiler.source>
+        <maven.compiler.target>17</maven.compiler.target>
         <flink.version>1.20.0</flink.version>
         <junit.version>4.13.2</junit.version>
-        <spark.version>3.5.3</spark.version>
-        <spark.version.gpu>3.5.1</spark.version.gpu>
+        <spark.version>4.0.0</spark.version>
+        <spark.version.gpu>4.0.0</spark.version.gpu>
         <fasterxml.jackson.version>2.15.0</fasterxml.jackson.version>
-        <scala.version>2.12.18</scala.version>
-        <scala.binary.version>2.12</scala.binary.version>
+        <scala.version>2.13.11</scala.version>
+        <scala.binary.version>2.13</scala.binary.version>
         <hadoop.version>3.4.1</hadoop.version>
         <maven.wagon.http.retryHandler.count>5</maven.wagon.http.retryHandler.count>
         <log.capi.invocation>OFF</log.capi.invocation>
@@ -89,6 +89,17 @@
             <name>central maven</name>
             <url>https://repo1.maven.org/maven2</url>
         </repository>
+      <repository>
+        <id>apache-snapshots</id>
+        <url>https://repository.apache.org/content/repositories/snapshots/</url>
+        <snapshots>
+          <enabled>true</enabled>
+        </snapshots>
+        <releases>
+          <enabled>false</enabled>
+        </releases>
+      </repository>
+
     </repositories>
     <modules>
     </modules>

jvm-packages/xgboost4j-example/pom.xml

@@ -5,11 +5,11 @@
     <modelVersion>4.0.0</modelVersion>
     <parent>
         <groupId>ml.dmlc</groupId>
-        <artifactId>xgboost-jvm_2.12</artifactId>
+        <artifactId>xgboost-jvm_2.13</artifactId>
         <version>3.1.0-SNAPSHOT</version>
     </parent>
     <name>xgboost4j-example</name>
-    <artifactId>xgboost4j-example_2.12</artifactId>
+    <artifactId>xgboost4j-example_2.13</artifactId>
     <version>3.1.0-SNAPSHOT</version>
     <packaging>jar</packaging>
     <build>
@@ -26,7 +26,7 @@
     <dependencies>
         <dependency>
             <groupId>ml.dmlc</groupId>
-            <artifactId>xgboost4j-spark_2.12</artifactId>
+            <artifactId>xgboost4j-spark_2.13</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>
@@ -37,7 +37,7 @@
         </dependency>
         <dependency>
             <groupId>ml.dmlc</groupId>
-            <artifactId>xgboost4j-flink_2.12</artifactId>
+            <artifactId>xgboost4j-flink_2.13</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>

jvm-packages/xgboost4j-flink/pom.xml

@@ -5,12 +5,12 @@
     <modelVersion>4.0.0</modelVersion>
     <parent>
         <groupId>ml.dmlc</groupId>
-        <artifactId>xgboost-jvm_2.12</artifactId>
+        <artifactId>xgboost-jvm_2.13</artifactId>
         <version>3.1.0-SNAPSHOT</version>
     </parent>
 
     <name>xgboost4j-flink</name>
-    <artifactId>xgboost4j-flink_2.12</artifactId>
+    <artifactId>xgboost4j-flink_2.13</artifactId>
     <version>3.1.0-SNAPSHOT</version>
     <properties>
       <flink-ml.version>2.2.0</flink-ml.version>
@@ -30,7 +30,7 @@
     <dependencies>
         <dependency>
             <groupId>ml.dmlc</groupId>
-            <artifactId>xgboost4j_2.12</artifactId>
+            <artifactId>xgboost4j_2.13</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>

jvm-packages/xgboost4j-spark-gpu/pom.xml

@@ -5,11 +5,11 @@
     <modelVersion>4.0.0</modelVersion>
     <parent>
         <groupId>ml.dmlc</groupId>
-        <artifactId>xgboost-jvm_2.12</artifactId>
+        <artifactId>xgboost-jvm_2.13</artifactId>
         <version>3.1.0-SNAPSHOT</version>
     </parent>
     <name>xgboost4j-spark-gpu</name>
-    <artifactId>xgboost4j-spark-gpu_2.12</artifactId>
+    <artifactId>xgboost4j-spark-gpu_2.13</artifactId>
     <description>JVM Package for XGBoost</description>
     <url>https://github.com/dmlc/xgboost/tree/master/jvm-packages</url>
     <licenses>
@@ -78,17 +78,17 @@
     <dependencies>
         <dependency>
             <groupId>ml.dmlc</groupId>
-            <artifactId>xgboost4j_2.12</artifactId>
+            <artifactId>xgboost4j_2.13</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>
             <groupId>ml.dmlc</groupId>
-            <artifactId>xgboost4j-spark_2.12</artifactId>
+            <artifactId>xgboost4j-spark_2.13</artifactId>
             <version>${project.version}</version>
             <exclusions>
               <exclusion>
                   <groupId>ml.dmlc</groupId>
-                  <artifactId>xgboost4j_2.12</artifactId>
+                  <artifactId>xgboost4j_2.13</artifactId>
               </exclusion>
             </exclusions>
         </dependency>

jvm-packages/xgboost4j-spark-gpu/src/main/java/ml/dmlc/xgboost4j/java/ExtMemQuantileDMatrix.java

@@ -31,15 +31,16 @@ public ExtMemQuantileDMatrix(Iterator<ColumnBatch> iter,
       DMatrix ref,
       int nthread,
       int maxQuantileBatches,
-      int minCachePageBytes) throws XGBoostError {
+      long minCachePageBytes,
+      float cacheHostRatio) throws XGBoostError {
     long[] out = new long[1];
     long[] refHandle = null;
     if (ref != null) {
       refHandle = new long[1];
       refHandle[0] = ref.getHandle();
     }
     String conf = this.getConfig(missing, maxBin, nthread,
-                                 maxQuantileBatches, minCachePageBytes);
+                                 maxQuantileBatches, minCachePageBytes, cacheHostRatio);
     XGBoostJNI.checkCall(XGBoostJNI.XGExtMemQuantileDMatrixCreateFromCallback(
         iter, refHandle, conf, out));
     handle = out[0];
@@ -50,7 +51,7 @@ public ExtMemQuantileDMatrix(
       float missing,
       int maxBin,
       DMatrix ref) throws XGBoostError {
-    this(iter, missing, maxBin, ref, 0, -1, -1);
+    this(iter, missing, maxBin, ref, 0, -1, -1, -1.0f);
   }
 
   public ExtMemQuantileDMatrix(
@@ -61,19 +62,25 @@ public ExtMemQuantileDMatrix(
   }
 
   private String getConfig(float missing, int maxBin, int nthread,
-                           int maxQuantileBatches, int minCachePageBytes) {
+                           int maxQuantileBatches, long minCachePageBytes, float cacheHostRatio) {
     Map<String, Object> conf = new java.util.HashMap<>();
     conf.put("missing", missing);
     conf.put("max_bin", maxBin);
     conf.put("nthread", nthread);
 
     if (maxQuantileBatches > 0) {
-      conf.put("max_quantile_batches", maxQuantileBatches);
+      conf.put("max_quantile_blocks", maxQuantileBatches);
     }
+    System.err.println("minCachePageBytes");
+    System.err.println(minCachePageBytes);
     if (minCachePageBytes > 0) {
       conf.put("min_cache_page_bytes", minCachePageBytes);
     }
 
+    if (cacheHostRatio > 0.0 && cacheHostRatio <= 1.0) {
+      conf.put("cache_host_ratio", cacheHostRatio);
+    }
+
     conf.put("on_host", true);
     conf.put("cache_prefix", ".");
     ObjectMapper mapper = new ObjectMapper();

jvm-packages/xgboost4j-spark-gpu/src/main/scala/ml/dmlc/xgboost4j/scala/ExtMemQuantileDMatrix.scala

@@ -28,10 +28,11 @@ class ExtMemQuantileDMatrix private[scala](
            ref: Option[QuantileDMatrix],
            nthread: Int,
            maxQuantileBatches: Int,
-           minCachePageBytes: Int) {
+           minCachePageBytes: Long,
+           cacheHostRatio: Float) {
     this(new jExtMemQuantileDMatrix(iter.asJava, missing, maxBin,
       ref.map(_.jDMatrix).orNull,
-      nthread, maxQuantileBatches, minCachePageBytes))
+      nthread, maxQuantileBatches, minCachePageBytes, cacheHostRatio))
   }
 
   def this(iter: Iterator[ColumnBatch], missing: Float, maxBin: Int) {