diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 83c2a9ab..106058eb 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -1948,6 +1948,8 @@ private ObTableQuery buildObTableQuery(ObHTableFilter filter, final Scan scan) { HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE)); obTableQuery.setObKVParams(buildOBKVParams(scan)); obTableQuery.setScanRangeColumns("K", "Q", "T"); + byte[] hotOnly = scan.getAttribute(HBASE_HTABLE_QUERY_HOT_ONLY); + obTableQuery.setHotOnly(hotOnly != null && Arrays.equals(hotOnly, "true".getBytes())); return obTableQuery; } @@ -1965,6 +1967,8 @@ private ObTableQuery buildObTableQuery(final Get get, Collection columnQ } obTableQuery.setObKVParams(buildOBKVParams(get)); obTableQuery.setScanRangeColumns("K", "Q", "T"); + byte[] hotOnly = get.getAttribute(HBASE_HTABLE_QUERY_HOT_ONLY); + obTableQuery.setHotOnly(hotOnly != null && Arrays.equals(hotOnly, "true".getBytes())); return obTableQuery; } diff --git a/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java b/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java index 19311744..deb94f75 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java +++ b/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java @@ -23,17 +23,17 @@ public final class OHConstants { /** - * ocenbase hbase root server http url + * oceanbase hbase root server http url */ public static final String HBASE_OCEANBASE_PARAM_URL = "hbase.oceanbase.paramURL"; /** - * ocenbase hbase connect server username + * oceanbase hbase connect server username */ public static final String HBASE_OCEANBASE_FULL_USER_NAME = "hbase.oceanbase.fullUserName"; /** - * ocenbase hbase connect server password + * oceanbase hbase connect server password */ public static final String HBASE_OCEANBASE_PASSWORD = "hbase.oceanbase.password"; @@ -48,39 +48,39 @@ public final class OHConstants { public static final String HBASE_OCEANBASE_SYS_PASSWORD = "hbase.oceanbase.sysPassword"; /** - * ocenbase hbase connect server password + * oceanbase hbase connect server password */ public static final String HBASE_OCEANBASE_BATCH_EXECUTOR = "hbase.oceanbase.batch.executor"; /** - * ocenbase hbase connect server ODP address + * oceanbase hbase connect server ODP address */ public static final String HBASE_OCEANBASE_ODP_ADDR = "hbase.oceanbase.odpAddr"; /** - * ocenbase hbase connect server ODP port + * oceanbase hbase connect server ODP port */ public static final String HBASE_OCEANBASE_ODP_PORT = "hbase.oceanbase.odpPort"; /** - * ocenbase hbase connect server ODP mode + * oceanbase hbase connect server ODP mode */ public static final String HBASE_OCEANBASE_ODP_MODE = "hbase.oceanbase.odpMode"; /** - * ocenbase hbase connect server database + * oceanbase hbase connect server database */ public static final String HBASE_OCEANBASE_DATABASE = "hbase.oceanbase.database"; /** - * ocenbase hbase model rowkey column is consist of following column + * oceanbase hbase model rowkey column is consist of following column * K, Q, T hbase value */ public static final String[] ROW_KEY_COLUMNS = new String[] { "K", "Q", "T" }; /** - * ocenbase hbase model value column is consist of following column + * oceanbase hbase model value column is consist of following column * V hbase value */ public static final String[] V_COLUMNS = new String[] { "V" }; @@ -124,6 +124,11 @@ public final class OHConstants { */ public static final String DEFAULT_HBASE_HTABLE_TEST_LOAD_SUFFIX = "_t"; + /** + * use to specify whether to query only the data in hot storage when performing a query. + */ + public static final String HBASE_HTABLE_QUERY_HOT_ONLY = "hbase.htable.query.hot_only"; + /*-------------------------------------------------------------------------------------------------------------*/ /** diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageSeriesTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageSeriesTest.java new file mode 100644 index 00000000..53fb2746 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageSeriesTest.java @@ -0,0 +1,187 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.ValueFilter; +import org.junit.*; + +import java.util.*; + +import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_QUERY_HOT_ONLY; +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.*; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; + + +public class OHTableShareStorageSeriesTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap<>(); + + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableType type : NORMAL_SERIES_PARTITIONED_TABLES) { + createTables(type, tableNames, group2tableNames, true); + } + for (TableType type : NORMAL_SERIES_PARTITIONED_TABLES) { + alterTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + + public static void testGetImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + // 0. prepare data - 100 key + long recordCount = 100; + String family = getColumnFamilyName(tableName); + String key = "Key"; + String column = "Column"; + String value = "Value"; + long curTs = System.currentTimeMillis(); + for (int i = 0; i < recordCount; i++) { + Put put = new Put(toBytes(key + i)); + put.add(family.getBytes(), (column + i).getBytes(), curTs, toBytes(value + i)); + hTable.put(put); + } + + // 1. get, expect less than 100 key + long getCount = 0; + for (int i = 0; i < recordCount; i++) { + Get get = new Get((key + i).getBytes()); + get.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + get.addColumn(family.getBytes(), (column + i).getBytes()); + Result r = hTable.get(get); + Cell cells[] = r.rawCells(); + getCount += cells.length; + } + Assert.assertTrue(getCount < recordCount); + + hTable.close(); + } + + public static void testScanImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + // 0. prepare data - 100 key + long recordCount = 100; + String family = getColumnFamilyName(tableName); + String key = "Key"; + String column = "Column"; + String value = "Value"; + long curTs = System.currentTimeMillis(); + for (int i = 0; i < recordCount; i++) { + Put put = new Put(toBytes(key + i)); + put.add(family.getBytes(), (column).getBytes(), curTs, toBytes(value + i)); + hTable.put(put); + } + + // 1. scan not specify column + { + Scan scan = new Scan(key.getBytes(), (key+recordCount).getBytes()); + scan.addFamily(family.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 2. scan specify column + { + Scan scan = new Scan(key.getBytes(), (key+recordCount).getBytes()); + scan.addColumn(family.getBytes(), column.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 3. scan specify versions + { + Scan scan = new Scan(key.getBytes(), (key+recordCount).getBytes()); + scan.setMaxVersions(2); + scan.addColumn(family.getBytes(), column.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 4. scan specify time range + { + Scan scan = new Scan(key.getBytes(), (key+recordCount).getBytes()); + scan.setMaxVersions(2); + scan.setTimeStamp(curTs); + scan.addColumn(family.getBytes(), column.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + hTable.close(); + } + + @Test + public void testGet() throws Throwable { + FOR_EACH(tableNames, OHTableShareStorageSeriesTest::testGetImpl); + } + + + @Test + public void testScan() throws Throwable { + FOR_EACH(tableNames, OHTableShareStorageSeriesTest::testScanImpl); + } + +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageTest.java new file mode 100644 index 00000000..684f5c53 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageTest.java @@ -0,0 +1,558 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.ValueFilter; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.*; + +import java.util.*; + +import static com.alipay.oceanbase.hbase.constants.OHConstants.*; +import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_OCEANBASE_SYS_PASSWORD; +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.ODP_MODE; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.*; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.assertEquals; + + +public class OHTableShareStorageTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap<>(); + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableTemplateManager.TableType type : NORMAL_PARTITIONED_TABLES) { + createTables(type, tableNames, group2tableNames, true); + } + for (TableTemplateManager.TableType type : NORMAL_PARTITIONED_TABLES) { + alterTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + + public static void testGetImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + // 0. prepare data - 100 key + long recordCount = 100; + String family = getColumnFamilyName(tableName); + String key = "Key"; + String column = "Column"; + String value = "Value"; + long curTs = System.currentTimeMillis(); + for (int i = 0; i < recordCount; i++) { + Put put = new Put(toBytes(key + i)); + put.add(family.getBytes(), (column + i).getBytes(), curTs, toBytes(value + i)); + hTable.put(put); + } + + // 1. get, expect less than 100 key + long getCount = 0; + for (int i = 0; i < recordCount; i++) { + Get get = new Get((key + i).getBytes()); + get.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + get.addColumn(family.getBytes(), (column + i).getBytes()); + Result r = hTable.get(get); + Cell cells[] = r.rawCells(); + getCount += cells.length; + } + System.out.println("getCount:" + getCount); + Assert.assertTrue(getCount < recordCount); + + hTable.close(); + } + + public static void testMultiCFGetImpl(Map.Entry> entry) throws Exception { + // 0. prepare data + long recordCount = 100; + String groupName = getTableName(entry.getKey()); + List tableNames = entry.getValue(); + String key = "Key"; + String column = "Column"; + String value = "Value"; + long curTs = System.currentTimeMillis(); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); + hTable.init(); + + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (int i = 0; i < recordCount; i++) { + Put put = new Put(toBytes(key + i)); + put.add(family.getBytes(), (column + i).getBytes(), curTs, toBytes(value + i)); + hTable.put(put); + } + } + + // 1. get specify column + { + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + long getCount = 0; + for (int i = 0; i < recordCount; i++) { + Get get = new Get((key + i).getBytes()); + get.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + get.addColumn(family.getBytes(), (column + i).getBytes()); + Result r = hTable.get(get); + Cell cells[] = r.rawCells(); + getCount += cells.length; + System.out.println("getCount:" + getCount); + } + Assert.assertTrue(getCount < recordCount); + } + } + + // 2. get do not specify column + { + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + long getCount = 0; + for (int i = 0; i < recordCount; i++) { + Get get = new Get((key + i).getBytes()); + get.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + get.addFamily(family.getBytes()); + Result r = hTable.get(get); + Cell cells[] = r.rawCells(); + getCount += cells.length; + System.out.println("getCount:" + getCount); + } + Assert.assertTrue(getCount < recordCount); + } + } + + // 3. get do not specify column family + { + long getCount = 0; + Get get = new Get(key.getBytes()); + get.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + Result r = hTable.get(get); + Cell cells[] = r.rawCells(); + getCount += cells.length; + System.out.println("getCount:" + getCount); + Assert.assertTrue(getCount < recordCount); + } + + // 4. get specify multi cf and column + { + long getCount = 0; + Get get = new Get(key.getBytes()); + get.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + get.addColumn(family.getBytes(), column.getBytes()); + } + Result r = hTable.get(get); + Cell cells[] = r.rawCells(); + getCount += cells.length; + System.out.println("getCount:" + getCount); + Assert.assertTrue(getCount < recordCount); + } + + hTable.close(); + } + + public static void testBatchGetImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + // 0. prepare data - 100 key + long recordCount = 100; + String family = getColumnFamilyName(tableName); + String key = "Key"; + String column = "Column"; + String value = "Value"; + long curTs = System.currentTimeMillis(); + for (int i = 0; i < recordCount; i++) { + Put put = new Put(toBytes(key + i)); + put.add(family.getBytes(), (column + i).getBytes(), curTs, toBytes(value)); + hTable.put(put); + } + + List gets = new ArrayList<>(); + // 1. get, expect less than 100 key + long getCount = 0; + for (int i = 0; i < recordCount; i++) { + String getKey = null; + if (i == 4) { + getKey = key + 101; + } else { + getKey = key + i; + } + Get get = new Get((getKey + i).getBytes()); + get.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + get.addColumn(family.getBytes(), (column + i).getBytes()); + gets.add(get); + } + Result[] results = hTable.get(gets); + for (Result result : results) { + getCount += result.size(); + } + Assert.assertTrue(getCount < recordCount); + + hTable.close(); + } + + public static void testScanImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + // 0. prepare data - 100 key + long recordCount = 100; + String family = getColumnFamilyName(tableName); + String key = "Key"; + String column = "Column"; + String value = "Value"; + long curTs = System.currentTimeMillis(); + for (int i = 0; i < recordCount; i++) { + Put put = new Put(toBytes(key + i)); + put.add(family.getBytes(), (column).getBytes(), curTs, toBytes(value + i)); + hTable.put(put); + } + + // 1. scan not specify column + { + Scan scan = new Scan(key.getBytes(), (key+recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.addFamily(family.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 2. scan specify column + { + Scan scan = new Scan(key.getBytes(), (key+recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.addColumn(family.getBytes(), column.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 3. scan specify versions + { + Scan scan = new Scan(key.getBytes(), (key+recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.setMaxVersions(2); + scan.addColumn(family.getBytes(), column.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + + // 4. scan specify filter + { + Scan scan = new Scan(key.getBytes(), (key+recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.setMaxVersions(2); + scan.addFamily(family.getBytes()); + ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, + new BinaryComparator(toBytes(value))); + scan.setFilter(valueFilter); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 5. scan in reverse + { + Scan scan = new Scan(key.getBytes(), (key+recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.addFamily(family.getBytes()); + scan.setReversed(true); + try { + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } catch (Exception e) { + Assert.assertTrue(e.getCause().getMessage().contains("secondary partitioned hbase table with reverse query not supported")); + } + } + hTable.close(); + } + + public static void testMultiCFScanImpl(Map.Entry> entry) throws Exception { + String groupName = getTableName(entry.getKey()); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); + hTable.init(); + + // 0. prepare data - 100 key + long recordCount = 100; + String key = "Key"; + String column = "Column"; + String value = "Value"; + long curTs = System.currentTimeMillis(); + List tableNames = entry.getValue(); + + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (int i = 0; i < recordCount; i++) { + Put put = new Put(toBytes(key + i)); + put.add(family.getBytes(), (column).getBytes(), curTs, toBytes(value)); + hTable.put(put); + } + } + + // 1. multi cf scan specify one cf and one column + { + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + Scan scan = new Scan(key.getBytes(), (key + recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.addColumn(family.getBytes(), column.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + } + + // 2. multi cf scan specify one cf without specify column + { + for (String tableName : tableNames) { + Scan scan = new Scan(key.getBytes(), (key + recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + String family = getColumnFamilyName(tableName); + scan.addFamily(family.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + } + + // 3. multi cf scan do not specify cf + { + Scan scan = new Scan(key.getBytes(), (key + recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 4. multi cf scan specify multi cf and multi column + { + Scan scan = new Scan(key.getBytes(), (key + recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + scan.addColumn(family.getBytes(), column.getBytes()); + } + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 5. multi cf scan specify versions + { + Scan scan = new Scan(key.getBytes(), (key + recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.setMaxVersions(2); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 6. multi cf scan specify time range + { + Scan scan = new Scan(key.getBytes(), (key + recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.setMaxVersions(2); + scan.setTimeStamp(curTs); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 7. multi cf scan specify filter + { + Scan scan = new Scan(key.getBytes(), (key + recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.setMaxVersions(2); + ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, + new BinaryComparator(toBytes(value))); + scan.setFilter(valueFilter); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 8. multi cf scan using setStartRow/setEndRow + { + Scan scan = new Scan(); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.setStartRow(key.getBytes()); + scan.setStopRow((key + recordCount).getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 9. multi cf scan using batch + { + Scan scan = new Scan(key.getBytes(), (key + recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.setBatch(2); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 10. multi cf scan with family scan and column-specific scan + { + Scan scan = new Scan(key.getBytes(), (key + recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + for (int i = 0; i < tableNames.size(); i++) { + String family = getColumnFamilyName(tableNames.get(i)); + if (i % 2 == 0) { + scan.addFamily(family.getBytes()); + } else { + scan.addColumn(family.getBytes(), column.getBytes()); + } + } + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + hTable.close(); + } + + @Test + public void testGet() throws Throwable { + FOR_EACH(tableNames, OHTableShareStorageTest::testGetImpl); + } + + @Test + public void testMultiCFGet() throws Throwable { + FOR_EACH(group2tableNames, OHTableShareStorageTest::testMultiCFGetImpl); + } + + @Test + public void testBatchGet() throws Throwable { + FOR_EACH(tableNames, OHTableShareStorageTest::testBatchGetImpl); + } + + @Test + public void testScan() throws Throwable { + FOR_EACH(tableNames, OHTableShareStorageTest::testScanImpl); + } + + @Test + public void testMultiCFScan() throws Throwable { + FOR_EACH(group2tableNames, OHTableShareStorageTest::testMultiCFScanImpl); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java index 693fa35a..2ee37e4e 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java @@ -57,6 +57,20 @@ public static void createTables(TableTemplateManager.TableType type, List tableNames, + Map> group2tableNames, boolean printSql) + throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + // single cf table + if (tableNames != null) { + alterTables(conn, type, tableNames, printSql); + } + // multi cf table + if (group2tableNames != null) { + alterTables(conn, type, group2tableNames, printSql); + } + } + public static void createTables(Connection conn, TableTemplateManager.TableType type, List tableNames, boolean printSql) throws Exception { // create single cf table @@ -85,6 +99,31 @@ public static void createTables(Connection conn, TableTemplateManager.TableType } } + public static void alterTables(Connection conn, TableTemplateManager.TableType type, + List tableNames, boolean printSql) throws Exception { + // create single cf table + if (tableNames != null) { + String tableGroup = TableTemplateManager.getTableGroupName(type, false); + String tableName = TableTemplateManager.generateTableName(tableGroup, false, 1); + String sql = TableTemplateManager.getAlterTableSQL(type, tableName); + try { + System.out.println(sql); + conn.createStatement().execute(sql); + System.out.println("============= alter table: " + tableName + " table_group: " + + getTableName(tableName) + " =============\n" + + (printSql ? sql : "") + + " \n============= done =============\n"); + } catch (SQLSyntaxErrorException e) { + if (!e.getMessage().contains("already exists")) { + throw e; + } else { + System.out.println("============= table: " + tableName + " table_group: " + + getTableName(tableName) + " alter failed ============="); + } + } + } + } + public static void createTables(Connection conn, TableTemplateManager.TableType type, Map> group2tableNames, boolean printSql) throws Exception { if (group2tableNames != null) { TimeGenerator.TimeRange timeRange = TimeGenerator.generateTestTimeRange(); @@ -121,6 +160,30 @@ public static void createTables(Connection conn, TableTemplateManager.TableType } } + public static void alterTables(Connection conn, TableTemplateManager.TableType type, Map> group2tableNames, boolean printSql) throws Exception { + if (group2tableNames != null) { + String tableGroup = TableTemplateManager.getTableGroupName(type, true); + group2tableNames.put(tableGroup, new LinkedList<>()); + for (int i = 1; i <= 3; ++i) { + String tableName = TableTemplateManager.generateTableName(tableGroup, true, i); + String sql = TableTemplateManager.getAlterTableSQL(type, tableName); + try { + conn.createStatement().execute(sql); + System.out.println("============= alter table: " + tableName + + " table_group: " + getTableName(tableName) + " =============\n" + + (printSql ? sql : "") + " \n============= done =============\n"); + } catch (SQLSyntaxErrorException e) { + if (!e.getMessage().contains("already exists")) { + throw e; + } else { + System.out.println("============= table: " + tableName + " table_group: " + getTableName(tableName) + " alter failed ============="); + } + } + } + } + } + public static void truncateTables(List tableNames, Map> group2tableNames) throws Exception { Connection conn = ObHTableTestUtil.getConnection(); diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java index 81fd3688..b2c94e6c 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java @@ -103,6 +103,8 @@ public static Configuration newConfiguration() { Configuration conf = HBaseConfiguration.create(); conf.set(HBASE_OCEANBASE_FULL_USER_NAME, FULL_USER_NAME); conf.set(HBASE_OCEANBASE_PASSWORD, PASSWORD); + conf.set("rpc.execute.timeout", "20000"); + conf.set("rpc.operation.timeout", "18000"); if (ODP_MODE) { // ODP mode conf.set(HBASE_OCEANBASE_ODP_ADDR, ODP_ADDR); diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java index 75a3e32d..2263bfcc 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java @@ -59,6 +59,19 @@ public enum TableType { SECONDARY_PARTITIONED_TIME_RANGE_KEY, SECONDARY_PARTITIONED_TIME_KEY_RANGE); + public static List NORMAL_PARTITIONED_TABLES = Arrays + .asList( + SINGLE_PARTITIONED_REGULAR, + SECONDARY_PARTITIONED_RANGE_KEY, + SECONDARY_PARTITIONED_RANGE_KEY_GEN, + SECONDARY_PARTITIONED_KEY_RANGE, + SECONDARY_PARTITIONED_KEY_RANGE_GEN); + public static List NORMAL_SERIES_PARTITIONED_TABLES = Arrays + .asList( + SINGLE_PARTITIONED_TIME_SERIES, + SECONDARY_PARTITIONED_TIME_RANGE_KEY, + SECONDARY_PARTITIONED_TIME_KEY_RANGE); + public static List SERIES_TABLES = Arrays .asList( NON_PARTITIONED_TIME_SERIES, @@ -284,6 +297,57 @@ public enum TableType { + ") TABLEGROUP = %s"); } + private static final Map ALTER_SQL_TEMPLATES = new EnumMap( + TableType.class); + + static { + // 普通表一级分区模板 + ALTER_SQL_TEMPLATES.put(TableType.SINGLE_PARTITIONED_REGULAR, + "ALTER TABLE `%s` ALTER PARTITION p0 STORAGE_CACHE_POLICY='hot';"); + // 时序表一级分区模板 + ALTER_SQL_TEMPLATES.put(TableType.SINGLE_PARTITIONED_TIME_SERIES, + "ALTER TABLE `%s` ALTER PARTITION p0 STORAGE_CACHE_POLICY='hot';"); + // 普通表RANGE-KEY分区（使用K） + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + + // 合并GEN类型的注释处理 + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY_GEN, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + + // 普通表KEY-RANGE分区（使用K） + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + + // 普通表KEY-RANGE分区（使用生成列） + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE_GEN, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + + // 时序表RANGE-KEY分区 + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_TIME_RANGE_KEY, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + + // 时序表KEY-RANGE分区 + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_TIME_KEY_RANGE, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + + /* ------------------ CELL TTL ----------------*/ + ALTER_SQL_TEMPLATES.put(TableType.SINGLE_PARTITIONED_REGULAR_CELL_TTL, + "ALTER TABLE `%s` ALTER PARTITION p0 STORAGE_CACHE_POLICY='hot';"); + + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY_CELL_TTL, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY_GEN_CELL_TTL, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE_CELL_TTL, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE_GEN_CELL_TTL, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + } + public static String getCreateTableSQL(TableType type, String tableName, TimeGenerator.TimeRange timeRange) { System.out.println(tableName); @@ -329,6 +393,12 @@ public static String getCreateTableSQL(TableType type, String tableName, return String.format(template, params); } + public static String getAlterTableSQL(TableType type, String tableName) { + System.out.println(tableName); + String template = ALTER_SQL_TEMPLATES.get(type); + return String.format(template, tableName); + } + private static String getGeneratedColumn(TableType type) { StringBuilder sb = new StringBuilder(); boolean needsKPrefix = type.name().startsWith("SECONDARY_PARTITIONED")