Merge branch 'master' into feature/iotconsensus-recv-snap

# Conflicts:
#	iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
#	iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/util/LoadUtil.java

Author: hongzhi-gao

dependencies.json

@@ -32,13 +32,13 @@
     "com.nimbusds:nimbus-jose-jwt",
     "com.nimbusds:oauth2-oidc-sdk",
     "com.sun.istack:istack-commons-runtime",
-    "com.timecho.ratis:ratis-client",
-    "com.timecho.ratis:ratis-common",
-    "com.timecho.ratis:ratis-grpc",
-    "com.timecho.ratis:ratis-metrics-api",
-    "com.timecho.ratis:ratis-proto",
-    "com.timecho.ratis:ratis-server",
-    "com.timecho.ratis:ratis-server-api",
+    "org.apache.ratis:ratis-client",
+    "org.apache.ratis:ratis-common",
+    "org.apache.ratis:ratis-grpc",
+    "org.apache.ratis:ratis-metrics-api",
+    "org.apache.ratis:ratis-proto",
+    "org.apache.ratis:ratis-server",
+    "org.apache.ratis:ratis-server-api",
     "com.zaxxer:HikariCP",
     "commons-cli:commons-cli",
     "commons-codec:commons-codec",

integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java

@@ -23,6 +23,7 @@
 import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant;
 import org.apache.iotdb.db.it.utils.TestUtils;
 import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
 import org.apache.iotdb.it.framework.IoTDBTestRunner;
 import org.apache.iotdb.it.utils.TsFileGenerator;
 import org.apache.iotdb.itbase.category.ClusterIT;
@@ -67,6 +68,7 @@
 import static org.apache.iotdb.db.it.utils.TestUtils.executeNonQuery;
 import static org.apache.iotdb.db.it.utils.TestUtils.grantUserSeriesPrivilege;
 import static org.apache.iotdb.db.it.utils.TestUtils.grantUserSystemPrivileges;
+import static org.apache.iotdb.it.env.cluster.ClusterConstant.USER_DIR;
 
 @RunWith(IoTDBTestRunner.class)
 @Category({LocalStandaloneIT.class, ClusterIT.class})
@@ -261,6 +263,108 @@ public void testLoad() throws Exception {
     }
   }
 
+  @Test
+  // Shall succeed with tablet conversion
+  public void testLoadWithAlignmentMismatch() throws Exception {
+    registerSchema();
+
+    final long writtenPoint1;
+    // device 0, sg 0
+    try (final TsFileGenerator generator =
+        new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) {
+      // Wrong, with 04-07 non-exist
+      generator.registerAlignedTimeseries(
+          SchemaConfig.DEVICE_0,
+          Arrays.asList(
+              SchemaConfig.MEASUREMENT_00,
+              SchemaConfig.MEASUREMENT_01,
+              SchemaConfig.MEASUREMENT_02,
+              SchemaConfig.MEASUREMENT_03,
+              SchemaConfig.MEASUREMENT_04,
+              SchemaConfig.MEASUREMENT_05,
+              SchemaConfig.MEASUREMENT_06,
+              SchemaConfig.MEASUREMENT_07));
+      generator.generateData(SchemaConfig.DEVICE_0, 100000, PARTITION_INTERVAL / 10_000, false);
+      writtenPoint1 = generator.getTotalNumber();
+    }
+
+    final long writtenPoint2;
+    // device 2, device 3, device4, sg 1
+    try (final TsFileGenerator generator =
+        new TsFileGenerator(new File(tmpDir, "2-0-0-0.tsfile"))) {
+      // right
+      generator.registerTimeseries(
+          SchemaConfig.DEVICE_2, Collections.singletonList(SchemaConfig.MEASUREMENT_20));
+      // right
+      generator.registerTimeseries(
+          SchemaConfig.DEVICE_3, Collections.singletonList(SchemaConfig.MEASUREMENT_30));
+      // Wrong, with 06 non-exist
+      generator.registerTimeseries(
+          SchemaConfig.DEVICE_4,
+          Arrays.asList(SchemaConfig.MEASUREMENT_40, SchemaConfig.MEASUREMENT_06));
+      generator.generateData(SchemaConfig.DEVICE_2, 10000, PARTITION_INTERVAL / 10_000, false);
+      generator.generateData(SchemaConfig.DEVICE_3, 10000, PARTITION_INTERVAL / 10_000, false);
+      generator.generateData(SchemaConfig.DEVICE_4, 10000, PARTITION_INTERVAL / 10_000, true);
+      for (int i = 0; i < 1000; i++) {
+        generator.generateData(SchemaConfig.DEVICE_4, 1, PARTITION_INTERVAL - 10, true);
+      }
+      writtenPoint2 = generator.getTotalNumber();
+    }
+
+    try (final Connection connection = EnvFactory.getEnv().getConnection();
+        final Statement statement = connection.createStatement()) {
+
+      try {
+        statement.execute(
+            String.format(
+                "load \"%s\" with ('database-level'='2', 'convert-on-type-mismatch'='false')",
+                tmpDir.getAbsolutePath() + File.separator + "1-0-0-0.tsfile"));
+        Assert.fail();
+      } catch (final Exception e) {
+        Assert.assertTrue(
+            e.getMessage()
+                .contains(
+                    "TimeSeries under this device is not aligned, please use createTimeSeries or change device. (Path: root.sg.test_0.d_0)."));
+      }
+
+      try {
+        statement.execute(
+            String.format(
+                "load \"%s\" with ('database-level'='2', 'convert-on-type-mismatch'='false')",
+                tmpDir.getAbsolutePath() + File.separator + "2-0-0-0.tsfile"));
+        Assert.fail();
+      } catch (final Exception e) {
+        Assert.assertTrue(
+            e.getMessage()
+                .contains(
+                    "TimeSeries under this device is aligned, please use createAlignedTimeSeries or change device. (Path: root.sg.test_1.a_4)."));
+      }
+
+      statement.execute(String.format("load \"%s\" sglevel=2", tmpDir.getAbsolutePath()));
+
+      try (final ResultSet resultSet =
+          statement.executeQuery("select count(*) from root.sg.** group by level=1,2")) {
+        if (resultSet.next()) {
+          long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)");
+          Assert.assertEquals(writtenPoint1, sg1Count);
+          long sg2Count = resultSet.getLong("count(root.sg.test_1.*.*)");
+          Assert.assertEquals(writtenPoint2, sg2Count);
+        } else {
+          Assert.fail("This ResultSet is empty.");
+        }
+      }
+    }
+
+    // Try to delete after loading. Expect no deadlock
+    try (final Connection connection = EnvFactory.getEnv().getConnection();
+        final Statement statement = connection.createStatement()) {
+      statement.execute(
+          String.format(
+              "delete timeseries %s.%s",
+              SchemaConfig.DEVICE_0, SchemaConfig.MEASUREMENT_00.getMeasurementName()));
+    }
+  }
+
   @Test
   public void testLoadAcrossMultipleTimePartitions() throws Exception {
     registerSchema();
@@ -742,6 +846,47 @@ public void testLoadWithOnNonStandardTsFileName() throws Exception {
     }
   }
 
+  @Test
+  public void testLoadWithRelativePathName() throws Exception {
+    DataNodeWrapper dataNodeWrapper = EnvFactory.getEnv().getDataNodeWrapper(0);
+
+    registerSchema();
+
+    final long writtenPoint1;
+    // device 0, device 1, sg 0
+    File relativePathFile = new File(System.getProperty(USER_DIR), "1-0-0-0.tsfile");
+    try {
+      try (final TsFileGenerator generator = new TsFileGenerator(relativePathFile)) {
+        generator.registerTimeseries(
+            SchemaConfig.DEVICE_0, Collections.singletonList(SchemaConfig.MEASUREMENT_00));
+        generator.generateData(SchemaConfig.DEVICE_0, 1, PARTITION_INTERVAL / 10_000, false);
+        writtenPoint1 = generator.getTotalNumber();
+      }
+
+      try (final Connection connection =
+              EnvFactory.getEnv().getConnectionWithSpecifiedDataNode(dataNodeWrapper);
+          final Statement statement = connection.createStatement()) {
+
+        statement.execute(String.format("load \"%s\" sglevel=2", "1-0-0-0.tsfile"));
+
+        try (final ResultSet resultSet =
+            statement.executeQuery("select count(*) from root.sg.** group by level=1,2")) {
+          if (resultSet.next()) {
+            final long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)");
+            Assert.assertEquals(writtenPoint1, sg1Count);
+          } else {
+            Assert.fail("This ResultSet is empty.");
+          }
+        }
+      }
+
+    } finally {
+      if (relativePathFile.exists()) {
+        relativePathFile.delete();
+      }
+    }
+  }
+
   @Test
   public void testLoadWithMods() throws Exception {
     final long writtenPoint1;

integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBAuthIT.java

@@ -54,6 +54,7 @@
 import static org.apache.iotdb.commons.auth.entity.User.INTERNAL_USER_END_ID;
 import static org.apache.iotdb.db.audit.DNAuditLogger.PREFIX_PASSWORD_HISTORY;
 import static org.apache.iotdb.db.it.utils.TestUtils.createUser;
+import static org.apache.iotdb.db.it.utils.TestUtils.executeNonQuery;
 import static org.apache.iotdb.db.it.utils.TestUtils.resultSetEqualTest;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -980,6 +981,8 @@ public void testGrantAndGrantOpt() throws SQLException {
     adminStmt.execute("CREATE USER user1 'password123456'");
     adminStmt.execute("CREATE USER user2 'password123456'");
     adminStmt.execute("CREATE USER user3 'password123456'");
+    adminStmt.execute("CREATE USER user4 'password123456'");
+    adminStmt.execute("CREATE USER user5 'password123456'");
     adminStmt.execute("CREATE ROLE testRole");
     adminStmt.execute("GRANT system ON root.** TO ROLE testRole WITH GRANT OPTION");
     adminStmt.execute("GRANT READ_DATA ON root.t1.** TO ROLE testRole");
@@ -1094,6 +1097,18 @@ public void testGrantAndGrantOpt() throws SQLException {
       }
     }
 
+    try (Connection userCon = EnvFactory.getEnv().getConnection("user4", "password123456");
+        Statement userStmt = userCon.createStatement()) {
+      adminStmt.execute("GRANT SYSTEM ON root.** TO USER user4");
+      try {
+        Assert.assertThrows(
+            SQLException.class, () -> userStmt.execute("GRANT SYSTEM ON root.** TO USER user5"));
+        adminStmt.execute("GRANT SYSTEM ON root.** TO USER user5");
+      } finally {
+        userStmt.close();
+      }
+    }
+
     adminStmt.close();
   }
 
@@ -1381,6 +1396,7 @@ public void noNeedPrivilegeTest() {
           "tempuser,",
         };
     resultSetEqualTest("show current_user", expectedHeader, retArray, "tempuser", "temppw123456");
+    executeNonQuery("SHOW AVAILABLE URLS", "tempuser", "temppw123456");
   }
 
   @Ignore

integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBCreateAlignedTimeseriesIT.java

@@ -19,6 +19,7 @@
 package org.apache.iotdb.db.it.schema;
 
 import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant;
+import org.apache.iotdb.db.it.utils.TestUtils;
 import org.apache.iotdb.it.env.EnvFactory;
 import org.apache.iotdb.itbase.category.ClusterIT;
 import org.apache.iotdb.itbase.category.LocalStandaloneIT;
@@ -34,6 +35,9 @@
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
+import java.util.Collections;
+
+import static org.junit.Assert.fail;
 
 /**
  * Notice that, all test begins with "IoTDB" is integration test. All test which will start the
@@ -141,4 +145,22 @@ private void assertTimeseriesEquals(String[] timeSeriesArray) throws SQLExceptio
     }
     Assert.assertEquals(timeSeriesArray.length, count);
   }
+
+  @Test
+  public void testDifferentDeviceAlignment() {
+    try (Connection connection = EnvFactory.getEnv().getConnection();
+        Statement statement = connection.createStatement()) {
+      // Should ignore the alignment difference
+      statement.execute("create aligned timeseries root.sg2.d (s2 int64, s3 int64)");
+      // Should use the existing alignment
+      statement.execute("create timeseries root.sg2.d.s1 with datatype=INT64");
+      statement.execute("insert into root.sg2.d (time, s4) values (-1, 1)");
+      TestUtils.assertResultSetEqual(
+          statement.executeQuery("select * from root.sg2.d"),
+          "Time,root.sg2.d.s3,root.sg2.d.s4,root.sg2.d.s1,root.sg2.d.s2,",
+          Collections.singleton("-1,null,1.0,null,null,"));
+    } catch (SQLException ignored) {
+      fail();
+    }
+  }
 }

integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBCreateTimeseriesIT.java

@@ -20,6 +20,7 @@
 package org.apache.iotdb.db.it.schema;
 
 import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant;
+import org.apache.iotdb.db.it.utils.TestUtils;
 import org.apache.iotdb.it.env.EnvFactory;
 import org.apache.iotdb.itbase.category.ClusterIT;
 import org.apache.iotdb.itbase.category.LocalStandaloneIT;
@@ -35,6 +36,7 @@
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
@@ -308,6 +310,12 @@ public void testDifferentDeviceAlignment() {
       statement.execute("create timeseries root.sg2.d.s1 with datatype=INT64");
       // Should ignore the alignment difference
       statement.execute("create aligned timeseries root.sg2.d (s2 int64, s3 int64)");
+      // Should use the existing alignment
+      statement.execute("insert into root.sg2.d (time, s4) aligned values (-1, 1)");
+      TestUtils.assertResultSetEqual(
+          statement.executeQuery("select * from root.sg2.d"),
+          "Time,root.sg2.d.s3,root.sg2.d.s4,root.sg2.d.s1,root.sg2.d.s2,",
+          Collections.singleton("-1,null,1.0,null,null,"));
     } catch (SQLException ignored) {
       fail();
     }

integration-test/src/test/java/org/apache/iotdb/db/it/selectinto/IoTDBSelectIntoIT.java

@@ -19,6 +19,7 @@
 
 package org.apache.iotdb.db.it.selectinto;
 
+import org.apache.iotdb.db.it.utils.TSDataTypeTestUtils;
 import org.apache.iotdb.it.env.EnvFactory;
 import org.apache.iotdb.it.framework.IoTDBTestRunner;
 import org.apache.iotdb.itbase.category.ClusterIT;
@@ -104,13 +105,11 @@ public class IoTDBSelectIntoIT {
   static {
     SELECT_INTO_SQL_LIST.add("CREATE DATABASE root.sg_type");
     for (int deviceId = 0; deviceId < 6; deviceId++) {
-      for (TSDataType dataType : TSDataType.values()) {
-        if (!dataType.equals(TSDataType.VECTOR) && !dataType.equals(TSDataType.UNKNOWN)) {
-          SELECT_INTO_SQL_LIST.add(
-              String.format(
-                  "CREATE TIMESERIES root.sg_type.d_%d.s_%s %s",
-                  deviceId, dataType.name().toLowerCase(), dataType));
-        }
+      for (TSDataType dataType : TSDataTypeTestUtils.getSupportedTypes()) {
+        SELECT_INTO_SQL_LIST.add(
+            String.format(
+                "CREATE TIMESERIES root.sg_type.d_%d.s_%s %s",
+                deviceId, dataType.name().toLowerCase(), dataType));
       }
     }
     for (int time = 0; time < 12; time++) {