Merge remote-tracking branch 'origin/main' into new-native-backup-provider

Author: JoaoJandre

api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java

@@ -37,7 +37,7 @@ public interface VolumeImportUnmanageService extends PluggableService, Configura
             Arrays.asList(Hypervisor.HypervisorType.KVM, Hypervisor.HypervisorType.VMware);
 
     List<Storage.StoragePoolType> SUPPORTED_STORAGE_POOL_TYPES_FOR_KVM = Arrays.asList(Storage.StoragePoolType.NetworkFilesystem,
-            Storage.StoragePoolType.Filesystem, Storage.StoragePoolType.RBD);
+            Storage.StoragePoolType.Filesystem, Storage.StoragePoolType.RBD, Storage.StoragePoolType.SharedMountPoint);
 
     ConfigKey<Boolean> AllowImportVolumeWithBackingFile = new ConfigKey<>(Boolean.class,
             "allow.import.volume.with.backing.file",

core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java

@@ -34,6 +34,7 @@ public class RestoreBackupCommand extends Command  {
     private List<String> backupVolumesUUIDs;
     private List<PrimaryDataStoreTO> restoreVolumePools;
     private List<String> restoreVolumePaths;
+    private List<Long> restoreVolumeSizes;
     private List<String> backupFiles;
     private String diskType;
     private Boolean vmExists;
@@ -92,6 +93,14 @@ public void setRestoreVolumePaths(List<String> restoreVolumePaths) {
         this.restoreVolumePaths = restoreVolumePaths;
     }
 
+    public List<Long> getRestoreVolumeSizes() {
+        return restoreVolumeSizes;
+    }
+
+    public void setRestoreVolumeSizes(List<Long> restoreVolumeSizes) {
+        this.restoreVolumeSizes = restoreVolumeSizes;
+    }
+
     public List<String> getBackupFiles() {
         return backupFiles;
     }

engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java

@@ -45,4 +45,9 @@ public interface HostTagsDao extends GenericDao<HostTagVO, Long> {
     HostTagResponse newHostTagResponse(HostTagVO hostTag);
 
     List<HostTagVO> searchByIds(Long... hostTagIds);
+
+    /**
+     * List all host tags defined on hosts within a cluster
+     */
+    List<String> listByClusterId(Long clusterId);
 }

engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java

@@ -23,6 +23,7 @@
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.Configurable;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Component;
 
@@ -43,9 +44,12 @@ public class HostTagsDaoImpl extends GenericDaoBase<HostTagVO, Long> implements
     private final SearchBuilder<HostTagVO> stSearch;
     private final SearchBuilder<HostTagVO> tagIdsearch;
     private final SearchBuilder<HostTagVO> ImplicitTagsSearch;
+    private final GenericSearchBuilder<HostTagVO, String> tagSearch;
 
     @Inject
     private ConfigurationDao _configDao;
+    @Inject
+    private HostDao hostDao;
 
     public HostTagsDaoImpl() {
         HostSearch = createSearchBuilder();
@@ -72,6 +76,11 @@ public HostTagsDaoImpl() {
         ImplicitTagsSearch.and("hostId", ImplicitTagsSearch.entity().getHostId(), SearchCriteria.Op.EQ);
         ImplicitTagsSearch.and("isImplicit", ImplicitTagsSearch.entity().getIsImplicit(), SearchCriteria.Op.EQ);
         ImplicitTagsSearch.done();
+
+        tagSearch = createSearchBuilder(String.class);
+        tagSearch.selectFields(tagSearch.entity().getTag());
+        tagSearch.and("hostIdIN", tagSearch.entity().getHostId(), SearchCriteria.Op.IN);
+        tagSearch.done();
     }
 
     @Override
@@ -235,4 +244,15 @@ public List<HostTagVO> searchByIds(Long... tagIds) {
 
         return tagList;
     }
+
+    @Override
+    public List<String> listByClusterId(Long clusterId) {
+        List<Long> hostIds = hostDao.listIdsByClusterId(clusterId);
+        if (CollectionUtils.isEmpty(hostIds)) {
+            return new ArrayList<>();
+        }
+        SearchCriteria<String> sc = tagSearch.create();
+        sc.setParameters("hostIdIN", hostIds.toArray());
+        return customSearch(sc, null);
+    }
 }

engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDaoImpl.java

@@ -178,6 +178,7 @@ protected void init() {
         CountSnapshotsByAccount.select(null, Func.COUNT, null);
         CountSnapshotsByAccount.and(ACCOUNT_ID, CountSnapshotsByAccount.entity().getAccountId(), SearchCriteria.Op.EQ);
         CountSnapshotsByAccount.and(STATUS, CountSnapshotsByAccount.entity().getState(), SearchCriteria.Op.NIN);
+        CountSnapshotsByAccount.and("snapshotTypeNEQ", CountSnapshotsByAccount.entity().getSnapshotType(), SearchCriteria.Op.NIN);
         CountSnapshotsByAccount.and(REMOVED, CountSnapshotsByAccount.entity().getRemoved(), SearchCriteria.Op.NULL);
         CountSnapshotsByAccount.done();
 
@@ -228,6 +229,7 @@ public Long countSnapshotsForAccount(long accountId) {
         SearchCriteria<Long> sc = CountSnapshotsByAccount.create();
         sc.setParameters(ACCOUNT_ID, accountId);
         sc.setParameters(STATUS, State.Error, State.Destroyed);
+        sc.setParameters("snapshotTypeNEQ", Snapshot.Type.GROUP.ordinal());
         return customSearch(sc, null).get(0);
     }
 

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java

@@ -51,6 +51,8 @@ public interface SnapshotDataStoreDao extends GenericDao<SnapshotDataStoreVO, Lo
 
     SnapshotDataStoreVO findBySnapshotIdAndDataStoreRoleAndState(long snapshotId, DataStoreRole role, ObjectInDataStoreStateMachine.State state);
 
+    List<SnapshotDataStoreVO> listBySnapshotIdAndDataStoreRoleAndStateIn(long snapshotId, DataStoreRole role, ObjectInDataStoreStateMachine.State... state);
+
     List<SnapshotDataStoreVO> listReadyByVolumeIdAndCheckpointPathNotNull(long volumeId);
 
     SnapshotDataStoreVO findOneBySnapshotId(long snapshotId, long zoneId);

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java

@@ -68,6 +68,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase<SnapshotDataStoreVO
     protected SearchBuilder<SnapshotDataStoreVO> searchFilteringStoreIdEqStateEqStoreRoleEqIdEqUpdateCountEqSnapshotIdEqVolumeIdEq;
     private SearchBuilder<SnapshotDataStoreVO> stateSearch;
     private SearchBuilder<SnapshotDataStoreVO> idStateNinSearch;
+    private SearchBuilder<SnapshotDataStoreVO> idEqRoleEqStateInSearch;
     protected SearchBuilder<SnapshotVO> snapshotVOSearch;
     private SearchBuilder<SnapshotDataStoreVO> snapshotCreatedSearch;
     private SearchBuilder<SnapshotDataStoreVO> dataStoreAndInstallPathSearch;
@@ -153,6 +154,11 @@ public boolean configure(String name, Map<String, Object> params) throws Configu
         idStateNinSearch.and(STATE, idStateNinSearch.entity().getState(), SearchCriteria.Op.NOTIN);
         idStateNinSearch.done();
 
+        idEqRoleEqStateInSearch = createSearchBuilder();
+        idEqRoleEqStateInSearch.and(SNAPSHOT_ID, idEqRoleEqStateInSearch.entity().getSnapshotId(), SearchCriteria.Op.EQ);
+        idEqRoleEqStateInSearch.and(STORE_ROLE, idEqRoleEqStateInSearch.entity().getRole(), SearchCriteria.Op.EQ);
+        idEqRoleEqStateInSearch.and(STATE, idEqRoleEqStateInSearch.entity().getState(), SearchCriteria.Op.IN);
+
         snapshotVOSearch = snapshotDao.createSearchBuilder();
         snapshotVOSearch.and(VOLUME_ID, snapshotVOSearch.entity().getVolumeId(), SearchCriteria.Op.EQ);
         snapshotVOSearch.done();
@@ -396,6 +402,15 @@ public SnapshotDataStoreVO findBySnapshotIdAndDataStoreRoleAndState(long snapsho
         return findOneBy(sc);
     }
 
+    @Override
+    public List<SnapshotDataStoreVO> listBySnapshotIdAndDataStoreRoleAndStateIn(long snapshotId, DataStoreRole role, State... state) {
+        SearchCriteria<SnapshotDataStoreVO> sc = idEqRoleEqStateInSearch.create();
+        sc.setParameters(SNAPSHOT_ID, snapshotId);
+        sc.setParameters(STORE_ROLE, role);
+        sc.setParameters(STATE, (Object[])state);
+        return listBy(sc);
+    }
+
     @Override
     public SnapshotDataStoreVO findOneBySnapshotId(long snapshotId, long zoneId) {
         try (TransactionLegacy transactionLegacy = TransactionLegacy.currentTxn()) {

engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql

@@ -687,22 +687,23 @@ CREATE TABLE IF NOT EXISTS `cloud`.`backup_details` (
 UPDATE `cloud`.`backups` b
 INNER JOIN `cloud`.`vm_instance` vm ON b.vm_id = vm.id
 SET b.backed_volumes = (
-    SELECT CONCAT("[",
-        GROUP_CONCAT(
-            CONCAT(
-                "{\"uuid\":\"", v.uuid, "\",",
-                "\"type\":\"", v.volume_type, "\",",
-                "\"size\":", v.`size`, ",",
-                "\"path\":\"", IFNULL(v.path, 'null'), "\",",
-                "\"deviceId\":", IFNULL(v.device_id, 'null'), ",",
-                "\"diskOfferingId\":\"", doff.uuid, "\",",
-                "\"minIops\":", IFNULL(v.min_iops, 'null'), ",",
-                "\"maxIops\":", IFNULL(v.max_iops, 'null'),
-                "}"
-            )
-            SEPARATOR ","
+    SELECT COALESCE(
+        CAST(
+            JSON_ARRAYAGG(
+                JSON_OBJECT(
+                     'uuid', v.uuid,
+                     'type', v.volume_type,
+                     'size', v.size,
+                     'path', v.path,
+                     'deviceId', v.device_id,
+                     'diskOfferingId', doff.uuid,
+                     'minIops', v.min_iops,
+                     'maxIops', v.max_iops
+                )
+            ) AS CHAR
         ),
-    "]")
+        '[]'
+    )
     FROM `cloud`.`volumes` v
     LEFT JOIN `cloud`.`disk_offering` doff ON v.disk_offering_id = doff.id
     WHERE v.instance_id = vm.id
@@ -711,22 +712,23 @@ SET b.backed_volumes = (
 -- Add diskOfferingId, deviceId, minIops and maxIops to backup_volumes in vm_instance table
 UPDATE `cloud`.`vm_instance` vm
 SET vm.backup_volumes = (
-    SELECT CONCAT("[",
-        GROUP_CONCAT(
-            CONCAT(
-                "{\"uuid\":\"", v.uuid, "\",",
-                "\"type\":\"", v.volume_type, "\",",
-                "\"size\":", v.`size`, ",",
-                "\"path\":\"", IFNULL(v.path, 'null'), "\",",
-                "\"deviceId\":", IFNULL(v.device_id, 'null'), ",",
-                "\"diskOfferingId\":\"", doff.uuid, "\",",
-                "\"minIops\":", IFNULL(v.min_iops, 'null'), ",",
-                "\"maxIops\":", IFNULL(v.max_iops, 'null'),
-                "}"
-            )
-            SEPARATOR ","
+    SELECT COALESCE(
+        CAST(
+            JSON_ARRAYAGG(
+                JSON_OBJECT(
+                     'uuid', v.uuid,
+                     'type', v.volume_type,
+                     'size', v.size,
+                     'path', v.path,
+                     'deviceId', v.device_id,
+                     'diskOfferingId', doff.uuid,
+                     'minIops', v.min_iops,
+                     'maxIops', v.max_iops
+                )
+            ) AS CHAR
         ),
-    "]")
+        '[]'
+    )
     FROM `cloud`.`volumes` v
     LEFT JOIN `cloud`.`disk_offering` doff ON v.disk_offering_id = doff.id
     WHERE v.instance_id = vm.id

engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql

@@ -18,5 +18,3 @@
 --;
 -- Schema upgrade cleanup from 4.22.0.0 to 4.22.1.0
 --;
-
-DROP VIEW IF EXISTS `cloud`.`account_netstats_view`;

engine/schema/src/main/resources/META-INF/db/views/cloud.account_netstats_view.sql

@@ -0,0 +1,31 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- cloud.account_netstats_view source
+
+
+DROP VIEW IF EXISTS `cloud`.`account_netstats_view`;
+
+CREATE VIEW `cloud`.`account_netstats_view` AS
+select
+    `user_statistics`.`account_id` AS `account_id`,
+    (sum(`user_statistics`.`net_bytes_received`) + sum(`user_statistics`.`current_bytes_received`)) AS `bytesReceived`,
+    (sum(`user_statistics`.`net_bytes_sent`) + sum(`user_statistics`.`current_bytes_sent`)) AS `bytesSent`
+from
+    `user_statistics`
+group by
+    `user_statistics`.`account_id`;