// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.storage.datastore.driver;

import com.linbit.linstor.api.ApiException;
import com.linbit.linstor.api.CloneWaiter;
import com.linbit.linstor.api.DevelopersApi;
import com.linbit.linstor.api.model.ApiCallRc;
import com.linbit.linstor.api.model.ApiCallRcList;
import com.linbit.linstor.api.model.AutoSelectFilter;
import com.linbit.linstor.api.model.LayerType;
import com.linbit.linstor.api.model.Properties;
import com.linbit.linstor.api.model.ResourceDefinition;
import com.linbit.linstor.api.model.ResourceDefinitionCloneRequest;
import com.linbit.linstor.api.model.ResourceDefinitionCloneStarted;
import com.linbit.linstor.api.model.ResourceDefinitionCreate;
import com.linbit.linstor.api.model.ResourceDefinitionModify;
import com.linbit.linstor.api.model.ResourceGroup;
import com.linbit.linstor.api.model.ResourceGroupSpawn;
import com.linbit.linstor.api.model.ResourceMakeAvailable;
import com.linbit.linstor.api.model.ResourceWithVolumes;
import com.linbit.linstor.api.model.Snapshot;
import com.linbit.linstor.api.model.SnapshotRestore;
import com.linbit.linstor.api.model.VolumeDefinition;
import com.linbit.linstor.api.model.VolumeDefinitionModify;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.inject.Inject;

import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;

import com.cloud.agent.api.Answer;
import com.cloud.agent.api.storage.ResizeVolumeAnswer;
import com.cloud.agent.api.storage.ResizeVolumeCommand;
import com.cloud.agent.api.to.DataObjectType;
import com.cloud.agent.api.to.DataStoreTO;
import com.cloud.agent.api.to.DataTO;
import com.cloud.agent.api.to.DiskTO;
import com.cloud.agent.api.to.StorageFilerTO;
import com.cloud.api.storage.LinstorBackupSnapshotCommand;
import com.cloud.api.storage.LinstorRevertBackupSnapshotCommand;
import com.cloud.configuration.Config;
import com.cloud.host.Host;
import com.cloud.host.dao.HostDao;
import com.cloud.resource.ResourceState;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.ResizeVolumePayload;
import com.cloud.storage.SnapshotVO;
import com.cloud.storage.Storage;
import com.cloud.storage.Storage.StoragePoolType;
import com.cloud.storage.StorageManager;
import com.cloud.storage.StoragePool;
import com.cloud.storage.VMTemplateStoragePoolVO;
import com.cloud.storage.VMTemplateStorageResourceAssoc;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.Volume;
import com.cloud.storage.VolumeDetailVO;
import com.cloud.storage.VolumeVO;
import com.cloud.storage.dao.SnapshotDao;
import com.cloud.storage.dao.SnapshotDetailsDao;
import com.cloud.storage.dao.SnapshotDetailsVO;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.dao.VMTemplatePoolDao;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.storage.dao.VolumeDetailsDao;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.Pair;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.VirtualMachineManager;
import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult;
import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreCapabilities;
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
import org.apache.cloudstack.framework.async.AsyncCompletionCallback;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.storage.RemoteHostEndPoint;
import org.apache.cloudstack.storage.command.CommandResult;
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
import org.apache.cloudstack.storage.command.CopyCommand;
import org.apache.cloudstack.storage.command.CreateObjectAnswer;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.storage.datastore.util.LinstorConfigurationManager;
import org.apache.cloudstack.storage.datastore.util.LinstorUtil;
import org.apache.cloudstack.storage.snapshot.SnapshotObject;
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
import org.apache.cloudstack.storage.volume.VolumeObject;

import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import org.apache.commons.collections.CollectionUtils;

import java.nio.charset.StandardCharsets;

public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver {
    protected Logger logger = LogManager.getLogger(getClass());
    @Inject private PrimaryDataStoreDao _storagePoolDao;
    @Inject private VolumeDao _volumeDao;
    @Inject private VolumeDetailsDao _volumeDetailsDao;
    @Inject private VMTemplatePoolDao _vmTemplatePoolDao;
    @Inject private SnapshotDao _snapshotDao;
    @Inject private SnapshotDetailsDao _snapshotDetailsDao;
    @Inject private StorageManager _storageMgr;
    @Inject
    ConfigurationDao _configDao;
    @Inject
    private HostDao _hostDao;
    @Inject private VMTemplateDao _vmTemplateDao;

    private final Map<String, Long> volumeStatsLastUpdate = new HashMap<>();
    private final Map<String, Pair<Long, Long>> volumeStats = new HashMap<>();

    public LinstorPrimaryDataStoreDriverImpl()
    {
    }

    @Override
    public Map<String, String> getCapabilities()
    {
        Map<String, String> mapCapabilities = new HashMap<>();

        // Linstor will be restricted to only run on LVM-THIN and ZFS storage pools with ACS
        // This enables template caching on our primary storage
        mapCapabilities.put(DataStoreCapabilities.CAN_CREATE_VOLUME_FROM_VOLUME.toString(), Boolean.TRUE.toString());

        // fetch if lvm-thin or ZFS
        boolean system_snapshot = !LinstorConfigurationManager.BackupSnapshots.value();
        mapCapabilities.put(DataStoreCapabilities.STORAGE_SYSTEM_SNAPSHOT.toString(), Boolean.toString(system_snapshot));

        // CAN_CREATE_VOLUME_FROM_SNAPSHOT see note from CAN_CREATE_VOLUME_FROM_VOLUME
        mapCapabilities.put(DataStoreCapabilities.CAN_CREATE_VOLUME_FROM_SNAPSHOT.toString(), Boolean.TRUE.toString());
        mapCapabilities.put(DataStoreCapabilities.CAN_REVERT_VOLUME_TO_SNAPSHOT.toString(), Boolean.TRUE.toString());

        return mapCapabilities;
    }

    @Override
    public DataTO getTO(DataObject data)
    {
        return null;
    }

    @Override
    public DataStoreTO getStoreTO(DataStore store)
    {
        return null;
    }

    @Override
    public ChapInfo getChapInfo(DataObject dataObject)
    {
        return null;
    }

    @Override
    public boolean grantAccess(DataObject dataObject, Host host, DataStore dataStore)
    {
        return false;
    }

    @Override
    public void revokeAccess(DataObject dataObject, Host host, DataStore dataStore)
    {
    }

    @Override
    public long getUsedBytes(StoragePool storagePool)
    {
        return 0;
    }

    @Override
    public long getUsedIops(StoragePool storagePool)
    {
        return 0;
    }

    @Override
    public long getDataObjectSizeIncludingHypervisorSnapshotReserve(DataObject dataObject, StoragePool pool)
    {
        return dataObject.getSize();
    }

    @Override
    public long getBytesRequiredForTemplate(TemplateInfo templateInfo, StoragePool storagePool)
    {
        return 0;
    }

    private String getSnapshotName(String snapshotUuid) {
        return LinstorUtil.RSC_PREFIX + snapshotUuid;
    }

    private void deleteResourceDefinition(StoragePoolVO storagePoolVO, String rscDefName)
    {
        DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(storagePoolVO.getHostAddress());

        try
        {
            ApiCallRcList answers = linstorApi.resourceDefinitionDelete(rscDefName);
            if (answers.hasError())
            {
                for (ApiCallRc answer : answers)
                {
                    logger.error(answer.getMessage());
                }
                throw new CloudRuntimeException("Linstor: Unable to delete resource definition: " + rscDefName);
            }
            logger.info("Linstor: Deleted resource {}", rscDefName);
        } catch (ApiException apiEx)
        {
            logger.error("Linstor: ApiEx - " + apiEx.getMessage());
            throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx);
        }
    }

    private void deleteSnapshot(@Nonnull DataStore dataStore, @Nonnull String rscDefName, @Nonnull String snapshotName)
    {
        StoragePoolVO storagePool = _storagePoolDao.findById(dataStore.getId());
        DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(storagePool.getHostAddress());

        try
        {
            ApiCallRcList answers = linstorApi.resourceSnapshotDelete(rscDefName, snapshotName, Collections.emptyList());
            if (answers.hasError())
            {
                for (ApiCallRc answer : answers)
                {
                    logger.error(answer.getMessage());
                }
                throw new CloudRuntimeException("Linstor: Unable to delete snapshot: " + rscDefName);
            }
            logger.info("Linstor: Deleted snapshot " + snapshotName + " for resource " + rscDefName);
        } catch (ApiException apiEx)
        {
            logger.error("Linstor: ApiEx - " + apiEx.getMessage());
            throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx);
        }
    }

    private long getCsIdForCloning(long volumeId, String cloneOf) {
        VolumeDetailVO volumeDetail = _volumeDetailsDao.findDetail(volumeId, cloneOf);

        if (volumeDetail != null && volumeDetail.getValue() != null) {
            return Long.parseLong(volumeDetail.getValue());
        }

        return Long.MIN_VALUE;
    }

    @Override
    public void deleteAsync(DataStore dataStore, DataObject dataObject, AsyncCompletionCallback<CommandResult> callback)
    {
        logger.debug("deleteAsync: " + dataObject.getType() + ";" + dataObject.getUuid());
        String errMsg = null;

        final long storagePoolId = dataStore.getId();
        final StoragePoolVO storagePool = _storagePoolDao.findById(storagePoolId);

        switch (dataObject.getType()) {
            case VOLUME:
            {
                final VolumeInfo volumeInfo = (VolumeInfo) dataObject;
                // if volume creation wasn't completely done .setPath wasn't called, so we fallback to vol.getUuid()
                final String volUuid = volumeInfo.getPath() != null ? volumeInfo.getPath() : volumeInfo.getUuid();
                final String rscName = LinstorUtil.RSC_PREFIX + volUuid;
                deleteResourceDefinition(storagePool, rscName);

                long usedBytes = storagePool.getUsedBytes();
                Long capacityIops = storagePool.getCapacityIops();

                if (capacityIops != null)
                {
                    if (volumeInfo.getMaxIops() != null)
                        capacityIops += volumeInfo.getMaxIops();
                    storagePool.setCapacityIops(Math.max(0, capacityIops));
                }

                usedBytes -= volumeInfo.getSize();
                storagePool.setUsedBytes(Math.max(0, usedBytes));

                _storagePoolDao.update(storagePoolId, storagePool);
            }
                break;
            case SNAPSHOT:
                final SnapshotInfo snapshotInfo = (SnapshotInfo) dataObject;
                final String rscName = LinstorUtil.RSC_PREFIX + snapshotInfo.getBaseVolume().getPath();
                deleteSnapshot(dataStore, rscName, getSnapshotName(snapshotInfo.getUuid()));
                long usedBytes = storagePool.getUsedBytes() - snapshotInfo.getSize();
                storagePool.setUsedBytes(Math.max(0, usedBytes));
                _storagePoolDao.update(storagePoolId, storagePool);
                break;
            default:
                errMsg = "Invalid DataObjectType (" + dataObject.getType() + ") passed to deleteAsync";
                logger.error(errMsg);
        }

        if (callback != null) {
            CommandResult result = new CommandResult();
            result.setResult(errMsg);

            callback.complete(result);
        }
    }

    private void logLinstorAnswer(@Nonnull ApiCallRc answer) {
        if (answer.isError()) {
            logger.error(answer.getMessage());
        } else if (answer.isWarning()) {
            logger.warn(answer.getMessage());
        } else if (answer.isInfo()) {
            logger.info(answer.getMessage());
        }
    }

    private void logLinstorAnswers(@Nonnull ApiCallRcList answers) {
        answers.forEach(this::logLinstorAnswer);
    }

    private void checkLinstorAnswersThrow(@Nonnull ApiCallRcList answers) {
        logLinstorAnswers(answers);
        if (answers.hasError())
        {
            String errMsg = answers.stream()
                .filter(ApiCallRc::isError)
                .findFirst()
                .map(ApiCallRc::getMessage).orElse("Unknown linstor error");
            throw new CloudRuntimeException(errMsg);
        }
    }

    private String checkLinstorAnswers(@Nonnull ApiCallRcList answers) {
        logLinstorAnswers(answers);
        return answers.stream().filter(ApiCallRc::isError).findFirst().map(ApiCallRc::getMessage).orElse(null);
    }

    private void applyQoSSettings(StoragePoolVO storagePool, DevelopersApi api, String rscName, Long maxIops)
        throws ApiException
    {
        Long currentQosIops = null;
        List<VolumeDefinition> vlmDfns = api.volumeDefinitionList(rscName, null, null);
        if (!vlmDfns.isEmpty())
        {
            Properties props = vlmDfns.get(0).getProps();
            long iops = Long.parseLong(props.getOrDefault("sys/fs/blkio_throttle_write_iops", "0"));
            currentQosIops = iops > 0 ? iops : null;
        }

        if (!Objects.equals(maxIops, currentQosIops))
        {
            VolumeDefinitionModify vdm = new VolumeDefinitionModify();
            if (maxIops != null)
            {
                Properties props = new Properties();
                props.put("sys/fs/blkio_throttle_read_iops", "" + maxIops);
                props.put("sys/fs/blkio_throttle_write_iops", "" + maxIops);
                vdm.overrideProps(props);
                logger.info("Apply qos setting: " + maxIops + " to " + rscName);
            }
            else
            {
                logger.info("Remove QoS setting for " + rscName);
                vdm.deleteProps(Arrays.asList("sys/fs/blkio_throttle_read_iops", "sys/fs/blkio_throttle_write_iops"));
            }
            ApiCallRcList answers = api.volumeDefinitionModify(rscName, 0, vdm);
            checkLinstorAnswersThrow(answers);

            Long capacityIops = storagePool.getCapacityIops();
            if (capacityIops != null)
            {
                long vcIops = currentQosIops != null ? currentQosIops * -1 : 0;
                long vMaxIops = maxIops != null ? maxIops : 0;
                long newIops = vcIops + vMaxIops;
                capacityIops -= newIops;
                logger.info(String.format("Current storagepool %s iops capacity:  %d", storagePool, capacityIops));
                storagePool.setCapacityIops(Math.max(0, capacityIops));
                _storagePoolDao.update(storagePool.getId(), storagePool);
            }
        }
    }

    private String getRscGrp(StoragePool storagePool) {
        return storagePool.getUserInfo() != null && !storagePool.getUserInfo().isEmpty() ?
            storagePool.getUserInfo() : "DfltRscGrp";
    }

    /**
     * Returns the layerlist of the resourceGroup with encryption(LUKS) added above STORAGE.
     * If the resourceGroup layer list already contains LUKS this layer list will be returned.
     * @param api Linstor developers API
     * @param resourceGroup Resource group to get the encryption layer list
     * @return layer list with LUKS added
     */
    public List<LayerType> getEncryptedLayerList(DevelopersApi api, String resourceGroup) {
        try {
            List<ResourceGroup> rscGrps = api.resourceGroupList(
                    Collections.singletonList(resourceGroup), Collections.emptyList(), null, null);

            if (CollectionUtils.isEmpty(rscGrps)) {
                throw new CloudRuntimeException(
                        String.format("Resource Group %s not found on Linstor cluster.", resourceGroup));
            }

            final ResourceGroup rscGrp = rscGrps.get(0);
            List<LayerType> layers = Arrays.asList(LayerType.DRBD, LayerType.LUKS, LayerType.STORAGE);
            List<String> curLayerStack = rscGrp.getSelectFilter() != null ?
                    rscGrp.getSelectFilter().getLayerStack() : Collections.emptyList();
            if (CollectionUtils.isNotEmpty(curLayerStack)) {
                layers = curLayerStack.stream().map(LayerType::valueOf).collect(Collectors.toList());
                if (!layers.contains(LayerType.LUKS)) {
                    layers.add(layers.size() - 1, LayerType.LUKS); // lowest layer is STORAGE
                }
            }
            return layers;
        } catch (ApiException e) {
            throw new CloudRuntimeException(
                    String.format("Resource Group %s not found on Linstor cluster.", resourceGroup));
        }
    }

    /**
     * Spawns a new Linstor resource with the given arguments.
     * @param api
     * @param newRscName
     * @param sizeInBytes
     * @param isTemplate
     * @param rscGrpName
     * @param volName
     * @param vmName
     * @throws ApiException
     */
    private void spawnResource(
            DevelopersApi api, String newRscName, long sizeInBytes, boolean isTemplate, String rscGrpName,
            String volName, String vmName, @Nullable Long passPhraseId, @Nullable byte[] passPhrase) throws ApiException
    {
        ResourceGroupSpawn rscGrpSpawn = new ResourceGroupSpawn();
        rscGrpSpawn.setResourceDefinitionName(newRscName);
        rscGrpSpawn.addVolumeSizesItem(sizeInBytes / 1024);
        if (passPhraseId != null) {
            AutoSelectFilter asf = new AutoSelectFilter();
            List<LayerType> luksLayers = getEncryptedLayerList(api, rscGrpName);
            asf.setLayerStack(luksLayers.stream().map(LayerType::toString).collect(Collectors.toList()));
            rscGrpSpawn.setSelectFilter(asf);
            if (passPhrase != null) {
                String utf8Passphrase = new String(passPhrase, StandardCharsets.UTF_8);
                rscGrpSpawn.setVolumePassphrases(Collections.singletonList(utf8Passphrase));
            }
        }

        if (isTemplate) {
            Properties props = new Properties();
            props.put(LinstorUtil.getTemplateForAuxPropKey(rscGrpName), "true");
            rscGrpSpawn.setResourceDefinitionProps(props);
        }

        logger.info("Linstor: Spawn resource " + newRscName);
        ApiCallRcList answers = api.resourceGroupSpawn(rscGrpName, rscGrpSpawn);
        checkLinstorAnswersThrow(answers);

        answers = LinstorUtil.applyAuxProps(api, newRscName, volName, vmName);
        checkLinstorAnswersThrow(answers);
    }

    /**
     * Condition if a template resource can be shared with the given resource group.
     * @param tgtRscGrp
     * @param tgtLayerStack
     * @param rg
     * @return True if the template resource can be shared, else false.
     */
    private boolean canShareTemplateForResourceGroup(
            ResourceGroup tgtRscGrp, List<String> tgtLayerStack, ResourceGroup rg) {
        List<String> rgLayerStack = rg.getSelectFilter() != null ?
                rg.getSelectFilter().getLayerStack() : null;
        return Objects.equals(tgtLayerStack, rgLayerStack) &&
                Objects.equals(tgtRscGrp.getSelectFilter().getStoragePoolList(),
                        rg.getSelectFilter().getStoragePoolList());
    }

    /**
     * Searches for a shareable template for this rscGrpName and sets the aux template property.
     * @param api
     * @param rscName
     * @param rscGrpName
     * @param existingRDs
     * @return
     * @throws ApiException
     */
    private boolean foundShareableTemplate(
            DevelopersApi api, String rscName, String rscGrpName,
            List<Pair<ResourceDefinition, ResourceGroup>> existingRDs) throws ApiException {
        if (!existingRDs.isEmpty()) {
            ResourceGroup tgtRscGrp = api.resourceGroupList(
                    Collections.singletonList(rscGrpName), null, null, null).get(0);
            List<String> tgtLayerStack = tgtRscGrp.getSelectFilter() != null ?
                    tgtRscGrp.getSelectFilter().getLayerStack() : null;

            // check if there is already a template copy, that we could reuse
            // this means if select filters are similar enough to allow cloning from
            for (Pair<ResourceDefinition, ResourceGroup> rdPair : existingRDs) {
                ResourceGroup rg = rdPair.second();
                if (canShareTemplateForResourceGroup(tgtRscGrp, tgtLayerStack, rg)) {
                    LinstorUtil.setAuxTemplateForProperty(api, rscName, rscGrpName);
                    return true;
                }
            }
        }
        return false;
    }

    /**
     * Creates a new Linstor resource.
     * @param rscName
     * @param sizeInBytes
     * @param volName
     * @param vmName
     * @param api
     * @param rscGrp
     * @param poolId
     * @param isTemplate indicates if the resource is a template
     * @return true if a new resource was created, false if it already existed or was reused.
     */
    private boolean createResourceBase(
        String rscName, long sizeInBytes, String volName, String vmName,
        @Nullable Long passPhraseId, @Nullable byte[] passPhrase, DevelopersApi api,
        String rscGrp, long poolId, boolean isTemplate)
    {
        try
        {
            logger.debug("createRscBase: {} :: {} :: {}", rscName, rscGrp, isTemplate);
            List<Pair<ResourceDefinition, ResourceGroup>> existingRDs = LinstorUtil.getRDAndRGListStartingWith(api, rscName);

            String fullRscName = String.format("%s-%d", rscName, poolId);
            boolean alreadyCreated = existingRDs.stream()
                    .anyMatch(p -> p.first().getName().equalsIgnoreCase(fullRscName)) ||
                    existingRDs.stream().anyMatch(p -> p.first().getProps().containsKey(LinstorUtil.getTemplateForAuxPropKey(rscGrp)));
            if (!alreadyCreated) {
                boolean createNewRsc = !foundShareableTemplate(api, rscName, rscGrp, existingRDs);
                if (createNewRsc) {
                    String newRscName = existingRDs.isEmpty() ? rscName : fullRscName;
                    spawnResource(api, newRscName, sizeInBytes, isTemplate, rscGrp,
                            volName, vmName, passPhraseId, passPhrase);
                }
                return createNewRsc;
            }
            return false;
        } catch (ApiException apiEx)
        {
            logger.error("Linstor: ApiEx - " + apiEx.getMessage());
            throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx);
        }
    }

    private String createResource(VolumeInfo vol, StoragePoolVO storagePoolVO) {
        DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(storagePoolVO.getHostAddress());
        final String rscGrp = getRscGrp(storagePoolVO);

        final String rscName = LinstorUtil.RSC_PREFIX + vol.getUuid();
        createResourceBase(
            rscName, vol.getSize(), vol.getName(), vol.getAttachedVmName(), vol.getPassphraseId(), vol.getPassphrase(),
                linstorApi, rscGrp, storagePoolVO.getId(), false);

        try
        {
            applyQoSSettings(storagePoolVO, linstorApi, rscName, vol.getMaxIops());

            return LinstorUtil.getDevicePath(linstorApi, rscName);
        } catch (ApiException apiEx)
        {
            logger.error("Linstor: ApiEx - " + apiEx.getMessage());
            throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx);
        }
    }

    private void resizeResource(DevelopersApi api, String resourceName, long sizeByte) throws ApiException {
        VolumeDefinitionModify dfm = new VolumeDefinitionModify();
        dfm.setSizeKib(sizeByte / 1024);

        ApiCallRcList answers = api.volumeDefinitionModify(resourceName, 0, dfm);
        if (answers.hasError()) {
            logger.error("Resize error: " + answers.get(0).getMessage());
            throw new CloudRuntimeException(answers.get(0).getMessage());
        } else {
            logger.info(String.format("Successfully resized %s to %d kib", resourceName, dfm.getSizeKib()));
        }
    }

    /**
     * Update resource-definitions resource-group to the correct one if it isn't already the intended.
     * @param api Linstor api
     * @param rscName resource name to check the resource group
     * @param tgtRscGrp resource group name to set
     * @throws ApiException exception if any api error occurred
     */
    private void updateRscGrpIfNecessary(DevelopersApi api, String rscName, String tgtRscGrp) throws ApiException {
        List<ResourceDefinition> rscDfns = api.resourceDefinitionList(
                Collections.singletonList(rscName), false, null, null, null);
        if (rscDfns != null && !rscDfns.isEmpty()) {
            ResourceDefinition rscDfn = rscDfns.get(0);

            if (!rscDfn.getResourceGroupName().equalsIgnoreCase(tgtRscGrp)) {
                ResourceDefinitionModify rdm = new ResourceDefinitionModify();
                rdm.setResourceGroup(tgtRscGrp);
                ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm);

                if (answers.hasError()) {
                    String bestError = LinstorUtil.getBestErrorMessage(answers);
                    logger.error("Update resource group on {} error: {}", rscName, bestError);
                    throw new CloudRuntimeException(bestError);
                } else {
                    logger.info("Successfully changed resource group to {} on {}", tgtRscGrp, rscName);
                }
            }
        }
    }

    /**
     * If a resource is cloned, all properties are cloned too, but the _cs-template-for properties,
     * should only stay on the template resource, so delete them in this method.
     * @param api
     * @param rscName
     * @throws ApiException
     */
    private void deleteTemplateForProps(
            DevelopersApi api, String rscName) throws ApiException {
        List<ResourceDefinition> rdList = api.resourceDefinitionList(
                Collections.singletonList(rscName), false, null, null, null);

        if (CollectionUtils.isNotEmpty(rdList)) {
            ResourceDefinitionModify rdm = new ResourceDefinitionModify();
            List<String> deleteProps = rdList.get(0).getProps().keySet().stream()
                    .filter(key -> key.startsWith("Aux/" + LinstorUtil.CS_TEMPLATE_FOR_PREFIX))
                    .collect(Collectors.toList());
            rdm.deleteProps(deleteProps);
            ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm);
            checkLinstorAnswers(answers);
        }
    }

    private String cloneResource(long csCloneId, VolumeInfo volumeInfo, StoragePoolVO storagePoolVO) {
        // get the cached template on this storage
        VMTemplateStoragePoolVO tmplPoolRef = _vmTemplatePoolDao.findByPoolTemplate(
            storagePoolVO.getId(), csCloneId, null);

        if (tmplPoolRef != null) {
            final String templateRscName;
            if (tmplPoolRef.getLocalDownloadPath() == null) {
                VMTemplateVO vmTemplateVO = _vmTemplateDao.findById(tmplPoolRef.getTemplateId());
                templateRscName = LinstorUtil.RSC_PREFIX + vmTemplateVO.getUuid();
            } else {
                templateRscName = LinstorUtil.RSC_PREFIX + tmplPoolRef.getLocalDownloadPath();
            }
            final String rscName = LinstorUtil.RSC_PREFIX + volumeInfo.getUuid();

            final DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(storagePoolVO.getHostAddress());

            try {
                ResourceDefinition templateRD = LinstorUtil.findResourceDefinition(
                        linstorApi, templateRscName, getRscGrp(storagePoolVO));
                final String cloneRes = templateRD != null ? templateRD.getName() : templateRscName;
                logger.info("Clone resource definition {} to {}", cloneRes, rscName);
                ResourceDefinitionCloneRequest cloneRequest = new ResourceDefinitionCloneRequest();
                cloneRequest.setName(rscName);
                if (volumeInfo.getPassphraseId() != null) {
                    List<LayerType> encryptionLayer = getEncryptedLayerList(linstorApi, getRscGrp(storagePoolVO));
                    cloneRequest.setLayerList(encryptionLayer);
                    if (volumeInfo.getPassphrase() != null) {
                        String utf8Passphrase = new String(volumeInfo.getPassphrase(), StandardCharsets.UTF_8);
                        cloneRequest.setVolumePassphrases(Collections.singletonList(utf8Passphrase));
                    }
                }
                ResourceDefinitionCloneStarted cloneStarted = linstorApi.resourceDefinitionClone(
                    cloneRes, cloneRequest);

                checkLinstorAnswersThrow(cloneStarted.getMessages());

                if (!CloneWaiter.waitFor(linstorApi, cloneStarted)) {
                    throw new CloudRuntimeException("Clone for resource " + rscName + " failed.");
                }

                logger.info("Clone resource definition " + cloneRes + " to " + rscName + " finished");

                if (volumeInfo.getSize() != null && volumeInfo.getSize() > 0) {
                    resizeResource(linstorApi, rscName, volumeInfo.getSize());
                }

                updateRscGrpIfNecessary(linstorApi, rscName, getRscGrp(storagePoolVO));

                deleteTemplateForProps(linstorApi, rscName);
                LinstorUtil.applyAuxProps(linstorApi, rscName, volumeInfo.getName(), volumeInfo.getAttachedVmName());
                applyQoSSettings(storagePoolVO, linstorApi, rscName, volumeInfo.getMaxIops());

                return LinstorUtil.getDevicePath(linstorApi, rscName);
            } catch (ApiException apiEx) {
                logger.error("Linstor: ApiEx - " + apiEx.getMessage());
                throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx);
            }
        } else {
            throw new CloudRuntimeException(
                "Unable to find Linstor resource for the following template data-object ID: " + csCloneId);
        }
    }

    private ResourceDefinitionCreate createResourceDefinitionCreate(String rscName, String rscGrpName)
            throws ApiException {
        ResourceDefinitionCreate rdCreate = new ResourceDefinitionCreate();
        ResourceDefinition rd = new ResourceDefinition();
        rd.setName(rscName);
        rd.setResourceGroupName(rscGrpName);
        rdCreate.setResourceDefinition(rd);
        return rdCreate;
    }

    private String createResourceFromSnapshot(long csSnapshotId, String rscName, StoragePoolVO storagePoolVO) {
        final String rscGrp = getRscGrp(storagePoolVO);
        final DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(storagePoolVO.getHostAddress());

        SnapshotVO snapshotVO = _snapshotDao.findById(csSnapshotId);
        String snapName = LinstorUtil.RSC_PREFIX + snapshotVO.getUuid();
        VolumeVO volumeVO = _volumeDao.findById(snapshotVO.getVolumeId());
        String cloneRes = LinstorUtil.RSC_PREFIX + volumeVO.getPath();

        try
        {
            logger.debug("Create new resource definition: " + rscName);
            ResourceDefinitionCreate rdCreate = createResourceDefinitionCreate(rscName, rscGrp);
            ApiCallRcList answers = linstorApi.resourceDefinitionCreate(rdCreate);
            checkLinstorAnswersThrow(answers);

            SnapshotRestore snapshotRestore = new SnapshotRestore();
            snapshotRestore.toResource(rscName);

            logger.debug("Create new volume definition for snapshot: " + cloneRes + ":" + snapName);
            answers = linstorApi.resourceSnapshotsRestoreVolumeDefinition(cloneRes, snapName, snapshotRestore);
            checkLinstorAnswersThrow(answers);

            // restore snapshot to new resource
            logger.info("Restore resource from snapshot: " + cloneRes + ":" + snapName);
            answers = linstorApi.resourceSnapshotRestore(cloneRes, snapName, snapshotRestore);
            checkLinstorAnswersThrow(answers);

            LinstorUtil.applyAuxProps(linstorApi, rscName, volumeVO.getName(), null);
            applyQoSSettings(storagePoolVO, linstorApi, rscName, volumeVO.getMaxIops());

            return LinstorUtil.getDevicePath(linstorApi, rscName);
        } catch (ApiException apiEx) {
            logger.error("Linstor: ApiEx - " + apiEx.getMessage());
            throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx);
        }
    }

    private String createVolume(VolumeInfo volumeInfo, StoragePoolVO storagePoolVO) {
        long csSnapshotId = getCsIdForCloning(volumeInfo.getId(), "cloneOfSnapshot");
        long csTemplateId = getCsIdForCloning(volumeInfo.getId(), "cloneOfTemplate");

        if (csSnapshotId > 0) {
            return createResourceFromSnapshot(csSnapshotId, LinstorUtil.RSC_PREFIX + volumeInfo.getUuid(), storagePoolVO);
        } else if (csTemplateId > 0) {
            return cloneResource(csTemplateId, volumeInfo, storagePoolVO);
        } else {
            return createResource(volumeInfo, storagePoolVO);
        }
    }

    private void handleSnapshotDetails(long csSnapshotId, String name, String value) {
        _snapshotDetailsDao.removeDetail(csSnapshotId, name);
        SnapshotDetailsVO snapshotDetails = new SnapshotDetailsVO(csSnapshotId, name, value, false);
        _snapshotDetailsDao.persist(snapshotDetails);
    }

    private void addTempVolumeToDb(long csSnapshotId, String tempVolumeName) {
        // TEMP_VOLUME_ID is needed, to find which temporary resource should be deleted after copying it on agent side
        handleSnapshotDetails(csSnapshotId, LinstorUtil.TEMP_VOLUME_ID, LinstorUtil.RSC_PREFIX + tempVolumeName);
        // the iqn will be used on the agent side to copy from, even though linstor doesn't have anything to do with IQN
        handleSnapshotDetails(csSnapshotId, DiskTO.IQN, tempVolumeName);
    }

    private void removeTempVolumeFromDb(long csSnapshotId) {
        SnapshotDetailsVO snapshotDetails = _snapshotDetailsDao.findDetail(csSnapshotId, LinstorUtil.TEMP_VOLUME_ID);

        if (snapshotDetails == null || snapshotDetails.getValue() == null) {
            throw new CloudRuntimeException(
                "'removeTempVolumeId' should not be invoked unless " + LinstorUtil.TEMP_VOLUME_ID + " exists.");
        }

        String originalVolumeId = snapshotDetails.getValue();

        handleSnapshotDetails(csSnapshotId, LinstorUtil.TEMP_VOLUME_ID, originalVolumeId);

        _snapshotDetailsDao.remove(snapshotDetails.getId());
    }

    private void createVolumeFromSnapshot(SnapshotInfo snapshotInfo, StoragePoolVO storagePoolVO) {
        long csSnapshotId = snapshotInfo.getId();

        SnapshotDetailsVO snapshotDetails = _snapshotDetailsDao.findDetail(csSnapshotId, "tempVolume");

        if (snapshotDetails != null && snapshotDetails.getValue() != null &&
            snapshotDetails.getValue().equalsIgnoreCase("create"))
        {
            final String csName = "Temp-" + snapshotInfo.getUuid();
            final String tempRscName = LinstorUtil.RSC_PREFIX + csName;
            createResourceFromSnapshot(csSnapshotId, tempRscName, storagePoolVO);

            logger.debug("Temp resource created: " + tempRscName);
            addTempVolumeToDb(csSnapshotId, csName);
        }
        else if (snapshotDetails != null && snapshotDetails.getValue() != null &&
            snapshotDetails.getValue().equalsIgnoreCase("delete"))
        {
            snapshotDetails = _snapshotDetailsDao.findDetail(csSnapshotId, LinstorUtil.TEMP_VOLUME_ID);

            deleteResourceDefinition(storagePoolVO, snapshotDetails.getValue());

            logger.debug("Temp resource deleted: " + snapshotDetails.getValue());
            removeTempVolumeFromDb(csSnapshotId);
        }
        else {
            throw new CloudRuntimeException("Invalid state in 'createVolumeFromSnapshot(SnapshotInfo, StoragePoolVO)'");
        }
    }

    @Override
    public void createAsync(DataStore dataStore, DataObject vol, AsyncCompletionCallback<CreateCmdResult> callback)
    {
        String devPath = null;
        String errMsg = null;
        StoragePoolVO storagePool = _storagePoolDao.findById(dataStore.getId());

        try
        {
            switch (vol.getType())
            {
                case VOLUME:
                    VolumeInfo volumeInfo = (VolumeInfo) vol;
                    VolumeVO volume = _volumeDao.findById(volumeInfo.getId());
                    logger.debug("createAsync - creating volume");
                    devPath = createVolume(volumeInfo, storagePool);
                    volume.setFolder("/dev/");
                    volume.setPoolId(storagePool.getId());
                    volume.setPoolType(storagePool.getPoolType());
                    volume.setUuid(vol.getUuid());
                    volume.setPath(vol.getUuid());

                    _volumeDao.update(volume.getId(), volume);
                    break;
                case SNAPSHOT:
                    logger.debug("createAsync - SNAPSHOT");
                    createVolumeFromSnapshot((SnapshotInfo) vol, storagePool);
                    break;
                case TEMPLATE:
                    errMsg = "creating template - not supported";
                    logger.error("createAsync - " + errMsg);
                    break;
                default:
                    errMsg = "Invalid DataObjectType (" + vol.getType() + ") passed to createAsync";
                    logger.error(errMsg);
            }
        } catch (Exception ex)
        {
            errMsg = ex.getMessage();

            logger.error("createAsync: " + errMsg);
            if (callback == null)
            {
                throw ex;
            }
        }

        if (callback != null)
        {
            CreateCmdResult result = new CreateCmdResult(devPath, new Answer(null, errMsg == null, errMsg));
            result.setResult(errMsg);
            callback.complete(result);
        }
    }

    private String revertSnapshotFromImageStore(
        final SnapshotInfo snapshot,
        final VolumeInfo volumeInfo,
        final DevelopersApi linstorApi,
        final String rscName)
    throws ApiException {
        String resultMsg = null;
        String value = _configDao.getValue(Config.BackupSnapshotWait.toString());
        int _backupsnapshotwait = NumbersUtil.parseInt(
            value, Integer.parseInt(Config.BackupSnapshotWait.getDefaultValue()));

        LinstorRevertBackupSnapshotCommand cmd = new LinstorRevertBackupSnapshotCommand(
            snapshot.getTO(),
            volumeInfo.getTO(),
            _backupsnapshotwait,
            VirtualMachineManager.ExecuteInSequence.value());

        Optional<RemoteHostEndPoint> optEP = getDiskfullEP(linstorApi, rscName);
        if (optEP.isEmpty()) {
            optEP = getLinstorEP(linstorApi, rscName);
        }

        if (optEP.isPresent()) {
            Answer answer = optEP.get().sendMessage(cmd);
            if (!answer.getResult()) {
                resultMsg = answer.getDetails();
            }
        } else {
            resultMsg = "Unable to get matching Linstor endpoint.";
        }
        return resultMsg;
    }

    private String doRevertSnapshot(final SnapshotInfo snapshot, final VolumeInfo volumeInfo) {
        final StoragePool pool = (StoragePool) volumeInfo.getDataStore();
        final DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(pool.getHostAddress());
        final String rscName = LinstorUtil.RSC_PREFIX + volumeInfo.getPath();
        String resultMsg;
        try {
            if (snapshot.getDataStore().getRole() == DataStoreRole.Primary) {
                final String snapName = LinstorUtil.RSC_PREFIX + snapshot.getUuid();

                ApiCallRcList answers = linstorApi.resourceSnapshotRollback(rscName, snapName);
                resultMsg = checkLinstorAnswers(answers);
            } else if (snapshot.getDataStore().getRole() == DataStoreRole.Image) {
                resultMsg = revertSnapshotFromImageStore(snapshot, volumeInfo, linstorApi, rscName);
            } else {
                resultMsg = "Linstor: Snapshot revert datastore not supported";
            }
        } catch (ApiException apiEx) {
            logger.error("Linstor: ApiEx - " + apiEx.getMessage());
            resultMsg = apiEx.getBestMessage();
        }

        return resultMsg;
    }

    @Override
    public void revertSnapshot(
        SnapshotInfo snapshot,
        SnapshotInfo snapshotOnPrimaryStore,
        AsyncCompletionCallback<CommandResult> callback)
    {
        logger.debug("Linstor: revertSnapshot");
        final VolumeInfo volumeInfo = snapshot.getBaseVolume();
        VolumeVO volumeVO = _volumeDao.findById(volumeInfo.getId());
        if (volumeVO == null || volumeVO.getRemoved() != null) {
            CommandResult commandResult = new CommandResult();
            commandResult.setResult("The volume that the snapshot belongs to no longer exists.");
            callback.complete(commandResult);
            return;
        }

        String resultMsg = doRevertSnapshot(snapshot, volumeInfo);

        if (callback != null)
        {
            CommandResult result = new CommandResult();
            result.setResult(resultMsg);
            callback.complete(result);
        }
    }

    private static boolean canCopySnapshotCond(DataObject srcData, DataObject dstData) {
        return srcData.getType() == DataObjectType.SNAPSHOT && dstData.getType() == DataObjectType.SNAPSHOT
            && (dstData.getDataStore().getRole() == DataStoreRole.Image
            || dstData.getDataStore().getRole() == DataStoreRole.ImageCache);
    }

    private static boolean canCopyTemplateCond(DataObject srcData, DataObject dstData) {
        return srcData.getType() == DataObjectType.TEMPLATE && dstData.getType() == DataObjectType.TEMPLATE
            && dstData.getDataStore().getRole() == DataStoreRole.Primary
            && (srcData.getDataStore().getRole() == DataStoreRole.Image
            || srcData.getDataStore().getRole() == DataStoreRole.ImageCache);
    }

    private static boolean canCopyVolumeCond(DataObject srcData, DataObject dstData) {
        // Volume download from Linstor primary storage
        return srcData.getType() == DataObjectType.VOLUME
                && (dstData.getType() == DataObjectType.VOLUME || dstData.getType() == DataObjectType.TEMPLATE)
                && srcData.getDataStore().getRole() == DataStoreRole.Primary
                && (dstData.getDataStore().getRole() == DataStoreRole.Image
                || dstData.getDataStore().getRole() == DataStoreRole.ImageCache);
    }

    @Override
    public boolean canCopy(DataObject srcData, DataObject dstData)
    {
        logger.debug("LinstorPrimaryDataStoreDriverImpl.canCopy: " + srcData.getType() + " -> " + dstData.getType());

        if (canCopySnapshotCond(srcData, dstData)) {
            SnapshotInfo sinfo = (SnapshotInfo) srcData;
            VolumeInfo volume = sinfo.getBaseVolume();
            StoragePoolVO storagePool = _storagePoolDao.findById(volume.getPoolId());
            return storagePool.getStorageProviderName().equals(LinstorUtil.PROVIDER_NAME);
        } else if (canCopyTemplateCond(srcData, dstData)) {
            TemplateInfo tInfo = (TemplateInfo) dstData;
            StoragePoolVO storagePoolVO = _storagePoolDao.findById(dstData.getDataStore().getId());
            return storagePoolVO != null
                && storagePoolVO.getPoolType() == Storage.StoragePoolType.Linstor
                && tInfo.getSize() != null;
        } else if (canCopyVolumeCond(srcData, dstData)) {
            VolumeInfo srcVolInfo = (VolumeInfo) srcData;
            StoragePoolVO storagePool = _storagePoolDao.findById(srcVolInfo.getPoolId());
            return storagePool.getStorageProviderName().equals(LinstorUtil.PROVIDER_NAME);
        }
        return false;
    }

    @Override
    public void copyAsync(DataObject srcData, DataObject dstData, AsyncCompletionCallback<CopyCommandResult> callback)
    {
        logger.debug("LinstorPrimaryDataStoreDriverImpl.copyAsync: "
            + srcData.getType() + " -> " + dstData.getType());

        final CopyCommandResult res;
        if (canCopySnapshotCond(srcData, dstData)) {
            String errMsg = null;
            Answer answer = copySnapshot(srcData, dstData);
            if (answer != null && !answer.getResult()) {
                errMsg = answer.getDetails();
            } else {
                // delete primary storage snapshot
                SnapshotInfo sinfo = (SnapshotInfo) srcData;
                VolumeInfo volume = sinfo.getBaseVolume();
                deleteSnapshot(
                    srcData.getDataStore(),
                    LinstorUtil.RSC_PREFIX + volume.getPath(),
                    LinstorUtil.RSC_PREFIX + sinfo.getUuid());
            }
            res = new CopyCommandResult(null, answer);
            res.setResult(errMsg);
        } else if (canCopyTemplateCond(srcData, dstData)) {
            Answer answer = copyTemplate(srcData, dstData);
            res = new CopyCommandResult(null, answer);
        } else if (canCopyVolumeCond(srcData, dstData)) {
            Answer answer = copyVolume(srcData, dstData);
            res = new CopyCommandResult(null, answer);
        } else {
            Answer answer = new Answer(null, false, "noimpl");
            res = new CopyCommandResult(null, answer);
            res.setResult("Not implemented yet");
        }
        callback.complete(res);
    }

    /**
     * Tries to get a Linstor cloudstack end point, that is at least diskless.
     *
     * @param api Linstor java api object
     * @param rscName resource name to make available on node
     * @return Optional RemoteHostEndPoint if one could get found.
     * @throws ApiException
     */
    private Optional<RemoteHostEndPoint> getLinstorEP(DevelopersApi api, String rscName) throws ApiException {
        List<String> linstorNodeNames = LinstorUtil.getLinstorNodeNames(api);
        Collections.shuffle(linstorNodeNames);  // do not always pick the first linstor node

        Host host = null;
        for (String nodeName : linstorNodeNames) {
            host = _hostDao.findByName(nodeName);
            if (host != null && host.getResourceState() == ResourceState.Enabled) {
                logger.info(String.format("Linstor: Make resource %s available on node %s ...", rscName, nodeName));
                ApiCallRcList answers = api.resourceMakeAvailableOnNode(rscName, nodeName, new ResourceMakeAvailable());
                if (!answers.hasError()) {
                    break; // found working host
                } else {
                    logger.error(
                        String.format("Linstor: Unable to make resource %s on node %s available: %s",
                            rscName,
                            nodeName,
                            LinstorUtil.getBestErrorMessage(answers)));
                }
            }
        }

        if (host == null)
        {
            logger.error("Linstor: Couldn't create a resource on any cloudstack host.");
            return Optional.empty();
        }
        else
        {
            return Optional.of(RemoteHostEndPoint.getHypervisorHostEndPoint(host));
        }
    }

    private Optional<RemoteHostEndPoint> getDiskfullEP(DevelopersApi api, String rscName) throws ApiException {
        List<com.linbit.linstor.api.model.StoragePool> linSPs = LinstorUtil.getDiskfulStoragePools(api, rscName);
        if (linSPs != null) {
            for (com.linbit.linstor.api.model.StoragePool sp : linSPs) {
                Host host = _hostDao.findByName(sp.getNodeName());
                if (host != null && host.getResourceState() == ResourceState.Enabled) {
                    return Optional.of(RemoteHostEndPoint.getHypervisorHostEndPoint(host));
                }
            }
        }
        logger.error("Linstor: No diskfull host found.");
        return Optional.empty();
    }

    private String restoreResourceFromSnapshot(
            DevelopersApi api,
            StoragePoolVO storagePoolVO,
            String rscName,
            String snapshotName,
            String restoredName) throws ApiException {
        final String rscGrp = getRscGrp(storagePoolVO);
        // try to delete -rst resource, could happen if the copy failed and noone deleted it.
        deleteResourceDefinition(storagePoolVO, restoredName);
        ResourceDefinitionCreate rdc = createResourceDefinitionCreate(restoredName, rscGrp);
        api.resourceDefinitionCreate(rdc);

        SnapshotRestore sr = new SnapshotRestore();
        sr.toResource(restoredName);
        api.resourceSnapshotsRestoreVolumeDefinition(rscName, snapshotName, sr);

        api.resourceSnapshotRestore(rscName, snapshotName, sr);

        return LinstorUtil.getDevicePath(api, restoredName);
    }

    /**
     * Updates the template_spool_ref DB entry to indicate that this template was fully downloaded and is ready.
     * @param templateId
     * @param destTemplateInfoUuid
     * @param destDataStoreId
     * @param templateSize
     */
    private void updateTemplateSpoolRef(
            long templateId, String destTemplateInfoUuid, long destDataStoreId, long templateSize) {
        VMTemplateStoragePoolVO destVolumeTemplateStoragePoolVO = _vmTemplatePoolDao.findByPoolTemplate(
                destDataStoreId, templateId, null);
        if (destVolumeTemplateStoragePoolVO == null) {
            throw new CloudRuntimeException(
                    String.format("Unable to find template_spool_ref entry for pool_id %d and template_id %d",
                            destDataStoreId, templateId));
        }
        destVolumeTemplateStoragePoolVO.setDownloadPercent(100);
        destVolumeTemplateStoragePoolVO.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED);
        destVolumeTemplateStoragePoolVO.setState(ObjectInDataStoreStateMachine.State.Ready);
        destVolumeTemplateStoragePoolVO.setTemplateSize(templateSize);
        destVolumeTemplateStoragePoolVO.setLocalDownloadPath(destTemplateInfoUuid);
        destVolumeTemplateStoragePoolVO.setInstallPath(destTemplateInfoUuid);
        _vmTemplatePoolDao.persist(destVolumeTemplateStoragePoolVO);
    }

    private Answer copyTemplate(DataObject srcData, DataObject dstData) {
        TemplateInfo tInfo = (TemplateInfo) dstData;
        final StoragePoolVO pool = _storagePoolDao.findById(dstData.getDataStore().getId());
        final DevelopersApi api = LinstorUtil.getLinstorAPI(pool.getHostAddress());
        final String rscName = LinstorUtil.RSC_PREFIX + dstData.getUuid();
        boolean newCreated = createResourceBase(
            LinstorUtil.RSC_PREFIX + dstData.getUuid(),
            tInfo.getSize(),
            tInfo.getName(),
            "",
            null,
            null,
            api,
            getRscGrp(pool),
            pool.getId(),
            true);

        Answer answer;
        if (newCreated) {
            int nMaxExecutionMinutes = NumbersUtil.parseInt(
                    _configDao.getValue(Config.SecStorageCmdExecutionTimeMax.key()), 30);
            CopyCommand cmd = new CopyCommand(
                    srcData.getTO(),
                    dstData.getTO(),
                    nMaxExecutionMinutes * 60 * 1000,
                    VirtualMachineManager.ExecuteInSequence.value());

            try {
                Optional<RemoteHostEndPoint> optEP = getLinstorEP(api, rscName);
                if (optEP.isPresent()) {
                    answer = optEP.get().sendMessage(cmd);
                } else {
                    answer = new Answer(cmd, false, "Unable to get matching Linstor endpoint.");
                    deleteResourceDefinition(pool, rscName);
                }
            } catch (ApiException exc) {
                logger.error("copy template failed: ", exc);
                deleteResourceDefinition(pool, rscName);
                throw new CloudRuntimeException(exc.getBestMessage());
            }
        } else {
            updateTemplateSpoolRef(dstData.getId(), tInfo.getUuid(), dstData.getDataStore().getId(), srcData.getSize());
            answer = new Answer(new CopyCmdAnswer(dstData.getTO()));
        }
        return answer;
    }

    private Answer copyVolume(DataObject srcData, DataObject dstData) {
        VolumeInfo srcVolInfo = (VolumeInfo) srcData;
        final StoragePoolVO pool = _storagePoolDao.findById(srcVolInfo.getDataStore().getId());
        final DevelopersApi api = LinstorUtil.getLinstorAPI(pool.getHostAddress());
        final String rscName = LinstorUtil.RSC_PREFIX + srcVolInfo.getPath();

        VolumeObjectTO to = (VolumeObjectTO) srcVolInfo.getTO();
        // patch source format
        // Linstor volumes are stored as RAW, but we can't set the correct format as RAW (we use QCOW2)
        // otherwise create template from snapshot won't work, because this operation
        // uses the format of the base volume and we backup snapshots as QCOW2
        // https://github.com/apache/cloudstack/pull/8802#issuecomment-2024019927
        to.setFormat(Storage.ImageFormat.RAW);
        int nMaxExecutionSeconds = NumbersUtil.parseInt(
                _configDao.getValue(Config.CopyVolumeWait.key()), 10800);
        CopyCommand cmd = new CopyCommand(
                to,
                dstData.getTO(),
                nMaxExecutionSeconds,
                VirtualMachineManager.ExecuteInSequence.value());
        Answer answer;

        try {
            Optional<RemoteHostEndPoint> optEP = getLinstorEP(api, rscName);
            if (optEP.isPresent()) {
                answer = optEP.get().sendMessage(cmd);
            }
            else {
                answer = new Answer(cmd, false, "Unable to get matching Linstor endpoint.");
            }
        } catch (ApiException exc) {
            logger.error("copy volume failed: ", exc);
            throw new CloudRuntimeException(exc.getBestMessage());
        }
        return answer;
    }

    /**
     * Create a temporary resource from the snapshot to backup, so we can copy the data on a diskless agent
     * @param api Linstor Developer api object
     * @param pool StoragePool this resource resides on
     * @param rscName rscName of the snapshotted resource
     * @param snapshotName Name of the snapshot to copy from
     * @param snapshotObject snapshot object of the origCmd, so the path can be modified
     * @param origCmd original LinstorBackupSnapshotCommand that needs to have a patched path
     * @return answer from agent operation
     * @throws ApiException if any Linstor api operation fails
     */
    private Answer copyFromTemporaryResource(
            DevelopersApi api,
            StoragePoolVO pool,
            String rscName,
            String snapshotName,
            SnapshotObject snapshotObject,
            CopyCommand origCmd)
            throws ApiException {
        Answer answer;
        String restoreName = rscName + "-rst";
        try {
            String devName = restoreResourceFromSnapshot(api, pool, rscName, snapshotName, restoreName);

            Optional<RemoteHostEndPoint> optEPAny = getLinstorEP(api, restoreName);
            if (optEPAny.isPresent()) {
                // patch the src device path to the temporary linstor resource
                snapshotObject.setPath(devName);
                origCmd.setSrcTO(snapshotObject.getTO());
                answer = optEPAny.get().sendMessage(origCmd);
            } else{
                answer = new Answer(origCmd, false, "Unable to get matching Linstor endpoint.");
            }
        } finally {
            // delete the temporary resource, noop if already gone
            api.resourceDefinitionDelete(restoreName);
        }
        return answer;
    }

    /**
     * vmsnapshots don't have our typical snapshot path set
     * instead the path is the internal snapshot name e.g.: {vm}_VS_{datestr}
     * we have to find out and modify the path here before
     * @return the original snapshotObject.getPath()
     */
    private String setCorrectSnapshotPath(DevelopersApi api, String rscName, SnapshotObject snapshotObject)
            throws ApiException {
        String originalPath = LinstorUtil.RSC_PREFIX + snapshotObject.getUuid();
        if (!(snapshotObject.getPath().startsWith("/dev/mapper/") ||
                snapshotObject.getPath().startsWith("zfs://"))) {
            originalPath = snapshotObject.getPath();
            com.linbit.linstor.api.model.StoragePool linStoragePool =
                    LinstorUtil.getDiskfulStoragePool(api, rscName);
            if (linStoragePool == null) {
                throw new CloudRuntimeException("Linstor: Unable to find storage pool for resource " + rscName);
            }
            final String path = LinstorUtil.getSnapshotPath(linStoragePool, rscName, snapshotObject.getPath());
            snapshotObject.setPath(path);
        }
        return originalPath;
    }

    protected Answer copySnapshot(DataObject srcData, DataObject destData) {
        String value = _configDao.getValue(Config.BackupSnapshotWait.toString());
        int _backupsnapshotwait = NumbersUtil.parseInt(
            value, Integer.parseInt(Config.BackupSnapshotWait.getDefaultValue()));

        SnapshotObject snapshotObject = (SnapshotObject)srcData;
        Boolean snapshotFullBackup = snapshotObject.getFullBackup();
        final StoragePoolVO pool = _storagePoolDao.findById(srcData.getDataStore().getId());
        final DevelopersApi api = LinstorUtil.getLinstorAPI(pool.getHostAddress());
        boolean fullSnapshot = true;
        if (snapshotFullBackup != null) {
            fullSnapshot = snapshotFullBackup;
        }
        Map<String, String> options = new HashMap<>();
        options.put("fullSnapshot", fullSnapshot + "");
        options.put(SnapshotInfo.BackupSnapshotAfterTakingSnapshot.key(),
            String.valueOf(SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value()));
        options.put("volumeSize", snapshotObject.getBaseVolume().getSize() + "");

        try {
            final String rscName = LinstorUtil.RSC_PREFIX + snapshotObject.getBaseVolume().getPath();
            String snapshotName = setCorrectSnapshotPath(api, rscName, snapshotObject);

            CopyCommand cmd = new LinstorBackupSnapshotCommand(
                snapshotObject.getTO(),
                destData.getTO(),
                _backupsnapshotwait,
                VirtualMachineManager.ExecuteInSequence.value());
            cmd.setOptions(options);

            Optional<RemoteHostEndPoint> optEP = getDiskfullEP(api, rscName);
            Answer answer;
            if (optEP.isPresent()) {
                answer = optEP.get().sendMessage(cmd);
            } else {
                logger.debug("No diskfull endpoint found to copy image, creating diskless endpoint");
                answer = copyFromTemporaryResource(api, pool, rscName, snapshotName, snapshotObject, cmd);
            }
            return answer;
        } catch (Exception e) {
            logger.debug("copy snapshot failed, please cleanup snapshot manually: ", e);
            throw new CloudRuntimeException(e.toString());
        }

    }

    @Override
    public void copyAsync(DataObject srcData, DataObject destData, Host destHost, AsyncCompletionCallback<CopyCommandResult> callback)
    {
        // as long as canCopy is false, this isn't called
        logger.debug("Linstor: copyAsync with host");
        copyAsync(srcData, destData, callback);
    }

    private CreateCmdResult notifyResize(
        VolumeObject vol,
        long oldSize,
        ResizeVolumePayload resizeParameter)
    {
        StoragePool pool = (StoragePool) vol.getDataStore();

        ResizeVolumeCommand resizeCmd =
            new ResizeVolumeCommand(vol.getPath(), new StorageFilerTO(pool), oldSize, resizeParameter.newSize, resizeParameter.shrinkOk,
                resizeParameter.instanceName, null);
        CreateCmdResult result = new CreateCmdResult(null, null);
        try {
            ResizeVolumeAnswer answer = (ResizeVolumeAnswer) _storageMgr.sendToPool(pool, resizeParameter.hosts, resizeCmd);
            if (answer != null && answer.getResult()) {
                logger.debug("Resize: notified hosts");
            } else if (answer != null) {
                result.setResult(answer.getDetails());
            } else {
                logger.debug("return a null answer, mark it as failed for unknown reason");
                result.setResult("return a null answer, mark it as failed for unknown reason");
            }

        } catch (Exception e) {
            logger.debug("sending resize command failed", e);
            result.setResult(e.toString());
        }

        return result;
    }

    @Override
    public void resize(DataObject data, AsyncCompletionCallback<CreateCmdResult> callback)
    {
        final VolumeObject vol = (VolumeObject) data;
        final StoragePoolVO pool = _storagePoolDao.findById(data.getDataStore().getId());
        final DevelopersApi api = LinstorUtil.getLinstorAPI(pool.getHostAddress());
        final ResizeVolumePayload resizeParameter = (ResizeVolumePayload) vol.getpayload();

        final String rscName = LinstorUtil.RSC_PREFIX + vol.getPath();
        final long oldSize = vol.getSize();

        String errMsg = null;
        VolumeDefinitionModify dfm = new VolumeDefinitionModify();
        dfm.setSizeKib(resizeParameter.newSize / 1024);
        try
        {
            resizeResource(api, rscName, resizeParameter.newSize);

            applyQoSSettings(pool, api, rscName, resizeParameter.newMaxIops);
            {
                final VolumeVO volume = _volumeDao.findById(vol.getId());
                volume.setMinIops(resizeParameter.newMinIops);
                volume.setMaxIops(resizeParameter.newMaxIops);
                volume.setSize(resizeParameter.newSize);
                _volumeDao.update(volume.getId(), volume);
            }
        } catch (ApiException apiExc)
        {
            logger.error(apiExc);
            errMsg = apiExc.getBestMessage();
        }

        CreateCmdResult result;
        if (errMsg != null) {
            result = new CreateCmdResult(null, new Answer(null, false, errMsg));
            result.setResult(errMsg);
        } else {
            // notify guests
            result = notifyResize(vol, oldSize, resizeParameter);
        }

        callback.complete(result);
    }

    @Override
    public void handleQualityOfServiceForVolumeMigration(
        VolumeInfo volumeInfo,
        QualityOfServiceState qualityOfServiceState)
    {
        logger.debug("Linstor: handleQualityOfServiceForVolumeMigration");
    }

    private Answer createAnswerAndPerstistDetails(DevelopersApi api, SnapshotInfo snapshotInfo, String rscName)
        throws ApiException {
        SnapshotObjectTO snapshotTO = (SnapshotObjectTO)snapshotInfo.getTO();
        com.linbit.linstor.api.model.StoragePool linStoragePool = LinstorUtil.getDiskfulStoragePool(api, rscName);
        if (linStoragePool == null) {
            throw new CloudRuntimeException("Linstor: Unable to find storage pool for resource " + rscName);
        }

        final String path = LinstorUtil.getSnapshotPath(linStoragePool, rscName, LinstorUtil.RSC_PREFIX + snapshotInfo.getUuid());
        snapshotTO.setPath(path);
        SnapshotDetailsVO details = new SnapshotDetailsVO(
            snapshotInfo.getId(), snapshotInfo.getUuid(), path, false);
        _snapshotDetailsDao.persist(details);

        return new CreateObjectAnswer(snapshotTO);
    }

    @Override
    public void takeSnapshot(SnapshotInfo snapshotInfo, AsyncCompletionCallback<CreateCmdResult> callback)
    {
        logger.debug(String.format("Linstor: takeSnapshot with snapshot: %s", snapshotInfo.getSnapshotVO()));

        final VolumeInfo volumeInfo = snapshotInfo.getBaseVolume();
        final VolumeVO volumeVO = _volumeDao.findById(volumeInfo.getId());

        long storagePoolId = volumeVO.getPoolId();
        final StoragePoolVO storagePool = _storagePoolDao.findById(storagePoolId);
        final DevelopersApi api = LinstorUtil.getLinstorAPI(storagePool.getHostAddress());
        final String rscName = LinstorUtil.RSC_PREFIX + volumeVO.getPath();

        Snapshot snapshot = new Snapshot();
        snapshot.setName(getSnapshotName(snapshotInfo.getUuid()));

        CreateCmdResult result;
        try
        {
            ApiCallRcList answers = api.resourceSnapshotCreate(rscName, snapshot);

            if (answers.hasError())
            {
                final String errMsg = answers.get(0).getMessage();
                logger.error("Snapshot error: " + errMsg);
                result = new CreateCmdResult(null, new Answer(null, false, errMsg));
                result.setResult(errMsg);
            } else
            {
                logger.info(String.format("Successfully took snapshot %s from %s", snapshot.getName(), rscName));

                Answer answer = createAnswerAndPerstistDetails(api, snapshotInfo, rscName);

                result = new CreateCmdResult(null, answer);
                result.setResult(null);
            }
        } catch (ApiException apiExc)
        {
            logger.error(apiExc);
            result = new CreateCmdResult(null, new Answer(null, false, apiExc.getBestMessage()));
            result.setResult(apiExc.getBestMessage());
        }

        callback.complete(result);
    }

    @Override
    public boolean canProvideStorageStats() {
        return true;
    }

    @Override
    public Pair<Long, Long> getStorageStats(StoragePool storagePool) {
        logger.debug(String.format("Requesting storage stats: %s", storagePool));
        return LinstorUtil.getStorageStats(storagePool.getHostAddress(), getRscGrp(storagePool));
    }

    @Override
    public boolean canProvideVolumeStats() {
        return LinstorConfigurationManager.VolumeStatsCacheTime.value() > 0;
    }

    /**
     * Updates the cache map containing current allocated size data.
     * @param linstorAddr Linstor cluster api address
     */
    private void fillVolumeStatsCache(String linstorAddr) {
        final DevelopersApi api = LinstorUtil.getLinstorAPI(linstorAddr);
        try {
            logger.trace("Start volume stats cache update for " + linstorAddr);
            List<ResourceWithVolumes> resources = api.viewResources(
                    Collections.emptyList(),
                    Collections.emptyList(),
                    Collections.emptyList(),
                    null,
                    null,
                    null);

            List<ResourceDefinition> rscDfns = api.resourceDefinitionList(
                    Collections.emptyList(), true, null, null, null);

            HashMap<String, Long> resSizeMap = new HashMap<>();
            for (ResourceDefinition rscDfn : rscDfns) {
                if (CollectionUtils.isNotEmpty(rscDfn.getVolumeDefinitions())) {
                    resSizeMap.put(rscDfn.getName(), rscDfn.getVolumeDefinitions().get(0).getSizeKib() * 1024);
                }
            }

            HashMap<String, Long> allocSizeMap = new HashMap<>();
            for (ResourceWithVolumes rsc : resources) {
                if (!LinstorUtil.isRscDiskless(rsc) && !rsc.getVolumes().isEmpty()) {
                    long allocatedBytes = allocSizeMap.getOrDefault(rsc.getName(), 0L);
                    allocSizeMap.put(rsc.getName(), Math.max(allocatedBytes, rsc.getVolumes().get(0).getAllocatedSizeKib() * 1024));
                }
            }

            volumeStats.keySet().removeIf(key -> key.startsWith(linstorAddr));
            for (Map.Entry<String, Long> entry : allocSizeMap.entrySet()) {
                Long reserved = resSizeMap.getOrDefault(entry.getKey(), 0L);
                Pair<Long, Long> volStat = new Pair<>(entry.getValue(), reserved);
                volumeStats.put(linstorAddr + "/" + entry.getKey(), volStat);
            }
            volumeStatsLastUpdate.put(linstorAddr, System.currentTimeMillis());
            logger.debug(String.format("Done volume stats cache update for %s: %d", linstorAddr, volumeStats.size()));
        } catch (ApiException e) {
            logger.error("Unable to fetch Linstor resources: {}", e.getBestMessage());
        }
    }

    @Override
    public Pair<Long, Long> getVolumeStats(StoragePool storagePool, String volumeId) {
        String linstorAddr = storagePool.getHostAddress();
        synchronized (volumeStats) {
            long invalidateCacheTime = volumeStatsLastUpdate.getOrDefault(storagePool.getHostAddress(), 0L) +
                    LinstorConfigurationManager.VolumeStatsCacheTime.value() * 1000;
            if (invalidateCacheTime < System.currentTimeMillis()) {
                fillVolumeStatsCache(storagePool.getHostAddress());
            }
            String volumeKey = linstorAddr + "/" + LinstorUtil.RSC_PREFIX + volumeId;
            Pair<Long, Long> sizePair = volumeStats.get(volumeKey);
            if (sizePair == null) {
                logger.warn(String.format("Volumestats for %s not found in cache", volumeKey));
            }
            return sizePair;
        }
    }

    @Override
    public boolean canHostAccessStoragePool(Host host, StoragePool pool) {
        return true;
    }

    @Override
    public boolean isVmInfoNeeded() {
        return false;
    }

    @Override
    public void provideVmInfo(long vmId, long volumeId) {
    }

    @Override
    public boolean isVmTagsNeeded(String tagKey) {
        return false;
    }

    @Override
    public void provideVmTags(long vmId, long volumeId, String tagValue) {
    }

    @Override
    public boolean isStorageSupportHA(StoragePoolType type) {
        return true;
    }

    @Override
    public void detachVolumeFromAllStorageNodes(Volume volume) {
    }
}
