Skip to content

Support Linstor Primary Storage for NAS BnR #12796

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
abh1sar wants to merge 1 commit into
base: 4.22
Choose a base branch
from
Open

Support Linstor Primary Storage for NAS BnR #12796

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ public class RestoreBackupCommand extends Command {
private List<String> backupVolumesUUIDs;
private List<PrimaryDataStoreTO> restoreVolumePools;
private List<String> restoreVolumePaths;
private List<Long> restoreVolumeSizes;
private List<String> backupFiles;
private String diskType;
private Boolean vmExists;
Expand Down Expand Up @@ -92,6 +93,14 @@ public void setRestoreVolumePaths(List<String> restoreVolumePaths) {
this.restoreVolumePaths = restoreVolumePaths;
}

public List<Long> getRestoreVolumeSizes() {
return restoreVolumeSizes;
}

public void setRestoreVolumeSizes(List<Long> restoreVolumeSizes) {
this.restoreVolumeSizes = restoreVolumeSizes;
}

public List<String> getBackupFiles() {
return backupFiles;
}
Expand Down
21 changes: 17 additions & 4 deletions plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -350,7 +350,8 @@ private Pair<List<PrimaryDataStoreTO>, List<String>> getVolumePoolsAndPaths(List
volumePools.add(dataStore != null ? (PrimaryDataStoreTO)dataStore.getTO() : null);

String volumePathPrefix = getVolumePathPrefix(storagePool);
volumePaths.add(String.format("%s/%s", volumePathPrefix, volume.getPath()));
String volumePathSuffix = getVolumePathSuffix(storagePool);
volumePaths.add(String.format("%s%s%s", volumePathPrefix, volume.getPath(), volumePathSuffix));
}
return new Pair<>(volumePools, volumePaths);
}
Expand All @@ -360,14 +361,24 @@ private String getVolumePathPrefix(StoragePoolVO storagePool) {
if (ScopeType.HOST.equals(storagePool.getScope()) ||
Storage.StoragePoolType.SharedMountPoint.equals(storagePool.getPoolType()) ||
Storage.StoragePoolType.RBD.equals(storagePool.getPoolType())) {
volumePathPrefix = storagePool.getPath();
volumePathPrefix = storagePool.getPath() + "/";
} else if (Storage.StoragePoolType.Linstor.equals(storagePool.getPoolType())) {
volumePathPrefix = "/dev/drbd/by-res/cs-";
Copy link
Contributor

@sureshanaparti sureshanaparti Mar 12, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can keep this path in LinstorUtil, and use it here

Copy link
Contributor Author

@abh1sar abh1sar Mar 13, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can't access LinstorUtil from here.

} else {
// Should be Storage.StoragePoolType.NetworkFilesystem
volumePathPrefix = String.format("/mnt/%s", storagePool.getUuid());
volumePathPrefix = String.format("/mnt/%s/", storagePool.getUuid());
}
return volumePathPrefix;
}

private String getVolumePathSuffix(StoragePoolVO storagePool) {
if (Storage.StoragePoolType.Linstor.equals(storagePool.getPoolType())) {
return "/0";
} else {
return "";
}
}

@Override
public Pair<Boolean, String> restoreBackedUpVolume(Backup backup, Backup.VolumeInfo backupVolumeInfo, String hostIp, String dataStoreUuid, Pair<String, VirtualMachine.State> vmNameAndState) {
final VolumeVO volume = volumeDao.findByUuid(backupVolumeInfo.getUuid());
Expand Down Expand Up @@ -412,7 +423,9 @@ public Pair<Boolean, String> restoreBackedUpVolume(Backup backup, Backup.VolumeI
restoreCommand.setBackupRepoType(backupRepository.getType());
restoreCommand.setBackupRepoAddress(backupRepository.getAddress());
restoreCommand.setVmName(vmNameAndState.first());
restoreCommand.setRestoreVolumePaths(Collections.singletonList(String.format("%s/%s", getVolumePathPrefix(pool), volumeUUID)));
String restoreVolumePath = String.format("%s%s%s", getVolumePathPrefix(pool), volumeUUID, getVolumePathSuffix(pool));
restoreCommand.setRestoreVolumePaths(Collections.singletonList(restoreVolumePath));
restoreCommand.setRestoreVolumeSizes(Collections.singletonList(volume.getSize()));
DataStore dataStore = dataStoreMgr.getDataStore(pool.getId(), DataStoreRole.Primary);
restoreCommand.setRestoreVolumePools(Collections.singletonList(dataStore != null ? (PrimaryDataStoreTO)dataStore.getTO() : null));
restoreCommand.setDiskType(backupVolumeInfo.getType().name().toLowerCase(Locale.ROOT));
Expand Down
88 changes: 62 additions & 26 deletions ...in/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,9 @@
import org.apache.commons.lang3.StringUtils;
import org.libvirt.LibvirtException;

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.Locale;
Expand All @@ -56,10 +56,25 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper<RestoreBa
private static final String UMOUNT_COMMAND = "sudo umount %s";
private static final String FILE_PATH_PLACEHOLDER = "%s/%s";
private static final String ATTACH_QCOW2_DISK_COMMAND = " virsh attach-disk %s %s %s --driver qemu --subdriver qcow2 --cache none";
private static final String ATTACH_RAW_DISK_COMMAND = " virsh attach-disk %s %s %s --driver qemu --cache none";
private static final String ATTACH_RBD_DISK_XML_COMMAND = " virsh attach-device %s /dev/stdin <<EOF%sEOF";
private static final String CURRRENT_DEVICE = "virsh domblklist --domain %s | tail -n 3 | head -n 1 | awk '{print $1}'";
private static final String RSYNC_COMMAND = "rsync -az %s %s";

private String getVolumeUuidFromPath(String volumePath, PrimaryDataStoreTO volumePool) {
if (Storage.StoragePoolType.Linstor.equals(volumePool.getPoolType())) {
Path path = Paths.get(volumePath);
String rscName = path.getParent().getFileName().toString();
if (rscName.startsWith("cs-")) {
rscName = rscName.substring(3);
}
return rscName;
} else {
int lastIndex = volumePath.lastIndexOf("/");
return volumePath.substring(lastIndex + 1);
}
}

@Override
public Answer execute(RestoreBackupCommand command, LibvirtComputingResource serverResource) {
String vmName = command.getVmName();
Expand All @@ -84,9 +99,9 @@ public Answer execute(RestoreBackupCommand command, LibvirtComputingResource ser
PrimaryDataStoreTO volumePool = restoreVolumePools.get(0);
String volumePath = restoreVolumePaths.get(0);
String backupFile = backupFiles.get(0);
int lastIndex = volumePath.lastIndexOf("/");
newVolumeId = volumePath.substring(lastIndex + 1);
restoreVolume(storagePoolMgr, backupPath, volumePool, volumePath, diskType, backupFile,
newVolumeId = getVolumeUuidFromPath(volumePath, volumePool);
Long size = command.getRestoreVolumeSizes().get(0);
restoreVolume(storagePoolMgr, backupPath, volumePool, volumePath, diskType, backupFile, size,
new Pair<>(vmName, command.getVmState()), mountDirectory, timeout);
} else if (Boolean.TRUE.equals(vmExists)) {
restoreVolumesOfExistingVM(storagePoolMgr, restoreVolumePools, restoreVolumePaths, backedVolumeUUIDs, backupPath, backupFiles, mountDirectory, timeout);
Expand Down Expand Up @@ -143,7 +158,7 @@ private void restoreVolumesOfDestroyedVMs(KVMStoragePoolManager storagePoolMgr,
String volumePath = volumePaths.get(i);
String backupFile = backupFiles.get(i);
String bkpPath = getBackupPath(mountDirectory, backupPath, backupFile, diskType);
String volumeUuid = volumePath.substring(volumePath.lastIndexOf(File.separator) + 1);
String volumeUuid = getVolumeUuidFromPath(volumePath, volumePool);
diskType = "datadisk";
verifyBackupFile(bkpPath, volumeUuid);
if (!replaceVolumeWithBackup(storagePoolMgr, volumePool, volumePath, bkpPath, timeout)) {
Expand All @@ -157,14 +172,14 @@ private void restoreVolumesOfDestroyedVMs(KVMStoragePoolManager storagePoolMgr,
}

private void restoreVolume(KVMStoragePoolManager storagePoolMgr, String backupPath, PrimaryDataStoreTO volumePool, String volumePath, String diskType, String backupFile,
Pair<String, VirtualMachine.State> vmNameAndState, String mountDirectory, int timeout) {
Long size, Pair<String, VirtualMachine.State> vmNameAndState, String mountDirectory, int timeout) {
String bkpPath;
String volumeUuid;
try {
bkpPath = getBackupPath(mountDirectory, backupPath, backupFile, diskType);
volumeUuid = volumePath.substring(volumePath.lastIndexOf(File.separator) + 1);
volumeUuid = getVolumeUuidFromPath(volumePath, volumePool);
verifyBackupFile(bkpPath, volumeUuid);
if (!replaceVolumeWithBackup(storagePoolMgr, volumePool, volumePath, bkpPath, timeout, true)) {
if (!replaceVolumeWithBackup(storagePoolMgr, volumePool, volumePath, bkpPath, timeout, true, size)) {
throw new CloudRuntimeException(String.format("Unable to restore contents from the backup volume [%s].", volumeUuid));

}
Expand Down Expand Up @@ -247,28 +262,38 @@ private boolean checkBackupPathExists(String backupPath) {
}

private boolean replaceVolumeWithBackup(KVMStoragePoolManager storagePoolMgr, PrimaryDataStoreTO volumePool, String volumePath, String backupPath, int timeout) {
return replaceVolumeWithBackup(storagePoolMgr, volumePool, volumePath, backupPath, timeout, false);
return replaceVolumeWithBackup(storagePoolMgr, volumePool, volumePath, backupPath, timeout, false, null);
}

private boolean replaceVolumeWithBackup(KVMStoragePoolManager storagePoolMgr, PrimaryDataStoreTO volumePool, String volumePath, String backupPath, int timeout, boolean createTargetVolume) {
if (volumePool.getPoolType() != Storage.StoragePoolType.RBD) {
int exitValue = Script.runSimpleBashScriptForExitValue(String.format(RSYNC_COMMAND, backupPath, volumePath));
return exitValue == 0;
private boolean replaceVolumeWithBackup(KVMStoragePoolManager storagePoolMgr, PrimaryDataStoreTO volumePool, String volumePath, String backupPath, int timeout, boolean createTargetVolume, Long size) {
if (List.of(Storage.StoragePoolType.RBD, Storage.StoragePoolType.Linstor).contains(volumePool.getPoolType())) {
return replaceBlockDeviceWithBackup(storagePoolMgr, volumePool, volumePath, backupPath, timeout, createTargetVolume, size);
}

return replaceRbdVolumeWithBackup(storagePoolMgr, volumePool, volumePath, backupPath, timeout, createTargetVolume);
int exitValue = Script.runSimpleBashScriptForExitValue(String.format(RSYNC_COMMAND, backupPath, volumePath));
return exitValue == 0;
}

private boolean replaceRbdVolumeWithBackup(KVMStoragePoolManager storagePoolMgr, PrimaryDataStoreTO volumePool, String volumePath, String backupPath, int timeout, boolean createTargetVolume) {
private boolean replaceBlockDeviceWithBackup(KVMStoragePoolManager storagePoolMgr, PrimaryDataStoreTO volumePool, String volumePath, String backupPath, int timeout, boolean createTargetVolume, Long size) {
KVMStoragePool volumeStoragePool = storagePoolMgr.getStoragePool(volumePool.getPoolType(), volumePool.getUuid());
QemuImg qemu;
try {
qemu = new QemuImg(timeout * 1000, true, false);
if (!createTargetVolume) {
KVMPhysicalDisk rdbDisk = volumeStoragePool.getPhysicalDisk(volumePath);
logger.debug("Restoring RBD volume: {}", rdbDisk.toString());
String volumeUuid = getVolumeUuidFromPath(volumePath, volumePool);
KVMPhysicalDisk disk = null;
if (createTargetVolume) {
if (Storage.StoragePoolType.Linstor.equals(volumePool.getPoolType())) {
disk = volumeStoragePool.createPhysicalDisk(volumeUuid, QemuImg.PhysicalDiskFormat.RAW, Storage.ProvisioningType.THIN, size, null);
}
} else {
if (Storage.StoragePoolType.Linstor.equals(volumePool.getPoolType())) {
storagePoolMgr.connectPhysicalDisk(volumePool.getPoolType(), volumePool.getUuid(), volumeUuid, null);
} else {
disk = volumeStoragePool.getPhysicalDisk(getVolumeUuidFromPath(volumePath, volumePool));
Copy link
Contributor Author

@abh1sar abh1sar Mar 13, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@sureshanaparti can you please review from RBD's regression perspective?

volumeStoragePool.getPhysicalDisk takes UUID as an argument. previously it was passing path

Copy link
Contributor

@sureshanaparti sureshanaparti Mar 13, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@abh1sar usually, better to lookup by volume path, not the uuid. I checked in my earlier testing, the uuid and path are the same for volumes on ceph, we've to check after migrate volume, any other operation which can changes the path and not uuid. to avoid any issue, can you check for StoragePoolType.RBD and get disk by volume path?

}
qemu.setSkipTargetVolumeCreation(true);
}
logger.debug("Restoring volume: {}", disk.toString());
} catch (LibvirtException ex) {
throw new CloudRuntimeException("Failed to create qemu-img command to restore RBD volume with backup", ex);
}
Expand All @@ -277,12 +302,21 @@ private boolean replaceRbdVolumeWithBackup(KVMStoragePoolManager storagePoolMgr,
QemuImgFile destVolumeFile = null;
try {
srcBackupFile = new QemuImgFile(backupPath, QemuImg.PhysicalDiskFormat.QCOW2);
String rbdDestVolumeFile = KVMPhysicalDisk.RBDStringBuilder(volumeStoragePool, volumePath);
destVolumeFile = new QemuImgFile(rbdDestVolumeFile, QemuImg.PhysicalDiskFormat.RAW);

logger.debug("Starting convert backup {} to RBD volume {}", backupPath, volumePath);
String destVolume;
switch(volumePool.getPoolType()) {
case Linstor:
destVolume = volumePath;
break;
case RBD:
destVolume = KVMPhysicalDisk.RBDStringBuilder(volumeStoragePool, volumePath);
break;
default:
throw new CloudRuntimeException(String.format("Unsupported storage pool type [%s] for block device restore with backup.", volumePool.getPoolType()));
}
destVolumeFile = new QemuImgFile(destVolume, QemuImg.PhysicalDiskFormat.RAW);
logger.debug("Starting convert backup {} to volume {}", backupPath, volumePath);
qemu.convert(srcBackupFile, destVolumeFile);
logger.debug("Successfully converted backup {} to RBD volume {}", backupPath, volumePath);
logger.debug("Successfully converted backup {} to volume {}", backupPath, volumePath);
} catch (QemuImgException | LibvirtException e) {
String srcFilename = srcBackupFile != null ? srcBackupFile.getFileName() : null;
String destFilename = destVolumeFile != null ? destVolumeFile.getFileName() : null;
Expand All @@ -296,12 +330,14 @@ private boolean replaceRbdVolumeWithBackup(KVMStoragePoolManager storagePoolMgr,
private boolean attachVolumeToVm(KVMStoragePoolManager storagePoolMgr, String vmName, PrimaryDataStoreTO volumePool, String volumePath) {
String deviceToAttachDiskTo = getDeviceToAttachDisk(vmName);
int exitValue;
if (volumePool.getPoolType() != Storage.StoragePoolType.RBD) {
exitValue = Script.runSimpleBashScriptForExitValue(String.format(ATTACH_QCOW2_DISK_COMMAND, vmName, volumePath, deviceToAttachDiskTo));
} else {
if (volumePool.getPoolType() == Storage.StoragePoolType.RBD) {
String xmlForRbdDisk = getXmlForRbdDisk(storagePoolMgr, volumePool, volumePath, deviceToAttachDiskTo);
logger.debug("RBD disk xml to attach: {}", xmlForRbdDisk);
exitValue = Script.runSimpleBashScriptForExitValue(String.format(ATTACH_RBD_DISK_XML_COMMAND, vmName, xmlForRbdDisk));
} else if (volumePool.getPoolType() == Storage.StoragePoolType.Linstor) {
exitValue = Script.runSimpleBashScriptForExitValue(String.format(ATTACH_RAW_DISK_COMMAND, vmName, volumePath, deviceToAttachDiskTo));
} else {
exitValue = Script.runSimpleBashScriptForExitValue(String.format(ATTACH_QCOW2_DISK_COMMAND, vmName, volumePath, deviceToAttachDiskTo));
}
return exitValue == 0;
}
Expand Down
40 changes: 32 additions & 8 deletions scripts/vm/hypervisor/kvm/nasbackup.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,17 +89,42 @@ sanity_checks() {

### Operation methods ###

get_ceph_uuid_from_path() {
local fullpath="$1"
# disk for rbd => rbd:<pool>/<uuid>:mon_host=<monitor_host>...
# sample: rbd:cloudstack/53d5c355-d726-4d3e-9422-046a503a0b12:mon_host=10.0.1.2...
beforeUuid="${fullpath#*/}" # Remove up to first slash after rbd:
volUuid="${beforeUuid%%:*}" # Remove everything after colon to get the uuid
echo $volUuid
}

get_linstor_uuid_from_path() {
local fullpath="$1"
# disk for linstor => /dev/drbd/by-res/cs-<uuid>/0
# sample: /dev/drbd/by-res/cs-53d5c355-d726-4d3e-9422-046a503a0b12/0
beforeUuid="${fullpath#/dev/drbd/by-res/}"
volUuid="${beforeUuid%%/*}"
volUuid="${volUuid#cs-}"
echo $volUuid
}

backup_running_vm() {
mount_operation
mkdir -p "$dest" || { echo "Failed to create backup directory $dest"; exit 1; }

name="root"
echo "<domainbackup mode='push'><disks>" > $dest/backup.xml
for disk in $(virsh -c qemu:///system domblklist $VM --details 2>/dev/null | awk '/disk/{print$3}'); do
volpath=$(virsh -c qemu:///system domblklist $VM --details | awk "/$disk/{print $4}" | sed 's/.*\///')
echo "<disk name='$disk' backup='yes' type='file' backupmode='full'><driver type='qcow2'/><target file='$dest/$name.$volpath.qcow2' /></disk>" >> $dest/backup.xml
while read -r disk fullpath; do
if [[ "$fullpath" == /dev/drbd/by-res/* ]]; then
volUuid=$(get_linstor_uuid_from_path "$fullpath")
else
volUuid="${fullpath##*/}"
fi
echo "<disk name='$disk' backup='yes' type='file' backupmode='full'><driver type='qcow2'/><target file='$dest/$name.$volUuid.qcow2' /></disk>" >> $dest/backup.xml
name="datadisk"
done
done < <(
virsh -c qemu:///system domblklist "$VM" --details 2>/dev/null | awk '$2=="disk"{print $3, $4}'
)
echo "</disks></domainbackup>" >> $dest/backup.xml

local thaw=0
Expand Down Expand Up @@ -166,10 +191,9 @@ backup_stopped_vm() {
name="root"
for disk in $DISK_PATHS; do
if [[ "$disk" == rbd:* ]]; then
# disk for rbd => rbd:<pool>/<uuid>:mon_host=<monitor_host>...
# sample: rbd:cloudstack/53d5c355-d726-4d3e-9422-046a503a0b12:mon_host=10.0.1.2...
beforeUuid="${disk#*/}" # Remove up to first slash after rbd:
volUuid="${beforeUuid%%:*}" # Remove everything after colon to get the uuid
volUuid=$(get_ceph_uuid_from_path "$disk")
elif [[ "$disk" == /dev/drbd/by-res/* ]]; then
volUuid=$(get_linstor_uuid_from_path "$disk")
else
volUuid="${disk##*/}"
fi
Expand Down
Loading