Moving files from Google Cloud Storage to Azure Blob Storage is pretty simple as they can just be streamed directly. My source file in Google Cloud Storage was a compressed gzip file which I needed to be unzipped at the Azure Blob Storage destination.

GetStorageObjects

public IEnumerable<Google.Apis.Storage.v1.Data.Object> GetStorageObjects(string bucketName, string prefix)
{
	var result = _storageClient.ListObjects(bucketName, prefix).GetEnumerator();

	while (result.MoveNext())
	{
		yield return result.Current;
	}
}

GetStorageFileStreamAsync

public async Task<Stream> GetStorageFileStreamAsync(Google.Apis.Storage.v1.Data.Object storageObject)
{
	return await DownloadStreamFromStorageAsync(storageObject);
}

DecompressStreamToStream

public static Stream DecompressStreamToStream(Stream stream)
{
	return new GZipStream(stream, CompressionMode.Decompress);
}

Usage

List<Google.Apis.Storage.v1.Data.Object> storageObjects = _googleClient.GetStorageObjects(transfer.Source.GoogleCloudStorage.BucketName, $"{transfer.Source.GoogleCloudStorage.ProjectId}/{transfer.Source.GoogleCloudStorage.DataSetId}/{transfer.Source.GoogleCloudStorage.FileName}").ToList();

foreach (Google.Apis.Storage.v1.Data.Object storageObject in storageObjects)
{
	using (Stream stream = await _googleClient.GetStorageFileStreamAsync(storageObject))
	{
		CloudBlockBlob blockBlob = _cloudBlobContainer.GetBlockBlobReference(storageObject.Name.Replace(".gzip", ""));
		using (Stream decompressedStream = Utilities.DecompressStreamToStream(stream))
		{
			await blockBlob.UploadFromStreamAsync(decompressedStream);
		}
	}
}

Leave a Reply

Your email address will not be published. Required fields are marked *