Data Transfer – Google Cloud Storage to Azure Blob Storage

Moving files from Google Cloud Storage to Azure Blob Storage is pretty simple as they can just be streamed directly. My source file in Google Cloud Storage was a compressed gzip file which I needed to be unzipped at the Azure Blob Storage destination.

public IEnumerable<Google.Apis.Storage.v1.Data.Object> GetStorageObjects(string bucketName, string prefix)
{
	var result = _storageClient.ListObjects(bucketName, prefix).GetEnumerator();

	while (result.MoveNext())
	{
		yield return result.Current;
	}
}

public async Task<Stream> GetStorageFileStreamAsync(Google.Apis.Storage.v1.Data.Object storageObject)
{
	return await DownloadStreamFromStorageAsync(storageObject);
}

public static Stream DecompressStreamToStream(Stream stream)
{
	return new GZipStream(stream, CompressionMode.Decompress);
}

Usage

List<Google.Apis.Storage.v1.Data.Object> storageObjects = _googleClient.GetStorageObjects(transfer.Source.GoogleCloudStorage.BucketName, $"{transfer.Source.GoogleCloudStorage.ProjectId}/{transfer.Source.GoogleCloudStorage.DataSetId}/{transfer.Source.GoogleCloudStorage.FileName}").ToList();

foreach (Google.Apis.Storage.v1.Data.Object storageObject in storageObjects)
{
	using (Stream stream = await _googleClient.GetStorageFileStreamAsync(storageObject))
	{
		CloudBlockBlob blockBlob = _cloudBlobContainer.GetBlockBlobReference(storageObject.Name.Replace(".gzip", ""));
		using (Stream decompressedStream = Utilities.DecompressStreamToStream(stream))
		{
			await blockBlob.UploadFromStreamAsync(decompressedStream);
		}
	}
}