Moving files from Google Cloud Storage to Azure Blob Storage is pretty simple as they can just be streamed directly. My source file in Google Cloud Storage was a compressed gzip file which I needed to be unzipped at the Azure Blob Storage destination.

GetStorageObjects

public IEnumerable<Google.Apis.Storage.v1.Data.Object> GetStorageObjects(string bucketName, string prefix)
{
	var result = _storageClient.ListObjects(bucketName, prefix).GetEnumerator();

	while (result.MoveNext())
	{
		yield return result.Current;
	}
}

GetStorageFileStreamAsync

public async Task<Stream> GetStorageFileStreamAsync(Google.Apis.Storage.v1.Data.Object storageObject)
{
	return await DownloadStreamFromStorageAsync(storageObject);
}

DecompressStreamToStream

public static Stream DecompressStreamToStream(Stream stream)
{
	return new GZipStream(stream, CompressionMode.Decompress);
}

Usage

List<Google.Apis.Storage.v1.Data.Object> storageObjects = _googleClient.GetStorageObjects(transfer.Source.GoogleCloudStorage.BucketName, $"{transfer.Source.GoogleCloudStorage.ProjectId}/{transfer.Source.GoogleCloudStorage.DataSetId}/{transfer.Source.GoogleCloudStorage.FileName}").ToList();

foreach (Google.Apis.Storage.v1.Data.Object storageObject in storageObjects)
{
	using (Stream stream = await _googleClient.GetStorageFileStreamAsync(storageObject))
	{
		CloudBlockBlob blockBlob = _cloudBlobContainer.GetBlockBlobReference(storageObject.Name.Replace(".gzip", ""));
		using (Stream decompressedStream = Utilities.DecompressStreamToStream(stream))
		{
			await blockBlob.UploadFromStreamAsync(decompressedStream);
		}
	}
}

Leave a Reply

Your email address will not be published. Required fields are marked *


Fatal error: Uncaught GuzzleHttp\Exception\ClientException: Client error: `POST https://dc.services.visualstudio.com/v2/track` resulted in a `400 Invalid instrumentation key` response: {"itemsReceived":1,"itemsAccepted":0,"errors":[{"index":0,"statusCode":400,"message":"Invalid instrumentation key"}]} in D:\home\site\wwwroot\wp-content\plugins\application-insights\vendor\guzzlehttp\guzzle\src\Exception\RequestException.php:113 Stack trace: #0 D:\home\site\wwwroot\wp-content\plugins\application-insights\vendor\guzzlehttp\guzzle\src\Middleware.php(66): GuzzleHttp\Exception\RequestException::create(Object(GuzzleHttp\Psr7\Request), Object(GuzzleHttp\Psr7\Response)) #1 D:\home\site\wwwroot\wp-content\plugins\application-insights\vendor\guzzlehttp\promises\src\Promise.php(203): GuzzleHttp\Middleware::GuzzleHttp\{closure}(Object(GuzzleHttp\Psr7\Response)) #2 D:\home\site\wwwroot\wp-content\plugins\application-insights\vendor\guzzlehttp\promises\src\Promise.php(156): GuzzleHttp\Promise\Promise::callHandler(1, Object(GuzzleHttp\P in D:\home\site\wwwroot\wp-content\plugins\application-insights\vendor\guzzlehttp\guzzle\src\Exception\RequestException.php on line 113