Speed up large object downloads
In this tutorial you will learn how to use striped downloads to speed up downloads of large Cloud Storage objects.
Prerequisites
The guide assumes you have an existing Google Cloud project with billing enabled, and a Cloud Storage bucket in that project.
You will create some large objects during this tutorial, remember to clean up any resources to avoid excessive billing.
The tutorial assumes you are familiar with the basics of using the client library. If not, read the quickstart guide.
Add the client library as a dependency
cargo add google-cloud-storage
Create source data
To run this tutorial you will need some large objects in Cloud Storage. You can create such objects by seeding a smaller object and then repeatedly composing it to create objects of the desired size.
You can put all the code for seeding the data in its own function. This function will receive the storage and storage control clients as parameters. For information on how to create these clients, consult the quickstart guide:
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = tokio::fs::File::create(destination).await?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = std::time::Instant::now() - start;
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &tokio::fs::File,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
use tokio::io::AsyncSeekExt;
let mut writer = file.try_clone().await?;
writer.seek(std::io::SeekFrom::Start(offset)).await?;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
while let Some(b) = reader.next().await.transpose()? {
use tokio::io::AsyncWriteExt;
writer.write_all(&b).await?;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
#[cfg(feature = "run-large-downloads")]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}
// ... details omitted ...
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = tokio::fs::File::create(destination).await?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = std::time::Instant::now() - start;
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &tokio::fs::File,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
use tokio::io::AsyncSeekExt;
let mut writer = file.try_clone().await?;
writer.seek(std::io::SeekFrom::Start(offset)).await?;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
while let Some(b) = reader.next().await.transpose()? {
use tokio::io::AsyncWriteExt;
writer.write_all(&b).await?;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
#[cfg(feature = "run-large-downloads")]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}As usual, the function starts with some use declarations to simplify the code:
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = tokio::fs::File::create(destination).await?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = std::time::Instant::now() - start;
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &tokio::fs::File,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
use tokio::io::AsyncSeekExt;
let mut writer = file.try_clone().await?;
writer.seek(std::io::SeekFrom::Start(offset)).await?;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
while let Some(b) = reader.next().await.transpose()? {
use tokio::io::AsyncWriteExt;
writer.write_all(&b).await?;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
#[cfg(feature = "run-large-downloads")]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}Using the storage client, you create a 1MiB object:
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = tokio::fs::File::create(destination).await?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = std::time::Instant::now() - start;
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &tokio::fs::File,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
use tokio::io::AsyncSeekExt;
let mut writer = file.try_clone().await?;
writer.seek(std::io::SeekFrom::Start(offset)).await?;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
while let Some(b) = reader.next().await.transpose()? {
use tokio::io::AsyncWriteExt;
writer.write_all(&b).await?;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
#[cfg(feature = "run-large-downloads")]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}Then you use the storage control client to concatenate 32 copies of this object into a larger object. This operation does not require transferring any object data to the client, it is performed by the service:
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = tokio::fs::File::create(destination).await?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = std::time::Instant::now() - start;
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &tokio::fs::File,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
use tokio::io::AsyncSeekExt;
let mut writer = file.try_clone().await?;
writer.seek(std::io::SeekFrom::Start(offset)).await?;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
while let Some(b) = reader.next().await.transpose()? {
use tokio::io::AsyncWriteExt;
writer.write_all(&b).await?;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
#[cfg(feature = "run-large-downloads")]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}You can repeat the operation to create larger and larger objects:
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = tokio::fs::File::create(destination).await?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = std::time::Instant::now() - start;
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &tokio::fs::File,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
use tokio::io::AsyncSeekExt;
let mut writer = file.try_clone().await?;
writer.seek(std::io::SeekFrom::Start(offset)).await?;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
while let Some(b) = reader.next().await.transpose()? {
use tokio::io::AsyncWriteExt;
writer.write_all(&b).await?;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
#[cfg(feature = "run-large-downloads")]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = tokio::fs::File::create(destination).await?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = std::time::Instant::now() - start;
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &tokio::fs::File,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
use tokio::io::AsyncSeekExt;
let mut writer = file.try_clone().await?;
writer.seek(std::io::SeekFrom::Start(offset)).await?;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
while let Some(b) = reader.next().await.transpose()? {
use tokio::io::AsyncWriteExt;
writer.write_all(&b).await?;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
#[cfg(feature = "run-large-downloads")]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}Striped downloads
Again, write a function to perform the striped download:
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = tokio::fs::File::create(destination).await?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = std::time::Instant::now() - start;
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &tokio::fs::File,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
use tokio::io::AsyncSeekExt;
let mut writer = file.try_clone().await?;
writer.seek(std::io::SeekFrom::Start(offset)).await?;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
while let Some(b) = reader.next().await.transpose()? {
use tokio::io::AsyncWriteExt;
writer.write_all(&b).await?;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
#[cfg(feature = "run-large-downloads")]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}
// ... details below ...
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = tokio::fs::File::create(destination).await?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = std::time::Instant::now() - start;
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &tokio::fs::File,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
use tokio::io::AsyncSeekExt;
let mut writer = file.try_clone().await?;
writer.seek(std::io::SeekFrom::Start(offset)).await?;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
while let Some(b) = reader.next().await.transpose()? {
use tokio::io::AsyncWriteExt;
writer.write_all(&b).await?;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
#[cfg(feature = "run-large-downloads")]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}Use the storage control client to query the object metadata. This metadata includes the object size and the current generation:
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = tokio::fs::File::create(destination).await?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = std::time::Instant::now() - start;
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &tokio::fs::File,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
use tokio::io::AsyncSeekExt;
let mut writer = file.try_clone().await?;
writer.seek(std::io::SeekFrom::Start(offset)).await?;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
while let Some(b) = reader.next().await.transpose()? {
use tokio::io::AsyncWriteExt;
writer.write_all(&b).await?;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
#[cfg(feature = "run-large-downloads")]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}We will split the download of each stripe to a separate function. You will see
the details of this function in a moment, for now just note that it is async,
so it returns a Future:
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = tokio::fs::File::create(destination).await?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = std::time::Instant::now() - start;
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &tokio::fs::File,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
use tokio::io::AsyncSeekExt;
let mut writer = file.try_clone().await?;
writer.seek(std::io::SeekFrom::Start(offset)).await?;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
while let Some(b) = reader.next().await.transpose()? {
use tokio::io::AsyncWriteExt;
writer.write_all(&b).await?;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
#[cfg(feature = "run-large-downloads")]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}
// ... details below ...
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = tokio::fs::File::create(destination).await?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = std::time::Instant::now() - start;
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &tokio::fs::File,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
use tokio::io::AsyncSeekExt;
let mut writer = file.try_clone().await?;
writer.seek(std::io::SeekFrom::Start(offset)).await?;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
while let Some(b) = reader.next().await.transpose()? {
use tokio::io::AsyncWriteExt;
writer.write_all(&b).await?;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
#[cfg(feature = "run-large-downloads")]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}You can compute the size of each stripe and then call write_stripe() to
download each of these stripes. Then collect the results into a vector:
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = tokio::fs::File::create(destination).await?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = std::time::Instant::now() - start;
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &tokio::fs::File,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
use tokio::io::AsyncSeekExt;
let mut writer = file.try_clone().await?;
writer.seek(std::io::SeekFrom::Start(offset)).await?;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
while let Some(b) = reader.next().await.transpose()? {
use tokio::io::AsyncWriteExt;
writer.write_all(&b).await?;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
#[cfg(feature = "run-large-downloads")]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}You can use the standard Rust facilities to concurrently await all these futures:
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = tokio::fs::File::create(destination).await?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = std::time::Instant::now() - start;
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &tokio::fs::File,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
use tokio::io::AsyncSeekExt;
let mut writer = file.try_clone().await?;
writer.seek(std::io::SeekFrom::Start(offset)).await?;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
while let Some(b) = reader.next().await.transpose()? {
use tokio::io::AsyncWriteExt;
writer.write_all(&b).await?;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
#[cfg(feature = "run-large-downloads")]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}Once they complete, the file is downloaded.
Now you should complete writing the write_stripe() function. First, duplicate
the write object and prepare to write starting at the desired offset:
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = tokio::fs::File::create(destination).await?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = std::time::Instant::now() - start;
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &tokio::fs::File,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
use tokio::io::AsyncSeekExt;
let mut writer = file.try_clone().await?;
writer.seek(std::io::SeekFrom::Start(offset)).await?;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
while let Some(b) = reader.next().await.transpose()? {
use tokio::io::AsyncWriteExt;
writer.write_all(&b).await?;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
#[cfg(feature = "run-large-downloads")]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}Start a download from Cloud Storage:
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = tokio::fs::File::create(destination).await?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = std::time::Instant::now() - start;
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &tokio::fs::File,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
use tokio::io::AsyncSeekExt;
let mut writer = file.try_clone().await?;
writer.seek(std::io::SeekFrom::Start(offset)).await?;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
while let Some(b) = reader.next().await.transpose()? {
use tokio::io::AsyncWriteExt;
writer.write_all(&b).await?;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
#[cfg(feature = "run-large-downloads")]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}To restrict the download to the desired stripe, use .with_read_offset() and
.with_read_limit():
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = tokio::fs::File::create(destination).await?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = std::time::Instant::now() - start;
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &tokio::fs::File,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
use tokio::io::AsyncSeekExt;
let mut writer = file.try_clone().await?;
writer.seek(std::io::SeekFrom::Start(offset)).await?;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
while let Some(b) = reader.next().await.transpose()? {
use tokio::io::AsyncWriteExt;
writer.write_all(&b).await?;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
#[cfg(feature = "run-large-downloads")]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}You may also want to restrict the download to the right object generation. This will avoid race conditions where another process writes over the object and you get inconsistent reads:
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = tokio::fs::File::create(destination).await?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = std::time::Instant::now() - start;
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &tokio::fs::File,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
use tokio::io::AsyncSeekExt;
let mut writer = file.try_clone().await?;
writer.seek(std::io::SeekFrom::Start(offset)).await?;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
while let Some(b) = reader.next().await.transpose()? {
use tokio::io::AsyncWriteExt;
writer.write_all(&b).await?;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
#[cfg(feature = "run-large-downloads")]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}Then you read the data and write it to the local file:
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = tokio::fs::File::create(destination).await?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = std::time::Instant::now() - start;
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &tokio::fs::File,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
use tokio::io::AsyncSeekExt;
let mut writer = file.try_clone().await?;
writer.seek(std::io::SeekFrom::Start(offset)).await?;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
while let Some(b) = reader.next().await.transpose()? {
use tokio::io::AsyncWriteExt;
writer.write_all(&b).await?;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
#[cfg(feature = "run-large-downloads")]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}Next steps
- Consider optimizing the case where the last stripe only has a few bytes
Expected performance
The performance of these downloads depends on:
- The I/O subsystem: if your local storage is not fast enough the downloads will be throttled by the writes to disk.
- The configuration of your VM: if you do not have enough CPUs, the downloads will be throttled on trying to decrypt the on data, as Cloud Storage and the client library always encrypt the data in transit.
- The location of the bucket and the particular object: the bucket may store all of the objects (or some objects) in a region different from your VM's location. In this case, you may be throttled by the wide-area network capacity.
With a large enough VM, using SSD for disk, and with a bucket in the same region as the VM you should get close to 1,000 MiB/s of effective throughput.
Full program
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = tokio::fs::File::create(destination).await?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = std::time::Instant::now() - start;
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &tokio::fs::File,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
use tokio::io::AsyncSeekExt;
let mut writer = file.try_clone().await?;
writer.seek(std::io::SeekFrom::Start(offset)).await?;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
while let Some(b) = reader.next().await.transpose()? {
use tokio::io::AsyncWriteExt;
writer.write_all(&b).await?;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
#[cfg(feature = "run-large-downloads")]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}