Speed up large object downloads
In this tutorial you will learn how to use striped downloads to speed up downloads of large Cloud Storage objects.
Prerequisites
The guide assumes you have an existing Google Cloud project with billing enabled, and a Cloud Storage bucket in that project.
You will create some large objects during this tutorial, remember to clean up any resources to avoid excessive billing.
The tutorial assumes you are familiar with the basics of using the client library. If not, read the quickstart guide.
Add the client library as a dependency
cargo add google-cloud-storage
Create source data
To run this tutorial you will need some large objects in Cloud Storage. You can create such objects by seeding a smaller object and then repeatedly composing it to create objects of the desired size.
You can put all the code for seeding the data in its own function. This function will receive the storage and storage control clients as parameters. For information on how to create these clients, consult the quickstart guide:
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use google_cloud_gax::options::RequestOptionsBuilder;
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.set_if_generation_match(0)
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = Arc::new(std::fs::File::create(destination)?);
file.set_len(metadata.size as u64)?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = start.elapsed();
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &Arc<std::fs::File>,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
let mut current_pos = offset;
while let Some(b) = reader.next().await.transpose()? {
let chunk_len = b.len() as u64;
let handle = file.clone();
#[cfg(unix)]
tokio::task::spawn_blocking(move || {
use std::os::unix::fs::FileExt;
handle.write_all_at(&b, current_pos)
})
.await??;
#[cfg(windows)]
tokio::task::spawn_blocking(move || {
use std::os::windows::fs::FileExt;
handle.seek_write(&b, current_pos)
})
.await??;
current_pos += chunk_len;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
// These only work on Linux because they use /dev/shm.
#[cfg(all(target_os = "linux", feature = "run-large-downloads"))]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}
// ... details omitted ...
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use google_cloud_gax::options::RequestOptionsBuilder;
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.set_if_generation_match(0)
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = Arc::new(std::fs::File::create(destination)?);
file.set_len(metadata.size as u64)?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = start.elapsed();
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &Arc<std::fs::File>,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
let mut current_pos = offset;
while let Some(b) = reader.next().await.transpose()? {
let chunk_len = b.len() as u64;
let handle = file.clone();
#[cfg(unix)]
tokio::task::spawn_blocking(move || {
use std::os::unix::fs::FileExt;
handle.write_all_at(&b, current_pos)
})
.await??;
#[cfg(windows)]
tokio::task::spawn_blocking(move || {
use std::os::windows::fs::FileExt;
handle.seek_write(&b, current_pos)
})
.await??;
current_pos += chunk_len;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
// These only work on Linux because they use /dev/shm.
#[cfg(all(target_os = "linux", feature = "run-large-downloads"))]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}As usual, the function starts with some use declarations to simplify the code:
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use google_cloud_gax::options::RequestOptionsBuilder;
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.set_if_generation_match(0)
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = Arc::new(std::fs::File::create(destination)?);
file.set_len(metadata.size as u64)?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = start.elapsed();
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &Arc<std::fs::File>,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
let mut current_pos = offset;
while let Some(b) = reader.next().await.transpose()? {
let chunk_len = b.len() as u64;
let handle = file.clone();
#[cfg(unix)]
tokio::task::spawn_blocking(move || {
use std::os::unix::fs::FileExt;
handle.write_all_at(&b, current_pos)
})
.await??;
#[cfg(windows)]
tokio::task::spawn_blocking(move || {
use std::os::windows::fs::FileExt;
handle.seek_write(&b, current_pos)
})
.await??;
current_pos += chunk_len;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
// These only work on Linux because they use /dev/shm.
#[cfg(all(target_os = "linux", feature = "run-large-downloads"))]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}Using the storage client, you create a 1MiB object:
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use google_cloud_gax::options::RequestOptionsBuilder;
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.set_if_generation_match(0)
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = Arc::new(std::fs::File::create(destination)?);
file.set_len(metadata.size as u64)?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = start.elapsed();
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &Arc<std::fs::File>,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
let mut current_pos = offset;
while let Some(b) = reader.next().await.transpose()? {
let chunk_len = b.len() as u64;
let handle = file.clone();
#[cfg(unix)]
tokio::task::spawn_blocking(move || {
use std::os::unix::fs::FileExt;
handle.write_all_at(&b, current_pos)
})
.await??;
#[cfg(windows)]
tokio::task::spawn_blocking(move || {
use std::os::windows::fs::FileExt;
handle.seek_write(&b, current_pos)
})
.await??;
current_pos += chunk_len;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
// These only work on Linux because they use /dev/shm.
#[cfg(all(target_os = "linux", feature = "run-large-downloads"))]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}Then you use the storage control client to concatenate 32 copies of this object into a larger object. This operation does not require transferring any object data to the client; the service performs it:
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use google_cloud_gax::options::RequestOptionsBuilder;
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.set_if_generation_match(0)
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = Arc::new(std::fs::File::create(destination)?);
file.set_len(metadata.size as u64)?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = start.elapsed();
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &Arc<std::fs::File>,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
let mut current_pos = offset;
while let Some(b) = reader.next().await.transpose()? {
let chunk_len = b.len() as u64;
let handle = file.clone();
#[cfg(unix)]
tokio::task::spawn_blocking(move || {
use std::os::unix::fs::FileExt;
handle.write_all_at(&b, current_pos)
})
.await??;
#[cfg(windows)]
tokio::task::spawn_blocking(move || {
use std::os::windows::fs::FileExt;
handle.seek_write(&b, current_pos)
})
.await??;
current_pos += chunk_len;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
// These only work on Linux because they use /dev/shm.
#[cfg(all(target_os = "linux", feature = "run-large-downloads"))]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}You can repeat the operation to create larger and larger objects:
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use google_cloud_gax::options::RequestOptionsBuilder;
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.set_if_generation_match(0)
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = Arc::new(std::fs::File::create(destination)?);
file.set_len(metadata.size as u64)?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = start.elapsed();
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &Arc<std::fs::File>,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
let mut current_pos = offset;
while let Some(b) = reader.next().await.transpose()? {
let chunk_len = b.len() as u64;
let handle = file.clone();
#[cfg(unix)]
tokio::task::spawn_blocking(move || {
use std::os::unix::fs::FileExt;
handle.write_all_at(&b, current_pos)
})
.await??;
#[cfg(windows)]
tokio::task::spawn_blocking(move || {
use std::os::windows::fs::FileExt;
handle.seek_write(&b, current_pos)
})
.await??;
current_pos += chunk_len;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
// These only work on Linux because they use /dev/shm.
#[cfg(all(target_os = "linux", feature = "run-large-downloads"))]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use google_cloud_gax::options::RequestOptionsBuilder;
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.set_if_generation_match(0)
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = Arc::new(std::fs::File::create(destination)?);
file.set_len(metadata.size as u64)?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = start.elapsed();
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &Arc<std::fs::File>,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
let mut current_pos = offset;
while let Some(b) = reader.next().await.transpose()? {
let chunk_len = b.len() as u64;
let handle = file.clone();
#[cfg(unix)]
tokio::task::spawn_blocking(move || {
use std::os::unix::fs::FileExt;
handle.write_all_at(&b, current_pos)
})
.await??;
#[cfg(windows)]
tokio::task::spawn_blocking(move || {
use std::os::windows::fs::FileExt;
handle.seek_write(&b, current_pos)
})
.await??;
current_pos += chunk_len;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
// These only work on Linux because they use /dev/shm.
#[cfg(all(target_os = "linux", feature = "run-large-downloads"))]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}Striped downloads
Again, write a function to perform the striped download:
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use google_cloud_gax::options::RequestOptionsBuilder;
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.set_if_generation_match(0)
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = Arc::new(std::fs::File::create(destination)?);
file.set_len(metadata.size as u64)?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = start.elapsed();
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &Arc<std::fs::File>,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
let mut current_pos = offset;
while let Some(b) = reader.next().await.transpose()? {
let chunk_len = b.len() as u64;
let handle = file.clone();
#[cfg(unix)]
tokio::task::spawn_blocking(move || {
use std::os::unix::fs::FileExt;
handle.write_all_at(&b, current_pos)
})
.await??;
#[cfg(windows)]
tokio::task::spawn_blocking(move || {
use std::os::windows::fs::FileExt;
handle.seek_write(&b, current_pos)
})
.await??;
current_pos += chunk_len;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
// These only work on Linux because they use /dev/shm.
#[cfg(all(target_os = "linux", feature = "run-large-downloads"))]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}
// ... details below ...
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use google_cloud_gax::options::RequestOptionsBuilder;
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.set_if_generation_match(0)
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = Arc::new(std::fs::File::create(destination)?);
file.set_len(metadata.size as u64)?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = start.elapsed();
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &Arc<std::fs::File>,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
let mut current_pos = offset;
while let Some(b) = reader.next().await.transpose()? {
let chunk_len = b.len() as u64;
let handle = file.clone();
#[cfg(unix)]
tokio::task::spawn_blocking(move || {
use std::os::unix::fs::FileExt;
handle.write_all_at(&b, current_pos)
})
.await??;
#[cfg(windows)]
tokio::task::spawn_blocking(move || {
use std::os::windows::fs::FileExt;
handle.seek_write(&b, current_pos)
})
.await??;
current_pos += chunk_len;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
// These only work on Linux because they use /dev/shm.
#[cfg(all(target_os = "linux", feature = "run-large-downloads"))]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}Use the storage control client to query the object metadata. This metadata includes the object size and the current generation:
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use google_cloud_gax::options::RequestOptionsBuilder;
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.set_if_generation_match(0)
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = Arc::new(std::fs::File::create(destination)?);
file.set_len(metadata.size as u64)?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = start.elapsed();
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &Arc<std::fs::File>,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
let mut current_pos = offset;
while let Some(b) = reader.next().await.transpose()? {
let chunk_len = b.len() as u64;
let handle = file.clone();
#[cfg(unix)]
tokio::task::spawn_blocking(move || {
use std::os::unix::fs::FileExt;
handle.write_all_at(&b, current_pos)
})
.await??;
#[cfg(windows)]
tokio::task::spawn_blocking(move || {
use std::os::windows::fs::FileExt;
handle.seek_write(&b, current_pos)
})
.await??;
current_pos += chunk_len;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
// These only work on Linux because they use /dev/shm.
#[cfg(all(target_os = "linux", feature = "run-large-downloads"))]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}Split the download of each stripe to a separate function. You will see the
details of this function in a moment, for now just note that it is async, so
it returns a Future:
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use google_cloud_gax::options::RequestOptionsBuilder;
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.set_if_generation_match(0)
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = Arc::new(std::fs::File::create(destination)?);
file.set_len(metadata.size as u64)?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = start.elapsed();
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &Arc<std::fs::File>,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
let mut current_pos = offset;
while let Some(b) = reader.next().await.transpose()? {
let chunk_len = b.len() as u64;
let handle = file.clone();
#[cfg(unix)]
tokio::task::spawn_blocking(move || {
use std::os::unix::fs::FileExt;
handle.write_all_at(&b, current_pos)
})
.await??;
#[cfg(windows)]
tokio::task::spawn_blocking(move || {
use std::os::windows::fs::FileExt;
handle.seek_write(&b, current_pos)
})
.await??;
current_pos += chunk_len;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
// These only work on Linux because they use /dev/shm.
#[cfg(all(target_os = "linux", feature = "run-large-downloads"))]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}
// ... details below ...
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use google_cloud_gax::options::RequestOptionsBuilder;
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.set_if_generation_match(0)
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = Arc::new(std::fs::File::create(destination)?);
file.set_len(metadata.size as u64)?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = start.elapsed();
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &Arc<std::fs::File>,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
let mut current_pos = offset;
while let Some(b) = reader.next().await.transpose()? {
let chunk_len = b.len() as u64;
let handle = file.clone();
#[cfg(unix)]
tokio::task::spawn_blocking(move || {
use std::os::unix::fs::FileExt;
handle.write_all_at(&b, current_pos)
})
.await??;
#[cfg(windows)]
tokio::task::spawn_blocking(move || {
use std::os::windows::fs::FileExt;
handle.seek_write(&b, current_pos)
})
.await??;
current_pos += chunk_len;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
// These only work on Linux because they use /dev/shm.
#[cfg(all(target_os = "linux", feature = "run-large-downloads"))]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}You can compute the size of each stripe and then call write_stripe() to
download each of these stripes. Then collect the results into a vector:
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use google_cloud_gax::options::RequestOptionsBuilder;
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.set_if_generation_match(0)
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = Arc::new(std::fs::File::create(destination)?);
file.set_len(metadata.size as u64)?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = start.elapsed();
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &Arc<std::fs::File>,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
let mut current_pos = offset;
while let Some(b) = reader.next().await.transpose()? {
let chunk_len = b.len() as u64;
let handle = file.clone();
#[cfg(unix)]
tokio::task::spawn_blocking(move || {
use std::os::unix::fs::FileExt;
handle.write_all_at(&b, current_pos)
})
.await??;
#[cfg(windows)]
tokio::task::spawn_blocking(move || {
use std::os::windows::fs::FileExt;
handle.seek_write(&b, current_pos)
})
.await??;
current_pos += chunk_len;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
// These only work on Linux because they use /dev/shm.
#[cfg(all(target_os = "linux", feature = "run-large-downloads"))]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}You can use the standard Rust facilities to concurrently await all these futures:
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use google_cloud_gax::options::RequestOptionsBuilder;
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.set_if_generation_match(0)
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = Arc::new(std::fs::File::create(destination)?);
file.set_len(metadata.size as u64)?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = start.elapsed();
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &Arc<std::fs::File>,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
let mut current_pos = offset;
while let Some(b) = reader.next().await.transpose()? {
let chunk_len = b.len() as u64;
let handle = file.clone();
#[cfg(unix)]
tokio::task::spawn_blocking(move || {
use std::os::unix::fs::FileExt;
handle.write_all_at(&b, current_pos)
})
.await??;
#[cfg(windows)]
tokio::task::spawn_blocking(move || {
use std::os::windows::fs::FileExt;
handle.seek_write(&b, current_pos)
})
.await??;
current_pos += chunk_len;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
// These only work on Linux because they use /dev/shm.
#[cfg(all(target_os = "linux", feature = "run-large-downloads"))]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}Once they complete, the file is downloaded.
Now you should complete writing the write_stripe() function. First, start a
download from Cloud Storage:
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use google_cloud_gax::options::RequestOptionsBuilder;
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.set_if_generation_match(0)
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = Arc::new(std::fs::File::create(destination)?);
file.set_len(metadata.size as u64)?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = start.elapsed();
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &Arc<std::fs::File>,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
let mut current_pos = offset;
while let Some(b) = reader.next().await.transpose()? {
let chunk_len = b.len() as u64;
let handle = file.clone();
#[cfg(unix)]
tokio::task::spawn_blocking(move || {
use std::os::unix::fs::FileExt;
handle.write_all_at(&b, current_pos)
})
.await??;
#[cfg(windows)]
tokio::task::spawn_blocking(move || {
use std::os::windows::fs::FileExt;
handle.seek_write(&b, current_pos)
})
.await??;
current_pos += chunk_len;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
// These only work on Linux because they use /dev/shm.
#[cfg(all(target_os = "linux", feature = "run-large-downloads"))]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}To restrict the download to the desired stripe, use .with_read_offset() and
.with_read_limit():
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use google_cloud_gax::options::RequestOptionsBuilder;
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.set_if_generation_match(0)
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = Arc::new(std::fs::File::create(destination)?);
file.set_len(metadata.size as u64)?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = start.elapsed();
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &Arc<std::fs::File>,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
let mut current_pos = offset;
while let Some(b) = reader.next().await.transpose()? {
let chunk_len = b.len() as u64;
let handle = file.clone();
#[cfg(unix)]
tokio::task::spawn_blocking(move || {
use std::os::unix::fs::FileExt;
handle.write_all_at(&b, current_pos)
})
.await??;
#[cfg(windows)]
tokio::task::spawn_blocking(move || {
use std::os::windows::fs::FileExt;
handle.seek_write(&b, current_pos)
})
.await??;
current_pos += chunk_len;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
// These only work on Linux because they use /dev/shm.
#[cfg(all(target_os = "linux", feature = "run-large-downloads"))]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}You may also want to restrict the download to the right object generation. This will avoid race conditions where another process writes over the object and you get inconsistent reads:
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use google_cloud_gax::options::RequestOptionsBuilder;
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.set_if_generation_match(0)
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = Arc::new(std::fs::File::create(destination)?);
file.set_len(metadata.size as u64)?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = start.elapsed();
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &Arc<std::fs::File>,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
let mut current_pos = offset;
while let Some(b) = reader.next().await.transpose()? {
let chunk_len = b.len() as u64;
let handle = file.clone();
#[cfg(unix)]
tokio::task::spawn_blocking(move || {
use std::os::unix::fs::FileExt;
handle.write_all_at(&b, current_pos)
})
.await??;
#[cfg(windows)]
tokio::task::spawn_blocking(move || {
use std::os::windows::fs::FileExt;
handle.seek_write(&b, current_pos)
})
.await??;
current_pos += chunk_len;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
// These only work on Linux because they use /dev/shm.
#[cfg(all(target_os = "linux", feature = "run-large-downloads"))]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}Then you read the data and write it to the local file. This example uses
write_all_at on Unix, and seek_write on Windows, as Rust lacks a portable
function to atomically write at an specific offset. Both these functions are
blocking, so you need to run them on a dedicated thread:
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use google_cloud_gax::options::RequestOptionsBuilder;
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.set_if_generation_match(0)
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = Arc::new(std::fs::File::create(destination)?);
file.set_len(metadata.size as u64)?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = start.elapsed();
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &Arc<std::fs::File>,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
let mut current_pos = offset;
while let Some(b) = reader.next().await.transpose()? {
let chunk_len = b.len() as u64;
let handle = file.clone();
#[cfg(unix)]
tokio::task::spawn_blocking(move || {
use std::os::unix::fs::FileExt;
handle.write_all_at(&b, current_pos)
})
.await??;
#[cfg(windows)]
tokio::task::spawn_blocking(move || {
use std::os::windows::fs::FileExt;
handle.seek_write(&b, current_pos)
})
.await??;
current_pos += chunk_len;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
// These only work on Linux because they use /dev/shm.
#[cfg(all(target_os = "linux", feature = "run-large-downloads"))]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}Next steps
- Consider optimizing the case where the last stripe only has a few bytes
Expected performance
The performance of these downloads depends on:
- The I/O subsystem: if your local storage is not fast enough writing to disk may be the rate limiting step for these downloads.
- The configuration of your VM: if the VM does not have enough CPUs, descrypting the data may be the rate limiting step for these downloads. Recall that Cloud Storage always encrypt the data in transit, and the client must always decrypt.
- The location of the bucket and the particular object: the bucket may store all of the objects (or some objects) in a region different from your VM’s location. In this case, you may be throttled by the wide-area network capacity.
With a large enough VM, using SSD for disk, and with a bucket in the same region as the VM you should get close to 1,000 MiB/s of effective throughput.
Full program
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use google_cloud_gax::options::RequestOptionsBuilder;
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
use google_cloud_storage::model::Object;
async fn seed(client: Storage, control: StorageControl, bucket_name: &str) -> anyhow::Result<()> {
use google_cloud_storage::model::compose_object_request::SourceObject;
let buffer = String::from_iter(('a'..='z').cycle().take(1024 * 1024));
let seed = client
.write_object(bucket_name, "1MiB.txt", bytes::Bytes::from_owner(buffer))
.set_if_generation_match(0)
.send_unbuffered()
.await?;
println!(
"Uploaded object {}, size={}KiB",
seed.name,
seed.size / 1024
);
let seed_32 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("32MiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed.name)
.set_generation(seed.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
let seed_1024 = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name("1GiB.txt"))
.set_source_objects((0..32).map(|_| {
SourceObject::new()
.set_name(&seed_32.name)
.set_generation(seed_32.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {}, size={}MiB",
seed.name,
seed.size / (1024 * 1024)
);
for s in [2, 4, 8, 16, 32] {
let name = format!("{s}GiB.txt");
let target = control
.compose_object()
.set_destination(Object::new().set_bucket(bucket_name).set_name(&name))
.set_source_objects((0..s).map(|_| {
SourceObject::new()
.set_name(&seed_1024.name)
.set_generation(seed_1024.generation)
}))
.set_if_generation_match(0)
.with_idempotency(true)
.send()
.await?;
println!(
"Created object {} size={} MiB",
target.name,
target.size / (1024 * 1024)
);
}
Ok(())
}
async fn download(
client: Storage,
control: StorageControl,
bucket_name: &str,
object_name: &str,
stripe_size: usize,
destination: &str,
) -> anyhow::Result<()> {
let metadata = control
.get_object()
.set_bucket(bucket_name)
.set_object(object_name)
.send()
.await?;
let file = Arc::new(std::fs::File::create(destination)?);
file.set_len(metadata.size as u64)?;
let start = std::time::Instant::now();
let size = metadata.size as u64;
let limit = stripe_size as u64;
let count = size / limit;
let mut stripes = (0..count)
.map(|i| write_stripe(client.clone(), &file, i * limit, limit, &metadata))
.collect::<Vec<_>>();
if size % limit != 0 {
stripes.push(write_stripe(
client.clone(),
&file,
count * limit,
limit,
&metadata,
))
}
futures::future::join_all(stripes)
.await
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;
let elapsed = start.elapsed();
let mib = metadata.size as f64 / (1024.0 * 1024.0);
let bw = mib / elapsed.as_secs_f64();
println!(
"Completed {mib:.2} MiB download in {elapsed:?}, using {count} stripes, effective bandwidth = {bw:.2} MiB/s"
);
Ok(())
}
async fn write_stripe(
client: Storage,
file: &Arc<std::fs::File>,
offset: u64,
limit: u64,
metadata: &Object,
) -> anyhow::Result<()> {
use google_cloud_storage::model_ext::ReadRange;
let mut reader = client
.read_object(&metadata.bucket, &metadata.name)
.set_generation(metadata.generation)
.set_read_range(ReadRange::segment(offset, limit))
.send()
.await?;
let mut current_pos = offset;
while let Some(b) = reader.next().await.transpose()? {
let chunk_len = b.len() as u64;
let handle = file.clone();
#[cfg(unix)]
tokio::task::spawn_blocking(move || {
use std::os::unix::fs::FileExt;
handle.write_all_at(&b, current_pos)
})
.await??;
#[cfg(windows)]
tokio::task::spawn_blocking(move || {
use std::os::windows::fs::FileExt;
handle.seek_write(&b, current_pos)
})
.await??;
current_pos += chunk_len;
}
Ok(())
}
pub async fn test(bucket_name: &str, destination: &str) -> anyhow::Result<()> {
const MB: usize = 1024 * 1024;
let client = Storage::builder().build().await?;
let control = StorageControl::builder().build().await?;
seed(client.clone(), control.clone(), bucket_name).await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
32 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32MiB.txt",
MB,
destination,
)
.await?;
// These only work on Linux because they use /dev/shm.
#[cfg(all(target_os = "linux", feature = "run-large-downloads"))]
{
let destination = "/dev/shm/output";
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"32GiB.txt",
256 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
1024 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"1GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"4GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"8GiB.txt",
512 * MB,
destination,
)
.await?;
download(
client.clone(),
control.clone(),
bucket_name,
"16GiB.txt",
512 * MB,
destination,
)
.await?;
}
Ok(())
}