From 423f0f0d13b4f406c93f191417a45f28ff9c4ffb Mon Sep 17 00:00:00 2001 From: Daniele Palaia Date: Tue, 30 Jul 2024 09:40:18 +0200 Subject: [PATCH 1/4] tutorial2: rust implementation --- rust-stream/Cargo.lock | 41 ++----- rust-stream/Cargo.toml | 2 +- rust-stream/README.md | 7 +- .../src/bin/receive_offset_tracking.rs | 104 ++++++++++++++++++ rust-stream/src/bin/send_offset_tracking.rs | 66 +++++++++++ 5 files changed, 188 insertions(+), 32 deletions(-) create mode 100644 rust-stream/src/bin/receive_offset_tracking.rs create mode 100644 rust-stream/src/bin/send_offset_tracking.rs diff --git a/rust-stream/Cargo.lock b/rust-stream/Cargo.lock index 2259e460..324f61da 100644 --- a/rust-stream/Cargo.lock +++ b/rust-stream/Cargo.lock @@ -382,13 +382,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.11" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4" dependencies = [ + "hermit-abi", "libc", "wasi", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -400,16 +401,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi", - "libc", -] - [[package]] name = "num_enum" version = "0.7.2" @@ -637,7 +628,7 @@ dependencies = [ "libc", "spin", "untrusted", - "windows-sys 0.52.0", + "windows-sys", ] [[package]] @@ -748,7 +739,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys", ] [[package]] @@ -816,28 +807,27 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.37.0" +version = "1.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" +checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1" dependencies = [ "backtrace", "bytes", "libc", "mio", - "num_cpus", "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] name = "tokio-macros" -version = "2.2.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", @@ -1040,15 +1030,6 @@ dependencies = [ "windows-targets 0.52.5", ] -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", -] - [[package]] name = "windows-sys" version = "0.52.0" diff --git a/rust-stream/Cargo.toml b/rust-stream/Cargo.toml index 024d3084..6b7676f4 100644 --- a/rust-stream/Cargo.toml +++ b/rust-stream/Cargo.toml @@ -7,5 +7,5 @@ edition = "2021" [dependencies] rabbitmq-stream-client = "0.4.2" -tokio = { version = "1.0.0", features = ["rt", "rt-multi-thread", "macros"] } +tokio = { version = "1.39.0", features = ["rt", "rt-multi-thread", "macros"] } futures = "0.3.30" diff --git a/rust-stream/README.md b/rust-stream/README.md index a02e81a5..e8c8cb47 100644 --- a/rust-stream/README.md +++ b/rust-stream/README.md @@ -19,4 +19,9 @@ Each cargo command should be launched in a separate shell. #### [Tutorial one: "Hello World!"](https://www.rabbitmq.com/tutorials/tutorial-one-rust-stream.html) cargo run --bin receive - cargo run --bin send \ No newline at end of file + cargo run --bin send + +#### [Tutorial one: "Offset tracking!"](https://www.rabbitmq.com/tutorials/tutorial-one-rust-stream.html) + + cargo run --bin send_offset_tracking + cargo run --bin receive_offset_tracking \ No newline at end of file diff --git a/rust-stream/src/bin/receive_offset_tracking.rs b/rust-stream/src/bin/receive_offset_tracking.rs new file mode 100644 index 00000000..ecc0daeb --- /dev/null +++ b/rust-stream/src/bin/receive_offset_tracking.rs @@ -0,0 +1,104 @@ +use futures::StreamExt; +use rabbitmq_stream_client::error::StreamCreateError; +use rabbitmq_stream_client::types::{ByteCapacity, OffsetSpecification, ResponseCode}; +use std::io::stdin; +use std::sync::atomic::{AtomicI64, Ordering}; +use std::sync::Arc; +use tokio::task; + +#[tokio::main] +async fn main() -> Result<(), Box> { + use rabbitmq_stream_client::Environment; + let environment = Environment::builder().build().await?; + let stream = "stream-offset-tracking-rust"; + let received_messages = Arc::new(AtomicI64::new(-1)); + let first_offset = Arc::new(AtomicI64::new(-1)); + let last_offset = Arc::new(AtomicI64::new(-1)); + let create_response = environment + .stream_creator() + .max_length(ByteCapacity::GB(2)) + .create(stream) + .await; + + if let Err(e) = create_response { + if let StreamCreateError::Create { stream, status } = e { + match status { + // we can ignore this error because the stream already exists + ResponseCode::StreamAlreadyExists => {} + err => { + println!("Error creating stream: {:?} {:?}", stream, err); + std::process::exit(1); + } + } + } + } + + let mut consumer = environment + .consumer() + .name("consumer-1") + .offset(OffsetSpecification::First) + .build(stream) + .await + .unwrap(); + + println!("Starting consuming"); + println!("Press any key to close the consumer"); + + let mut stored_offset: u64 = consumer.query_offset().await.unwrap_or_else(|_| 0); + + if stored_offset > 0 { + stored_offset += 1; + } + consumer = environment + .consumer() + .name("consumer-1") + .offset(OffsetSpecification::Offset(stored_offset)) + .build(stream) + .await + .unwrap(); + + let first_cloned_offset = first_offset.clone(); + let last_cloned_offset = last_offset.clone(); + + task::spawn(async move { + while let Some(delivery) = consumer.next().await { + let d = delivery.unwrap(); + + if first_offset.load(Ordering::Relaxed) == -1 { + println!("consuming first message"); + _ = first_offset.compare_exchange( + first_offset.load(Ordering::Relaxed), + d.offset() as i64, + Ordering::Relaxed, + Ordering::Relaxed, + ); + } + + if received_messages.fetch_add(1, Ordering::Relaxed) % 10 == 0 + || String::from_utf8_lossy(d.message().data().unwrap()).contains("marker") + { + let _ = consumer + .store_offset(d.offset()) + .await + .unwrap_or_else(|e| println!("Err: {}", e)); + if String::from_utf8_lossy(d.message().data().unwrap()).contains("marker") { + last_offset.store(d.offset() as i64, Ordering::Relaxed); + let handle = consumer.handle(); + _ = handle.close().await; + + } + } + } + }); + + _ = stdin().read_line(&mut "".to_string()); + + if first_cloned_offset.load(Ordering::Relaxed) != -1 { + println!( + "Done consuming first_offset: {:?} last_offset: {:?} ", + first_cloned_offset, last_cloned_offset + ); + } + + Ok(()) +} diff --git a/rust-stream/src/bin/send_offset_tracking.rs b/rust-stream/src/bin/send_offset_tracking.rs new file mode 100644 index 00000000..abe0d979 --- /dev/null +++ b/rust-stream/src/bin/send_offset_tracking.rs @@ -0,0 +1,66 @@ +use rabbitmq_stream_client::error::StreamCreateError; +use rabbitmq_stream_client::types::{ByteCapacity, Message, ResponseCode}; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::Arc; +use tokio::sync::Notify; + +#[tokio::main] +async fn main() -> Result<(), Box> { + use rabbitmq_stream_client::Environment; + let environment = Environment::builder().build().await?; + let stream = "stream-offset-tracking-rust"; + let message_count = 100; + let confirmed_messages = Arc::new(AtomicU32::new(0)); + let notify_on_send = Arc::new(Notify::new()); + + let create_response = environment + .stream_creator() + .max_length(ByteCapacity::GB(2)) + .create(stream) + .await; + + if let Err(e) = create_response { + if let StreamCreateError::Create { stream, status } = e { + match status { + // we can ignore this error because the stream already exists + ResponseCode::StreamAlreadyExists => {} + err => { + println!("Error creating stream: {:?} {:?}", stream, err); + std::process::exit(1); + } + } + } + } + + println!("Publishing {:?} messages", message_count); + + let producer = environment.producer().build(stream).await?; + + for i in 0..message_count { + let msg; + if i < message_count - 1 { + msg = Message::builder().body(format!("hello{}", i)).build(); + } else { + msg = Message::builder().body(format!("marker{}", i)).build(); + }; + + let counter = confirmed_messages.clone(); + let notifier = notify_on_send.clone(); + producer + .send(msg, move |_| { + let inner_counter = counter.clone(); + let inner_notifier = notifier.clone(); + async move { + if inner_counter.fetch_add(1, Ordering::Relaxed) == message_count - 1 { + inner_notifier.notify_one(); + } + } + }) + .await?; + } + + notify_on_send.notified().await; + println!("Messages confirmed."); + producer.close().await?; + Ok(()) +} From 03a853bfe971a2eda9e8f46fbcba741001431c2c Mon Sep 17 00:00:00 2001 From: Daniele Palaia Date: Sat, 3 Aug 2024 16:41:08 +0200 Subject: [PATCH 2/4] replacing command key exit with Notify --- rust-stream/src/bin/receive_offset_tracking.rs | 12 +++++++----- rust-stream/src/bin/send_offset_tracking.rs | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/rust-stream/src/bin/receive_offset_tracking.rs b/rust-stream/src/bin/receive_offset_tracking.rs index ecc0daeb..c1290565 100644 --- a/rust-stream/src/bin/receive_offset_tracking.rs +++ b/rust-stream/src/bin/receive_offset_tracking.rs @@ -1,9 +1,9 @@ use futures::StreamExt; use rabbitmq_stream_client::error::StreamCreateError; use rabbitmq_stream_client::types::{ByteCapacity, OffsetSpecification, ResponseCode}; -use std::io::stdin; use std::sync::atomic::{AtomicI64, Ordering}; use std::sync::Arc; +use tokio::sync::Notify; use tokio::task; #[tokio::main] @@ -14,6 +14,7 @@ async fn main() -> Result<(), Box> { let received_messages = Arc::new(AtomicI64::new(-1)); let first_offset = Arc::new(AtomicI64::new(-1)); let last_offset = Arc::new(AtomicI64::new(-1)); + let notify_on_close = Arc::new(Notify::new()); let create_response = environment .stream_creator() .max_length(ByteCapacity::GB(2)) @@ -41,8 +42,7 @@ async fn main() -> Result<(), Box> { .await .unwrap(); - println!("Starting consuming"); - println!("Press any key to close the consumer"); + println!("Started consuming"); let mut stored_offset: u64 = consumer.query_offset().await.unwrap_or_else(|_| 0); @@ -59,13 +59,14 @@ async fn main() -> Result<(), Box> { let first_cloned_offset = first_offset.clone(); let last_cloned_offset = last_offset.clone(); + let notify_on_close_cloned = notify_on_close.clone(); task::spawn(async move { while let Some(delivery) = consumer.next().await { let d = delivery.unwrap(); if first_offset.load(Ordering::Relaxed) == -1 { - println!("consuming first message"); + println!("First message received"); _ = first_offset.compare_exchange( first_offset.load(Ordering::Relaxed), d.offset() as i64, @@ -85,13 +86,14 @@ async fn main() -> Result<(), Box> { last_offset.store(d.offset() as i64, Ordering::Relaxed); let handle = consumer.handle(); _ = handle.close().await; + notify_on_close_cloned.notify_one(); } } } }); - _ = stdin().read_line(&mut "".to_string()); + notify_on_close.notified().await; if first_cloned_offset.load(Ordering::Relaxed) != -1 { println!( diff --git a/rust-stream/src/bin/send_offset_tracking.rs b/rust-stream/src/bin/send_offset_tracking.rs index abe0d979..d40be89e 100644 --- a/rust-stream/src/bin/send_offset_tracking.rs +++ b/rust-stream/src/bin/send_offset_tracking.rs @@ -60,7 +60,7 @@ async fn main() -> Result<(), Box> { } notify_on_send.notified().await; - println!("Messages confirmed."); + println!("Messages confirmed: True"); producer.close().await?; Ok(()) } From 575c8aa9bec0e8603e66bc4561afbb24f97de210 Mon Sep 17 00:00:00 2001 From: Daniele Palaia Date: Tue, 6 Aug 2024 15:11:55 +0200 Subject: [PATCH 3/4] make received_messages variable local to the consuming thread --- rust-stream/src/bin/receive_offset_tracking.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/rust-stream/src/bin/receive_offset_tracking.rs b/rust-stream/src/bin/receive_offset_tracking.rs index c1290565..ecafd9a7 100644 --- a/rust-stream/src/bin/receive_offset_tracking.rs +++ b/rust-stream/src/bin/receive_offset_tracking.rs @@ -11,7 +11,6 @@ async fn main() -> Result<(), Box> { use rabbitmq_stream_client::Environment; let environment = Environment::builder().build().await?; let stream = "stream-offset-tracking-rust"; - let received_messages = Arc::new(AtomicI64::new(-1)); let first_offset = Arc::new(AtomicI64::new(-1)); let last_offset = Arc::new(AtomicI64::new(-1)); let notify_on_close = Arc::new(Notify::new()); @@ -46,7 +45,7 @@ async fn main() -> Result<(), Box> { let mut stored_offset: u64 = consumer.query_offset().await.unwrap_or_else(|_| 0); - if stored_offset > 0 { + if stored_offset > 0 { stored_offset += 1; } consumer = environment @@ -62,6 +61,7 @@ async fn main() -> Result<(), Box> { let notify_on_close_cloned = notify_on_close.clone(); task::spawn(async move { + let mut received_messages = -1; while let Some(delivery) = consumer.next().await { let d = delivery.unwrap(); @@ -74,8 +74,8 @@ async fn main() -> Result<(), Box> { Ordering::Relaxed, ); } - - if received_messages.fetch_add(1, Ordering::Relaxed) % 10 == 0 + received_messages = received_messages + 1; + if received_messages % 10 == 0 || String::from_utf8_lossy(d.message().data().unwrap()).contains("marker") { let _ = consumer @@ -87,7 +87,6 @@ async fn main() -> Result<(), Box> { let handle = consumer.handle(); _ = handle.close().await; notify_on_close_cloned.notify_one(); - } } } From a8d1638d5ada2c70c016b56bfa7ce4aa27f9e730 Mon Sep 17 00:00:00 2001 From: Daniele Palaia Date: Tue, 6 Aug 2024 16:39:38 +0200 Subject: [PATCH 4/4] test --- rust-stream/Cargo.lock | 4 ++-- rust-stream/Cargo.toml | 2 +- .../src/bin/receive_offset_tracking.rs | 22 ++++++++++--------- rust-stream/src/bin/send_offset_tracking.rs | 2 +- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/rust-stream/Cargo.lock b/rust-stream/Cargo.lock index 324f61da..c10a2af4 100644 --- a/rust-stream/Cargo.lock +++ b/rust-stream/Cargo.lock @@ -542,9 +542,9 @@ dependencies = [ [[package]] name = "rabbitmq-stream-client" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88174e13a7d829a79af91437e439722bd6b095c819f7f2973c25e6ff2b94e173" +checksum = "38019b93705e3371296fe57a10740a2ea669fee1a210ad7ff647a26e8ccd697e" dependencies = [ "async-trait", "bytes", diff --git a/rust-stream/Cargo.toml b/rust-stream/Cargo.toml index 6b7676f4..4da2cce0 100644 --- a/rust-stream/Cargo.toml +++ b/rust-stream/Cargo.toml @@ -6,6 +6,6 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -rabbitmq-stream-client = "0.4.2" +rabbitmq-stream-client = "0.4.3" tokio = { version = "1.39.0", features = ["rt", "rt-multi-thread", "macros"] } futures = "0.3.30" diff --git a/rust-stream/src/bin/receive_offset_tracking.rs b/rust-stream/src/bin/receive_offset_tracking.rs index ecafd9a7..cd2fe6ea 100644 --- a/rust-stream/src/bin/receive_offset_tracking.rs +++ b/rust-stream/src/bin/receive_offset_tracking.rs @@ -10,7 +10,7 @@ use tokio::task; async fn main() -> Result<(), Box> { use rabbitmq_stream_client::Environment; let environment = Environment::builder().build().await?; - let stream = "stream-offset-tracking-rust"; + let stream = "pippo"; let first_offset = Arc::new(AtomicI64::new(-1)); let last_offset = Arc::new(AtomicI64::new(-1)); let notify_on_close = Arc::new(Notify::new()); @@ -33,17 +33,18 @@ async fn main() -> Result<(), Box> { } } + let stored_offset:u64 = 45; let mut consumer = environment .consumer() .name("consumer-1") - .offset(OffsetSpecification::First) + .offset(OffsetSpecification::Offset(stored_offset)) .build(stream) .await .unwrap(); println!("Started consuming"); - let mut stored_offset: u64 = consumer.query_offset().await.unwrap_or_else(|_| 0); + /*let mut stored_offset: u64 = consumer.query_offset().await.unwrap_or_else(|_| 0); if stored_offset > 0 { stored_offset += 1; @@ -51,10 +52,10 @@ async fn main() -> Result<(), Box> { consumer = environment .consumer() .name("consumer-1") - .offset(OffsetSpecification::Offset(stored_offset)) + .offset(OffsetSpecification::Offset(42)) .build(stream) .await - .unwrap(); + .unwrap();*/ let first_cloned_offset = first_offset.clone(); let last_cloned_offset = last_offset.clone(); @@ -65,6 +66,7 @@ async fn main() -> Result<(), Box> { while let Some(delivery) = consumer.next().await { let d = delivery.unwrap(); + println!("offset {} ", d.offset()); if first_offset.load(Ordering::Relaxed) == -1 { println!("First message received"); _ = first_offset.compare_exchange( @@ -74,19 +76,19 @@ async fn main() -> Result<(), Box> { Ordering::Relaxed, ); } - received_messages = received_messages + 1; - if received_messages % 10 == 0 - || String::from_utf8_lossy(d.message().data().unwrap()).contains("marker") + //received_messages = received_messages + 1; + if String::from_utf8_lossy(d.message().data().unwrap()).contains("marker") { - let _ = consumer + /*let _ = consumer .store_offset(d.offset()) .await - .unwrap_or_else(|e| println!("Err: {}", e)); + .unwrap_or_else(|e| println!("Err: {}", e));*/ if String::from_utf8_lossy(d.message().data().unwrap()).contains("marker") { last_offset.store(d.offset() as i64, Ordering::Relaxed); let handle = consumer.handle(); _ = handle.close().await; notify_on_close_cloned.notify_one(); + break; } } } diff --git a/rust-stream/src/bin/send_offset_tracking.rs b/rust-stream/src/bin/send_offset_tracking.rs index d40be89e..bc5b8e2c 100644 --- a/rust-stream/src/bin/send_offset_tracking.rs +++ b/rust-stream/src/bin/send_offset_tracking.rs @@ -8,7 +8,7 @@ use tokio::sync::Notify; async fn main() -> Result<(), Box> { use rabbitmq_stream_client::Environment; let environment = Environment::builder().build().await?; - let stream = "stream-offset-tracking-rust"; + let stream = "pippo"; let message_count = 100; let confirmed_messages = Arc::new(AtomicU32::new(0)); let notify_on_send = Arc::new(Notify::new()); pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy