Skip to content

Commit 757bc5e

Browse files
acogoluegnesacogoluegnes
authored
Merge pull request #421 from rabbitmq/stream-tutorial-2
Stream tutorial 2
2 parents b4d6186 + 8f674f6 commit 757bc5e

File tree

15 files changed

+379
-6
lines changed

15 files changed

+379
-6
lines changed

‎.gitignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,3 +35,7 @@ target/
3535
*.log
3636
.packages
3737
.python-version
38+
39+
.classpath
40+
.project
41+
.settings

‎dotnet-stream/OffsetTrackingReceive/OffsetTrackingReceive.cs

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
using System.Text;
2+
using System.Threading;
3+
using System.Threading.Tasks;
4+
using RabbitMQ.Stream.Client;
5+
using RabbitMQ.Stream.Client.Reliable;
6+
7+
var streamSystem = await StreamSystem.Create(new StreamSystemConfig());
8+
9+
var stream = "stream-offset-tracking-dotnet";
10+
await streamSystem.CreateStream(new StreamSpec(stream));
11+
12+
var consumerName = "offset-tracking-tutorial";
13+
IOffsetType offsetSpecification;
14+
try {
15+
ulong storedOffset = await streamSystem.QueryOffset(consumerName, stream).ConfigureAwait(false);
16+
offsetSpecification = new OffsetTypeOffset(storedOffset + 1);
17+
} catch (OffsetNotFoundException) {
18+
offsetSpecification = new OffsetTypeFirst();
19+
}
20+
ulong initialValue = UInt64.MaxValue;
21+
ulong firstOffset = initialValue;
22+
int messageCount = 0;
23+
ulong lastOffset = initialValue;
24+
var consumedCde = new CountdownEvent(1);
25+
var consumer = await Consumer.Create(new ConsumerConfig(streamSystem, stream)
26+
{
27+
OffsetSpec = offsetSpecification,
28+
Reference = consumerName,
29+
MessageHandler = async (_, consumer, context, message) => {
30+
if (Interlocked.CompareExchange(ref firstOffset, context.Offset, initialValue) == initialValue) {
31+
Console.WriteLine("First message received.");
32+
}
33+
if (Interlocked.Increment(ref messageCount) % 10 == 0) {
34+
await consumer.StoreOffset(context.Offset).ConfigureAwait(false);
35+
}
36+
if ("marker".Equals(Encoding.UTF8.GetString(message.Data.Contents))) {
37+
Interlocked.Exchange(ref lastOffset, context.Offset);
38+
await consumer.StoreOffset(context.Offset).ConfigureAwait(false);
39+
await consumer.Close();
40+
consumedCde.Signal();
41+
}
42+
await Task.CompletedTask;
43+
}
44+
});
45+
Console.WriteLine("Started consuming...");
46+
47+
consumedCde.Wait();
48+
Console.WriteLine("Done consuming, first offset {0}, last offset {1}", firstOffset, lastOffset);
49+
await streamSystem.Close();

‎dotnet-stream/OffsetTrackingReceive/OffsetTrackingReceive.csproj

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<OutputType>Exe</OutputType>
5+
<TargetFramework>net8.0</TargetFramework>
6+
<ImplicitUsings>enable</ImplicitUsings>
7+
<Nullable>enable</Nullable>
8+
</PropertyGroup>
9+
10+
<ItemGroup>
11+
<PackageReference Include="RabbitMQ.Stream.Client" Version="1.8.7" />
12+
</ItemGroup>
13+
14+
</Project>

‎dotnet-stream/OffsetTrackingSend/OffsetTrackingSend.cs

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
using System.Text;
2+
using System.Threading.Tasks;
3+
using RabbitMQ.Stream.Client;
4+
using RabbitMQ.Stream.Client.Reliable;
5+
6+
var streamSystem = await StreamSystem.Create(new StreamSystemConfig());
7+
8+
var stream = "stream-offset-tracking-dotnet";
9+
await streamSystem.CreateStream(new StreamSpec(stream));
10+
11+
var messageCount = 100;
12+
var confirmedCde = new CountdownEvent(messageCount);
13+
var producer = await Producer.Create(new ProducerConfig(streamSystem, stream) {
14+
ConfirmationHandler = async confirmation => {
15+
if (confirmation.Status == ConfirmationStatus.Confirmed) {
16+
confirmedCde.Signal();
17+
}
18+
await Task.CompletedTask.ConfigureAwait(false);
19+
}
20+
});
21+
22+
Console.WriteLine("Publishing {0} messages...", messageCount);
23+
for (int i = 0; i < messageCount; i++) {
24+
var body = i == messageCount - 1 ? "marker" : "hello";
25+
await producer.Send(new Message(Encoding.UTF8.GetBytes(body)));
26+
}
27+
28+
confirmedCde.Wait();
29+
Console.WriteLine("Messages confirmed.");
30+
await producer.Close();
31+
await streamSystem.Close();

‎dotnet-stream/OffsetTrackingSend/OffsetTrackingSend.csproj

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<OutputType>Exe</OutputType>
5+
<TargetFramework>net8.0</TargetFramework>
6+
<ImplicitUsings>enable</ImplicitUsings>
7+
<Nullable>enable</Nullable>
8+
</PropertyGroup>
9+
10+
<ItemGroup>
11+
<PackageReference Include="RabbitMQ.Stream.Client" Version="1.8.7" />
12+
</ItemGroup>
13+
14+
</Project>

‎go-stream/README.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,4 +23,9 @@ Code examples are executed via `go run`:
2323
go run receive.go
2424

2525

26+
[Tutorial two: Offset Tracking](https://www.rabbitmq.com/tutorials/tutorial-two-go-stream.html):
27+
28+
go run offset_tracking_send.go
29+
go run offset_tracking_receive.go
30+
2631
To learn more, see [`rabbitmq/rabbitmq-stream-go-client`](https://github.com/rabbitmq/rabbitmq-stream-go-client).

‎go-stream/go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@ module github.com/rabbitmq/rabbitmq-tutorials
22

33
go 1.22.0
44

5-
require github.com/rabbitmq/rabbitmq-stream-go-client v1.4.1
5+
require github.com/rabbitmq/rabbitmq-stream-go-client v1.4.6
66

77
require (
88
github.com/golang/snappy v0.0.4 // indirect
9-
github.com/klauspost/compress v1.17.8 // indirect
9+
github.com/klauspost/compress v1.17.9 // indirect
1010
github.com/pierrec/lz4 v2.6.1+incompatible // indirect
1111
github.com/pkg/errors v0.9.1 // indirect
1212
github.com/spaolacci/murmur3 v1.1.0 // indirect

‎go-stream/go.sum

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,15 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
22
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
33
github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
44
github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
5+
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
6+
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
57
github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM=
68
github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
79
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
810
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
911
github.com/rabbitmq/rabbitmq-stream-go-client v1.4.1 h1:QWHiXPio2rhQU98TKFhyooGWH2Xviwwz9RFGdIpIlUg=
1012
github.com/rabbitmq/rabbitmq-stream-go-client v1.4.1/go.mod h1:CbFPhxC2+ctPq0FmWD3CiZWWGiP7sP4joRpw4YTlyL4=
13+
github.com/rabbitmq/rabbitmq-stream-go-client v1.4.6 h1:aStxoALHUFdUouWWtG6f82rW0VE6885q2SGLKGOCZ50=
14+
github.com/rabbitmq/rabbitmq-stream-go-client v1.4.6/go.mod h1:CDeYQ8E+cC96SWnkmdDc5NVEWjmtRBA9JAQrMG/y+MI=
1115
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
1216
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=

‎go-stream/offset_tracking_receive.go

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
package main
2+
3+
import (
4+
"errors"
5+
"fmt"
6+
"github.com/rabbitmq/rabbitmq-stream-go-client/pkg/amqp"
7+
"github.com/rabbitmq/rabbitmq-stream-go-client/pkg/stream"
8+
"os"
9+
"sync/atomic"
10+
)
11+
12+
func main() {
13+
env, err := stream.NewEnvironment(
14+
stream.NewEnvironmentOptions().
15+
SetHost("localhost").
16+
SetPort(5552).
17+
SetUser("guest").
18+
SetPassword("guest"))
19+
CheckErrReceive(err)
20+
21+
streamName := "stream-offset-tracking-go"
22+
err = env.DeclareStream(streamName,
23+
&stream.StreamOptions{
24+
MaxLengthBytes: stream.ByteCapacity{}.GB(2),
25+
},
26+
)
27+
CheckErrReceive(err)
28+
29+
var firstOffset int64 = -1
30+
var messageCount int64 = -1
31+
var lastOffset atomic.Int64
32+
ch := make(chan bool)
33+
messagesHandler := func(consumerContext stream.ConsumerContext, message *amqp.Message) {
34+
if atomic.CompareAndSwapInt64(&firstOffset, -1, consumerContext.Consumer.GetOffset()) {
35+
fmt.Println("First message received.")
36+
}
37+
if atomic.AddInt64(&messageCount, 1)%10 == 0 {
38+
_ = consumerContext.Consumer.StoreOffset()
39+
}
40+
if string(message.GetData()) == "marker" {
41+
lastOffset.Store(consumerContext.Consumer.GetOffset())
42+
_ = consumerContext.Consumer.StoreOffset()
43+
_ = consumerContext.Consumer.Close()
44+
ch <- true
45+
}
46+
}
47+
48+
var offsetSpecification stream.OffsetSpecification
49+
consumerName := "offset-tracking-tutorial"
50+
storedOffset, err := env.QueryOffset(consumerName, streamName)
51+
if errors.Is(err, stream.OffsetNotFoundError) {
52+
offsetSpecification = stream.OffsetSpecification{}.First()
53+
} else {
54+
offsetSpecification = stream.OffsetSpecification{}.Offset(storedOffset + 1)
55+
}
56+
57+
_, err = env.NewConsumer(streamName, messagesHandler,
58+
stream.NewConsumerOptions().
59+
SetManualCommit().
60+
SetConsumerName(consumerName).
61+
SetOffset(offsetSpecification))
62+
CheckErrReceive(err)
63+
fmt.Println("Started consuming...")
64+
_ = <-ch
65+
fmt.Printf("Done consuming, first offset %d, last offset %d.\n", firstOffset, lastOffset.Load())
66+
}
67+
68+
func CheckErrReceive(err error) {
69+
if err != nil {
70+
fmt.Printf("%s ", err)
71+
os.Exit(1)
72+
}
73+
}

‎go-stream/offset_tracking_send.go

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"github.com/rabbitmq/rabbitmq-stream-go-client/pkg/amqp"
6+
"github.com/rabbitmq/rabbitmq-stream-go-client/pkg/stream"
7+
"os"
8+
)
9+
10+
func main() {
11+
env, err := stream.NewEnvironment(
12+
stream.NewEnvironmentOptions().
13+
SetHost("localhost").
14+
SetPort(5552).
15+
SetUser("guest").
16+
SetPassword("guest"))
17+
CheckErrSend(err)
18+
19+
streamName := "stream-offset-tracking-go"
20+
err = env.DeclareStream(streamName,
21+
&stream.StreamOptions{
22+
MaxLengthBytes: stream.ByteCapacity{}.GB(2),
23+
},
24+
)
25+
CheckErrSend(err)
26+
27+
producer, err := env.NewProducer(streamName, stream.NewProducerOptions())
28+
CheckErrSend(err)
29+
30+
messageCount := 100
31+
chPublishConfirm := producer.NotifyPublishConfirmation()
32+
ch := make(chan bool)
33+
handlePublishConfirm(chPublishConfirm, messageCount, ch)
34+
35+
fmt.Printf("Publishing %d messages...\n", messageCount)
36+
for i := 0; i < messageCount; i++ {
37+
var body string
38+
if i == messageCount-1 {
39+
body = "marker"
40+
} else {
41+
body = "hello"
42+
}
43+
err = producer.Send(amqp.NewMessage([]byte(body)))
44+
CheckErrSend(err)
45+
}
46+
_ = <-ch
47+
fmt.Println("Messages confirmed.")
48+
49+
err = producer.Close()
50+
CheckErrSend(err)
51+
}
52+
53+
func handlePublishConfirm(confirms stream.ChannelPublishConfirm, messageCount int, ch chan bool) {
54+
go func() {
55+
confirmedCount := 0
56+
for confirmed := range confirms {
57+
for _, msg := range confirmed {
58+
if msg.IsConfirmed() {
59+
confirmedCount++
60+
if confirmedCount == messageCount {
61+
ch <- true
62+
}
63+
}
64+
}
65+
}
66+
}()
67+
}
68+
69+
func CheckErrSend(err error) {
70+
if err != nil {
71+
fmt.Printf("%s ", err)
72+
os.Exit(1)
73+
}
74+
}

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy