Content-Length: 548083 | pFad | http://github.com/electric-sql/electric/commit/50f3ddc2ab3c2a02146def90b879cb298cb59b7b

C7 feat(sync-service): Configure GC telemetry via env vars (#2836) · electric-sql/electric@50f3ddc · GitHub
Skip to content

Commit 50f3ddc

Browse files
robacourtrobacourt
authored
feat(sync-service): Configure GC telemetry via env vars (#2836)
1 parent 37c5902 commit 50f3ddc

File tree

6 files changed

+41
-15
lines changed

6 files changed

+41
-15
lines changed

‎.changeset/clever-rules-clap.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@core/sync-service": patch
3+
---
4+
5+
Add env vars to configure telemetry: ELECTRIC_TELEMETRY_LONG_GC_THRESHOLD, ELECTRIC_TELEMETRY_LONG_SCHEDULE_THRESHOLD, ELECTRIC_TELEMETRY_LONG_MESSAGE_QUEUE_ENABLE_THRESHOLD and ELECTRIC_TELEMETRY_LONG_MESSAGE_QUEUE_DISABLE_THRESHOLD

‎packages/sync-service/config/runtime.exs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,13 @@ config :electric,
188188
otel_export_period: otel_export_period,
189189
otel_per_process_metrics?: env!("ELECTRIC_OTEL_PER_PROCESS_METRICS", :boolean, nil),
190190
telemetry_top_process_count: env!("ELECTRIC_TELEMETRY_TOP_PROCESS_COUNT", :integer, nil),
191+
telemetry_long_gc_threshold: env!("ELECTRIC_TELEMETRY_LONG_GC_THRESHOLD", :integer, nil),
192+
telemetry_long_schedule_threshold:
193+
env!("ELECTRIC_TELEMETRY_LONG_SCHEDULE_THRESHOLD", :integer, nil),
194+
telemetry_long_message_queue_enable_threshold:
195+
env!("ELECTRIC_TELEMETRY_LONG_MESSAGE_QUEUE_ENABLE_THRESHOLD", :integer, nil),
196+
telemetry_long_message_queue_disable_threshold:
197+
env!("ELECTRIC_TELEMETRY_LONG_MESSAGE_QUEUE_DISABLE_THRESHOLD", :integer, nil),
191198
telemetry_statsd_host: statsd_host,
192199
prometheus_port: prometheus_port,
193200
db_pool_size: env!("ELECTRIC_DB_POOL_SIZE", :integer, nil),

‎packages/sync-service/lib/electric/application.ex

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -295,7 +295,13 @@ defmodule Electric.Application do
295295
otel_metrics?: not is_nil(Application.get_env(:otel_metric_exporter, :otlp_endpoint)),
296296
otel_export_period: get_env(opts, :otel_export_period),
297297
otel_per_process_metrics?: get_env(opts, :otel_per_process_metrics?),
298-
top_process_count: get_env(opts, :telemetry_top_process_count)
298+
top_process_count: get_env(opts, :telemetry_top_process_count),
299+
long_gc_threshold: get_env(opts, :telemetry_long_gc_threshold),
300+
long_schedule_threshold: get_env(opts, :telemetry_long_schedule_threshold),
301+
long_message_queue_enable_threshold:
302+
get_env(opts, :telemetry_long_message_queue_enable_threshold),
303+
long_message_queue_disable_threshold:
304+
get_env(opts, :telemetry_long_message_queue_disable_threshold)
299305
]
300306
end
301307
end

‎packages/sync-service/lib/electric/config.ex

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,10 @@ defmodule Electric.Config do
6666
otel_export_period: :timer.seconds(30),
6767
otel_per_process_metrics?: false,
6868
telemetry_top_process_count: 5,
69+
telemetry_long_gc_threshold: 500,
70+
telemetry_long_schedule_threshold: 500,
71+
telemetry_long_message_queue_enable_threshold: 1000,
72+
telemetry_long_message_queue_disable_threshold: 100,
6973
## Memory
7074
shape_hibernate_after: :timer.seconds(30),
7175
## Performance tweaks

‎packages/sync-service/lib/electric/telemetry/opts.ex

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,11 @@ defmodule Electric.Telemetry.Opts do
1010
otel_metrics?: [type: :boolean, default: false],
1111
otel_export_period: [type: :integer, default: :timer.seconds(30)],
1212
otel_per_process_metrics?: [type: :boolean, default: false],
13-
top_process_count: [type: :integer, default: 5]
13+
top_process_count: [type: :integer, default: 5],
14+
long_gc_threshold: [type: :integer, default: 500],
15+
long_schedule_threshold: [type: :integer, default: 500],
16+
long_message_queue_enable_threshold: [type: :integer, default: 1000],
17+
long_message_queue_disable_threshold: [type: :integer, default: 100]
1418
]
1519
end
1620
end

‎packages/sync-service/lib/electric/telemetry/system_monitor.ex

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,12 @@ defmodule Electric.Telemetry.SystemMonitor do
66
GenServer.start_link(__MODULE__, opts, name: __MODULE__)
77
end
88

9-
def init(_) do
10-
# Monitor for: GC taking more than 500ms, process not interrupted for more than 500ms, message queue length > 1000
9+
def init(opts) do
1110
:erlang.system_monitor(self(),
12-
long_gc: 500,
13-
long_schedule: 500,
14-
long_message_queue: {100, 1000}
11+
long_gc: opts.long_gc_threshold,
12+
long_schedule: opts.long_schedule_threshold,
13+
long_message_queue:
14+
{opts.long_message_queue_disable_threshold, opts.long_message_queue_enable_threshold}
1515
)
1616

1717
:timer.send_interval(2000, :report_long_queues)
@@ -20,12 +20,12 @@ defmodule Electric.Telemetry.SystemMonitor do
2020
end
2121

2222
def handle_info({:monitor, gc_pid, :long_gc, info}, state) do
23+
type = Electric.Debug.Process.type(gc_pid)
24+
2325
Logger.debug(
24-
"Long GC detected for pid #{inspect(gc_pid)}: took #{Keyword.fetch!(info, :timeout)}ms. #{inspect(info, limit: :infinity)}"
26+
"Long GC detected for pid #{inspect(gc_pid)} (#{inspect(type)}): took #{Keyword.fetch!(info, :timeout)}ms. #{inspect(info, limit: :infinity)}"
2527
)
2628

27-
type = Electric.Debug.Process.type(gc_pid)
28-
2929
:telemetry.execute([:vm, :monitor, :long_gc], Map.new(info), %{process_type: type})
3030
{:noreply, state}
3131
end
@@ -45,12 +45,12 @@ defmodule Electric.Telemetry.SystemMonitor do
4545
end
4646

4747
def handle_info({:monitor, pid, :long_schedule, info}, state) when is_pid(pid) do
48+
type = Electric.Debug.Process.type(pid)
49+
4850
Logger.debug(
49-
"Long schedule detected for pid #{inspect(pid)}, took #{Keyword.fetch!(info, :timeout)}ms"
51+
"Long schedule detected for pid #{inspect(pid)} (#{inspect(type)}), took #{Keyword.fetch!(info, :timeout)}ms"
5052
)
5153

52-
type = Electric.Debug.Process.type(pid)
53-
5454
:telemetry.execute(
5555
[:vm, :monitor, :long_schedule],
5656
%{timeout: Keyword.fetch!(info, :timeout)},
@@ -61,10 +61,10 @@ defmodule Electric.Telemetry.SystemMonitor do
6161
end
6262

6363
def handle_info({:monitor, pid, :long_message_queue, true}, state) do
64-
Logger.debug("Long message queue detected for pid #{inspect(pid)}")
65-
6664
type = Electric.Debug.Process.type(pid)
6765

66+
Logger.debug("Long message queue detected for pid #{inspect(pid)} (#{inspect(type)})")
67+
6868
:telemetry.execute([:vm, :monitor, :long_message_queue], %{present: 1}, %{process_type: type})
6969

7070
{:noreply,

0 commit comments

Comments
 (0)








ApplySandwichStrip

pFad - (p)hone/(F)rame/(a)nonymizer/(d)eclutterfier!      Saves Data!


--- a PPN by Garber Painting Akron. With Image Size Reduction included!

Fetched URL: http://github.com/electric-sql/electric/commit/50f3ddc2ab3c2a02146def90b879cb298cb59b7b

Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy