🎓 A1Learner
🔍
0 of 46 lessons completed

Elixir Restart Intensity

Restart intensity is the limit a supervisor places on how many restarts it allows in a time window. When restarts pass that limit, the supervisor gives up and stops itself instead of restarting the worker again.

Three crashes pass a limit of two

The supervisor allows two restarts in five seconds. A third quick crash passes the limit, so the supervisor stops itself.

Logger.configure(level: :none)
Process.flag(:trap_exit, true)
defmodule Waiter do
  def until_named(name) do
    if Process.whereis(name) == nil do
      Process.sleep(10)
      until_named(name)
    else
      :ok
    end
  end
  def until_dead(pid) do
    if Process.alive?(pid) do
      Process.sleep(10)
      until_dead(pid)
    else
      :ok
    end
  end
end
defmodule Flaky do
  use GenServer
  def start_link(arg), do: GenServer.start_link(__MODULE__, arg, name: __MODULE__)
  def init(count), do: {:ok, count}
  def handle_call(:get, _from, count), do: {:reply, count, count}
  def handle_cast(:boom, _state), do: raise "boom"
end
{:ok, sup} = Supervisor.start_link([%{id: Flaky, start: {Flaky, :start_link, [0]}}], strategy: :one_for_one, max_restarts: 2, max_seconds: 5)
old = Process.whereis(Flaky)
GenServer.cast(Flaky, :boom)
Waiter.until_dead(old)
Waiter.until_named(Flaky)
old = Process.whereis(Flaky)
GenServer.cast(Flaky, :boom)
Waiter.until_dead(old)
Waiter.until_named(Flaky)
GenServer.cast(Flaky, :boom)
Waiter.until_dead(sup)
IO.puts("supervisor alive: #{Process.alive?(sup)}")

Two crashes stay inside a limit of two

The supervisor allows two restarts in five seconds. Two quick crashes use the limit exactly, so the supervisor keeps running.

Logger.configure(level: :none)
Process.flag(:trap_exit, true)
defmodule Waiter do
  def until_named(name) do
    if Process.whereis(name) == nil do
      Process.sleep(10)
      until_named(name)
    else
      :ok
    end
  end
  def until_dead(pid) do
    if Process.alive?(pid) do
      Process.sleep(10)
      until_dead(pid)
    else
      :ok
    end
  end
end
defmodule Flaky do
  use GenServer
  def start_link(arg), do: GenServer.start_link(__MODULE__, arg, name: __MODULE__)
  def init(count), do: {:ok, count}
  def handle_call(:get, _from, count), do: {:reply, count, count}
  def handle_cast(:boom, _state), do: raise "boom"
end
{:ok, sup} = Supervisor.start_link([%{id: Flaky, start: {Flaky, :start_link, [0]}}], strategy: :one_for_one, max_restarts: 2, max_seconds: 5)
old = Process.whereis(Flaky)
GenServer.cast(Flaky, :boom)
Waiter.until_dead(old)
Waiter.until_named(Flaky)
old = Process.whereis(Flaky)
GenServer.cast(Flaky, :boom)
Waiter.until_dead(old)
Waiter.until_named(Flaky)
IO.puts("supervisor alive: #{Process.alive?(sup)}")
Supervisor.stop(sup)

A limit of zero allows no restarts

The supervisor allows zero restarts in five seconds. One crash is already too many, so the supervisor stops itself at once.

Logger.configure(level: :none)
Process.flag(:trap_exit, true)
defmodule Waiter do
  def until_named(name) do
    if Process.whereis(name) == nil do
      Process.sleep(10)
      until_named(name)
    else
      :ok
    end
  end
  def until_dead(pid) do
    if Process.alive?(pid) do
      Process.sleep(10)
      until_dead(pid)
    else
      :ok
    end
  end
end
defmodule Flaky do
  use GenServer
  def start_link(arg), do: GenServer.start_link(__MODULE__, arg, name: __MODULE__)
  def init(count), do: {:ok, count}
  def handle_call(:get, _from, count), do: {:reply, count, count}
  def handle_cast(:boom, _state), do: raise "boom"
end
{:ok, sup} = Supervisor.start_link([%{id: Flaky, start: {Flaky, :start_link, [0]}}], strategy: :one_for_one, max_restarts: 0, max_seconds: 5)
GenServer.cast(Flaky, :boom)
Waiter.until_dead(sup)
IO.puts("supervisor alive: #{Process.alive?(sup)}")

Example

Logger.configure(level: :none)
Process.flag(:trap_exit, true)
defmodule Waiter do
  def until_named(name) do
    if Process.whereis(name) == nil do
      Process.sleep(10)
      until_named(name)
    else
      :ok
    end
  end
  def until_dead(pid) do
    if Process.alive?(pid) do
      Process.sleep(10)
      until_dead(pid)
    else
      :ok
    end
  end
end
defmodule Flaky do
  use GenServer
  def start_link(arg), do: GenServer.start_link(__MODULE__, arg, name: __MODULE__)
  def init(count), do: {:ok, count}
  def handle_call(:get, _from, count), do: {:reply, count, count}
  def handle_cast(:boom, _state), do: raise "boom"
end
{:ok, sup} = Supervisor.start_link([%{id: Flaky, start: {Flaky, :start_link, [0]}}], strategy: :one_for_one, max_restarts: 2, max_seconds: 5)
old = Process.whereis(Flaky)
GenServer.cast(Flaky, :boom)
Waiter.until_dead(old)
Waiter.until_named(Flaky)
IO.puts("after 1 crash: #{Process.alive?(sup)}")
old = Process.whereis(Flaky)
GenServer.cast(Flaky, :boom)
Waiter.until_dead(old)
Waiter.until_named(Flaky)
IO.puts("after 2 crashes: #{Process.alive?(sup)}")
GenServer.cast(Flaky, :boom)
Waiter.until_dead(sup)
IO.puts("after 3 crashes: #{Process.alive?(sup)}")

Key points

  • Max restarts: the largest number of restarts allowed inside the window.
  • Max seconds: the length of the window in seconds.
  • Giving up: the supervisor stops itself when restarts pass the limit.
  • Boundary: restarts exactly at the limit still leave the supervisor running.
  • Repeated failure: a worker that crashes again and again quickly reaches the limit.

In this example, a supervisor allows two restarts in five seconds and its worker crashes three times in a row. After the first crash the supervisor is alive, and after the second crash it is still alive. After the third crash it is gone, which shows the limit being passed.

Note

a parent that traps exits can keep running after the supervisor it started gives up.

Try it Yourself

Edit the code, then press Run.

Running is simulated

Exercise

A supervisor allows two restarts in five seconds, and its worker crashes three times quickly. What happens to the supervisor?