How to use leveled, a pure erlang leveldb implementation

Yesterday at the riak_core tutorial at CodeBEAMSF I was trying to implement a leveled based backend for the key value store we were building, I was having troubles with leveled crashing when trying to destroy it (stop and remove files in leveled parlance), after fighting for a while I needed a smaller example to see if it was my mistake or not.

I decided to do the smaller example and to share the process here.

First we need some erlang application to hold our leveled dependency and configuration, let's do it by creating an erlang release with rebar3:

rebar3 new release name=lvld
cd lvld

Now that the skeleton is ready, we need to change rebar.config to add the information to use leveled, the resulting rebar.config below, see comments:

{erl_opts, [debug_info]}.

{deps, [
    % add leveled dependency
    {leveled, {git, "", {branch, "master"}}}

{relx, [{release, { lvld, "0.1.0" },
    % leveled needs crypto
    % make sure to load leveled, don't start it, it's not an app
    {leveled, load},
    % required by leveled
    {lz4, load},

    {sys_config, "./config/sys.config"},
    {vm_args, "./config/vm.args"},

    {dev_mode, true},
    {include_erts, false},

    {extended_start_script, true}]

{profiles, [{prod, [{relx, [{dev_mode, false},
    {include_erts, true}]}]

% leveled generates lots of warnings and has warnings_as_errors set, we need
% to override that by copying the erl_opts field without warnings_as_errors
    [{override, leveled,
        [{erl_opts, [{platform_define, "^1[7-8]{1}", old_rand},
            {platform_define, "^R", old_rand},
            {platform_define, "^R", no_sync}]}]}

We will build a wrapper for leveled that exposes a simple kv store in apps/lvld/src/lvld_kv.erl:

-export([new/1, get/3, put/4, delete/3, keys/2, close/1, delete/1, is_empty/1, foldl/3]).


-record(state, {bookie, base_path}).

new(Opts=#{path := Path}) ->
    LedgerCacheSize = maps:get(ledger_cache_size, Opts, 2000),
    JournalSize = maps:get(journal_size, Opts, 500000000),
    SyncStrategy = maps:get(sync_strategy, Opts, none),
    {ok, Bookie} = leveled_bookie:book_start(Path, LedgerCacheSize,
                                             JournalSize, SyncStrategy),
    State = #state{bookie=Bookie, base_path=Path},
    {ok, State}.

put(State=#state{bookie=Bookie}, Bucket, Key, Value) ->
    R = leveled_bookie:book_put(Bookie, Bucket, Key, Value, []),
    {R, State}.

get(State=#state{bookie=Bookie}, Bucket, Key) ->
    K = {Bucket, Key},
    Res = case leveled_bookie:book_get(Bookie, Bucket, Key) of
              not_found -> {not_found, K};
              {ok, Value} -> {found, {K, Value}}
    {Res, State}.

delete(State=#state{bookie=Bookie}, Bucket, Key) ->
    R = leveled_bookie:book_delete(Bookie, Bucket, Key, []),
    {R, State}.

keys(State=#state{bookie=Bookie}, Bucket) ->
    FoldHeadsFun = fun(_B, K, _ProxyV, Acc) -> [K | Acc] end,
    {async, FoldFn} = leveled_bookie:book_returnfolder(Bookie,
                                true, true, false}),
    Keys = FoldFn(),
    {Keys, State}.

is_empty(State=#state{bookie=Bookie}) ->
    FoldBucketsFun = fun(B, Acc) -> [B | Acc] end,
    {async, FoldFn} = leveled_bookie:book_returnfolder(Bookie,
                                                        {FoldBucketsFun, []}}),
    IsEmpty = case FoldFn() of
                  [] -> true;
                  _ -> false
    {IsEmpty, State}.

close(State=#state{bookie=Bookie}) ->
    R = leveled_bookie:book_close(Bookie),
    {R, State}.

delete(State=#state{base_path=Path}) ->
    R = remove_path(Path),
    {R, State}.

foldl(Fun, Acc0, State=#state{bookie=Bookie}) ->
    FoldObjectsFun = fun(B, K, V, Acc) -> Fun({{B, K}, V}, Acc) end,
    {async, FoldFn} = leveled_bookie:book_returnfolder(Bookie, {foldobjects_allkeys,
                                                                {FoldObjectsFun, Acc0},
    AccOut = FoldFn(),
    {AccOut, State}.

% private functions

sub_files(From) ->
    {ok, SubFiles} = file:list_dir(From),
    [filename:join(From, SubFile) || SubFile <- SubFiles].

remove_path(Path) ->
    case filelib:is_dir(Path) of
        false ->
        true ->
            lists:foreach(fun(ChildPath) -> remove_path(ChildPath) end,

We are ready to build a release and try our kv api on the repl:

rebar3 release
./_build/default/rel/lvld/bin/lvld console

This is the code we will run in the repl, I put it here so it's easy to read and copy and paste:

Nums = lists:seq(1, 10).
Buckets = lists:map(fun (N) -> list_to_binary("bucket-" ++ integer_to_list(N)) end,
Keys = lists:map(fun (N) -> list_to_binary("key-" ++ integer_to_list(N)) end, Nums).

GenValue = fun (Bucket, Key) -> <<"v/", Bucket/binary, "/", Key/binary>> end.

{ok, Kv} = lvld_kv:new(#{path => "/tmp/lvld_test"}).
lists:foreach(fun (Bucket) ->
        lists:foreach(fun (Key) ->
                Val = GenValue(Bucket, Key),
                lvld_kv:put(Kv, Bucket, Key, Val)
        end, Keys)
end, Buckets).

B1 = <<"bucket-1">>.
K1 = <<"key-1">>.
V1 = <<"value-1">>.
B2 = <<"bucket-2">>.
K2 = <<"key-2">>.

FoldFn = fun ({{B, K}, V}, AccIn) -> [{B, K, V} | AccIn] end.
lvld_kv:foldl(FoldFn, [], Kv).

lvld_kv:put(Kv, B1, K1, V1).
lvld_kv:get(Kv, B1, K1).
lvld_kv:delete(Kv, B1, K1).
lvld_kv:get(Kv, B1, K1).

lvld_kv:keys(Kv, B1).


The results of running it (removing some of the verbose logging):

(lvld@ganesha)1> Nums = lists:seq(1, 10).


(lvld@ganesha)2> Buckets = lists:map(fun (N) -> list_to_binary("bucket-" ++ integer_to_list(N)) end, Nums).


(lvld@ganesha)3> Keys = lists:map(fun (N) -> list_to_binary("key-" ++ integer_to_list(N)) end, Nums).


(lvld@ganesha)4> GenValue = fun (Bucket, Key) -> <<"v/", Bucket/binary, "/", Key/binary>> end.


(lvld@ganesha)5> {ok, Kv} = lvld_kv:new(#{path => "/tmp/lvld_test"}).


(lvld@ganesha)6> B1 = <<"bucket-1">>.

(lvld@ganesha)7> K1 = <<"key-1">>.

(lvld@ganesha)8> V1 = <<"value-1">>.

(lvld@ganesha)9> B2 = <<"bucket-2">>.

(lvld@ganesha)10> K2 = <<"key-2">>.

(lvld@ganesha)11> FoldFn = fun ({{B, K}, V}, AccIn) -> [{B, K, V} | AccIn] end.

(lvld@ganesha)13> lists:foreach(fun (Bucket) ->
(lvld@ganesha)13>         lists:foreach(fun (Key) ->
(lvld@ganesha)13>                 Val = GenValue(Bucket, Key),
(lvld@ganesha)13>                 lvld_kv:put(Kv, Bucket, Key, Val)
(lvld@ganesha)13>         end, Keys)
(lvld@ganesha)13> end, Buckets).

(lvld@ganesha)14> lvld_kv:foldl(FoldFn, [], Kv).



(lvld@ganesha)15> lvld_kv:put(Kv, B1, K1, V1).

(lvld@ganesha)16> lvld_kv:get(Kv, B1, K1).

(lvld@ganesha)17> lvld_kv:delete(Kv, B1, K1).

(lvld@ganesha)18> lvld_kv:get(Kv, B1, K1).

(lvld@ganesha)19> lvld_kv:keys(Kv, B1).

(lvld@ganesha)20> lvld_kv:close(Kv).

(lvld@ganesha)21> lvld_kv:delete(Kv).

In case you want to know the case for the crashing, when calling destroy on leveled, it returns destroy as reason for gen_server stop, which doesn't seem to make Erlang happy and it crashes the process and propagates the error.

The solution here is to just close it and remove the files myself (the difference between close and destroy is file removal).

ameo: Redis compatible GET,SET,DEL and PUBLISH/SUBSCRIBE on riak_core with WebSocket API

Last week I was invited by Erlang Solutions to give a talk at the London's Erlang Meetup and to help at a riak_core themed hackathon at their offices the next day.

The talk slides: Building distributed applications: riak_core vs partisan in case you are interested.

After giving the introduction to riak_core the teams started to work on their projects and since they were pretty busy and didn't need much help I decided to "participate" too by implementing an idea I had in mind for a while.

The result is called ameo and can be described as:

a Redis compatible, distributed in-memory key-value store and pubsub server
implemented using riak_core that exposes the topics via WebSockets.


Many languages make it hard to provide websocket connections and live connections with clients, they work on a request/response basis and/or make it really hard/expensive to handle multiple persistent connections.

I've seen solutions that involve starting a redis server and putting usually nodejs in the front to expose redis topics via websockets, this involves two moving parts and for many projects, managing nodejs which they may not have experience with.

The solution is to just start one instance of ameo or a cluster of ameos and expose the WebSocket API to clients and the Redis API to the servers.

Servers can use their preferred Redis client library and as long as they only use GET, PUT, DEL, PUBLISH, SUBSCRIBE and UNSUBSCRIBE it will look like they are talking to a Redis server.

On my way back I had some extra time at the airport so I implemented a basic web UI to play with the websocket client and to provide a reference implementation others can use.

You can see the result in this screencast:

Implementation details:

As said earlier I use riak_core for clustering, for the Redis part I took some modules from an Erlang implementation of Redis called edis and I created a library called edis_proto that allows any project to expose a Redis compatible API to their servers with a couple lines of code.

The WebSocket part is implemented using the Cowboy Web Server.

Playing with Lasp in a 3 Node Cluster

After playing with Lasp in a single node in the previous post: Playing with Lasp and CRDTs

I created a rebar3 template to easily setup a 3 node cluster to play with Lasp where it shines, in a distributed environment.

Install rebar3 if you haven't already.

then install this template:

mkdir -p ~/.config/rebar3/templates
git clone ~/.config/rebar3/templates/rebar3_template_lasp


rebar3 new rebar3_template_lasp name=laspy
cd laspy

    # build the 3 node cluster
make devrel

# on 3 different shells
make dev1-console
make dev2-console
make dev3-console

# join all nodes:
make devrel-join

# check node members
make devrel-status

On one of the nodes' shell run:

Key1 = <<"key1">>.
Key2 = <<"key2">>.
Timestamp = fun () -> erlang:unique_integer([monotonic, positive]) end.

AwMapType = {state_awmap, [state_mvregister]}.
AwMapVarName = <<"awmap">>.
AwMapVal = #{what => i_am_an_awmap_value}.

% declare the variable
{ok, {AwMap, _, _, _}} = lasp:declare({AwMapVarName, AwMapType}, AwMapType).

% update its content setting Key1 = AwMapVal
{ok, {AwMap1, _, _, _}} = lasp:update(AwMap, {apply, Key1,
                                              {set, Timestamp(), AwMapVal}},
% timestamp argument is not needed in mvregister, it's only for compatibility
% with lwwregister
{ok, _} = lasp:update(AwMap, {apply, Key1, {set, nil, AwMapVal}}, self()).

% get the value
{ok, AwMapRes} = lasp:query(AwMap1).

% {ok,[{<<"key1">>, {set, ...#{what => i_am_an_awmap_value} ... }}]}

[{_, AwMapSet}] = AwMapRes.
% [#{what => i_am_an_awmap_value}]

On another one run:

{ok, AwMapRes} = lasp:query({<<"awmap">>,{state_awmap,[state_mvregister]}}).


[{_, AwMapSet}] = AwMapRes.

You should get:

[#{what => i_am_an_awmap_value}]

All the examples from the previous post should run.

A termcast (?) of the process:

Happy Lasping!

Playing with Lasp and CRDTs

For a long time I was looking for some time to play with Lasp:

Lasp is a suite of libraries aimed at providing a comprehensive programming
system for planetary scale Elixir and Erlang applications.

Yesterday I had some time and played a little bit with it, and thanks to the help from @cmeik and @vitorenesduarte I got some code running and understood some things.

Here is what I achieved until now, note that I'm learning this stuff so I may say many things that are wrong, I will try to get people to review it and update it with the corrections.

Some initial snippets of code we will reuse:

% clear all variable bindings
Key1 = <<"key1">>.
Key2 = <<"key2">>.
Timestamp = fun () -> erlang:unique_integer([monotonic, positive]) end.

A map with any type as value

If we want to have data which can be represented as a dict/map/key-value mapping, there are multiple alternatives we can use in lasp, here we will explore some.

First we are going to use an AwMap CRDT as a map which is defined as:

A dictionary where keys can be anything and the values are causal-CRDTs.

Since values must be casual-CRDTs we can't use a bare erlang type as value nor a non causal-CRDT type like lwwregister, which we will use later with another map type.

AwMapType = {state_awmap, [state_mvregister]}.
AwMapVarName = <<"awmap">>.
AwMapVal = #{what => i_am_an_awmap_value}.

% declare the variable
{ok, {AwMap, _, _, _}} = lasp:declare({AwMapVarName, AwMapType}, AwMapType).

% update its content setting Key1 = AwMapVal
{ok, {AwMap1, _, _, _}} = lasp:update(AwMap, {apply, Key1,
                                              {set, Timestamp(), AwMapVal}},
% timestamp argument is not needed in mvregister, it's only for compatibility
% with lwwregister
{ok, _} = lasp:update(AwMap, {apply, Key1, {set, nil, AwMapVal}}, self()).

% get the value
{ok, AwMapRes} = lasp:query(AwMap1).

% {ok,[{<<"key1">>, {set, ...#{what => i_am_an_awmap_value} ... }}]}

[{_, AwMapSet}] = AwMapRes.
% [#{what => i_am_an_awmap_value}]

We can also use a GMap CRDT (Grow Only Map, keys can't be removed):

A dictionary where keys can be anything and the values are join-semilattices.

In this case we can use non causal-CRDTs as values, to make it a little different we are going to use another register type, a lwwregister

GMapType = {state_gmap, [state_lwwregister]}.
GMapVarName = <<"gmap">>.
GMapVal = #{what => i_am_a_gmap_value}.

{ok, {GMap, _, _, _}} = lasp:declare({GMapVarName, GMapType}, GMapType).
{ok, {GMap1, _, _, _}} = lasp:update(GMap, {apply, Key1,
                                            {set, Timestamp(), GMapVal}},

{ok, GMapRes} = lasp:query(GMap1).

% [{<<"key1">>,#{thing => 42,what => i_am_a_gmap_value}}]

[{_, GMapResVal}] = GMapRes.

% #{what => i_am_a_gmap_value}

We can use a gmap with an ivar as value type, an ivar is a register that can only be set once.

GMapIVarType = {state_gmap, [state_ivar]}.
GMapIVarVarName = <<"gmapivar">>.
GMapIVarVal1 = #{what => i_am_a_gmap_ivar_value}.
GMapIVarVal2 = #{what => i_am_a_gmap_ivar_update}.

{ok, {GMapIVar, _, _, _}} = lasp:declare({GMapIVarVarName, GMapIVarType}, GMapIVarType).
{ok, {GMapIVar1, _, _, _}} = lasp:update(GMapIVar, {apply, Key1,
                                            {set, GMapIVarVal1}},
% try updating it, will throw an error (the value of GMapIVar1 will be lost)
{ok, {GMapIVar2, _, _, _}} = lasp:update(GMapIVar1, {apply, Key1,
                                            {set, GMapIVarVal2}},

{ok, GMapIVarRes} = lasp:query(GMapIVar1).


[{_, GMapIVarResVal}] = GMapIVarRes.

% #{what => i_am_a_gmap_ivar_value}


My examples are all maps with some value type because that's the use case I'm most interested, here's a list of types and their operations:




Can be set to true once


BoolType = state_boolean.
BoolVarName = <<"boolvar">>.

{ok, {Bool, _, _, _}} = lasp:declare({BoolVarName, BoolType}, BoolType).
{ok, {Bool1, _, _, _}} = lasp:update(Bool, true, self()).

{ok, BoolRes} = lasp:query(Bool1).

% true


Enable-Wins Flag CRDT


Enable the flag
Disable the flag


EWType = state_ewflag.
EWVarName = <<"ewvar">>.

{ok, {EW, _, _, _}} = lasp:declare({EWVarName, EWType}, EWType).
{ok, {EW1, _, _, _}} = lasp:update(EW, enable, self()).
{ok, EWRes1} = lasp:query(EW1).
{ok, {EW2, _, _, _}} = lasp:update(EW1, disable, self()).
{ok, EWRes2} = lasp:query(EW2).
{ok, {EW3, _, _, _}} = lasp:update(EW2, enable, self()).
{ok, EWRes3} = lasp:query(EW3).

% true
% false
% true


Disable-Wins Flag CRDT

Follows the same strategy used in Enable-Wins Flag but,
instead of creating a new dot when enabling the flag,
we create a new dot when disabling it.


Enable the flag
Disable the flag


DWType = state_dwflag.
DWVarName = <<"dwvar">>.

{ok, {DW, _, _, _}} = lasp:declare({DWVarName, DWType}, DWType).
{ok, {DW1, _, _, _}} = lasp:update(DW, enable, self()).
{ok, DWRes1} = lasp:query(DW1).
{ok, {DW2, _, _, _}} = lasp:update(DW1, disable, self()).
{ok, DWRes2} = lasp:query(DW2).
{ok, {DW3, _, _, _}} = lasp:update(DW2, enable, self()).
{ok, DWRes3} = lasp:query(DW3).

% true
% false
% true

GCounter CRDT: grow only counter


Increment the counter by 1


GCountType = state_gcounter.
GCountVarName = <<"gcountvar">>.

{ok, {GCount, _, _, _}} = lasp:declare({GCountVarName, GCountType}, GCountType).
{ok, {GCount1, _, _, _}} = lasp:update(GCount, increment, self()).
{ok, GCountRes1} = lasp:query(GCount1).
{ok, {GCount2, _, _, _}} = lasp:update(GCount1, increment, self()).
{ok, GCountRes2} = lasp:query(GCount2).
{ok, {GCount3, _, _, _}} = lasp:update(GCount2, increment, self()).
{ok, GCountRes3} = lasp:query(GCount3).

% 1
% 2
% 3

Some more I don't get completely what they could be used for:

Bounded Counter CRDT:

Modeled as a pair where the first component is a PNCounter and the second
component is a GMap.

This counter has sub counter for different ids (actors), each of which can't go below 0, with this you can model things like seats or some resource where you allocate counts to different parties (actors) and each can decrement their own count but not others, also each counter can't go below 0, if a given actor needs to decrement it has to move counts from other actor.


{move, term()}
Moves permissions to decrement to another replica (if it has enough permissions)
Increment counter, can always happen
Decrement counter, can happen when the replica has enough local increments, or has permissions received from other replicas


BCountType = state_bcounter.
BCountVarName = <<"bcountvar">>.
Actor1 = self().
Actor2 = <<"actor2-id">>.

{ok, {BCount, _, _, _}} = lasp:declare({BCountVarName, BCountType}, BCountType).
{ok, {BCount1, _, _, _}} = lasp:update(BCount, increment, Actor1).
{ok, BCountRes1} = lasp:query(BCount1).
{ok, {BCount2, _, _, _}} = lasp:update(BCount1, increment, Actor2).
{ok, BCountRes2} = lasp:query(BCount2).
{ok, {BCount3, _, _, _}} = lasp:update(BCount2, decrement, Actor1).
{ok, BCountRes3} = lasp:query(BCount3).

% here Actor1 has counter set to 0, can't go below 0, if it want's to
% decrement it has to move a 1 from another actor, Actor2 has 1 in its
% counter so we will move it and then decrement

BCountMoveFrom = Actor2.
BCountMoveTo = Actor1.
{ok, {BCount4, _, _, _}} = lasp:update(BCount3, {move, 1, BCountMoveTo}, BCountMoveFrom).

% now we can decrement from Actor1
{ok, {BCount5, _, _, _}} = lasp:update(BCount4, decrement, Actor1).
{ok, BCountRes5} = lasp:query(BCount5).

% 1
% 2
% 1

% 0

Max Int CRDT


  • increment


MaxIntType = state_max_int.
MaxIntVarName = <<"maxintvar">>.

{ok, {MaxInt, _, _, _}} = lasp:declare({MaxIntVarName, MaxIntType}, MaxIntType).
{ok, {MaxInt1, _, _, _}} = lasp:update(MaxInt, increment, self()).
{ok, MaxIntRes1} = lasp:query(MaxInt1).
{ok, {MaxInt2, _, _, _}} = lasp:update(MaxInt1, increment, self()).
{ok, MaxIntRes2} = lasp:query(MaxInt2).
{ok, {MaxInt3, _, _, _}} = lasp:update(MaxInt2, increment, self()).
{ok, MaxIntRes3} = lasp:query(MaxInt3).

% 1
% 2
% 3

Lexicographic Counter


  • increment
  • decrement


LCountType = state_lexcounter.
LCountVarName = <<"lexcountvar">>.

{ok, {LCount, _, _, _}} = lasp:declare({LCountVarName, LCountType}, LCountType).
{ok, {LCount1, _, _, _}} = lasp:update(LCount, increment, self()).
{ok, LCountRes1} = lasp:query(LCount1).
{ok, {LCount2, _, _, _}} = lasp:update(LCount1, increment, self()).
{ok, LCountRes2} = lasp:query(LCount2).
{ok, {LCount3, _, _, _}} = lasp:update(LCount2, decrement, self()).
{ok, LCountRes3} = lasp:query(LCount3).

% 1
% 2
% 1

PNCounter CRDT:

Counter that allows both increments and decrements.

Modeled as a dictionary where keys are replicas ids and values are pairs
where the first component is the number of increments and the second component
is the number of decrements.

An actor may only update its own entry in the dictionary.

The value of the counter is the sum of all first components minus the sum
of all second components.


  • increment
  • decrement


PNCountType = state_pncounter.
PNCountVarName = <<"pncountvar">>.

{ok, {PNCount, _, _, _}} = lasp:declare({PNCountVarName, PNCountType}, PNCountType).
{ok, {PNCount1, _, _, _}} = lasp:update(PNCount, increment, self()).
{ok, PNCountRes1} = lasp:query(PNCount1).
{ok, {PNCount2, _, _, _}} = lasp:update(PNCount1, increment, self()).
{ok, PNCountRes2} = lasp:query(PNCount2).
{ok, {PNCount3, _, _, _}} = lasp:update(PNCount2, decrement, self()).
{ok, PNCountRes3} = lasp:query(PNCount3).

% 1
% 2
% 1



We assume timestamp are unique, totally ordered and consistent
with causal order. We use integers as timestamps.
When using this, make sure you provide globally unique timestamps.


{set, Timestamp, Value}
Set register to Value


LWWRegType = state_lwwregister.
LWWRegVarName = <<"lwwregister">>.

{ok, {LWWReg, _, _, _}} = lasp:declare({LWWRegVarName, LWWRegType}, LWWRegType).
{ok, {LWWReg1, _, _, _}} = lasp:update(LWWReg, {set, Timestamp(), foo}, self()).
{ok, LWWRegRes1} = lasp:query(LWWReg1).
{ok, {LWWReg2, _, _, _}} = lasp:update(LWWReg1, {set, Timestamp(), bar}, self()).
{ok, LWWRegRes2} = lasp:query(LWWReg2).
{ok, {LWWReg3, _, _, _}} = lasp:update(LWWReg2, {set, Timestamp(), baz}, self()).
{ok, LWWRegRes3} = lasp:query(LWWReg3).

% foo
% bar
% baz


Single-assignment variable.
Write once register.


{set, Value}
Set register to Value (only possible once)
IVarRegType = state_ivar.
IVarRegVarName = <<"ivar">>.

{ok, {IVarReg, _, _, _}} = lasp:declare({IVarRegVarName, IVarRegType}, IVarRegType).
{ok, {IVarReg1, _, _, _}} = lasp:update(IVarReg, {set, foo}, self()).
{ok, IVarRegRes1} = lasp:query(IVarReg1).

% foo


Multi-Value Register CRDT.


{set, Timestamp, Value}
the Timestamp will not be used. in order to have an unified API for all registers (since LWWRegister needs to receive a timestamp), the timestamp is also supplied here


MVRegType = state_mvregister.
MVRegVarName = <<"mvregister">>.

{ok, {MVReg, _, _, _}} = lasp:declare({MVRegVarName, MVRegType}, MVRegType).
{ok, {MVReg1, _, _, _}} = lasp:update(MVReg, {set, Timestamp(), foo}, self()).
{ok, MVRegRes1} = lasp:query(MVReg1).
{ok, {MVReg2, _, _, _}} = lasp:update(MVReg1, {set, nil, bar}, self()).
{ok, MVRegRes2} = lasp:query(MVReg2).
{ok, {MVReg3, _, _, _}} = lasp:update(MVReg2, {set, Timestamp(), baz}, self()).
{ok, MVRegRes3} = lasp:query(MVReg3).

% [foo]
% [bar]
% [baz]



Modeled as a dictionary where keys can be anything and the values are


{apply, Key, Op}
Update Key with operation Op (since value is a type, you have to supply an operation, for example if it's a register it may be set, if it's a counter it could be increment).
{rmv, Key}
Remove Key


AwMapType = {state_awmap, [state_mvregister]}.
AwMapVarName = <<"awmap">>.
AwMapVal1 = foo.
AwMapVal2 = bar.

{ok, {AwMap, _, _, _}} = lasp:declare({AwMapVarName, AwMapType}, AwMapType).

{ok, {AwMap1, _, _, _}} = lasp:update(AwMap, {apply, Key1,
                                              {set, Timestamp(), AwMapVal1}},

{ok, {AwMap2, _, _, _}} = lasp:update(AwMap1, {apply, Key1,
                                              {set, Timestamp(), AwMapVal2}},

{ok, {AwMap3, _, _, _}} = lasp:update(AwMap2, {apply, Key2,
                                              {set, Timestamp(), AwMapVal1}},

{ok, AwMapRes3} = lasp:query(AwMap3).

{ok, {AwMap4, _, _, _}} = lasp:update(AwMap3, {rmv, Key2}, self()).

{ok, AwMapRes4} = lasp:query(AwMap4).

% before removing Key2
[{K, sets:to_list(V)} || {K, V} <- AwMapRes3].
% [{<<"key1">>,[bar]},{<<"key2">>,[foo]}]

[{K, sets:to_list(V)} || {K, V} <- AwMapRes4].
% [{<<"key1">>,[bar]}]


Grow only map.
Modeled as a dictionary where keys can be anything and the
values are join-semilattices.
{apply, Key, Op}
Update Key with operation Op (since value is a type, you have to supply an operation, for example if it's a register it may be set, if it's a counter it could be increment).


GMapType = {state_gmap, [state_lwwregister]}.
GMapVarName = <<"gmap">>.
GMapVal1 = foo.
GMapVal2 = bar.

{ok, {GMap, _, _, _}} = lasp:declare({GMapVarName, GMapType}, GMapType).

{ok, {GMap1, _, _, _}} = lasp:update(GMap, {apply, Key1,
                                              {set, Timestamp(), GMapVal1}},

{ok, {GMap2, _, _, _}} = lasp:update(GMap1, {apply, Key1,
                                              {set, Timestamp(), GMapVal2}},

{ok, {GMap3, _, _, _}} = lasp:update(GMap2, {apply, Key2,
                                              {set, Timestamp(), GMapVal1}},

{ok, GMapRes3} = lasp:query(GMap3).
% {ok,[{<<"key1">>,bar},{<<"key2">>,foo}]}


Multi-Value Map CRDT.
MVMap = AWMap<MVRegister<V>>


{set, Key, Value}
Set Key to Value


MVMapType = state_mvmap.
MVMapVarName = <<"mvmap">>.
MVMapVal1 = foo.
MVMapVal2 = bar.

{ok, {MVMap, _, _, _}} = lasp:declare({MVMapVarName, MVMapType}, MVMapType).

{ok, {MVMap1, _, _, _}} = lasp:update(MVMap, {set, Key1, MVMapVal1}, self()).
{ok, {MVMap2, _, _, _}} = lasp:update(MVMap1, {set, Key1, MVMapVal2}, self()).
{ok, {MVMap3, _, _, _}} = lasp:update(MVMap2, {set, Key2, MVMapVal1}, self()).

{ok, MVMapRes3} = lasp:query(MVMap3).

[{K, sets:to_list(V)} || {K, V} <- MVMapRes3].
% [{<<"key1">>,[bar]},{<<"key2">>,[foo]}]



{fst, Value}
Mutates the first item in the pair
{snd, Value}
Mutates the second item in the pair


PairLeftType = state_lwwregister.
PairRightType = state_gcounter.
PairType = {state_pair, [PairLeftType, PairRightType]}.
PairVarName = <<"pair">>.
PairVal1 = foo.
PairVal2 = bar.

{ok, {Pair, _, _, _}} = lasp:declare({PairVarName, PairType}, PairType).

{ok, {Pair1, _, _, _}} = lasp:update(Pair, {fst, {set, Timestamp(), PairVal1}}, self()).
{ok, {Pair2, _, _, _}} = lasp:update(Pair1, {snd, increment}, self()).
{ok, {Pair3, _, _, _}} = lasp:update(Pair2, {fst, {set, Timestamp(), PairVal2}}, self()).
{ok, {Pair4, _, _, _}} = lasp:update(Pair3, {snd, increment}, self()).

{ok, PairRes4} = lasp:query(Pair4).

% {bar,2}


Observed-Remove Set with tombstones
{add, Element}
Add Element to the Set
{add_by_token, token(), element()}
Add Element to the Set by Token (orset internally generates a unique token for the element, therefore, it's not deterministic, this allows it to be explicit, and therefore deterministic)
{add_all, ListOfElements}
Add a list of elements to the set
{rmv, Element}
Remove Element from the set
{rmv_all, ListOfElements}
Remove a list of elements from the set


ORSetType = state_orset.
ORSetVarName = <<"orset">>.
ORSetVal1 = foo.
ORSetVal2 = bar.
ORSetVal3 = baz.
ORSetAllVals = [ORSetVal1, ORSetVal2, ORSetVal3].

{ok, {ORSet, _, _, _}} = lasp:declare({ORSetVarName, ORSetType}, ORSetType).

{ok, {ORSet1, _, _, _}} = lasp:update(ORSet, {add, ORSetVal1}, self()).
{ok, {ORSet2, _, _, _}} = lasp:update(ORSet1, {add, ORSetVal2}, self()).
% repeat value
{ok, {ORSet3, _, _, _}} = lasp:update(ORSet2, {add, ORSetVal1}, self()).
{ok, {ORSet4, _, _, _}} = lasp:update(ORSet3, {add, ORSetVal3}, self()).

{ok, ORSetRes4} = lasp:query(ORSet4).
% [bar,baz,foo]

{ok, {ORSet5, _, _, _}} = lasp:update(ORSet4, {rmv, ORSetVal3}, self()).
{ok, ORSetRes5} = lasp:query(ORSet5).
% [bar,foo]

{ok, {ORSet6, _, _, _}} = lasp:update(ORSet5, {rmv_all, ORSetAllVals}, self()).
{ok, ORSetRes6} = lasp:query(ORSet6).
% []

{ok, {ORSet7, _, _, _}} = lasp:update(ORSet6, {add_all, ORSetAllVals}, self()).
{ok, ORSetRes7} = lasp:query(ORSet7).
% [bar,baz,foo]

Add-Wins ORSet CRDT:

Observed-Remove Set without tombstones


{add, Element}
Add Element to the Set
{add_all, ListOfElements}
Add a list of elements to the set
{rmv, Element}
Remove Element from the set
{rmv_all, ListOfElements}
Remove a list of elements from the set
{filter, Function}
Filter elements in the set


AWSetType = state_awset.
AWSetVarName = <<"awset">>.
AWSetVal1 = foo.
AWSetVal2 = bar.
AWSetVal3 = baz.
AWSetAllVals = [AWSetVal1, AWSetVal2, AWSetVal3].

{ok, {AWSet, _, _, _}} = lasp:declare({AWSetVarName, AWSetType}, AWSetType).

{ok, {AWSet1, _, _, _}} = lasp:update(AWSet, {add, AWSetVal1}, self()).
{ok, {AWSet2, _, _, _}} = lasp:update(AWSet1, {add, AWSetVal2}, self()).
% repeat value
{ok, {AWSet3, _, _, _}} = lasp:update(AWSet2, {add, AWSetVal1}, self()).
{ok, {AWSet4, _, _, _}} = lasp:update(AWSet3, {add, AWSetVal3}, self()).

{ok, AWSetRes4} = lasp:query(AWSet4).
% [bar,baz,foo]

{ok, {AWSet5, _, _, _}} = lasp:update(AWSet4, {rmv, AWSetVal3}, self()).
{ok, AWSetRes5} = lasp:query(AWSet5).
% [bar,foo]

{ok, {AWSet6, _, _, _}} = lasp:update(AWSet5, {rmv_all, AWSetAllVals}, self()).
{ok, AWSetRes6} = lasp:query(AWSet6).
% []

{ok, {AWSet7, _, _, _}} = lasp:update(AWSet6, {add_all, AWSetAllVals}, self()).
{ok, AWSetRes7} = lasp:query(AWSet7).
% [bar,baz,foo]

Add-Wins Set CRDT with the provenance semiring:

add-wins set without tombstones


{add, Element}
Add Element to the Set
{add_all, ListOfElements}
Add a list of elements to the set
{rmv, Element}
Remove Element from the set
{rmv_all, ListOfElements}
Remove a list of elements from the set


AWPSSetType = state_awset_ps.
AWPSSetVarName = <<"awset_ps">>.
AWPSSetVal1 = foo.
AWPSSetVal2 = bar.
AWPSSetVal3 = baz.
AWPSSetAllVals = [AWPSSetVal1, AWPSSetVal2, AWPSSetVal3].

{ok, {AWPSSet, _, _, _}} = lasp:declare({AWPSSetVarName, AWPSSetType}, AWPSSetType).

{ok, {AWPSSet1, _, _, _}} = lasp:update(AWPSSet, {add, AWPSSetVal1}, self()).
{ok, {AWPSSet2, _, _, _}} = lasp:update(AWPSSet1, {add, AWPSSetVal2}, self()).
% repeat value
{ok, {AWPSSet3, _, _, _}} = lasp:update(AWPSSet2, {add, AWPSSetVal1}, self()).
{ok, {AWPSSet4, _, _, _}} = lasp:update(AWPSSet3, {add, AWPSSetVal3}, self()).

{ok, AWPSSetRes4} = lasp:query(AWPSSet4).
% [bar,baz,foo]

{ok, {AWPSSet5, _, _, _}} = lasp:update(AWPSSet4, {rmv, AWPSSetVal3}, self()).
{ok, AWPSSetRes5} = lasp:query(AWPSSet5).
% [bar,foo]

{ok, {AWPSSet6, _, _, _}} = lasp:update(AWPSSet5, {rmv_all, AWPSSetAllVals}, self()).
{ok, AWPSSetRes6} = lasp:query(AWPSSet6).
% []

{ok, {AWPSSet7, _, _, _}} = lasp:update(AWPSSet6, {add_all, AWPSSetAllVals}, self()).
{ok, AWPSSetRes7} = lasp:query(AWPSSet7).
% [bar,baz,foo]


Grow only set


{add, Element}
Add Element to the Set


GSetType = state_gset.
GSetVarName = <<"gset">>.
GSetVal1 = foo.
GSetVal2 = bar.
GSetVal3 = baz.
GSetAllVals = [GSetVal1, GSetVal2, GSetVal3].

{ok, {GSet, _, _, _}} = lasp:declare({GSetVarName, GSetType}, GSetType).

{ok, {GSet1, _, _, _}} = lasp:update(GSet, {add, GSetVal1}, self()).
{ok, {GSet2, _, _, _}} = lasp:update(GSet1, {add, GSetVal2}, self()).
% repeat value
{ok, {GSet3, _, _, _}} = lasp:update(GSet2, {add, GSetVal1}, self()).
{ok, {GSet4, _, _, _}} = lasp:update(GSet3, {add, GSetVal3}, self()).

{ok, GSetRes4} = lasp:query(GSet4).
% [bar,baz,foo]


Two-Phased Set

Once removed, elements cannot be added again.

Also, this is not an observed removed variant.

This means elements can be removed before being in the set.


{add, Element}
Add Element to the Set
{rmv, Element}
Remove Element from the set


TPSetType = state_awset_ps.
TPSetVarName = <<"awset_ps">>.
TPSetVal1 = foo.
TPSetVal2 = bar.
TPSetVal3 = baz.
TPSetAllVals = [TPSetVal1, TPSetVal2, TPSetVal3].

{ok, {TPSet, _, _, _}} = lasp:declare({TPSetVarName, TPSetType}, TPSetType).

{ok, {TPSet1, _, _, _}} = lasp:update(TPSet, {add, TPSetVal1}, self()).
{ok, {TPSet2, _, _, _}} = lasp:update(TPSet1, {add, TPSetVal2}, self()).
% repeat value
{ok, {TPSet3, _, _, _}} = lasp:update(TPSet2, {add, TPSetVal1}, self()).
{ok, {TPSet4, _, _, _}} = lasp:update(TPSet3, {add, TPSetVal3}, self()).

{ok, TPSetRes4} = lasp:query(TPSet4).
% [bar,baz,foo]

{ok, {TPSet5, _, _, _}} = lasp:update(TPSet4, {rmv, TPSetVal3}, self()).
{ok, TPSetRes5} = lasp:query(TPSet5).
% [bar,foo]

rebar3 partisan template

I created a rebar3 template to start a partisan project, the template is here:

rebar3 partisan template

What is partisan? from the docs:

Partisan is the technology that provides Lasp's scalable cluster
membership. Partisan bypasses the use of Distributed Erlang for manual
connection management via TCP, and has several pluggable backends for
different deployment scenarios.

To use:

mkdir -p ~/.config/rebar3/templates
git clone ~/.config/rebar3/templates/rebar3_template_partisan
rebar3 new rebar3_template_partisan name=party
cd party
make release
make console


make devrel

# on 3 different shells
make dev1-console
make dev2-console
make dev3-console

# join all nodes:
make devrel-join

# check node members
make devrel-status

# join node1 to node2 manually:
./_build/dev1/rel/party/bin/party-admin cluster join party2@

# check node1 members
./_build/dev1/rel/party/bin/party-admin cluster members

# check node1 connections
./_build/dev1/rel/party/bin/party-admin cluster connections

A video to show how to use:

Let's build a key value store in erlang

The code for this post is at

We first need to have erlang installed, I will show you how to setup any version you want to use, and a way to have the version I will use for this without affecting any other installation you may have.

Setting up kerl

For this we will use kerl, from it's github README:

Easy building and installing of Erlang/OTP instances.

Kerl aims to be shell agnostic and its only dependencies, excluding what's
required to actually build Erlang/OTP, are curl and git.

So, first we need to fetch kerl:

# create bin folder in our home directory if it's not already there
mkdir -p ~/bin

# cd to it
cd ~/bin

# download kerl script
curl -O

# set execution permitions for our user
chmod u+x kerl

You will need to add ~/bin to your PATH variable so your shell can find the kerl script, you can do it like this in your shell:

# set the PATH environment variable to the value it had before plus a colon
# (path separator) and a new path which points to the bin folder we just
# created

If you want to make this work every time you start a shell you need to put it it the rc file of your shell of choice, for bash it's ~/.bashrc, for zsh it's .zshrc, check your shell's docs for other shells, you will have to add a line like this:

export PATH=$PATH:$HOME/bin

After this, start a new shell or source your rc file so that it picks up your new PATH variable, you can check that it's set correctly by running:

echo $PATH

Building an Erlang release with kerl

We have kerl installed and available in our shell, now we need to build an Erlang release of our choice, for this we will need a compiler and other tools and libraries needed to compile it:

This are instructions on ubuntu 17.10, check the names for those packages on your distribution.

# required: basic tools and libraries needed
# (compiler, curses for the shell, ssl for crypto)
sudo apt-get -y install build-essential m4 libncurses5-dev libssl-dev

# optonal: if you want odbc support (database connectivity)
sudo apt-get install unixodbc-dev

# optonal: if you want pdf docs you need apache fop and xslt tools and java (fop is a java project)
sudo apt-get install -y fop xsltproc default-jdk

# optional: if you want to build jinterface you need a JDK
sudo apt-get install -y default-jdk

# optional: if you want wx (desktop GUI modules)
sudo apt-get install -y libwxgtk3.0-dev

Now that we have everything we need we can finally build our erlang release.

First we fetch an updated list of releases:

kerl update releases

The output in my case:

The available releases are:

R10B-0 R10B-10 R10B-1a R10B-2 R10B-3 R10B-4 R10B-5 R10B-6 R10B-7 R10B-8
R10B-9 R11B-0 R11B-1 R11B-2 R11B-3 R11B-4 R11B-5 R12B-0 R12B-1 R12B-2 R12B-3
R12B-4 R12B-5 R13A R13B01 R13B02-1 R13B02 R13B03 R13B04 R13B R14A R14B01
R14B02 R14B03 R14B04 R14B_erts- R14B R15B01 R15B02
R15B02_with_MSVCR100_installer_fix R15B03-1 R15B03 R15B
R16A_RELEASE_CANDIDATE R16B01 R16B02 R16B03-1 R16B03 R16B 17.0-rc1 17.0-rc2
17.0 17.1 17.3 17.4 17.5 18.0 18.1 18.2.1 18.2 18.3 19.0 19.1 19.2 19.3
20.0 20.1

Let's build the 20.1 version:

# this will take a while
kerl build 20.1 20.1

And install it:

kerl install 20.1 ~/bin/erl-20.1

Now everytime we want to use this version of erlang we need to run:

. $HOME/bin/erl-20.1/activate

Setting up rebar3

Now we have erlang, we need a build tool, we are going to use rebar3:

# download rebar3 to our bin directory
wget -O $HOME/bin/rebar3

# set execution permissions for our user
chmod u+x rebar3

Just in case you have problems running the rebar3 commands with a different version, here's the version I'm using:

rebar3 version


rebar 3.4.7 on Erlang/OTP 20 Erts 9.1

Setting up our project

We are ready to start our project, go to a folder where you keep your code and if you haven't done it yet, add $HOME/bin to your path and activate erlang 20.1 as shown above, then run:

rebar3 new release name=akvs

The output should be something like this:

===> Writing akvs/apps/akvs/src/akvs_app.erl
===> Writing akvs/apps/akvs/src/akvs_sup.erl
===> Writing akvs/apps/akvs/src/
===> Writing akvs/rebar.config
===> Writing akvs/config/sys.config
===> Writing akvs/config/vm.args
===> Writing akvs/.gitignore
===> Writing akvs/LICENSE
===> Writing akvs/

Let's see what each file does:

First of all, we created a release, which is a kind of project that can have more than one application (a common way to structure a project is into applications and libraries)

Under the apps folder are all the applications we mantain for this release, in our case we only have one application, named akvc.

Under the akvs application folder we have a src folder where all the source code for that application will live, we can add other folders there, for tests, header files, private files etc.


The $APPNAME_app module is called when starting and stopping the app to do the setup and tear down of the application.

Check Erlang's manual for application or the user's guide entry for application for more information.


The $APPNAME_sup module defines the root supervisor for the application, it implements the supervisor behavior and will be "hooked" into the supervisor hierarchy of this release when initialized.

Check Erlang's manual for supervisor or the user's guide entry for supervisor for more information.


The $ is a file that contains metadata about this app.

Check ERlang's manual for application resource file for more information.


Contains information about the project, dependencies, how to build it, test it, and how to build a release.

Check rebar3 docs for details.


Configuration parameters for the application.

Check sys.config's manual page for more information.


Configuration parameters for the Erlang VM.


Git specific, files to ignore.


The license for this project, you should change it if the Apache License 2.0 isn't the one you want.

Project's readme.

Starting it for the first time

First we need to build a release:

cd akvs

# build a release, the result will be at _build/default/rel/akvs
rebar3 release

# start the release and attach to the console
./_build/default/rel/akvs/bin/akvs console

The output in my case is (redacted for clarity):

Exec: bin/erl-20.1/erts-9.1/bin/erlexec
         -boot src/erl/akvs/_build/default/rel/akvs/releases/0.1.0/akvs
         -mode embedded -boot_var ERTS_LIB_DIR bin/erl-20.1/lib
         -config src/erl/akvs/_build/default/rel/akvs/releases/0.1.0/sys.config
         -args_file src/erl/akvs/_build/default/rel/akvs/releases/0.1.0/vm.args
         -pa -- console

Root: src/erl/akvs/_build/default/rel/akvs

Erlang/OTP 20 [erts-9.1] [source] [64-bit] [smp:4:4] [ds:4:4:10]
                          [async-threads:30] [kernel-poll:true]

=PROGRESS REPORT==== 25-Nov-2017::22:28:34 ===
                  supervisor: {local,sasl_safe_sup}
                         started: [{pid,<0.225.0>},

=PROGRESS REPORT==== 25-Nov-2017::22:28:34 ===
                  supervisor: {local,sasl_sup}
                         started: [{pid,<0.224.0>},

=PROGRESS REPORT==== 25-Nov-2017::22:28:34 ===
                  supervisor: {local,sasl_sup}
                         started: [{pid,<0.226.0>},

=PROGRESS REPORT==== 25-Nov-2017::22:28:34 ===
                 application: sasl
                  started_at: akvs@ganesha
Eshell V9.1  (abort with ^G)

There's not much we can do with our project at this stage, so we will just stop it and exit by running the q(). function in the shell:

(akvs@ganesha)1> q().

Coding (and testing) the Key Value store modules

The way I usually code in erlang is to first build a stateless module that has an init function that returns some state, all other functions expect that state as first parameter, then those functions do something and return the state and the result.

This modules are really easy to use in the shell and test.

This will be our first module, we will call it akvs_kv and it will have the following API:

%% types:

-type error() :: {error, {atom(), iolist(), map()}}.
-type key()   :: binary().
-type value() :: any().

% we don't want other modules to know/care about the internal structure of
% the state type
-opaque state() :: map().

%% functions:

%% @doc create a new instance of a key value store
-spec new(map()) -> {ok, state()} | error().

%% @doc dispose resources associated with a previously created kv store
-spec dispose(state()) -> ok | error().

%% @doc set a value for a key in a kv store
-spec set(state(), key(), value()) -> {ok, state()} | error().

%% @doc get a value for a key or an error if not found
-spec get(state(), key()) -> {ok, value()} | error().

%% @doc get a value for a key or a default value if not found
-spec get(state(), key(), value()) -> {ok, value()} | error().

%% @doc remove a value for a key, if not found do nothing
-spec del(state(), key()) -> {ok, state()} | error().

Notice that to specify the API I used a specification of the types and functions, this is called spec, read more about it at the Types and Function Specifications sectio in the erlang reference manual.

Also for documentation comments I'm using the edoc format, read more about it at the edoc user's guide section.

You can see the full code of this module here: akvs_kv

But how do we know if it works?

At this point there are two ways: testing it in the shell, or writing tests for it, let's do the right thing and write some tests.

We are going to use Common Test for our tests.

First we need to create the test folder for our tests:

mkdir apps/akvs/test

Inside it we will create a module called akvs_kv_SUITE that will contain the tests for the akvs_kv module.

You can see the full code of this module here: akvs_kv_SUITE

To run the tests:

rebar3 ct

We can also use the type specs we defined to check our code using dialyzer:

rebar3 dialyzer

Everything seems to be right, let's move on to the next step.

But before that, in case you want to generate API docs for our code taking advantage of the edoc annotations, you can do so by running:

rebar3 edoc

And opening apps/akvs/doc/index.html with a browser.

Wrapping the state

Stateless modules are a good start and are really easy to test and use, but we don't want to pass the burden of threading the state to the users of our code, also we want to centralize the state management so that more than one process can call our module and see the state changes of other callers.

In this case we are using ETS to make it simpler but if our kv was backed by a map, or if we had some kind of cache, then state management would become really important to get right, otherwise the results seen by each caller would diverge.

To manage the state of our module we are going to wrap it in a process, a gen_server in this case.

The module will be called akvs_kv_s (_s for server, don't know if there's a convention for it).

The module is a basic gen_server that exposes a couple functions to call the kv API from the akvs_kv module, you can read the code here: akvs_kv_s.

We write tests for this module too, you can read the test's code here: akvs_kv_s_SUITE.

Run the tests:

rebar3 ct

An API for our key value stores

Now we can spawn a key value store in a gen_server and apply operations to it, but like with the stateless module, someone has to keep a reference to the process and provide a nicer way to find and operate on our key value stores, if it was only one it's easy to just start it as a registered process with a name and send messages to it by it's name, but in our case, we want to provide namespaces where each namespace holds a key value store of its own.

The abstract API or this module should be like this:

-type ns() :: binary().
-type key() :: akvs_kv:key().
-type value() :: akvs_kv:value().
-type error() :: akvs_kv:value().

%% @doc set Key to Value in namespace Ns
-spec set(ns(), key(), value()) -> ok | error().

%% @doc get Key from namespace Ns
-spec get(ns(), key()) -> {ok, value()} | error().

%% @doc get Key from namespace Ns or DefaultValue if Key not found
-spec get(ns(), key(), value()) -> {ok, value()} | error().

%% @doc delete  Key in namespace Ns
-spec del(ns(), key()) -> ok | error().

Right now we are going to solve the problem of who keeps the namespace to process mapping really simple so we can continue, we are going to setup a public ETS table at application startup and lookup the processes by namespace there, if not found we are going to start the process and register it under that namespace.

This solution is not recommendable at all but it will allow us to continue and since the API doesn't know a thing about the way we register/lookup namespaces we can explore different alternatives later.

You can view the source code for akvs module here: akvs and the tests here akvs_SUITE.

An HTTP API for our key value stores

We are at the point where we can expose our APIs to the world, we are going to do it by exposing a really basic HTTP API for it.

The API will look like this:

# set key in namespace to the binary value sent in body
# return status: 201
POST /kv/<namespace>/<key>

# get key in namespace
# return status:
#  200: if found
#  404: if not found
GET /kv/<namespace>/<key>

# delete key from namespace
# return status: 200
DELETE /kv/<namespace>/<key>

To create an HTTP API we need an HTTP server, in this case we will use Cowboy 2.

First we need to add it as a dependency in our rebar.config file in the deps section and in the release dependencies section.

Then we need to setup the routes in our application initialization code.

We are going to have only one route and handler, we are going to use a basic HTTP to keep it simple, you can read the handler's code here: akvs_h_kv.

Now we can test it by building a release, starting it and playing with the API using curl:

rebar3 release
_build/default/rel/akvs/bin/akvs console

In another shell:

curl http://localhost:8080/kv/foo/bar
Not Found

curl -X POST http://localhost:8080/kv/foo/bar -d "hello world"

curl http://localhost:8080/kv/foo/bar
hello world

curl -X DELETE http://localhost:8080/kv/foo/bar

curl http://localhost:8080/kv/foo/bar
Not Found

curl -X PUT http://localhost:8080/kv/foo/bar -d "hello world"
Method Not Allowed

Seems to work fine.

Now we can build a production release and try it:

rebar3 as prod release
cd _build/prod/rel
tar -czf akvs.tar.gz akvs
cd -
mv _build/prod/rel/akvs.tar.gz /tmp
cd /tmp
tar -xzf akvs.tar.gz
cd akvs
./bin/akvs start

The application is started, you can check it's running by pinging it:

./bin/akvs ping

In case you need, you can attach to it (you should exit with Ctrl+D, using q() won't only detach your console but also stop the system!):

./bin/akvs attach

You can try it again:

curl http://localhost:8080/kv/foo/bar
curl -X POST http://localhost:8080/kv/foo/bar -d "hello world"
curl http://localhost:8080/kv/foo/bar
curl -X DELETE http://localhost:8080/kv/foo/bar
curl http://localhost:8080/kv/foo/bar
curl -X PUT http://localhost:8080/kv/foo/bar -d "hello world"

When you are finished, you can stop it:

./bin/akvs stop

Now you can upload akvs.tar.gz to any bare server and start akvs there, as long as the operating system is similar (better if the same) as the one where you built the release, this is because when building the release we bundle the erlang runtime for simplicity, this assumes specific versions of libraries like libssl which may not be available on the target system if it's too different.

Another way is to build the release without bundling the erlang runtime and having it available on the target system, just make sure that the erlang runtime in the target system has the same version you used to build it, otherwise you may experience errors due to modules/functions not being available or bytecode incompatibility if the target runtime is older than the one used for the release.

State of the BEAM 2017: Survey Results


You can't improve what you don't measure, and since I think there are areas in the BEAM community (Erlang, Elixir, LFE, Efene, Alpaca, Clojerl et al.) to improve we need to have a better picture of it.

That's why some months ago I decided to create this survey, I told to some people and started researching other "State of the X Community" yearly surveys, I wrote some draft questions and published to some people for feedback, after a couple of rounds I made a Form and ran a test survey for more feedback, after a couple dozen answers I cleared the results and announced it publicly with a weakly reminder on multiple channels.

Result Analysis

We got 423 Responses up to this point.

I present the results of the State of the BEAM Survey 2017 here in two ways:

  • Bar charts sorted by most answers to less
    • On questions with many answers I make a cut at some point
  • Raw data tables sorted by most answers to less
    • Here I did some consolidation of answers to avoid making them too large

I was thinking on doing a deep analysis on the answers but later I realized that if I did an analysis many people would read mine and avoid analyzing it themselves in detail.

Instead I decided to open an analysis thread in some forum and later maybe summarize the most interesting comments.

To ease the discussion I will do some light observations where I see it makes sense and make some questions to open the discussion.

Before diving into the result I want to make explicit two things that may make the results less representative than they should:

1. The "Elixir Effect"

I think the Elixir community is bigger or at least more active than the rest of the BEAM community, because of that and the fact that Elixir already has its own survey, I decided not to promote this survey there, to avoid the number of Elixir specific answers to skew the results and make this survey just be yet another Elixir survey with some BEAMers also replying.

With this clarification, and looking at the answers, I can identify some answers that are from Elixir-only developers, you can see that when some Elixir specific tools appear in the answers (Mix, ExUnit, Distillery, deploy to Heroku etc.), just keep that in mind when analyzing the results.

2. The "Survivorship Bias Effect"

From the wikipedia article on Survivorship bias

Survivorship bias or survival bias is the logical error of concentrating on the people or things that made it past some selection process and overlooking those that did not, typically because of their lack of visibility. This can lead to false conclusions in several different ways. It is a form of selection bias.

Survivorship bias can lead to overly optimistic beliefs because failures are ignored, such as when companies that no longer exist are excluded from analyses of financial performance.


The damaged portions of returning planes show locations where they can take a hit and still return home safely; those hit in other places do not survive.

This survey is done on people that wanted to learn Erlang, learned it, and are still active enough on the community to see the survey announcement.

This means that the answers are from the ones that "survived", which makes it really hard to get good feedback on the bad parts of the language, tooling and community since the most affected by it aren't going to stay around to fill this survey.

How to reach those? I don't know, propose solutions on the discussion.

I forgot to ask if I could make public the name of the companies so I won't, but I can say that I got 202 responses and most of them are not duplicates.

Things to improve for next year

  • Ask users if they want their answers available to be distributed in raw form for others to analyze
  • Ask users if I can share publicly the name of the company where they use Erlang
  • Decide what to do about Elixir-only replies, maybe make a question about it
  • Make specific questions regarding better tooling
  • I forgot Russia and Central America options, maybe next time do Latin America?

Let's see the results!

Which languages of the BEAM do you use?


Clearly Erlang is the most used language, ignoring the Elixir Effect, I'm kind of disappointed by the lack of users trying alternative languages. More so given the fact that many of the complaints or requests in other questions are already solved by other languages in the ecosystem, for example "better macros" or lisp inspired features being solved by LFE, static/stronger typing or better static analysis being solved by Alpaca, Elixir's pipe operator and a more mainstream syntax being solved by Efene.

My advice to the community, try the other languages, blog/tweet about it and share feedback with their creators, there's a language for each taste!

Erlang 326 54.42%
Elixir 231 38.56%
LFE 14 2.34%
Luerl 12 2.00%
Alpaca 9 1.50%
Clojerl 4 0.67%
Erlog 1 0.17%
Efene 1 0.17%
PHP 1 0.17%

How would you characterize your use of BEAM Languages today?


Many people using it for serious stuff, the Open Source answer is really low here but is contradicted by another answer below.

I think I should add another option for something like "experiments", "try new ideas".

I use it at work 327 48.66%
I use it for serious "hobby" projects 245 36.46%
I'm just tinkering 62 9.23%
I use it for my studies 35 5.21%
Learning 1 0.15%
katas 1 0.15%
Open Source Software 1 0.15%

In which domains are you applying it?

Distributed Systems 225 15.20%
Web development 214 14.46%
Building and delivering commercial services 172 11.62%
Open source projects 149 10.07%
Network programming 136 9.19%
Enterprise apps 92 6.22%
Databases 80 5.41%
IoT / home automation / physical computing 75 5.07%
System administration / dev ops 60 4.05%
Big Data 51 3.45%
Mobile app development (non-web) 46 3.11%
Research 33 2.23%
AI / NLP / machine learning 28 1.89%
Games 28 1.89%
Math / data analysis 23 1.55%
Scientific computing / simulations / data visualization 21 1.42%
Desktop apps 14 0.95%
Graphics / Art 4 0.27%
Music 3 0.20%
Industrial Automation 2 0.14%
log system 1 0.07%
videostreaming 1 0.07%
soft real time analytics 1 0.07%
Security Event Processing 1 0.07%
Media encoding and distribution 1 0.07%
Ad delivery 1 0.07%
Telecom Apps 1 0.07%
telecom and chat 1 0.07%
video 1 0.07%
Developer Tooling 1 0.07%
Telecommunications 1 0.07%
embedded systems 1 0.07%
Advertising/RTB 1 0.07%
Prototyping network apps 1 0.07%
Real time systems 1 0.07%
Real-Time Bidding 1 0.07%
Instant messaging / VoIP / Communications 1 0.07%
ad traffic management 1 0.07%
REST/GraphQL API 1 0.07%
Test systems 1 0.07%
Learning 1 0.07%
telecommunications 1 0.07%
VoIP 1 0.07%
Code static analysis 1 0.07%

What industry or industries do you develop for?

Enterprise software 117 15.04%
Communications / Networking 103 13.24%
Consumer software 85 10.93%
IT / Cloud Provider 83 10.67%
Financial services / FinTech 69 8.87%
Telecom 67 8.61%
Media / Advertising 46 5.91%
Retail / ecommerce 41 5.27%
Academic 29 3.73%
Healthcare 28 3.60%
Education 26 3.34%
Government / Military 22 2.83%
Scientific 16 2.06%
Legal Tech 6 0.77%
Energy 5 0.64%
Gaming 2 0.26%
HR 2 0.26%
Security 2 0.26%
Logistics 2 0.26%
sports/fitness 1 0.13%
Retired 1 0.13%
Sport 1 0.13%
Business Intelligence 1 0.13%
Telematics / Car industry 1 0.13%
Manufacturing / Automotive 1 0.13%
Cultural/Museum 1 0.13%
Utilities 1 0.13%
Open source 1 0.13%
Travel 1 0.13%
Sport analysis 1 0.13%
Fitness 1 0.13%
Online Games 1 0.13%
Automotive 1 0.13%
Marketing 1 0.13%
Real estate 1 0.13%
Consumer electronics 1 0.13%
Non profit 1 0.13%
Client driven 1 0.13%
Industrial IoT 1 0.13%
Electric utility 1 0.13%
SaaS 1 0.13%
Automobile 1 0.13%
energy sector 1 0.13%
utilities 1 0.13%
Recruitment 1 0.13%
Energetics 1 0.13%

How long have you been using Erlang?


The entrants (1 year or less) being less than 2 and 3 years may be discouraging or maybe as a sign that this survey didn't reach as many newcomers as it should.

> 6 Years 116 27.62%
2 Years 76 18.10%
3 Years 58 13.81%
1 Year 52 12.38%
Less than a year 45 10.71%
5 Years 36 8.57%
4 Years 34 8.10%
I've stopped using it 3 0.71%

What's your age


Similar to the previous one, the survey shows that we are not interesting to young programmers (or this survey is not interesting to them :)

30-40 179 42.42%
20-30 112 26.54%
40-50 93 22.04%
> 50 31 7.35%
< 20 7 1.66%

What's your gender


One I was expecting, but bad nonetheless.

Male 401 95.02%
Prefer not to say 15 3.55%
Female 5 1.18%
attack helicopter 1 0.24%

Where are you located?

North America 127 30.09%
Western Europe 117 27.73%
Eastern Europe 42 9.95%
Northern Europe 39 9.24%
South America 30 7.11%
Asia 25 5.92%
Oceania 11 2.61%
Russia 7 1.66%
India 6 1.42%
China 6 1.42%
South Saharan Afica 3 0.71%
Middle East 2 0.47%
Europe 1 0.24%
Iran 1 0.24%
Central America 1 0.24%
Australia 1 0.24%
Thailand 1 0.24%
East Africa 1 0.24%
Central Europe 1 0.24%

What is your level of experience with functional programming?


7 answers got the joke or are really awesome programmers :)

Intermediate 202 48.44%
Advanced 148 35.49%
Beginner 57 13.67%
Profunctor Optics Level 7 1.68%
None 3 0.72%

Prior to using Erlang, which were your primary development languages?

C or C++ 163 14.75%
Python 145 13.12%
Javascript 144 13.03%
Ruby 138 12.49%
Java 135 12.22%
PHP 72 6.52%
C# 56 5.07%
Perl 46 4.16%
Go 26 2.35%
Haskell 25 2.26%
Swift or Objective-C 24 2.17%
Common Lisp 20 1.81%
Scala 20 1.81%
Scheme or Racket 14 1.27%
Visual Basic 11 1.00%
Clojure 8 0.72%
R 8 0.72%
Rust 7 0.63%
None 6 0.54%
OCaml 3 0.27%
F# 3 0.27%
Kotlin 2 0.18%
Standard ML 2 0.18%
Fortran 2 0.18%
Pascal 1 0.09%
Ocaml 1 0.09%
KDB 1 0.09%
so "primary" here for me is "what was most used at work" 1 0.09%
TypeScript 1 0.09%
Microsoft Access 1 0.09%
Groovy 1 0.09%
but I am a self-proclaimed polyglot 1 0.09%
Shell 1 0.09%
Tcl/Tk 1 0.09%
Limbo 1 0.09%
Smalltalk 1 0.09%
clojure 1 0.09%
ActionScript 1 0.09%
Actionscript 1 0.09%
Prolog 1 0.09%
Racket 1 0.09%
Bash 1 0.09%
ML 1 0.09%
TCL 1 0.09%
Elixir 1 0.09%
C ANSI POSIX 1 0.09%
D 1 0.09%
ocaml 1 0.09%
Assembly 1 0.09%

Which client-side language are you using with Erlang?

Javascript 257 44.93%
None 90 15.73%
Elm 69 12.06%
Java 36 6.29%
Swift/Objective-C 36 6.29%
Clojurescript 13 2.27%
ReasonML/Ocaml 10 1.75%
Kotlin 8 1.40%
Typescript 7 1.22%
Scala 7 1.22%
Purescript 6 1.05%
C++ 4 0.70%
TypeScript 3 0.52%
Go 2 0.35%
typescript 2 0.35%
Python 2 0.35%
Erlang 2 0.35%
Flow + Javascript 1 0.17%
HTML-CSS 1 0.17%
Haskell 1 0.17%
What do you mean by "client-side language"? 1 0.17%
other 1 0.17%
Action Script 3 1 0.17%
Coffeescript 1 0.17%
d3.js 1 0.17%
lua 1 0.17%
Python/PyQt 1 0.17%
Dart 1 0.17%
Golang 1 0.17%
Ruby 1 0.17%
M$ C# 1 0.17%
Python (interface to legacy system - not web based) 1 0.17%
clojure 1 0.17%
C# 1 0.17%
Tcl/Tk 1 0.17%

In your Erlang projects, do you interoperate with other languages? if so, which ones?

C or C++ 156 24.19%
None 92 14.26%
Python 87 13.49%
Javascript 72 11.16%
Java 51 7.91%
Ruby 37 5.74%
Rust 27 4.19%
Go 27 4.19%
Swift or Objective-C 14 2.17%
C# 12 1.86%
Scala 11 1.71%
PHP 9 1.40%
Perl 8 1.24%
R 8 1.24%
Haskell 6 0.93%
Common Lisp 4 0.62%
Clojure 3 0.47%
OCaml 3 0.47%
Elixir 2 0.31%
Scheme or Racket 2 0.31%
Bash 2 0.31%
Kotlin 1 0.16%
KDB 1 0.16%
I use Erlang from Elixir 1 0.16%
lua 1 0.16%
SQL 1 0.16%
java 1 0.16%
Ocaml 1 0.16%
go 1 0.16%
Not directly via NIFs/ports but via HTTP/rabbit with ruby 1 0.16%
Tcl/Tk 1 0.16%
Lua 1 0.16%
python 1 0.16%

Which is your primary development environment?


I thought Emacs would win here by the fact that the Erlang creators use Emacs.

Vim 116 27.49%
Emacs 114 27.01%
IntelliJ 47 11.14%
Visual Studio Code 47 11.14%
Sublime Text 39 9.24%
Atom 32 7.58%
Eclipse 6 1.42%
spacemacs 2 0.47%
nano 2 0.47%
linux with vim as my text editor 1 0.24%
Kate 1 0.24%
textmate 1 0.24%
TextPad 1 0.24%
Simple text editor 1 0.24%
Notepad++ 1 0.24%
Also nvi. 1 0.24%
mcedit 1 0.24%
PSPad 1 0.24%
geany with erlang syntax support 1 0.24%
Kakoune 1 0.24%
Neovim 1 0.24%
Acme 1 0.24%
Spacemacs 1 0.24%
Atom and Emacs both very equally 1 0.24%
ed 1 0.24%
elvis 1 0.24%

Where do you go for Erlang news and discussions?

Twitter 204 20.20%
Mailing List 188 18.61%
Slack 116 11.49%
Reddit 111 10.99%
Stack Overflow 103 10.20%
IRC 62 6.14%
Erlang Central 60 5.94%
Newsletters 50 4.95%
Podcasts 33 3.27%
Planet Erlang 31 3.07%
ElixirForum 31 3.07%
Elixir Community 1 0.10% 1 0.10%
EUC 1 0.10%
Reddit and ElixirForum 1 0.10%
This week in Erlang 1 0.10%
Gitter 1 0.10%
Awesome Elixir 1 0.10%
OTP's github for PRs and commit log 1 0.10% 1 0.10%
Google Plus 1 0.10%
youtube 1 0.10%
Search on github 1 0.10%
Erlang solutions 1 0.10%
not interesting 1 0.10%
Medium 1 0.10%
None 1 0.10%
Watch talks 1 0.10%
Conference Videos 1 0.10%
Elixir Sips 1 0.10% 1 0.10% 1 0.10%

Which versions of the Erlang VM do you currently use for development?


We are really up to date, we are near a point where we can assume maps on our libraries :)

20 305 46.71%
19 213 32.62%
18 84 12.86%
17 30 4.59%
16 16 2.45%
<= 15 5 0.77%

Which versions of the Erlang VM do you currently use in production?


Of course production is a little more conservative

19 215 37.65%
20 183 32.05%
18 94 16.46%
17 43 7.53%
16 26 4.55%
<= 15 10 1.75%

Which build tool do you use?


Nice to see Rebar3 picking up momentum, Mix being mainly the Elixir Effect, next year I should add an option for "Mix for erlang or mixed projects".

Rebar3 220 32.59%
Mix 177 26.22%
Makefile 111 16.44%
Rebar 71 10.52% 46 6.81%
Custom build scripts 44 6.52%
Distillery 1 0.15%
maven 1 0.15%
redo 1 0.15%
mix 1 0.15%
synrc/mad 1 0.15%
MBU 1 0.15%

How do you test your code?


Surprised by EUnit being on top, why do people prefer it over Common Test?

EUnit 216 34.67%
Common Test 158 25.36%
ExUnit 74 11.88%
PropEr 69 11.08%
I don't write tests 45 7.22%
QuickCheck 33 5.30%
Custom 4 0.64%
Triq 4 0.64%
ESpec 3 0.48%
CutEr 3 0.48%
StreamData 2 0.32%
Lux 2 0.32%
py.test 2 0.32%
Functional tests 1 0.16%
Don't have time to write tests 1 0.16%
katana-test 1 0.16%
riak_test 1 0.16%
Dialyzer 1 0.16%
Integration tests 1 0.16%
Elixir tests. 1 0.16%
Concuerror 1 0.16%

How do you deploy your application?


Lot of custom deploy scripts and docker/kubernetes here, maybe we should have better deploy support in our tools?

Custom deploy scripts 186 32.75%
Docker 128 22.54%
Kubernetes 50 8.80%
Ansible 44 7.75%
I don't deploy in other servers 40 7.04%
Chef 39 6.87%
Puppet 11 1.94%
SaltStack 11 1.94%
Heroku 11 1.94%
Edeliver 8 1.41%
deb 7 1.23%
Distillery 7 1.23%
Zones 5 0.88%
AWS CodeDeploy 3 0.53%
Rancher 1 0.18%
VM image 1 0.18%
boot from flash 1 0.18% 1 0.18%
Not my job 1 0.18%
copy paste 1 0.18%
mad 1 0.18%
CD 1 0.18%
Exrm 1 0.18%
rpm 1 0.18%
Nomad 1 0.18%
AWS ECS 1 0.18%
FreeBSD Jails 1 0.18%
lxc 1 0.18%
WIX 1 0.18%
os packages 1 0.18%
nanobox 1 0.18%
cloudfoundry 1 0.18%

What is your organization's size?


Can we say that Erlang works on organizations of any size?

11-50 109 26.20%
2-10 93 22.36%
Just me 75 18.03%
500+ 65 15.62%
101-500 45 10.82%
51-100 29 6.97%

Which operating system(s) you use for development?


Almost same amount of Windows and FreeBSD, is it because Windows support is bad? or is this a reflection of the usual developer OS choice in any programming language?

Linux 307 47.01%
MacOS 253 38.74%
Windows 38 5.82%
FreeBSD 34 5.21%
Illumos 8 1.23%
OpenBSD 7 1.07%
Solaris 3 0.46%
NetBSD 1 0.15%
GRiSP 1 0.15%
ChromeOS 1 0.15%

Which operating system(s) you use for deployment?

Linux 378 75.15%
FreeBSD 43 8.55%
MacOS 25 4.97%
Windows 22 4.37%
I don't deploy in other servers 11 2.19%
Solaris 9 1.79%
Illumos 8 1.59%
OpenBSD 3 0.60%
RTEMS 1 0.20%
GRiSP 1 0.20%
OSv 1 0.20%
NetBSD 1 0.20%

Where do you deploy your applications?


I don't think this question provides useful information, maybe I should add options?

Public Cloud 188 27.85%
Use on local machine(s) 162 24.00%
Traditional Infrastructure 157 23.26%
Private Cloud (or hybrid) 156 23.11%
Distillery 1 0.15%
VMs on ESXi 1 0.15%
embedded systems 1 0.15%
Rented physical server 1 0.15%
Vagrant vms 1 0.15%
Embedded appliances 1 0.15%
Heroku 1 0.15%
VPS 1 0.15% 1 0.15%
Containers (Docker) 1 0.15%
Edeliver 1 0.15%
GitHub 1 0.15%

Which events have you attended in the last year?


Local Meetups at the top is a nice one, we can work to promote more of these.

Local Meetup 124 46.97%
Erlang Factory 39 14.77%
Erlang User Conference 38 14.39%
Erlang Factory Light 20 7.58%
ElixirConf 12 4.55%
ElixirConfEU 9 3.41%
None 6 2.27%
Lonestar Elixir 3 1.14%
Code Mesh 2 0.76%
Lambda Days 1 0.38%
ElixirConfEU + ElixirLDN 1 0.38%
ElixirConf USA 2017 1 0.38%
Elixir London 1 0.38%
ElixirConf 2017 1 0.38%
Elixir meetup in Leeds UK 1 0.38%
ElixirLive Conference in Warsaw 1 0.38%
J on the Beach 1 0.38%
Peer gatherings in region 1 0.38%
Empex 1 0.38%
Elixir Camp 1 0.38%

Do you use HiPE?

No 301 74.32%
Yes 104 25.68%

Do you use dialyzer?

Yes 280 66.99%
No 138 33.01%

How important have each of these aspects of Erlang been to you and your projects?

Remember that charts and tables are sorted by most to less answers to compare correctly in the following set of questions.


Very Important 157 37.74%
Fairly Important 112 26.92%
Important 79 18.99%
Slightly Important 52 12.50%
No Opinion 8 1.92%
Not Important at All 8 1.92%

Concurrency facilities

Very Important 306 73.73%
Fairly Important 58 13.98%
Important 36 8.67%
No Opinion 7 1.69%
Slightly Important 7 1.69%
Not Important at All 1 0.24%

Ease of development

Very Important 205 49.52%
Fairly Important 98 23.67%
Important 72 17.39%
Slightly Important 27 6.52%
No Opinion 10 2.42%
Not Important at All 2 0.48%

Functional Programming

Very Important 207 49.88%
Fairly Important 105 25.30%
Important 53 12.77%
Slightly Important 33 7.95%
No Opinion 9 2.17%
Not Important at All 8 1.93%


Very Important 222 53.62%
Fairly Important 90 21.74%
Important 60 14.49%
Slightly Important 30 7.25%
No Opinion 8 1.93%
Not Important at All 4 0.97%

Runtime performance

Very Important 148 35.75%
Fairly Important 122 29.47%
Important 95 22.95%
Slightly Important 36 8.70%
Not Important at All 7 1.69%
No Opinion 6 1.45%


Very Important 145 35.02%
Fairly Important 106 25.60%
Important 74 17.87%
Slightly Important 61 14.73%
No Opinion 19 4.59%
Not Important at All 9 2.17%


Slightly Important 96 23.02%
Very Important 95 22.78%
Fairly Important 90 21.58%
Important 82 19.66%
Not Important at All 29 6.95%
No Opinion 25 6.00%

What has been most frustrating or has prevented you from using Erlang more than you do now?

App deployment

Not Frustrating at All 120 30.08%
Slightly Frustrating 93 23.31%
Frustrating 54 13.53%
Fairly Frustrating 41 10.28%
Quite the contrary: I love this feature 38 9.52%
No Opinion 29 7.27%
Very Frustrating 24 6.02%

Error messages

Slightly Frustrating 119 29.82%
Not Frustrating at All 89 22.31%
Frustrating 48 12.03%
Fairly Frustrating 43 10.78%
Quite the contrary: I love this feature 39 9.77%
Very Frustrating 38 9.52%
No Opinion 23 5.76%

Finding libraries

Slightly Frustrating 137 34.08%
Not Frustrating at All 121 30.10%
Frustrating 64 15.92%
Fairly Frustrating 31 7.71%
Quite the contrary: I love this feature 21 5.22%
Very Frustrating 15 3.73%
No Opinion 13 3.23%

Hard to Learn it

Not Frustrating at All 204 51.13%
Slightly Frustrating 77 19.30%
Quite the contrary: I love this feature 48 12.03%
Frustrating 29 7.27%
No Opinion 21 5.26%
Fairly Frustrating 14 3.51%
Very Frustrating 6 1.50%

Hiring and staffing

No Opinion 124 31.47%
Slightly Frustrating 78 19.80%
Not Frustrating at All 71 18.02%
Fairly Frustrating 43 10.91%
Frustrating 41 10.41%
Very Frustrating 25 6.35%
Quite the contrary: I love this feature 12 3.05%

Installation process

Not Frustrating at All 218 55.05%
Slightly Frustrating 67 16.92%
Quite the contrary: I love this feature 54 13.64%
Frustrating 23 5.81%
Fairly Frustrating 16 4.04%
No Opinion 14 3.54%
Very Frustrating 4 1.01%

Long term viability

Not Frustrating at All 194 48.87%
Quite the contrary: I love this feature 74 18.64%
Slightly Frustrating 46 11.59%
No Opinion 41 10.33%
Frustrating 28 7.05%
Fairly Frustrating 9 2.27%
Very Frustrating 5 1.26%

Need more docs/tutorials

Not Frustrating at All 127 32.32%
Slightly Frustrating 124 31.55%
Frustrating 44 11.20%
Fairly Frustrating 37 9.41%
Quite the contrary: I love this feature 22 5.60%
No Opinion 22 5.60%
Very Frustrating 17 4.33%

Need more text editor support/IDEs

Not Frustrating at All 168 42.00%
Slightly Frustrating 93 23.25%
Frustrating 39 9.75%
Quite the contrary: I love this feature 32 8.00%
Fairly Frustrating 28 7.00%
Very Frustrating 22 5.50%
No Opinion 18 4.50%

Need more tools

Slightly Frustrating 128 32.16%
Not Frustrating at All 99 24.87%
Frustrating 58 14.57%
Fairly Frustrating 40 10.05%
Very Frustrating 34 8.54%
No Opinion 26 6.53%
Quite the contrary: I love this feature 13 3.27%

No static typing

Not Frustrating at All 113 28.18%
Slightly Frustrating 105 26.18%
Quite the contrary: I love this feature 63 15.71%
Frustrating 40 9.98%
Fairly Frustrating 34 8.48%
Very Frustrating 25 6.23%
No Opinion 21 5.24%

Release schedule

Not Frustrating at All 258 64.99%
Quite the contrary: I love this feature 57 14.36%
No Opinion 43 10.83%
Slightly Frustrating 26 6.55%
Frustrating 9 2.27%
Very Frustrating 2 0.50%
Fairly Frustrating 2 0.50%

Runtime performance

Not Frustrating at All 185 46.25%
Slightly Frustrating 72 18.00%
Quite the contrary: I love this feature 57 14.25%
Frustrating 32 8.00%
No Opinion 25 6.25%
Fairly Frustrating 17 4.25%
Very Frustrating 12 3.00%

Unpleasant community

Not Frustrating at All 224 56.14%
Quite the contrary: I love this feature 79 19.80%
No Opinion 45 11.28%
Slightly Frustrating 26 6.52%
Frustrating 14 3.51%
Very Frustrating 7 1.75%
Fairly Frustrating 4 1.00%

Version incompatibility

Not Frustrating at All 212 53.13%
Slightly Frustrating 84 21.05%
No Opinion 40 10.03%
Quite the contrary: I love this feature 29 7.27%
Frustrating 19 4.76%
Fairly Frustrating 11 2.76%
Very Frustrating 4 1.00%

Any feature you would like to see added to the language?

This was an open ended question, I'm summarizing similar answers here in groups

Static Typing 20
Performance 7
Pipe operator 7
Currying 6
Better GUI lib 5
Better macros 4
Docs in shell 4
Better language interop 4
JSON in stdlib 3
Compile to single binary 3
Namespaces 3
Rebind variables 2
Numeric performance 2
Elixir protocols 2
Language server protocol 2
Non full mesh disterl 2
Consensus implementations in stdlib 2
More than 2 version of same module 2
Atom GC 2

Other answers with one vote:

  • Backward compatibility
  • BEAM on browsers
  • Better binary syntax
  • Better container support
  • Better datetime support
  • Better documentation
  • Better errors
  • Better global registry
  • Better if expression
  • Better map support in mnesia
  • Better ML integration
  • Better module system
  • Better proc_lib
  • Better profiler
  • Better site
  • Better string module
  • Better unicode support
  • Bigger standard library
  • Bring back parameterized modules
  • Cleanup standard library
  • Code change watcher and loader
  • Consistent error return
  • CRDTs
  • Curses version of observer
  • Database drivers/support
  • Early return statement
  • Encrypted inter node communication
  • Erlang leveldb/rocksdb (better DETS)
  • First class records (not as tuples)
  • Function composition
  • IPv6
  • Laziness
  • LLVM based Hipe
  • Map comprehensions
  • Monads
  • More behaviors
  • More Lispy
  • More robust on_load
  • Multi-poll sets
  • Native compilation
  • New logo
  • Numerical/GPU support
  • Orleans
  • Package manager
  • Rational numbers
  • Remove stuff
  • Short circuit folding a list
  • Single file distribution
  • String performance
  • Top-like tool
  • Type checking as you type
  • Type inference
  • WebRTC
  • With expression like Elixir

Any advise on how we can make Erlang more welcoming and easy to use?

This was an open ended question, I'm summarizing similar answers here in groups

Better guides 20
Better documentation 18
Better error messages 13
Better tooling 9
Central/better landing page 4
Better REPL 3
Translated documentation 2
Better libraries 2
Friendlier community 2
Better learning curve 2
Learn from Elixir community 3
IDE support 3
Marketing 2
Searchable docs 2
Throw away legacy 2
Simpler release process 2

Other answers with one vote:

  • A killer app
  • Better tracing tools
  • Better Windows experience
  • Embrace BEAM languages
  • Erlang forum
  • Introductory workshops
  • More conferences
  • More welcoming mailing list
  • Nicer syntax
  • Performance
  • Templates

Would you like to share any frustrating experience?

This was an open ended question, I'm summarizing similar answers here in groups

Bad tooling 6
Lack of libraries, immaturity, not maintained 5
Lack of guides 5
Unwelcome community 4
Lack of strong/static typing 4
Syntax 4
No examples on documentation/in general 3
Bad documentation 3
Mnesia 3
Bad debugging experience 3
Ignoring other languages/communities/feedback 3
Niche language 2
Bad shell 2
No jobs 2
Learning curve 2
Difficulty contributing to the BEAM and OTP 2
Confusing errors 2
Performance 2
Package management 2

Other answers with one vote:

  • Atoms not garbage collected
  • Elitism
  • Flat namespace
  • Hard to hire
  • Lack of features in standard library
  • Lack of language features
  • Not a clear overview of the ecosystem
  • People complaining about syntax
  • String vs binary
  • Upgrades break code

Papers (and other things) of the LargeSpanOfTime III

I got to this papers by following the references from another paper, they were interesting but not as much as I was expecting:

This was on my queue:

A good one, I found it quite hard to find good overviews about datalog before this one.

Finished reading the book (or collection of 19 papers), really good chapters/papers in it: Your Wish is My Command: Giving Users the Power to Instruct their Software


Papers this not so long span of time: 5 (count books as 1)

Papers+Books so far: 59

Multi-Paxos with riak_ensemble Part 3

In the previous post I showed how to use riak_ensemble in a rebar3 project, now I will show how to create an HTTP API for the Key/Value store using Cowboy and jsone.

This post assumes that you have erlang and rebar3 installed, I'm using erlang 19.3 and rebar3 3.4.3.

The source code for this post is at check the commits for the steps.

Dependency Setup

To have an HTTP API we will need an HTTP server, in our case we will use Cowboy 2.0 RC 3, for that we need to:

  1. Add it as a dependency (we will load if from git since it's still a release candidate)
  2. Add it to our list of applications to start when our application starts
  3. Add it to the list of dependencies to include in our release
  4. Set up the HTTP listener and routes when our application starts

We setup just one route that is handled by the cadena_h_keys module, it's a plain HTTP handler, no fancy REST stuff for now, there we handle the request on the init/2 function itself, we pattern match against the method field on the request object and handle:

set a key in a given ensemble to the value sent in the JSON request body
get a key in a given ensemble, if not found null will be returned in the value field in the response
delete a key in a given ensemble, returns null both if the key existed and if itdidn't

Any other method would get a 405 Method Not Allowed response.

The route has the format /keys/<ensemble>/<key>, for now we only allow the root ensemble to be set in the <ensemble> part of the path.

We also add the jsone library to encode/decode JSON and the lager library to log messages.

We add both to the list of dependencies to include in the release.

We will also need to have a way to override the HTTP port where each instance listens to so we can run a cluster on one computer and each node can listen for HTTP requests on a different port.

The dev and prod releases will listen on 8080 as specified in vars.config.

node1 will listen on port 8081 (override in vars_node1.config)

node2 will listen on port 8082 (override in vars_node2.config)

node3 will listen on port 8083 (override in vars_node3.config)

To avoid having to configure this in sys.config we will define a cuttlefish schema in config.schema that cuttlefish will use to generate a default config file and validation code for us.

We have to replace the variables from variable overrides in our config.schema file for each release before it's processed by cuttlefish itself, for that we use the template directive on an overlay section on the release config.

Build devrel:

make revrel

Check the configuration file generated for each node at:


The first part is of interest to us, it looks like this for node1, the port number is different in node2 and node3:

## port to listen to for HTTP API
## Default: 8081
## Acceptable values:
##   - an integer
http.port = 8081

## number of acceptors to user for HTTP API
## Default: 100
## Acceptable values:
##   - an integer
http.acceptors = 100

## folder where ensemble data is stored
## Default: ./cadena_data
## Acceptable values:
##   - text
data.dir = ./cadena_data

Start 3 nodes in 3 different shells:

make node1-console
make node2-console
make node3-console

Start enseble and join nodes, I created a target called devrel-setup in the Makefile to make it easier:

make devrel-setup

Let's set key1 in ensemble root to 42 on node1 (port 8081):

curl -X POST http://localhost:8081/keys/root/key1 -d 42



Let's get key1 in ensemble root to 42 on node2 (port 8082):

curl -X GET http://localhost:8082/keys/root/key1



Same on node3:

curl -X GET http://localhost:8083/keys/root/key1



Overwrite on node1:

curl -X POST http://localhost:8081/keys/root/key1 -d '{"number": 42}'



Get on node2:

curl -X GET http://localhost:8082/keys/root/key2

Let's set key2 in ensemble root to {"number": 42} on node1 (port 8081):

curl -X POST http://localhost:8081/keys/root/key2 -d '{"number": 42}'



Get it on node2:

curl -X GET http://localhost:8082/keys/root/key2



Delete key2 in ensemble root on node2:

curl -X DELETE http://localhost:8082/keys/root/key2



Check that it was removed by trying to get it again on node2:

curl -X GET http://localhost:8082/keys/root/key2



There you go, now you have a Consistent Key Value Store with an HTTP API.

Multi-Paxos with riak_ensemble Part 2

In the previous post I showed how to use riak_ensemble from the interactive shell, now I will show how to use rebar3 to use riak_ensemble from a real project.

This post assumes that you have erlang and rebar3 installed, I'm using erlang 19.3 and rebar3 3.4.3.

The source code for this post is at check the commits for the steps.

Create Project

rebar3 new app name=cadena
cd cadena

The project structure should look like this:

├── rebar.config
└── src
        ├── cadena_app.erl
        └── cadena_sup.erl

1 directory, 6 files

Configuring Dev Release

We do the following steps, check the links for comments on what's going on for each step:

  1. Add Dependencies
  2. Configure relx section
    1. Add overlay variables file vars.config
    2. Add sys.config
    3. Add vm.args

Build a release to test that everything is setup correctly:

$ rebar3 release

Run the release interactively with a console:

$ _build/default/rel/cadena/bin/cadena console

Output (edited and paths redacted for clarity):

Exec: erlexec
        -boot _build/default/rel/cadena/releases/0.1.0/cadena
        -boot_var ERTS_LIB_DIR erts-8.3/../lib
        -mode embedded
        -config    _build/default/rel/cadena/generated.conf/app.1.config
        -args_file _build/default/rel/cadena/generated.conf/vm.1.args
        -vm_args   _build/default/rel/cadena/generated.conf/vm.1.args
        -- console

Root: _build/default/rel/cadena
Erlang/OTP 19 [erts-8.3] [source] [64-bit] [smp:4:4] [async-threads:64]

18:31:12.150 [info] Application lager started on node 'cadena@'
18:31:12.151 [info] Application cadena started on node 'cadena@'
Eshell V8.3  (abort with ^G)


(cadena@> q().

Non interactive start:

$ _build/default/rel/cadena/bin/cadena start

No output is generated if it's started, we can check if it's running by pinging the application:

$ _build/default/rel/cadena/bin/cadena ping

We should get:


If we want we can attach a console to the running system:

$ _build/default/rel/cadena/bin/cadena attach


Attaching to /tmp/erl_pipes/cadena@ (^D to exit)


If we press Ctrl+d we can dettach the console without stopping the system:

(cadena@> [Quit]

We can stop the system whenever we want issuing the stop command:

$ _build/default/rel/cadena/bin/cadena stop




Use Ctrl+d to exit, if we write q(). not only we dettach the console but we also stop the system!

Let's try it.

Non interactive start:

$ _build/default/rel/cadena/bin/cadena start

No output is generated if it's started, we can check if it's running by pinging the application:

$ _build/default/rel/cadena/bin/cadena ping

We should get:


If we want we can attach a console to the running system:

$ _build/default/rel/cadena/bin/cadena attach


Attaching to /tmp/erl_pipes/cadena@ (^D to exit)


Now let's quit with q():

(cadena@> q().



Now let's see if it's alive:

$ _build/default/rel/cadena/bin/cadena ping

Node 'cadena@' not responding to pings.

Be careful with how you quit attached consoles in production systems :)

Configure Prod and Dev Cluster Releases

Building Prod Release

We start by adding a new section to rebar.config called profiles, and define 4 profiles that override the default release config with specific values, let's start by trying the prod profile, which we will use to create production releases of the project:

rebar3 as prod release


===> Verifying dependencies...
===> Compiling cadena
===> Running cuttlefish schema generator
===> Starting relx build process ...
===> Resolving OTP Applications from directories:
===> Resolved cadena-0.1.0
===> Including Erts from erl-19.3
===> release successfully created!

Notice now that we have a new folder in the _build directory:

$ ls -1 _build



The results of the commands run "as prod" are stored in the prod folder.

You will notice if you explore the prod/rel/cadena folder that there's a folder called erts-8.3 (the version may differ if you are using a different erlang version), that folder is there because of the include_erts option we overrided in the prod profile.

This means you can zip the _build/prod/rel/cadena folder, upload it to a server that doesn't have erlang installed in it and still run your release there.

This is a good way to be sure that the version running in production is the same you use in development or at build time in your build server.

Just be careful with deploying to an operating system too different to the one you used to create the release becase you may have problems with bindings like libc or openssl.

Running it is done as usual, only the path changes:

_build/prod/rel/cadena/bin/cadena console

_build/prod/rel/cadena/bin/cadena start
_build/prod/rel/cadena/bin/cadena ping
_build/prod/rel/cadena/bin/cadena attach
_build/prod/rel/cadena/bin/cadena stop

Building Dev Cluster Releases

To build a cluster we need at least 3 nodes, that's why the last 3 profiles are node1, node2 and node3, they need to have different node names, for that we use the overlay var files to override the name of each, that is achieved on config/vars_node1.config for node1, config/vars_node2.config for node2 and config/vars_node3.config for node3.

Now let's build them:

rebar3 as node1 release
rebar3 as node2 release
rebar3 as node3 release

The output for each should be similar to the one for the prod release.

Now on three different shells start each node:

./_build/node1/rel/cadena/bin/cadena console

Check the name of the node in the shell:


Do the same for node2 and node3 on different shells:

./_build/node2/rel/cadena/bin/cadena console
./_build/node3/rel/cadena/bin/cadena console

You should get respectively:




In case you don't remember, you can quit with q().

Joining the Cluster Together

Until here we built 3 releases of the same code with slight modifications to allow running a cluster on one computer, but 3 nodes running doesn't mean we have a cluster, for that we need to use what we learned in the Multi-Paxos with riak_ensemble Part 1 but now on code and not interactively.

For that we will create a cadena_console module that we will use to make calls from the outside and trigger actions on each node, the code is similar to the one presented in Multi-Paxos with riak_ensemble Part 1.

join([NodeStr]) ->
    % node name comes as a list string, we need it as an atom
    Node = list_to_atom(NodeStr),
    % check that the node exists and is alive
    case net_adm:ping(Node) of
        % if not, return an error
        pang ->
            {error, not_reachable};
        % if it replies, let's join him passing our node reference
        pong ->
            riak_ensemble_manager:join(Node, node())

create([]) ->
    % enable riak_ensemble_manager
    % wait until it stabilizes

cluster_status() ->
    case riak_ensemble_manager:enabled() of
        false ->
            {error, not_enabled};
        true ->
            Nodes = lists:sort(riak_ensemble_manager:cluster()),
            io:format("Nodes in cluster: ~p~n",[Nodes]),
            LeaderNode = node(riak_ensemble_manager:get_leader_pid(root)),
            io:format("Leader: ~p~n",[LeaderNode])

We also need to add the riak_ensemble supervisor to our supervisor tree in cadena_sup:

init([]) ->
    % get the configuration from sys.config
    DataRoot = application:get_env(riak_ensemble, data_root, "./data"),
    % create a unique path for each node to avoid clashes if running more
    % than one node in the same computer
    NodeDataDir = filename:join(DataRoot, atom_to_list(node())),

    Ensemble = {riak_ensemble_sup,
                {riak_ensemble_sup, start_link,
                permanent, 20000, supervisor, [riak_ensemble_sup]},

    {ok, { {one_for_all, 0, 1}, [Ensemble]} }.

Before building the dev cluster we need to add the crypto app to since it's needed by riak_ensemble to create the cluster.

Now let's build the dev cluster, I created a Makefile to make it simpler:

make devrel

On three different shells run one command on each:

make node1-console
make node2-console
make node3-console

Let's make an rpc call to enable the riak_ensemble cluster on node1:

./_build/node1/rel/cadena/bin/cadena rpc cadena_console create

On node1 you should see something like:

[info] {root,'node1@'}: Leading

Let's join node2 to node1:

./_build/node2/rel/cadena/bin/cadena rpc cadena_console join node1@

On node1 you should see:

[info] join(Vsn): {1,152} :: 'node2@' :: ['node1@']

On node2:

[info] JOIN: success

Finally let's join node3:

./_build/node3/rel/cadena/bin/cadena rpc cadena_console join node1@

Output on node1:

[info] join(Vsn): {1,453} :: 'node3@' :: ['node1@','node2@']

On node3:

[info] JOIN: success

Let's check that the 3 nodes have the same view of the cluster, let's ask node1 what's the ensemble status:

./_build/node1/rel/cadena/bin/cadena rpc cadena_console ensemble_status
Nodes in cluster: ['node1@','node2@','node3@']
Leader: 'node1@'


$ ./_build/node2/rel/cadena/bin/cadena rpc cadena_console ensemble_status
Nodes in cluster: ['node1@','node2@','node3@']
Leader: 'node1@'


$ ./_build/node3/rel/cadena/bin/cadena rpc cadena_console ensemble_status
Nodes in cluster: ['node1@','node2@','node3@']
Leader: 'node1@'

Everything looks right, stop the 3 nodes (q().) and start them again, you will see that after starting up node1 logs:

[info] {root,'node1@'}: Leading

And if you call ensemble_status on any node you get the same outputs as before, this means they remember the cluster topology even after restarts.