Damien Krotkine

This post is a compilation of four old posts. They are gathered here in one piece for more clarity, and for archiving purpose. The work described in this post was done over the years 2014-2016

compress => Sereal::Encoder::SRL_ZLIB,
dedupe_strings => 1

[
  { cpu => 5  },
  { cpu => 99 }
]

curl -d <data> -XPUT "http://node:8098/buckets/events/keys/1413813813:1:type1:subtype1:0"
# ...
curl -d <data> -XPUT "http://node:8098/buckets/events/keys/1413813813:2:type3::0"

curl -d "type1:subtype1:0|type1:subtype1:1|type3::0" -XPUT "http://riak_host:8098/buckets/epochs/keys/1413813813-2"

"n_val" : 3,

"allow_mult"      : false,
"last_write_wins" : true,

"w"  : 3,
"dw" : 0,
"pw" : 0,

bucket: epochs
key: 1428415043-1
value: 1:cell0:WEB:app:chunk0|1:cell0:EMK::chunk0

bucket: events
key: 1428415043:1:cell0:WEB:app:chunk0
value: <binary sereal blob>

bucket: events
key: 1428415043:1:cell0:EMK::chunk0
value: <binary sereal blob>

1435748400
1435748401
1435748402
...
1435749000

"r"            : 1,
"pr"           : 0,
"rw"           : "quorum",
"basic_quorum" : true,
"notfound_ok"  : true,

:::erlang
metadata_stored_hook(RiakObject) ->
    Key = riak_object:key(RiakObject),
    Bucket = riak_object:bucket(RiakObject),
    [ Epoch, DC ] = binary:split(Key, <<"-">>),
    MetaData = riak_object:get_value(RiakObject),
    DataKeys = binary:split(MetaData, <<"|">>, [ global ]),
    send_to_REST(Epoch, Hostname, DataKeys),
    ok.

send_to_REST(Epoch, Hostname, DataKeys) ->
    Method = post,
    URL = "http://" ++ binary_to_list(Hostname)
       ++ ":5000?epoch=" ++ binary_to_list(Epoch),
    HTTPOptions = [ { timeout, 4000 } ],
    Options = [ { body_format, string },
    		         { sync, false },
            		  { receiver, fun(ReplyInfo) -> ok end }
              ],
    Body = iolist_to_binary(mochijson2:encode( DataKeys )),
    httpc:request(Method,
                  {URL, [], "application/json", Body},
                  HTTPOptions, Options),
    ok.

riak-admin bucket-type create metadata_with_post_commit \
'{"props":{"postcommit":["metadata_stored_hook"]}'

riak-admin bucket-type activate metadata_with_post_commit

    open my $fh, $file or die "failed to open '$file', error: $!";
    eval { code_that_may_die(); 1; }
      or say "exception has been caught: $@"
    try {
      croak "exception";
    } catch {
      warn "caught error: $_";
    };
    open $file or die MyExceptions::IO::File->new(
      filename => $file,
      error => $!
    );
    package MyExceptions;

    use Exception::Class (
        'MyException::IO',
        'MyException::IO::File' => {
            isa => 'MyException::IO',    
            fields => [ 'filename' ],
        },
    );
    use MyExceptions;

    open $file or MyException::IO::File->throw(
      filename => $file,
      error => $!
    );
    use MyException;

    try {
        open $file or MyException::IO::File->throw(
          filename => $file,
          error => $!
        );
    } catch {
        my $exception = $_;
        if ($exception->isa(MyException::IO)) {
            # we know how to handle these
        } else {
            $exception->rethrow
        }
    };
    use MyException;
    use Scalar::Util qw(blessed);

    try {
        # ... code that may die
    } catch {
        my $exception = $_;
        if (blessed $exception) {
            # exception is an object
            # ...
        } else {
            # exception is a normal string
            # ...
        }
    };
    sub do_stuff {
        try {
            # ... code that can only throw objects exceptions
        } catch {
            my $exception = $_;
            # exception is always an object
            if ($exception->isa(...)) {
                # ...
            }
        };
    }
    $SIG{__DIE__} = sub { die "FATAL: $_[0]" };
    do_stuff();
    "permission denied"
    "[MyException::IO]permission denied"
    "[MyException::IO|filename:/tmp/file|]permission denied"
    $exception->message();
    $exception->$message();
    $message->($exception);
    $exception->$xthrow()
    $exception->$xrethrow()
    $exception->$xraise()
    $exception->$xclass()
    $exception->$xisa()
    $exception->$xfields()
    $exception->$xfield()
    $exception->$xmessage()
    $exception->$xerror()
    MyException->throw(...)
    $exception->$xthrow();
    use Exception::Stringy;
    Exception::Stringy->declare_exceptions(
        'MyException',

        'YetAnotherException' => {
            isa         => 'AnotherException',
        },

        'ExceptionWithFields' => {
            isa    => 'YetAnotherException',
            fields => [ 'grandiosity', 'quixotic' ],
            throw_alias  => 'throw_fields',
        },
    );

    ### with Try::Tiny

    use Try::Tiny;

    try {
        # throw an exception
        MyException->throw('I feel funny.');

        # or use an alias
        throw_fields 'Error message', grandiosity => 1;

        # or with fields
        ExceptionWithFields->throw('I feel funny.',
                                   quixotic => 1,
                                   grandiosity => 2);

        # you can build exception step by step
        my $e = ExceptionWithFields->new("The error message");
        $e->$xfield(quixotic => "some_value");
        $e->$xthrow();

    }
    catch {
        if ( $_->$xisa('Exception::Stringy') ) {
            warn $_->$xerror, "\n";
        }

        if ( $_->$xisa('ExceptionWithFields') ) {
            if ( $_->$xfield('quixotic') ) {
                handle_quixotic_exception();
            }
            else {
                handle_non_quixotic_exception();
            }
        }
        else {
            $_->$xrethrow;
        }
    };

    ### without Try::Tiny

    eval {
        # ...
        MyException->throw('I feel funny.');
        1;
    } or do {
        my $e = $@;
        # .. same as above with $e instead of $_
    }
use mop;

class Point {
has $!x is ro = 0;
    has $!y is ro = 0;

    method set_x ($x) {
        $!x = $x;
    }

    method set_y ($y) {
        $!y = $y;
    }

    method clear {
        ($!x, $!y) = (0, 0);
    }

    method pack {
        +{ x => $self->x, y => $self->y }
    }
}

# ... subclass it ...

class Point3D extends Point {
    has $!z is ro = 0;

    method set_z ($z) {
        $!z = $z;
    }

    method pack {
        my $data = $self->next::method;
        $data->{z} = $!z;
        $data;
    }
}
class Foo {
    has $!stuff;
	method do_stuff ($stuff) {
        $!stuff = $stuff;
    }
}
has $!bar is ro, lazy = $_->foo + 2;
has $!foo = 'default value';
has $!foo = sub { 'default value' };
has $!foo = $_->_init_foo;
method foo ($arg1, $arg2=10) {
    say $arg1;
}
# clearer
method clear_foo { undef $!foo }

# predicate
method has_foo { defined $!foo }
foo();
sub foo {
    my ($arg) = @_; # $arg is undef
}
role Bar {
    has $!additional_attr = 42;
    method more_feature { say $!additional_attr }
}
class Foo with Bar, Baz {
    # ...
}
sub modifier {
    if ($_[0]->isa('mop::method')) {
        my $method = shift;
        my $type   = shift;
        my $meta   = $method->associated_meta;
        if ($meta->isa('mop::role')) {
            if ( $type eq 'around' ) {
                $meta->bind('after:COMPOSE' => sub {
                    my ($self, $other) = @_;
                    if ($other->has_method( $method->name )) {
                        my $old_method = $other->remove_method( $method->name );
                        $other->add_method(
                            $other->method_class->new(
                                name => $method->name,
                                body => sub {
                                    local ${^NEXT} = $old_method->body;
                                    my $self = shift;
                                    $method->execute( $self, [ @_ ] );
                                }
                            )
                        );
                    }
                });
            } elsif ( $type eq 'before' ) {
                die "before not yet supported";
            } elsif ( $type eq 'after' ) {
                die "after not yet supported";
            } else {
                die "I have no idea what to do with $type";
            }
        } elsif ($meta->isa('mop::class')) {
            die "modifiers on classes not yet supported";
        }
    }
}
method my_method is modifier('around') ($arg) {
    $arg % 2 and return $self->${^NEXT}(@_);
    die "foo";
}
around foo { }
method foo is around { ... }
method foo is modifier(around) { ... }

class Point is rw {
    has ($.x, $.y);
    method gist { "Point a x=$.x y=$.y" }
}
sub foo is exportable { ... }
{
    package MyRole;
    use Moo::Role;
    use MooX::LvalueAttribute;
}

{
    package MyApp;
    use Moo;

    with ('MyRole');

    has name => ( is => 'rw',
                  lvalue => 1,
                );
}

my $object = MyApp->new();
$object->name = 'Joe';
    use Bloomd::Client;
    my $b = Bloomd::Client->new;

    my $filter = 'test_filter';
    $b->create($filter);
    my $hash_ref = $b->info($filter);

    $b->set($filter, 'u1');
    if ($b->check($filter, 'u1')) {
	  say "it exists!"
    }
package App;
use Moo;
use MooX::LvalueAttribute;

has name => (
  is => 'rw',
  lvalue => 1,
);

# Elsewhere
my $app = App->new(name => 'foo');

$app->name = 'Bar';

print $app->name;  # Bar
use Action::Retry qw(retry);
retry { ... };
Action::Retry->new( attempt_code => sub { ... } )->run();
my $action = Action::Retry->new(
  attempt_code => sub { ... },
  retry_if_code => sub { $_[0] =~ /Connection lost/ || $_[1] > 20 },
  strategy => { Fibonacci => { multiplicator => 2000,
                               initial_term_index => 3,
                               max_retries_number => 5,
                             }
              },
  on_failure_code => sub { say "Given up retrying" },
);
$action->run();
  use Action::Retry qw(retry);
  retry { ... }
  retry_if_code => sub { $_[0] =~ /Connection lost/ || $_[1] > 20 },
  strategy => { Fibonacci => { multiplicator => 2000,
                               initial_term_index => 3,
                               max_retries_number => 5,
                             }
              },
  on_failure_code => sub { say "Given up retrying" };
my $action = Action::Retry->new(
  attempt_code => sub { ... },
  non_blocking => 1,
  strategy => { 'Constant' }
);
while (1) {
  # if the action failed, it doesn't sleep
  # next time it's called, it won't do anything until it's time to retry
  $action->run();

  do_something_else();
  # do something else while time goes on

}
#!/usr/bin/env perl

use Modern::Perl;
use ElasticSearch;

my $es = ElasticSearch->new( servers => 'api.metacpan.org', no_refresh => 1 );

my $scroller = $es->scrolled_search(
    query       => { match_all => { } },
    search_type => 'scan',
    scroll      => '5m',
    index       => 'v0',
    type        => 'release',
    size        => 100,
    filter => {
            term => {
                     'release.dependency.module' => 'Dancer'
                    }
              },

);

my $result = $scroller->next;

my %plugins;
while ( my $result = $scroller->next ) {
    $result->{_source}->{name} =~ /Dancer-Plugin/
      or next;
    my $name = $result->{_source}->{name};
    $name =~ s/-\d.*//;
    $name =~ s/-/::/g;
    $plugins{$name} = 1;
}
say $_ foreach sort keys %plugins;
HERE=${PWD}

# set up the local Perlbrew location
PERLBREW_DIR=${HERE}/perlbrew
PERLBREW=export PERLBREW_ROOT=${PERLBREW_DIR} \
    && source ${PERLBREW_DIR}/etc/bashrc \
    && PERLBREW_ROOT=${PERLBREW_DIR} ${PERLBREW_DIR}/bin/perlbrew

# This will run the cpanm from the local Perlbrew, on each of the 2 perls
CPANM_DANCER1=${PERLBREW} exec --with dancer1_plugin_tests ${PERLBREW_DIR}/bin/cpanm
CPANM_DANCER2=${PERLBREW} exec --with dancer2_plugin_tests ${PERLBREW_DIR}/bin/cpanm

# loop on the plugin list, install the deps and test each plugin and output result
all: got_curl got_bash got_perlbrew_perl plugins_list
	echo "Plugin name,Pass on Dancer 1,Pass on Dancer 2" > result.csv
	@for i in `cat plugins_list`; do pass_d1=0; pass_d2=0; \
	  echo " ---------- TESTING on Dancer 1 : $$i"; \
          ${CPANM_DANCER1} -n --installdeps $$i \
              && ${CPANM_DANCER1} --test-only $$i && pass_d1=1; \
	  echo " ---------- TESTING on Dancer 2 : $$i"; \
          ${CPANM_DANCER2} -n --installdeps $$i \
              && DANCER_FORCE_PLUGIN_REGISTRATION=1 ${CPANM_DANCER2} --test-only $$i \
              && pass_d2=1; \
          echo "$$i,$$pass_d1,$$pass_d2" >> result.csv; \
        done;

clean:
	rm -rf plugins_list

fullclean:
	rm -rf ${PERLBREW_DIR}

# Get and install Perlbrew locally
${PERLBREW_DIR}/bin/perlbrew:
	@echo " - creating a local perlbrew"
	export PERLBREW_ROOT=${PERLBREW_DIR} && curl -kL http://install.perlbrew.pl | bash

# Get and install cpanm in the local Perlbrew
${PERLBREW_DIR}/bin/cpanm:
	${PERLBREW} install-cpanm

got_curl:
	@which curl >/dev/null \
        || ( echo "you don't have curl, please install it and retry" && false )

got_bash:
	@which bash >/dev/null \
        || ( echo "you don't have bash, please install it and retry" && false )

got_perlbrew_perl: ${PERLBREW_DIR}/bin/perlbrew ${PERLBREW_DIR}/bin/cpanm perlbrew_dancer1_plugin_tests perlbrew_dancer2_plugin_tests

# Build perl from scratch and call this instance dancer1
perlbrew_dancer1_plugin_tests:
	${PERLBREW} list | grep dancer1_plugin_tests > /dev/null \
        || ( ${PERLBREW} install -j 2 -n perl-5.16.1 --as dancer1_plugin_tests )

# Build perl from scratch and call this instance dancer2
perlbrew_dancer2_plugin_tests:
	${PERLBREW} list | grep dancer2_plugin_tests > /dev/null \
        || ( ${PERLBREW} install -j 2 -n perl-5.16.1 --as dancer2_plugin_tests \
             && ${CPANM_DANCER2} ${HERE}/../.. )

# This gets the list of plugins, as previously described
plugins_list:
	${HERE}/get_modules_list.pl > plugins_list
use Moose;

my $singleton;

sub instance {
    return $singleton //= $CLASS->new();
}

# to protect against people using new() instead of instance()
around 'new' => sub {
    my $orig = shift;
    my $self = shift;
    return $singleton //= $self->$orig(@_);
};

sub initialize {
    defined $singleton
      and croak __PACKAGE__ . ' singleton has already been instanciated'; 
    shift;
    return __PACKAGE__->new(@_);
}
# file ~/.corsspost.ini
[ main_blog ]
title = dams blog
url = http://damien.krotkine.com

[ blogs.perl.org ]
type = mt
username = foo
password = bar
url = http://blogs.perl.org/rsd.xml

# file crosspost.plcorsspost.ini

use Modern::Perl;
use Carp;

use Net::MovableType;
use Config::Any::Merge;
use Getopt::Long;
use Path::Class;
use Text::Markdown qw(markdown);

my $file;
GetOptions ("file=s"   => \$file)
  or croak "failed to parse command line options";

$file
  or croak "need a file";

$file = file($file);

# Get a hash representing the config file
my $cfg = (values %{Config::Any->load_files({files => [ $ENV{HOME} . '/.crosspost.ini' ], use_ext => 1 })->[0]})[0];

# Grab some main info
my $main_blog_title = $cfg->{main_blog}{title};
my $main_blog_url = $cfg->{main_blog}{url};

my $text = $file->slurp;
# In reality, I do some transformation on $text to interpret jekyll specific
# syntax

$text = "\n <i>cross-posted from [$main_blog_title]($main_blog_url)</i>\n\n" . $text;

# So now we have the content to be posted, but it's in markdown. Let's
# transform it in HTML
my $html = markdown($text);

# Then we loop on all the other blog to cross post to
foreach my $site (grep { $_ ne 'main_blog'} keys %$cfg) {
    say "posting to $site";
    my %properties = %{$cfg->{$site}};

    # for now we handle only movable type
    if ($properties{type} eq 'mt') {
        my $username = $properties{username};
        my $password = $properties{password};
        my $url = $properties{url};

        my $mt = Net::MovableType->new($url);
        $mt->username($username)
          or croak $mt->errstr;
        $mt->password($password)
          or croak $mt->errstr;
        We need to get the user's blog id
        my $user_blogs = $mt->getUsersBlogs
          or croak $mt->errstr;
        $mt->blogId($user_blogs->[0]->{blogid})
          or croak $mt->errstr;

        # finally, the post
        $mt->newPost({
                      title       => $title,
                      description => $html,
                      mt_allow_comments => $properties{allow_comments} // 1,
                     },
                     # uncomment this line to directly post it
                     # 1
                    );

    }

}
# In your configuration file
plugins:
  Params::Normalization:
    method: lowercase

$ screen
# hit ctrl A :password
# enter the new password twice
# hit ctrl A ]
# the encrypted password should be pasted in the console
# add this line, with your encrypted password
password VGdGzMopF
dams@foo:~$ screen -r -d plop
Screen password: 
# works fine
$attribute->task('profit');

# fails
$attribute->task('failure');
# type definition
use Moose::Util::TypeConstraints;
subtype 'My:Type:Task' => as 'Str' =>
  where { my $r = join( '|', __PACKAGE__->get_possible_tasks ); /^(?:$r)$/ };
coerce 'My:Type:Task' => from 'Str' => via {lc};
sub get_possible_tasks {qw(profit world_domination)}

# attribute declaration
has task => ( isa => 'My:Type:Task' );
# using a named enum subtype:
use Moose::Util::TypeConstraints;
enum 'My:Enum:Task', [qw(profit world_domination)];
has task => ( isa => 'My:Enum:Task' );

# using an anonymous enum subtype:
use Moose::Util::TypeConstraints;
has task => ( isa => enum([qw(profit world_domination)]) );
(require 'perlbrew-mini)
;; change to your username below
(perlbrew-mini-set-perls-dir "/home/uername/perl5/perlbrew/perls/")
;; change the version you wish to use
(perlbrew-mini-use "perl-5.12.2")

(require 'flymake)

(defun flymake-perl-init ()
  (let* ((temp-file (flymake-init-create-temp-buffer-copy
                     'flymake-create-temp-inplace))
         (local-file (file-relative-name
                      temp-file
                      (file-name-directory buffer-file-name))))
    (list (perlbrew-mini-get-current-perl-path)
          (list "-MProject::Libs" "-wc" local-file)))
)

(add-hook 'cperl-mode-hook (lambda () (flymake-mode t)))

my $duration = $dt2->subtract_datetime_absolute($dt1);
my $seconds = $duration->seconds();
package Foo;
{
  my $structure = {};
  sub plop {
      #...
      $structure->{$foo} = 'bar';
      #...
  }
}
package Foo;
{
  my $cache;
  sub plop {
      #...
      defined $cache or $cache = _load_cache();
      #...
  }
}
package Foo;
sub plop {
    state $structure = {};
    $structure->{$foo} = 'bar';
    #...
}
package Foo;
sub plop {
    #...
    state $cache = _load_cache();
    #...
}
   MooseX::Getopt::Meta::Attribute::Trait::NoGetopt->apply(
       __PACKAGE__->meta->get_attribute('some_attr')
   );
    has '+some_attr' => (traits => ['NoGetopt'])
flagedit allows you to edit the use flags or the keywords for a particular
ebuild, or for the whole system. a backup is done for each modified file, named
file.old.

Usage: flagedit [PACKAGE] [ACTIONS | -- KEYWORD_ACTIONS] [ OPTIONS ]

Examples:
 flagedit net-im/amsn --show # shows the use flag set for net-im/amsn
 flagedit net-im/amsn +gnome # adds the gnome use flag to net-im/amsn
 flagedit net-im/amsn -kde +xmms # adds the xmms use flag and set the -kde one
 flagedit net-im/amsn %kde # reset the kde use flag (it's removed from the line)

 flagedit %kde # reset the global use flag (it's removed in make.conf)
 flagedit +gnome -qt # add gnome and -qt in make.conf

 flagedit net-im/amsn -- %x86 # reset the x86 keyword for net-im/amsn
 flagedit net-im/amsn -- +~ppc +~x86 # adds the ~ppc and ~x86 keywords for net-im/amsn
 flagedit net-im/amsn -- % # resets the keywords for this package
 flagedit -- +~x86 # sets ACCEPT_KEYWORDS to "~x86" in /etc/make.conf

You can mix the flags and keywords :
 flagedit net-im/amsn +gnome -- +~x86

PACKAGE is a package name (like dev-ruby/ruby-atk). If no package is given,
flagedit will edit the maine USE flags (in make.conf), or the main
ACCEPT_KEYWORDS (in make.conf)

ACTIONS are :
+FLAG enable the FLAG. Example : +sse
-FLAG disable the FLAG. Example : -sse
%FLAG reset the FLAG to default. Example : %sse
% reset the whole flags of PACKAGE to default. In this case, PACKAGE is not optional

KEYWORD_ACTIONS are :
+KEYWORD enable the keyword. Example : +x86
-KEYWORD disable the KEYWORD. Example : -~x86
%KEYWORD reset the KEYWORD to default. Example : %x86
% reset the whole keywords of PACKAGE to default. In this case, PACKAGE is not optional

OPTIONS are :
 --package-file <path> specify an alternate package.use file (default is /etc/portage/package.use)
 --keywords-file <path> specify an alternate package.keywords file (default is /etc/portage/package.keywords)
 --make-conf-file <path> specify an alternate make.conf file (default is /etc/make.conf)
 --portage-dir <path> specify an alternate portage directory path (default is /usr/portage)
 --alpha-order sort the flags or keywords alphabetically instead of keeping the original order
 --show don't edit, display the flags or keywords of the PACKAGE. If no package is given, display the system USE flags or the system ACCEPT_KEYWORDS.
 --list don't edit, display the entire list of possible flags or possible keywords.
 --desc if specified with --list, display the flags or keywords description also.
 --keywords specifies that actions are to be done on keywords, not on use flags
 -- same as --keywords
 --strict if a specified flag or keyword name is invalid, dies, instead of just warning.
 --nowarn if a specified flag or keyword name is invalid, don't warn.
 --help this help
 --version prints the version

see http://dkrotkine.com/flagedit/

Damien Krotkine

Riak as Events Storage

Riak as Events Storage

Introduction to Events Storage

High Level overview

Individual Events

Aggregated Events

Data flow size and properties

Why Riak

Riak 101

Riak clusters configuration

Data Design

Pushing to Riak

PUT to Riak

PUT options

Reading from Riak

GET options

Real time data processing outside of Riak

Limitations of data processing outside of Riak

Next: data filtering and processing inside Riak

Real-time server-side data processing: the theory

A first attempt: MapReduce

The metrics

Metrics when doing external processing

External bandwidth usage

Internal bandwidth usage

Deserialise time

Processing time

Metrics when using MapReduce

External bandwidth usage

Internal bandwidth usage

Deserialise time

Processing time

Limitations of MapReduce

A better solution: post-commit hooks

Strategy and Features##

Real-time isolated data transformation

Performance and scalability

No back-processing

Only fast transformations

The solution: Substreams

Data decompressed only once

Transformation at write time, not at query time

Hard timeout - open platform

Implementation using a Riak commit hook

Hook implementation

Enabling the Hook

The companion app

Optimising intra-cluster network usage

Back-pressure management strategy

Conclusion

Notes

PromCon2017 - Prometheus Conference 2017

PromCon2017 - Prometheus Conference 2017

Best practises - general

Best practises - metrics and label naming

Best practises - alerting

Remote storage

InfluxDB

Exporters

Alerting tools

Meta Alerting

Grafana

Cortex

Various

Prometheus limitations

Prometheus v2

Exception::Stringy - Modern exceptions for legacy code

Exception::Stringy - Modern exceptions for legacy code

A small recap of Perl exceptions

Basic Usage Of Exceptions

Throwing Objects

Catching Objects Exceptions

When things go wrong

Mixing Objects And String Exceptions

Mixed Exceptions Issues

The Overloaded Stringification Route

Exceptions::Stringy from scratch

The Needed Features

Class Instance

why `is` ? we already have `has` !