Support for combined experiments (see README) (#493)

* Support for combined experiments (see README)

* Refactor combined experiments into its own helper

* Update README, add descriptive error messages

Author: daviddening

README.md

@@ -1,4 +1,4 @@
-# [Split](http://libraries.io/rubygems/split)
+# [Split](http://libraries.io/rubygems/split) 
 
 [![Gem Version](https://badge.fury.io/rb/split.svg)](http://badge.fury.io/rb/split)
 [![Build Status](https://secure.travis-ci.org/splitrb/split.svg?branch=master)](http://travis-ci.org/splitrb/split)
@@ -623,6 +623,35 @@ Once you finish one of the goals, the test is considered to be completed, and fi
 
 **Bad Example**: Test both how button color affects signup *and* how it affects login, at the same time. THIS WILL NOT WORK.
 
+#### Combined Experiments
+If you want to test how how button color affects signup *and* how it affects login, at the same time. Use combined tests
+Configure like so 
+```ruby
+  Split.configuration.experiments = {
+        :button_color_experiment => {
+          :alternatives => ["blue", "green"],
+          :combined_experiments => ["button_color_on_signup", "button_color_on_login"]
+        }
+      }
+```
+
+Starting the combined test starts all combined experiments
+```ruby
+ ab_combined_test(:button_color_experiment)
+```
+Finish each combined test as normal
+
+```ruby
+   ab_finished(:button_color_on_login)
+   ab_finished(:button_color_on_signup)
+```
+
+**Additional Configuration**: 
+* Be sure to enable `allow_multiple_experiments` 
+* In Sinatra include the CombinedExperimentsHelper
+  ```
+    helpers Split::CombinedExperimentsHelper
+  ```
 ### DB failover solution
 
 Due to the fact that Redis has no automatic failover mechanism, it's

lib/split.rb

@@ -13,6 +13,7 @@
 require 'split/extensions/string'
 require 'split/goals_collection'
 require 'split/helper'
+require 'split/combined_experiments_helper'
 require 'split/metric'
 require 'split/persistence'
 require 'split/redis_interface'

lib/split/combined_experiments_helper.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+module Split
+  module CombinedExperimentsHelper
+    def ab_combined_test(metric_descriptor, control = nil, *alternatives)
+      return nil unless experiment = find_combined_experiment(metric_descriptor)
+      raise(Split::InvalidExperimentsFormatError, 'Unable to find experiment #{metric_descriptor} in configuration') if experiment[:combined_experiments].nil?
+
+      alternative = nil
+      experiment[:combined_experiments].each do |combined_experiment|
+        if alternative.nil?
+          if control
+            alternative = ab_test(combined_experiment, control, alternatives)
+          else
+            normalized_alternatives = Split::Configuration.new.normalize_alternatives(experiment[:alternatives])
+            alternative = ab_test(combined_experiment, normalized_alternatives[0], *normalized_alternatives[1])
+          end
+        else
+          ab_test(combined_experiment, [{alternative => 1}])
+        end
+      end
+    end
+
+    def find_combined_experiment(metric_descriptor)
+      raise(Split::InvalidExperimentsFormatError, 'Invalid descriptor class (String or Symbol required)') unless metric_descriptor.class == String || metric_descriptor.class == Symbol
+      raise(Split::InvalidExperimentsFormatError, 'Enable configuration') unless Split.configuration.enabled
+      raise(Split::InvalidExperimentsFormatError, 'Enable `allow_multiple_experiments`') unless Split.configuration.allow_multiple_experiments
+      experiment = Split::configuration.experiments[metric_descriptor.to_sym]
+    end
+  end
+end

lib/split/engine.rb

@@ -5,6 +5,8 @@ class Engine < ::Rails::Engine
       if Split.configuration.include_rails_helper
         ActionController::Base.send :include, Split::Helper
         ActionController::Base.helper Split::Helper
+        ActionController::Base.send :include, Split::CombinedExperimentsHelper
+        ActionController::Base.helper Split::CombinedExperimentsHelper
       end
     end
   end

lib/split/helper.rb

@@ -10,6 +10,7 @@ def ab_test(metric_descriptor, control = nil, *alternatives)
         experiment = ExperimentCatalog.find_or_initialize(metric_descriptor, control, *alternatives)
         alternative = if Split.configuration.enabled
           experiment.save
+          raise(Split::InvalidExperimentsFormatError) unless Split::configuration.experiments&.dig(experiment.name.to_sym,:combined_experiments).nil?
           trial = Trial.new(:user => ab_user, :experiment => experiment,
               :override => override_alternative(experiment.name), :exclude => exclude_visitor?,
               :disabled => split_generically_disabled?)

spec/combined_experiments_helper_spec.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+require 'spec_helper'
+require 'split/combined_experiments_helper'
+
+describe Split::CombinedExperimentsHelper do
+  include Split::CombinedExperimentsHelper
+
+  describe 'ab_combined_test' do
+    let!(:config_enabled) { true }
+    let!(:combined_experiments) { [:exp_1_click, :exp_1_scroll ]}
+    let!(:allow_multiple_experiments) { true }
+
+    before do
+      Split.configuration.experiments = {
+        :combined_exp_1 => {
+          :alternatives => [ {"control"=> 0.5}, {"test-alt"=> 0.5} ],
+          :metric => :my_metric,
+          :combined_experiments => combined_experiments
+        }
+      }
+      Split.configuration.enabled = config_enabled
+      Split.configuration.allow_multiple_experiments = allow_multiple_experiments
+    end
+
+    context 'without config enabled' do
+      let!(:config_enabled) { false }
+
+      it "raises an error" do
+        expect(lambda { ab_combined_test :combined_exp_1 }).to raise_error(Split::InvalidExperimentsFormatError )
+      end
+    end
+
+    context 'multiple experiments disabled' do
+      let!(:allow_multiple_experiments) { false }
+
+      it "raises an error if multiple experiments is disabled" do
+        expect(lambda { ab_combined_test :combined_exp_1 }).to raise_error(Split::InvalidExperimentsFormatError)
+      end
+    end
+
+    context 'without combined experiments' do
+      let!(:combined_experiments) { nil }
+
+      it "raises an error" do
+        expect(lambda { ab_combined_test :combined_exp_1 }).to raise_error(Split::InvalidExperimentsFormatError )
+      end
+    end
+
+    it "uses same alternatives for all sub experiments " do
+      allow(self).to receive(:get_alternative) { "test-alt" }
+      expect(self).to receive(:ab_test).with(:exp_1_click, {"control"=>0.5}, {"test-alt"=>0.5}) { "test-alt" }
+      expect(self).to receive(:ab_test).with(:exp_1_scroll, [{"test-alt" => 1}] )
+
+      ab_combined_test('combined_exp_1')
+    end
+  end
+end

spec/helper_spec.rb

@@ -35,6 +35,18 @@
       expect(lambda { ab_test({'link_color' => "purchase"}, 'blue', 'red') }).not_to raise_error
     end
 
+    it "raises an appropriate error when processing combined expirements" do
+      Split.configuration.experiments = {
+        :combined_exp_1 => {
+          :alternatives => [  { name: "control", percent: 50 },  { name: "test-alt", percent: 50 } ],
+          :metric => :my_metric,
+          :combined_experiments => [:combined_exp_1_sub_1]
+        }
+      }
+      Split::ExperimentCatalog.find_or_create('combined_exp_1')
+      expect(lambda { ab_test('combined_exp_1')}).to raise_error(Split::InvalidExperimentsFormatError )
+    end
+
     it "should assign a random alternative to a new user when there are an equal number of alternatives assigned" do
       ab_test('link_color', 'blue', 'red')
       expect(['red', 'blue']).to include(ab_user['link_color'])