Prise3D
/
Thesis-OptimizationModules


			
							123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351
							

<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>macop.policies.reinforcement &mdash; macop v1.0.5 documentation</title>
  

  <script type="text/javascript" src="../_static/js/modernizr.min.js"></script>
  
    
      <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
        <script type="text/javascript" src="../_static/jquery.js"></script>
        <script type="text/javascript" src="../_static/underscore.js"></script>
        <script type="text/javascript" src="../_static/doctools.js"></script>
        <script type="text/javascript" src="../_static/language_data.js"></script>
    
    <script type="text/javascript" src="../_static/js/theme.js"></script>

    
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/custom.css" type="text/css" />
    <link rel="index" title="Index" href="../genindex.html" />
    <link rel="search" title="Search" href="../search.html" />
    <link rel="next" title="macop.solutions.base" href="macop.solutions.base.html" />
    <link rel="prev" title="macop.policies.classicals" href="macop.policies.classicals.html" /> 
</head>

<body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search" >
          

            <a href="../index.html" class="icon icon-home"> macop
          

          </a>

          
              <div class="version">
                1.0.5
              </div>
            
          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
              <p class="caption"><span class="caption-text">Contents:</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../description.html">Description</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="../api.html">API</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="../api.html#macop">macop</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api.html#macop-algorithms">macop.algorithms</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api.html#macop-callbacks">macop.callbacks</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api.html#macop-evaluators">macop.evaluators</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api.html#macop-operators">macop.operators</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="../api.html#macop-policies">macop.policies</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="macop.policies.base.html">macop.policies.base</a></li>
<li class="toctree-l3"><a class="reference internal" href="macop.policies.classicals.html">macop.policies.classicals</a></li>
<li class="toctree-l3 current"><a class="current reference internal" href="#">macop.policies.reinforcement</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../api.html#macop-solution">macop.solution</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api.html#macop-utils">macop.utils</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../documentations/index.html">Documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../examples.html">Some examples</a></li>
<li class="toctree-l1"><a class="reference internal" href="../contributing.html">Contributing</a></li>
</ul>

            
        </div>
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" aria-label="top navigation">
        
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="../index.html">macop</a>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        
          
<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../index.html">Docs</a> &raquo;</li>
        
          <li><a href="../api.html">API</a> &raquo;</li>
        
      <li>macop.policies.reinforcement</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
            
              <a href="https://github.com/prise-3d/macop/blob/master/docs/source/macop/macop.policies.reinforcement.rst" class="fa fa-github"> Edit on GitHub</a>
            
          
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="module-macop.policies.reinforcement">
<span id="macop-policies-reinforcement"></span><h1>macop.policies.reinforcement<a class="headerlink" href="#module-macop.policies.reinforcement" title="Permalink to this headline">¶</a></h1>
<p>Reinforcement learning policy classes implementations for Operator Selection Strategy</p>
<p class="rubric">Classes</p>
<table class="longtable docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 90%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><a class="reference internal" href="#macop.policies.reinforcement.UCBPolicy" title="macop.policies.reinforcement.UCBPolicy"><code class="xref py py-obj docutils literal notranslate"><span class="pre">UCBPolicy</span></code></a>(operators[, C, exp_rate])</p></td>
<td><p>Upper Confidence Bound (UCB) policy class which is used for applying UCB strategy when selecting and applying operator</p></td>
</tr>
</tbody>
</table>
<dl class="class">
<dt id="macop.policies.reinforcement.UCBPolicy">
<em class="property">class </em><code class="sig-prename descclassname">macop.policies.reinforcement.</code><code class="sig-name descname">UCBPolicy</code><span class="sig-paren">(</span><em class="sig-param">operators</em>, <em class="sig-param">C=100.0</em>, <em class="sig-param">exp_rate=0.5</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/macop/policies/reinforcement.html#UCBPolicy"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#macop.policies.reinforcement.UCBPolicy" title="Permalink to this definition">¶</a></dt>
<dd><p>Upper Confidence Bound (UCB) policy class which is used for applying UCB strategy when selecting and applying operator</p>
<p>Rather than performing exploration by simply selecting an arbitrary action, chosen with a probability that remains constant,
the UCB algorithm changes its exploration-exploitation balance as it gathers more knowledge of the environment.
It moves from being primarily focused on exploration, when actions that have been tried the least are preferred,
to instead concentrate on exploitation, selecting the action with the highest estimated reward.</p>
<ul class="simple">
<li><p>Resource link: <a class="reference external" href="https://banditalgs.com/2016/09/18/the-upper-confidence-bound-algorithm/">https://banditalgs.com/2016/09/18/the-upper-confidence-bound-algorithm/</a></p></li>
</ul>
<dl class="attribute">
<dt id="macop.policies.reinforcement.UCBPolicy.operators">
<code class="sig-name descname">operators</code><a class="headerlink" href="#macop.policies.reinforcement.UCBPolicy.operators" title="Permalink to this definition">¶</a></dt>
<dd><p>{[Operator]} – list of selected operators for the algorithm</p>
</dd></dl>

<dl class="attribute">
<dt id="macop.policies.reinforcement.UCBPolicy.C">
<code class="sig-name descname">C</code><a class="headerlink" href="#macop.policies.reinforcement.UCBPolicy.C" title="Permalink to this definition">¶</a></dt>
<dd><p>{float} – The second half of the UCB equation adds exploration, with the degree of exploration being controlled by the hyper-parameter <cite>C</cite>.</p>
</dd></dl>

<dl class="attribute">
<dt id="macop.policies.reinforcement.UCBPolicy.exp_rate">
<code class="sig-name descname">exp_rate</code><a class="headerlink" href="#macop.policies.reinforcement.UCBPolicy.exp_rate" title="Permalink to this definition">¶</a></dt>
<dd><p>{float} – exploration rate (probability to choose randomly next operator)</p>
</dd></dl>

<dl class="attribute">
<dt id="macop.policies.reinforcement.UCBPolicy.rewards">
<code class="sig-name descname">rewards</code><a class="headerlink" href="#macop.policies.reinforcement.UCBPolicy.rewards" title="Permalink to this definition">¶</a></dt>
<dd><p>{[float]} – list of summed rewards obtained for each operator</p>
</dd></dl>

<dl class="attribute">
<dt id="macop.policies.reinforcement.UCBPolicy.occurrences">
<code class="sig-name descname">occurrences</code><a class="headerlink" href="#macop.policies.reinforcement.UCBPolicy.occurrences" title="Permalink to this definition">¶</a></dt>
<dd><p>{[int]} – number of use (selected) of each operator</p>
</dd></dl>

<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># operators import</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">macop.operators.discrete.crossovers</span> <span class="kn">import</span> <span class="n">SimpleCrossover</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">macop.operators.discrete.mutators</span> <span class="kn">import</span> <span class="n">SimpleMutation</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># policy import</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">macop.policies.reinforcement</span> <span class="kn">import</span> <span class="n">UCBPolicy</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># solution and algorithm</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">macop.solutions.discrete</span> <span class="kn">import</span> <span class="n">BinarySolution</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">macop.algorithms.mono</span> <span class="kn">import</span> <span class="n">IteratedLocalSearch</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># evaluator import</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">macop.evaluators.discrete.mono</span> <span class="kn">import</span> <span class="n">KnapsackEvaluator</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># evaluator initialization (worths objects passed into data)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">worths</span> <span class="o">=</span> <span class="p">[</span> <span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">20</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">20</span><span class="p">)</span> <span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">evaluator</span> <span class="o">=</span> <span class="n">KnapsackEvaluator</span><span class="p">(</span><span class="n">data</span><span class="o">=</span><span class="p">{</span><span class="s1">&#39;worths&#39;</span><span class="p">:</span> <span class="n">worths</span><span class="p">})</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># validator specification (based on weights of each objects)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">weights</span> <span class="o">=</span> <span class="p">[</span> <span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">30</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">20</span><span class="p">)</span> <span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">validator</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">solution</span><span class="p">:</span> <span class="kc">True</span> <span class="k">if</span> <span class="nb">sum</span><span class="p">([</span><span class="n">weights</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">solution</span><span class="o">.</span><span class="n">_data</span><span class="p">)</span> <span class="k">if</span> <span class="n">value</span> <span class="o">==</span> <span class="mi">1</span><span class="p">])</span> <span class="o">&lt;</span> <span class="mi">200</span> <span class="k">else</span> <span class="kc">False</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># initializer function with lambda function</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">initializer</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">x</span><span class="o">=</span><span class="mi">20</span><span class="p">:</span> <span class="n">BinarySolution</span><span class="o">.</span><span class="n">random</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">validator</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># operators list with crossover and mutation</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">operators</span> <span class="o">=</span> <span class="p">[</span><span class="n">SimpleCrossover</span><span class="p">(),</span> <span class="n">SimpleMutation</span><span class="p">()]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">policy</span> <span class="o">=</span> <span class="n">UCBPolicy</span><span class="p">(</span><span class="n">operators</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">algo</span> <span class="o">=</span> <span class="n">IteratedLocalSearch</span><span class="p">(</span><span class="n">initializer</span><span class="p">,</span> <span class="n">evaluator</span><span class="p">,</span> <span class="n">operators</span><span class="p">,</span> <span class="n">policy</span><span class="p">,</span> <span class="n">validator</span><span class="p">,</span> <span class="n">maximise</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">policy</span><span class="o">.</span><span class="n">_occurences</span>
<span class="go">[0, 0]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">solution</span> <span class="o">=</span> <span class="n">algo</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="mi">100</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">type</span><span class="p">(</span><span class="n">solution</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span>
<span class="go">&#39;BinarySolution&#39;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">policy</span><span class="o">.</span><span class="n">_occurences</span> <span class="c1"># one more due to first evaluation</span>
<span class="go">[51, 53]</span>
</pre></div>
</div>
<dl class="method">
<dt id="macop.policies.reinforcement.UCBPolicy.apply">
<code class="sig-name descname">apply</code><span class="sig-paren">(</span><em class="sig-param">solution</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/macop/policies/reinforcement.html#UCBPolicy.apply"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#macop.policies.reinforcement.UCBPolicy.apply" title="Permalink to this definition">¶</a></dt>
<dd><p>Apply specific operator chosen to create new solution, computes its fitness and returns solution</p>
<ul class="simple">
<li><p>fitness improvment is saved as rewards</p></li>
<li><p>selected operator occurence is also increased</p></li>
</ul>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>solution</strong> – {Solution} – the solution to use for generating new solution</p>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>{Solution} – new generated solution</p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="macop.policies.reinforcement.UCBPolicy.select">
<code class="sig-name descname">select</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/macop/policies/reinforcement.html#UCBPolicy.select"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#macop.policies.reinforcement.UCBPolicy.select" title="Permalink to this definition">¶</a></dt>
<dd><p>Select using Upper Confidence Bound the next operator to use (using acquired rewards)</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>the selected operator</p>
</dd>
<dt class="field-even">Return type</dt>
<dd class="field-even"><p>{Operator}</p>
</dd>
</dl>
</dd></dl>

</dd></dl>

</div>


           </div>
           
          </div>
          <footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="macop.solutions.base.html" class="btn btn-neutral float-right" title="macop.solutions.base" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
      
      
        <a href="macop.policies.classicals.html" class="btn btn-neutral float-left" title="macop.policies.classicals" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
      
    </div>
  

  <hr/>

  <div role="contentinfo">
    <p>
        &copy; Copyright 2020, Jérôme BUISINE

    </p>
  </div>
  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  
</body>
</html>