(PDOC-35) First pass at documenting providers

Adding a provider page and menu
* Add categories to html search bar
* Require provider handler and object classes
* Fetch provider code objects from registry
* Add function to generate the provider list
* Fetch providers from registry in monkey patches
* Add provider templates
* Add provider code object
* Add provider handler
* Add erb file to populate the provider list
* Don't emit type information for providers in html
* Add tests for provider handler

Refactor heredoc:
* Remove heredoc annotations
* Move heredoc functions into a heredoc helper
* Add heredoc helper class

lib/puppet_x/puppetlabs/strings.rb

@@ -27,6 +27,7 @@ module PuppetX::PuppetLabs
         require 'puppet_x/puppetlabs/strings/yard/code_objects/puppet_namespace_object'
         require 'puppet_x/puppetlabs/strings/yard/code_objects/defined_type_object'
         require 'puppet_x/puppetlabs/strings/yard/code_objects/host_class_object'
+        require 'puppet_x/puppetlabs/strings/yard/code_objects/provider_object'
       end
 
       # This submodule contains handlers which are used to extract relevant data about
@@ -38,6 +39,7 @@ module PuppetX::PuppetLabs
         require 'puppet_x/puppetlabs/strings/yard/handlers/host_class_handler'
         require 'puppet_x/puppetlabs/strings/yard/handlers/puppet_3x_function_handler'
         require 'puppet_x/puppetlabs/strings/yard/handlers/puppet_4x_function_handler'
+        require 'puppet_x/puppetlabs/strings/yard/handlers/provider_handler'
       end
 
       ::YARD::Parser::SourceParser.register_parser_type(:puppet,

lib/puppet_x/puppetlabs/strings/yard/code_objects/provider_object.rb

@@ -0,0 +1,5 @@
+class PuppetX::PuppetLabs::Strings::YARD::CodeObjects::ProviderObject < PuppetX::PuppetLabs::Strings::YARD::CodeObjects::PuppetNamespaceObject
+  # A list of parameters attached to this class.
+  # @return [Array<Array(String, String)>]
+  attr_accessor :parameters
+end

lib/puppet_x/puppetlabs/strings/yard/handlers/heredoc_helper.rb

@@ -0,0 +1,80 @@
+class HereDocHelper
+  # NOTE: The following methods duplicate functionality from
+  # Puppet::Util::Reference and Puppet::Parser::Functions.functiondocs
+  #
+  # However, implementing this natively in YARD is a good test for the
+  # feasibility of extracting custom Ruby documentation. In the end, the
+  # existing approach taken by Puppet::Util::Reference may be the best due to
+  # the heavy use of metaprogramming in Types and Providers.
+
+  # Extracts the Puppet function name and options hash from the parsed
+  # definition.
+  #
+  # @return [(String, Hash{String => String})]
+  def process_parameters(statement)
+    # Passing `false` to prameters excludes the block param from the returned
+    # list.
+    name, opts = statement.parameters(false).compact
+
+    name = process_element(name)
+
+    # Don't try to process options if we don't have any
+    if !opts.nil?
+      opts = opts.map do |tuple|
+        # Jump down into the S-Expression that represents a hashrocket, `=>`,
+        # and the values on either side of it.
+        tuple.jump(:assoc).map{|e| process_element(e)}
+      end
+
+      options = Hash[opts]
+    else
+      options = {}
+    end
+
+    [name, options]
+  end
+
+  # Sometimes the YARD parser returns Heredoc strings that start with `<-`
+  # instead of `<<-`.
+  HEREDOC_START = /^<?<-/
+
+  def is_heredoc?(str)
+    HEREDOC_START.match(str)
+  end
+
+    # Turns an entry in the method parameter list into a string.
+    #
+    # @param ele [YARD::Parser::Ruby::AstNode]
+    # @return [String]
+    def process_element(ele)
+      ele = ele.jump(:ident, :string_content)
+
+      case ele.type
+      when :ident
+        ele.source
+      when :string_content
+        source = ele.source
+        if is_heredoc? source
+          process_heredoc(source)
+        else
+          source
+        end
+      end
+    end
+
+  # Cleans up and formats Heredoc contents parsed by YARD.
+  #
+  # @param source [String]
+  # @return [String]
+  def process_heredoc(source)
+    source = source.lines.to_a
+
+    # YARD adds a line of source context on either side of the Heredoc
+    # contents.
+    source.shift
+    source.pop
+
+    # This utility method normalizes indentation and trims whitespace.
+    Puppet::Util::Docs.scrub(source.join)
+  end
+end

lib/puppet_x/puppetlabs/strings/yard/handlers/provider_handler.rb

@@ -0,0 +1,128 @@
+# Handles `dispatch` calls within a future parser function declaration. For
+# now, it just treats any docstring as an `@overlaod` tag and attaches the
+# overload to the parent function.
+class PuppetX::PuppetLabs::Strings::YARD::Handlers::PuppetProviderHandler < YARD::Handlers::Ruby::Base
+  include PuppetX::PuppetLabs::Strings::YARD::CodeObjects
+
+  handles :call
+
+  process do
+    @heredoc_helper = HereDocHelper.new
+    # Puppet providers always begin with:
+    # Puppet::Types.newtype...
+    # Therefore, we match the corresponding trees which look like this:
+    # s(:call,
+    #   s(:const_path_ref,
+    #     s(:var_ref, s(:const, "Puppet", ...), ...),
+    #   s(:const, "Type", ...),
+    # You think this is ugly? It's better than the alternative.
+    return unless statement.children.length > 2
+    first = statement.children.first
+    return unless first.type == :const_path_ref and
+      first.children.length == 2 and
+      first.children.map { |o| o.source } == ["Puppet", "Type"] and
+      statement.children[1].source == "newtype"
+
+    # Fetch the docstring for the provider. The docstring is the string literal
+    # assigned to the @doc parameter or absent, like this:
+    # @doc "docstring goes here"
+    # We assume that docstrings nodes have the following shape in the source
+    # code:
+    # ...
+    # s(s(:assign,
+    #        s(:..., s(:ivar, "@doc", ...), ...),
+    #        s(:...,
+    #           s(:...,
+    #              s(:tstring_content,
+    #                 "Manages files, including their content, etc.", ...
+    # Initialize the docstring to nil, the default value if we don't find
+    # anything
+    docstring = nil
+    # Walk the tree searching for assignments
+    statement.traverse do |node|
+      if node.type == :assign
+        # Once we have found and assignment, jump to the first ivar
+        # (the l-value)
+        # If we can't find an ivar return the node.
+        ivar = node.jump(:ivar)
+        # If we found and  ivar and its source reads '@doc' then...
+        if ivar != node and ivar.source == '@doc'
+          # find the next string content
+          content = node.jump(:tstring_content)
+          # if we found the string content extract its source
+          if content != node
+            # The docstring is either the source stripped of heredoc
+            # annotations or the raw source.
+            if @heredoc_helper.is_heredoc? content.source
+              docstring = @heredoc_helper.process_heredoc content.source
+            else
+              docstring = content.source
+            end
+          end
+          # Since we found the @doc parameter (regardless of whether we
+          # successfully extracted its source), we're done.
+          break
+        # But if we didn't find the ivar loop around again.
+        else
+          next
+        end
+      end
+    end
+
+    # The providers begin with:
+    # Puppet::Types.newtype(:symbol)
+    # Jump to the first identifier (':symbol') after the third argument
+    # ('(:symbol)') to the current statement
+    name = statement.children[2].jump(:ident).source
+    parameter_details = []
+    obj = ProviderObject.new(:root, name) do |o|
+      # FIXME: This block gets yielded twice for whatever reason
+      parameter_details = []
+      o.parameters = []
+      # Find the de block following the Provider.
+      do_block = statement.jump(:do_block)
+      # traverse the do block's children searching for function calls whose
+      # identifier is newparam (we're calling the newparam function)
+      do_block.traverse do |node|
+        if node.type == :fcall and node.children.first.source == 'newparam'
+          # The first member of the parameter tuple is the parameter name.
+          # Find the second identifier node under the fcall tree. The first one
+          # is 'newparam', the second one is the function name.
+          # Get its source.
+          # The second parameter is nil because we cannot infer types for these
+          # functions. In fact, that's a silly thing to ask because ruby
+          # providers were deprecated with puppet 4 at the same time the type
+          # system was created.
+          param_name = node.children[1].jump(:ident).source
+          o.parameters << [param_name, nil]
+          parameter_details << {:name => param_name,
+            :desc => fetch_description(node), :exists? => true,
+            :provider => true}
+        end
+      end
+    end
+    obj.parameter_details = parameter_details
+
+    register_docstring(obj, docstring, nil)
+
+    register obj
+  end
+
+  def fetch_description(fcall)
+    fcall.traverse do |node|
+      if node.type == :command and node.children.first.source == 'desc'
+        content = node.jump(:string_content)
+        if content != node
+          @heredoc_helper = HereDocHelper.new
+          if @heredoc_helper.is_heredoc? content.source
+            docstring = @heredoc_helper.process_heredoc content.source
+          else
+            docstring = content.source
+          end
+          return docstring
+        end
+      end
+    end
+    return nil
+  end
+end

lib/puppet_x/puppetlabs/strings/yard/handlers/puppet_3x_function_handler.rb

@@ -1,10 +1,13 @@
+require File.join(File.dirname(__FILE__),'./heredoc_helper')
+
 class PuppetX::PuppetLabs::Strings::YARD::Handlers::Puppet3xFunctionHandler < YARD::Handlers::Ruby::Base
   include PuppetX::PuppetLabs::Strings::YARD::CodeObjects
 
   handles method_call(:newfunction)
 
   process do
-    name, options = process_parameters
+    @heredoc_helper = HereDocHelper.new
+    name, options = @heredoc_helper.process_parameters statement
 
     obj = MethodObject.new(function_namespace, name)
 
@@ -47,78 +50,4 @@ class PuppetX::PuppetLabs::Strings::YARD::Handlers::Puppet3xFunctionHandler < YA
     obj
   end
 
-  # NOTE: The following methods duplicate functionality from
-  # Puppet::Util::Reference and Puppet::Parser::Functions.functiondocs
-  #
-  # However, implementing this natively in YARD is a good test for the
-  # feasibility of extracting custom Ruby documentation. In the end, the
-  # existing approach taken by Puppet::Util::Reference may be the best due to
-  # the heavy use of metaprogramming in Types and Providers.
-
-  # Extracts the Puppet function name and options hash from the parsed
-  # definition.
-  #
-  # @return [(String, Hash{String => String})]
-  def process_parameters
-    # Passing `false` to prameters excludes the block param from the returned
-    # list.
-    name, opts = statement.parameters(false).compact
-
-    name = process_element(name)
-
-    # Don't try to process options if we don't have any
-    if !opts.nil?
-      opts = opts.map do |tuple|
-        # Jump down into the S-Expression that represents a hashrocket, `=>`,
-        # and the values on either side of it.
-        tuple.jump(:assoc).map{|e| process_element(e)}
-      end
-
-      options = Hash[opts]
-    else
-      options = {}
-    end
-
-    [name, options]
-  end
-
-  # Sometimes the YARD parser returns Heredoc strings that start with `<-`
-  # instead of `<<-`.
-  HEREDOC_START = /^<?<-/
-
-    # Turns an entry in the method parameter list into a string.
-    #
-    # @param ele [YARD::Parser::Ruby::AstNode]
-    # @return [String]
-    def process_element(ele)
-      ele = ele.jump(:ident, :string_content)
-
-      case ele.type
-      when :ident
-        ele.source
-      when :string_content
-        source = ele.source
-        if HEREDOC_START.match(source)
-          process_heredoc(source)
-        else
-          source
-        end
-      end
-    end
-
-  # Cleans up and formats Heredoc contents parsed by YARD.
-  #
-  # @param source [String]
-  # @return [String]
-  def process_heredoc(source)
-    source = source.lines.to_a
-
-    # YARD adds a line of source context on either side of the Heredoc
-    # contents.
-    source.shift
-    source.pop
-
-    # This utility method normalizes indentation and trims whitespace.
-    Puppet::Util::Docs.scrub(source.join)
-  end
 end

lib/puppet_x/puppetlabs/strings/yard/monkey_patches.rb

@@ -6,7 +6,7 @@ require 'yard'
 
 class YARD::CLI::Yardoc
   def all_objects
-    YARD::Registry.all(:root, :module, :class, :puppetnamespace, :hostclass, :definedtype)
+    YARD::Registry.all(:root, :module, :class, :provider, :puppetnamespace, :hostclass, :definedtype)
   end
 end
 

lib/puppet_x/puppetlabs/strings/yard/templates/default/fulldoc/html/full_list_puppet_provider.erb

@@ -0,0 +1 @@
+<%= namespace_list(:namespace_types => [:provider]) %>

lib/puppet_x/puppetlabs/strings/yard/templates/default/fulldoc/html/setup.rb

@@ -27,6 +27,14 @@ def generate_puppet_plugin_list
   generate_list_contents
 end
 
+def generate_puppet_provider_list
+  @items = options.objects.select{|o| [:provider].include? o.type} if options.objects
+  @list_title = "Puppet Provider List"
+  @list_type = "puppet_provider"
+  generate_list_contents
+end
+
+
 # A hacked version of class_list that can be instructed to only display certain
 # namespace types. This allows us to separate Puppet bits from Ruby bits.
 def namespace_list(opts = {})

lib/puppet_x/puppetlabs/strings/yard/templates/default/html_helper.rb

@@ -59,7 +59,7 @@ class HTMLHelper
           result << "(" << "<tt>" << possible_types.join(", ") << "</tt>" << ")"
         end
       # Give up. It can probably be anything.
-      elsif !param[:puppet_3_func]
+      elsif not (param[:puppet_3_func] or param[:provider])
         result << "(<tt>Unknown</tt>)"
       end
 

lib/puppet_x/puppetlabs/strings/yard/templates/default/layout/html/setup.rb

@@ -2,7 +2,7 @@
 # @objects_by_letter prevents that. Submit a pull request.
 def index
   @objects_by_letter = {}
-  objects = Registry.all(:class, :module, :puppetnamespace, :hostclass, :definedtype).sort_by {|o| o.name.to_s }
+  objects = Registry.all(:class, :module, :provider, :puppetnamespace, :hostclass, :definedtype).sort_by {|o| o.name.to_s }
   objects = run_verifier(objects)
   objects.each {|o| (@objects_by_letter[o.name.to_s[0,1].upcase] ||= []) << o }
   erb(:index)
@@ -11,6 +11,7 @@ end
 def menu_lists
   [
     {:type => 'puppet_manifest', :title => 'Puppet Manifests', :search_title => "Puppet Manifest List"},
-    {:type => 'puppet_plugin', :title => 'Puppet Plugins', :search_title => "Puppet Plugin List"}
+    {:type => 'puppet_plugin', :title => 'Puppet Plugins', :search_title => "Puppet Plugin List"},
+    {:type => 'puppet_provider', :title => 'Puppet Providers', :search_title => "Puppet Providers  List"}
   ] + super
 end

lib/puppet_x/puppetlabs/strings/yard/templates/default/provider/html/docstring.erb

@@ -0,0 +1,34 @@
+<div class="docstring">
+  <div class="discussion">
+    <p><%= htmlify(@class_details[:desc]) %></p>
+  </div>
+</div>
+<div class="tags">
+  <% if @class_details[:examples] != {}%>
+    <div class="examples">
+    <p class="tag_title">Examples:</p>
+    <% @class_details[:examples].each do |title, text| %>
+      <div class="inline"><p><%= title %></p></div>
+      <pre class="example code"><code><span><%= text %></span></code></pre>
+    <% end %>
+    </div>
+  <% end %>
+  <% if @class_details[:since] %>
+    <p class="tag_title">Since:</p>
+    <ul class="since">
+      <li>
+        <div class="inline">
+          <p><%= @class_details[:since] %></p>
+        </div>
+      </li>
+    </ul>
+  <% end %>
+  <% if @class_details[:return] %>
+    <p class="tag_title">Return:</p>
+    <ul class="return">
+      <li>
+        <%= @html_helper.generate_return_types(@class_details[:return][1], @class_details[:return][0]) %>
+      </li>
+    </ul>
+  <% end %>
+</div>

lib/puppet_x/puppetlabs/strings/yard/templates/default/provider/html/header.erb

@@ -0,0 +1,5 @@
+<div class='module_header'>
+  <h1>
+    <%= @header_text %>
+  </h1>
+</div>

lib/puppet_x/puppetlabs/strings/yard/templates/default/provider/html/parameter_details.erb

@@ -0,0 +1,6 @@
+<h2>Parameter Summary</h2>
+<div class="tags">
+  <ul class="param">
+    <%= @html_helper.generate_parameters(@param_details, object) %>
+  </ul>
+</div>

lib/puppet_x/puppetlabs/strings/yard/templates/default/provider/html/setup.rb

@@ -0,0 +1 @@
+include T('default/module/html')

lib/puppet_x/puppetlabs/strings/yard/templates/default/provider/setup.rb

@@ -0,0 +1,35 @@
+include T('default/module')
+
+require File.join(File.dirname(__FILE__),'../html_helper')
+require File.join(File.dirname(__FILE__),'../template_helper')
+
+def init
+  sections :header, :box_info, :pre_docstring, :docstring, :parameter_details
+
+  @template_helper = TemplateHelper.new
+  @html_helper = HTMLHelper.new
+end
+
+def parameter_details
+
+  params = object.parameter_details.map { |h| h[:name] }
+
+  @param_details = []
+  @param_details = object.parameter_details
+  @template_helper.check_parameters_match_docs object
+
+  erb(:parameter_details)
+end
+
+def header
+  @header_text = "Puppet Provider: #{object.name}"
+
+  erb(:header)
+end
+
+def docstring
+
+  @class_details = @template_helper.extract_tag_data(object)
+
+  erb(:docstring)
+end

spec/unit/puppet_x/puppetlabs/strings/yard/provider_handler_spec.rb

@@ -0,0 +1,63 @@
+require 'spec_helper'
+require 'puppet_x/puppetlabs/strings/yard/handlers/provider_handler'
+require 'strings_spec/parsing'
+
+
+describe PuppetX::PuppetLabs::Strings::YARD::Handlers::PuppetProviderHandler do
+  include StringsSpec::Parsing
+
+  def the_provider()
+    YARD::Registry.at("file")
+  end
+
+  it "should have the proper docstring" do
+    parse  <<-RUBY
+      Puppet::Type.newtype(:file) do
+        @doc = "Manages files, including their content, ownership, and perms."
+        newparam(:path) do
+          desc <<-'EOT'
+            The path to the file to manage.  Must be fully qualified.
+          EOT
+        end
+        isnamevar
+      end
+    RUBY
+
+    expect(the_provider.docstring).to eq("Manages files, including their " +
+      "content, ownership, and perms.")
+  end
+
+  it "should have the proper parameter details" do
+    parse  <<-RUBY
+      Puppet::Type.newtype(:file) do
+        @doc = "Manages files, including their content, ownership, and perms."
+        newparam(:path) do
+          desc <<-'EOT'
+            The path to the file to manage.  Must be fully qualified.
+          EOT
+        end
+        isnamevar
+      end
+    RUBY
+
+    expect(the_provider.parameter_details).to eq([{ :name => "path",
+      :desc => "The path to the file to manage.  Must be fully qualified.",
+      :exists? => true, :provider => true, }])
+  end
+
+  it "should have the proper parameters" do
+    parse  <<-RUBY
+      Puppet::Type.newtype(:file) do
+        @doc = "Manages files, including their content, ownership, and perms."
+        newparam(:path) do
+          desc <<-'EOT'
+            The path to the file to manage.  Must be fully qualified.
+          EOT
+        end
+        isnamevar
+      end
+    RUBY
+
+    expect(the_provider.parameters).to eq([["path", nil]])
+  end
+end