From 676364bd172cc5cbbd67ced6b30e66a38283c474 Mon Sep 17 00:00:00 2001 From: Ian Kronquist Date: Fri, 31 Jul 2015 14:59:15 -0700 Subject: [PATCH] (PDOC-35) First pass at documenting providers Adding a provider page and menu * Add categories to html search bar * Require provider handler and object classes * Fetch provider code objects from registry * Add function to generate the provider list * Fetch providers from registry in monkey patches * Add provider templates * Add provider code object * Add provider handler * Add erb file to populate the provider list * Don't emit type information for providers in html * Add tests for provider handler Refactor heredoc: * Remove heredoc annotations * Move heredoc functions into a heredoc helper * Add heredoc helper class --- lib/puppet_x/puppetlabs/strings.rb | 2 + .../yard/code_objects/provider_object.rb | 5 + .../strings/yard/handlers/heredoc_helper.rb | 80 +++++++++++ .../strings/yard/handlers/provider_handler.rb | 128 ++++++++++++++++++ .../handlers/puppet_3x_function_handler.rb | 79 +---------- .../puppetlabs/strings/yard/monkey_patches.rb | 2 +- .../html/full_list_puppet_provider.erb | 1 + .../templates/default/fulldoc/html/setup.rb | 8 ++ .../yard/templates/default/html_helper.rb | 2 +- .../templates/default/layout/html/setup.rb | 5 +- .../default/provider/html/docstring.erb | 34 +++++ .../default/provider/html/header.erb | 5 + .../provider/html/parameter_details.erb | 6 + .../templates/default/provider/html/setup.rb | 1 + .../yard/templates/default/provider/setup.rb | 35 +++++ .../strings/yard/provider_handler_spec.rb | 63 +++++++++ 16 files changed, 377 insertions(+), 79 deletions(-) create mode 100644 lib/puppet_x/puppetlabs/strings/yard/code_objects/provider_object.rb create mode 100644 lib/puppet_x/puppetlabs/strings/yard/handlers/heredoc_helper.rb create mode 100644 lib/puppet_x/puppetlabs/strings/yard/handlers/provider_handler.rb create mode 100644 lib/puppet_x/puppetlabs/strings/yard/templates/default/fulldoc/html/full_list_puppet_provider.erb create mode 100644 lib/puppet_x/puppetlabs/strings/yard/templates/default/provider/html/docstring.erb create mode 100644 lib/puppet_x/puppetlabs/strings/yard/templates/default/provider/html/header.erb create mode 100644 lib/puppet_x/puppetlabs/strings/yard/templates/default/provider/html/parameter_details.erb create mode 100644 lib/puppet_x/puppetlabs/strings/yard/templates/default/provider/html/setup.rb create mode 100644 lib/puppet_x/puppetlabs/strings/yard/templates/default/provider/setup.rb create mode 100644 spec/unit/puppet_x/puppetlabs/strings/yard/provider_handler_spec.rb diff --git a/lib/puppet_x/puppetlabs/strings.rb b/lib/puppet_x/puppetlabs/strings.rb index 2bf4c16..619bf0d 100644 --- a/lib/puppet_x/puppetlabs/strings.rb +++ b/lib/puppet_x/puppetlabs/strings.rb @@ -27,6 +27,7 @@ module PuppetX::PuppetLabs require 'puppet_x/puppetlabs/strings/yard/code_objects/puppet_namespace_object' require 'puppet_x/puppetlabs/strings/yard/code_objects/defined_type_object' require 'puppet_x/puppetlabs/strings/yard/code_objects/host_class_object' + require 'puppet_x/puppetlabs/strings/yard/code_objects/provider_object' end # This submodule contains handlers which are used to extract relevant data about @@ -38,6 +39,7 @@ module PuppetX::PuppetLabs require 'puppet_x/puppetlabs/strings/yard/handlers/host_class_handler' require 'puppet_x/puppetlabs/strings/yard/handlers/puppet_3x_function_handler' require 'puppet_x/puppetlabs/strings/yard/handlers/puppet_4x_function_handler' + require 'puppet_x/puppetlabs/strings/yard/handlers/provider_handler' end ::YARD::Parser::SourceParser.register_parser_type(:puppet, diff --git a/lib/puppet_x/puppetlabs/strings/yard/code_objects/provider_object.rb b/lib/puppet_x/puppetlabs/strings/yard/code_objects/provider_object.rb new file mode 100644 index 0000000..95f8d54 --- /dev/null +++ b/lib/puppet_x/puppetlabs/strings/yard/code_objects/provider_object.rb @@ -0,0 +1,5 @@ +class PuppetX::PuppetLabs::Strings::YARD::CodeObjects::ProviderObject < PuppetX::PuppetLabs::Strings::YARD::CodeObjects::PuppetNamespaceObject + # A list of parameters attached to this class. + # @return [Array] + attr_accessor :parameters +end diff --git a/lib/puppet_x/puppetlabs/strings/yard/handlers/heredoc_helper.rb b/lib/puppet_x/puppetlabs/strings/yard/handlers/heredoc_helper.rb new file mode 100644 index 0000000..73fe2aa --- /dev/null +++ b/lib/puppet_x/puppetlabs/strings/yard/handlers/heredoc_helper.rb @@ -0,0 +1,80 @@ +class HereDocHelper + # NOTE: The following methods duplicate functionality from + # Puppet::Util::Reference and Puppet::Parser::Functions.functiondocs + # + # However, implementing this natively in YARD is a good test for the + # feasibility of extracting custom Ruby documentation. In the end, the + # existing approach taken by Puppet::Util::Reference may be the best due to + # the heavy use of metaprogramming in Types and Providers. + + # Extracts the Puppet function name and options hash from the parsed + # definition. + # + # @return [(String, Hash{String => String})] + def process_parameters(statement) + # Passing `false` to prameters excludes the block param from the returned + # list. + name, opts = statement.parameters(false).compact + + name = process_element(name) + + # Don't try to process options if we don't have any + if !opts.nil? + opts = opts.map do |tuple| + # Jump down into the S-Expression that represents a hashrocket, `=>`, + # and the values on either side of it. + tuple.jump(:assoc).map{|e| process_element(e)} + end + + options = Hash[opts] + else + options = {} + end + + [name, options] + end + + # Sometimes the YARD parser returns Heredoc strings that start with `<-` + # instead of `<<-`. + HEREDOC_START = /^ 2 + first = statement.children.first + return unless first.type == :const_path_ref and + first.children.length == 2 and + first.children.map { |o| o.source } == ["Puppet", "Type"] and + statement.children[1].source == "newtype" + + # Fetch the docstring for the provider. The docstring is the string literal + # assigned to the @doc parameter or absent, like this: + # @doc "docstring goes here" + # We assume that docstrings nodes have the following shape in the source + # code: + # ... + # s(s(:assign, + # s(:..., s(:ivar, "@doc", ...), ...), + # s(:..., + # s(:..., + # s(:tstring_content, + # "Manages files, including their content, etc.", ... + # Initialize the docstring to nil, the default value if we don't find + # anything + docstring = nil + # Walk the tree searching for assignments + statement.traverse do |node| + if node.type == :assign + # Once we have found and assignment, jump to the first ivar + # (the l-value) + # If we can't find an ivar return the node. + ivar = node.jump(:ivar) + # If we found and ivar and its source reads '@doc' then... + if ivar != node and ivar.source == '@doc' + # find the next string content + content = node.jump(:tstring_content) + # if we found the string content extract its source + if content != node + # The docstring is either the source stripped of heredoc + # annotations or the raw source. + if @heredoc_helper.is_heredoc? content.source + docstring = @heredoc_helper.process_heredoc content.source + else + docstring = content.source + end + end + # Since we found the @doc parameter (regardless of whether we + # successfully extracted its source), we're done. + break + # But if we didn't find the ivar loop around again. + else + next + end + end + end + + # The providers begin with: + # Puppet::Types.newtype(:symbol) + # Jump to the first identifier (':symbol') after the third argument + # ('(:symbol)') to the current statement + name = statement.children[2].jump(:ident).source + parameter_details = [] + obj = ProviderObject.new(:root, name) do |o| + # FIXME: This block gets yielded twice for whatever reason + parameter_details = [] + o.parameters = [] + # Find the de block following the Provider. + do_block = statement.jump(:do_block) + # traverse the do block's children searching for function calls whose + # identifier is newparam (we're calling the newparam function) + do_block.traverse do |node| + if node.type == :fcall and node.children.first.source == 'newparam' + # The first member of the parameter tuple is the parameter name. + # Find the second identifier node under the fcall tree. The first one + # is 'newparam', the second one is the function name. + # Get its source. + # The second parameter is nil because we cannot infer types for these + # functions. In fact, that's a silly thing to ask because ruby + # providers were deprecated with puppet 4 at the same time the type + # system was created. + param_name = node.children[1].jump(:ident).source + o.parameters << [param_name, nil] + parameter_details << {:name => param_name, + :desc => fetch_description(node), :exists? => true, + :provider => true} + end + end + end + obj.parameter_details = parameter_details + + register_docstring(obj, docstring, nil) + + register obj + end + + def fetch_description(fcall) + fcall.traverse do |node| + if node.type == :command and node.children.first.source == 'desc' + content = node.jump(:string_content) + if content != node + @heredoc_helper = HereDocHelper.new + if @heredoc_helper.is_heredoc? content.source + docstring = @heredoc_helper.process_heredoc content.source + else + docstring = content.source + end + return docstring + end + end + end + return nil + end +end diff --git a/lib/puppet_x/puppetlabs/strings/yard/handlers/puppet_3x_function_handler.rb b/lib/puppet_x/puppetlabs/strings/yard/handlers/puppet_3x_function_handler.rb index 7ae9bd3..5529b44 100644 --- a/lib/puppet_x/puppetlabs/strings/yard/handlers/puppet_3x_function_handler.rb +++ b/lib/puppet_x/puppetlabs/strings/yard/handlers/puppet_3x_function_handler.rb @@ -1,10 +1,13 @@ +require File.join(File.dirname(__FILE__),'./heredoc_helper') + class PuppetX::PuppetLabs::Strings::YARD::Handlers::Puppet3xFunctionHandler < YARD::Handlers::Ruby::Base include PuppetX::PuppetLabs::Strings::YARD::CodeObjects handles method_call(:newfunction) process do - name, options = process_parameters + @heredoc_helper = HereDocHelper.new + name, options = @heredoc_helper.process_parameters statement obj = MethodObject.new(function_namespace, name) @@ -47,78 +50,4 @@ class PuppetX::PuppetLabs::Strings::YARD::Handlers::Puppet3xFunctionHandler < YA obj end - # NOTE: The following methods duplicate functionality from - # Puppet::Util::Reference and Puppet::Parser::Functions.functiondocs - # - # However, implementing this natively in YARD is a good test for the - # feasibility of extracting custom Ruby documentation. In the end, the - # existing approach taken by Puppet::Util::Reference may be the best due to - # the heavy use of metaprogramming in Types and Providers. - - # Extracts the Puppet function name and options hash from the parsed - # definition. - # - # @return [(String, Hash{String => String})] - def process_parameters - # Passing `false` to prameters excludes the block param from the returned - # list. - name, opts = statement.parameters(false).compact - - name = process_element(name) - - # Don't try to process options if we don't have any - if !opts.nil? - opts = opts.map do |tuple| - # Jump down into the S-Expression that represents a hashrocket, `=>`, - # and the values on either side of it. - tuple.jump(:assoc).map{|e| process_element(e)} - end - - options = Hash[opts] - else - options = {} - end - - [name, options] - end - - # Sometimes the YARD parser returns Heredoc strings that start with `<-` - # instead of `<<-`. - HEREDOC_START = /^ [:provider]) %> diff --git a/lib/puppet_x/puppetlabs/strings/yard/templates/default/fulldoc/html/setup.rb b/lib/puppet_x/puppetlabs/strings/yard/templates/default/fulldoc/html/setup.rb index 69fb957..c6dd7c8 100644 --- a/lib/puppet_x/puppetlabs/strings/yard/templates/default/fulldoc/html/setup.rb +++ b/lib/puppet_x/puppetlabs/strings/yard/templates/default/fulldoc/html/setup.rb @@ -27,6 +27,14 @@ def generate_puppet_plugin_list generate_list_contents end +def generate_puppet_provider_list + @items = options.objects.select{|o| [:provider].include? o.type} if options.objects + @list_title = "Puppet Provider List" + @list_type = "puppet_provider" + generate_list_contents +end + + # A hacked version of class_list that can be instructed to only display certain # namespace types. This allows us to separate Puppet bits from Ruby bits. def namespace_list(opts = {}) diff --git a/lib/puppet_x/puppetlabs/strings/yard/templates/default/html_helper.rb b/lib/puppet_x/puppetlabs/strings/yard/templates/default/html_helper.rb index 2b4c8d6..d633fa9 100644 --- a/lib/puppet_x/puppetlabs/strings/yard/templates/default/html_helper.rb +++ b/lib/puppet_x/puppetlabs/strings/yard/templates/default/html_helper.rb @@ -59,7 +59,7 @@ class HTMLHelper result << "(" << "" << possible_types.join(", ") << "" << ")" end # Give up. It can probably be anything. - elsif !param[:puppet_3_func] + elsif not (param[:puppet_3_func] or param[:provider]) result << "(Unknown)" end diff --git a/lib/puppet_x/puppetlabs/strings/yard/templates/default/layout/html/setup.rb b/lib/puppet_x/puppetlabs/strings/yard/templates/default/layout/html/setup.rb index 53a1ee0..62a6369 100644 --- a/lib/puppet_x/puppetlabs/strings/yard/templates/default/layout/html/setup.rb +++ b/lib/puppet_x/puppetlabs/strings/yard/templates/default/layout/html/setup.rb @@ -2,7 +2,7 @@ # @objects_by_letter prevents that. Submit a pull request. def index @objects_by_letter = {} - objects = Registry.all(:class, :module, :puppetnamespace, :hostclass, :definedtype).sort_by {|o| o.name.to_s } + objects = Registry.all(:class, :module, :provider, :puppetnamespace, :hostclass, :definedtype).sort_by {|o| o.name.to_s } objects = run_verifier(objects) objects.each {|o| (@objects_by_letter[o.name.to_s[0,1].upcase] ||= []) << o } erb(:index) @@ -11,6 +11,7 @@ end def menu_lists [ {:type => 'puppet_manifest', :title => 'Puppet Manifests', :search_title => "Puppet Manifest List"}, - {:type => 'puppet_plugin', :title => 'Puppet Plugins', :search_title => "Puppet Plugin List"} + {:type => 'puppet_plugin', :title => 'Puppet Plugins', :search_title => "Puppet Plugin List"}, + {:type => 'puppet_provider', :title => 'Puppet Providers', :search_title => "Puppet Providers List"} ] + super end diff --git a/lib/puppet_x/puppetlabs/strings/yard/templates/default/provider/html/docstring.erb b/lib/puppet_x/puppetlabs/strings/yard/templates/default/provider/html/docstring.erb new file mode 100644 index 0000000..900985f --- /dev/null +++ b/lib/puppet_x/puppetlabs/strings/yard/templates/default/provider/html/docstring.erb @@ -0,0 +1,34 @@ +
+
+

<%= htmlify(@class_details[:desc]) %>

+
+
+
+ <% if @class_details[:examples] != {}%> +
+

Examples:

+ <% @class_details[:examples].each do |title, text| %> +

<%= title %>

+
<%= text %>
+ <% end %> +
+ <% end %> + <% if @class_details[:since] %> +

Since:

+
    +
  • +
    +

    <%= @class_details[:since] %>

    +
    +
  • +
+ <% end %> + <% if @class_details[:return] %> +

Return:

+
    +
  • + <%= @html_helper.generate_return_types(@class_details[:return][1], @class_details[:return][0]) %> +
  • +
+ <% end %> +
diff --git a/lib/puppet_x/puppetlabs/strings/yard/templates/default/provider/html/header.erb b/lib/puppet_x/puppetlabs/strings/yard/templates/default/provider/html/header.erb new file mode 100644 index 0000000..74b3dba --- /dev/null +++ b/lib/puppet_x/puppetlabs/strings/yard/templates/default/provider/html/header.erb @@ -0,0 +1,5 @@ +
+

+ <%= @header_text %> +

+
diff --git a/lib/puppet_x/puppetlabs/strings/yard/templates/default/provider/html/parameter_details.erb b/lib/puppet_x/puppetlabs/strings/yard/templates/default/provider/html/parameter_details.erb new file mode 100644 index 0000000..d03666e --- /dev/null +++ b/lib/puppet_x/puppetlabs/strings/yard/templates/default/provider/html/parameter_details.erb @@ -0,0 +1,6 @@ +

Parameter Summary

+
+
    + <%= @html_helper.generate_parameters(@param_details, object) %> +
+
diff --git a/lib/puppet_x/puppetlabs/strings/yard/templates/default/provider/html/setup.rb b/lib/puppet_x/puppetlabs/strings/yard/templates/default/provider/html/setup.rb new file mode 100644 index 0000000..91e114a --- /dev/null +++ b/lib/puppet_x/puppetlabs/strings/yard/templates/default/provider/html/setup.rb @@ -0,0 +1 @@ +include T('default/module/html') diff --git a/lib/puppet_x/puppetlabs/strings/yard/templates/default/provider/setup.rb b/lib/puppet_x/puppetlabs/strings/yard/templates/default/provider/setup.rb new file mode 100644 index 0000000..0b249f3 --- /dev/null +++ b/lib/puppet_x/puppetlabs/strings/yard/templates/default/provider/setup.rb @@ -0,0 +1,35 @@ +include T('default/module') + +require File.join(File.dirname(__FILE__),'../html_helper') +require File.join(File.dirname(__FILE__),'../template_helper') + +def init + sections :header, :box_info, :pre_docstring, :docstring, :parameter_details + + @template_helper = TemplateHelper.new + @html_helper = HTMLHelper.new +end + +def parameter_details + + params = object.parameter_details.map { |h| h[:name] } + + @param_details = [] + @param_details = object.parameter_details + @template_helper.check_parameters_match_docs object + + erb(:parameter_details) +end + +def header + @header_text = "Puppet Provider: #{object.name}" + + erb(:header) +end + +def docstring + + @class_details = @template_helper.extract_tag_data(object) + + erb(:docstring) +end diff --git a/spec/unit/puppet_x/puppetlabs/strings/yard/provider_handler_spec.rb b/spec/unit/puppet_x/puppetlabs/strings/yard/provider_handler_spec.rb new file mode 100644 index 0000000..16f0487 --- /dev/null +++ b/spec/unit/puppet_x/puppetlabs/strings/yard/provider_handler_spec.rb @@ -0,0 +1,63 @@ +require 'spec_helper' +require 'puppet_x/puppetlabs/strings/yard/handlers/provider_handler' +require 'strings_spec/parsing' + + +describe PuppetX::PuppetLabs::Strings::YARD::Handlers::PuppetProviderHandler do + include StringsSpec::Parsing + + def the_provider() + YARD::Registry.at("file") + end + + it "should have the proper docstring" do + parse <<-RUBY + Puppet::Type.newtype(:file) do + @doc = "Manages files, including their content, ownership, and perms." + newparam(:path) do + desc <<-'EOT' + The path to the file to manage. Must be fully qualified. + EOT + end + isnamevar + end + RUBY + + expect(the_provider.docstring).to eq("Manages files, including their " + + "content, ownership, and perms.") + end + + it "should have the proper parameter details" do + parse <<-RUBY + Puppet::Type.newtype(:file) do + @doc = "Manages files, including their content, ownership, and perms." + newparam(:path) do + desc <<-'EOT' + The path to the file to manage. Must be fully qualified. + EOT + end + isnamevar + end + RUBY + + expect(the_provider.parameter_details).to eq([{ :name => "path", + :desc => "The path to the file to manage. Must be fully qualified.", + :exists? => true, :provider => true, }]) + end + + it "should have the proper parameters" do + parse <<-RUBY + Puppet::Type.newtype(:file) do + @doc = "Manages files, including their content, ownership, and perms." + newparam(:path) do + desc <<-'EOT' + The path to the file to manage. Must be fully qualified. + EOT + end + isnamevar + end + RUBY + + expect(the_provider.parameters).to eq([["path", nil]]) + end +end