From a9a387a05a914ffee96c5b992ba02427f9adc351 Mon Sep 17 00:00:00 2001 From: Ian Kronquist Date: Fri, 4 Sep 2015 14:22:33 -0700 Subject: [PATCH] (PDOC-23) Define JSON Structure for Puppet code --- lib/puppet_x/puppetlabs/strings.rb | 1 + .../yard/code_objects/defined_type_object.rb | 23 ++++ .../yard/code_objects/host_class_object.rb | 16 ++- .../yard/code_objects/method_object.rb | 56 +++++++++ .../yard/code_objects/provider_object.rb | 16 +++ .../code_objects/puppet_namespace_object.rb | 15 ++- .../strings/yard/code_objects/type_object.rb | 34 ++++++ .../handlers/puppet_3x_function_handler.rb | 1 + .../strings/yard/handlers/type_handler.rb | 112 ++++++++++-------- .../strings/yard/json_registry_store.rb | 72 +++++++---- .../strings/yard/type_handler_spec.rb | 2 +- 11 files changed, 270 insertions(+), 78 deletions(-) create mode 100644 lib/puppet_x/puppetlabs/strings/yard/code_objects/method_object.rb diff --git a/lib/puppet_x/puppetlabs/strings.rb b/lib/puppet_x/puppetlabs/strings.rb index 727bd22..299af0b 100644 --- a/lib/puppet_x/puppetlabs/strings.rb +++ b/lib/puppet_x/puppetlabs/strings.rb @@ -29,6 +29,7 @@ module PuppetX::PuppetLabs # aspects of puppet code in YARD's Registry module CodeObjects require 'puppet_x/puppetlabs/strings/yard/code_objects/puppet_namespace_object' + require 'puppet_x/puppetlabs/strings/yard/code_objects/method_object' require 'puppet_x/puppetlabs/strings/yard/code_objects/defined_type_object' require 'puppet_x/puppetlabs/strings/yard/code_objects/host_class_object' require 'puppet_x/puppetlabs/strings/yard/code_objects/type_object' diff --git a/lib/puppet_x/puppetlabs/strings/yard/code_objects/defined_type_object.rb b/lib/puppet_x/puppetlabs/strings/yard/code_objects/defined_type_object.rb index fba8cef..83d6d49 100644 --- a/lib/puppet_x/puppetlabs/strings/yard/code_objects/defined_type_object.rb +++ b/lib/puppet_x/puppetlabs/strings/yard/code_objects/defined_type_object.rb @@ -1,7 +1,30 @@ +require 'json' + class PuppetX::PuppetLabs::Strings::YARD::CodeObjects::DefinedTypeObject < PuppetX::PuppetLabs::Strings::YARD::CodeObjects::PuppetNamespaceObject # A list of parameters attached to this class. # @return [Array] attr_accessor :parameters attr_accessor :type_info + def to_s + name.to_s + end + + def to_json(*a) + { + "name" => @name, + "file" => file, + "line" => line, + "parameters" => Hash[@parameters], + "docstring" => Puppet::Util::Docs.scrub(@docstring), + "signatures" => @type_info.map do |signature| + signature.map do |key, value| + { + "name" => key, + "type" => value, + } + end + end, + }.to_json(*a) + end end diff --git a/lib/puppet_x/puppetlabs/strings/yard/code_objects/host_class_object.rb b/lib/puppet_x/puppetlabs/strings/yard/code_objects/host_class_object.rb index 4b7e0ec..c595fa9 100644 --- a/lib/puppet_x/puppetlabs/strings/yard/code_objects/host_class_object.rb +++ b/lib/puppet_x/puppetlabs/strings/yard/code_objects/host_class_object.rb @@ -2,8 +2,22 @@ class PuppetX::PuppetLabs::Strings::YARD::CodeObjects::HostClassObject < PuppetX # The {HostClassObject} that this class inherits from, if any. # @return [HostClassObject, Proxy, nil] attr_accessor :parent_class - attr_accessor :type_info +# def to_json(*a) +# { +# "name" => @name, +# "file" => file, +# "line" => line, +# "docstring" => Puppet::Util::Docs.scrub(@docstring), +# "parameters" => Hash[@parameters], +# "signatures" => @type_info.map do |key, value| +# { +# "name" => key, +# "type" => value, +# } +# end, +# }.to_json(*a) +# end # NOTE: `include_mods` is never used as it makes no sense for Puppet, but # this is called by `YARD::Registry` and it will pass a parameter. diff --git a/lib/puppet_x/puppetlabs/strings/yard/code_objects/method_object.rb b/lib/puppet_x/puppetlabs/strings/yard/code_objects/method_object.rb new file mode 100644 index 0000000..7ebcb18 --- /dev/null +++ b/lib/puppet_x/puppetlabs/strings/yard/code_objects/method_object.rb @@ -0,0 +1,56 @@ +class YARD::CodeObjects::MethodObject + + # Override to_s and to_json methods in Yard's MethodObject so that they + # return output formatted as I like for puppet 3x and 4x methods. + def to_s + if self[:puppet_4x_function] || self[:puppet_3x_function] + name.to_s + else + super + end + end + + def to_json(*a) + if self[:puppet_4x_function] + { + "name" => @name, + "file" => file, + "line" => line, + "puppet_version" => 4, + "docstring" => Puppet::Util::Docs.scrub(@docstring), + "documented_params" => @parameters.map do |tuple| + { + "name" => tuple[0], + "type" => tuple[1], + } + end, + "signatures" => @type_info.map do |signature| + signature.map do |key, value| + { + "name" => key, + "type" => value, + } + end + end, + }.to_json(*a) + elsif self[:puppet_3x_function] + { + "name" => @name, + "file" => file, + "line" => line, + "puppet_version" => 3, + "docstring" => Puppet::Util::Docs.scrub(@docstring), + "documented_params" => @parameters.map do |tuple| + { + "name" => tuple[0], + "type" => tuple[1], + } + end, + }.to_json(*a) + else + super + end + end + + +end diff --git a/lib/puppet_x/puppetlabs/strings/yard/code_objects/provider_object.rb b/lib/puppet_x/puppetlabs/strings/yard/code_objects/provider_object.rb index 95f8d54..a7a416c 100644 --- a/lib/puppet_x/puppetlabs/strings/yard/code_objects/provider_object.rb +++ b/lib/puppet_x/puppetlabs/strings/yard/code_objects/provider_object.rb @@ -2,4 +2,20 @@ class PuppetX::PuppetLabs::Strings::YARD::CodeObjects::ProviderObject < PuppetX: # A list of parameters attached to this class. # @return [Array] attr_accessor :parameters + + def to_json(*a) + { + "name" => @name, + "type_name" => @type_name, + "file" => file, + "line" => line, + "docstring" => Puppet::Util::Docs.scrub(@docstring), + "commands" => @commands, + "confines" => @confines, + "defaults" => @defaults, + "features" => @features, + }.to_json(*a) + end + + end diff --git a/lib/puppet_x/puppetlabs/strings/yard/code_objects/puppet_namespace_object.rb b/lib/puppet_x/puppetlabs/strings/yard/code_objects/puppet_namespace_object.rb index 8fe2329..1584503 100644 --- a/lib/puppet_x/puppetlabs/strings/yard/code_objects/puppet_namespace_object.rb +++ b/lib/puppet_x/puppetlabs/strings/yard/code_objects/puppet_namespace_object.rb @@ -1,11 +1,24 @@ class PuppetX::PuppetLabs::Strings::YARD::CodeObjects::PuppetNamespaceObject < YARD::CodeObjects::NamespaceObject + + attr_accessor :type_info # NOTE: `YARD::Registry#resolve` requires a method with this signature to # be present on all subclasses of `NamespaceObject`. def inheritance_tree(include_mods = false) [self] end - attr_accessor :type_info + def to_s + name.to_s + end + + def to_json(*a) + { + "name" => @name, + "file" => file, + "line" => line, + "docstring" => @docstring, + }.to_json(*a) + end # FIXME: We used to override `self.new` to ensure no YARD proxies were # created for namespaces segments that did not map to a host class or diff --git a/lib/puppet_x/puppetlabs/strings/yard/code_objects/type_object.rb b/lib/puppet_x/puppetlabs/strings/yard/code_objects/type_object.rb index 0ca38b1..09d2f1b 100644 --- a/lib/puppet_x/puppetlabs/strings/yard/code_objects/type_object.rb +++ b/lib/puppet_x/puppetlabs/strings/yard/code_objects/type_object.rb @@ -2,4 +2,38 @@ class PuppetX::PuppetLabs::Strings::YARD::CodeObjects::TypeObject < PuppetX::Pup # A list of parameters attached to this class. # @return [Array] attr_accessor :parameters + + def to_json(*a) + { + "name" => @name, + "file" => file, + "line" => line, + "docstring" => Puppet::Util::Docs.scrub(@docstring), + "parameters" => @parameter_details.map do |obj| + { + "allowed_values" => obj[:allowed_values] ? obj[:allowed_values].flatten : [], + "default" => obj[:default], + "docstring" => Puppet::Util::Docs.scrub(obj[:desc] || ''), + "namevar" => obj[:namevar], + "name" => obj[:name], + } + end, + "properties" => @property_details.map do |obj| + { + "allowed_values" => obj[:allowed_values] ? obj[:allowed_values].flatten : [], + "default" => obj[:default], + "docstring" => Puppet::Util::Docs.scrub(obj[:desc] || ''), + "name" => obj[:name], + } + end, + "features" => @features.map do |obj| + { + "docstring" => Puppet::Util::Docs.scrub(obj[:desc] || ''), + "methods" => obj[:methods], + "name" => obj[:name], + } + end, + }.to_json(*a) + end + end diff --git a/lib/puppet_x/puppetlabs/strings/yard/handlers/puppet_3x_function_handler.rb b/lib/puppet_x/puppetlabs/strings/yard/handlers/puppet_3x_function_handler.rb index 5529b44..f55ad4f 100644 --- a/lib/puppet_x/puppetlabs/strings/yard/handlers/puppet_3x_function_handler.rb +++ b/lib/puppet_x/puppetlabs/strings/yard/handlers/puppet_3x_function_handler.rb @@ -10,6 +10,7 @@ class PuppetX::PuppetLabs::Strings::YARD::Handlers::Puppet3xFunctionHandler < YA name, options = @heredoc_helper.process_parameters statement obj = MethodObject.new(function_namespace, name) + obj[:puppet_3x_function] = true register obj if options['doc'] diff --git a/lib/puppet_x/puppetlabs/strings/yard/handlers/type_handler.rb b/lib/puppet_x/puppetlabs/strings/yard/handlers/type_handler.rb index ffc1409..346277b 100644 --- a/lib/puppet_x/puppetlabs/strings/yard/handlers/type_handler.rb +++ b/lib/puppet_x/puppetlabs/strings/yard/handlers/type_handler.rb @@ -78,60 +78,68 @@ class PuppetX::PuppetLabs::Strings::YARD::Handlers::PuppetTypeHandler < YARD::Ha parameter_details = [] property_details = [] features = [] - obj = TypeObject.new(:root, "#{name}_type") do |o| - # FIXME: This block gets yielded twice for whatever reason - parameter_details = [] - property_details = [] - o.parameters = [] - # Find the do block following the Type. - do_block = statement.jump(:do_block) - # traverse the do block's children searching for function calls whose - # identifier is newparam (we're calling the newparam function) - do_block.traverse do |node| - if is_param? node - # The first member of the parameter tuple is the parameter name. - # Find the second identifier node under the fcall tree. The first one - # is 'newparam', the second one is the function name. - # Get its source. - # The second parameter is nil because we cannot infer types for these - # functions. In fact, that's a silly thing to ask because ruby - # types were deprecated with puppet 4 at the same time the type - # system was created. + obj = TypeObject.new(:root, name) + obj.parameters = [] - # Because of a ripper bug a symbol identifier is sometimes incorrectly parsed as a keyword. - # That is, the symbol `:true` will be represented as s(:symbol s(:kw, true... - param_name = node.children[1].jump(:ident) - if param_name == node.children[1] - param_name = node.children[1].jump(:kw) - end - param_name = param_name.source - o.parameters << [param_name, nil] - parameter_details << {:name => param_name, - :desc => fetch_description(node), :exists? => true, - :puppet_type => true, - :default => fetch_default(node), - :namevar => is_namevar?(node, param_name, name), - :parameter => true, - :allowed_values => get_parameter_allowed_values(node), - } - elsif is_prop? node - # Because of a ripper bug a symbol identifier is sometimes incorrectly parsed as a keyword. - # That is, the symbol `:true` will be represented as s(:symbol s(:kw, true... - prop_name = node.children[1].jump(:ident) - if prop_name == node.children[1] - prop_name = node.children[1].jump(:kw) - end - prop_name = prop_name.source - property_details << {:name => prop_name, - :desc => fetch_description(node), :exists? => true, - :default => fetch_default(node), - :puppet_type => true, - :property => true, - :allowed_values => get_property_allowed_values(node), - } - elsif is_feature? node - features << get_feature(node) + # Find the do block following the Type. + do_block = statement.jump(:do_block) + # traverse the do block's children searching for function calls whose + # identifier is newparam (we're calling the newparam function) + do_block.traverse do |node| + if is_param? node + # The first member of the parameter tuple is the parameter name. + # Find the second identifier node under the fcall tree. The first one + # is 'newparam', the second one is the function name. + # Get its source. + # The second parameter is nil because we cannot infer types for these + # functions. In fact, that's a silly thing to ask because ruby + # types were deprecated with puppet 4 at the same time the type + # system was created. + + # Because of a ripper bug a symbol identifier is sometimes incorrectly parsed as a keyword. + # That is, the symbol `:true` will be represented as s(:symbol s(:kw, true... + param_name = node.children[1].jump(:ident) + if param_name == node.children[1] + param_name = node.children[1].jump(:kw) end + param_name = param_name.source + obj.parameters << [param_name, nil] + parameter_details << {:name => param_name, + :desc => fetch_description(node), :exists? => true, + :puppet_type => true, + :default => fetch_default(node), + :namevar => is_namevar?(node, param_name, name), + :parameter => true, + :allowed_values => get_parameter_allowed_values(node), + } + elsif is_prop? node + # Because of a ripper bug a symbol identifier is sometimes incorrectly parsed as a keyword. + # That is, the symbol `:true` will be represented as s(:symbol s(:kw, true... + prop_name = node.children[1].jump(:ident) + if prop_name == node.children[1] + prop_name = node.children[1].jump(:kw) + end + prop_name = prop_name.source + property_details << {:name => prop_name, + :desc => fetch_description(node), :exists? => true, + :default => fetch_default(node), + :puppet_type => true, + :property => true, + :allowed_values => get_property_allowed_values(node), + } + elsif is_feature? node + features << get_feature(node) + elsif is_a_func_call_named? 'ensurable', node + # Someone could call the ensurable method and create an ensure + # property. If that happens, they it will be documented twice. Serves + # them right. + property_details << {:name => 'ensure', + :desc => '', :exists? => true, + :default => nil, + :puppet_type => true, + :property => true, + :allowed_values => [], + } end end obj.parameter_details = parameter_details diff --git a/lib/puppet_x/puppetlabs/strings/yard/json_registry_store.rb b/lib/puppet_x/puppetlabs/strings/yard/json_registry_store.rb index 63ee59b..f436501 100644 --- a/lib/puppet_x/puppetlabs/strings/yard/json_registry_store.rb +++ b/lib/puppet_x/puppetlabs/strings/yard/json_registry_store.rb @@ -4,45 +4,71 @@ module YARD def save(merge=true, file=nil) super - # FIXME: do we need this? - if file && file != @file - @file = file - @serializer = Serializers::JsonSerializer.new(@file) - end @serializer = Serializers::JsonSerializer.new(@file) sdb = Registry.single_object_db - original_extension = @serializer.extension - @serializer.extension = 'json' - @serializer.basepath = 'yardoc_json' - interesting_entries = proc { |key, val| - [:puppetnamespace, :hostclass,].include? val.type or - (val.type == :method and (val['puppet_4x_function'] or - val['puppet_3x_function'])) - } - rename_methods = proc { |key, value| - [value.type == :method ? value.name.to_sym : key, - value] - } if sdb == true || sdb == nil - @serializer.serialize(Hash[@store.select(&interesting_entries).map(&rename_methods)].to_json) + serialize_output_schema(@store) else values(false).each do |object| - @serializer.serialize(Hash[object.select(&interesting_entries).map(&rename_methods)].to_json) + serialize_output_schema(object) end end - @serializer.extension = original_extension true end + + # @param obj [Hash] A hash representing the registry or part of the + # registry. + def serialize_output_schema(obj) + schema = { + :puppet_functions => [], + :puppet_providers => [], + :puppet_classes => [], + :defined_types => [], + :puppet_types => [], + } + + schema[:puppet_functions] += obj.select do |key, val| + val.type == :method and (val['puppet_4x_function'] or + val['puppet_3x_function']) + end.values + + schema[:puppet_classes] += obj.select do |key, val| + val.type == :hostclass + end.values + + schema[:defined_types] += obj.select do |key, val| + val.type == :definedtype + end.values + + schema[:puppet_providers] += obj.select do |key, val| + val.type == :provider + end.values + + schema[:puppet_types] += obj.select do |key, val| + val.type == :type + end.values + + @serializer.serialize(schema.to_json) + end end - # Override the serializer because it puts the data at a whacky path and, more - # importantly, mashals the data with a bunch of non-printable characters. + # Override the serializer because it puts the data at a wacky path and, more + # importantly, marshals the data with a bunch of non-printable characters. module Serializers class JsonSerializer < YardocSerializer + + def initialize o + super + @options = { + :basepath => 'doc', + :extension => 'json', + } + @extension = 'json' + @basepath = 'doc' + end def serialize(data) path = File.join(basepath, "registry_dump.#{extension}") - require 'pry'; binding.pry log.debug "Serializing json to #{path}" File.open!(path, "wb") {|f| f.write data } end diff --git a/spec/unit/puppet_x/puppetlabs/strings/yard/type_handler_spec.rb b/spec/unit/puppet_x/puppetlabs/strings/yard/type_handler_spec.rb index 7c12f54..338ce5c 100644 --- a/spec/unit/puppet_x/puppetlabs/strings/yard/type_handler_spec.rb +++ b/spec/unit/puppet_x/puppetlabs/strings/yard/type_handler_spec.rb @@ -7,7 +7,7 @@ describe PuppetX::PuppetLabs::Strings::YARD::Handlers::PuppetTypeHandler do include StringsSpec::Parsing def the_type() - YARD::Registry.at("file_type") + YARD::Registry.at("file") end it "should have the proper docstring" do