(PDOC-23) Define JSON Structure for Puppet code

lib/puppet_x/puppetlabs/strings.rb

@@ -29,6 +29,7 @@ module PuppetX::PuppetLabs
       # aspects of puppet code in YARD's Registry
       module CodeObjects
         require 'puppet_x/puppetlabs/strings/yard/code_objects/puppet_namespace_object'
+        require 'puppet_x/puppetlabs/strings/yard/code_objects/method_object'
         require 'puppet_x/puppetlabs/strings/yard/code_objects/defined_type_object'
         require 'puppet_x/puppetlabs/strings/yard/code_objects/host_class_object'
         require 'puppet_x/puppetlabs/strings/yard/code_objects/type_object'

lib/puppet_x/puppetlabs/strings/yard/code_objects/defined_type_object.rb

@@ -1,7 +1,30 @@
+require 'json'
+
 class PuppetX::PuppetLabs::Strings::YARD::CodeObjects::DefinedTypeObject < PuppetX::PuppetLabs::Strings::YARD::CodeObjects::PuppetNamespaceObject
   # A list of parameters attached to this class.
   # @return [Array<Array(String, String)>]
   attr_accessor :parameters
   attr_accessor :type_info
 
+  def to_s
+    name.to_s
+  end
+
+  def to_json(*a)
+    {
+      "name"             => @name,
+      "file"             => file,
+      "line"             => line,
+      "parameters"       => Hash[@parameters],
+      "docstring"        => Puppet::Util::Docs.scrub(@docstring),
+      "signatures"       => @type_info.map do |signature|
+        signature.map do |key, value|
+          {
+            "name" => key,
+            "type" => value,
+          }
+        end
+      end,
+    }.to_json(*a)
+  end
 end

lib/puppet_x/puppetlabs/strings/yard/code_objects/host_class_object.rb

@@ -2,8 +2,22 @@ class PuppetX::PuppetLabs::Strings::YARD::CodeObjects::HostClassObject < PuppetX
   # The {HostClassObject} that this class inherits from, if any.
   # @return [HostClassObject, Proxy, nil]
   attr_accessor :parent_class
-  attr_accessor :type_info
 
+#  def to_json(*a)
+#    {
+#      "name"             => @name,
+#      "file"             => file,
+#      "line"             => line,
+#      "docstring"        => Puppet::Util::Docs.scrub(@docstring),
+#      "parameters"       => Hash[@parameters],
+#      "signatures"       => @type_info.map do |key, value|
+#          {
+#            "name" => key,
+#            "type" => value,
+#          }
+#      end,
+#    }.to_json(*a)
+#  end
 
   # NOTE: `include_mods` is never used as it makes no sense for Puppet, but
   # this is called by `YARD::Registry` and it will pass a parameter.

lib/puppet_x/puppetlabs/strings/yard/code_objects/method_object.rb

@@ -0,0 +1,56 @@
+class YARD::CodeObjects::MethodObject
+
+  # Override to_s and to_json methods in Yard's MethodObject so that they
+  # return output formatted as I like for puppet 3x and 4x methods.
+  def to_s
+    if self[:puppet_4x_function] || self[:puppet_3x_function]
+      name.to_s
+    else
+      super
+    end
+  end
+
+  def to_json(*a)
+    if self[:puppet_4x_function]
+      {
+        "name"                => @name,
+        "file"                => file,
+        "line"                => line,
+        "puppet_version"      => 4,
+        "docstring"           => Puppet::Util::Docs.scrub(@docstring),
+        "documented_params"   => @parameters.map do |tuple|
+          {
+            "name" => tuple[0],
+            "type" => tuple[1],
+          }
+        end,
+       "signatures"       => @type_info.map do |signature|
+          signature.map do |key, value|
+            {
+              "name" => key,
+              "type" => value,
+            }
+          end
+        end,
+      }.to_json(*a)
+    elsif self[:puppet_3x_function]
+      {
+        "name"                => @name,
+        "file"                => file,
+        "line"                => line,
+        "puppet_version"      => 3,
+        "docstring"           => Puppet::Util::Docs.scrub(@docstring),
+        "documented_params"   => @parameters.map do |tuple|
+          {
+            "name" => tuple[0],
+            "type" => tuple[1],
+          }
+        end,
+      }.to_json(*a)
+    else
+      super
+    end
+  end
+
+
+end

lib/puppet_x/puppetlabs/strings/yard/code_objects/provider_object.rb

@@ -2,4 +2,20 @@ class PuppetX::PuppetLabs::Strings::YARD::CodeObjects::ProviderObject < PuppetX:
   # A list of parameters attached to this class.
   # @return [Array<Array(String, String)>]
   attr_accessor :parameters
+
+  def to_json(*a)
+    {
+      "name"             => @name,
+      "type_name"        => @type_name,
+      "file"             => file,
+      "line"             => line,
+      "docstring"        => Puppet::Util::Docs.scrub(@docstring),
+      "commands"         => @commands,
+      "confines"         => @confines,
+      "defaults"         => @defaults,
+      "features"         => @features,
+    }.to_json(*a)
+  end
+
+
 end

lib/puppet_x/puppetlabs/strings/yard/code_objects/puppet_namespace_object.rb

@@ -1,11 +1,24 @@
 class PuppetX::PuppetLabs::Strings::YARD::CodeObjects::PuppetNamespaceObject < YARD::CodeObjects::NamespaceObject
+
+  attr_accessor :type_info
   # NOTE: `YARD::Registry#resolve` requires a method with this signature to
   # be present on all subclasses of `NamespaceObject`.
   def inheritance_tree(include_mods = false)
     [self]
   end
 
-  attr_accessor :type_info
+  def to_s
+    name.to_s
+  end
+
+  def to_json(*a)
+    {
+      "name"             => @name,
+      "file"             => file,
+      "line"             => line,
+      "docstring"        => @docstring,
+    }.to_json(*a)
+  end
 
   # FIXME: We used to override `self.new` to ensure no YARD proxies were
   # created for namespaces segments that did not map to a host class or

lib/puppet_x/puppetlabs/strings/yard/code_objects/type_object.rb

@@ -2,4 +2,38 @@ class PuppetX::PuppetLabs::Strings::YARD::CodeObjects::TypeObject < PuppetX::Pup
   # A list of parameters attached to this class.
   # @return [Array<Array(String, String)>]
   attr_accessor :parameters
+
+  def to_json(*a)
+    {
+      "name"             => @name,
+      "file"             => file,
+      "line"             => line,
+      "docstring"        => Puppet::Util::Docs.scrub(@docstring),
+      "parameters"       => @parameter_details.map do |obj|
+        {
+          "allowed_values" => obj[:allowed_values] ? obj[:allowed_values].flatten : [],
+          "default"        => obj[:default],
+          "docstring"      => Puppet::Util::Docs.scrub(obj[:desc] || ''),
+          "namevar"        => obj[:namevar],
+          "name"           => obj[:name],
+        }
+      end,
+      "properties"         => @property_details.map do |obj|
+        {
+          "allowed_values" => obj[:allowed_values] ? obj[:allowed_values].flatten : [],
+          "default"        => obj[:default],
+          "docstring"      => Puppet::Util::Docs.scrub(obj[:desc] || ''),
+          "name"           => obj[:name],
+        }
+      end,
+      "features"         => @features.map do |obj|
+        {
+          "docstring" => Puppet::Util::Docs.scrub(obj[:desc] || ''),
+          "methods"   => obj[:methods],
+          "name"      => obj[:name],
+        }
+      end,
+    }.to_json(*a)
+  end
+
 end

lib/puppet_x/puppetlabs/strings/yard/handlers/puppet_3x_function_handler.rb

@@ -10,6 +10,7 @@ class PuppetX::PuppetLabs::Strings::YARD::Handlers::Puppet3xFunctionHandler < YA
     name, options = @heredoc_helper.process_parameters statement
 
     obj = MethodObject.new(function_namespace, name)
+    obj[:puppet_3x_function] = true
 
     register obj
     if options['doc']

lib/puppet_x/puppetlabs/strings/yard/handlers/type_handler.rb

@@ -78,60 +78,68 @@ class PuppetX::PuppetLabs::Strings::YARD::Handlers::PuppetTypeHandler < YARD::Ha
     parameter_details = []
     property_details = []
     features = []
-    obj = TypeObject.new(:root, "#{name}_type") do |o|
+    obj = TypeObject.new(:root, name)
-      # FIXME: This block gets yielded twice for whatever reason
+    obj.parameters = []
-      parameter_details = []
-      property_details = []
-      o.parameters = []
-      # Find the do block following the Type.
-      do_block = statement.jump(:do_block)
-      # traverse the do block's children searching for function calls whose
-      # identifier is newparam (we're calling the newparam function)
-      do_block.traverse do |node|
-        if is_param? node
-          # The first member of the parameter tuple is the parameter name.
-          # Find the second identifier node under the fcall tree. The first one
-          # is 'newparam', the second one is the function name.
-          # Get its source.
-          # The second parameter is nil because we cannot infer types for these
-          # functions. In fact, that's a silly thing to ask because ruby
-          # types were deprecated with puppet 4 at the same time the type
-          # system was created.
 
-          # Because of a ripper bug a symbol identifier is sometimes incorrectly parsed as a keyword.
+    # Find the do block following the Type.
-          # That is, the symbol `:true` will be represented as s(:symbol s(:kw, true...
+    do_block = statement.jump(:do_block)
-          param_name = node.children[1].jump(:ident)
+    # traverse the do block's children searching for function calls whose
-          if param_name == node.children[1]
+    # identifier is newparam (we're calling the newparam function)
-            param_name = node.children[1].jump(:kw)
+    do_block.traverse do |node|
-          end
+      if is_param? node
-          param_name = param_name.source
+        # The first member of the parameter tuple is the parameter name.
-          o.parameters << [param_name, nil]
+        # Find the second identifier node under the fcall tree. The first one
-          parameter_details << {:name => param_name,
+        # is 'newparam', the second one is the function name.
-            :desc => fetch_description(node), :exists? => true,
+        # Get its source.
-            :puppet_type => true,
+        # The second parameter is nil because we cannot infer types for these
-            :default => fetch_default(node),
+        # functions. In fact, that's a silly thing to ask because ruby
-            :namevar => is_namevar?(node, param_name, name),
+        # types were deprecated with puppet 4 at the same time the type
-            :parameter => true,
+        # system was created.
-            :allowed_values => get_parameter_allowed_values(node),
+
-          }
+        # Because of a ripper bug a symbol identifier is sometimes incorrectly parsed as a keyword.
-        elsif is_prop? node
+        # That is, the symbol `:true` will be represented as s(:symbol s(:kw, true...
-          # Because of a ripper bug a symbol identifier is sometimes incorrectly parsed as a keyword.
+        param_name = node.children[1].jump(:ident)
-          # That is, the symbol `:true` will be represented as s(:symbol s(:kw, true...
+        if param_name == node.children[1]
-          prop_name = node.children[1].jump(:ident)
+          param_name = node.children[1].jump(:kw)
-          if prop_name == node.children[1]
-            prop_name = node.children[1].jump(:kw)
-          end
-          prop_name = prop_name.source
-          property_details << {:name => prop_name,
-            :desc => fetch_description(node), :exists? => true,
-            :default => fetch_default(node),
-            :puppet_type => true,
-            :property => true,
-            :allowed_values => get_property_allowed_values(node),
-            }
-        elsif is_feature? node
-          features << get_feature(node)
         end
+        param_name = param_name.source
+        obj.parameters << [param_name, nil]
+        parameter_details << {:name => param_name,
+          :desc => fetch_description(node), :exists? => true,
+          :puppet_type => true,
+          :default => fetch_default(node),
+          :namevar => is_namevar?(node, param_name, name),
+          :parameter => true,
+          :allowed_values => get_parameter_allowed_values(node),
+        }
+      elsif is_prop? node
+        # Because of a ripper bug a symbol identifier is sometimes incorrectly parsed as a keyword.
+        # That is, the symbol `:true` will be represented as s(:symbol s(:kw, true...
+        prop_name = node.children[1].jump(:ident)
+        if prop_name == node.children[1]
+          prop_name = node.children[1].jump(:kw)
+        end
+        prop_name = prop_name.source
+        property_details << {:name => prop_name,
+          :desc => fetch_description(node), :exists? => true,
+          :default => fetch_default(node),
+          :puppet_type => true,
+          :property => true,
+          :allowed_values => get_property_allowed_values(node),
+        }
+      elsif is_feature? node
+        features << get_feature(node)
+      elsif is_a_func_call_named? 'ensurable', node
+        # Someone could call the ensurable method and create an ensure 
+        # property. If that happens, they it will be documented twice. Serves
+        # them right.
+        property_details << {:name => 'ensure',
+          :desc => '', :exists? => true,
+          :default => nil,
+          :puppet_type => true,
+          :property => true,
+          :allowed_values => [],
+        }
       end
     end
     obj.parameter_details = parameter_details

lib/puppet_x/puppetlabs/strings/yard/json_registry_store.rb

@@ -4,45 +4,71 @@ module YARD
     def save(merge=true, file=nil)
       super
 
-      # FIXME: do we need this?
-      if file && file != @file
-        @file = file
-        @serializer = Serializers::JsonSerializer.new(@file)
-      end
       @serializer = Serializers::JsonSerializer.new(@file)
 
       sdb = Registry.single_object_db
-      original_extension = @serializer.extension
-      @serializer.extension = 'json'
-      @serializer.basepath = 'yardoc_json'
-      interesting_entries = proc { |key, val|
-        [:puppetnamespace, :hostclass,].include? val.type or
-        (val.type == :method and (val['puppet_4x_function'] or
-        val['puppet_3x_function']))
-      }
-      rename_methods = proc { |key, value|
-        [value.type == :method ? value.name.to_sym : key,
-        value]
-      }
       if sdb == true || sdb == nil
-        @serializer.serialize(Hash[@store.select(&interesting_entries).map(&rename_methods)].to_json)
+        serialize_output_schema(@store)
       else
         values(false).each do |object|
-          @serializer.serialize(Hash[object.select(&interesting_entries).map(&rename_methods)].to_json)
+          serialize_output_schema(object)
         end
       end
-      @serializer.extension = original_extension
       true
     end
+
+    # @param obj [Hash] A hash representing the registry or part of the
+    # registry.
+    def serialize_output_schema(obj)
+        schema = {
+          :puppet_functions => [],
+          :puppet_providers => [],
+          :puppet_classes => [],
+          :defined_types => [],
+          :puppet_types => [],
+        }
+
+        schema[:puppet_functions] += obj.select do |key, val|
+          val.type == :method and (val['puppet_4x_function'] or
+                                   val['puppet_3x_function'])
+        end.values
+
+        schema[:puppet_classes] += obj.select do |key, val|
+          val.type == :hostclass
+        end.values
+
+        schema[:defined_types] += obj.select do |key, val|
+          val.type == :definedtype
+        end.values
+
+        schema[:puppet_providers] += obj.select do |key, val|
+          val.type == :provider
+        end.values
+
+        schema[:puppet_types] += obj.select do |key, val|
+          val.type == :type
+        end.values
+
+        @serializer.serialize(schema.to_json)
+    end
   end
 
-  # Override the serializer because it puts the data at a whacky path and, more
+  # Override the serializer because it puts the data at a wacky path and, more
-  # importantly, mashals the data with a bunch of non-printable characters.
+  # importantly, marshals the data with a bunch of non-printable characters.
   module Serializers
     class JsonSerializer < YardocSerializer
+
+      def initialize o
+        super
+        @options = {
+          :basepath => 'doc',
+          :extension => 'json',
+        }
+        @extension = 'json'
+        @basepath = 'doc'
+      end
       def serialize(data)
         path = File.join(basepath, "registry_dump.#{extension}")
-        require 'pry'; binding.pry
         log.debug "Serializing json to #{path}"
         File.open!(path, "wb") {|f| f.write data }
       end

spec/unit/puppet_x/puppetlabs/strings/yard/type_handler_spec.rb

@@ -7,7 +7,7 @@ describe PuppetX::PuppetLabs::Strings::YARD::Handlers::PuppetTypeHandler do
   include StringsSpec::Parsing
 
   def the_type()
-    YARD::Registry.at("file_type")
+    YARD::Registry.at("file")
   end
 
   it "should have the proper docstring" do