Tags

redhat
employment
ripple
interfaces
ncurses
ruby
refs
filesystems
retro gaming
raspberry pi
sinatra
3d printing
nethack
gcc
compiler
fedora
virtfs
project
gaming
vim
grep
sed
aikido
philosophy
splix
android
lvm
storage
bitcoin
projects
sig315
miq
db
polisher
meditation
hopex
conferences
omega
simulator
bundler_ext
rubygems
book review
google code in
isitfedoraruby
svn
gsoc
design patters
jsonrpc
rjr
aeolus
ohiolinuxfest
rome
europe
travel
brno
gtk
python
puppet
conference
fudcon
snap
html5
tips
ssh
linux
hardware
libvirt
virtualization
engineering expo
cloud
rpm
yum
rake
redmine
plugins
screencasting
jruby
fosscon
pidgin
gnome-shell
distros
notacon
presentation
rails
deltacloud
apache
qmf
passenger
syrlug
hackerspace
music
massive attack
crypto
backups
vnc
xsd
rxsd
x3d
mercurial
ovirt
qpid
webdev
haikus
poetry
legaleese
jquery
selenium
testing
xpath
git
sshfs
svg
ldap
autotools
pygtk
xmlrpc
slackware

Feb 29 2012 puppet

Updated Puppet Web Resource Module

Some of you may recall the Puppet Web Resource type and provider which I wrote for Aeolus a while back. The module has since undergone alot of improvment from various contributers, and was finally accepted and uploaded to the Puppet module forge. You may download it from there and simply drop it into your puppet module load path to access the resource in all its glory!

Some additional features since I last shared this include

For reference here is the updated web resource type and provider:

lib/puppet/provider/type/web_request.rb
require 'uri'

# A puppet resource type used to access resources on the World Wide Web
Puppet::Type.newtype(:web_request) do
    @doc = "Issue a request to a resource on the world wide web"

    private

    # Validates uris passed in
    def self.validate_uri(url)
      begin
        uri = URI.parse(url)
        raise ArgumentError, "Specified uri #{url} is not valid" if ![URI::HTTP, URI::HTTPS].include?(uri.class)
      rescue URI::InvalidURIError
        raise ArgumentError, "Specified uri #{url} is not valid"
      end
    end

    # Validates http statuses passed in
    def self.validate_http_status(status)
      status = [status] unless status.is_a?(Array)
      status.each { |stat|
        stat = stat.to_s
        unless ['100', '101', '102', '122',
                '200', '201', '202', '203', '204', '205', '206', '207', '226',
                '300', '301', '302', '303', '304', '305', '306', '307',
                '400', '401', '402', '403', '404', '405', '406', '407', '408', '409',
                '410', '411', '412', '413', '414', '415', '416', '417', '418',
                '422', '423', '424', '425', '426', '444', '449', '450', '499',
                '500', '501', '502', '503', '504', '505', '506', '507', '508', ' 509', '510'
                ].include?(stat)
          raise ArgumentError, "Invalid http status code #{stat} specified"
        end
      }
    end

    # Convert singular params into arrays of strings
    def self.munge_array_params(value)
      value = [value] unless value.is_a?(Array)
      value = value.collect { |val| val.to_s }
      value
    end

    newparam :name

    newproperty(:get) do
      desc "Issue get request to the specified uri"
      validate do |value| Puppet::Type::Web_request.validate_uri(value) end
    end

    newproperty(:post) do
      desc "Issue post request to the specified uri"
      validate do |value| Puppet::Type::Web_request.validate_uri(value) end
    end

    newproperty(:delete) do
      desc "Issue delete request to the specified uri"
      validate do |value| Puppet::Type::Web_request.validate_uri(value) end
    end

    newproperty(:put) do
      desc "Issue put request to the specified uri"
      validate do |value| Puppet::Type::Web_request.validate_uri(value) end
    end

    newparam(:parameters) do
      desc "Hash of parameters to include in the web request"
    end

    newparam(:file_parameters) do
      desc "Hash of file parameters to include in the web request"
    end

    newparam(:follow) do
      desc "Boolean indicating if redirects should be followed"
      newvalues(:true, :false)
    end

    newparam(:store_cookies_at) do
      desc "String indicating where session cookies should be stored"
    end

    newparam(:use_cookies_at) do
      desc "String indicating where session cookies should be read from"
    end

    newparam(:remove_cookies) do
      desc "Boolean indicating if cookies should be removed after using them"
      newvalues(:true, :false)
    end

    newparam(:returns) do
      desc "Expected http return codes of the request"
      defaultto ["200"]
      validate do |value| Puppet::Type::Web_request.validate_http_status(value) end
      munge    do |value| Puppet::Type::Web_request.munge_array_params(value)   end
    end

    newparam(:does_not_return) do
      desc "Unexecpected http return codes of the request"
      validate do |value| Puppet::Type::Web_request.validate_http_status(value) end
      munge    do |value| Puppet::Type::Web_request.munge_array_params(value)   end
    end

    newparam(:contains) do
      desc "XPath to verify as part of the result"
      munge    do |value| Puppet::Type::Web_request.munge_array_params(value)   end
    end

    newparam(:does_not_contain) do
      desc "XPath to verify as not being part of the result"
      munge    do |value| Puppet::Type::Web_request.munge_array_params(value)   end
    end

    newparam(:log_to) do
      desc "Log requests/responses to the specified file or directory"
    end

    newparam(:only_log_errors) do
      desc "Boolean indicating if we should only log responses which did not pass validations"
      newvalues(:true, :false)
    end

    newparam(:if) do
      desc "Invoke request only if the specified request returns true"
    end

    newparam(:unless) do
      desc "Invoke request unless the specified request returns true"
    end

    newparam(:username) do
      desc "HTTP authentication username"
    end
 
    newparam(:password) do
      desc "HTTP authentication password"
    end

end
lib/puppet/provider/web_request/curl.rb
require 'fileutils'

# Provides an interface to curl using the curb gem for puppet
require 'curb'

# uses nokogiri to verify responses w/ xpath
require 'nokogiri'

class Curl::Easy

  # Format request parameters for the specified request method
  def self.format_params(method, params, file_params)
    if([:get, :delete].include?(method))
      return params.collect { |k,v| "#{k}=#{v}" }.join("&") unless params.nil?
      return ""
    end
    # post, put:
    cparams = []
    params.each_pair      { |k,v| cparams << Curl::PostField.content(k,v) } unless params.nil?
    file_params.each_pair { |k,v| cparams << Curl::PostField.file(k,v)    } unless file_params.nil?
    return cparams
  end

  # Format a url for the specified request method, base uri, and parameters
  def self.format_url(method, uri, params)
    if([:get, :delete].include?(method))
      url = uri
      url +=  ";" + format_params(method, params)
      return url
    end
    # post, put:
    return uri
  end

  # Invoke a new curl request and return result
  def self.web_request(method, uri, params = {})
    raise Puppet::Error, "Must specify http method (#{method}) and uri (#{uri})" if method.nil? || uri.nil?

    curl = self.new

    if params.has_key?(:cookie) && !params[:cookie].nil?
      curl.enable_cookies = true
      curl.cookiefile = params[:cookie]
      curl.cookiejar  = params[:cookie]
    end

    if params.has_key?(:username) && !params[:username].nil?
      curl.username = params[:username]
    end
 
    if params.has_key?(:password) && !params[:password].nil?
      curl.password = params[:password]
    end

    curl.follow_location = (params.has_key?(:follow) && params[:follow])
    request_params = params[:parameters]
    file_params    = params[:file_parameters]

    case(method)
    when 'get'
      curl.url = format_url(method, uri, request_params)
      curl.http_get
      return curl

    when 'post'
      curl.url = format_url(method, uri, request_params)
      curl.multipart_form_post = true if !file_params.nil? && file_params.size > 0
      curl.http_post(*format_params(method, request_params, file_params))
      return curl

    when 'put'
      curl.url = format_url(method, uri, request_params)
      curl.multipart_form_post = true if !file_params.nil? && file_params.size > 0
      curl.http_put(*format_params(method, request_params, file_params))
      return curl

    when 'delete'
      curl.url = format_url(method, uri, request_params)
      curl.http_delete
      return curl
    end
  end

  def valid_status_code?(valid_values=[])
    valid_values.include?(response_code.to_s)
  end

  def valid_xpath?(xpath="/")
    !Nokogiri::HTML(body_str.to_s).xpath(xpath.to_s).empty?
  end

end

# Puppet provider definition
Puppet::Type.type(:web_request).provide :curl do
  desc "Use curl to access web resources"

  def get
    @uri
  end

  def post
    @uri
  end

  def delete
    @uri
  end

  def put
    @uri
  end

  def get=(uri)
    @uri = uri
    process_params('get', @resource, uri)
  end

  def post=(uri)
    @uri = uri
    process_params('post', @resource, uri)
  end

  def delete=(uri)
    @uri = uri
    process_params('delete', @resource, uri)
  end

  def put=(uri)
    @uri = uri
    process_params('put', @resource, uri)
  end

  private

  # Helper to process/parse web parameters
  def process_params(request_method, params, uri)
    begin
      error = nil
      cookies = nil
      if params[:store_cookies_at]
        if File.exist?(params[:store_cookies_at])
          File.truncate(params[:store_cookies_at], 0)
        else
          FileUtils.touch(params[:store_cookies_at])
        end
        cookies = params[:store_cookies_at]
      elsif params[:use_cookies_at]
        cookies = params[:use_cookies_at]
      end

      # verify that we should actually run the request
      return if skip_request?(params, cookies)

      # Actually run the request and verify the result
      result = Curl::Easy::web_request(request_method, uri,
                                       :parameters => params[:parameters],
                                       :file_parameters => params[:file_parameters],
                                       :cookie => cookies,
                                       :follow => params[:follow],
                                       :username => params[:username],
                                       :password => params[:password])

      result_body = result.body_str.to_s

      verify_result(result,
                    :returns          => params[:returns],
                    :does_not_return  => params[:does_not_return],
                    :contains         => params[:contains],
                    :does_not_contain => params[:does_not_contain] )

      result.close

    rescue Exception => e
      error = e
      raise Puppet::Error, "An exception was raised when invoking web request: #{e}"

    ensure
      unless result.nil?
        log_response(:result => result_body,
                     :method => request_method, :uri => uri,
                     :puppet_params => params,  :error => error)
      end
      FileUtils.rm_f(cookies) if params[:remove_cookies]
    end
  end

  # Helper to determine if we should skip the request
  def skip_request?(params, cookie = nil)
    [:if, :unless].each { |c|
      condition = params[c]
      unless condition.nil?
        method = (condition.keys & ['get', 'post', 'delete', 'put']).first
        result = Curl::Easy::web_request(method, condition[method],
                                         :parameters => condition['parameters'],
                                         :file_parameters => condition['file_parameters'],
                                         :cookie => cookie, :follow => params[:follow])
        result_succeeded = true
        begin
          verify_result(result, condition)
        rescue Puppet::Error
          result_succeeded = false
        end
        return true if (c == :if && !result_succeeded) || (c == :unless && result_succeeded)
      end
    }
    return false
  end

  # Helper to verify the response
  def verify_result(result, verify = {})
    verify[:returns]          = verify['returns']          if verify[:returns].nil?          && !verify['returns'].nil?
    verify[:does_not_return]  = verify['does_not_return']  if verify[:does_not_return].nil?  && !verify['does_not_return'].nil?
    verify[:contains]         = verify['contains']         if verify[:contains].nil?         && !verify['contains'].nil?
    verify[:does_not_contain] = verify['does_not_contain'] if verify[:does_not_contain].nil? && !verify['does_not_contain'].nil?

    if !verify[:returns].nil? &&
       !result.valid_status_code?(verify[:returns])
         raise Puppet::Error, "Invalid HTTP Return Code: #{result.response_code},
                               was expecting one of #{verify[:returns].join(", ")}"
    end

    if !verify[:does_not_return].nil? &&
       result.valid_status_code?(verify[:does_not_return])
         raise Puppet::Error, "Invalid HTTP Return Code: #{result.response_code},
                               was not expecting one of #{verify[:does_not_return].join(", ")}"
    end

    if !verify[:contains].nil? &&
       !result.valid_xpath?(verify[:contains])
         raise Puppet::Error, "Expecting #{verify[:contains]} in the result"
    end

    if !verify[:does_not_contain].nil? &&
       result.valid_xpath?(verify[:does_not_contain])
         raise Puppet::Error, "Not expecting #{verify[:does_not_contain]} in the result"
    end
  end

  def log_response(params)
    method  = params[:method]
    uri     = params[:uri]
    result  = params[:result]
    error   = params[:error]
    puppet_params = params[:puppet_params]

    if puppet_params[:log_to]
      return if puppet_params[:only_log_errors] == :true && error.nil?
      logfile = puppet_params[:log_to].strip
      exists = File.exists?(logfile)
      isfile = File.file?(logfile) || (!exists && (logfile[-1].chr != '/'))
      if !isfile
        FileUtils.mkdir_p(logfile) if !exists
	      logfile += puppet_params[:name]
      end

      f = File.open(logfile, 'a')
      f.write("=====BEGIN=====\n")
      f.write(Time.now.strftime("%Y-%m-%d %H:%M:%S"))
      f.write(" #{method} request to #{uri}\n")
      f.write(result.to_s)
      f.write("\n=====END=====\n\n")
      f.close
    end
  end
end
As always, Happy Puppeteering!