Tags

redhat
employment
ripple
interfaces
ncurses
ruby
refs
filesystems
retro gaming
raspberry pi
sinatra
3d printing
nethack
gcc
compiler
fedora
virtfs
project
gaming
vim
grep
sed
aikido
philosophy
splix
android
lvm
storage
bitcoin
projects
sig315
miq
db
polisher
meditation
hopex
conferences
omega
simulator
bundler_ext
rubygems
book review
google code in
isitfedoraruby
svn
gsoc
design patters
jsonrpc
rjr
aeolus
ohiolinuxfest
rome
europe
travel
brno
gtk
python
puppet
conference
fudcon
snap
html5
tips
ssh
linux
hardware
libvirt
virtualization
engineering expo
cloud
rpm
yum
rake
redmine
plugins
screencasting
jruby
fosscon
pidgin
gnome-shell
distros
notacon
presentation
rails
deltacloud
apache
qmf
passenger
syrlug
hackerspace
music
massive attack
crypto
backups
vnc
xsd
rxsd
x3d
mercurial
ovirt
qpid
webdev
haikus
poetry
legaleese
jquery
selenium
testing
xpath
git
sshfs
svg
ldap
autotools
pygtk
xmlrpc
slackware

Sep 5 2011 puppet

A Web Resource Provider for Puppet

As part of the aeolus puppet recipe we are issuing a few web requests via curl to perform various operations against the aeolus APIs. The problem is this is that the results of these requests are somewhat tricky to parse as they require piping the curl output into grep or some other pattern matching mechanism in conjunction with analyzing the returned HTTP status code.

To address this issue, we implemented a puppet resource type for generic web requests and a provider of that type utilizing the curl API to issue requests and analyze the results. To use it to query google for example, simply add the following to your puppet script:

web_request{google:
       get => "http://google.com",
       follow => true
}

Or to invoke the aeolus API, a more complicated use case involving logging in / out, you can include the following in your puppet recipe:

Web_request{
  login       => { 'http_method' => 'post',
                   'uri'         =>  'https://localhost/conductor/user_session',
                   'user_session[login]'    => "$admin_user",
                   'user_session[password]' => "$admin_password",
                   'commit'                 => 'submit' },
  logout      => { 'http_method' => 'post',
                   'uri'         =>  'https://localhost/conductor/logout' }
}

web_request{"provider-foobar":
  post =>  "https://localhost/conductor/admin/providers",
  parameters  => { 'provider[name]'  => 'foobar',
                   'provider[url]'   => 'http://localhost:3003/api',
                   'provider[provider_type_codename]' => 'ec2' },
  returns     => '200',
  verify      => '.*Provider added.*',
  follow      => true,
  unless      => { 'http_method'     => 'get',
                   'uri'             => 'https://localhost/conductor/admin/providers',
                   'verify'          => '.*foobar.*' }
}

web_request{ "hwp1"
  post        => "https://localhost/conductor/admin/hardware_profiles",
  parameters  => {'hardware_profile[name]'  => "hwp1",
                  'hardware_profile[memory_attributes][value]'       => "1024",
                  'hardware_profile[cpu_attributes][value]'          => "1",
                  'hardware_profile[storage_attributes][value]'      => "1",
                  'hardware_profile[architecture_attributes][value]' => "x86_64",
                  'hardware_profile[memory_attributes][name]'        => 'memory',
                  'hardware_profile[memory_attributes][unit]'        => 'MB',
                  'hardware_profile[cpu_attributes][name]'           => 'cpu',
                  'hardware_profile[cpu_attributes][unit]'           => 'count',
                  'hardware_profile[storage_attributes][name]'       => 'storage',
                  'hardware_profile[storage_attributes][unit]'       => 'GB',
                  'hardware_profile[architecture_attributes][name]'  => 'architecture',
                  'hardware_profile[architecture_attributes][unit]'  => 'label',
                  'commit' => 'Save'},
  returns     => '200',
  verify      => '.*Hardware profile added.*',
  follow      => true,
  unless      => { 'http_method'     => 'get',
                   'uri'             => 'https://localhost/conductor/admin/hardware_profiles',
                   'verify'          => ".*hwp1.*" },
  require    => Service['aeolus-conductor']
}
The actual resource type is fairly straightforward, it defines an API which web resources can be accessed in puppet. To use this resource type, place it in lib/puppet/type/web_request.rb under the module which you want to install it in (or in the system wide modules dir)
Puppet::Type.newtype(:web_request) do
    @doc = "Issue a request via the world wide web"

    newparam :name

    newproperty(:get) do
      desc "Issue get request to the specified uri"
    end 

    newproperty(:post) do
      desc "Issue get request to the specified uri"
    end 

    #newproperty(:delete)
    #newproperty(:put)

    newparam(:parameters) do
      desc "Hash of parameters to include in the web request"
    end 

    newparam(:returns) do
      desc "Expected http return codes of the request"
      defaultto "200"
    end 

    newparam(:follow) do
      desc "Boolean indicating if redirects should be followed"
      newvalues(:true, :false)
    end 

    newparam(:verify) do
      desc "String to verify as being part of the result"
    end 

    newparam(:login) do
      desc "Login parameters to be used if a login is required before making the request"
    end 

    newparam(:logout) do
      desc "Logout parameters to be used if a logout is requred after making the request"
    end 

    newparam(:unless) do
      desc "Do not run request if the request specified here succeeds"
    end 
end
And finally here is the resource type provider which uses curl (provided in the rubygem 'curb') to implement the web resource interface. Place this under lib/puppet/provider/web_request/curl.rb in your module directory to use
require 'curb'
require 'uuid'
require 'fileutils'

# Helper to invoke the web request w/ curl
def web_request(method, uri, request_params, params = {})
  raise Puppet::Error, "Must specify http method and uri" if method.nil? || uri.nil?

  curl = Curl::Easy.new

  if params.has_key?(:cookie)
    curl.enable_cookies = true
    curl.cookiefile = params[:cookie]
    curl.cookiejar  = params[:cookie]
  end

  curl.follow_location = (params.has_key?(:follow) && params[:follow])

  case(method)
  when 'get'
    url = uri
    url += ";" + request_params.collect { |k,v| "#{k}=#{v}" }.join("&") unless request_params.nil?
    curl.url = url
    curl.http_get
    return curl

  when 'post'
    cparams = []
    request_params.each_pair { |k,v| cparams << Curl::PostField.content(k,v) } unless request_params.nil?
    curl.url = uri
    curl.http_post(cparams)
    return curl

  #when 'put'
  #when 'delete'
  end
end

# Helper to verify the response
def verify_result(result, verify = {})
  returns = (verify.has_key?(:returns) && !verify[:returns].nil?) ? verify[:returns] : "200"
  returns = [returns] unless returns.is_a? Array
  unless returns.include?(result.response_code.to_s)
    raise Puppet::Error, "Invalid HTTP Return Code: #{result.response_code}, 
                          was expecting one of #{returns.join(", ")}"
  end

  if verify.has_key?(:body) && !verify[:body].nil? && !(result.body_str =~ Regexp.new(verify[:body]))
    raise Puppet::Error, "Expecting #{verify[:body]} in the result"
  end
end

# Helper to process/parse web parameters
def process_params(request_method, params, uri)
  begin
    # Set request method and generate a unique session key
    session = "/tmp/#{UUID.new.generate}"

    # Invoke a login request if necessary
    if params[:login]
      login_params = params[:login].reject { |k,v| ['http_method', 'uri'].include?(k) }
      web_request(params[:login]['http_method'], params[:login]['uri'],
                  login_params, :cookie => session, :follow => params[:follow]).close
    end

    # Check to see if we should actually run the request
    skip_request = !params[:unless].nil?
    if params[:unless]
      result = web_request(params[:unless]['http_method'], params[:unless]['uri'],
                           params[:unless]['parameters'],
                           :cookie => session, :follow => params[:follow])
      begin
        verify_result(result,
                      :returns => params[:unless]['returns'],
                      :body    => params[:unless]['verify'])
      rescue Puppet::Error => e
        skip_request = false
      end
      result.close
    end
    return if skip_request

    # Actually run the request and verify the result
    uri = params[:name] if uri.nil?
    result = web_request(request_method, uri, params[:parameters],
                         :cookie => session, :follow => params[:follow])
    verify_result(result,
                  :returns => params[:returns],
                  :body    => params[:verify])
    result.close

    # Invoke a logout request if necessary
    if params[:logout]
      logout_params = params[:login].reject { |k,v| ['http_method', 'uri'].include?(k) }
      web_request(params[:logout]['http_method'], params[:logout]['uri'],
                  logout_params, :cookie => session, :follow => params[:follow]).close
    end

  rescue Exception => e
    raise Puppet::Error, "An exception was raised when invoking web request: #{e}"

  ensure
    FileUtils.rm_f(session) if params[:logout]
  end
end

# Puppet provider definition
Puppet::Type.type(:web_request).provide :curl do
  desc "Use curl to access web resources"

  def get
    @uri
  end

  def post
    @uri
  end

  def get=(uri)
    @uri = uri
    process_params('get', @resource, uri)
  end

  def post=(uri)
    @uri = uri
    process_params('post', @resource, uri)
  end
end

Happy Puppeteering!