lost and found ( for me ? )

lost and found ( for me ? )

↑ Grab this Headline Animator

Python : calculate HTTP content size

Here is how to calculate HTTP content size.
This script will fetch all images in a html file and calculate content size from HTTP content-length header.
This script uses beautifulsoup module, so install the module if you have not installed.
# pip install beautifulsoup4
Here is the script.
# cat HTTP_calc_content_size.py
#!/usr/bin/env python

from bs4 import BeautifulSoup
import urllib2
import re

i = []
j = []

# connect to the site
url = "http://jp.msn.com"
#url = "http://www.apple.com"
#url = "http://www.amazon.co.jp"
#url = "http://www.yahoo.co.jp"
#url = "http://www.google.co.jp"

content = urllib2.urlopen(url).read()
soup = BeautifulSoup(content)

# find img tags
elements = soup.findAll('img')

for element in elements:
   if re.match('^http', element['src']):
#        print element['src']
       j.append(element['src'])
   else:
#        print "%s%s" % (url, element['src'])
       j.append("%s%s" % (url, element['src']))

# fetch all images and retrieve content-length header
for tmp in j:
   conn = urllib2.urlopen(tmp, timeout=3)
   i.append(int(conn.headers['content-length']))

# calculate average content size
average = (sum(i)/(len(i)*1.0))

# print results
print "site name : %s" % url
print "number of objects : %s" % len(i)
print "average : %d bytes" % average
print "max : %d bytes" % max(i)
print "minimum : %d bytes" % min(i)
Here is outputs.
- msn
# ./HTTP_calc_content_size.py
site name : http://jp.msn.com
number of objects : 51
average : 2540 bytes
max : 9270 bytes
minimum : 42 bytes
- amazon
# ./HTTP_calc_content_size.py
site name : http://www.amazon.co.jp
number of objects : 52
average : 8457 bytes
max : 77708 bytes
minimum : 43 bytes
- google
# ./HTTP_calc_content_size.py
site name : http://www.google.co.jp
number of objects : 1
average : 1834 bytes
max : 1834 bytes
minimum : 1834 bytes

No comments:
Labels: , , ,

make presentation materials with html5 ( S5 template )

Reference

Here is how to make presentation materials with S5 template.
very easy!

# unzip s5-blank.zip

# ls s5-blank
s5-blank.html  ui

backup the original file and edit my-s5-blank.html
# cd s5-blank/

# cp s5-blank.html my-s5-blank.html

[ title ]

     4 <html xmlns="http://www.w3.org/1999/xhtml">
     5
     6 <head>
     7 <title>Test title hello</title>
     8 <!-- metadata -->
     9 <meta name="generator" content="S5" />
    10 <meta name="version" content="S5 1.1" />


    30 <div id="header"></div>
    31 <div id="footer">
    32 <h1>Date 2014/1/1</h1>
    33 <h2>main title</h2>
    34 </div>


   39 <div class="presentation">
    40
    41 <div class="slide">
    42 <h1>main title</h1>
    43 <h2>sub title</h2>
    44 <h3>my name</h3>
    45 </div>

[ slide ]

add three slides

   39 <div class="presentation">
    40
    41 <div class="slide">
    42 <h1>main title</h1>
    43 <h2>sub title</h2>
    44 <h3>my name</h3>
    45 </div>
    46
    47
    48 <div class="slide">
    49 <h1>hello world slide1</h1>
    50 <ul>
    51 <li>point 1</li>
    52 <li>point 2</li>
    53 <li>point 3</li>
    54 </ul>
    55 <div class="handout">
    56 [any material that should appear in print but not on the slide]
    57 </div>
    58 </div>
    59
    60
    61 <div class="slide">
    62 <h1>hello world slide2</h1>
    63 <ul>
    64 <li>point 1</li>
    65 <li>point 2</li>
    66 <li>point 3</li>
    67 </ul>
    68 <div class="handout">
    69 [any material that should appear in print but not on the slide]
    70 </div>
    71 </div>
    72
    73
    74 <div class="slide">
    75 <h1>hello world slide3</h1>
    76 <ul>
    77 <li>point 1</li>
    78 <li>point 2</li>
    79 <li>point 3</li>
    80 </ul>
    81 <div class="handout">
    82 [any material that should appear in print but not on the slide]
    83 </div>
    84 </div>
    85
    86
    87
    88 </div>
    89
    90 </body>

after editing the file, open this file with browser

# firefox my-s5-blank.html

great!


change background

edit pretty.css file under s5-blank/ui/default.
I changed gif to blank.gif
# grep gif pretty.css
body {background: #FFF url(blank.gif) -16px 0 no-repeat; color: #000; font-size: 2em;}
div#header {background: #005 url(blank.gif) -16px 0 no-repeat;

No comments:
Labels: , ,

Python : how to use virtualenv

Reference
Small tips.
With virtualenv, you can create python virtual environment separately for each projects.
install virtualenv
root@ubuntu:~# apt-cache install python-pip

root@ubuntu:~# pip install virtualenv

# virtualenv --version
1.11.6
create python virtual environment under dest_dir1 directory.
root@ubuntu:/home/hattori/Python_works# virtualenv dest_dir1
New python executable in dest_dir1/bin/python
Installing setuptools, pip...done.
python environment has been created under dest_dir1.
root@ubuntu:/home/hattori/Python_works# ls dest_dir1/
bin  include  lib  local

root@ubuntu:/home/hattori/Python_works# ls dest_dir1/bin/
activate      activate.fish     easy_install      pip   pip2.7  python2
activate.csh  activate_this.py  easy_install-2.7  pip2  python  python2.7
use this env and install requests module within that environment
python env has changed from “/usr/bin/python” to “dest_dir1/bin/python”
root@ubuntu:/home/hattori/Python_works# which python
/usr/bin/python


root@ubuntu:/home/hattori/Python_works# source dest_dir1/bin/activate
(dest_dir1)root@ubuntu:/home/hattori/Python_works#

(dest_dir1)root@ubuntu:/home/hattori/Python_works# which python
/home/hattori/Python_works/dest_dir1/bin/python

(dest_dir1)root@ubuntu:/home/hattori/Python_works# which pip
/home/hattori/Python_works/dest_dir1/bin/pip

(dest_dir1)root@ubuntu:/home/hattori/Python_works# pip install requests
“activate” file is a bash script, so you can check that to know the script executes what kind of things.
# unset irrelevant variables
deactivate nondestructive

VIRTUAL_ENV="/home/hattori/Python_works/dest_dir1"
export VIRTUAL_ENV

_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
exit this env
(dest_dir1)root@ubuntu:/home/hattori/Python_works# deactivate
root@ubuntu:/home/hattori/Python_works#
root@ubuntu:/home/hattori/Python_works# which python
/usr/bin/python
You can use different python version.
root@ubuntu:/home/hattori/Python_works# virtualenv -p /usr/bin/python3.4 dest_dir2
Running virtualenv with interpreter /usr/bin/python3.4
Using base prefix '/usr'
New python executable in dest_dir2/bin/python3.4
Also creating executable in dest_dir2/bin/python
Installing setuptools, pip...done.


(dest_dir2)root@ubuntu:/home/hattori/Python_works# which python
/home/hattori/Python_works/dest_dir2/bin/python

(dest_dir2)root@ubuntu:/home/hattori/Python_works# python --version
Python 3.4.0

(dest_dir2)root@ubuntu:/home/hattori/Python_works# deactivate
root@ubuntu:/home/hattori/Python_works#

No comments:
Labels: ,

FreeBSD 9.3 : make bootable USB stick for FreeBSD 9.3 on Ubuntu 14.04

Rederence
https://www.freebsd.org/releases/9.3R/announce.html
download images for memory stick
# unxz /home/hattori/FreeBSD-9.3-RELEASE-amd64-memstick.img.xz

prepare a USB stick
# dd if=FreeBSD-9.3-RELEASE-amd64-memstick.img of=/dev/sdg1 bs=10240 conv=sync
confirm that you can boot FreeBSD from the memory stick.
I have tested this under KVM.
# qemu-system-x86_64 -enable-kvm -m 256 -usb /dev/sdg1

No comments:
Labels: ,

Install puppet master and agent on Ubuntu 14.04

Here are trial and error logs when installing puppet master and agent.
Reference
http://terokarvinen.com/2012/puppetmaster-on-ubuntu-12-04
Prepare two Ubuntu 14.04 boxes, one is for puppet master and the other is for agent.
# tail -1 /etc/lsb-release ;uname -ri
DISTRIB_DESCRIPTION="Ubuntu 14.04.1 LTS"
3.13.0-34-generic x86_64
Before installing puppet master and agent, edit /etc/hosts so that each machine can communicate with the other with hostname.
on the master
root@puppet-master:/home/hattori# domainname
(none)
root@puppet-master:/home/hattori# cat /etc/hosts
127.0.0.1       localhost
127.0.1.1       puppet-master

# The following lines are desirable for IPv6 capable hosts
::1     localhost ip6-localhost ip6-loopback
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters

192.168.122.56 puppet-master
192.168.122.197 puppet-agent01

root@puppet-master:/home/hattori# ping -c 1 puppet-agent01
PING puppet-agent01 (192.168.122.197) 56(84) bytes of data.
64 bytes from puppet-agent01 (192.168.122.197): icmp_seq=1 ttl=64 time=0.775 ms

--- puppet-agent01 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.775/0.775/0.775/0.000 ms
on the agent
root@puppet-agent01:/home/hattori# hostname
puppet-agent01
root@puppet-agent01:/home/hattori# domainname
(none)
root@puppet-agent01:/home/hattori# cat /etc/hosts
127.0.0.1       localhost
127.0.1.1       puppet-agent01

# The following lines are desirable for IPv6 capable hosts
::1     localhost ip6-localhost ip6-loopback
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters

192.168.122.56 puppet-master
192.168.122.197 puppet-agent01

root@puppet-agent01:/home/hattori# ping -c 1 puppet-master
PING puppet-master (192.168.122.56) 56(84) bytes of data.
64 bytes from puppet-master (192.168.122.56): icmp_seq=1 ttl=64 time=0.330 ms

--- puppet-master ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.330/0.330/0.330/0.000 ms
Install puppet master
On the master box
root@puppet-master:/home/hattori# apt-get install puppetmaster –y

root@puppet-master:/home/hattori# service puppetmaster status
* master is running
start puppetmaster when booting the OS
root@puppet-master:/home/hattori# update-rc.d puppetmaster enable
Install puppet agent
On the agent box
root@puppet-agent01:/home/hattori# apt-get install puppet –y

root@puppet-agent01:/home/hattori# service puppet status
* agent is running
Start puppet agent when booting the OS
root@puppet-agent01:/home/hattori# update-rc.d puppet enable
[ configure puppet master ]
create a certificate file
root@puppet-master:/home/hattori# service puppetmaster stop
root@puppet-master:/home/hattori# rm -fr /var/lib/puppet/ssl/
edit /etc/puppet/puppet.conf and add puppetmaster hostname in dns_alt_names statement.
root@puppet-master:~# cat /etc/puppet/puppet.conf
[main]
logdir=/var/log/puppet
vardir=/var/lib/puppet
ssldir=/var/lib/puppet/ssl
rundir=/var/run/puppet
factpath=$vardir/lib/facter
templatedir=$confdir/templates
prerun_command=/etc/puppet/etckeeper-commit-pre
postrun_command=/etc/puppet/etckeeper-commit-post

[master]
# These are needed when the puppetmaster is run by passenger
# and can safely be removed if webrick is used.
ssl_client_header = SSL_CLIENT_S_DN
ssl_client_verify_header = SSL_CLIENT_VERIFY
dns_alt_names = puppet-master
root@puppet-master:~#  service puppetmaster start
confirm that there is the string of master’s hostname (puppet-master) in the SSL cert file.
puppet agent check this.
root@puppet-master:~# openssl x509 -in /var/lib/puppet/ssl/certs/puppet-master.pem  -text | grep -i dns
               DNS:puppet-master
root@puppet-master:~#
[ agent ]
on the agent
edit /etc/puppet/puppet.conf
root@puppet-agent01:~# grep -v ^# /etc/puppet/puppet.conf
[main]
logdir=/var/log/puppet
vardir=/var/lib/puppet
ssldir=/var/lib/puppet/ssl
rundir=/var/run/puppet
factpath=$vardir/lib/facter
templatedir=$confdir/templates
prerun_command=/etc/puppet/etckeeper-commit-pre
postrun_command=/etc/puppet/etckeeper-commit-post

[master]
ssl_client_header = SSL_CLIENT_S_DN
ssl_client_verify_header = SSL_CLIENT_VERIFY

[agent]
server = puppet-master
restart puppet
root@puppet-agent01:~# service puppet restart
* Restarting puppet agent
  ...done.
on the master
root@puppet-master:~# puppet cert --list
root@puppet-master:~#
nnn, can’t see the agent on the master.
on the agent. delete SSL files and restart puppet.
root@puppet-agent01:~# service puppet stop
* Stopping puppet agent
  ...done.
root@puppet-agent01:~# rm -r /var/lib/puppet/ssl/

root@puppet-agent01:~# service puppet start
* Starting puppet agent
  ...done.
on the master.
Okay, I was able to see the agent on the master
root@puppet-master:~# puppet cert –list
 "puppet-agent01" (SHA256) xx:xx:xx
sign the agent certificate on the master.
root@puppet-master:~# puppet cert --sign puppet-agent01
Notice: Signed certificate request for puppet-agent01
Notice: Removing file Puppet::SSL::CertificateRequest puppet-agent01 at '/var/lib/puppet/ssl/ca/requests/puppet-agent01.pem'
[ create site manifests and modules ]
create the manifest
on the master
root@puppet-master:~# cd /etc/puppet/
root@puppet-master:/etc/puppet# mkdir -p manifests/ modules/helloworld/manifests
root@puppet-master:/etc/puppet# echo 'include helloworld' > manifests/site.pp
root@puppet-master:/etc/puppet# cat manifests/site.pp
include helloworld
create the module
root@puppet-master:/etc/puppet# vi modules/helloworld/manifests/init.pp

root@puppet-master:/etc/puppet# cat modules/helloworld/manifests/init.pp
class helloworld {
       file { '/tmp/helloFromMaster':
               content => "Hello from Puppet Master\n"
       }
}
on the agent, restart puppet agent.
root@puppet-agent01:~# service puppet restart
* Restarting puppet agent
  ...done.
on the agent, check whether or not /tmp/helloFromMaster file exits.
nnn, no file exist..
root@puppet-agent01:~# ls /tmp/
root@puppet-agent01:~#
let’s run puppet as verbose mode to find the problem.
root@puppet-agent01:~# service puppet stop
* Stopping puppet agent
  ...done.
root@puppet-agent01:~# puppet agent --verbose --no-daemonize --onetime
Info: Retrieving plugin
Error: Could not retrieve catalog from remote server: Error 400 on SERVER: Could not parse for environment production: Syntax error at 'file' at /etc/puppet/manifests/site.pp:3 on node puppet-agent01
Notice: Using cached catalog
Error: Could not retrieve catalog; skipping run
Seen from the errors, something is wrong with the manifests and the modules
let’s re-create manifests by following instructions as below.
on the master
root@puppet-master:~# cat /etc/puppet/manifests/site.pp
node "puppet-agent01" {

 file { "/root/helloworld.txt":
   ensure => file,
   owner  => "root",
   group  => "root",
   mode   => 0644
 }
}
on the agent, run puppet as verbose mode.
Okay, the agent created the file.
root@puppet-agent01:~# service puppet stop

root@puppet-agent01:~# puppet agent --verbose --no-daemonize --onetime
Info: Retrieving plugin
Info: Caching catalog for puppet-agent01
Info: Applying configuration version '1408594607'
Notice: /Stage[main]/Main/Node[puppet-agent01]/File[/root/helloworld.txt]/ensure: created
Info: Creating state file /var/lib/puppet/state/state.yaml
Notice: Finished catalog run in 0.04 seconds

root@puppet-agent01:~# ls /root/helloworld.txt
/root/helloworld.txt
change permissions of this file on the agent
root@puppet-agent01:~# chmod 0444 /root/helloworld.txt
run puppet
permission has been changed based on manifests
root@puppet-agent01:~# puppet agent --verbose --no-daemonize --onetime
Info: Retrieving plugin
Info: Caching catalog for puppet-agent01
Info: Applying configuration version '1408594607'
Notice: /Stage[main]/Main/Node[puppet-agent01]/File[/root/helloworld.txt]/mode: mode changed '0444' to '0644'
Notice: Finished catalog run in 0.05 seconds

No comments:
Labels: ,
Subscribe to: Comments (Atom)