lost and found ( for me ? ): June 2012

Python : convert to localtime from epoch time

small tips.

# python --version
Python 2.7.3

1. convert space-demilited text file’s timestamp from epoch to localtime

# cat space-demilited-ephoc.txt
1342199723 hello bye

python script

# cat -n space-demilited-text-file-convert-to-localtime-from-epoch.py
    1 #!/usr/bin/env python
    2 # -*- coding: UTF-8 -*-
    3
    4 import sys
    5 import time
    6
    7 arg = sys.argv[1]
    8 f = open(arg, 'r')
    9
   10 for line in f:
   11        s = line.split()
   12        a = s[0]
   13        s[0] = time.ctime(float(a))
   14        print " ".join(s)
   15
   16 f.close()

convert to localtime from epoch time.

# ./space-demilited-text-file-convert-to-localtime-from-epoch.py space-demilited-ephoc.txt
Sat Jul 14 02:15:23 2012 hello bye

2. convert comma-demilited text file’s timestamp from epoch to localtime

# cat comma-demilited-ephoc.txt
1342199723,hello,bye

python script

# cat -n comma-demilited-text-file-convert-to-localtime-from-epoch.py
    1 #!/usr/bin/env python
    2 # -*- coding: UTF-8 -*-
    3
    4 import sys
    5 import time
    6 import csv
    7
    8 arg = sys.argv[1]
    9 f = open(arg, 'r')
   10 reader = csv.reader(f)
   11
   12 for line in reader:
   13        a = line[0]
   14        line[0] = time.ctime(float(a))
   15        print ",".join(line)
   16
   17 f.close()

convert to localtime from epoch

# ./comma-demilited-text-file-convert-to-localtime-from-epoch.py comma-demilited-ephoc.txt
Sat Jul 14 02:15:23 2012,hello,bye

How to install perf ( kernel performance monitor ) on Ubuntu , CentOS

[ install perf on Ubuntu 12.04 ]

# tail -1 /etc/lsb-release

DISTRIB_DESCRIPTION="Ubuntu 12.04 LTS"

# uname -ri

3.2.0-25-generic x86_64

install

# apt-get install linux-tools -y

monitor a chromium-browser process.

fly chromium-browser via perf like this

on GUI , issue the following command

# perf record chromium-browser

nnn, there appears a waning message.. , kernel pointer ??

Kernel address maps (/proc/{kallsyms,modules}) were restricted.

Check /proc/sys/kernel/kptr_restrict before running 'perf record'.

to solve this:

# cat /proc/sys/kernel/kptr_restrict

1

# echo 0 > /proc/sys/kernel/kptr_restrict

# cat /proc/sys/kernel/kptr_restrict

0

# cat /proc/kallsyms | head -10

0000000000000000 D irq_stack_union

0000000000000000 D __per_cpu_start

0000000000004000 D gdt_page

0000000000005000 d exception_stacks

000000000000b000 d tlb_vector_offset

000000000000b080 d cpu_loops_per_jiffy

000000000000b0c0 D xen_vcpu_info

000000000000b100 D xen_vcpu

000000000000b108 d idt_desc

000000000000b118 d xen_cr0_value

try again.

Okay , there are no warning messages.

check the result file

$ perf report

Events: 3K cycles

73.37% chromium-browse [unknown]                [.] 0x7f613588d718

2.80% chromium-browse [kernel.kallsyms]        [k] native_write_msr_safe

1.25% chromium-browse [kernel.kallsyms]        [k] page_fault

1.02% chromium-browse libfreebl3.so            [.] 0x27d5e

<snip>

you can specify events you’d like to collect.

# perf list

List of pre-defined events (to be used in -e):

cpu-cycles OR cycles                               [Hardware event]

stalled-cycles-frontend OR idle-cycles-frontend    [Hardware event]

stalled-cycles-backend OR idle-cycles-backend      [Hardware event]

instructions                                       [Hardware event]

cache-references                                   [Hardware event]

cache-misses                                       [Hardware event]

branch-instructions OR branches                    [Hardware event]

branch-misses                                      [Hardware event]

bus-cycles                                         [Hardware event]

cpu-clock                                          [Software event]

task-clock                                         [Software event]

page-faults OR faults                              [Software event]

minor-faults                                       [Software event]

major-faults                                       [Software event]

context-switches OR cs                             [Software event]

cpu-migrations OR migrations                       [Software event]

$ perf record -e cpu-cycles -e cpu-clock -e context-switches chromium-browser

$ perf report

perf stat

$ perf stat chromium-browser

Performance counter stats for 'chromium-browser':

      2349.438123 task-clock                #    0.211 CPUs utilized

            6,912 context-switches          #    0.003 M/sec

              635 CPU-migrations            #    0.000 M/sec

           87,726 page-faults               #    0.037 M/sec

    4,824,461,601 cycles                    #    2.053 GHz                     [83.95%]

    2,375,916,048 stalled-cycles-frontend   #   49.25% frontend cycles idle    [82.50%]

    1,467,114,781 stalled-cycles-backend    #   30.41% backend cycles idle    [67.95%]

    5,208,469,775 instructions              #    1.08 insns per cycle

                                            #    0.46 stalled cycles per insn [83.95%]

    1,091,803,683 branches                  # 464.708 M/sec                   [82.98%]

       31,296,254 branch-misses             #    2.87% of all branches         [83.44%]

     11.149857740 seconds time elapsed

[ install perf on CentOS6 ]

# cat /etc/centos-release

CentOS release 6.2 (Final)

# uname -ri

2.6.32-220.17.1.el6.x86_64 x86_64

# yum install -y perf

perf : Performance monitoring for the Linux kernel

- about perf

https://perf.wiki.kernel.org/index.php/Tutorial

[root@fc17 ~]# cat /etc/fedora-release
Fedora release 17 (Beefy Miracle)

[root@fc17 ~]# uname -ri
3.4.0-1.fc17.x86_64 x86_64

install perf via yum

[root@fc17 ~]# yum install perf -y

[ example usage ]

# perf top

you can see which processes affect linux kernel performance.

you could also see the detail of specific process info.

select a process -> type enter

here’s an output of map details.

- perf record and perf repot

record kernel performance into a file

[root@fc17 ~]# perf record -a sleep 5
[ perf record: Woken up 5 times to write data ]
[ perf record: Captured and wrote 2.234 MB perf.data (~97596 samples) ]

info is written into perf.data file
.
read that data
“perf report” command reads info from perf.data file.

# perf report
Events: 14K cycles
12.32%          swapper [kernel.kallsyms]                   [k] native_safe_ha
5.25%              dig ld-2.15.so                          [.] do_lookup_x
2.75%              dig ld-2.15.so                          [.] strcmp
2.37%           chrome libflashplayer.so                   [.] 0x00000000003d
1.75%              dig [kernel.kallsyms]                   [k] clear_page_c
1.66%              dig [kernel.kallsyms]                   [k] page_fault
1.49%              dig libc-2.15.so                        [.] __memset_x86_6
1.38%              dig ld-2.15.so                          [.] _dl_lookup_sym
1.32%   gnome-terminal libvte2_90.so.9.3200.1              [.] 0x000000000003
1.20%              dig [kernel.kallsyms]                   [k] unmap_single_v
1.13%              dig ld-2.15.so                          [.] _dl_relocate_o
0.97%      gnome-shell r600_dri.so                         [.] 0x000000000005

# perf report --sort=cpu
Events: 14K cycles
26.35% 3
25.43% 0
24.50% 1
23.73% 2

- monitor a specific process

check PID you want to monitor

[root@fc17 ~]# ps aux | grep unbound | grep -v grep
unbound   1772 0.1 0.1 51596 9884 ?        Ss   00:23   0:03 unbound -c /etc/unbound/unbound.conf

[root@fc17 ~]# perf stat -p 1772 sleep 5

Performance counter stats for process id '1772':

      2619.300788 task-clock                #    0.524 CPUs utilized           [100.00%]
            2,294 context-switches          #    0.876 K/sec                   [100.00%]
              147 CPU-migrations            #    0.056 K/sec                   [100.00%]
              252 page-faults               #    0.096 K/sec
    2,104,000,503 cycles                    #    0.803 GHz                     [49.81%]
      576,211,571 stalled-cycles-frontend   #   27.39% frontend cycles idle    [49.55%]
      702,855,782 stalled-cycles-backend    #   33.41% backend cycles idle    [49.55%]
    1,512,642,754 instructions              #    0.72 insns per cycle
                                            #    0.46 stalled cycles per insn [50.19%]
      296,171,521 branches                  # 113.073 M/sec                   [50.45%]
       23,773,996 branch-misses             #    8.03% of all branches         [50.45%]

      5.001789912 seconds time elapsed

interesting tool.

unbound 1.4.17 supports rrset-rouundrobin and minimal-responses

unbound 1.4.17 supports rrset-roundrobin and minimal-responses.

download unbound 1.4.17 and compile it on Fedora 17

[root@fc17 unbound-1.4.17]# cat /etc/fedora-release
Fedora release 17 (Beefy Miracle)

[root@fc17 unbound-1.4.17]# uname -ri
3.4.0-1.fc17.x86_64 x86_64

before compiling unbound , install ldns-devel if you have not installed it

# yum install ldns-devel

compile

495 ./configure --sysconfdir=/etc/ --disable-gost --disable-ecdsa;make;make install
497 make
498 make install

# unbound-control status
version: 1.4.17
verbosity: 1
threads: 1
modules: 2 [ validator iterator ]
uptime: 47 seconds
unbound (pid 786) is running...

[ rrset-roundrobin ]

unbound.conf

add “rrset-roundrobin: yes” in server section

server:

rrset-roundrobin: yes

reload

# unbound-control reload
ok

it works well

# dig @127.1 unbound.net +short
213.248.210.39
213.154.224.1

# dig @127.1 unbound.net +short
213.154.224.1
213.248.210.39

[ minimal-responses ]

add “minimal-responses: yes” in server section and issue “unbound-control reload”

server:

minimal-responses: yes

before

# dig @127.1 unbound.net

; <<>> DiG 9.9.1-RedHat-9.9.1-1.fc17 <<>> @127.1 unbound.net
; (1 server found)
;; global options: +cmd
;; Got answer:
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 13942
;; flags: qr rd ra; QUERY: 1, ANSWER: 2, AUTHORITY: 3, ADDITIONAL: 1

;; OPT PSEUDOSECTION:
; EDNS: version: 0, flags:; udp: 4096
;; QUESTION SECTION:
;unbound.net. IN A

;; ANSWER SECTION:
unbound.net. 7200 IN A 213.154.224.1
unbound.net. 7200 IN A 213.248.210.39

;; AUTHORITY SECTION:
unbound.net. 7200 IN NS open.nlnetlabs.nl.
unbound.net. 7200 IN NS nom-ns1.nominet.org.uk.
unbound.net. 7200 IN NS ns.secret-wg.org.

after
no auth section.

# dig @127.1 unbound.net

; <<>> DiG 9.9.1-RedHat-9.9.1-1.fc17 <<>> @127.1 unbound.net
; (1 server found)
;; global options: +cmd
;; Got answer:
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 19704
;; flags: qr rd ra; QUERY: 1, ANSWER: 2, AUTHORITY: 0, ADDITIONAL: 1

;; OPT PSEUDOSECTION:
; EDNS: version: 0, flags:; udp: 4096
;; QUESTION SECTION:
;unbound.net. IN A

;; ANSWER SECTION:
unbound.net. 7200 IN A 213.154.224.1
unbound.net. 7200 IN A 213.248.210.39

;; Query time: 1012 msec

dos2unix , unix2dos : convert DOS format text file to UNIX format or vice versa

dos2unix converts a text file line endings from CRLF to LF or LF to CRLF

install dos2unix

mint-13 Script_works # apt-get install dos2unix -y

- convert CRLF to LF

mint-13 Script_works # file zzz.py
zzz.py: Python script, ASCII text executable, with CRLF line terminators

mint-13 Script_works # dos2unix zzz.py
dos2unix: converting file zzz.py to Unix format ...

- convert LF to CRLF

mint-13 Script_works # unix2dos zzz.py
unix2dos: converting file zzz.py to DOS format ...
mint-13 Script_works # file zzz.py
zzz.py: Python script, ASCII text executable, with CRLF line terminators