lost and found ( for me ? )

Python : convert to localtime from epoch time

small tips.


# python --version
Python 2.7.3

1. convert space-demilited text file’s timestamp from epoch to localtime



# cat space-demilited-ephoc.txt
1342199723 hello bye

python script
# cat -n space-demilited-text-file-convert-to-localtime-from-epoch.py
    1 #!/usr/bin/env python
    2 # -*- coding: UTF-8 -*-
    3
    4 import sys
    5 import time
    6
    7 arg = sys.argv[1]
    8 f = open(arg, 'r')
    9
   10 for line in f:
   11        s = line.split()
   12        a = s[0]
   13        s[0] = time.ctime(float(a))
   14        print " ".join(s)
   15
   16 f.close()

convert to localtime from epoch time.
# ./space-demilited-text-file-convert-to-localtime-from-epoch.py space-demilited-ephoc.txt
Sat Jul 14 02:15:23 2012 hello bye

2. convert comma-demilited text file’s timestamp from epoch to localtime

# cat comma-demilited-ephoc.txt
1342199723,hello,bye

python script
# cat -n comma-demilited-text-file-convert-to-localtime-from-epoch.py
    1 #!/usr/bin/env python
    2 # -*- coding: UTF-8 -*-
    3
    4 import sys
    5 import time
    6 import csv
    7
    8 arg = sys.argv[1]
    9 f = open(arg, 'r')
   10 reader = csv.reader(f)
   11
   12 for line in reader:
   13        a = line[0]
   14        line[0] = time.ctime(float(a))
   15        print ",".join(line)
   16
   17 f.close()

convert to localtime from epoch
# ./comma-demilited-text-file-convert-to-localtime-from-epoch.py comma-demilited-ephoc.txt
Sat Jul 14 02:15:23 2012,hello,bye

How to install perf ( kernel performance monitor ) on Ubuntu , CentOS


[ install perf on Ubuntu 12.04 ]



# tail -1 /etc/lsb-release
DISTRIB_DESCRIPTION="Ubuntu 12.04 LTS"

# uname -ri
3.2.0-25-generic x86_64

install
# apt-get install linux-tools -y


monitor a chromium-browser process.
fly chromium-browser via perf like this

on GUI , issue the following command
# perf record chromium-browser




nnn, there appears a waning message.. , kernel pointer ??
Kernel address maps (/proc/{kallsyms,modules}) were restricted.                                                                        
Check /proc/sys/kernel/kptr_restrict before running 'perf record'.


to solve this:
# cat /proc/sys/kernel/kptr_restrict
1

# echo 0 > /proc/sys/kernel/kptr_restrict

# cat /proc/sys/kernel/kptr_restrict
0

# cat /proc/kallsyms  | head -10
0000000000000000 D irq_stack_union
0000000000000000 D __per_cpu_start
0000000000004000 D gdt_page
0000000000005000 d exception_stacks
000000000000b000 d tlb_vector_offset
000000000000b080 d cpu_loops_per_jiffy
000000000000b0c0 D xen_vcpu_info
000000000000b100 D xen_vcpu
000000000000b108 d idt_desc
000000000000b118 d xen_cr0_value


try again.
Okay , there are no warning messages.

check the result file
$ perf report
Events: 3K cycles
73.37%  chromium-browse  [unknown]                [.] 0x7f613588d718
 2.80%  chromium-browse  [kernel.kallsyms]        [k] native_write_msr_safe
 1.25%  chromium-browse  [kernel.kallsyms]        [k] page_fault
 1.02%  chromium-browse  libfreebl3.so            [.] 0x27d5e
<snip>

you can specify events you’d like to collect.
# perf list

List of pre-defined events (to be used in -e):
 cpu-cycles OR cycles                               [Hardware event]
 stalled-cycles-frontend OR idle-cycles-frontend    [Hardware event]
 stalled-cycles-backend OR idle-cycles-backend      [Hardware event]
 instructions                                       [Hardware event]
 cache-references                                   [Hardware event]
 cache-misses                                       [Hardware event]
 branch-instructions OR branches                    [Hardware event]
 branch-misses                                      [Hardware event]
 bus-cycles                                         [Hardware event]

 cpu-clock                                          [Software event]
 task-clock                                         [Software event]
 page-faults OR faults                              [Software event]
 minor-faults                                       [Software event]
 major-faults                                       [Software event]
 context-switches OR cs                             [Software event]
 cpu-migrations OR migrations                       [Software event]

$ perf record -e cpu-cycles -e cpu-clock -e context-switches chromium-browser

$ perf report

perf stat
$ perf stat chromium-browser

Performance counter stats for 'chromium-browser':

      2349.438123 task-clock                #    0.211 CPUs utilized          
            6,912 context-switches          #    0.003 M/sec                  
              635 CPU-migrations            #    0.000 M/sec                  
           87,726 page-faults               #    0.037 M/sec                  
    4,824,461,601 cycles                    #    2.053 GHz                     [83.95%]
    2,375,916,048 stalled-cycles-frontend   #   49.25% frontend cycles idle    [82.50%]
    1,467,114,781 stalled-cycles-backend    #   30.41% backend  cycles idle    [67.95%]
    5,208,469,775 instructions              #    1.08  insns per cycle        
                                            #    0.46  stalled cycles per insn [83.95%]
    1,091,803,683 branches                  #  464.708 M/sec                   [82.98%]
       31,296,254 branch-misses             #    2.87% of all branches         [83.44%]

     11.149857740 seconds time elapsed

[ install perf on CentOS6 ]

# cat /etc/centos-release
CentOS release 6.2 (Final)

# uname -ri
2.6.32-220.17.1.el6.x86_64 x86_64

# yum install -y perf

perf : Performance monitoring for the Linux kernel

- about perf

https://perf.wiki.kernel.org/index.php/Tutorial



[root@fc17 ~]# cat /etc/fedora-release
Fedora release 17 (Beefy Miracle)

[root@fc17 ~]# uname -ri
3.4.0-1.fc17.x86_64 x86_64

install perf via yum
[root@fc17 ~]# yum install perf -y

[ example usage ]

# perf top

you can see which processes affect linux kernel performance.

you could also see the detail of specific process info.

select a process -> type enter





here’s an output of map details.





- perf record and perf repot

record kernel performance into a file

[root@fc17 ~]# perf record -a sleep 5
[ perf record: Woken up 5 times to write data ]
[ perf record: Captured and wrote 2.234 MB perf.data (~97596 samples) ]

info is written into perf.data file
.
read that data
“perf report” command reads info from perf.data file.
# perf report
Events: 14K cycles                                                              
12.32%          swapper  [kernel.kallsyms]                   [k] native_safe_ha
 5.25%              dig  ld-2.15.so                          [.] do_lookup_x
 2.75%              dig  ld-2.15.so                          [.] strcmp
 2.37%           chrome  libflashplayer.so                   [.] 0x00000000003d
 1.75%              dig  [kernel.kallsyms]                   [k] clear_page_c
 1.66%              dig  [kernel.kallsyms]                   [k] page_fault
 1.49%              dig  libc-2.15.so                        [.] __memset_x86_6
 1.38%              dig  ld-2.15.so                          [.] _dl_lookup_sym
 1.32%   gnome-terminal  libvte2_90.so.9.3200.1              [.] 0x000000000003
 1.20%              dig  [kernel.kallsyms]                   [k] unmap_single_v
 1.13%              dig  ld-2.15.so                          [.] _dl_relocate_o
 0.97%      gnome-shell  r600_dri.so                         [.] 0x000000000005

# perf report --sort=cpu
Events: 14K cycles                                                              
26.35%  3
25.43%  0
24.50%  1
23.73%  2

- monitor a specific process

check PID you want to monitor
[root@fc17 ~]# ps aux | grep unbound | grep -v grep
unbound   1772  0.1  0.1  51596  9884 ?        Ss   00:23   0:03 unbound -c /etc/unbound/unbound.conf

[root@fc17 ~]# perf stat -p 1772 sleep 5

Performance counter stats for process id '1772':

      2619.300788 task-clock                #    0.524 CPUs utilized           [100.00%]
            2,294 context-switches          #    0.876 K/sec                   [100.00%]
              147 CPU-migrations            #    0.056 K/sec                   [100.00%]
              252 page-faults               #    0.096 K/sec                  
    2,104,000,503 cycles                    #    0.803 GHz                     [49.81%]
      576,211,571 stalled-cycles-frontend   #   27.39% frontend cycles idle    [49.55%]
      702,855,782 stalled-cycles-backend    #   33.41% backend  cycles idle    [49.55%]
    1,512,642,754 instructions              #    0.72  insns per cycle        
                                            #    0.46  stalled cycles per insn [50.19%]
      296,171,521 branches                  #  113.073 M/sec                   [50.45%]
       23,773,996 branch-misses             #    8.03% of all branches         [50.45%]

      5.001789912 seconds time elapsed

interesting tool.

unbound 1.4.17 supports rrset-rouundrobin and minimal-responses

unbound 1.4.17 supports rrset-roundrobin and minimal-responses.


download unbound 1.4.17 and compile it on Fedora 17

[root@fc17 unbound-1.4.17]# cat /etc/fedora-release
Fedora release 17 (Beefy Miracle)

[root@fc17 unbound-1.4.17]# uname -ri
3.4.0-1.fc17.x86_64 x86_64

before compiling unbound , install ldns-devel if you have not installed it

# yum install ldns-devel

compile
 495  ./configure --sysconfdir=/etc/ --disable-gost --disable-ecdsa;make;make install
 497  make
 498  make install

# unbound-control status
version: 1.4.17
verbosity: 1
threads: 1
modules: 2 [ validator iterator ]
uptime: 47 seconds
unbound (pid 786) is running...


[ rrset-roundrobin ]

unbound.conf

add “rrset-roundrobin: yes” in server section
server:

       rrset-roundrobin: yes

reload
# unbound-control reload
ok

it works well
# dig @127.1 unbound.net +short
213.248.210.39
213.154.224.1

# dig @127.1 unbound.net +short
213.154.224.1
213.248.210.39

[ minimal-responses ]

add “minimal-responses: yes” in server section and issue “unbound-control reload”
server:

       minimal-responses: yes

before
# dig @127.1 unbound.net

; <<>> DiG 9.9.1-RedHat-9.9.1-1.fc17 <<>> @127.1 unbound.net
; (1 server found)
;; global options: +cmd
;; Got answer:
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 13942
;; flags: qr rd ra; QUERY: 1, ANSWER: 2, AUTHORITY: 3, ADDITIONAL: 1

;; OPT PSEUDOSECTION:
; EDNS: version: 0, flags:; udp: 4096
;; QUESTION SECTION:
;unbound.net. IN A

;; ANSWER SECTION:
unbound.net. 7200 IN A 213.154.224.1
unbound.net. 7200 IN A 213.248.210.39

;; AUTHORITY SECTION:
unbound.net. 7200 IN NS open.nlnetlabs.nl.
unbound.net. 7200 IN NS nom-ns1.nominet.org.uk.
unbound.net. 7200 IN NS ns.secret-wg.org.


after
no auth section.
# dig @127.1 unbound.net

; <<>> DiG 9.9.1-RedHat-9.9.1-1.fc17 <<>> @127.1 unbound.net
; (1 server found)
;; global options: +cmd
;; Got answer:
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 19704
;; flags: qr rd ra; QUERY: 1, ANSWER: 2, AUTHORITY: 0, ADDITIONAL: 1

;; OPT PSEUDOSECTION:
; EDNS: version: 0, flags:; udp: 4096
;; QUESTION SECTION:
;unbound.net. IN A

;; ANSWER SECTION:
unbound.net. 7200 IN A 213.154.224.1
unbound.net. 7200 IN A 213.248.210.39

;; Query time: 1012 msec


dos2unix , unix2dos : convert DOS format text file to UNIX format or vice versa

dos2unix converts a text file line endings from CRLF to LF or LF to CRLF

install dos2unix

mint-13 Script_works # apt-get install dos2unix -y

- convert CRLF to LF
mint-13 Script_works # file zzz.py
zzz.py: Python script, ASCII text executable, with CRLF line terminators

mint-13 Script_works # dos2unix zzz.py
dos2unix: converting file zzz.py to Unix format ...

- convert LF to CRLF
mint-13 Script_works # unix2dos zzz.py
unix2dos: converting file zzz.py to DOS format ...
mint-13 Script_works # file zzz.py
zzz.py: Python script, ASCII text executable, with CRLF line terminators