■debian busterにsysstatを導入する。
$ lsb_release -d
Description: Debian GNU/Linux 10 (buster)
■概要の検索
$ apt-file search bin/sar
geda-utils: /usr/bin/sarlacc_schem
geda-utils: /usr/bin/sarlacc_sym
pcp-import-sar2pcp: /usr/bin/sar2pcp
python-sardana: /usr/bin/sardanatestsuite
sysstat: /usr/bin/sar.sysstat
$ apt-cache show sysstat | ./myscripts/lsec Description-en:
Description-en: system performance tools for Linux
The sysstat package contains the following system performance tools:
- sar: collects and reports system activity information;
- iostat: reports CPU utilization and disk I/O statistics;
- tapestat: reports statistics for tapes connected to the system;
- mpstat: reports global and per-processor statistics;
- pidstat: reports statistics for Linux tasks (processes);
- sadf: displays data collected by sar in various formats;
- cifsiostat: reports I/O statistics for CIFS filesystems.
.
The statistics reported by sar deal with I/O transfer rates,
paging activity, process-related activities, interrupts,
network activity, memory and swap space utilization, CPU
utilization, kernel activities and TTY statistics, among
others. Both UP and SMP machines are fully supported.
■sysstatをインストールするとサービス登録は自動的にされている。
$ sudo apt-get install -y sysstat
$ systemctl list-unit-files -t service | awk 'NR==1||/sysstat/'
UNIT FILE STATE
sysstat.service enabled
$ sudo systemctl status sysstat
● sysstat.service - Resets System Activity Data Collector
Loaded: loaded (/lib/systemd/system/sysstat.service; enabled; vendor preset: enabled)
Active: inactive (dead)
Docs: man:sa1(8)
man:sadc(8)
man:sar(1)
■サービス起動設定
$ sudo cp /etc/default/sysstat{,.org}
$ sudo sed -i -e 's%ENABLED="false"%ENABLED="true"%' /etc/default/sysstat
$ echo /etc/default/sysstat{,.org} | awk '{printf "%-63s %-63s\n",$1,$2}';sdiff -s /etc/default/sysstat{,.org}
/etc/default/sysstat /etc/default/sysstat.org
ENABLED="true" | ENABLED="false"
$ sudo systemctl start sysstat
$ sudo systemctl status sysstat | awk -v host=$(hostname -s) '{gsub(host,"target-host",$0);print $0}'
● sysstat.service - Resets System Activity Data Collector
Loaded: loaded (/lib/systemd/system/sysstat.service; enabled; vendor preset: enabled)
Active: active (exited) since Tue 2020-07-21 00:27:05 JST; 2min 13s ago
Docs: man:sa1(8)
man:sadc(8)
man:sar(1)
Process: 5483 ExecStart=/usr/lib/sysstat/debian-sa1 --boot (code=exited, status=0/SUCCESS)
Main PID: 5483 (code=exited, status=0/SUCCESS)
7月 21 00:27:05 target-host systemd[1]: Starting Resets System Activity Data Collector...
7月 21 00:27:05 target-host systemd[1]: Started Resets System Activity Data Collector.
$ sudo ls -l --time-style='+%Y/%m/%d %H:%M:%S' /var/log/sysstat/sa21
-rw-r--r-- 1 root root 3680 2020/07/21 00:27:05 /var/log/sysstat/sa21
■デフォルトでは10分に1回。
$ cat /etc/cron.d/sysstat
PATH=/usr/lib/sysstat:/usr/sbin:/usr/sbin:/usr/bin:/sbin:/bin
5-55/10 * * * * root command -v debian-sa1 > /dev/null && debian-sa1 1 1
59 23 * * * root command -v debian-sa1 > /dev/null && debian-sa1 60 2
■10分に1回、15分に1回、5分に1回のタイミングで実施される。
※当たり前だけど、5分に1回だと0分には行わないので11回のサンプリングになる。
$ seq 5 55 | awk 'BEGIN{c=0}{if($1%10==5){c++;print c,$1}}'
1 5
2 15
3 25
4 35
5 45
6 55
$ seq 5 55 | awk 'BEGIN{c=0}{if($1%15==5){c++;print c,$1}}'
1 5
2 20
3 35
4 50
$ seq 5 55 | awk 'BEGIN{c=0}{if($1%5==0){c++;print c,$1}}'
1 5
2 10
3 15
4 20
5 25
6 30
7 35
8 40
9 45
10 50
11 55
■15分に1回のサンプリングに変更。reloadは出来ないので、restartする。
途中まで出来たsaファイルを削除するので、stop、削除、startの方が良い。
$ sudo cp /etc/cron.d/sysstat{,.org}
$ sudo sed -i -e 's%5-55/10%5-55/15%' /etc/cron.d/sysstat
$ echo /etc/cron.d/sysstat{,.org} | awk '{printf "%-63s %-63s\n",$1,$2}';sdiff -s /etc/cron.d/sysstat{,.org}
/etc/cron.d/sysstat /etc/cron.d/sysstat.org
5-55/15 * * * * root command -v debian-sa1 > /dev/null && deb | 5-55/10 * * * * root command -v debian-sa1 > /dev/null && deb
$ sudo rm /var/log/sysstat/sa21
$ sudo systemctl reload sysstat
Failed to reload sysstat.service: Job type reload is not applicable for unit sysstat.service.
$ sudo systemctl restart sysstat
$ sudo systemctl status sysstat | awk -v host=$(hostname -s) '{gsub(host,"target-host",$0);print $0}'
● sysstat.service - Resets System Activity Data Collector
Loaded: loaded (/lib/systemd/system/sysstat.service; enabled; vendor preset: enabled)
Active: active (exited) since Tue 2020-07-21 00:40:46 JST; 43s ago
Docs: man:sa1(8)
man:sadc(8)
man:sar(1)
Process: 5657 ExecStart=/usr/lib/sysstat/debian-sa1 --boot (code=exited, status=0/SUCCESS)
Main PID: 5657 (code=exited, status=0/SUCCESS)
7月 21 00:40:46 target-host systemd[1]: Starting Resets System Activity Data Collector...
7月 21 00:40:46 target-host systemd[1]: Started Resets System Activity Data Collector.
■こんな感じで更新を待つ。
$ watch -d -n 10 "sar -u -f /var/log/sysstat/sa21"
Every 10.0s: sar -u -f /var/log/sysstat/sa21 target-host: Tue Jul 21 00:53:41 2020
Linux 4.19.0-9-amd64 (target-host) 2020年07月21日 _x86_64_ (4 CPU)
00時42分58秒 LINUX RESTART (4 CPU)
■待っている間に。
$ man sar 2>&1 | ./myscripts/lsec EXAMPLES
EXAMPLES
sar -u 2 5
Report CPU utilization for each 2 seconds. 5 lines are dis‐
played.
sar -I 14 -o int14.file 2 10
Report statistics on IRQ 14 for each 2 seconds. 10 lines are
displayed. Data are stored in a file called int14.file.
sar -r -n DEV -f /var/log/sysstat/sa16
Display memory and network statistics saved in daily data file
'sa16'.
sar -A
Display all the statistics saved in current daily data file.
Linux OCTOBER 2018 SAR(1)
■使うオプション
lsecのセクションの先頭文字に「-」を追加して、よく使うオプションの説明を抽出する。
$ grep 'if($0' myscripts/lsec
{if($0 ~ /^[A-Za-z0-9-]/){f++;a[f]=$0}else{a[f]=a[f]"___"$0}}\
{if($0 ~ /^[A-Za-z0-9-]/){f++;a[f]=$0}else{a[f]=a[f]"___"$0}}\
$ echo "`seq 32 126`" | awk '{printf "%c,\\%03o\n",$1,$1}' | grep "-"
-,\055
$ man sar 2>&1 | ./myscripts/lsec OPTIONS | sed -e 's/^ //g' | \
./myscripts/lsec "^\055[qubrsef] "
-b Report I/O and transfer rate statistics. The following values
are displayed:
tps
Total number of transfers per second that were issued to
physical devices. A transfer is an I/O request to a
physical device. Multiple logical requests can be com‐
bined into a single I/O request to the device. A trans‐
fer is of indeterminate size.
rtps
Total number of read requests per second issued to physi‐
cal devices.
wtps
Total number of write requests per second issued to phys‐
ical devices.
bread/s
Total amount of data read from the devices in blocks per
second. Blocks are equivalent to sectors and therefore
have a size of 512 bytes.
bwrtn/s
Total amount of data written to devices in blocks per
second.
-e [ hh:mm[:ss] ]
Set the ending time of the report. The default ending time is
18:00:00. Hours must be given in 24-hour format. This option
can be used when data are read from or written to a file (op‐
tions -f or -o).
-f [ filename ]
Extract records from filename (created by the -o filename flag).
The default value of the filename parameter is the current stan‐
dard system activity daily data file. If filename is a direc‐
tory instead of a plain file then it is considered as the direc‐
tory where the standard system activity daily data files are lo‐
cated. The -f option is exclusive of the -o option.
-q Report queue length and load averages. The following values are
displayed:
runq-sz
Run queue length (number of tasks waiting for run time).
plist-sz
Number of tasks in the task list.
ldavg-1
System load average for the last minute. The load aver‐
age is calculated as the average number of runnable or
running tasks (R state), and the number of tasks in unin‐
terruptible sleep (D state) over the specified interval.
ldavg-5
System load average for the past 5 minutes.
ldavg-15
System load average for the past 15 minutes.
blocked
Number of tasks currently blocked, waiting for I/O to
complete.
-r [ ALL ]
Report memory utilization statistics. The ALL keyword indicates
that all the memory fields should be displayed. The following
values may be displayed:
kbmemfree
Amount of free memory available in kilobytes.
kbavail
Estimate of how much memory in kilobytes is available for
starting new applications, without swapping. The esti‐
mate takes into account that the system needs some page
cache to function well, and that not all reclaimable slab
will be reclaimable, due to items being in use. The im‐
pact of those factors will vary from system to system.
kbmemused
Amount of used memory in kilobytes (calculated as total
installed memory - kbmemfree - kbbuffers - kbcached - kb‐
slab).
%memused
Percentage of used memory.
kbbuffers
Amount of memory used as buffers by the kernel in kilo‐
bytes.
kbcached
Amount of memory used to cache data by the kernel in
kilobytes.
kbcommit
Amount of memory in kilobytes needed for current work‐
load. This is an estimate of how much RAM/swap is needed
to guarantee that there never is out of memory.
%commit
Percentage of memory needed for current workload in rela‐
tion to the total amount of memory (RAM+swap). This num‐
ber may be greater than 100% because the kernel usually
overcommits memory.
kbactive
Amount of active memory in kilobytes (memory that has
been used more recently and usually not reclaimed unless
absolutely necessary).
kbinact
Amount of inactive memory in kilobytes (memory which has
been less recently used. It is more eligible to be re‐
claimed for other purposes).
kbdirty
Amount of memory in kilobytes waiting to get written back
to the disk.
kbanonpg
Amount of non-file backed pages in kilobytes mapped into
userspace page tables.
kbslab
Amount of memory in kilobytes used by the kernel to cache
data structures for its own use.
kbkstack
Amount of memory in kilobytes used for kernel stack
space.
kbpgtbl
Amount of memory in kilobytes dedicated to the lowest
level of page tables.
kbvmused
Amount of memory in kilobytes of used virtual address
space.
-s [ hh:mm[:ss] ]
Set the starting time of the data, causing the sar command to
extract records time-tagged at, or following, the time speci‐
fied. The default starting time is 08:00:00. Hours must be
given in 24-hour format. This option can be used only when data
are read from a file (option -f).
-u [ ALL ]
Report CPU utilization. The ALL keyword indicates that all the
CPU fields should be displayed. The report may show the follow‐
ing fields:
%user
Percentage of CPU utilization that occurred while execut‐
ing at the user level (application). Note that this field
includes time spent running virtual processors.
%usr
Percentage of CPU utilization that occurred while execut‐
ing at the user level (application). Note that this field
does NOT include time spent running virtual processors.
%nice
Percentage of CPU utilization that occurred while execut‐
ing at the user level with nice priority.
%system
Percentage of CPU utilization that occurred while execut‐
ing at the system level (kernel). Note that this field
includes time spent servicing hardware and software in‐
terrupts.
%sys
Percentage of CPU utilization that occurred while execut‐
ing at the system level (kernel). Note that this field
does NOT include time spent servicing hardware or soft‐
ware interrupts.
%iowait
Percentage of time that the CPU or CPUs were idle during
which the system had an outstanding disk I/O request.
%steal
Percentage of time spent in involuntary wait by the vir‐
tual CPU or CPUs while the hypervisor was servicing an‐
other virtual processor.
%irq
Percentage of time spent by the CPU or CPUs to service
hardware interrupts.
%soft
Percentage of time spent by the CPU or CPUs to service
software interrupts.
%guest
Percentage of time spent by the CPU or CPUs to run a vir‐
tual processor.
%gnice
Percentage of time spent by the CPU or CPUs to run a
niced guest.
%idle
Percentage of time that the CPU or CPUs were idle and the
system did not have an outstanding disk I/O request.
Linux OCTOBER 2018 SAR(1)
■CPUのidle値だけを抽出
$ env LANG=C sar -u -f /var/log/sysstat/sa21 | awk '(NR==5||NR==6)'
00:50:01 CPU %user %nice %system %iowait %steal %idle
01:05:01 all 3.68 0.00 0.65 0.04 0.00 95.62
$ env LANG=C sar -u -f /var/log/sysstat/sa21 | \
awk -v search=idle '(NR==5){for(a=2;a<=NF;a++){if($a ~ search){target=a}}} \
(NR>5&&$1 !~ /Average/&& $2 ~ /all/){print $1","$target}'
01:05:01,95.62
01:20:01,95.71
01:35:01,96.80
01:50:01,92.32
02:05:02,95.79
■100-CPUのidle値を抽出(あくまで見栄えのCPU使用率であって正確な値では無いので、要件に注意)
$ env LANG=C sar -u -f /var/log/sysstat/sa21 | \
awk -v search=idle '(NR==5){for(a=2;a<=NF;a++){if($a ~ search){target=a}}} \
(NR>5&&$1 !~ /Average/&& $2 ~ /all/){print $1","100-$target}'
01:05:01,4.38
01:20:01,4.29
01:35:01,3.2
01:50:01,7.68
02:05:02,4.21
■メモリ使用率の値だけを抽出
$ env LANG=C sar -r -f /var/log/sysstat/sa21 | awk '(NR==5||NR==6)'
00:50:01 kbmemfree kbavail kbmemused %memused kbbuffers kbcached kbcommit %commit kbactive kbinact kbdirty
01:05:01 284412 11925376 3962084 24.18 139556 11560376 10945320 33.05 4617104 10887628 8
$ env LANG=C sar -r -f /var/log/sysstat/sa21 | \
awk -v search=memused '(NR==5){for(a=2;a<=NF;a++){if($a ~ search){target=a}}} \
(NR>5&&$1 !~ /Average/){print $1","$target}'
01:05:01,24.18
01:20:01,24.18
01:35:01,24.15
01:50:01,24.17
02:05:02,24.15
■100-メモリ使用率の値だけを抽出(あくまで見栄えのメモリ空き容量であって正確な値では無いので、要件に注意)
$ env LANG=C sar -r -f /var/log/sysstat/sa21 | \
awk -v search=memused '(NR==5){for(a=2;a<=NF;a++){if($a ~ search){target=a}}} \
(NR>5&&$1 !~ /Average/){print $1","100-$target}'
01:05:01,75.82
01:20:01,75.82
01:35:01,75.85
01:50:01,75.83
02:05:02,75.85
■スワップ使用率
$ env LANG=C sar -S -f /var/log/sysstat/sa21 | awk '(NR==5||NR==6)'
00:50:01 kbswpfree kbswpused %swpused kbswpcad %swpcad
01:05:01 16733936 1292 0.01 32 2.48
$ env LANG=C sar -S -f /var/log/sysstat/sa21 | \
awk -v search="^.swpused" '(NR==5){for(a=2;a<=NF;a++){if($a ~ search){target=a}}} \
(NR>5&&$1 !~ /Average/){print $1","$target}'
01:05:01,0.01
01:20:01,0.01
01:35:01,0.01
01:50:01,0.01
02:05:02,0.01
■100-スワップ使用率
$ env LANG=C sar -S -f /var/log/sysstat/sa21 | \
awk -v search="^.swpused" '(NR==5){for(a=2;a<=NF;a++){if($a ~ search){target=a}}} \
(NR>5&&$1 !~ /Average/){print $1","100-$target}'
01:05:01,99.99
01:20:01,99.99
01:35:01,99.99
01:50:01,99.99
02:05:02,99.99
■ディスクはtps、rtps、wtpsを抽出。
$ env LANG=C sar -b -f /var/log/sysstat/sa21 | awk '(NR==5||NR==6)'
00:50:01 tps rtps wtps bread/s bwrtn/s
01:05:01 0.23 0.01 0.22 0.20 2.66
$ env LANG=C sar -b -f /var/log/sysstat/sa21 | \
awk -v search="^tps" '(NR==5){for(a=2;a<=NF;a++){if($a ~ search){target=a}}} \
(NR>5&&$1 !~ /Average/){OFS=",";print $1,$target,$(target+1),$(target+2)}'
01:05:01,0.23,0.01,0.22
01:20:01,0.39,0.11,0.28
01:35:01,0.24,0.00,0.24
01:50:01,0.16,0.00,0.16
02:05:02,0.19,0.01,0.18
■virbr0のネットワークインターフェイス使用率
$ env LANG=C sar -n DEV -f /var/log/sysstat/sa21 | awk '(NR==5||NR==6)'
00:50:01 IFACE rxpck/s txpck/s rxkB/s txkB/s rxcmp/s txcmp/s rxmcst/s %ifutil
01:05:01 virbr0 0.01 0.01 0.00 0.00 0.00 0.00 0.00 0.00
$ env LANG=C sar -n DEV -f /var/log/sysstat/sa21 | awk '(NR>6&&$1 !~ /Average/){print $2 | "sort -uV"}'
br0
br1
docker0
eno1
lo
tap0
virbr0
virbr0-nic
vnet0
$ env LANG=C sar -n DEV -f /var/log/sysstat/sa21 | \
awk -v search="^.ifutil" -v interface="virbr0$" \
'(NR==5){for(a=2;a<=NF;a++){if($a ~ search){target=a}}} \
(NR>5&&$1 !~ /Average/&&$2 ~ interface){print $1","$target}'
01:05:01,0.00
01:20:01,0.00
01:35:01,0.00
01:50:01,0.00
02:05:02,0.00
■br0のrxkB/S、wxkB/Sの取得。
$ env LANG=C sar -n DEV -f /var/log/sysstat/sa21 | \
awk -v search="^rxkB/s" -v interface="^br0" '(NR==5){for(a=2;a<=NF;a++){if($a ~ search){target=a}}} \
(NR>5&&$1 !~ /Average/&&$2 ~ interface){OFS=",";print $1","$target,$(target+1)}'
01:05:01,0.17,0.87
01:20:01,0.17,0.30
01:35:01,0.09,0.10
01:50:01,0.12,0.16
02:05:02,0.10,0.12
