labunix's blog

labunixのラボUnix

ワンライナーで$PATHから自身のユーザ名のパスを除き、scriptファイルを種類分けしてみる。

■ワンライナーで$PATHから自身のユーザ名のパスを除き、scriptファイルを種類分けしてみる。

$ echo $PATH | \
    awk -F\: -v WHOAMI=$(whoami) \
      '{for(a=1;a<=NF;a++){if($a !~ WHOAMI) \
       {print "find "$a" -type f -exec file -N -F\134; {} +" | "sh"}}}' | \
    awk -F\; '{a[$2]=a[$2]";"$1}END{for(n in a){if(n ~ /[Ss]cript/&& a[n] !~ /log/){gsub(";","\n\t",a[n]);print "["n"]",a[n]}}}'
[ Ruby script, ASCII text executable] 
	/usr/bin/irb1.8
	/usr/bin/ri1.9.1
	/usr/bin/testrb2.1
	/usr/bin/gem2.5
	/usr/bin/irb1.9.1
	/usr/bin/ri2.5
	/usr/bin/prime-refresh
	/usr/bin/irb2.5
	/usr/bin/rake
	/usr/bin/rake2.1
	/usr/bin/rdoc1.9.1
	/usr/bin/rake1.9.1
	/usr/bin/ri2.1
	/usr/bin/gem2.1
	/usr/bin/rdoc2.1
	/usr/bin/irb2.1
	/usr/bin/ri1.8
	/usr/bin/rdoc2.5
	/usr/bin/testrb1.9.1
	/usr/bin/gem1.9.1
	/usr/bin/testrb1.8
[ POSIX shell script, ISO-8859 text executable, with CR, LF line terminators] 
	/usr/bin/card
[ a /usr/bin/python3.7 script, ASCII text executable] 
	/usr/bin/pydoc3.7
	/usr/bin/pyvenv-3.7
[ a /usr/bin/python2 script, ASCII text executable] 
	/usr/bin/chardetect
[ POSIX shell script, ASCII text executable, with CR, LF line terminators] 
	/usr/bin/mktexlsr
	/usr/bin/fixps
[ a /usr/bin/python3 script, ASCII text executable] 
	/usr/bin/pygmentize
	/usr/bin/spd-conf
	/usr/bin/chardetect3
	/usr/bin/ansible
	/usr/bin/gdbus-codegen
	/usr/bin/ssh-audit
	/usr/bin/ansible-connection
	/usr/bin/glib-genmarshal
	/usr/bin/ansible-test
	/usr/bin/glib-mkenums
	/usr/bin/aptdcon
	/usr/bin/g-ir-doc-tool
	/usr/bin/gtester-report
	/usr/bin/g-ir-scanner
	/usr/bin/g-ir-annotation-tool
[ a /usr/bin/python script, ASCII text executable] 
	/usr/bin/pcsc-spy
	/usr/bin/dh_python2
[ a /usr/bin/python3.7 script, ISO-8859 text executable] 
	/usr/bin/pygettext3.7
[ a /usr/bin/python3 script, UTF-8 Unicode text executable] 
	/usr/bin/py3compile
	/usr/bin/py3clean
[ Bourne-Again shell script, ASCII text executable, with escape sequences] 
	/usr/bin/libtool
[ a /usr/bin/python2 script, UTF-8 Unicode text executable] 
	/usr/bin/pycompile
	/usr/bin/pyclean
[ a /usr/bin/mawk -f script, ASCII text executable] 
	/usr/bin/checkmk
[ a /usr/bin/python2.7 script, ASCII text executable] 
	/usr/bin/2to3-2.7
	/usr/bin/pydoc2.7
[ Ruby script, ASCII text] 
	/usr/bin/prime-dict-convert
	/usr/bin/prime-dict-index
	/usr/bin/erb2.5
	/usr/bin/erb2.1
	/usr/bin/erb1.8
	/usr/bin/whatweb
	/usr/bin/rdoc1.8
	/usr/bin/prime
	/usr/bin/erb1.9.1
	/usr/bin/prime-userdict-convert
	/usr/bin/prime-userdict-update
[ Bourne-Again shell script, UTF-8 Unicode text executable] 
	/usr/bin/deb-reversion
	/usr/bin/hwloc-assembler-remote
	/usr/bin/who-uploads
[ Bourne-Again shell script, UTF-8 Unicode text executable, with very long lines] 
	/usr/bin/funcoeszz
[ a /usr/bin/python2.7 script, ISO-8859 text executable] 
	/usr/bin/pygettext2.7
[ POSIX shell script, ASCII text executable, with very long lines] 
	/usr/bin/discover-config
	/usr/bin/jarwrapper
	/usr/bin/net-snmp-config
[ a /usr/bin/env nickle script, UTF-8 Unicode text executable] 
	/usr/bin/xkeystone
[ POSIX shell script, UTF-8 Unicode text executable, with escape sequences] 
	/usr/bin/dpkg-maintscript-helper
[ Bourne-Again shell script, ASCII text executable, with very long lines] 
	/usr/bin/checkinstall
[ a /usr/bin/gosh script, ASCII text executable] 
	/usr/bin/gauche-cesconv
[ POSIX shell script, ASCII text executable, with very long lines, with escape sequences] 
	/usr/bin/libtoolize
[ Python script, UTF-8 Unicode text executable] 
	/usr/bin/chrome-gnome-shell
	/usr/bin/meld
	/usr/bin/sadt
[ a /usr/bin/php script executable (binary data)] 
	/usr/bin/phar5
[ awk script, ASCII text executable] 
	/usr/bin/mcolor
[ Python script, ASCII text executable] 
	/usr/local/bin/pyuic5
	/usr/local/bin/jsonschema
	/usr/local/bin/gns3
	/usr/local/bin/pip2
	/usr/local/bin/gns3loopback
	/usr/local/bin/gns3server
	/usr/local/bin/gns3vmnet
	/usr/local/bin/pip
	/usr/local/bin/pip2.7
	/usr/local/bin/raven
	/usr/local/bin/pip3.7
	/usr/local/bin/pyrcc5
	/usr/local/bin/brew_tools
	/usr/local/bin/pip3
	/usr/local/bin/pylupdate5
	/usr/bin/swfparser
	/usr/bin/reproducible-check
	/usr/bin/fail2ban-client
	/usr/bin/wapiti-getcookie
	/usr/bin/querybts
	/usr/bin/py3rsa-decrypt
	/usr/bin/gnome-shell-perf-tool
	/usr/bin/fail2ban-server
	/usr/bin/py3rsa-keygen
	/usr/bin/openshot-qt
	/usr/bin/reportbug
	/usr/bin/fail2ban-testcases
	/usr/bin/apt-listchanges
	/usr/bin/py3rsa-sign
	/usr/bin/pycodestyle
	/usr/bin/py3rsa-encrypt
	/usr/bin/binwalk
	/usr/bin/debianbts
	/usr/bin/gnome-shell-extension-tool
	/usr/bin/suspicious-source
	/usr/bin/debdiff-apply
	/usr/bin/f2py3.7
	/usr/bin/lsb_release
	/usr/bin/aws_completer
	/usr/bin/youtube-dl
	/usr/bin/wapiti
	/usr/bin/wrap-and-sort
	/usr/bin/stepic
	/usr/bin/popbugs
	/usr/bin/fail2ban-regex
	/usr/bin/aws
	/usr/bin/gtk-builder-convert
	/usr/bin/f2py3
	/usr/bin/dnsping
	/usr/bin/py3rsa-priv2pub
	/usr/bin/which-pkg-broke
	/usr/bin/orca
	/usr/bin/dnseval
	/usr/bin/unattended-upgrade
	/usr/bin/python3-cymruwhois
	/usr/bin/py3rsa-verify

■環境

$ env LANG=C bash --version | grep ^GNU
GNU bash, version 5.0.3(1)-release (x86_64-pc-linux-gnu)

$ awk --version | grep ^GNU
GNU Awk 4.2.1, API: 2.0 (GNU MPFR 4.0.2, GNU MP 6.1.2)
GNU General Public License for more details.

$ lsb_release -d
Description:	Debian GNU/Linux 10 (buster)

■fileコマンドの結果の一覧

$ echo $PATH | \
    awk -F\: -v WHOAMI=$(whoami) \
      '{for(a=1;a<=NF;a++){if($a !~ WHOAMI) \
       {print "find "$a" -type f -exec file -N -F\134; {} +" | "sh"}}}' | \
    awk -F\; '{a[$2]=a[$2]";"$1}END{for(n in a){if(n ~ /[Ss]cript/&& a[n] !~ /log/){print n | "sort -uV"}}}'
 Bourne-Again shell script, ASCII text executable, with escape sequences
 Bourne-Again shell script, ASCII text executable, with very long lines
 Bourne-Again shell script, UTF-8 Unicode text executable
 Bourne-Again shell script, UTF-8 Unicode text executable, with very long lines
 POSIX shell script, ASCII text executable, with CR, LF line terminators
 POSIX shell script, ASCII text executable, with very long lines
 POSIX shell script, ASCII text executable, with very long lines, with escape sequences
 POSIX shell script, ISO-8859 text executable, with CR, LF line terminators
 POSIX shell script, UTF-8 Unicode text executable, with escape sequences
 Python script, ASCII text executable
 Python script, UTF-8 Unicode text executable
 Ruby script, ASCII text
 Ruby script, ASCII text executable
 awk script, ASCII text executable
 a /usr/bin/env nickle script, UTF-8 Unicode text executable
 a /usr/bin/gosh script, ASCII text executable
 a /usr/bin/mawk -f script, ASCII text executable
 a /usr/bin/php script executable (binary data)
 a /usr/bin/python2 script, ASCII text executable
 a /usr/bin/python2 script, UTF-8 Unicode text executable
 a /usr/bin/python2.7 script, ASCII text executable
 a /usr/bin/python2.7 script, ISO-8859 text executable
 a /usr/bin/python3 script, ASCII text executable
 a /usr/bin/python3 script, UTF-8 Unicode text executable
 a /usr/bin/python3.7 script, ASCII text executable
 a /usr/bin/python3.7 script, ISO-8859 text executable
 a /usr/bin/python script, ASCII text executable

■シェルスクリプトを抽出

$ echo $PATH | \
    awk -F\: -v WHOAMI=$(whoami) \
      '{for(a=1;a<=NF;a++){if($a !~ WHOAMI) \
       {print "find "$a" -type f -exec file -N -F\134; {} +" | "sh"}}}' | \
    awk -F\; '{a[$2]=a[$2]";"$1}END{for(n in a){if(n ~ /[Ss]cript/&& a[n] !~ /log/&&n ~ /Bourne-Again|POSIX/) \
       {gsub(";","\n\t",a[n]);print "["n"]",a[n]}}}'
[ POSIX shell script, ISO-8859 text executable, with CR, LF line terminators] 
	/usr/bin/card
[ POSIX shell script, ASCII text executable, with CR, LF line terminators] 
	/usr/bin/mktexlsr
	/usr/bin/fixps
[ Bourne-Again shell script, ASCII text executable, with escape sequences] 
	/usr/bin/libtool
[ Bourne-Again shell script, UTF-8 Unicode text executable] 
	/usr/bin/deb-reversion
	/usr/bin/hwloc-assembler-remote
	/usr/bin/who-uploads
[ Bourne-Again shell script, UTF-8 Unicode text executable, with very long lines] 
	/usr/bin/funcoeszz
[ POSIX shell script, ASCII text executable, with very long lines] 
	/usr/bin/discover-config
	/usr/bin/jarwrapper
	/usr/bin/net-snmp-config
[ POSIX shell script, UTF-8 Unicode text executable, with escape sequences] 
	/usr/bin/dpkg-maintscript-helper
[ Bourne-Again shell script, ASCII text executable, with very long lines] 
	/usr/bin/checkinstall
[ POSIX shell script, ASCII text executable, with very long lines, with escape sequences] 
	/usr/bin/libtoolize

■シェルスクリプト種類でファイルに書き出す
 ファイルの種類から、「/usr/bin/」を削除するなり「/」を別の記号に置き換えるなどは今回は不要なのでそのまま。

$ echo $PATH | \
    awk -F\: -v WHOAMI=$(whoami) \
      '{for(a=1;a<=NF;a++){if($a !~ WHOAMI) \
       {print "find "$a" -type f -exec file -N -F\134; {} +" | "sh"}}}' | \
    awk -F\; '{a[$2]=a[$2]";"$1}END{for(n in a){if(n ~ /[Ss]cript/&& a[n] !~ /log/&&n ~ /Bourne-Again|POSIX/) \
       {gsub(";","\n\t",a[n]);b=n;gsub(",.*","",b);gsub(" ","_",b);print "echo \047"a[n]"\047 >> "b}}}' | sh

■シェルスクリプトの種類毎のファイル一覧
 上記のワンライナーが追記型なのでsortコマンドで
 「-V」でバージョン順に並び替え、「-u」でユニーク表示する。

$ ls _* | awk '{a=$0;gsub("_"," ",a);print "echo ["a"];sort -uV "$0 | "sh"}' | grep -v "^\$"
[ Bourne-Again shell script]
	/usr/bin/checkinstall
	/usr/bin/deb-reversion
	/usr/bin/funcoeszz
	/usr/bin/hwloc-assembler-remote
	/usr/bin/libtool
	/usr/bin/who-uploads
[ POSIX shell script]
	/usr/bin/card
	/usr/bin/discover-config
	/usr/bin/dpkg-maintscript-helper
	/usr/bin/fixps
	/usr/bin/jarwrapper
	/usr/bin/libtoolize
	/usr/bin/mktexlsr
	/usr/bin/net-snmp-config

■例えばcase文の内ファイルの抽出。

$ sort -uV _Bourne-Again_shell_script | grep -v "^\$" | tr -d '\t' > _Bourne-Again_shell_script_sorted
$ awk '/case.*in/{print FILENAME | "sort -uV"}' $(sort -uV _Bo*) > _Bourne-Again_shell_script_case
$ diff -s _Bourne-Again_shell_script{_sorted,_case}
ファイル _Bourne-Again_shell_script_sorted と _Bourne-Again_shell_script_case は同一です

$ sort -uV _POSIX_shell_script  | grep -v "^\$" | tr -d '\t' > _POSIX_shell_script_sorted
$ awk '/case.*in/{print FILENAME | "sort -uV"}' $(sort -uV _PO*) > _POSIX_shell_script_case
$ diff -s _POSIX_shell_script{_sorted,_case}
5d4
< /usr/bin/jarwrapper

■$PATHは「:」で区切られているので、for文で列分割して抽出する例。
 for文のaに0を与えると列分割前のデータ、つまり元の行が与えられる。

$ echo $PATH | \
    awk -F\: -v WHOAMI=$(whoami) \
      '{for(a=1;a<=NF;a++){if($a !~ WHOAMI) \
       {print $a}}}' 
/usr/local/bin
/usr/bin
/bin
/usr/local/games
/usr/games

■メインとなるfindコマンドとfileコマンドの組み合わせの例。
 シェル行の終端文字にも取れる「;」は「\」でエスケープする必要があり、
 awk文内の「\」は「\134」で8進数エスケープする必要がある。

$ find /usr/local/bin/ -type f -exec file -N -F\; \{} + | head -2
/usr/local/bin/pyuic5; Python script, ASCII text executable
/usr/local/bin/jsonschema; Python script, ASCII text executable

■2列目の文字列(1文字でも文字列として扱う)をキーに、1列目がどのキーに属するか集計する例。

$ echo -e "a b\nc b\nd d" | awk '{a[$2]=$1";"a[$2]}END{for(n in a){print n,a[n]}}' 
b c;a;
d d;