This document discusses the awk command and how it can be used to parse and extract data from text files and output. Awk allows specifying patterns to match and actions to perform on matches. Examples show using awk to extract specific fields from the /etc/passwd file and Apache log files. Pipelines combining awk with other commands like grep, sed and cut are demonstrated.

1. awk

2. awk
• ·
Peter Weinberger Brian Kernighan
•

3. awk
• sed , awk
.

4. awk
•
•
•
•

5. • last
chenggan pts/467 192.168.130.29 Thu Aug 18 15:39 still logged in
chenggan pts/446 192.168.130.29 Thu Aug 18 15:38 still logged in
chenggan pts/414 192.168.130.29 Thu Aug 18 15:33 still logged in
chenggan pts/414 192.168.130.29 Thu Aug 18 15:26 - 15:32 (00:06)
lixinhe pts/413 192.168.130.34 Thu Aug 18 15:25 still logged in
lixinhe pts/521 192.168.130.34 Thu Aug 18 15:09 still logged in
xuzhang pts/509 192.168.129.2 Thu Aug 18 15:03 still logged in

6. • ip

7. • last | awk ‘{printf $1, $3}’
chenggan 192.168.130.29
chenggan 192.168.130.29
chenggan 192.168.130.29
chenggan 192.168.130.29
lixinhe 192.168.130.34
lixinhe 192.168.130.34
xuzhang 192.168.129.2

8. awk
• ,
$0,$1,$2... .
• Pattern { Actions }
Action
• { Actions }
• ,

9. chenggan pts/467 192.168.130.29 Thu Aug 18 15:39 still logged in
$0 = chenggan pts/467 192.168.130.29 Thu Aug 18 15:39 still logged in
$1 = chenggan
$2 = pts/467
$3 = 192.168.130.29
$4 = Thu
$5 = Aug
Pattern . Action
print $1, $3 $1 $3

10. awk
• FILENAME
• FS ,
• NF ,
• OFS
• FNR

11. • cat /etc/passwd
beanstalkd:x:107:110:beanstalkd user:/var/lib/beanstalkd:/bin/false
postfix:x:89:89::/var/spool/postfix:/sbin/nologin
privoxy:x:73:73::/etc/privoxy:/sbin/nologin
_tor:x:108:514:Tor user:/var/lib/tor:/bin/false
murongjun:x:518:501::/home/murongjun:/bin/bash
liying:x:519:501::/home/liying:/bin/bash

12. •
• ?

13. • cat /etc/passwd | awk '{ FS=":"; print $1, $6;}'
beanstalkd /var/lib/beanstalkd
postfix /var/spool/postfix
privoxy /etc/privoxy
_tor /var/lib/tor
murongjun /home/murongjun
liying /home/liying

14. • cat /etc/passwd | awk '{ FS=":"; OFS = " || "; print $1, $6;}'
beanstalkd || /var/lib/beanstalkd
postfix || /var/spool/postfix
privoxy || /etc/privoxy
_tor || /var/lib/tor
murongjun || /home/murongjun
liying || /home/liying

15. • ,
, id, , .
cityid.
• php ,
sql .
• php 20 ,awk 1

16. 107255,"2011-06-17"," ",15201095922,0
663805,"2011-07-18"," ",13521050527,0
346569,"2011-05-31"," ",84920575,84920575
268831,"2011-05-29"," ",13601180538,0
551017,"2011-04-20"," ",84920575,84855168

17. • awk -F "," '{ printf "insert into dealrevisit values
(%s,%s,%s,10,0,UNIX_TIMESTAMP());n", $1, $3,
$4;}' beijing > insert.sql

18. insert into dealrevisit value (967898," ",13681190863,10,0,UNIX_TIMESTAMP());
insert into dealrevisit value (582907," ",13901316805,10,0,UNIX_TIMESTAMP());
insert into dealrevisit value (734974," ",15010372631,10,0,UNIX_TIMESTAMP());
insert into dealrevisit value (868768," ",18600011579,10,0,UNIX_TIMESTAMP());

19. • FS ,FS ,
.
• FS AWK

20. Pattern { Action }
• Pattern { Action } AWK
.
• Pattern,
Action

21. Pattern { Action }
• AWK Pattern
• awk ‘BEGIN{Action}; Pattern{Action};Pattern {Action};...; END{Action}’ file

22. BEGIN AND END
• BEGIN AND Pattern
• BEGIN
• END

23. ......
create mode 100644 mis/template/mobileads/list.php
create mode 100644 model/MobileadsModel.php
create mode 100644 model/TpBankBranchModel.php
create mode 100644 model/TpBankInfoModel.php
create mode 100644 model/TpPaymentLogModel.php
create mode 100644 sql/110818_moddeallog.sql
<em:version>0.1.3</em:version>
......

24. • 0.1.3

25. • <em:version>...

26. • awk '$0 ~ /<em:version>[^<]*</em:version>/ {print $0;}' install.rdf |awk -F
'>' '{print $2}'|awk -F '<' '{print $1}'

27. • $0 ~ /<em:version>[^<]*</em:version>/
Pattern.
• $0 , /
<em:version>[^<]*</em:version>/
• {print $0;} Action

28. • awk <em:version>0.1.3</
em:version>
• awk > 0.1.3</
em:version
• awk < 0.1.3

29. Pattern { Action }
• AWK

30. 202.106.58.18 - - [15/Aug/2011:23:59:42 +0800] 735802 -
"GET /acl/account/checkurl?url=/mis/deals/default/all/
jining&citySlug=jilin HTTP/1.0" 302 - "http://
www.meituan.com/mis/" ....
221.223.96.234 - - [15/Aug/2011:23:59:58 +0800] 450858 -
"GET /deal/1105.html HTTP/1.0" 200 2132 "http://
www.meituan.com/mis/account/login"

31. • apache
• url , url
url
• url mis
• grep , grep

32. • awk ' $9 ~ /mis/ { print $0}' log

33. • awk awk B

34. Pattern { Action }
• AWK Pattern >,<
• awk '$1 + $2 < 100' test
100 Pattern
Action, Action
• AWK Pattern !~ .

35. awk
• awk shell
, awk

36. 3199, , ,187****9445, BD ( )
3228, , ,136****9587, BD ( )
2962, , ,136****7116, BD ( )
2963, , ,137****0897, BD ( )
5554, , ,186****2154,
4838, , ,134****7617, BD ( )
4882, , ,139****5562, BD ( )

37. • .

38. • php , 20 ,awk 1

39. • awk 'BEGIN {FS=","} { print $0 >>$2; close($2);}'
cityuser

40. • , .
$2 .
• close
• awk
close()

41. awk
• getline .
• getline ,
.
• ,getline 1,
0, ,
, -1.

42. • awk 'BEGIN { while("who" | getline) n++;
print n;}'
• .

43. • awk 'BEGIN {"date" | getline dtime; print dtime}'
•
• data dtime,

44. 202.106.58.18 - - [17/Aug/2011:23:59:51 +0800] 1042839 - "GET /mis/deals/default/all/ningbo HTTP/1.0" 200 17505 "http://
www.meituan.com/mis/frame/side?city=ningbo" "Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0"
"uuid=13ce7442b68505d69df7.1297390972.68.4.1; __utma=1.64891309.1297390974.1313584928.1313594207.499;
__utmz=1.1313594207.499.164.utmcsr=%28none%29|utmccn=UserReferral|utmcmd=im|utmctr=index|utmcct=u313552;
mb=15210890172; ci=135; __utmv=1.|1=city=yuncheng=1,2=user=BD=1; abt=1313544681%7CADF; m=chuan_ye_lony
%40163.com; al=HRPAxN_gimR2EVusR69-AhDTM2t0V-Z7ENMza3RX5nvXmpHDt8V-XQ;
SID=vp8csbhtgc1g42f3villuod0g7; u=2506429; n=%E7%82%B9%E7%82%B9%E5%B0%8F%E9%9B%A8%E5%B0%98; __utmc=1;
au=2367; MisCitySlug=ningbo; em=Tjs; rf=1313584922-4752116b-313552; __utmb=1.87.9.1313594355400; om=Tjs"
www.meituan.com

45. • apache ,
mis , au,
cookie acluserid .
au url.

46. • ,
, au id
url
•
,

47. • cat www.meituan.com-110810-access_log | grep '/mis' | grep 'au=' |
awk '{ FS="""; printf("%sau=%sn"),$2, $0;}' | awk '{FS="au=";
printf("%s %sn"), $1,$3;}'| awk '{FS="[;”]"; print $1}' | awk '{print
$2,$1}' |awk '{FS="?"; print $1}'| awk '{FS="/"; for(i=1;i<=NF;i++)
{printf("%s/"),$i; if (i==4) {break;}};printf("n");}' | sed "s_//_/_g" > gui

48. • awk ” GET /mis/
deals/default/all/ningbo HTTP/1.0au=
• awk ”au=” GET /
mis/deals/default/all/ningbo HTTP/1.0
2367; MisCitySlug=ningbo; em=Tjs;

49. • [;”] GET /mis/
deals/default/all/ningbo HTTP/1.0 2367
•

50. ,3349,,1422,/mis/frame/toggle/ || /mis/frame/side/ || /
mtm/ || /mtm/index/list/ || /mis/deals/default/ || /acl/
account/checkurl/ || /mis/deal/edit/
,2435,,974,/mis/frame/toggle/ || /mis/frame/side/
|| /mtm/ || /mtm/index/list/ || /acl/account/checkurl/ || /
mtm/index/ || /mis/deals/default/

51. grep in awk
• ,
50 , id ip,
.
• gui id
apache log .

52. grep in awk
• awk
grep , grep
. $0
grep

53. grep in awk
• awk 'BEGIN { while (( getline < "/var/
sankuai/wwwlogs/
www.meituan.com-110816-access_log" >
0)) { f2[lc] = $0; lc++;}} -F "," { name =
"au="$2"[;|"]"; for (i = 1; i < lc; i++) {if (f2[i]
~ name) print f2[i];} printf("nn");}'
zhuagui.ouput > zhuagui50

54. grep in awk
• getline log
f2 . BEGIN Pattern
.
• gui f2
.
• name name =

55. • ,
awk ,

56. ^-^