awk Cheatsheet

One-Liners

Use this awk reference while you build software engineering projects, review code, or refresh the syntax you reach for most.

Selecting and filtering

awk 'NF'                          file   # drop blank lines
awk '!/^[[:space:]]*(#|$)/'       file   # drop comments and blanks
awk 'length > 80'                 file   # lines longer than 80 chars
awk 'NR >= 10 && NR <= 20'        file   # lines 10-20
awk 'NR == 42 { print; exit }'    file   # line 42, stop reading
awk 'END { print }'               file   # last line, like tail -1
awk 'END { print NR }'            file   # line count, like wc -l
awk 'NR % 3 == 0'                 file   # every third line
awk '$3 == "ERROR"'               file   # exact field match
awk '$2 + 0 > 1000'               file   # numeric field threshold
awk '/start/ { f = 1 } f; /stop/ { f = 0 }' file   # inclusive block

exit on a match is the difference between reading 42 lines and reading a 10 GB file.

Columns and reshaping

awk '{ print $2, $1 }'                    file   # swap two columns
awk '{ print $NF }'                       file   # last field
awk '{ print $(NF-1) }'                   file   # second to last
# drop column 2 without leaving a doubled separator behind
awk '{ for (i=1;i<=NF;i++) if (i!=2) printf "%s%s", $i, (i<NF?OFS:ORS) }' f
awk -F, 'BEGIN{OFS="\t"} { $1=$1; print }' in.csv  # csv to tsv
awk '{ print NR ": " $0 }'                file   # number every line
awk '{ printf "%s%s", (NR>1 ? "," : ""), $1 } END { print "" }' file  # join
awk 'BEGIN{ RS="[,\n]" } NF { print }' one.csv   # split on commas

Cutting a column is the one job where cut -f2 -d, is both shorter and faster. Reach for awk when you need a condition or a computation with it. Note the RS recipe uses [,\n], not ,, or the file's final newline lands inside the last record.

Arithmetic and stats

# sum column 2
awk '{ s += $2 } END { print s+0 }' data.txt

# mean, guarding the empty-input divide
awk '{ s += $2; n++ } END { if (n) printf "%.4f\n", s/n }' data.txt

# min, max, and count in one pass
awk 'NR == 1 { lo = hi = $2 }
     { if ($2 < lo) lo = $2; if ($2 > hi) hi = $2; s += $2 }
     END { printf "n=%d min=%s max=%s mean=%.3f\n", NR, lo, hi, s/NR }' data.txt

# running total as a new column
awk '{ s += $2; print $0, s }' data.txt

# sum per key
awk '{ s[$1] += $2 } END { for (k in s) printf "%s\t%.2f\n", k, s[k] }' data.txt

# percentage of a total, two passes over one file
awk 'NR==FNR { t+=$2; next } { printf "%s %.1f%%\n", $1, 100*$2/t }' d.txt d.txt

s+0 in the first recipe prints 0 rather than an empty line when the file has no rows, since an unset variable stringifies to "".

Counting and deduping

awk '!seen[$0]++'                       file   # dedupe, keep first, no sort
awk 'seen[$0]++'                        file   # print only the duplicates
awk '!seen[$1]++'                       file   # dedupe by first field
awk '{ c[$1]++ } END { for (k in c) print c[k], k }' file | sort -rn | head
awk '{ c[$0]++ } END { for (k in c) if (c[k] > 1) print c[k], k }' file
awk 'END { print NR }' file; awk '!x[$0]++ { u++ } END { print u }' file

A sorted frequency report, the thing you actually want from a log:

awk '{ c[$1]++ }
     END { for (k in c) printf "%7d  %s\n", c[k], k }' access.log |
  sort -rn | head -20

!seen[$0]++ holds every distinct line in memory. On a file bigger than RAM use sort -u instead, which spills to disk.

Joining two files

The NR == FNR test is true only while awk is reading the first file, because FNR restarts per file and NR does not.

# keep rows of data.txt whose key appears in allow.txt
awk 'NR == FNR { k[$1]; next } $1 in k' allow.txt data.txt

# the inverse, an anti-join
awk 'NR == FNR { k[$1]; next } !($1 in k)' deny.txt data.txt

# add a column looked up from a mapping file
awk 'NR == FNR { name[$1] = $2; next }
     { print $0, ($1 in name ? name[$1] : "UNKNOWN") }' users.tsv events.tsv

# lines present in both files
awk 'NR == FNR { a[$0]; next } $0 in a' f1 f2

NR == FNR breaks if the first file is empty. FNR restarts at 1 for the second file while NR is still 1, so awk treats the second file as if it were the first, fills the lookup table with it, and prints nothing at all. Guard with FILENAME == ARGV[1] when an empty input is possible.

Transposing and multi-line records

# transpose a whitespace table
awk '{ for (i = 1; i <= NF; i++) m[i, NR] = $i; if (NF > w) w = NF }
     END { for (i = 1; i <= w; i++) {
             row = ""
             for (j = 1; j <= NR; j++) row = row (j > 1 ? OFS : "") m[i, j]
             print row } }' table.txt

# reverse the line order, like tac
awk '{ a[NR] = $0 } END { for (i = NR; i >= 1; i--) print a[i] }' file

# join every pair of lines
awk 'NR % 2 { printf "%s ", $0; next } { print }' file

# paragraph records, print the first line of each
awk 'BEGIN{ RS = ""; FS = "\n" } { print $1 }' notes.txt

CSV, and why awk is the wrong tool for it

-F, splits on every comma, including the ones inside quotes.

# WRONG on a,"b,c",d  ->  NF is 4, $2 is "b
awk -F, '{ print $2 }' quoted.csv

# gawk only: FPAT describes what a field IS, so quotes survive
gawk -v FPAT='([^,]*)|("[^"]*")' '{ print NF, $2 }' quoted.csv
# 3 "b,c"

# gawk 5 has a real CSV mode
gawk --csv '{ print $2 }' quoted.csv

macOS awk and mawk have neither FPAT nor --csv. For real CSV with quotes, embedded newlines, or escaped quotes, use a CSV-aware tool (`csvkit`, `xsv`, python -c 'import csv') instead of building a regex.

Text munging

awk '{ gsub(/[ \t]+$/, ""); print }'        file   # strip trailing whitespace
awk '{ gsub(/^[ \t]+|[ \t]+$/, ""); print }' file  # trim both ends
awk '{ $1 = $1; print }'                    file   # squeeze internal whitespace
awk '{ print toupper($0) }'                 file   # uppercase
awk '{ print tolower($1), $2 }'             file   # lowercase one field
awk '{ gsub(/foo/, "bar"); print }'         file   # global replace
awk 'BEGIN{ while ((getline l < "a.txt") > 0) print l }'   # cat, the hard way
awk '{ printf "%s", $0 } END { print "" }'  file   # strip all newlines
awk 'BEGIN{ ORS = "" } { print; print (NR%2 ? "\t" : "\n") }' file  # 2 per line

Every one of these runs on macOS awk with no flags. Nothing above needs gawk except the rows marked gawk.