AWK Demystified - Text Processing Essentials

# Basic structure: 'pattern { action }'
# Prints every line (default action if none specified)
awk '{ print }' data.txt

# Prints lines containing "error"
awk '/error/ { print }' log.txt

# BEGIN block: executed once before any input is read
# END block: executed once after all input is processed
awk 'BEGIN { print "--- Log Analysis Start ---" } /FAIL/ { count++ } END { print "Total failures:", count }' system.log

# Print the second and first fields, separated by a comma
awk '{ print $2, $1 }' names.txt

# Print the last field of each line
awk '{ print $NF }' data.csv

# Change field separator to comma using -F option
awk -F',' '{ print $1, $3 }' data.csv

# Change field separator within the script (BEGIN block is ideal)
awk 'BEGIN { FS=":"; OFS=" -> " } { print $1, $3 }' /etc/passwd

# Print line number and the entire line
awk '{ print NR, $0 }' file.txt

# Process only lines with at least 3 fields
awk 'NF >= 3 { print $1, $NF }' data.log

# If-else statement: check a condition on a field
awk '{
    if ($3 > 100) {
        print "High value:", $0
    } else {
        print "Normal value:", $0
    }
}' metrics.txt

# For loop: iterate through fields
awk '{
    for (i = 1; i <= NF; i++) { # Loop from first field to last
        print "Field", i, ":", $i
    }
}' single_line.txt

# While loop: example for specific conditions (less common than for)
awk '{
    i = 1
    while (i <= NF && length($i) < 5) { # Process fields while length is less than 5
        print "Short field:", $i
        i++
    }
}' words.txt

# 'next' statement: skip remaining actions for current record
awk '/^#/ { next } { print $0 }' config.ini # Skip comment lines

# 'exit' statement: stop processing immediately
awk '/ERROR/ { print "Found error, exiting."; exit } { print $0 }' log.txt

# User-defined function: calculate average
awk '
function average(a, b) { # Define a function 'average'
    return (a + b) / 2
}
{
    avg = average($1, $2) # Call the function
    print $1, $2, avg
}' numbers.txt

# Associative arrays: count occurrences of field values
awk '{
    count[$1]++ # Increment count for the value of the first field
}
END {
    for (item in count) { # Iterate through array keys
        print item, count[item]
    }
}' access.log

# Associative arrays for summing values by key
awk '{
    sum[$1] += $2 # Add second field to sum for key (first field)
}
END {
    for (key in sum) {
        print key, sum[key]
    }
}' sales.csv

# printf for formatted output
awk '{
    printf "Item: %-10s Price: $%5.2f Quantity: %d\n", $1, $2, $3
}' inventory.txt

# Example with padding and precision
# %-10s: left-justified string, 10 chars wide
# %5.2f: float, 5 total width (including decimal), 2 decimal places
# %d: integer

# Example: Extract IP, timestamp, and request from Apache access logs
# Log format: 192.168.1.1 - - [29/Dec/2025:10:00:00 +0000] "GET /index.html HTTP/1.1" 200 1234
awk -F'[][]' '{ # Split by square brackets
    ip = $1; sub(/ - - $/, "", ip) # Clean up IP field
    timestamp = $2 # Timestamp is in second field
    request = $3; sub(/^"|"$ /, "", request) # Clean up request field
    print "IP:", ip, "Time:", timestamp, "Request:", request
}' access.log

# Example: Sum bytes transferred by IP
awk '{
    ip = $1
    bytes = $NF # Last field is bytes transferred
    if (bytes ~ /^[0-9]+$/) { # Ensure bytes is a number
        total_bytes[ip] += bytes
    }
}
END {
    print "--- Bytes Transferred by IP ---"
    for (addr in total_bytes) {
        printf "%-15s %10d bytes\n", addr, total_bytes[addr]
    }
}' access.log

# Pre-filter with grep for massive files to reduce AWK's workload
# This is often faster than AWK's pattern matching on its own for simple regex
grep "specific_pattern" large_file.log | awk '{ print $1, $NF }'

# Use 'next' early to avoid unnecessary processing
# If a line doesn't meet initial criteria, skip it immediately
awk '{
    if ($1 == "SKIP") { next } # Skip lines starting with "SKIP"
    # ... rest of complex processing ...
    print $0
}' data.txt

# Avoid unnecessary regex matching in loops
# Pre-compile regex if possible (gawk specific, `~` operator is fast enough for most)
# For very complex patterns, consider using `match()` and `RSTART`/`RLENGTH`