Apache日誌分析與伺服器管理

$ wc -l access_log
7836 access_log
$ head -1 access_log ; tail -1 access_log
69.195.124.69 - - [29/Jun/2016:03:35:37 +0000] ...
65.55.219.126 - - [04/Jul/2016:14:07:23 +0000] ...

#!/bin/bash
# webaccess--分析Apache格式的access_log檔案，提取有用且有趣的統計資訊
bytes_in_gb=1048576

# 您需要更改以下內容以符合您自己的主機名，以幫助篩選出內部參照的點選。
host="intuitive.com"

if [ $# -eq 0 ] ; then
  echo "用法：$(basename $0) 日誌檔" >&2
  exit 1
fi

if [ ! -r "$1" ] ; then
  echo "錯誤：日誌檔 $1 未找到。" >&2
  exit 1
fi

firstdate="$(head -1 "$1" | awk '{print $4}' | sed 's/\[//')"
lastdate="$(tail -1 "$1" | awk '{print $4}' | sed 's/\[//')"

echo "分析日誌檔 $1 的結果"
echo ""
echo " 起始日期：$(echo $firstdate|sed 's/:/ 在 /')"
echo " 結束日期：$(echo $lastdate|sed 's/:/ 在 /')"
hits="$(wc -l < "$1" | sed 's/[^[:digit:]]//g')"
echo " 點選次數：$(nicenumber $hits) (總存取量)"
pages="$(grep -ivE '(.gif|.jpg|.png)' "$1" | wc -l | sed 's/[^[:digit:]]//g')"
echo " 頁面瀏覽量：$(nicenumber $pages) (點選次數減去圖形檔案)"
totalbytes="$(awk '{sum+=$10} END {print sum}' "$1")"
/bin/echo -n " 已傳輸：$(nicenumber $totalbytes) 位元組 "
if [ $totalbytes -gt $bytes_in_gb ] ; then
  echo "($(/usr/local/bin/scriptbc $totalbytes / $bytes_in_gb) GB)"
elif [ $totalbytes -gt 1024 ] ; then
  echo "($(/usr/local/bin/scriptbc $totalbytes / 1024) MB)"
else
  echo ""
fi

# 現在讓我們從日誌檔中抓取一些有用的資料。
echo ""
echo "前10個最受歡迎的頁面是："
awk '{print $7}' "$1" | grep -ivE '(.gif|.jpg|.png)' | sed 's/\/$//g' | sort | uniq -c | sort -rn | head -10
echo ""
echo "前10個最常見的參照者URL是："
awk '{print $11}' "$1" | grep -vE "(^\"-\"$|/www.$host|/$host)" | sort | uniq -c | sort -rn | head -10
echo ""
exit 0

#!/bin/bash
# searchinfo--Extracts and analyzes search engine traffic indicated in the
# referrer field of a Common Log Format access log
host="intuitive.com" # 變更為你的網域名稱
maxmatches=20
count=0
temp="/tmp/$(basename $0).$$"
trap "$(which rm) -f $temp" 0
if [ $# -eq 0 ] ; then
  echo "Usage: $(basename $0) logfile" >&2
  exit 1
fi
if [ ! -r "$1" ] ; then
  echo "Error: can't open file $1 for analysis." >&2
  exit 1
fi

for URL in $(awk '{ if (length($11) > 4) { print $11 } }' "$1" | \
             grep -vE "(/www.$host|/$host)" | grep '?')
do
  searchengine="$(echo $URL | cut -d/ -f3 | rev | cut -d. -f1-2 | rev)"
  args="$(echo $URL | cut -d\? -f2 | tr '&' '\n' | \
          grep -E '(^q=|^sid=|^p=|query=|item=|ask=|name=|topic=)' | \
          sed -e 's/+/ /g' -e 's/%20/ /g' -e 's/"//g' | cut -d= -f2)"
  if [ ! -z "$args" ] ; then
    echo "${searchengine}: $args" >> $temp
  else
    # 若非已知格式，則顯示整個 GET 字串
    echo "${searchengine} $(echo $URL | cut -d\? -f2)" >> $temp
  fi
  count="$(( $count + 1 ))"
done

echo "Search engine referrer info extracted from ${1}:"
sort $temp | uniq -c | sort -rn | head -$maxmatches | sed 's/^/ /g'
echo ""
echo Scanned $count entries in log file out of $(wc -l < "$1") total.
exit 0

$ head -1 error_log
[Mon Jun 06 08:08:35 2016] [error] [client 54.204.131.75] File does not exist: /var/www/vhosts/default/htdocs/clientaccesspolicy.xml

$ awk '{print $9" "$10" "$11" "$12 }' error_log | sort -u
File does not exist:
Invalid error redirection directive:
Premature end of script
execution failure for parameter
premature EOF in parsed
script not found or
malformed header from script

#!/bin/bash
# weberrors--掃描Apache錯誤日誌，報告最重要的錯誤，然後列出其他條目
temp="/tmp/$(basename $0).$$"

# 為了使指令碼更好地工作，請根據您的安裝自定義以下三行。
htdocs="/usr/local/etc/httpd/htdocs/"
myhome="/usr/home/taylor/"
cgibin="/usr/local/etc/httpd/cgi-bin/"

sedstr="s/^/ /g;s|$htdocs|[htdocs] |;s|$myhome|[homedir] "
sedstr=$sedstr"|;s|$cgibin|[cgi-bin] "
screen="(File does not exist|Invalid error redirect|premature EOF"
screen=$screen"|Premature end of script|script not found)"
length=5 # 每類別顯示的條目數

checkfor()
{
    grep "${2}:" "$1" | awk '{print $NF}' \
    | sort | uniq -c | sort -rn | head -$length | sed "$sedstr" > $temp
    if [ $(wc -l < $temp) -gt 0 ] ; then
        echo ""
        echo "$2 errors:"
        cat $temp
    fi
}

trap "$(which rm) -f $temp" 0

if [ "$1" = "-l" ] ; then
    length=$2; shift 2
fi

if [ $# -ne 1 -o ! -r "$1" ] ; then
    echo "用法: $(basename $0) [-l len] error_log" >&2
    exit 1
fi

echo "輸入檔案 $1 有 $(wc -l < "$1") 個條目。"
start="$(grep -E '\[.*:.*:.*\]' "$1" | head -1 \
| awk '{print $1" "$2" "$3" "$4" "$5 }')"
end="$(grep -E '\[.*:.*:.*\]' "$1" | tail -1 \
| awk '{print $1" "$2" "$3" "$4" "$5 }')"

checkfor "$1" "File does not exist"
checkfor "$1" "Invalid error redirection directive"
checkfor "$1" "Premature EOF"
checkfor "$1" "Script not found or unable to stat"
checkfor "$1" "Premature end of script headers"

X grep -vE "$screen" "$1" | grep "\[error\]" | grep "\[client " \
| sed 's/\[error\]/\`/' | cut -d\` -f2 | cut -d\ -f4- \
Y | sort | uniq -c | sort -rn | sed 's/^/ /' | head -$length > $temp

outfile="/tmp/rb.$$.tgz"
outfname="backup.$(date +%y%m%d).tgz"
infile="/tmp/rb.$$.in"
trap "$(which rm) -f $outfile $infile" 0

X while read entry; do
echo "$entry" | sed -e 's/ /\\ /g' >> $infile
done < "$1"

Y tar czf - $(cat $infile) | \
uuencode $outfname | \
mail -s "${3:-Backup archive for $(date)}" "$2"