SEO Log File Analysis: Uncover How Crawlers See Your Site

# Apache access.log entry
66.249.66.1 - - [01/Feb/2025:10:32:15 +0000] "GET /page HTTP/1.1" 200 15234 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"

# Breaking down the log entry:
66.249.66.1                    # IP address (Googlebot)
-                              # Remote logname (unused)
-                              # Remote user (unused)
[01/Feb/2025:10:32:15 +0000]   # Timestamp
"GET /page HTTP/1.1"           # HTTP method, path, protocol
200                            # Status code
15234                          # Response size (bytes)
"-"                            # Referer
"Mozilla/5.0..."               # User-Agent (identifies crawler)

# Googlebot (main web crawler)
Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)

# Googlebot Mobile
Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)

# Bingbot
Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)

# DuckDuckBot
DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)

# Yandex
Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)

# Baiduspider
Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)

# Extract Googlebot requests
grep "Googlebot" access.log

# Extract all major search engine bots
grep -E "Googlebot|bingbot|Slurp|DuckDuckBot|Baiduspider|YandexBot" access.log

# Count requests per bot
grep -oE "Googlebot|bingbot|Slurp|DuckDuckBot" access.log | sort | uniq -c | sort -rn

# Extract Googlebot requests for specific date
grep "01/Feb/2025" access.log | grep "Googlebot"

# Get hourly Googlebot activity
grep "Googlebot" access.log | awk '{print $4}' | cut -d: -f2 | sort | uniq -c

# Daily Googlebot crawl rate
grep "Googlebot" access.log | awk '{print $4}' | cut -d: -f1 | cut -d[ -f2 | sort | uniq -c

# Hourly breakdown
grep "Googlebot" access.log | awk '{print $4}' | cut -d: -f1-2 | sort | uniq -c

# Output example:
#  1247 01/Feb/2025:00
#  1523 01/Feb/2025:01
#  1689 01/Feb/2025:02
#   892 01/Feb/2025:03

# Visualize crawl patterns
# High activity = good (if server handles it)
# Sudden drops = potential issues

# Top 20 pages crawled by Googlebot
grep "Googlebot" access.log | awk '{print $7}' | sort | uniq -c | sort -rn | head -20

# Example output:
#  523 /products/
#  412 /
#  289 /products/category-1/
#  201 /blog/
#  156 /products/item-123/

# Analyze results:
# - Are important pages being crawled?
# - Are low-value pages wasting crawl budget?
# - Are infinite scroll pages being over-crawled?

# Count status codes for Googlebot
grep "Googlebot" access.log | awk '{print $9}' | sort | uniq -c | sort -rn

# Example output:
#  5234 200  # Success
#   423 304  # Not Modified (good, efficient)
#   156 301  # Permanent Redirect
#    89 404  # Not Found (investigate!)
#    34 500  # Server Error (critical!)
#    12 503  # Service Unavailable (critical!)

# Alarm thresholds:
# 4xx errors > 5% = content issues
# 5xx errors > 1% = server issues

# Apache with %D (microseconds) or %T (seconds) in log format
# Requires custom log format

# Custom Apache log format:
LogFormat "%h %l %u %t "%r" %>s %b %D" combined_with_time

# Extract slow pages (>1 second = 1000000 microseconds)
awk '$NF > 1000000 {print $7, $NF/1000000 "s"}' access.log | sort -k2 -rn

# Average response time per page
awk '{url=$7; time=$NF; sum[url]+=time; count[url]++} END {for(url in sum) print url, sum[url]/count[url]/1000000 "s"}' access.log | sort -k2 -rn

# Find URLs with parameters (often duplicate content)
grep "Googlebot" access.log | awk '{print $7}' | grep "?" | sort | uniq -c | sort -rn

# Find crawled admin pages (should be blocked)
grep "Googlebot" access.log | grep -E "/admin/|/wp-admin/|/login" | wc -l

# Find crawled search result pages
grep "Googlebot" access.log | grep -E "/search?|/results?" | wc -l

# Calculate waste percentage
total=$(grep "Googlebot" access.log | wc -l)
waste=$(grep "Googlebot" access.log | grep -E "?|/admin/|/search?" | wc -l)
echo "Crawl budget waste: $(($waste * 100 / $total))%"

#!/bin/bash
# Calculate crawl efficiency

LOG_FILE="access.log"

# Total Googlebot requests
total=$(grep "Googlebot" $LOG_FILE | wc -l)

# Successful requests (200, 304)
successful=$(grep "Googlebot" $LOG_FILE | awk '$9 == 200 || $9 == 304' | wc -l)

# Unique URLs crawled
unique=$(grep "Googlebot" $LOG_FILE | awk '{print $7}' | sort -u | wc -l)

# Error requests
errors=$(grep "Googlebot" $LOG_FILE | awk '$9 >= 400' | wc -l)

# Calculate metrics
success_rate=$(($successful * 100 / $total))
error_rate=$(($errors * 100 / $total))
unique_rate=$(($unique * 100 / $total))

echo "Total requests: $total"
echo "Unique URLs: $unique"
echo "Success rate: $success_rate%"
echo "Error rate: $error_rate%"
echo "Unique crawl rate: $unique_rate%"

# Efficiency score (higher = better)
efficiency=$((success_rate - error_rate - (100 - unique_rate)))
echo "Efficiency score: $efficiency"

# Find pages in sitemap but not in logs (potential orphans)
# 1. Extract URLs from sitemap
curl -s https://example.com/sitemap.xml | grep -oP '(?<=<loc>)[^<]+' > sitemap-urls.txt

# 2. Extract URLs from logs
grep "Googlebot" access.log | awk '{print $7}' | sort -u > crawled-urls.txt

# 3. Find differences
comm -23 sitemap-urls.txt crawled-urls.txt > orphan-candidates.txt

# These pages are in sitemap but not crawled
# Reasons:
# - Not linked internally
# - Deep in site hierarchy
# - Recently added
# - Low priority

# Analyze crawl frequency for different page types
grep "Googlebot" access.log | awk '{print $7}' | awk -F/ '{
  if ($2 == "") print "Homepage"
  else if ($2 == "products") print "Products"
  else if ($2 == "blog") print "Blog"
  else if ($2 == "category") print "Category"
  else print "Other"
}' | sort | uniq -c | sort -rn

# Example output:
#  2341 Products
#  1523 Blog
#   892 Category
#   456 Homepage
#   234 Other

# Insights:
# - Are important pages crawled frequently?
# - Are low-value pages over-crawled?
# - Does frequency match business priority?

# Extract specific fields
awk '{print $1, $7, $9}' access.log

# Filter and count
awk '$9 == 404 {print $7}' access.log | sort | uniq -c | sort -rn

# Calculate averages
awk '{sum+=$10; count++} END {print sum/count}' access.log

# Time-based filtering
awk '$4 ~ /01/Feb/2025:1[0-2]:/ {print $7}' access.log

import re
from collections import Counter, defaultdict
from datetime import datetime
import pandas as pd

def parse_log_line(line):
  """Parse Apache/Nginx log line"""
  pattern = r'(S+) S+ S+ [(.*?)] "(S+) (S+) S+" (d+) (d+) "(.*?)" "(.*?)"'
  match = re.match(pattern, line)
  
  if match:
      return {
          'ip': match.group(1),
          'timestamp': match.group(2),
          'method': match.group(3),
          'url': match.group(4),
          'status': int(match.group(5)),
          'size': int(match.group(6)),
          'referer': match.group(7),
          'user_agent': match.group(8)
      }
  return None

def is_search_bot(user_agent):
  """Identify search engine bots"""
  bots = ['Googlebot', 'bingbot', 'Slurp', 'DuckDuckBot', 'Baiduspider', 'YandexBot']
  return any(bot in user_agent for bot in bots)

def analyze_logs(log_file):
  """Comprehensive log analysis"""
  bot_requests = []
  url_counts = Counter()
  status_counts = Counter()
  hourly_activity = defaultdict(int)
  
  with open(log_file, 'r') as f:
      for line in f:
          data = parse_log_line(line)
          if not data:
              continue
          
          if is_search_bot(data['user_agent']):
              bot_requests.append(data)
              url_counts[data['url']] += 1
              status_counts[data['status']] += 1
              
              # Extract hour
              dt = datetime.strptime(data['timestamp'], '%d/%b/%Y:%H:%M:%S %z')
              hour = dt.strftime('%Y-%m-%d %H:00')
              hourly_activity[hour] += 1
  
  # Generate report
  print(f"Total bot requests: {len(bot_requests)}")
  print(f"\nTop 10 crawled URLs:")
  for url, count in url_counts.most_common(10):
      print(f"  {count:5d} {url}")
  
  print(f"\nStatus code distribution:")
  for status, count in sorted(status_counts.items()):
      print(f"  {status}: {count:5d} ({count*100/len(bot_requests):.1f}%)")
  
  print(f"\nPeak crawl hours:")
  for hour, count in sorted(hourly_activity.items(), key=lambda x: x[1], reverse=True)[:5]:
      print(f"  {hour}: {count} requests")
  
  return bot_requests, url_counts, status_counts, hourly_activity

# Usage
if __name__ == '__main__':
  analyze_logs('access.log')

# Create prioritized fix list
cat access.log | grep "Googlebot" | awk '{print $7, $9}' | awk '$2 != 200 && $2 != 304 {print $1, $2}' | sort | uniq -c | sort -rn > crawl-waste-report.txt

# Fix priority:
# 1. High-traffic 404s (content gaps)
# 2. 500 errors (server issues)
# 3. Redirect chains (efficiency)
# 4. Parameter-heavy URLs (duplicate content)
# 5. Low-value pages (robots.txt blocking)

#!/bin/bash
# Compare crawl rates over time

current_week=$(grep "Googlebot" access.log | awk '$4 >= "[01/Feb/2025" && $4 < "[08/Feb/2025"' | wc -l)

previous_week=$(grep "Googlebot" access.log | awk '$4 >= "[25/Jan/2025" && $4 < "[01/Feb/2025"' | wc -l)

change=$(echo "scale=2; ($current_week - $previous_week) * 100 / $previous_week" | bc)

echo "Current week: $current_week requests"
echo "Previous week: $previous_week requests"
echo "Change: $change%"

if (( $(echo "$change < -20" | bc -l) )); then
echo "⚠️ WARNING: Crawl rate dropped significantly!"
echo "Possible causes:"
echo "- Server performance issues"
echo "- robots.txt changes"
echo "- Site structure changes"
echo "- Algorithm update"
fi

# Compare regular Googlebot vs Googlebot rendering
regular=$(grep "Googlebot/2.1" access.log | wc -l)
rendering=$(grep "Chrome.*Googlebot" access.log | wc -l)

render_ratio=$(echo "scale=2; $rendering * 100 / $regular" | bc)

echo "Regular Googlebot: $regular"
echo "Rendering Googlebot: $rendering"
echo "Render ratio: $render_ratio%"

# High render ratio (>30%) indicates:
# - JavaScript-heavy site
# - Potential rendering delays
# - Higher crawl budget consumption

if (( $(echo "$render_ratio > 30" | bc -l) )); then
echo "⚠️ High rendering ratio - consider SSR/SSG"
fi

#!/bin/bash
# Real-time log monitoring

tail -f access.log | grep --line-buffered "Googlebot" | while read line; do
  status=$(echo $line | awk '{print $9}')
  url=$(echo $line | awk '{print $7}')
  
  # Alert on errors
  if [ $status -ge 500 ]; then
    echo "🚨 SERVER ERROR: $status on $url"
    # Send notification (email, Slack, etc.)
  fi
  
  if [ $status -eq 404 ]; then
    echo "⚠️ NOT FOUND: $url"
  fi
  
  echo "Googlebot: $status $url"
done

#!/bin/bash
# Daily log analysis report

DATE=$(date +%Y-%m-%d)
YESTERDAY=$(date -d "yesterday" +%d/%b/%Y)
REPORT="daily-crawl-report-$DATE.txt"

{
echo "=== Crawl Report for $YESTERDAY ==="
echo ""

echo "Total Requests:"
grep "$YESTERDAY" access.log | grep "Googlebot" | wc -l

echo ""
echo "Status Code Distribution:"
grep "$YESTERDAY" access.log | grep "Googlebot" |   awk '{print $9}' | sort | uniq -c | sort -rn

echo ""
echo "Top 10 Crawled Pages:"
grep "$YESTERDAY" access.log | grep "Googlebot" |   awk '{print $7}' | sort | uniq -c | sort -rn | head -10

echo ""
echo "Errors (4xx, 5xx):"
grep "$YESTERDAY" access.log | grep "Googlebot" |   awk '$9 >= 400 {print $7, $9}' | sort | uniq -c | sort -rn | head -10
  
} > $REPORT

# Email report
mail -s "Daily Crawl Report - $DATE" admin@example.com < $REPORT

# Rotate logs to prevent disk space issues
# /etc/logrotate.d/apache2

/var/log/apache2/*.log {
  daily
  rotate 30        # Keep 30 days
  compress         # Compress old logs
  delaycompress    # Don't compress latest
  notifempty       # Skip empty logs
  create 640 root adm
  sharedscripts
  postrotate
      systemctl reload apache2 > /dev/null
  endscript
}

# For SEO analysis, keep logs for:
# - 30 days: Active monitoring
# - 90 days: Trend analysis
# - 1 year: Historical comparison

# For large log files (>1GB), use faster tools

# Use zgrep for compressed logs
zgrep "Googlebot" access.log.gz

# Parallel processing with GNU parallel
cat access.log | parallel --pipe --block 10M 'grep "Googlebot" | wc -l'

# Sample large logs
awk 'NR % 10 == 0' huge-access.log > sampled.log

# Use specific date ranges
awk '$4 >= "[01/Feb/2025" && $4 < "[02/Feb/2025"' access.log

# Anonymize IP addresses for GDPR compliance
# Apache mod_remoteip + custom logging

# Anonymize IPs in logs
awk '{$1="XXX.XXX.XXX.XXX"; print}' access.log > anonymized.log

# Or hash IPs
awk '{$1=substr(sha256($1),1,16); print}' access.log

# Remove sensitive parameters
sed 's/(email|token|password)=[^&]*//g' access.log