爬虫相关 #

判断是否搜索引擎爬虫 #

text
server {
    listen 80;
    location / {
        if ($http_user_agent ~* "Baiduspider|360Spider|qihoobot|Googlebot|bingbot|Sogou web spider|Sogou spider") {
            proxy_pass http://127.0.0.1:3000;
            break;
        }
        root html;
        try_files $uri $uri/ /index.html;
    }
}

完全禁止爬虫 #

nginx
location /robots.txt {
    root /data;
}

/data/robots.txt:

txt
Usage-agent: *
Disallow: /