Nginx限制搜索引擎爬虫频率、禁止屏蔽网络爬虫配置示例

        proxy_buffers           4 32k; 
        proxy_busy_buffers_size 64k; 
        proxy_temp_file_write_size 64k; 
    } 
     
        #error_page  404              /404.html; 
 
        # redirect server error pages to the static page /50x.html 
        # 
        error_page   500 502 503 504  /50x.html; 
        location = /50x.html { 
            root   html; 
        } 
 
        # proxy the PHP scripts to Apache listening on 127.0.0.1:80 
        # 
        #location ~ .php$ { 
        #    proxy_pass   http://127.0.0.1; 
        #} 
 
        # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000 
        # 
        #location ~ .php$ { 
        #    root           html; 
        #    fastcgi_pass   127.0.0.1:9000; 
        #    fastcgi_index  index.php; 
        #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name; 
        #    include        fastcgi_params; 
        #} 
 
        # deny access to .htaccess files, if Apache's document root 
        # concurs with nginx's one 
        # 
        #location ~ /.ht { 
        #    deny  all; 
        #} 
    } 

可以用 curl 测试一下