50%-60% of all internet traffic is bots/spiders. Add this to your .htaccess file if you want to block the bad ones
1. RewriteEngine on
2. # Abuse Agent Blocking
3. RewriteCond %{HTTP_USER_AGENT} ^[Ww]eb[Bb]andit [NC,OR]
4. RewriteCond %{HTTP_USER_AGENT} ^$ [NC,OR]
5. RewriteCond %{HTTP_USER_AGENT} ^Acunetix [NC,OR]
6. RewriteCond %{HTTP_USER_AGENT} ^binlar [NC,OR]
7. RewriteCond %{HTTP_USER_AGENT} ^BlackWidow [NC,OR]
8. RewriteCond %{HTTP_USER_AGENT} ^Bolt\ 0 [NC,OR]
9. RewriteCond %{HTTP_USER_AGENT} ^BOT\ for\ JCE [NC,OR]
10. RewriteCond %{HTTP_USER_AGENT} ^Bot\ mailto:craftbot\@yahoo\.com [NC,OR]
11. RewriteCond %{HTTP_USER_AGENT} ^casper [NC,OR]
12. RewriteCond %{HTTP_USER_AGENT} ^checkprivacy [NC,OR]
13. RewriteCond %{HTTP_USER_AGENT} ^ChinaClaw [NC,OR]
14. RewriteCond %{HTTP_USER_AGENT} ^clshttp [NC,OR]
15. RewriteCond %{HTTP_USER_AGENT} ^cmsworldmap [NC,OR]
16. RewriteCond %{HTTP_USER_AGENT} ^comodo [NC,OR]
17. RewriteCond %{HTTP_USER_AGENT} ^Custo [NC,OR]
18. RewriteCond %{HTTP_USER_AGENT} ^Default\ Browser\ 0 [NC,OR]
19. RewriteCond %{HTTP_USER_AGENT} ^diavol [NC,OR]
20. RewriteCond %{HTTP_USER_AGENT} ^DIIbot [NC,OR]
21. RewriteCond %{HTTP_USER_AGENT} ^DISCo [NC,OR]
22. RewriteCond %{HTTP_USER_AGENT} ^dotbot [NC,OR]
23. RewriteCond %{HTTP_USER_AGENT} ^Download\ Demon [NC,OR]
24. RewriteCond %{HTTP_USER_AGENT} ^eCatch [NC,OR]
25. RewriteCond %{HTTP_USER_AGENT} ^EirGrabber [NC,OR]
26. RewriteCond %{HTTP_USER_AGENT} ^EmailCollector [NC,OR]
27. RewriteCond %{HTTP_USER_AGENT} ^EmailSiphon [NC,OR]
28. RewriteCond %{HTTP_USER_AGENT} ^EmailWolf [NC,OR]
29. RewriteCond %{HTTP_USER_AGENT} ^Express\ WebPictures [NC,OR]
30. RewriteCond %{HTTP_USER_AGENT} ^extract [NC,OR]
31. RewriteCond %{HTTP_USER_AGENT} ^ExtractorPro [NC,OR]
32. RewriteCond %{HTTP_USER_AGENT} ^EyeNetIE [NC,OR]
33. RewriteCond %{HTTP_USER_AGENT} ^feedfinder [NC,OR]
34. RewriteCond %{HTTP_USER_AGENT} ^FHscan [NC,OR]
35. RewriteCond %{HTTP_USER_AGENT} ^FlashGet [NC,OR]
36. RewriteCond %{HTTP_USER_AGENT} ^flicky [NC,OR]
37. RewriteCond %{HTTP_USER_AGENT} ^FunWebProducts [NC,OR]
38. RewriteCond %{HTTP_USER_AGENT} ^g00g1e [NC,OR]
39. RewriteCond %{HTTP_USER_AGENT} ^GetRight [NC,OR]
40. RewriteCond %{HTTP_USER_AGENT} ^GetWeb! [NC,OR]
41. RewriteCond %{HTTP_USER_AGENT} ^Go-Ahead-Got-It [NC,OR]
42. RewriteCond %{HTTP_USER_AGENT} ^Go!Zilla [NC,OR]
43. RewriteCond %{HTTP_USER_AGENT} ^grab [NC,OR]
44. RewriteCond %{HTTP_USER_AGENT} ^GrabNet [NC,OR]
45. RewriteCond %{HTTP_USER_AGENT} ^Grafula [NC,OR]
46. RewriteCond %{HTTP_USER_AGENT} ^harvest [NC,OR]
47. RewriteCond %{HTTP_USER_AGENT} ^HMView [NC,OR]
48. RewriteCond %{HTTP_USER_AGENT} ^ia_archiver [NC,OR]
49. RewriteCond %{HTTP_USER_AGENT} ^Image\ Stripper [NC,OR]
50. RewriteCond %{HTTP_USER_AGENT} ^Image\ Sucker [NC,OR]
51. RewriteCond %{HTTP_USER_AGENT} ^InterGET [NC,OR]
52. RewriteCond %{HTTP_USER_AGENT} ^Internet\ Ninja [NC,OR]
53. RewriteCond %{HTTP_USER_AGENT} ^InternetSeer\.com [NC,OR]
54. RewriteCond %{HTTP_USER_AGENT} ^jakarta [NC,OR]
55. RewriteCond %{HTTP_USER_AGENT} ^Java [NC,OR]
56. RewriteCond %{HTTP_USER_AGENT} ^JetCar [NC,OR]
57. RewriteCond %{HTTP_USER_AGENT} ^JOC\ Web\ Spider [NC,OR]
58. RewriteCond %{HTTP_USER_AGENT} ^kanagawa [NC,OR]
59. RewriteCond %{HTTP_USER_AGENT} ^kmccrew [NC,OR]
60. RewriteCond %{HTTP_USER_AGENT} ^larbin [NC,OR]
61. RewriteCond %{HTTP_USER_AGENT} ^LeechFTP [NC,OR]
62. RewriteCond %{HTTP_USER_AGENT} ^libwww [NC,OR]
63. RewriteCond %{HTTP_USER_AGENT} ^Mass\ Downloader [NC,OR]
64. RewriteCond %{HTTP_USER_AGENT} ^Maxthon$ [NC,OR]
65. RewriteCond %{HTTP_USER_AGENT} ^microsoft\.url [NC,OR]
66. RewriteCond %{HTTP_USER_AGENT} ^MIDown\ tool [NC,OR]
67. RewriteCond %{HTTP_USER_AGENT} ^miner [NC,OR]
68. RewriteCond %{HTTP_USER_AGENT} ^Mister\ PiX [NC,OR]
69. RewriteCond %{HTTP_USER_AGENT} ^Mozilla\.*Indy [NC,OR]
70. RewriteCond %{HTTP_USER_AGENT} ^Mozilla\.*NEWT [NC,OR]
71. RewriteCond %{HTTP_USER_AGENT} ^MSFrontPage [NC,OR]
72. RewriteCond %{HTTP_USER_AGENT} ^Navroad [NC,OR]
73. RewriteCond %{HTTP_USER_AGENT} ^NearSite [NC,OR]
74. RewriteCond %{HTTP_USER_AGENT} ^Net\ Vampire [NC,OR]
75. RewriteCond %{HTTP_USER_AGENT} ^NetAnts [NC,OR]
76. RewriteCond %{HTTP_USER_AGENT} ^NetSpider [NC,OR]
77. RewriteCond %{HTTP_USER_AGENT} ^NetZIP [NC,OR]
78. RewriteCond %{HTTP_USER_AGENT} ^nutch [NC,OR]
79. RewriteCond %{HTTP_USER_AGENT} ^Octopus [NC,OR]
80. RewriteCond %{HTTP_USER_AGENT} ^Offline\ Explorer [NC,OR]
81. RewriteCond %{HTTP_USER_AGENT} ^Offline\ Navigator [NC,OR]
82. RewriteCond %{HTTP_USER_AGENT} ^PageGrabber [NC,OR]
83. RewriteCond %{HTTP_USER_AGENT} ^Papa\ Foto [NC,OR]
84. RewriteCond %{HTTP_USER_AGENT} ^pavuk [NC,OR]
85. RewriteCond %{HTTP_USER_AGENT} ^pcBrowser [NC,OR]
86. RewriteCond %{HTTP_USER_AGENT} ^PeoplePal [NC,OR]
87. RewriteCond %{HTTP_USER_AGENT} ^planetwork [NC,OR]
88. RewriteCond %{HTTP_USER_AGENT} ^psbot [NC,OR]
89. RewriteCond %{HTTP_USER_AGENT} ^purebot [NC,OR]
90. RewriteCond %{HTTP_USER_AGENT} ^pycurl [NC,OR]
91. RewriteCond %{HTTP_USER_AGENT} ^RealDownload [NC,OR]
92. RewriteCond %{HTTP_USER_AGENT} ^ReGet [NC,OR]
93. RewriteCond %{HTTP_USER_AGENT} ^Rippers\ 0 [NC,OR]
94. RewriteCond %{HTTP_USER_AGENT} ^SeaMonkey$ [NC,OR]
95. RewriteCond %{HTTP_USER_AGENT} ^sitecheck\.internetseer\.com [NC,OR]
96. RewriteCond %{HTTP_USER_AGENT} ^SiteSnagger [NC,OR]
97. RewriteCond %{HTTP_USER_AGENT} ^skygrid [NC,OR]
98. RewriteCond %{HTTP_USER_AGENT} ^SmartDownload [NC,OR]
99. RewriteCond %{HTTP_USER_AGENT} ^sucker [NC,OR]
100. RewriteCond %{HTTP_USER_AGENT} ^SuperBot [NC,OR]
101. RewriteCond %{HTTP_USER_AGENT} ^SuperHTTP [NC,OR]
102. RewriteCond %{HTTP_USER_AGENT} ^Surfbot [NC,OR]
103. RewriteCond %{HTTP_USER_AGENT} ^tAkeOut [NC,OR]
104. RewriteCond %{HTTP_USER_AGENT} ^Teleport\ Pro [NC,OR]
105. RewriteCond %{HTTP_USER_AGENT} ^Toata\ dragostea\ mea\ pentru\ diavola [NC,OR]
106. RewriteCond %{HTTP_USER_AGENT} ^turnit [NC,OR]
107. RewriteCond %{HTTP_USER_AGENT} ^vikspider [NC,OR]
108. RewriteCond %{HTTP_USER_AGENT} ^VoidEYE [NC,OR]
109. RewriteCond %{HTTP_USER_AGENT} ^Web\ Image\ Collector [NC,OR]
110. RewriteCond %{HTTP_USER_AGENT} ^Web\ Sucker [NC,OR]
111. RewriteCond %{HTTP_USER_AGENT} ^WebAuto [NC,OR]
112. RewriteCond %{HTTP_USER_AGENT} ^WebCopier [NC,OR]
113. RewriteCond %{HTTP_USER_AGENT} ^WebFetch [NC,OR]
114. RewriteCond %{HTTP_USER_AGENT} ^WebGo\ IS [NC,OR]
115. RewriteCond %{HTTP_USER_AGENT} ^WebLeacher [NC,OR]
116. RewriteCond %{HTTP_USER_AGENT} ^WebReaper [NC,OR]
117. RewriteCond %{HTTP_USER_AGENT} ^WebSauger [NC,OR]
118. RewriteCond %{HTTP_USER_AGENT} ^Website\ eXtractor [NC,OR]
119. RewriteCond %{HTTP_USER_AGENT} ^Website\ Quester [NC,OR]
120. RewriteCond %{HTTP_USER_AGENT} ^WebStripper [NC,OR]
121. RewriteCond %{HTTP_USER_AGENT} ^WebWhacker [NC,OR]
122. RewriteCond %{HTTP_USER_AGENT} ^WebZIP [NC,OR]
123. RewriteCond %{HTTP_USER_AGENT} ^Wget [NC,OR]
124. RewriteCond %{HTTP_USER_AGENT} ^Widow [NC,OR]
125. RewriteCond %{HTTP_USER_AGENT} ^WPScan [NC,OR]
126. RewriteCond %{HTTP_USER_AGENT} ^WWW-Mechanize [NC,OR]
127. RewriteCond %{HTTP_USER_AGENT} ^WWWOFFLE [NC,OR]
128. RewriteCond %{HTTP_USER_AGENT} ^Xaldon\ WebSpider [NC,OR]
129. RewriteCond %{HTTP_USER_AGENT} ^Yandex [NC,OR]
130. RewriteCond %{HTTP_USER_AGENT} ^Zeus [NC,OR]
131. RewriteCond %{HTTP_USER_AGENT} ^zmeu [NC,OR]
132. RewriteCond %{HTTP_USER_AGENT} 360Spider [NC,OR]
133. RewriteCond %{HTTP_USER_AGENT} AhrefsBot [NC,OR]
134. RewriteCond %{HTTP_USER_AGENT} CazoodleBot [NC,OR]
135. RewriteCond %{HTTP_USER_AGENT} discobot [NC,OR]
136. RewriteCond %{HTTP_USER_AGENT} EasouSpider [NC,OR]
137. RewriteCond %{HTTP_USER_AGENT} ecxi [NC,OR]
138. RewriteCond %{HTTP_USER_AGENT} GT::WWW [NC,OR]
139. RewriteCond %{HTTP_USER_AGENT} heritrix [NC,OR]
140. RewriteCond %{HTTP_USER_AGENT} HTTP::Lite [NC,OR]
141. RewriteCond %{HTTP_USER_AGENT} HTTrack [NC,OR]
142. RewriteCond %{HTTP_USER_AGENT} ia_archiver [NC,OR]
143. RewriteCond %{HTTP_USER_AGENT} id-search [NC,OR]
144. RewriteCond %{HTTP_USER_AGENT} id-search\.org [NC,OR]
145. RewriteCond %{HTTP_USER_AGENT} IDBot [NC,OR]
146. RewriteCond %{HTTP_USER_AGENT} Indy\ Library [NC,OR]
147. RewriteCond %{HTTP_USER_AGENT} IRLbot [NC,OR]
148. RewriteCond %{HTTP_USER_AGENT} ISC\ Systems\ iRc\ Search\ 2\.1 [NC,OR]
149. RewriteCond %{HTTP_USER_AGENT} LinksCrawler [NC,OR]
150. RewriteCond %{HTTP_USER_AGENT} LinksManager.com_bot [NC,OR]
151. RewriteCond %{HTTP_USER_AGENT} linkwalker [NC,OR]
152. RewriteCond %{HTTP_USER_AGENT} lwp-trivial [NC,OR]
153. RewriteCond %{HTTP_USER_AGENT} MFC_Tear_Sample [NC,OR]
154. RewriteCond %{HTTP_USER_AGENT} Microsoft\ URL\ Control [NC,OR]
155. RewriteCond %{HTTP_USER_AGENT} Missigua\ Locator [NC,OR]
156. RewriteCond %{HTTP_USER_AGENT} MJ12bot [NC,OR]
157. RewriteCond %{HTTP_USER_AGENT} panscient.com [NC,OR]
158. RewriteCond %{HTTP_USER_AGENT} PECL::HTTP [NC,OR]
159. RewriteCond %{HTTP_USER_AGENT} PHPCrawl [NC,OR]
160. RewriteCond %{HTTP_USER_AGENT} PleaseCrawl [NC,OR]
161. RewriteCond %{HTTP_USER_AGENT} SBIder [NC,OR]
162. RewriteCond %{HTTP_USER_AGENT} SearchmetricsBot [NC,OR]
163. RewriteCond %{HTTP_USER_AGENT} Snoopy [NC,OR]
164. RewriteCond %{HTTP_USER_AGENT} Sogou [NC,OR]
165. RewriteCond %{HTTP_USER_AGENT} Steeler [NC,OR]
166. RewriteCond %{HTTP_USER_AGENT} URI::Fetch [NC,OR]
167. RewriteCond %{HTTP_USER_AGENT} urllib [NC,OR]
168. RewriteCond %{HTTP_USER_AGENT} Web\ Sucker [NC,OR]
169. RewriteCond %{HTTP_USER_AGENT} webalta [NC,OR]
170. RewriteCond %{HTTP_USER_AGENT} WebCollage [NC,OR]
171. RewriteCond %{HTTP_USER_AGENT} Wells\ Search\ II [NC,OR]
172. RewriteCond %{HTTP_USER_AGENT} WEP\ Search [NC,OR]
173. RewriteCond %{HTTP_USER_AGENT} XoviBot [NC,OR]
174. RewriteCond %{HTTP_USER_AGENT} YisouSpider [NC,OR]
175. RewriteCond %{HTTP_USER_AGENT} zermelo [NC,OR]
176. RewriteCond %{HTTP_USER_AGENT} ZyBorg
177. RewriteRule ^.* - [F,L]
178. # Abuse bot blocking rule end
* # Abuse HTTP Referrer Blocking
179. RewriteCond %{HTTP_REFERER} free-social-buttions\.com [NC,OR]
180. RewriteCond %{HTTP_REFERER} best-seo-offer\.com [NC,OR]
181. RewriteCond %{HTTP_REFERER} buttons-for-your-website\.com [NC,OR]
182. RewriteCond %{HTTP_REFERER} www1.free-social-buttons\.com [NC,OR]
183. RewriteCond %{HTTP_REFERER} www2.free-social-buttons\.com [NC,OR]
184. RewriteCond %{HTTP_REFERER} www3.free-social-buttons\.com [NC,OR]
185. RewriteCond %{HTTP_REFERER} 100dollars-seo.com\.com [NC,OR]
186. RewriteCond %{HTTP_REFERER} www4.free-social-buttons\.com
187. RewriteRule ^.* - [F,L]
188. # Abuse HTTP Referrer Blocking end
* # Start Custom Blocks
189. # Aboundex
190. Deny from 173.192.34.95
191. # Bluecoat
192. deny from 8.21.4.254
193. deny from 65.46.48.192/30
194. deny from 65.160.238.176/28
195. deny from 85.92.222.0/24
196. deny from 206.51.36.0/22
197. deny from 216.52.23.0/24
198. # cyveillance
199. deny from 38.100.19.8/29
200. deny from 38.100.21.0/24
201. deny from 38.100.41.64/26
202. deny from 38.105.71.0/25
203. deny from 38.105.83.0/27
204. deny from 38.112.21.140/30
205. deny from 38.118.42.32/29
206. deny from 65.213.208.128/27
207. deny from 65.222.176.96/27
208. deny from 65.222.185.72/29
209. # Cyberpatrol
210. deny from 38.103.17.160/27
211. # Internet Identity - Anti-Phishing
212. deny from 66.113.96.0/20
213. deny from 70.35.113.192/27
214. # Ironport
215. deny from 204.15.80.0/22
216. # Lightspeed Systems Security
217. deny from 66.17.15.128/26
218. deny from 69.84.207.32/27
219. deny from 69.84.207.128/25
220. # Layered Technologies
221. deny from 72.36.128.0/17
222. deny from 72.232.0.0/16
223. deny from 72.233.0.0/17
224. deny from 216.32.0.0/14
225. # M86
226. deny from 67.192.231.224/29
227. deny from 208.90.236.0/22
228. # Phish-Inspector.com
229. deny from 209.147.127.208/28
230. # Prescient Software, Inc. Phishmongers
231. deny from 198.186.190.0/23
232. deny from 198.186.192.0/23
233. deny from 198.186.194.0/24
234. # urlfilterdb
235. deny from 207.210.99.32/29
236. # websense-in.car1.sandiego1.level3.net
237. deny from 4.53.120.22
238. # Websense
239. deny from 66.194.6.0/24
240. deny from 67.117.201.128/28
241. deny from 69.67.32.0/20
242. deny from 131.191.87.0/24
243. deny from 204.15.64.0/21
244. deny from 208.80.192.0/21
245. deny from 212.62.26.64/27
246. deny from 213.168.226.0/24
247. deny from 213.168.241.0/30
248. deny from 213.168.242.0/30
249. deny from 213.236.150.16/28
250. # End Custom Blocks. Add a couple line breaks below this as well.
* # Block comment spammers and bad bots
251. # Add your custom IP block list here. Example format:
252. # deny from 4.53.120.22
253. # End Block comment spammers, bad bots and some proxies