avdhut86 · March 19, 2021 12:53
diff --git a/nginx fail2ban badbots b/nginx fail2ban badbots
 #access log format 

 log_format  main  '$remote_addr - $remote_user [$time_local] "$request" - $request_filename '
  '$status $body_bytes_sent "$http_referer" '
  '"$http_user_agent" "$http_x_forwarded_for"';
diff --git a/nginx fail2ban badbots - jail.local b/nginx fail2ban badbots - jail.local
 #/etc/fail2ban/jail.local

 [nginx-badbots]
 enabled  = true
 port     = http,https
 filter   = nginx-badbots
 #logpath  = %(nginx_access_log)s
 logpath  = /var/log/nginx/*access.log
 maxretry = 2
diff --git a/nginx fail2ban badbots - nginx-badbots.conf b/nginx fail2ban badbots - nginx-badbots.conf
 #/etc/fail2ban/filter.d/nginx-badbots.conf
 #test it with:
 #sudo fail2ban-regex /var/log/nginx/test.log /etc/fail2ban/filter.d/nginx-badbots.conf

 [Definition]

 badbotscustom = 360Spider|404checker|404enemy|80legs|Abonti|Aboundex|Acunetix|ADmantX|AfD-Verbotsverfahren|AhrefsBot|AIBOT|AiHitBot|Aipbot|Alexibot|Alligator|AllSubmitter|AlphaBot|Anarchie|Apexoo|ASPSeek|Asterias|Attach|autoemailspider|BackDoorBot|Backlink-Ceck|backlink-check|BacklinkCrawler|BackStreet|BackWeb|Badass|Bandit|Barkrowler|BatchFTP|Battleztar Bazinga|BBBike|BDCbot|BDFetch|BetaBot|Bigfoot|Bitacle|Blackboard|Black Hole|BlackWidow|BLEXBot|Blow|BlowFish|Boardreader|Bolt|BotALot|Brandprotect|Brandwatch|Bubing|Buddy|BuiltBotTough|BuiltWith|Bullseye|BunnySlippers|BuzzSumo|Calculon|CATExplorador|CazoodleBot|CCBot|Cegbfeieh|CheeseBot|CherryPicker|ChinaClaw|Chlooe|Claritybot|Cliqzbot|Cloud mapping|coccocbot-web|Cogentbot|cognitiveseo|Collector|com\.plumanalytics|Copier|CopyRightCheck|Copyscape|Cosmos|Craftbot|crawler4j|crawler\.feedback|CrazyWebCrawler|Crescent|CSHttp|Curious|Custo|DatabaseDriverMysqli|DataCha0s|DBLBot|demandbase-bot|Demon|Deusu|Devil|Digincore|DigitalPebble|DIIbot|Dirbuster|Disco|Discobot|Discoverybot|DittoSpyder|DnyzBot|DomainAppender|DomainCrawler|DomainSigmaCrawler|DomainStatsBot|Dotbot|Download Wonder|Dragonfly|Drip|DTS Agent|EasyDL|Ebingbong|eCatch|ECCP/1\.0|Ecxi|EirGrabber|EMail Siphon|EMail Wolf|EroCrawler|evc-batch|Evil|Exabot|Express WebPictures|ExtLinksBot|Extractor|ExtractorPro|Extreme Picture Finder|EyeNetIE|Ezooms|FDM|FemtosearchBot|FHscan|Fimap|Firefox/7\.0|FlashGet|Flunky|Foobot|Freeuploader|FrontPage|Fyrebot|GalaxyBot|Genieo|GermCrawler|Getintent|GetRight|GetWeb|Gigablast|Gigabot|G-i-g-a-b-o-t|Go-Ahead-Got-It|Gotit|GoZilla|Go!Zilla|Grabber|GrabNet|Grafula|GrapeFX|GrapeshotCrawler|GridBot|GT\:\:WWW|Haansoft|HaosouSpider|Harvest|Havij|HEADMasterSEO|Heritrix|Hloader|HMView|HTMLparser|HTTP\:\:Lite|HTTrack|Humanlinks|HybridBot|Iblog|IDBot|Id-search|IlseBot|Image Fetch|Image Sucker|IndeedBot|Indy Library|InfoNaviRobot|InfoTekies|instabid|Intelliseek|InterGET|Internet Ninja|InternetSeer|internetVista monitor|ips-agent|Iria|IRLbot|Iskanie|IstellaBot|JamesBOT|Jbrofuzz|JennyBot|JetCar|JikeSpider|JOC Web Spider|Joomla|Jorgee|JustView|Jyxobot|Kenjin Spider|Keyword Density|Kozmosbot|Lanshanbot|Larbin|LeechFTP|LeechGet|LexiBot|Lftp|LibWeb|Libwhisker|Lightspeedsystems|Likse|Linkdexbot|LinkextractorPro|LinkpadBot|LinkScan|LinksManager|LinkWalker|LinqiaMetadataDownloaderBot|LinqiaRSSBot|LinqiaScrapeBot|Lipperhey|Litemage_walker|Lmspider|LNSpiderguy|Ltx71|lwp-request|LWP\:\:Simple|lwp-trivial|Magnet|Mag-Net|magpie-crawler|Mail\.RU_Bot|Majestic12|MarkMonitor|MarkWatch|Masscan|Mass Downloader|Mata Hari|MauiBot|Meanpathbot|mediawords|MegaIndex\.ru|Metauri|MFC_Tear_Sample|Microsoft Data Access|Microsoft URL Control|MIDown tool|MIIxpc|Mister PiX|MJ12bot|Mojeek|Morfeus Fucking Scanner|Mr\.4x3|MSFrontPage|MSIECrawler|Msrabot|MS Web Services Client Protocol|muhstik-scan|Musobot|Name Intelligence|Nameprotect|Navroad|NearSite|Needle|Nessus|NetAnts|Netcraft|netEstate NE Crawler|NetLyzer|NetMechanic|NetSpider|Nettrack|Net Vampire|Netvibes|NetZIP|NextGenSearchBot|Nibbler|NICErsPRO|Niki-bot|Nikto|NimbleCrawler|Ninja|Nmap|NPbot|Nutch|oBot|Octopus|Offline Explorer|Offline Navigator|Openfind|OpenLinkProfiler|Openvas|OrangeBot|OrangeSpider|OutclicksBot|OutfoxBot|PageAnalyzer|Page Analyzer|PageGrabber|page scorer|PageScorer|Panscient|Papa Foto|Pavuk|pcBrowser|PECL\:\:HTTP|PeoplePal|PHPCrawl|Picscout|Picsearch|PictureFinder|Pimonster|Pi-Monster|Pixray|PleaseCrawl|plumanalytics|Pockey|POE-Component-Client-HTTP|Probethenet|ProPowerBot|ProWebWalker|Psbot|Pump|PxBroker|PyCurl|QueryN Metasearch|Quick-Crawler|RankActive|RankActiveLinkBot|RankFlex|RankingBot|RankingBot2|Rankivabot|RankurBot|RealDownload|Reaper|RebelMouse|Recorder|RedesScrapy|ReGet|RepoMonkey|Ripper|RocketCrawler|Rogerbot|SalesIntelligent|SBIder|ScanAlert|Scanbot|scan\.lol|Scrapy|Screaming|ScreenerBot|Searchestate|SearchmetricsBot|Semrush|SemrushBot|SEOkicks|SEOlyticsCrawler|Seomoz|SEOprofiler|seoscanners|SEOstats|sexsearcher|Seznam|SeznamBot|Shodan|Siphon|SISTRIX|Sitebeam|SiteExplorer|Siteimprove|SiteLockSpider|SiteSnagger|SiteSucker|Site Sucker|Sitevigil|Slackbot-LinkExpanding|SlySearch|SmartDownload|SMTBot|Snake|Snapbot|Snoopy|SocialRankIOBot|Sogou web spider|Sosospider|Sottopop|SpaceBison|Spammen|SpankBot|Spanner|Spbot|Spinn3r|SputnikBot|Sqlmap|Sqlworm|Sqworm|Steeler|Stripper|Sucker|Sucuri|SuperBot|SuperHTTP|Surfbot|SurveyBot|Suzuran|Swiftbot|sysscan|Szukacz|T0PHackTeam|T8Abot|tAkeOut|Teleport|TeleportPro|Telesoft|Telesphoreo|Telesphorep|The Intraformant|TheNomad|TightTwatBot|Titan|Toata|Toweyabot|Trendiction|Trendictionbot|trendiction\.com|trendiction\.de|True_Robot|Turingos|Turnitin|TurnitinBot|TwengaBot|Twice|Typhoeus|UnisterBot|URLy\.Warning|URLy Warning|Vacuum|Vagabondo|VB Project|VCI|VeriCiteCrawler|VidibleScraper|Virusdie|VoidEYE|Voil|Voltron|Wallpapers/3\.0|WallpapersHD|WASALive-Bot|WBSearchBot|Webalta|WebAuto|Web Auto|WebBandit|WebCollage|Web Collage|WebCopier|WEBDAV|WebEnhancer|Web Enhancer|WebFetch|Web Fetch|WebFuck|Web Fuck|WebGo IS|WebImageCollector|WebLeacher|WebmasterWorldForumBot|webmeup-crawler|WebPix|Web Pix|WebReaper|WebSauger|Web Sauger|Webshag|WebsiteExtractor|WebsiteQuester|Website Quester|Webster|WebStripper|WebSucker|Web Sucker|WebWhacker|WebZIP|WeSEE|Whack|Whacker|Whatweb|Who\.is Bot|Widow|WinHTTrack|WiseGuys Robot|WISENutbot|Wonderbot|Woobot|Wotbox|Wprecon|WPScan|WWW-Collector-E|WWW-Mechanize|WWW\:\:Mechanize|WWWOFFLE|x09Mozilla|x22Mozilla|Xaldon_WebSpider|Xaldon WebSpider|Xenu|xpymep1\.exe|YoudaoBot|Zade|Zauba|zauba\.io|Zermelo|Zeus|zgrab|Zitebot|ZmEu|ZumBot|ZyBorg|PetalBot|EmailCollector|WebEMailExtrac|TrackBack/1\.02|sogou music spider|(?:Mozilla/\d+\.\d+ )?Jorgee
 badbots = Atomic_Email_Hunter/4\.0|atSpider/1\.0|autoemailspider|bwh3_user_agent|China Local Browse 2\.6|ContactBot/0\.2|ContentSmartz|DataCha0s/2\.0|DBrowse 1\.4b|DBrowse 1\.4d|Demo Bot DOT 16b|Demo Bot Z 16b|DSurf15a 01|DSurf15a 71|DSurf15a 81|DSurf15a VA|EBrowse 1\.4b|Educate Search VxB|EmailSiphon|EmailSpider|EmailWolf 1\.00|ESurf15a 15|ExtractorPro|Franklin Locator 1\.8|FSurf15a 01|Full Web Bot 0416B|Full Web Bot 0516B|Full Web Bot 2816B|Guestbook Auto Submitter|Industry Program 1\.0\.x|ISC Systems iRc Search 2\.1|IUPUI Research Bot v 1\.9a|LARBIN-EXPERIMENTAL \(efp@gmx\.net\)|LetsCrawl\.com/1\.0 \+http\://letscrawl\.com/|Lincoln State Web Browser|LMQueueBot/0\.2|LWP\:\:Simple/5\.803|Mac Finder 1\.0\.xx|MFC Foundation Class Library 4\.0|Microsoft URL Control - 6\.00\.8xxx|Missauga Locate 1\.0\.0|Missigua Locator 1\.9|Missouri College Browse|Mizzu Labs 2\.2|Mo College 1\.9|MVAClient|Mozilla/2\.0 \(compatible; NEWT ActiveX; Win32\)|Mozilla/3\.0 \(compatible; Indy Library\)|Mozilla/3\.0 \(compatible; scan4mail \(advanced version\) http\://www\.peterspages\.net/?scan4mail\)|Mozilla/4\.0 \(compatible; Advanced Email Extractor v2\.xx\)|Mozilla/4\.0 \(compatible; Iplexx Spider/1\.0 http\://www\.iplexx\.at\)|Mozilla/4\.0 \(compatible; MSIE 5\.0; Windows NT; DigExt; DTS Agent|Mozilla/4\.0 efp@gmx\.net|Mozilla/5\.0 \(Version\: xxxx Type\:xx\)|NameOfAgent \(CMS Spider\)|NASA Search 1\.0|Nsauditor/1\.x|PBrowse 1\.4b|PEval 1\.4b|Poirot|Port Huron Labs|Production Bot 0116B|Production Bot 2016B|Production Bot DOT 3016B|Program Shareware 1\.0\.2|PSurf15a 11|PSurf15a 51|PSurf15a VA|psycheclone|RSurf15a 41|RSurf15a 51|RSurf15a 81|searchbot admin@google\.com|ShablastBot 1\.0|snap\.com beta crawler v0|Snapbot/1\.0|Snapbot/1\.0 \(Snap Shots&#44; \+http\://www\.snap\.com\)|sogou develop spider|Sogou Orion spider/3\.0\(\+http\://www\.sogou\.com/docs/help/webmasters\.htm#07\)|sogou spider|Sogou web spider/3\.0\(\+http\://www\.sogou\.com/docs/help/webmasters\.htm#07\)|sohu agent|SSurf15a 11 |TSurf15a 11|Under the Rainbow 2\.2|User-Agent\: Mozilla/4\.0 \(compatible; MSIE 6\.0; Windows NT 5\.1\)|VadixBot|WebVulnCrawl\.unknown/1\.0 libwww-perl/5\.803|Wells Search II|WEP Search 00

 failregex = ^<HOST> -.*"(GET|POST|HEAD).*HTTP.*".*(?:%(badbots)s|%(badbotscustom)s).*"$

 ignoreregex =

 datepattern = ^[^\[]*\[({DATE})
              {^LN-BEG}
	#access log format

	log_format main '$remote_addr - $remote_user [$time_local] "$request" - $request_filename '
	'$status $body_bytes_sent "$http_referer" '
	'"$http_user_agent" "$http_x_forwarded_for"';
	#/etc/fail2ban/jail.local

	[nginx-badbots]
	enabled = true
	port = http,https
	filter = nginx-badbots
	#logpath = %(nginx_access_log)s
	logpath = /var/log/nginx/*access.log
	maxretry = 2
	#/etc/fail2ban/filter.d/nginx-badbots.conf
	#test it with:
	#sudo fail2ban-regex /var/log/nginx/test.log /etc/fail2ban/filter.d/nginx-badbots.conf

	[Definition]

	badbotscustom = 360Spider\|404checker\|404enemy\|80legs\|Abonti\|Aboundex\|Acunetix\|ADmantX\|AfD-Verbotsverfahren\|AhrefsBot\|AIBOT\|AiHitBot\|Aipbot\|Alexibot\|Alligator\|AllSubmitter\|AlphaBot\|Anarchie\|Apexoo\|ASPSeek\|Asterias\|Attach\|autoemailspider\|BackDoorBot\|Backlink-Ceck\|backlink-check\|BacklinkCrawler\|BackStreet\|BackWeb\|Badass\|Bandit\|Barkrowler\|BatchFTP\|Battleztar Bazinga\|BBBike\|BDCbot\|BDFetch\|BetaBot\|Bigfoot\|Bitacle\|Blackboard\|Black Hole\|BlackWidow\|BLEXBot\|Blow\|BlowFish\|Boardreader\|Bolt\|BotALot\|Brandprotect\|Brandwatch\|Bubing\|Buddy\|BuiltBotTough\|BuiltWith\|Bullseye\|BunnySlippers\|BuzzSumo\|Calculon\|CATExplorador\|CazoodleBot\|CCBot\|Cegbfeieh\|CheeseBot\|CherryPicker\|ChinaClaw\|Chlooe\|Claritybot\|Cliqzbot\|Cloud mapping\|coccocbot-web\|Cogentbot\|cognitiveseo\|Collector\|com\.plumanalytics\|Copier\|CopyRightCheck\|Copyscape\|Cosmos\|Craftbot\|crawler4j\|crawler\.feedback\|CrazyWebCrawler\|Crescent\|CSHttp\|Curious\|Custo\|DatabaseDriverMysqli\|DataCha0s\|DBLBot\|demandbase-bot\|Demon\|Deusu\|Devil\|Digincore\|DigitalPebble\|DIIbot\|Dirbuster\|Disco\|Discobot\|Discoverybot\|DittoSpyder\|DnyzBot\|DomainAppender\|DomainCrawler\|DomainSigmaCrawler\|DomainStatsBot\|Dotbot\|Download Wonder\|Dragonfly\|Drip\|DTS Agent\|EasyDL\|Ebingbong\|eCatch\|ECCP/1\.0\|Ecxi\|EirGrabber\|EMail Siphon\|EMail Wolf\|EroCrawler\|evc-batch\|Evil\|Exabot\|Express WebPictures\|ExtLinksBot\|Extractor\|ExtractorPro\|Extreme Picture Finder\|EyeNetIE\|Ezooms\|FDM\|FemtosearchBot\|FHscan\|Fimap\|Firefox/7\.0\|FlashGet\|Flunky\|Foobot\|Freeuploader\|FrontPage\|Fyrebot\|GalaxyBot\|Genieo\|GermCrawler\|Getintent\|GetRight\|GetWeb\|Gigablast\|Gigabot\|G-i-g-a-b-o-t\|Go-Ahead-Got-It\|Gotit\|GoZilla\|Go!Zilla\|Grabber\|GrabNet\|Grafula\|GrapeFX\|GrapeshotCrawler\|GridBot\|GT\:\:WWW\|Haansoft\|HaosouSpider\|Harvest\|Havij\|HEADMasterSEO\|Heritrix\|Hloader\|HMView\|HTMLparser\|HTTP\:\:Lite\|HTTrack\|Humanlinks\|HybridBot\|Iblog\|IDBot\|Id-search\|IlseBot\|Image Fetch\|Image Sucker\|IndeedBot\|Indy Library\|InfoNaviRobot\|InfoTekies\|instabid\|Intelliseek\|InterGET\|Internet Ninja\|InternetSeer\|internetVista monitor\|ips-agent\|Iria\|IRLbot\|Iskanie\|IstellaBot\|JamesBOT\|Jbrofuzz\|JennyBot\|JetCar\|JikeSpider\|JOC Web Spider\|Joomla\|Jorgee\|JustView\|Jyxobot\|Kenjin Spider\|Keyword Density\|Kozmosbot\|Lanshanbot\|Larbin\|LeechFTP\|LeechGet\|LexiBot\|Lftp\|LibWeb\|Libwhisker\|Lightspeedsystems\|Likse\|Linkdexbot\|LinkextractorPro\|LinkpadBot\|LinkScan\|LinksManager\|LinkWalker\|LinqiaMetadataDownloaderBot\|LinqiaRSSBot\|LinqiaScrapeBot\|Lipperhey\|Litemage_walker\|Lmspider\|LNSpiderguy\|Ltx71\|lwp-request\|LWP\:\:Simple\|lwp-trivial\|Magnet\|Mag-Net\|magpie-crawler\|Mail\.RU_Bot\|Majestic12\|MarkMonitor\|MarkWatch\|Masscan\|Mass Downloader\|Mata Hari\|MauiBot\|Meanpathbot\|mediawords\|MegaIndex\.ru\|Metauri\|MFC_Tear_Sample\|Microsoft Data Access\|Microsoft URL Control\|MIDown tool\|MIIxpc\|Mister PiX\|MJ12bot\|Mojeek\|Morfeus Fucking Scanner\|Mr\.4x3\|MSFrontPage\|MSIECrawler\|Msrabot\|MS Web Services Client Protocol\|muhstik-scan\|Musobot\|Name Intelligence\|Nameprotect\|Navroad\|NearSite\|Needle\|Nessus\|NetAnts\|Netcraft\|netEstate NE Crawler\|NetLyzer\|NetMechanic\|NetSpider\|Nettrack\|Net Vampire\|Netvibes\|NetZIP\|NextGenSearchBot\|Nibbler\|NICErsPRO\|Niki-bot\|Nikto\|NimbleCrawler\|Ninja\|Nmap\|NPbot\|Nutch\|oBot\|Octopus\|Offline Explorer\|Offline Navigator\|Openfind\|OpenLinkProfiler\|Openvas\|OrangeBot\|OrangeSpider\|OutclicksBot\|OutfoxBot\|PageAnalyzer\|Page Analyzer\|PageGrabber\|page scorer\|PageScorer\|Panscient\|Papa Foto\|Pavuk\|pcBrowser\|PECL\:\:HTTP\|PeoplePal\|PHPCrawl\|Picscout\|Picsearch\|PictureFinder\|Pimonster\|Pi-Monster\|Pixray\|PleaseCrawl\|plumanalytics\|Pockey\|POE-Component-Client-HTTP\|Probethenet\|ProPowerBot\|ProWebWalker\|Psbot\|Pump\|PxBroker\|PyCurl\|QueryN Metasearch\|Quick-Crawler\|RankActive\|RankActiveLinkBot\|RankFlex\|RankingBot\|RankingBot2\|Rankivabot\|RankurBot\|RealDownload\|Reaper\|RebelMouse\|Recorder\|RedesScrapy\|ReGet\|RepoMonkey\|Ripper\|RocketCrawler\|Rogerbot\|SalesIntelligent\|SBIder\|ScanAlert\|Scanbot\|scan\.lol\|Scrapy\|Screaming\|ScreenerBot\|Searchestate\|SearchmetricsBot\|Semrush\|SemrushBot\|SEOkicks\|SEOlyticsCrawler\|Seomoz\|SEOprofiler\|seoscanners\|SEOstats\|sexsearcher\|Seznam\|SeznamBot\|Shodan\|Siphon\|SISTRIX\|Sitebeam\|SiteExplorer\|Siteimprove\|SiteLockSpider\|SiteSnagger\|SiteSucker\|Site Sucker\|Sitevigil\|Slackbot-LinkExpanding\|SlySearch\|SmartDownload\|SMTBot\|Snake\|Snapbot\|Snoopy\|SocialRankIOBot\|Sogou web spider\|Sosospider\|Sottopop\|SpaceBison\|Spammen\|SpankBot\|Spanner\|Spbot\|Spinn3r\|SputnikBot\|Sqlmap\|Sqlworm\|Sqworm\|Steeler\|Stripper\|Sucker\|Sucuri\|SuperBot\|SuperHTTP\|Surfbot\|SurveyBot\|Suzuran\|Swiftbot\|sysscan\|Szukacz\|T0PHackTeam\|T8Abot\|tAkeOut\|Teleport\|TeleportPro\|Telesoft\|Telesphoreo\|Telesphorep\|The Intraformant\|TheNomad\|TightTwatBot\|Titan\|Toata\|Toweyabot\|Trendiction\|Trendictionbot\|trendiction\.com\|trendiction\.de\|True_Robot\|Turingos\|Turnitin\|TurnitinBot\|TwengaBot\|Twice\|Typhoeus\|UnisterBot\|URLy\.Warning\|URLy Warning\|Vacuum\|Vagabondo\|VB Project\|VCI\|VeriCiteCrawler\|VidibleScraper\|Virusdie\|VoidEYE\|Voil\|Voltron\|Wallpapers/3\.0\|WallpapersHD\|WASALive-Bot\|WBSearchBot\|Webalta\|WebAuto\|Web Auto\|WebBandit\|WebCollage\|Web Collage\|WebCopier\|WEBDAV\|WebEnhancer\|Web Enhancer\|WebFetch\|Web Fetch\|WebFuck\|Web Fuck\|WebGo IS\|WebImageCollector\|WebLeacher\|WebmasterWorldForumBot\|webmeup-crawler\|WebPix\|Web Pix\|WebReaper\|WebSauger\|Web Sauger\|Webshag\|WebsiteExtractor\|WebsiteQuester\|Website Quester\|Webster\|WebStripper\|WebSucker\|Web Sucker\|WebWhacker\|WebZIP\|WeSEE\|Whack\|Whacker\|Whatweb\|Who\.is Bot\|Widow\|WinHTTrack\|WiseGuys Robot\|WISENutbot\|Wonderbot\|Woobot\|Wotbox\|Wprecon\|WPScan\|WWW-Collector-E\|WWW-Mechanize\|WWW\:\:Mechanize\|WWWOFFLE\|x09Mozilla\|x22Mozilla\|Xaldon_WebSpider\|Xaldon WebSpider\|Xenu\|xpymep1\.exe\|YoudaoBot\|Zade\|Zauba\|zauba\.io\|Zermelo\|Zeus\|zgrab\|Zitebot\|ZmEu\|ZumBot\|ZyBorg\|PetalBot\|EmailCollector\|WebEMailExtrac\|TrackBack/1\.02\|sogou music spider\|(?:Mozilla/\d+\.\d+ )?Jorgee
	badbots = Atomic_Email_Hunter/4\.0\|atSpider/1\.0\|autoemailspider\|bwh3_user_agent\|China Local Browse 2\.6\|ContactBot/0\.2\|ContentSmartz\|DataCha0s/2\.0\|DBrowse 1\.4b\|DBrowse 1\.4d\|Demo Bot DOT 16b\|Demo Bot Z 16b\|DSurf15a 01\|DSurf15a 71\|DSurf15a 81\|DSurf15a VA\|EBrowse 1\.4b\|Educate Search VxB\|EmailSiphon\|EmailSpider\|EmailWolf 1\.00\|ESurf15a 15\|ExtractorPro\|Franklin Locator 1\.8\|FSurf15a 01\|Full Web Bot 0416B\|Full Web Bot 0516B\|Full Web Bot 2816B\|Guestbook Auto Submitter\|Industry Program 1\.0\.x\|ISC Systems iRc Search 2\.1\|IUPUI Research Bot v 1\.9a\|LARBIN-EXPERIMENTAL \(efp@gmx\.net\)\|LetsCrawl\.com/1\.0 \+http\://letscrawl\.com/\|Lincoln State Web Browser\|LMQueueBot/0\.2\|LWP\:\:Simple/5\.803\|Mac Finder 1\.0\.xx\|MFC Foundation Class Library 4\.0\|Microsoft URL Control - 6\.00\.8xxx\|Missauga Locate 1\.0\.0\|Missigua Locator 1\.9\|Missouri College Browse\|Mizzu Labs 2\.2\|Mo College 1\.9\|MVAClient\|Mozilla/2\.0 \(compatible; NEWT ActiveX; Win32\)\|Mozilla/3\.0 \(compatible; Indy Library\)\|Mozilla/3\.0 \(compatible; scan4mail \(advanced version\) http\://www\.peterspages\.net/?scan4mail\)\|Mozilla/4\.0 \(compatible; Advanced Email Extractor v2\.xx\)\|Mozilla/4\.0 \(compatible; Iplexx Spider/1\.0 http\://www\.iplexx\.at\)\|Mozilla/4\.0 \(compatible; MSIE 5\.0; Windows NT; DigExt; DTS Agent\|Mozilla/4\.0 efp@gmx\.net\|Mozilla/5\.0 \(Version\: xxxx Type\:xx\)\|NameOfAgent \(CMS Spider\)\|NASA Search 1\.0\|Nsauditor/1\.x\|PBrowse 1\.4b\|PEval 1\.4b\|Poirot\|Port Huron Labs\|Production Bot 0116B\|Production Bot 2016B\|Production Bot DOT 3016B\|Program Shareware 1\.0\.2\|PSurf15a 11\|PSurf15a 51\|PSurf15a VA\|psycheclone\|RSurf15a 41\|RSurf15a 51\|RSurf15a 81\|searchbot admin@google\.com\|ShablastBot 1\.0\|snap\.com beta crawler v0\|Snapbot/1\.0\|Snapbot/1\.0 \(Snap Shots, \+http\://www\.snap\.com\)\|sogou develop spider\|Sogou Orion spider/3\.0\(\+http\://www\.sogou\.com/docs/help/webmasters\.htm#07\)\|sogou spider\|Sogou web spider/3\.0\(\+http\://www\.sogou\.com/docs/help/webmasters\.htm#07\)\|sohu agent\|SSurf15a 11 \|TSurf15a 11\|Under the Rainbow 2\.2\|User-Agent\: Mozilla/4\.0 \(compatible; MSIE 6\.0; Windows NT 5\.1\)\|VadixBot\|WebVulnCrawl\.unknown/1\.0 libwww-perl/5\.803\|Wells Search II\|WEP Search 00

	failregex = ^<HOST> -."(GET\|POST\|HEAD).HTTP.".(?:%(badbots)s\|%(badbotscustom)s).*"$

	ignoreregex =

	datepattern = ^[^\[]*\[({DATE})
	{^LN-BEG}