Regex fix, checkip argument & updated tld list

plugins/uribl

@@ -66,6 +66,12 @@ Net::DNS documentation for details.
 If set true, any headers found in the URIs will be checked as well.  Disabled
 by default.
 
+=item checkip [yes|no]
+
+Specifies whether ip-addresses should be searched for or not. Some URIBLs
+only allows domain names and triggers positive on any ip-address.
+Default value: yes.
+
 =back
 
 =head1 CAUTIONS
@@ -139,6 +145,7 @@ sub init {
 
     $self->{action}  = $args{action}  || 'add-header';
     $self->{timeout} = $args{timeout} || 30;
+    $self->{checkip} = $args{checkip} || 'yes';
 
     # scan-headers was the originally documented name for this option, while
     # check-headers actually implements it, so tolerate both.
@@ -257,6 +264,8 @@ sub lookup_start {
     my $queries = 0;
     my %pending;
     my @qp_continuations;
+
+    my $tlds4regex = $self->get_tlds4regex;
 
     $transaction->body_resetpos;
 
@@ -266,13 +275,15 @@ sub lookup_start {
         chomp $l;
         last if !$l;
     }
-    while ($l = $transaction->body_getline) {
+    while ($l = $transaction->body_getline || @qp_continuations) {
         chomp $l;
 
         if ($l =~ /(.*)=$/) {
             push @qp_continuations, $1;
+            next;
         }
         elsif (@qp_continuations) {
+            $l = '' if $l eq '1';
             $l = join('', @qp_continuations, $l);
             @qp_continuations = ();
         }
@@ -286,6 +297,8 @@ sub lookup_start {
         # Dodge inserted-semicolon munging
         $l =~ tr/;//d;
 
+        # Check for IP-addresses?
+        if ($self->{checkip} eq 'yes') {
         while (
             $l =~ m{
             \w{3,16}:/+            # protocol
@@ -338,14 +351,12 @@ sub lookup_start {
                 $pending{$rev} = 1;
             }
         }
+        }
         while (
             $l =~ m{
             ((?:www\.)?                             # www?
-             [a-zA-Z0-9][a-zA-Z0-9\-.]+\.           # hostname
-             (?:aero|arpa|asia|biz|cat|com|coop|    # tld
-                edu|gov|info|int|jobs|mil|mobi|
-                museum|name|net|org|pro|tel|travel|
-                [a-zA-Z]{2})
+             [a-zA-Z0-9](?:[a-zA-Z0-9\-]+\.)+       # hostname
+             (?:$tlds4regex|[a-zA-Z]{2})            # tld
             )(?!\w)        
             }gix
               )
@@ -386,12 +397,9 @@ sub lookup_start {
             \w{3,16}:/+                 # protocol
             (?:\S+@)?                   # user/pass
             (
-	     [a-zA-Z0-9][a-zA-Z0-9\-.]+\.           # hostname
-	     (?:aero|arpa|asia|biz|cat|com|coop|    # tld
-                edu|gov|info|int|jobs|mil|mobi|
-                museum|name|net|org|pro|tel|travel|
-                [a-zA-Z]{2})
-            )
+	     [a-zA-Z0-9](?:[a-zA-Z0-9\-]+\.)+       # hostname
+	     (?:$tlds4regex|[a-zA-Z]{2})            # tld
+            )(?!\w)
             }gix
               )
         {
@@ -532,3 +540,118 @@ sub hook_data_post {
     return DECLINED;
 }
 
+sub get_tlds4regex {
+
+    # TLD list from http://data.iana.org/TLD/tlds-alpha-by-domain.txt as of 2017-03-03
+
+    return 'aaa|aarp|abarth|abb|abbott|abbvie|abc|able|abogado|abudhabi|ac|academy|
+    accenture|accountant|accountants|aco|active|actor|ad|adac|ads|adult|ae|aeg|aero|
+    aetna|af|afamilycompany|afl|ag|agakhan|agency|ai|aig|aigo|airbus|airforce|
+    airtel|akdn|al|alfaromeo|alibaba|alipay|allfinanz|allstate|ally|alsace|alstom|
+    am|americanexpress|americanfamily|amex|amfam|amica|amsterdam|analytics|android|
+    anquan|anz|ao|aol|apartments|app|apple|aq|aquarelle|ar|aramco|archi|army|arpa|
+    art|arte|as|asda|asia|associates|at|athleta|attorney|au|auction|audi|audible|
+    audio|auspost|author|auto|autos|avianca|aw|aws|ax|axa|az|azure|ba|baby|baidu|
+    banamex|bananarepublic|band|bank|bar|barcelona|barclaycard|barclays|barefoot|
+    bargains|baseball|basketball|bauhaus|bayern|bb|bbc|bbt|bbva|bcg|bcn|bd|be|beats|
+    beauty|beer|bentley|berlin|best|bestbuy|bet|bf|bg|bh|bharti|bi|bible|bid|bike|
+    bing|bingo|bio|biz|bj|black|blackfriday|blanco|blockbuster|blog|bloomberg|blue|
+    bm|bms|bmw|bn|bnl|bnpparibas|bo|boats|boehringer|bofa|bom|bond|boo|book|booking|
+    boots|bosch|bostik|bot|boutique|br|bradesco|bridgestone|broadway|broker|brother|
+    brussels|bs|bt|budapest|bugatti|build|builders|business|buy|buzz|bv|bw|by|bz|
+    bzh|ca|cab|cafe|cal|call|calvinklein|cam|camera|camp|cancerresearch|canon|
+    capetown|capital|capitalone|car|caravan|cards|care|career|careers|cars|cartier|
+    casa|case|caseih|cash|casino|cat|catering|cba|cbn|cbre|cbs|cc|cd|ceb|center|ceo|
+    cern|cf|cfa|cfd|cg|ch|chanel|channel|chase|chat|cheap|chintai|chloe|christmas|
+    chrome|chrysler|church|ci|cipriani|circle|cisco|citadel|citi|citic|city|
+    cityeats|ck|cl|claims|cleaning|click|clinic|clinique|clothing|cloud|club|
+    clubmed|cm|cn|co|coach|codes|coffee|college|cologne|com|comcast|commbank|
+    community|company|compare|computer|comsec|condos|construction|consulting|
+    contact|contractors|cooking|cookingchannel|cool|coop|corsica|country|coupon|
+    coupons|courses|cr|credit|creditcard|creditunion|cricket|crown|crs|cruises|csc|
+    cu|cuisinella|cv|cw|cx|cy|cymru|cyou|cz|dabur|dad|dance|date|dating|datsun|day|
+    dclk|dds|de|deal|dealer|deals|degree|delivery|dell|deloitte|delta|democrat|
+    dental|dentist|desi|design|dev|dhl|diamonds|diet|digital|direct|directory|
+    discount|discover|dish|diy|dj|dk|dm|dnp|do|docs|doctor|dodge|dog|doha|domains|
+    dot|download|drive|dtv|dubai|duck|dunlop|duns|dupont|durban|dvag|dvr|dz|earth|
+    eat|ec|eco|edeka|edu|education|ee|eg|email|emerck|energy|engineer|engineering|
+    enterprises|epost|epson|equipment|er|ericsson|erni|es|esq|estate|esurance|et|eu|
+    eurovision|eus|events|everbank|exchange|expert|exposed|express|extraspace|fage|
+    fail|fairwinds|faith|family|fan|fans|farm|farmers|fashion|fast|fedex|feedback|
+    ferrari|ferrero|fi|fiat|fidelity|fido|film|final|finance|financial|fire|
+    firestone|firmdale|fish|fishing|fit|fitness|fj|fk|flickr|flights|flir|florist|
+    flowers|fly|fm|fo|foo|foodnetwork|football|ford|forex|forsale|forum|foundation|
+    fox|fr|free|fresenius|frl|frogans|frontdoor|frontier|ftr|fujitsu|fujixerox|fund|
+    furniture|futbol|fyi|ga|gal|gallery|gallo|gallup|game|games|gap|garden|gb|gbiz|
+    gd|gdn|ge|gea|gent|genting|george|gf|gg|ggee|gh|gi|gift|gifts|gives|giving|gl|
+    glade|glass|gle|global|globo|gm|gmail|gmbh|gmo|gmx|gn|godaddy|gold|goldpoint|
+    golf|goo|goodhands|goodyear|goog|google|gop|got|gov|gp|gq|gr|grainger|graphics|
+    gratis|green|gripe|group|gs|gt|gu|guardian|gucci|guge|guide|guitars|guru|gw|gy|
+    hamburg|hangout|haus|hbo|hdfc|hdfcbank|health|healthcare|help|helsinki|here|
+    hermes|hgtv|hiphop|hisamitsu|hitachi|hiv|hk|hkt|hm|hn|hockey|holdings|holiday|
+    homedepot|homegoods|homes|homesense|honda|honeywell|horse|host|hosting|hot|
+    hoteles|hotmail|house|how|hr|hsbc|ht|htc|hu|hughes|hyatt|hyundai|ibm|icbc|ice|
+    icu|id|ie|ieee|ifm|iinet|ikano|il|im|imamat|imdb|immo|immobilien|in|industries|
+    infiniti|info|ing|ink|institute|insurance|insure|int|intel|international|intuit|
+    investments|io|ipiranga|iq|ir|irish|is|iselect|ismaili|ist|istanbul|it|itau|itv|
+    iveco|iwc|jaguar|java|jcb|jcp|je|jeep|jetzt|jewelry|jlc|jll|jm|jmp|jnj|jo|jobs|
+    joburg|jot|joy|jp|jpmorgan|jprs|juegos|juniper|kaufen|kddi|ke|kerryhotels|
+    kerrylogistics|kerryproperties|kfh|kg|kh|ki|kia|kim|kinder|kindle|kitchen|kiwi|
+    km|kn|koeln|komatsu|kosher|kp|kpmg|kpn|kr|krd|kred|kuokgroup|kw|ky|kyoto|kz|la|
+    lacaixa|ladbrokes|lamborghini|lamer|lancaster|lancia|lancome|land|landrover|
+    lanxess|lasalle|lat|latino|latrobe|law|lawyer|lb|lc|lds|lease|leclerc|lefrak|
+    legal|lego|lexus|lgbt|li|liaison|lidl|life|lifeinsurance|lifestyle|lighting|
+    like|lilly|limited|limo|lincoln|linde|link|lipsy|live|living|lixil|lk|loan|
+    loans|locker|locus|loft|lol|london|lotte|lotto|love|lpl|lplfinancial|lr|ls|lt|
+    ltd|ltda|lu|lundbeck|lupin|luxe|luxury|lv|ly|ma|macys|madrid|maif|maison|makeup|
+    man|management|mango|market|marketing|markets|marriott|marshalls|maserati|
+    mattel|mba|mc|mcd|mcdonalds|mckinsey|md|me|med|media|meet|melbourne|meme|
+    memorial|men|menu|meo|metlife|mg|mh|miami|microsoft|mil|mini|mint|mit|
+    mitsubishi|mk|ml|mlb|mls|mm|mma|mn|mo|mobi|mobily|moda|moe|moi|mom|monash|money|
+    monster|montblanc|mopar|mormon|mortgage|moscow|motorcycles|mov|movie|movistar|
+    mp|mq|mr|ms|msd|mt|mtn|mtpc|mtr|mu|museum|mutual|mutuelle|mv|mw|mx|my|mz|na|nab|
+    nadex|nagoya|name|nationwide|natura|navy|nba|nc|ne|nec|net|netbank|netflix|
+    network|neustar|new|newholland|news|next|nextdirect|nexus|nf|nfl|ng|ngo|nhk|ni|
+    nico|nike|nikon|ninja|nissan|nissay|nl|no|nokia|northwesternmutual|norton|now|
+    nowruz|nowtv|np|nr|nra|nrw|ntt|nu|nyc|nz|obi|observer|off|office|okinawa|olayan|
+    olayangroup|oldnavy|ollo|om|omega|one|ong|onl|online|onyourside|ooo|open|oracle|
+    orange|org|organic|orientexpress|origins|osaka|otsuka|ott|ovh|pa|page|
+    pamperedchef|panasonic|panerai|paris|pars|partners|parts|party|passagens|pay|
+    pccw|pe|pet|pf|pfizer|pg|ph|pharmacy|philips|photo|photography|photos|physio|
+    piaget|pics|pictet|pictures|pid|pin|ping|pink|pioneer|pizza|pk|pl|place|play|
+    playstation|plumbing|plus|pm|pn|pnc|pohl|poker|politie|porn|post|pr|pramerica|
+    praxi|press|prime|pro|prod|productions|prof|progressive|promo|properties|
+    property|protection|pru|prudential|ps|pt|pub|pw|pwc|py|qa|qpon|quebec|quest|qvc|
+    racing|radio|raid|re|read|realestate|realtor|realty|recipes|red|redstone|
+    redumbrella|rehab|reise|reisen|reit|ren|rent|rentals|repair|report|republican|
+    rest|restaurant|review|reviews|rexroth|rich|richardli|ricoh|rightathome|rio|rip|
+    ro|rocher|rocks|rodeo|rogers|room|rs|rsvp|ru|ruhr|run|rw|rwe|ryukyu|sa|saarland|
+    safe|safety|sakura|sale|salon|samsclub|samsung|sandvik|sandvikcoromant|sanofi|
+    sap|sapo|sarl|sas|save|saxo|sb|sbi|sbs|sc|sca|scb|schaeffler|schmidt|
+    scholarships|school|schule|schwarz|science|scjohnson|scor|scot|sd|se|seat|
+    secure|security|seek|select|sener|services|ses|seven|sew|sex|sexy|sfr|sg|sh|
+    shangrila|sharp|shaw|shell|shia|shiksha|shoes|shop|shopping|shouji|show|
+    showtime|shriram|si|silk|sina|singles|site|sj|sk|ski|skin|sky|skype|sl|sling|sm|
+    smart|smile|sn|sncf|so|soccer|social|softbank|software|sohu|solar|solutions|
+    song|sony|soy|space|spiegel|spot|spreadbetting|sr|srl|srt|st|stada|staples|star|
+    starhub|statebank|statefarm|statoil|stc|stcgroup|stockholm|storage|store|stream|
+    studio|study|style|su|sucks|supplies|supply|support|surf|surgery|suzuki|sv|
+    swatch|swiftcover|swiss|sx|sy|sydney|symantec|systems|sz|tab|taipei|talk|taobao|
+    target|tatamotors|tatar|tattoo|tax|taxi|tc|tci|td|tdk|team|tech|technology|tel|
+    telecity|telefonica|temasek|tennis|teva|tf|tg|th|thd|theater|theatre|tiaa|
+    tickets|tienda|tiffany|tips|tires|tirol|tj|tjmaxx|tjx|tk|tkmaxx|tl|tm|tmall|tn|
+    to|today|tokyo|tools|top|toray|toshiba|total|tours|town|toyota|toys|tr|trade|
+    trading|training|travel|travelchannel|travelers|travelersinsurance|trust|trv|tt|
+    tube|tui|tunes|tushu|tv|tvs|tw|tz|ua|ubank|ubs|uconnect|ug|uk|unicom|university|
+    uno|uol|ups|us|uy|uz|va|vacations|vana|vanguard|vc|ve|vegas|ventures|verisign|
+    versicherung|vet|vg|vi|viajes|video|vig|viking|villas|vin|vip|virgin|visa|
+    vision|vista|vistaprint|viva|vivo|vlaanderen|vn|vodka|volkswagen|volvo|vote|
+    voting|voto|voyage|vu|vuelos|wales|walmart|walter|wang|wanggou|warman|watch|
+    watches|weather|weatherchannel|webcam|weber|website|wed|wedding|weibo|weir|wf|
+    whoswho|wien|wiki|williamhill|win|windows|wine|winners|wme|wolterskluwer|
+    woodside|work|works|world|wow|ws|wtc|wtf|xbox|xerox|xfinity|xihuan|xin|xperia|
+    xxx|xyz|yachts|yahoo|yamaxun|yandex|ye|yodobashi|yoga|yokohama|you|youtube|yt|
+    yun|za|zappos|zara|zero|zip|zippo|zm|zone|zuerich|zw';
+}
+
+