From 068743545bdbb42f54a97c51ba1828f1a9602ba6 Mon Sep 17 00:00:00 2001 From: Benjamyn Love Date: Sun, 26 Aug 2018 23:59:23 +1000 Subject: [PATCH 01/11] First test of py3 version --- setup.py | 2 +- whois/whois.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 78cd3fb..3a53954 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from setuptools import setup, find_packages import sys, os -__version__ = '0.1' +__version__ = '0.2' setup(name='whois', version=__version__, diff --git a/whois/whois.py b/whois/whois.py index 5c4cee0..cfd49f5 100644 --- a/whois/whois.py +++ b/whois/whois.py @@ -12,9 +12,9 @@ import socket import re import logging import urllib -import urllib2 +#import urllib2 -import error +#import errormyhost import flags class Whois(object): @@ -61,13 +61,13 @@ class Whois(object): if self.settings.get("http-method").lower() == "post": logging.debug("sendHTTPQuery: Connecting to whois server using POST") - req = urllib2.Request(whoisServer, param) + req = urllib.Request(whoisServer, param) else: # GET logging.debug("sendHTTPQuery: Connecting to whois server using GET") - req = urllib2.Request((whoisServer.endswith("?") and whoisServer or whoisServer+"?") + param) + req = urllib.Request((whoisServer.endswith("?") and whoisServer or whoisServer+"?") + param) - data = urllib2.urlopen(req).read() - print data + data = urllib.urlopen(req).read() + print(data) return data -- 2.43.5 From f636c9f2ad9117e4893f86f7bb6ad6c4070e7773 Mon Sep 17 00:00:00 2001 From: Benjamyn Love Date: Mon, 27 Aug 2018 22:25:56 +1000 Subject: [PATCH 02/11] Broken mess --- whois/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/whois/parser.py b/whois/parser.py index c14667a..0c3130f 100644 --- a/whois/parser.py +++ b/whois/parser.py @@ -134,7 +134,7 @@ class Parser(object): for key in self.parseConf: matches = re.findall(self.parseConf[key], self.text, re.MULTILINE) if matches: - logging.debug("run: regex matches found for key %s. %s"%(key, matches)) + logging.debug("run: regex matches found for key %s. %s"%(key, matches)) result.update({key: map(lambda x: x.strip(), matches)}) else: logging.debug("run: No match for %s"%(key)) -- 2.43.5 From d9f11956c2edfd98ecaf5da3997434be76750b8b Mon Sep 17 00:00:00 2001 From: Benjamyn Love Date: Mon, 27 Aug 2018 23:31:41 +1000 Subject: [PATCH 03/11] Fixed, hopefully, issues with enoding --- whois/py3/whois.py | 136 +++++++++++++++++++++++++++++++++++++++++++++ whois/whois.py | 28 +++++----- whois/whois.py2 | 136 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 286 insertions(+), 14 deletions(-) create mode 100644 whois/py3/whois.py create mode 100644 whois/whois.py2 diff --git a/whois/py3/whois.py b/whois/py3/whois.py new file mode 100644 index 0000000..d450070 --- /dev/null +++ b/whois/py3/whois.py @@ -0,0 +1,136 @@ +# -*- coding: utf-8 -*- + +# __ __ __ __ ______ __ ______ +#/\ \ _ \ \ /\ \_\ \ /\ __ \ /\ \ /\ ___\ +#\ \ \/ ".\ \\ \ __ \\ \ \/\ \\ \ \\ \___ \ +# \ \__/".~\_\\ \_\ \_\\ \_____\\ \_\\/\_____\ +# \/_/ \/_/ \/_/\/_/ \/_____/ \/_/ \/_____/ + +import sys +import os +import socket +import re +import logging +import urllib.request, urllib.parse, urllib.error +import urllib.request, urllib.error, urllib.parse + +from . import error +from . import flags + +class Whois(object): + def __init__(self, domain, debug=False): + if debug: + logging.basicConfig(level=logging.DEBUG) + logging.debug("__init__: DEBUG is set to True") + + self.domain = str(domain, "utf-8").encode("idna") + self.tld = self.domain.split(".")[-1] + + self.currPath = os.path.dirname(os.path.realpath(__file__)) + self.tldPath = os.path.join(self.currPath, "tlds") + self.tldList = os.listdir(self.tldPath) + + logging.debug("__init__: Setting initial variables.. self.currPath = %s / self.tldPath = %s / self.tldList = %s" + %(self.currPath, self.tldPath, self.tldList)) + + self.settings = {} + + if self.tld in self.tldList: + logging.debug("__init__: Loading tld configuration file...") + + _settings = {} + exec(compile(open(os.path.join(self.tldPath, self.tld)).read(), os.path.join(self.tldPath, self.tld), 'exec'), {}, _settings) + + if "server" in _settings: + logging.debug("__init__: Settings: %s"%(_settings["server"])) + self.settings.update(_settings["server"]) + else: + logging.debug("__init__: No server settings found") + + def chooseServer(self): + '''Choose whois server by detecting tld of given domain.''' + if "host" in self.settings: + logging.debug("chooseServer: Whois server addr: %s"%(self.settings["host"])) + return self.settings["host"] + else: + logging.debug("chooseServer: Whois server addr: %s"%(self.tld + ".whois-servers.net")) + return self.tld + ".whois-servers.net" + + def sendHTTPQuery(self, whoisServer): + param = urllib.parse.urlencode({self.settings["http-arg"]: self.domain}) + + if self.settings.get("http-method").lower() == "post": + logging.debug("sendHTTPQuery: Connecting to whois server using POST") + req = urllib.request.Request(whoisServer, param) + else: # GET + logging.debug("sendHTTPQuery: Connecting to whois server using GET") + req = urllib.request.Request((whoisServer.endswith("?") and whoisServer or whoisServer+"?") + param) + + data = urllib.request.urlopen(req).read() + print(data) + + return data + + def sendQuery(self, whoisServer): + '''Send query to whois server.''' + logging.debug("sendQuery: Connecting to whois server") + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + try: + s.connect((whoisServer, 43)) + + except: + # FIXME: Create a exception class for this + logging.error("sendQuery: Error connecting to whois server %s"%(whoisServer)) + return False + + try: + msg = self.settings['format'][whoisServer].replace("%DOMAIN%", self.domain) + "\r\n" + + except: + msg = self.domain + "\r\n" + + logging.debug("sendQuery: Sending data.. %s"%(msg)) + + s.send(msg) + + result = "" + + while True: + buffer = s.recv(512) + + if not buffer: + break + + result += buffer + + finalResult = result.replace("\r\n", "\n") + + logging.debug("sendQuery: result: %s"%(finalResult)) + + return finalResult + + def query(self, redirect=True, return_type=flags.RETURN_TYPE_LIST): + '''Start whole process of whois query. This method will do them all.''' + whoisServer = self.chooseServer() + + if self.settings.get("method") == "http": + result = self.sendHTTPQuery(whoisServer) + else: + result = self.sendQuery(whoisServer) + + if redirect and "redirect" in self.settings: + logging.debug("query: Redirection found. Connecting to given server address") + + redirection = re.findall(self.settings["redirect"], result, re.MULTILINE) + + while redirection and len(redirection) >= 1: + whoisServer = redirection[0] + result = self.sendQuery(whoisServer) + redirection = re.findall(self.settings["redirect"], result) + + + if return_type == flags.RETURN_TYPE_LIST: + return whoisServer, result + else: + return {"whoisServer": whoisServer, "result": result} diff --git a/whois/whois.py b/whois/whois.py index cfd49f5..ac26be5 100644 --- a/whois/whois.py +++ b/whois/whois.py @@ -11,19 +11,19 @@ import os import socket import re import logging -import urllib -#import urllib2 +import urllib.request, urllib.parse, urllib.error -#import errormyhost +import error import flags class Whois(object): - def __init__(self, domain, debug=False): + def __init__(self, domain, debug=True): if debug: logging.basicConfig(level=logging.DEBUG) logging.debug("__init__: DEBUG is set to True") - self.domain = unicode(domain, "utf-8").encode("idna") + self.domain = domain + self.tld = self.domain.split(".")[-1] self.currPath = os.path.dirname(os.path.realpath(__file__)) @@ -39,7 +39,7 @@ class Whois(object): logging.debug("__init__: Loading tld configuration file...") _settings = {} - execfile(os.path.join(self.tldPath, self.tld), {}, _settings) + exec(compile(open(os.path.join(self.tldPath, self.tld)).read(), os.path.join(self.tldPath, self.tld), 'exec'), {}, _settings) if "server" in _settings: logging.debug("__init__: Settings: %s"%(_settings["server"])) @@ -57,17 +57,17 @@ class Whois(object): return self.tld + ".whois-servers.net" def sendHTTPQuery(self, whoisServer): - param = urllib.urlencode({self.settings["http-arg"]: self.domain}) + param = urllib.parse.urlencode({self.settings["http-arg"]: self.domain}) if self.settings.get("http-method").lower() == "post": logging.debug("sendHTTPQuery: Connecting to whois server using POST") - req = urllib.Request(whoisServer, param) + req = urllib.request.Request(whoisServer, param) else: # GET logging.debug("sendHTTPQuery: Connecting to whois server using GET") - req = urllib.Request((whoisServer.endswith("?") and whoisServer or whoisServer+"?") + param) + req = urllib.request.Request((whoisServer.endswith("?") and whoisServer or whoisServer+"?") + param) - data = urllib.urlopen(req).read() - print(data) + data = urllib.request.urlopen(req).read() + print(data) return data @@ -91,8 +91,8 @@ class Whois(object): msg = self.domain + "\r\n" logging.debug("sendQuery: Sending data.. %s"%(msg)) - - s.send(msg) + # print(msg.encode()) + s.send(msg.encode('utf-8')) result = "" @@ -102,7 +102,7 @@ class Whois(object): if not buffer: break - result += buffer + result += buffer.decode() finalResult = result.replace("\r\n", "\n") diff --git a/whois/whois.py2 b/whois/whois.py2 new file mode 100644 index 0000000..5c4cee0 --- /dev/null +++ b/whois/whois.py2 @@ -0,0 +1,136 @@ +# -*- coding: utf-8 -*- + +# __ __ __ __ ______ __ ______ +#/\ \ _ \ \ /\ \_\ \ /\ __ \ /\ \ /\ ___\ +#\ \ \/ ".\ \\ \ __ \\ \ \/\ \\ \ \\ \___ \ +# \ \__/".~\_\\ \_\ \_\\ \_____\\ \_\\/\_____\ +# \/_/ \/_/ \/_/\/_/ \/_____/ \/_/ \/_____/ + +import sys +import os +import socket +import re +import logging +import urllib +import urllib2 + +import error +import flags + +class Whois(object): + def __init__(self, domain, debug=False): + if debug: + logging.basicConfig(level=logging.DEBUG) + logging.debug("__init__: DEBUG is set to True") + + self.domain = unicode(domain, "utf-8").encode("idna") + self.tld = self.domain.split(".")[-1] + + self.currPath = os.path.dirname(os.path.realpath(__file__)) + self.tldPath = os.path.join(self.currPath, "tlds") + self.tldList = os.listdir(self.tldPath) + + logging.debug("__init__: Setting initial variables.. self.currPath = %s / self.tldPath = %s / self.tldList = %s" + %(self.currPath, self.tldPath, self.tldList)) + + self.settings = {} + + if self.tld in self.tldList: + logging.debug("__init__: Loading tld configuration file...") + + _settings = {} + execfile(os.path.join(self.tldPath, self.tld), {}, _settings) + + if "server" in _settings: + logging.debug("__init__: Settings: %s"%(_settings["server"])) + self.settings.update(_settings["server"]) + else: + logging.debug("__init__: No server settings found") + + def chooseServer(self): + '''Choose whois server by detecting tld of given domain.''' + if "host" in self.settings: + logging.debug("chooseServer: Whois server addr: %s"%(self.settings["host"])) + return self.settings["host"] + else: + logging.debug("chooseServer: Whois server addr: %s"%(self.tld + ".whois-servers.net")) + return self.tld + ".whois-servers.net" + + def sendHTTPQuery(self, whoisServer): + param = urllib.urlencode({self.settings["http-arg"]: self.domain}) + + if self.settings.get("http-method").lower() == "post": + logging.debug("sendHTTPQuery: Connecting to whois server using POST") + req = urllib2.Request(whoisServer, param) + else: # GET + logging.debug("sendHTTPQuery: Connecting to whois server using GET") + req = urllib2.Request((whoisServer.endswith("?") and whoisServer or whoisServer+"?") + param) + + data = urllib2.urlopen(req).read() + print data + + return data + + def sendQuery(self, whoisServer): + '''Send query to whois server.''' + logging.debug("sendQuery: Connecting to whois server") + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + try: + s.connect((whoisServer, 43)) + + except: + # FIXME: Create a exception class for this + logging.error("sendQuery: Error connecting to whois server %s"%(whoisServer)) + return False + + try: + msg = self.settings['format'][whoisServer].replace("%DOMAIN%", self.domain) + "\r\n" + + except: + msg = self.domain + "\r\n" + + logging.debug("sendQuery: Sending data.. %s"%(msg)) + + s.send(msg) + + result = "" + + while True: + buffer = s.recv(512) + + if not buffer: + break + + result += buffer + + finalResult = result.replace("\r\n", "\n") + + logging.debug("sendQuery: result: %s"%(finalResult)) + + return finalResult + + def query(self, redirect=True, return_type=flags.RETURN_TYPE_LIST): + '''Start whole process of whois query. This method will do them all.''' + whoisServer = self.chooseServer() + + if self.settings.get("method") == "http": + result = self.sendHTTPQuery(whoisServer) + else: + result = self.sendQuery(whoisServer) + + if redirect and "redirect" in self.settings: + logging.debug("query: Redirection found. Connecting to given server address") + + redirection = re.findall(self.settings["redirect"], result, re.MULTILINE) + + while redirection and len(redirection) >= 1: + whoisServer = redirection[0] + result = self.sendQuery(whoisServer) + redirection = re.findall(self.settings["redirect"], result) + + + if return_type == flags.RETURN_TYPE_LIST: + return whoisServer, result + else: + return {"whoisServer": whoisServer, "result": result} -- 2.43.5 From c07728c41fbe1c42f55361dbf539dd2fd381c0de Mon Sep 17 00:00:00 2001 From: Benjamyn Love Date: Mon, 27 Aug 2018 23:57:28 +1000 Subject: [PATCH 04/11] Fixed issue with loading relative paths Updated codebase to use python3 Seems to work --- whois/__init__.py | 4 ++-- whois/whois.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/whois/__init__.py b/whois/__init__.py index 0717af5..e84b071 100644 --- a/whois/__init__.py +++ b/whois/__init__.py @@ -7,5 +7,5 @@ # \/_/ \/_/ \/_/ \/_/ \/_/ from whois import Whois as whois -from parser import * -from flags import * +import flags +import parser diff --git a/whois/whois.py b/whois/whois.py index ac26be5..234b893 100644 --- a/whois/whois.py +++ b/whois/whois.py @@ -17,7 +17,7 @@ import error import flags class Whois(object): - def __init__(self, domain, debug=True): + def __init__(self, domain, debug=False): if debug: logging.basicConfig(level=logging.DEBUG) logging.debug("__init__: DEBUG is set to True") -- 2.43.5 From 17aec68fb3da656cff910b7e502a0933a5fa7eea Mon Sep 17 00:00:00 2001 From: Benjamyn Love Date: Wed, 29 Aug 2018 18:20:51 +1000 Subject: [PATCH 05/11] Fixed the imports, they now use python3 style --- whois/__init__.py | 5 +++-- whois/whois.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/whois/__init__.py b/whois/__init__.py index e84b071..cae3d54 100644 --- a/whois/__init__.py +++ b/whois/__init__.py @@ -6,6 +6,7 @@ # \ \_\\ \_\\"\_\\ \_\ \ \_\ # \/_/ \/_/ \/_/ \/_/ \/_/ -from whois import Whois as whois -import flags +from .whois import Whois +# import flags +from . import flags import parser diff --git a/whois/whois.py b/whois/whois.py index 234b893..0fba3d8 100644 --- a/whois/whois.py +++ b/whois/whois.py @@ -13,8 +13,9 @@ import re import logging import urllib.request, urllib.parse, urllib.error -import error -import flags +from . import flags +#import error +#import flags class Whois(object): def __init__(self, domain, debug=False): -- 2.43.5 From 01e2e50ce7ba15db19c9ac45a9404cff1259b336 Mon Sep 17 00:00:00 2001 From: Benjamyn Love Date: Thu, 30 Aug 2018 02:18:36 +1000 Subject: [PATCH 06/11] fixed init loading and have started porting the parser --- whois/__init__.py | 2 +- whois/parser.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/whois/__init__.py b/whois/__init__.py index cae3d54..17219fd 100644 --- a/whois/__init__.py +++ b/whois/__init__.py @@ -9,4 +9,4 @@ from .whois import Whois # import flags from . import flags -import parser +from . import parser diff --git a/whois/parser.py b/whois/parser.py index 0c3130f..e450ec1 100644 --- a/whois/parser.py +++ b/whois/parser.py @@ -6,7 +6,7 @@ # \ \_\ \ \_\ \_\\ \_\ \_\\/\_____\\ \_____\\ \_\ \_\ # \/_/ \/_/\/_/ \/_/ /_/ \/_____/ \/_____/ \/_/ /_/ -import error +from . import error import re import sys import os -- 2.43.5 From 74675c6fb0d82c97e491380fe7bb7e9007492a56 Mon Sep 17 00:00:00 2001 From: Benjamyn Love Date: Sun, 2 Sep 2018 15:57:46 +1000 Subject: [PATCH 07/11] Fixed up parser code for python3 --- whois/parser.py | 10 +-- whois/parser.py2 | 145 ++++++++++++++++++++++++++++++++++++++++++++ whois/py3/parser.py | 145 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 295 insertions(+), 5 deletions(-) create mode 100644 whois/parser.py2 create mode 100644 whois/py3/parser.py diff --git a/whois/parser.py b/whois/parser.py index e450ec1..2844953 100644 --- a/whois/parser.py +++ b/whois/parser.py @@ -45,7 +45,7 @@ class Parser(object): logging.basicConfig(level=logging.DEBUG) logging.debug("__init__: DEBUG is set to True") - self.domain = unicode(domain, "utf-8").encode("idna") + self.domain = str(domain) if not text: raise error.InvalidInputText(text) @@ -63,13 +63,13 @@ class Parser(object): self.parseDefaultConf = {} logging.debug("__init__: Loading default tld configuration file") - execfile(os.path.join(self.tldPath, "default"), {}, self.parseDefaultConf) + exec(compile(open(os.path.join(self.tldPath, "default")).read(), os.path.join(self.tldPath, "default"), 'exec'), {}, self.parseDefaultConf) self.parseDefaultConf = self.parseDefaultConf.get("parse") self.parseConf = {} try: - execfile(os.path.join(self.tldPath, self.tld), {}, self.parseConf) + exec(compile(open(os.path.join(self.tldPath, self.tld)).read(), os.path.join(self.tldPath, self.tld), 'exec'), {}, self.parseConf) self.parseConf = self.parseConf.get("parse") @@ -113,7 +113,7 @@ class Parser(object): logging.debug("__init__: Loading configuration file of tld name %s"%(lcTLD)) - execfile(os.path.join(self.tldPath, "%s"%(lcTLD)), {}, lcConf) + exec(compile(open(os.path.join(self.tldPath, "%s"%(lcTLD))).read(), os.path.join(self.tldPath, "%s"%(lcTLD)), 'exec'), {}, lcConf) lcConf = lcConf.get("parse") self.parseConf.update(lcConf.get(lcWS)) @@ -135,7 +135,7 @@ class Parser(object): matches = re.findall(self.parseConf[key], self.text, re.MULTILINE) if matches: logging.debug("run: regex matches found for key %s. %s"%(key, matches)) - result.update({key: map(lambda x: x.strip(), matches)}) + result.update({key: [x.strip() for x in matches]}) else: logging.debug("run: No match for %s"%(key)) diff --git a/whois/parser.py2 b/whois/parser.py2 new file mode 100644 index 0000000..e450ec1 --- /dev/null +++ b/whois/parser.py2 @@ -0,0 +1,145 @@ +# -*- coding: utf-8 -*- + +# ______ ______ ______ ______ ______ ______ +#/\ == \/\ __ \ /\ == \ /\ ___\ /\ ___\ /\ == \ +#\ \ _-/\ \ __ \\ \ __< \ \___ \\ \ __\ \ \ __< +# \ \_\ \ \_\ \_\\ \_\ \_\\/\_____\\ \_____\\ \_\ \_\ +# \/_/ \/_/\/_/ \/_/ /_/ \/_____/ \/_____/ \/_/ /_/ + +from . import error +import re +import sys +import os +import time +import logging + +def convertDate(s): + """Convert any date string found in WHOIS to a datetime object. + """ + # Source from https://code.google.com/p/pywhois/source/browse/whois/parser.py + known_formats = [ + '%d-%b-%Y', # 02-jan-2000 + '%Y-%m-%d', # 2000-01-02 + '%d.%m.%Y', # 2.1.2000 + '%Y.%m.%d', # 2000.01.02 + '%Y/%m/%d', # 2000/01/02 + '%d-%b-%Y %H:%M:%S %Z', # 24-Jul-2009 13:20:03 UTC + '%a %b %d %H:%M:%S %Z %Y', # Tue Jun 21 23:59:59 GMT 2011 + '%Y-%m-%dT%H:%M:%SZ', # 2007-01-26T19:10:31Z + '%Y. %m. %d.', # 2012. 04. 03. - whois.krnic.net + '%d/%m/%Y %H:%M:%S', # 14/09/2013 00:59:59 - whois.nic.im + '%Y/%m/%d %H:%M:%S (%Z)', # 2012/07/01 01:05:01 (JST) - whois.jprs.jp + ] + + for known_format in known_formats: + try: + return time.mktime(time.strptime(s.strip(), known_format)) + except ValueError as e: + pass # Wrong format, keep trying + + return s + +class Parser(object): + def __init__(self, domain, text, whoisServer=None, debug=False): + if debug: + logging.basicConfig(level=logging.DEBUG) + logging.debug("__init__: DEBUG is set to True") + + self.domain = unicode(domain, "utf-8").encode("idna") + + if not text: + raise error.InvalidInputText(text) + + self.text = text + self.whoisServer = whoisServer and whoisServer or "default" + + self.tld = self.domain.split(".")[-1] + + self.currPath = os.path.dirname(os.path.realpath(__file__)) + self.tldPath = os.path.join(self.currPath, "tlds") + + logging.debug("__init__: Setting initial variables...\nself.domain: %s\nself.text = %s\nself.whoisServer = %s\nself.tld = %s\nself.currPath = %s\nself.tldPath = %s" + %(self.domain, self.text, self.whoisServer, self.tld, self.currPath, self.tldPath)) + + self.parseDefaultConf = {} + logging.debug("__init__: Loading default tld configuration file") + execfile(os.path.join(self.tldPath, "default"), {}, self.parseDefaultConf) + self.parseDefaultConf = self.parseDefaultConf.get("parse") + + self.parseConf = {} + + try: + execfile(os.path.join(self.tldPath, self.tld), {}, self.parseConf) + + self.parseConf = self.parseConf.get("parse") + + # THERE IS NO "parse" in the tld config AND THERE IS regex for specified server in default conf + if not self.parseConf and whoisServer not in self.parseDefaultConf: + self.parseConf = self.parseDefaultConf.get("default") + + # THERE IS NO "parse" in the tld config + elif not self.parseConf: + self.parseConf = self.parseDefaultConf.get(whoisServer) + + # THERE IS "parse" in the tld config AND THERE IS regex for specified server + elif self.whoisServer in self.parseConf: + self.parseConf = self.parseConf.get(self.whoisServer) + + # THERE IS "parse" in the tld config AND THERE IS "default" regex in the tld config AND + # THERE IS NO regex for specified server + elif "default" in self.parseConf: + self.parseConf = self.parseConf.get("default") + + # THEE IS "parse" in the tld config AND THERE IS NO "default" regex in the tld config + # MAYBE empty file? + else: + self.parseConf = self.parseDefaultConf.get("default") + + # Check for LoadConf + _parseConf = self.parseConf + self.parseConf = {} + + if "LoadConf" in _parseConf: + logging.debug("__init__: LoadConf found in parser config") + try: + # / + # e.g. org/whois.publicinternetregistry.net + lc = _parseConf["LoadConf"].split("/", 1) + + lcTLD = lc[0] + lcWS = lc[1] + + lcConf = {} + + logging.debug("__init__: Loading configuration file of tld name %s"%(lcTLD)) + + execfile(os.path.join(self.tldPath, "%s"%(lcTLD)), {}, lcConf) + lcConf = lcConf.get("parse") + + self.parseConf.update(lcConf.get(lcWS)) + + except: + pass + + self.parseConf.update(_parseConf) + + except: + self.parseConf = self.parseDefaultConf.get("default") + + + logging.debug("__init__: self.parseConf = %s"%(self.parseConf)) + + def parse(self): + result = {} + for key in self.parseConf: + matches = re.findall(self.parseConf[key], self.text, re.MULTILINE) + if matches: + logging.debug("run: regex matches found for key %s. %s"%(key, matches)) + result.update({key: map(lambda x: x.strip(), matches)}) + else: + logging.debug("run: No match for %s"%(key)) + + return result + + + diff --git a/whois/py3/parser.py b/whois/py3/parser.py new file mode 100644 index 0000000..8fe590f --- /dev/null +++ b/whois/py3/parser.py @@ -0,0 +1,145 @@ +# -*- coding: utf-8 -*- + +# ______ ______ ______ ______ ______ ______ +#/\ == \/\ __ \ /\ == \ /\ ___\ /\ ___\ /\ == \ +#\ \ _-/\ \ __ \\ \ __< \ \___ \\ \ __\ \ \ __< +# \ \_\ \ \_\ \_\\ \_\ \_\\/\_____\\ \_____\\ \_\ \_\ +# \/_/ \/_/\/_/ \/_/ /_/ \/_____/ \/_____/ \/_/ /_/ + +from . import error +import re +import sys +import os +import time +import logging + +def convertDate(s): + """Convert any date string found in WHOIS to a datetime object. + """ + # Source from https://code.google.com/p/pywhois/source/browse/whois/parser.py + known_formats = [ + '%d-%b-%Y', # 02-jan-2000 + '%Y-%m-%d', # 2000-01-02 + '%d.%m.%Y', # 2.1.2000 + '%Y.%m.%d', # 2000.01.02 + '%Y/%m/%d', # 2000/01/02 + '%d-%b-%Y %H:%M:%S %Z', # 24-Jul-2009 13:20:03 UTC + '%a %b %d %H:%M:%S %Z %Y', # Tue Jun 21 23:59:59 GMT 2011 + '%Y-%m-%dT%H:%M:%SZ', # 2007-01-26T19:10:31Z + '%Y. %m. %d.', # 2012. 04. 03. - whois.krnic.net + '%d/%m/%Y %H:%M:%S', # 14/09/2013 00:59:59 - whois.nic.im + '%Y/%m/%d %H:%M:%S (%Z)', # 2012/07/01 01:05:01 (JST) - whois.jprs.jp + ] + + for known_format in known_formats: + try: + return time.mktime(time.strptime(s.strip(), known_format)) + except ValueError as e: + pass # Wrong format, keep trying + + return s + +class Parser(object): + def __init__(self, domain, text, whoisServer=None, debug=False): + if debug: + logging.basicConfig(level=logging.DEBUG) + logging.debug("__init__: DEBUG is set to True") + + self.domain = str(domain, "utf-8").encode("idna") + + if not text: + raise error.InvalidInputText(text) + + self.text = text + self.whoisServer = whoisServer and whoisServer or "default" + + self.tld = self.domain.split(".")[-1] + + self.currPath = os.path.dirname(os.path.realpath(__file__)) + self.tldPath = os.path.join(self.currPath, "tlds") + + logging.debug("__init__: Setting initial variables...\nself.domain: %s\nself.text = %s\nself.whoisServer = %s\nself.tld = %s\nself.currPath = %s\nself.tldPath = %s" + %(self.domain, self.text, self.whoisServer, self.tld, self.currPath, self.tldPath)) + + self.parseDefaultConf = {} + logging.debug("__init__: Loading default tld configuration file") + exec(compile(open(os.path.join(self.tldPath, "default")).read(), os.path.join(self.tldPath, "default"), 'exec'), {}, self.parseDefaultConf) + self.parseDefaultConf = self.parseDefaultConf.get("parse") + + self.parseConf = {} + + try: + exec(compile(open(os.path.join(self.tldPath, self.tld)).read(), os.path.join(self.tldPath, self.tld), 'exec'), {}, self.parseConf) + + self.parseConf = self.parseConf.get("parse") + + # THERE IS NO "parse" in the tld config AND THERE IS regex for specified server in default conf + if not self.parseConf and whoisServer not in self.parseDefaultConf: + self.parseConf = self.parseDefaultConf.get("default") + + # THERE IS NO "parse" in the tld config + elif not self.parseConf: + self.parseConf = self.parseDefaultConf.get(whoisServer) + + # THERE IS "parse" in the tld config AND THERE IS regex for specified server + elif self.whoisServer in self.parseConf: + self.parseConf = self.parseConf.get(self.whoisServer) + + # THERE IS "parse" in the tld config AND THERE IS "default" regex in the tld config AND + # THERE IS NO regex for specified server + elif "default" in self.parseConf: + self.parseConf = self.parseConf.get("default") + + # THEE IS "parse" in the tld config AND THERE IS NO "default" regex in the tld config + # MAYBE empty file? + else: + self.parseConf = self.parseDefaultConf.get("default") + + # Check for LoadConf + _parseConf = self.parseConf + self.parseConf = {} + + if "LoadConf" in _parseConf: + logging.debug("__init__: LoadConf found in parser config") + try: + # / + # e.g. org/whois.publicinternetregistry.net + lc = _parseConf["LoadConf"].split("/", 1) + + lcTLD = lc[0] + lcWS = lc[1] + + lcConf = {} + + logging.debug("__init__: Loading configuration file of tld name %s"%(lcTLD)) + + exec(compile(open(os.path.join(self.tldPath, "%s"%(lcTLD))).read(), os.path.join(self.tldPath, "%s"%(lcTLD)), 'exec'), {}, lcConf) + lcConf = lcConf.get("parse") + + self.parseConf.update(lcConf.get(lcWS)) + + except: + pass + + self.parseConf.update(_parseConf) + + except: + self.parseConf = self.parseDefaultConf.get("default") + + + logging.debug("__init__: self.parseConf = %s"%(self.parseConf)) + + def parse(self): + result = {} + for key in self.parseConf: + matches = re.findall(self.parseConf[key], self.text, re.MULTILINE) + if matches: + logging.debug("run: regex matches found for key %s. %s"%(key, matches)) + result.update({key: [x.strip() for x in matches]}) + else: + logging.debug("run: No match for %s"%(key)) + + return result + + + -- 2.43.5 From 772c0d5779cad8cb2054319277a00708b94aa572 Mon Sep 17 00:00:00 2001 From: benjamyn Date: Wed, 21 Nov 2018 02:39:35 +1100 Subject: [PATCH 08/11] Merged TLDS with server hotfixes --- whois/tlds/online | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 whois/tlds/online diff --git a/whois/tlds/online b/whois/tlds/online new file mode 100644 index 0000000..524d1fa --- /dev/null +++ b/whois/tlds/online @@ -0,0 +1,25 @@ +server = { + "host": "whois.nic.online", + "redirect": "\s+Whois Server: (.*)", # Should never apply to .au domains +} + +parse = { + "default": { + "NotFound": "No Data Found", + "DomainName": "Domain Name:\s+(.+)", + "Registrar": "Registrar Name:\s+(.+)", + "NameServer": "Name Server:\s+(.+)", + "Status": "Status:\s+(.+)", + "UpdatedDate": "Last Modified:\s+(.+)", +# "CreationDate": "Creation Date:\s+(.+)", +# "ExpirationDate": "Expiration Date:\s+(.+)", + "RegistrantName": "Registrant:(.+)", + "EligibilityName": "Eligibility Name:(.+)", + "EligibilityType": "Eligibility Type:(.+)", + "EligibilityID": "Eligibility ID:(.+)", + "AdminID": "Registrant Contact ID:(.+)", + "AdminName": "Registrant Contact Name:(.+)", + "TechName": "Tech Contact Name:(.+)", + "TechID": "Tech Contact ID:(.+)", + }, +} -- 2.43.5 From a09941ccaeef7eff34dda53feda8f47b78954e09 Mon Sep 17 00:00:00 2001 From: Benjamyn Date: Tue, 19 May 2020 09:48:54 -0400 Subject: [PATCH 09/11] Fixed AU and UK --- whois/tlds/au | 3 +- whois/tlds/uk | 2 +- whois/whois.py | 201 ++++++++++++++++++++++++++----------------------- 3 files changed, 110 insertions(+), 96 deletions(-) diff --git a/whois/tlds/au b/whois/tlds/au index 7ad082d..46004ea 100644 --- a/whois/tlds/au +++ b/whois/tlds/au @@ -1,5 +1,5 @@ server = { - "host": "whois.auda.ltd", + "host": "whois.auda.org.au", "redirect": "\s+Whois Server: (.*)", # Should never apply to .au domains } @@ -17,6 +17,7 @@ parse = { "EligibilityName": "Eligibility Name:(.+)", "EligibilityType": "Eligibility Type:(.+)", "EligibilityID": "Eligibility ID:(.+)", + "RegistrantID": "Registrant ID:(.+)", "AdminID": "Registrant Contact ID:(.+)", "AdminName": "Registrant Contact Name:(.+)", "TechName": "Tech Contact Name:(.+)", diff --git a/whois/tlds/uk b/whois/tlds/uk index d8dd84e..0949f13 100644 --- a/whois/tlds/uk +++ b/whois/tlds/uk @@ -8,7 +8,7 @@ parse = { "NotFound": "No match for", "DomainName": "Domain name:\s+(.+|\n)", "Registrar": "Registrar:\s+(.+)", - "NameServer": "Name servers:\s+(.+)", + "NameServer": "Name servers:\s+(.+?\n\n)", "Status": "Registration status:\s+(.+)", "UpdatedDate": "Last updated:\s+(.+)", "CreationDate": "Registered on:\s+(.+)", diff --git a/whois/whois.py b/whois/whois.py index 0fba3d8..e02a0b7 100644 --- a/whois/whois.py +++ b/whois/whois.py @@ -1,137 +1,150 @@ # -*- coding: utf-8 -*- -# __ __ __ __ ______ __ ______ -#/\ \ _ \ \ /\ \_\ \ /\ __ \ /\ \ /\ ___\ -#\ \ \/ ".\ \\ \ __ \\ \ \/\ \\ \ \\ \___ \ -# \ \__/".~\_\\ \_\ \_\\ \_____\\ \_\\/\_____\ -# \/_/ \/_/ \/_/\/_/ \/_____/ \/_/ \/_____/ +# __ __ __ __ ______ __ ______ +# /\ \ _ \ \ /\ \_\ \ /\ __ \ /\ \ /\ ___\ +# \ \ \/ ".\ \\ \ __ \\ \ \/\ \\ \ \\ \___ \ +# \ \__/".~\_\\ \_\ \_\\ \_____\\ \_\\/\_____\ +# \/_/ \/_/ \/_/\/_/ \/_____/ \/_/ \/_____/ import sys import os import socket import re import logging -import urllib.request, urllib.parse, urllib.error +import urllib.request +import urllib.parse +import urllib.error from . import flags #import error -#import flags +# import flags + class Whois(object): - def __init__(self, domain, debug=False): - if debug: - logging.basicConfig(level=logging.DEBUG) - logging.debug("__init__: DEBUG is set to True") + def __init__(self, domain, debug=False): + if debug: + logging.basicConfig(level=logging.DEBUG) + logging.debug("__init__: DEBUG is set to True") - self.domain = domain - - self.tld = self.domain.split(".")[-1] + self.domain = domain - self.currPath = os.path.dirname(os.path.realpath(__file__)) - self.tldPath = os.path.join(self.currPath, "tlds") - self.tldList = os.listdir(self.tldPath) + self.tld = self.domain.split(".")[-1] - logging.debug("__init__: Setting initial variables.. self.currPath = %s / self.tldPath = %s / self.tldList = %s" - %(self.currPath, self.tldPath, self.tldList)) + self.currPath = os.path.dirname(os.path.realpath(__file__)) + self.tldPath = os.path.join(self.currPath, "tlds") + self.tldList = os.listdir(self.tldPath) - self.settings = {} + logging.debug("__init__: Setting initial variables.. self.currPath = %s / self.tldPath = %s / self.tldList = %s" + % (self.currPath, self.tldPath, self.tldList)) - if self.tld in self.tldList: - logging.debug("__init__: Loading tld configuration file...") + self.settings = {} - _settings = {} - exec(compile(open(os.path.join(self.tldPath, self.tld)).read(), os.path.join(self.tldPath, self.tld), 'exec'), {}, _settings) + if self.tld in self.tldList: + logging.debug("__init__: Loading tld configuration file...") - if "server" in _settings: - logging.debug("__init__: Settings: %s"%(_settings["server"])) - self.settings.update(_settings["server"]) - else: - logging.debug("__init__: No server settings found") + _settings = {} + exec(compile(open(os.path.join(self.tldPath, self.tld)).read(), + os.path.join(self.tldPath, self.tld), 'exec'), {}, _settings) - def chooseServer(self): - '''Choose whois server by detecting tld of given domain.''' - if "host" in self.settings: - logging.debug("chooseServer: Whois server addr: %s"%(self.settings["host"])) - return self.settings["host"] - else: - logging.debug("chooseServer: Whois server addr: %s"%(self.tld + ".whois-servers.net")) - return self.tld + ".whois-servers.net" + if "server" in _settings: + logging.debug("__init__: Settings: %s" % (_settings["server"])) + self.settings.update(_settings["server"]) + else: + logging.debug("__init__: No server settings found") - def sendHTTPQuery(self, whoisServer): - param = urllib.parse.urlencode({self.settings["http-arg"]: self.domain}) + def chooseServer(self): + '''Choose whois server by detecting tld of given domain.''' + if "host" in self.settings: + logging.debug("chooseServer: Whois server addr: %s" % + (self.settings["host"])) + return self.settings["host"] + else: + logging.debug("chooseServer: Whois server addr: %s" % + (self.tld + ".whois-servers.net")) + return self.tld + ".whois-servers.net" - if self.settings.get("http-method").lower() == "post": - logging.debug("sendHTTPQuery: Connecting to whois server using POST") - req = urllib.request.Request(whoisServer, param) - else: # GET - logging.debug("sendHTTPQuery: Connecting to whois server using GET") - req = urllib.request.Request((whoisServer.endswith("?") and whoisServer or whoisServer+"?") + param) + def sendHTTPQuery(self, whoisServer): + param = urllib.parse.urlencode( + {self.settings["http-arg"]: self.domain}) - data = urllib.request.urlopen(req).read() - print(data) + if self.settings.get("http-method").lower() == "post": + logging.debug( + "sendHTTPQuery: Connecting to whois server using POST") + req = urllib.request.Request(whoisServer, param) + else: # GET + logging.debug( + "sendHTTPQuery: Connecting to whois server using GET") + req = urllib.request.Request((whoisServer.endswith( + "?") and whoisServer or whoisServer+"?") + param) - return data - - def sendQuery(self, whoisServer): - '''Send query to whois server.''' - logging.debug("sendQuery: Connecting to whois server") - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + data = urllib.request.urlopen(req).read() + print(data) - try: - s.connect((whoisServer, 43)) + return data - except: - # FIXME: Create a exception class for this - logging.error("sendQuery: Error connecting to whois server %s"%(whoisServer)) - return False + def sendQuery(self, whoisServer): + '''Send query to whois server.''' + logging.debug("sendQuery: Connecting to whois server") + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - try: - msg = self.settings['format'][whoisServer].replace("%DOMAIN%", self.domain) + "\r\n" + try: + s.connect((whoisServer, 43)) - except: - msg = self.domain + "\r\n" + except: + # FIXME: Create a exception class for this + logging.error( + "sendQuery: Error connecting to whois server %s" % (whoisServer)) + return False - logging.debug("sendQuery: Sending data.. %s"%(msg)) - # print(msg.encode()) - s.send(msg.encode('utf-8')) - - result = "" + try: + msg = self.settings['format'][whoisServer].replace( + "%DOMAIN%", self.domain) + "\r\n" - while True: - buffer = s.recv(512) + except: + msg = self.domain + "\r\n" - if not buffer: - break + logging.debug("sendQuery: Sending data.. %s" % (msg)) + # print(msg.encode()) + s.send(msg.encode('utf-8')) - result += buffer.decode() + result = "" - finalResult = result.replace("\r\n", "\n") + while True: + buffer = s.recv(512) - logging.debug("sendQuery: result: %s"%(finalResult)) + if not buffer: + break - return finalResult + result += buffer.decode() - def query(self, redirect=True, return_type=flags.RETURN_TYPE_LIST): - '''Start whole process of whois query. This method will do them all.''' - whoisServer = self.chooseServer() + finalResult = result.replace("\r\n", "\n") - if self.settings.get("method") == "http": - result = self.sendHTTPQuery(whoisServer) - else: - result = self.sendQuery(whoisServer) + logging.debug("sendQuery: result: %s" % (finalResult)) - if redirect and "redirect" in self.settings: - logging.debug("query: Redirection found. Connecting to given server address") + return finalResult - redirection = re.findall(self.settings["redirect"], result, re.MULTILINE) + def query(self, redirect=True, return_type=flags.RETURN_TYPE_LIST): + '''Start whole process of whois query. This method will do them all.''' + whoisServer = self.chooseServer() - while redirection and len(redirection) >= 1: - whoisServer = redirection[0] - result = self.sendQuery(whoisServer) - redirection = re.findall(self.settings["redirect"], result) + if self.settings.get("method") == "http": + result = self.sendHTTPQuery(whoisServer) + else: + result = self.sendQuery(whoisServer) + if redirect and "redirect" in self.settings: + logging.debug( + "query: Redirection found. Connecting to given server address") - if return_type == flags.RETURN_TYPE_LIST: - return whoisServer, result - else: - return {"whoisServer": whoisServer, "result": result} + redirection = re.findall( + self.settings["redirect"], result, re.MULTILINE) + + while redirection and len(redirection) >= 1: + whoisServer = redirection[0] + result = self.sendQuery(whoisServer) + redirection = re.findall(self.settings["redirect"], result) + + if return_type == flags.RETURN_TYPE_LIST: + return whoisServer, result + else: + return {"whoisServer": whoisServer, "result": result} -- 2.43.5 From cabdaa795146badd6c7cc6eb899cb22e32a591d2 Mon Sep 17 00:00:00 2001 From: Benjamyn Love Date: Sat, 10 Oct 2020 18:39:29 -0400 Subject: [PATCH 10/11] Update 'whois/tlds/uk' --- whois/tlds/uk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/whois/tlds/uk b/whois/tlds/uk index 0949f13..92f2b86 100644 --- a/whois/tlds/uk +++ b/whois/tlds/uk @@ -8,7 +8,7 @@ parse = { "NotFound": "No match for", "DomainName": "Domain name:\s+(.+|\n)", "Registrar": "Registrar:\s+(.+)", - "NameServer": "Name servers:\s+(.+?\n\n)", + "NameServer": "Name servers:\s(.+\n\s+.+)", "Status": "Registration status:\s+(.+)", "UpdatedDate": "Last updated:\s+(.+)", "CreationDate": "Registered on:\s+(.+)", @@ -21,4 +21,4 @@ parse = { # "TechName": "Tech Contact Name:(.+)", # "TechID": "Tech Contact ID:(.+)", }, -} +} \ No newline at end of file -- 2.43.5 From 949bf206c8aa65fa8e5b9becd27c98bc73886e45 Mon Sep 17 00:00:00 2001 From: Benjamyn Love Date: Sun, 11 Oct 2020 09:48:12 +1100 Subject: [PATCH 11/11] fixed uk --- whois/tlds/uk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/whois/tlds/uk b/whois/tlds/uk index 92f2b86..a358247 100644 --- a/whois/tlds/uk +++ b/whois/tlds/uk @@ -21,4 +21,4 @@ parse = { # "TechName": "Tech Contact Name:(.+)", # "TechID": "Tech Contact ID:(.+)", }, -} \ No newline at end of file +} -- 2.43.5