emdee/exclude_badExits
1
0
Fork 0
Code Issues Pull requests Releases Packages Wiki Activity

Added lookupdns.py

Browse source
This commit is contained in:
emdee 2022-11-14 11:59:33 +00:00
parent 2ea65cc181
commit 94c0834092
3 changed files with 97 additions and 86 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

149
exclude_badExits.py
View file

@ -160,7 +160,7 @@ def lYamlBadNodes(sFile,
global oBAD_NODES
global lKNOWN_NODNS
global lMAYBE_NODNS
l = []
if not yaml: return l
if os.path.exists(sFile):
@ -198,7 +198,7 @@ def lYamlGoodNodes(sFile='/etc/tor/torrc-goodnodes.yaml'):
# yq '.Nodes.IntroductionPoints|.[]' < /etc/tor/torrc-goodnodes.yaml
return l
def bdomain_is_bad(domain):
def bdomain_is_bad(domain, fp):
global lKNOWN_NODNS
if domain in lKNOWN_NODNS: return True
if domain in lMAYBE_NODNS:
@ -208,10 +208,11 @@ def bdomain_is_bad(domain):
lKNOWN_NODNS.append(domain)
lMAYBE_NODNS.remove(domain)
return True
if '@' in domain:
LOG.warn(f"@ in domain {domain}")
return True
for elt in '@(){}$!':
if elt in domain:
LOG.warn(f"{elt} in domain {domain}")
return True
return False
tBAD_URLS = set()
@ -256,7 +257,7 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
if aCachedContact['email'] == a['email']:
LOG.info(f"{fp} in aTRUST_DB_INDEX")
return aCachedContact
if 'url' not in keys:
if 'uri' not in keys:
a['url'] = ''
@ -270,20 +271,21 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
c = a['url'].lstrip('https://').lstrip('http://').strip('/')
a['url'] = 'https://' +c
# domain should be a unique key for contacts
domain = a['url'][8:]
if bdomain_is_bad(domain):
if bdomain_is_bad(domain, fp):
LOG.warn(f"{domain} is bad from {a['url']}")
LOG.info(f"{domain} is bad from {a}")
LOG.debug(f"{fp} is bad from {a}")
return a
ip = zResolveDomain(domain)
if ip == '':
aFP_EMAIL[fp] = a['email']
LOG.debug(f"{fp} {domain} does not resolve")
lKNOWN_NODNS.append(domain)
return {}
if a['proof'] not in ['uri-rsa']:
# only support uri for now
if False and ub_ctx:
@ -295,7 +297,7 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
pass
LOG.warn(f"{fp} proof={a['proof']} not supported yet")
return a
LOG.debug(f"{len(keys)} contact fields for {fp}")
url="https://"+domain+"/.well-known/tor-relay/rsa-fingerprint.txt"
try:
@ -328,7 +330,7 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
# any reason retry?
tBAD_URLS.add(a['url'])
return a
if hasattr(o, 'text'):
data = o.text
else:
@ -347,7 +349,7 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
def aParseContactYaml(contact, fp):
"""
See the Tor ContactInfo Information Sharing Specification v2
See the Tor ContactInfo Information Sharing Specification v2
https://nusenu.github.io/ContactInfo-Information-Sharing-Specification/
"""
lelts = contact.split()
@ -357,7 +359,7 @@ def aParseContactYaml(contact, fp):
LOG.debug(f"{fp} {a}")
return a
key = ''
for elt in lets:
for elt in lelts:
if key == '':
key = elt
continue
@ -368,7 +370,7 @@ def aParseContactYaml(contact, fp):
def aParseContact(contact, fp):
"""
See the Tor ContactInfo Information Sharing Specification v2
See the Tor ContactInfo Information Sharing Specification v2
https://nusenu.github.io/ContactInfo-Information-Sharing-Specification/
"""
l = [line for line in contact.strip().replace('"', '').split(' ')
@ -424,22 +426,22 @@ def vsetup_logging(log_level, logfile=''):
LOG.info(f"SSetting log_level to {log_level!s}")
logging._levelToName = {
CRITICAL: 'CRITICAL',
ERROR: 'ERROR',
WARNING: 'WARN',
INFO: 'INFO',
DEBUG: 'DEBUG',
NOTSET: 'NOTSET',
logging.CRITICAL: 'CRITICAL',
logging.ERROR: 'ERROR',
logging.WARNING: 'WARN',
logging.INFO: 'INFO',
logging.DEBUG: 'DEBUG',
logging.NOTSET: 'NOTSET',
}
logging._nameToLevel = {
'CRITICAL': CRITICAL,
'FATAL': FATAL,
'ERROR': ERROR,
'WARN': WARNING,
'WARNING': WARNING,
'INFO': INFO,
'DEBUG': DEBUG,
'NOTSET': NOTSET,
'CRITICAL': logging.CRITICAL,
'FATAL': logging.FATAL,
'ERROR': logging.ERROR,
'WARN': logging.WARNING,
'WARNING': logging.WARNING,
'INFO': logging.INFO,
'DEBUG': logging.DEBUG,
'NOTSET': logging.NOTSET,
}
def oMainArgparser(_=None):
@ -490,7 +492,7 @@ def oMainArgparser(_=None):
parser.add_argument('--bad_contacts', type=str,
default=os.path.join(ETC_DIR, 'badcontacts.yaml'),
help="Yaml file of bad contacts that bad FPs are using")
parser.add_argument('--strict_nodes', type=int, default=0,
choices=[0,1],
help="Set StrictNodes: 1 is less anonymous but more secure, although some sites may be unreachable")
@ -498,7 +500,7 @@ def oMainArgparser(_=None):
help="Seconds to wait for Tor to booststrap")
parser.add_argument('--points_timeout', type=int, default=0,
help="Timeout for getting introduction points - must be long >120sec. 0 means disabled looking for IPs")
parser.add_argument('--log_level', type=int, default=10,
parser.add_argument('--log_level', type=int, default=20,
help="10=debug 20=info 30=warn 40=error")
parser.add_argument('--bad_sections', type=str,
default='MyBadExit',
@ -523,24 +525,24 @@ def vwrite_badnodes(oArgs, oBAD_NODES, slen):
if os.path.exists(oArgs.bad_nodes):
os.rename(oArgs.bad_nodes, bak)
os.rename(tmp, oArgs.bad_nodes)
def vwrite_goodnodes(oArgs, oGOOD_NODES, slen):
def vwrite_goodnodes(oArgs, oGOOD_NODES, ilen):
if oArgs.good_nodes:
tmp = oArgs.good_nodes +'.tmp'
bak = oArgs.good_nodes +'.bak'
with open(tmp, 'wt') as oFYaml:
yaml.dump(oGOOD_NODES, indent=2, stream=oFYaml)
LOG.info(f"Wrote {slen} good nodes to {oArgs.good_nodes}")
LOG.info(f"Wrote {ilen} good relays to {oArgs.good_nodes}")
oFYaml.close()
if os.path.exists(oArgs.good_nodes):
os.rename(oArgs.good_nodes, bak)
os.rename(tmp, oArgs.good_nodes)
def iMain(lArgs):
global aTRUST_DB
global aTRUST_DB_INDEX
global oBAD_NODES
global oGOOD_NODES
global oGOOD_NODES
global lKNOWN_NODNS
parser = oMainArgparser()
oArgs = parser.parse_args(lArgs)
@ -573,7 +575,7 @@ def iMain(lArgs):
continue
aTRUST_DB_INDEX[fp] = v
LOG.info(f"{len(aTRUST_DB_INDEX.keys())} good relays from {sFile}")
except Exception as e:
LOG.exception(f"Error reading YAML TrustDB {sFile} {e}")
@ -625,17 +627,17 @@ def iMain(lArgs):
texclude_set = set(lYamlBadNodes(oArgs.bad_nodes,
lWanted=sections,
section=sEXCLUDE_EXIT_KEY))
LOG.info(f"Preloaded {len(texclude_set)} bad fps")
LOG.info(f"Preloaded {len(texclude_set)} bad fps")
ttrust_db_index = aTRUST_DB_INDEX.keys()
tdns_contacts = set()
tdns_urls = set()
iFakeContact = 0
iTotalContacts = 0
aBadContacts = {}
lConds = oArgs.contact.split(',')
iR = 0
relays = controller.get_server_descriptors()
for relay in relays:
iR += 1
@ -643,15 +645,15 @@ def iMain(lArgs):
LOG.warn('Invalid Fingerprint: %s' % relay.fingerprint)
continue
relay.fingerprint = relay.fingerprint.upper()
sofar = f"G:{len(aTRUST_DB.keys())} U:{len(tdns_contacts)} F:{iFakeContact} BF:{len(texclude_set)} GF:{len(ttrust_db_index)} TC:{iTotalContacts} #{iR}"
sofar = f"G:{len(aTRUST_DB.keys())} U:{len(tdns_urls)} F:{iFakeContact} BF:{len(texclude_set)} GF:{len(ttrust_db_index)} TC:{iTotalContacts} #{iR}"
if not relay.exit_policy.is_exiting_allowed():
if sEXCLUDE_EXIT_KEY == 'ExcludeNodes':
pass # LOG.debug(f"{relay.fingerprint} not an exit {sofar}")
else:
pass # LOG.warn(f"{relay.fingerprint} not an exit {sofar}")
# continue
# great contact had good fps and we are in them
if relay.fingerprint in aTRUST_DB_INDEX.keys():
# a cached entry
@ -660,54 +662,55 @@ def iMain(lArgs):
if type(relay.contact) == bytes:
# dunno
relay.contact = str(relay.contact, 'UTF-8')
if ('Empty' in lConds and not relay.contact) or \
('NoEmail' in lConds and relay.contact and not 'email:' in relay.contact):
texclude_set.add(relay.fingerprint)
continue
if not relay.contact or not 'ciissversion:' in relay.contact:
# should be unreached 'Empty' should always be in lConds
continue
iTotalContacts += 1
fp = relay.fingerprint
if relay.contact and not 'url:' in relay.contact:
LOG.info(f"{relay.fingerprint} skipping bad contact - no url: {sofar}")
LOG.debug(f"{relay.fingerprint} {relay.contact} {sofar}")
texclude_set.add(relay.fingerprint)
LOG.info(f"{fp} skipping bad contact - no url: {sofar}")
LOG.debug(f"{fp} {relay.contact} {sofar}")
texclude_set.add(fp)
continue
c = relay.contact.lower()
# first rough cut
i = c.find('url:')
if i >=0:
c = c[i+4:]
c = c[i+4:]
i = c.find(' ')
if i >=0: c = c[:i]
c = c.lstrip('https://').lstrip('http://').strip('/')
i = c.find('/')
if i >=0: c = c[:i]
domain = c
if domain and bdomain_is_bad(domain):
LOG.info(f"{relay.fingerprint} skipping bad {domain} {sofar}")
LOG.debug(f"{relay.fingerprint} {relay.contact} {sofar}")
texclude_set.add(relay.fingerprint)
if domain and bdomain_is_bad(domain, fp):
LOG.info(f"{fp} skipping bad {domain} {sofar}")
LOG.debug(f"{fp} {relay.contact} {sofar}")
texclude_set.add(fp)
continue
if domain:
ip = zResolveDomain(domain)
if not ip:
LOG.warn(f"{relay.fingerprint} {domain} did not resolve {sofar}")
texclude_set.add(relay.fingerprint)
LOG.warn(f"{fp} {domain} did not resolve {sofar}")
texclude_set.add(fp)
lKNOWN_NODNS.append(domain)
iFakeContact += 1
continue
if 'dns-rsa' in relay.contact.lower():
target = f"{relay.fingerprint}.{domain}"
if 'dns-rsa' in relay.contact.lower():
target = f"{relay.fingerprint}.{domain}"
LOG.info(f"skipping 'dns-rsa' {target} {sofar}")
tdns_contacts.add(target)
tdns_urls.add(target)
elif 'proof:uri-rsa' in relay.contact.lower():
a = aParseContact(relay.contact, relay.fingerprint)
if not a:
@ -730,7 +733,7 @@ def iMain(lArgs):
iFakeContact += 1
texclude_set.add(relay.fingerprint)
continue
b = aVerifyContact(list(a.values())[0],
relay.fingerprint,
@ -738,7 +741,7 @@ def iMain(lArgs):
timeout=oArgs.timeout,
host=oArgs.proxy_host,
port=oArgs.proxy_port)
if not b or not 'fps' in b or not b['fps'] or not b['url']:
LOG.warn(f"{relay.fingerprint} did NOT VERIFY {sofar}")
LOG.debug(f"{relay.fingerprint} {b} {sofar}")
@ -747,7 +750,7 @@ def iMain(lArgs):
texclude_set.add(relay.fingerprint)
aBadContacts[relay.fingerprint] = b
continue
if relay.fingerprint not in b['fps']:
LOG.warn(f"{relay.fingerprint} the FP IS NOT in the list of fps {sofar}")
# assume a fp is using a bogus contact
@ -766,9 +769,11 @@ def iMain(lArgs):
with open(proof_output_tmp, 'wt') as oFYaml:
yaml.dump(aTRUST_DB, indent=2, stream=oFYaml)
oFYaml.close()
LOG.info(f"Filtered {len(twhitelist_set)} whitelisted relays")
texclude_set = texclude_set.difference(twhitelist_set)
# accept the dns-rsa urls for now until we test them
texclude_set = texclude_set.difference(tdns_urls)
LOG.info(f"{len(list(aTRUST_DB.keys()))} good contacts out of {iTotalContacts}")
if oArgs.proof_output and aTRUST_DB:
@ -785,7 +790,7 @@ def iMain(lArgs):
with open(oArgs.torrc_output, 'wt') as oFTorrc:
oFTorrc.write(f"{sEXCLUDE_EXIT_KEY} {','.join(texclude_set)}\n")
oFTorrc.write(f"{sINCLUDE_EXIT_KEY} {','.join(aTRUST_DB_INDEX.keys())}\n")
oFTorrc.write(f"{sINCLUDE_GUARD_KEY} {','.join(o[oGOOD_ROOT]['GuardNodes'])}\n")
oFTorrc.write(f"{sINCLUDE_GUARD_KEY} {','.join(oGOOD_NODES[oGOOD_ROOT]['GuardNodes'])}\n")
LOG.info(f"Wrote tor configuration to {oArgs.torrc_output}")
oFTorrc.close()
@ -798,10 +803,10 @@ def iMain(lArgs):
oBAD_NODES[oBAD_ROOT]['ExcludeNodes']['BadExit'] = list(texclude_set)
oBAD_NODES[oBAD_ROOT]['ExcludeDomains'] = lKNOWN_NODNS
vwrite_badnodes(oArgs, oBAD_NODES, str(len(texclude_set)))
oGOOD_NODES['GoodNodes']['Relays']['ExitNodes'] = list(aTRUST_DB_INDEX.keys())
# GuardNodes are readonl
vwrite_goodnodes(oArgs, oGOOD_NODES, str(len(ttrust_db_index)))
vwrite_goodnodes(oArgs, oGOOD_NODES, len(aTRUST_DB_INDEX.keys()))
retval = 0
try:
logging.getLogger('stem').setLevel(30)
@ -838,7 +843,7 @@ def iMain(lArgs):
LOG.errro(f"Failed setting {sINCLUDE_EXIT_KEY} good exit nodes in Tor")
retval += 1
LOG.info("dns-rsa domains:\n{'\n'.join(tdns_contacts)}")
sys.stdout.write("dns-rsa domains:\n" +'\n'.join(tdns_urls) +'\n')
return retval
except InvalidRequest as e:

6
support_onions.py
View file

@ -27,9 +27,8 @@ LOG = logging.getLogger()
bHAVE_TORR = shutil.which('tor-resolve')
# maybe we should check these each time but we
# got them by sorting bad relays in the wild
# we'll keep a copy here
# we check these each time but we got them by sorting bad relays
# in the wild we'll keep a copy here so we can avoid restesting
yKNOWN_NODNS = """
---
- 0x0.is
@ -50,6 +49,7 @@ yKNOWN_NODNS = """
- or.wowplanet.de
- ormycloud.org
- plied-privacy.net
- rivacysvcs.net
- redacted.org
- rification-for-nusenu.net
- rofl.cat

28
trustor_poc.py
View file

@ -52,7 +52,7 @@ def read_local_trust_config(trust_config):
'''
result = []
# for now we support max_depth = 0 only
# this PoC version has no support for recursion
# https://github.com/nusenu/tor-relay-operator-ids-trust-information#trust-information-consumers
@ -140,7 +140,11 @@ def get_controller(address='127.0.0.1', port=9151, password=''):
return controller
def find_validation_candidates(controller, trusted_domains=[],validation_cache=[],accept_all=False):
def find_validation_candidates(controller,
trusted_domains=[],
validation_cache=[],
CAfile='/etc/ssl/certs/ca-certificates.crt',
accept_all=False):
'''
connect to a tor client via controlport and return a dict of all
not yet validated fingerprints per trusted operators
@ -221,14 +225,14 @@ def oDownloadUrlRequests(uri, sCAfile, timeout=30, host='127.0.0.1', port=9050):
head = requests.head(uri, timeout=timeout, proxies=proxy, headers=headers)
except Exception as e:
raise TrustorError(f"HTTP HEAD request failed for {uri} {e}")
if head.status_code >= 300:
raise TrustorError(f"HTTP Errorcode {head.status_code}")
if not head.headers['Content-Type'].startswith('text/plain'):
raise TrustorError(f"HTTP Content-Type != text/plain")
if not os.path.exists(sCAfile):
raise TrustorError(f"File not found CAfile {sCAfile}")
try:
with requests.sessions.Session() as session:
oReqResp = session.request(method="get", url=uri,
@ -336,7 +340,7 @@ def my_match_hostname(cert, hostname):
else:
raise CertificateError(
"no appropriate commonName or subjectAltName fields were found"
)
)
match_hostname = my_match_hostname
from urllib3.util.ssl_ import (
is_ipaddress,
@ -393,15 +397,15 @@ def oDownloadUrlUrllib3(uri, sCAfile, timeout=30, host='127.0.0.1', port=9050):
retries=False)
except Exception as e:
LOG.error(f"HTTP HEAD request failed for {uri} {e}")
raise
raise
if head.status >= 300:
raise TrustorError(f"HTTP Errorcode {head.status}")
if not head.headers['Content-Type'].startswith('text/plain'):
raise TrustorError(f"HTTP Content-Type != text/plain")
if not os.path.exists(sCAfile):
raise TrustorError(f"File not found CAfile {sCAfile}")
try:
oReqResp = proxy.request("GET", uri,
headers=headers,
@ -420,7 +424,7 @@ def oDownloadUrlUrllib3(uri, sCAfile, timeout=30, host='127.0.0.1', port=9050):
LOG.error(f'Redirect detected %s vs %s (final)' % (uri, oReqResp.geturl()))
raise TrustorError(f'Redirect detected %s vs %s (final)' % (uri, oReqResp.geturl()))
oReqResp.decode_content = True
return oReqResp
import urllib3.connectionpool
urllib3.connectionpool.VerifiedHTTPSConnection = HTTPSConnection
@ -483,7 +487,7 @@ def idns_validate(domain,
# this is not the system wide /etc/resolv.conf
# use dnscrypt-proxy to encrypt your DNS and route it via tor's SOCKSPort
ctx = ub_ctx()
if (os.path.isfile(libunbound_resolv_file)):
@ -529,6 +533,7 @@ def configure_tor(controller, trusted_fingerprints, exitonly=True):
if __name__ == '__main__':
CAfile = '/etc/ssl/certs/ca-certificates.crt'
trust_config = 'trust_config'
assert os.path.exists(trust_config)
trusted_domains = read_local_trust_config(trust_config)
@ -546,7 +551,8 @@ if __name__ == '__main__':
r = find_validation_candidates(controller,
validation_cache=trusted_fingerprints,
trusted_domains=trusted_domains)
trusted_domains=trusted_domains,
CAfile=CAfile)
validate_proofs(r, validation_cache_file,
timeout=timeout,
host=controller_address,