emdee/exclude_badExits
1
0
Fork 0
Code Issues Pull requests Releases Packages Wiki Activity

Fixes

Browse source
This commit is contained in:
emdee 2022-11-13 20:13:56 +00:00
parent 6d5f8781aa
commit f85f5c6bd8
4 changed files with 341 additions and 172 deletions
Download patch file Download diff file Expand all files Collapse all files

173
exclude_badExits.py
View file

@ -86,6 +86,7 @@ import re
import socket
import time
import argparse
import string
from io import StringIO
import ipaddr
# list(ipaddress._find_address_range(ipaddress.IPv4Network('172.16.0.0/12'))
@ -112,9 +113,6 @@ try:
import coloredlogs
except ImportError as e:
coloredlogs = False
from trustor_poc import oDownloadUrl, idns_validate, TrustorError
from support_onions import icheck_torrc, bAreWeConnected, lIntroductionPoints, zResolveDomain, vwait_for_controller, yKNOWN_NODNS
global LOG
import logging
@ -122,6 +120,12 @@ import warnings
warnings.filterwarnings('ignore')
LOG = logging.getLogger()
import requests
from trustor_poc import oDownloadUrlUrllib3 as oDownloadUrl
from trustor_poc import idns_validate, TrustorError
from support_onions import icheck_torrc, bAreWeConnected, lIntroductionPoints, zResolveDomain, vwait_for_controller, yKNOWN_NODNS
LOG.info("imported HTTPSAdapter")
ETC_DIR = '/etc/tor/yaml'
aTRUST_DB = {}
aTRUST_DB_INDEX = {}
@ -145,28 +149,37 @@ def oMakeController(sSock='', port=9051):
oBAD_NODES = {}
oBAD_ROOT = 'BadNodes'
oBAD_NODES[oBAD_ROOT] = {}
oBAD_NODES[oBAD_ROOT]['ExcludeNodes'] = {}
lKNOWN_NODNS = []
lMAYBE_NODNS = []
def lYamlBadNodes(sFile,
section=sEXCLUDE_EXIT_KEY,
lWanted=['BadExit']):
global oBAD_NODES
global lKNOWN_NODNS
global lMAYBE_NODNS
l = []
if not yaml: return l
if os.path.exists(sFile):
with open(sFile, 'rt') as oFd:
oBAD_NODES = yaml.safe_load(oFd)
# BROKEN
# BROKEN
# root = 'ExcludeNodes'
# for elt in o[oBAD_ROOT][root][section].keys():
# if lWanted and elt not in lWanted: continue
# # l += o[oBAD_ROOT][root][section][elt]
global lKNOWN_NODNS
l = oBAD_NODES[oBAD_ROOT]['ExcludeNodes']['BadExit']
root = 'ExcludeDomains'
if root not in oBAD_NODES[oBAD_ROOT] or not oBAD_NODES[oBAD_ROOT][root]:
oBAD_NODES[oBAD_ROOT][root] = yaml.safe_load(StringIO(yKNOWN_NODNS))
lMAYBE_NODNS = yaml.safe_load(StringIO(yKNOWN_NODNS))
else:
lKNOWN_NODNS = oBAD_NODES[oBAD_ROOT][root]
lMAYBE_NODNS = oBAD_NODES[oBAD_ROOT][root]
return l
oGOOD_NODES = {}
@ -185,6 +198,22 @@ def lYamlGoodNodes(sFile='/etc/tor/torrc-goodnodes.yaml'):
# yq '.Nodes.IntroductionPoints|.[]' < /etc/tor/torrc-goodnodes.yaml
return l
def bdomain_is_bad(domain):
global lKNOWN_NODNS
if domain in lKNOWN_NODNS: return True
if domain in lMAYBE_NODNS:
ip = zResolveDomain(domain)
if ip == '':
LOG.debug(f"{fp} {domain} does not resolve")
lKNOWN_NODNS.append(domain)
lMAYBE_NODNS.remove(domain)
return True
if '@' in domain:
LOG.warn(f"@ in domain {domain}")
return True
return False
tBAD_URLS = set()
lATS = ['abuse', 'email']
lINTS = ['ciissversion', 'uplinkbw', 'signingkeylifetime', 'memory']
@ -225,6 +254,7 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
if aTRUST_DB_INDEX and fp in aTRUST_DB_INDEX.keys():
aCachedContact = aTRUST_DB_INDEX[fp]
if aCachedContact['email'] == a['email']:
LOG.info(f"{fp} in aTRUST_DB_INDEX")
return aCachedContact
if 'url' not in keys:
@ -237,18 +267,17 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
aFP_EMAIL[fp] = a['email']
LOG.debug(f"{fp} 'uri' but not 'url' in {keys}")
# drop through
if a['url'].startswith('http:'):
a['url'] = 'https:' +a['url'][5:]
elif not a['url'].startswith('https:'):
a['url'] = 'https:' +a['url']
c = a['url'].lstrip('https://').lstrip('http://').strip('/')
a['url'] = 'https://' +c
# domain should be a unique key for contacts
domain = a['url'][8:].strip('/')
if lKNOWN_NODNS and domain in lKNOWN_NODNS:
LOG.warn(f"{domain} in lKNOWN_NODNS")
return {}
ip = zResolveDomain(domain, lKNOWN_NODNS)
domain = a['url'][8:]
if bdomain_is_bad(domain):
LOG.warn(f"{domain} is bad from {a['url']}")
LOG.info(f"{domain} is bad from {a}")
return a
ip = zResolveDomain(domain)
if ip == '':
aFP_EMAIL[fp] = a['email']
LOG.debug(f"{fp} {domain} does not resolve")
@ -268,9 +297,10 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
return a
LOG.debug(f"{len(keys)} contact fields for {fp}")
url="https://"+domain+"/.well-known/tor-relay/rsa-fingerprint.txt"
try:
LOG.debug(f"Downloading from {domain} for {fp}")
o = oDownloadUrl(domain, https_cafile,
o = oDownloadUrl(url, https_cafile,
timeout=timeout, host=host, port=port)
# requests response: text "reason", "status_code"
except AttributeError as e:
@ -288,20 +318,52 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
except (BaseException ) as e:
LOG.error(f"Exception {type(e)} downloading from {domain} {e}")
else:
if o.status_code >= 300:
if hasattr(o, 'status'):
status_code = o.status
else:
status_code = o.status_code
if status_code >= 300:
aFP_EMAIL[fp] = a['email']
LOG.warn(f"Error from {domain} {o.status_code} {o.reason}")
LOG.warn(f"Error from {domain} {status_code} {o.reason}")
# any reason retry?
tBAD_URLS.add(a['url'])
return a
l = o.text.upper().strip().split('\n')
a['modified'] = time.time()
if hasattr(o, 'text'):
data = o.text
else:
data = str(o.data, 'UTF-8')
l = data.upper().strip().split('\n')
LOG.debug(f"Downloaded from {domain} {len(l)} lines {len(data)} bytes")
a['modified'] = int(time.time())
if not l:
LOG.warn(f"Downloading from {domain} empty for {fp}")
else:
a['fps'] = [elt for elt in l if elt and len(elt) == 40
and not elt.startswith('#')]
LOG.info(f"Downloaded from {domain} {len(a['fps'])} FPs")
return a
def aParseContactYaml(contact, fp):
"""
See the Tor ContactInfo Information Sharing Specification v2
https://nusenu.github.io/ContactInfo-Information-Sharing-Specification/
"""
lelts = contact.split()
a = {}
if len(lelts) % 1 != 0:
LOG.warn(f"bad contact for {fp} odd number of components")
LOG.debug(f"{fp} {a}")
return a
key = ''
for elt in lets:
if key == '':
key = elt
continue
a[key] = elt
key = ''
LOG.debug(f"{fp} {len(a.keys())} fields")
return a
def aParseContact(contact, fp):
@ -419,7 +481,7 @@ def oMainArgparser(_=None):
parser.add_argument('--good_nodes', type=str,
default=os.path.join(ETC_DIR, 'torrc-goodnodes.yaml'),
help="Yaml file of good nodes that should not be excluded")
help="Yaml file of good info that should not be excluded")
parser.add_argument('--bad_nodes', type=str,
default=os.path.join(ETC_DIR, 'torrc-badnodes.yaml'),
help="Yaml file of bad nodes that should also be excluded")
@ -434,7 +496,9 @@ def oMainArgparser(_=None):
help="Set StrictNodes: 1 is less anonymous but more secure, although some sites may be unreachable")
parser.add_argument('--wait_boot', type=int, default=120,
help="Seconds to wait for Tor to booststrap")
parser.add_argument('--log_level', type=int, default=20,
parser.add_argument('--points_timeout', type=int, default=0,
help="Timeout for getting introduction points - must be long >120sec. 0 means disabled looking for IPs")
parser.add_argument('--log_level', type=int, default=10,
help="10=debug 20=info 30=warn 40=error")
parser.add_argument('--bad_sections', type=str,
default='MyBadExit',
@ -501,7 +565,7 @@ def iMain(lArgs):
# but...
for k,v in aTRUST_DB.items():
if 'modified' not in v.keys():
v['modified'] = time.time()
v['modified'] = int(time.time())
aTRUST_DB_INDEX[k] = v
if 'fps' in aTRUST_DB[k].keys():
for fp in aTRUST_DB[k]['fps']:
@ -546,9 +610,11 @@ def iMain(lArgs):
w = set(oGOOD_NODES[oGOOD_ROOT]['Services'])
if oArgs.white_services:
w.update(oArgs.white_services.split(','))
t.update(lIntroductionPoints(controller, w))
if oArgs.points_timeout > 0:
LOG.info(f"{len(w)} services will be checked from IntroductionPoints")
t.update(lIntroductionPoints(controller, w, itimeout=oArgs.points_timeout))
if len(t) > 0:
LOG.info(f"IntroductionPoints {len(t)} nodes")
LOG.info(f"IntroductionPoints {len(t)} relays from {len(w)} services")
twhitelist_set.update(t)
texclude_set = set()
@ -556,17 +622,15 @@ def iMain(lArgs):
if False and oArgs.bad_sections:
# BROKEN
sections = oArgs.bad_sections.split(',')
lexclude_list = set(lYamlBadNodes(oArgs.bad_nodes,
texclude_set = set(lYamlBadNodes(oArgs.bad_nodes,
lWanted=sections,
section=sEXCLUDE_EXIT_KEY))
else:
texclude_set = set(lYamlBadNodes(oArgs.bad_nodes))
LOG.info(f"lYamlBadNodes {len(texclude_set)}")
LOG.info(f"Preloaded {len(texclude_set)} bad fps")
ttrust_db_index = aTRUST_DB_INDEX.keys()
iDnsContact = 0
tdns_contacts = set()
iFakeContact = 0
iTotalContacts = 0
aBadContacts = {}
lConds = oArgs.contact.split(',')
@ -580,12 +644,12 @@ def iMain(lArgs):
continue
relay.fingerprint = relay.fingerprint.upper()
sofar = f"G:{len(aTRUST_DB.keys())} U:{iDnsContact} F:{iFakeContact} BF:{len(texclude_set)} GF:{len(ttrust_db_index)} #{iR}"
sofar = f"G:{len(aTRUST_DB.keys())} U:{len(tdns_contacts)} F:{iFakeContact} BF:{len(texclude_set)} GF:{len(ttrust_db_index)} TC:{iTotalContacts} #{iR}"
if not relay.exit_policy.is_exiting_allowed():
if sEXCLUDE_EXIT_KEY == 'ExcludeNodes':
LOG.debug(f"{relay.fingerprint} not an exit {sofar}")
pass # LOG.debug(f"{relay.fingerprint} not an exit {sofar}")
else:
LOG.warn(f"{relay.fingerprint} not an exit {sofar}")
pass # LOG.warn(f"{relay.fingerprint} not an exit {sofar}")
# continue
# great contact had good fps and we are in them
@ -598,43 +662,51 @@ def iMain(lArgs):
relay.contact = str(relay.contact, 'UTF-8')
if ('Empty' in lConds and not relay.contact) or \
('NoEmail' in lConds and relay.contact and not '@' in relay.contact):
('NoEmail' in lConds and relay.contact and not 'email:' in relay.contact):
texclude_set.add(relay.fingerprint)
continue
if not relay.contact or not 'ciissversion:' in relay.contact:
# should be unreached 'Empty' should always be in lConds
continue
iTotalContacts += 1
if relay.contact and not 'url:' in relay.contact:
LOG.info(f"{relay.fingerprint} skipping bad contact - no url: {sofar}")
LOG.debug(f"{relay.fingerprint} {relay.contact} {sofar}")
texclude_set.add(relay.fingerprint)
continue
c = relay.contact.lower()
# first rough cut
i = c.find('url:')
if i >=0: c = c[i+4:]
if i >=0:
c = c[i+4:]
i = c.find(' ')
if i >=0: c = c[:i]
c = c.replace('https://', '').replace('http://', '').strip('/')
c = c.lstrip('https://').lstrip('http://').strip('/')
i = c.find('/')
if i >=0: c = c[:i]
domain = c
LOG.info(f"{relay.fingerprint} domain={domain}")
if domain and domain in lKNOWN_NODNS:
LOG.info(f"{relay.fingerprint} skipping in lKNOWN_NODNS {domain} {sofar}")
if domain and bdomain_is_bad(domain):
LOG.info(f"{relay.fingerprint} skipping bad {domain} {sofar}")
LOG.debug(f"{relay.fingerprint} {relay.contact} {sofar}")
texclude_set.add(relay.fingerprint)
continue
if domain:
ip = zResolveDomain(domain, lKNOWN_NODNS)
ip = zResolveDomain(domain)
if not ip:
LOG.warn(f"{relay.fingerprint} {domain} did not resolve {sofar}")
texclude_set.add(relay.fingerprint)
lKNOWN_NODNS.append(domain)
iFakeContact += 1
continue
if 'dns-rsa' in relay.contact.lower():
target = f"{relay.fingerprint}.{domain}"
target = f"{relay.fingerprint}.{domain}"
LOG.info(f"skipping 'dns-rsa' {target} {sofar}")
iDnsContact += 1
tdns_contacts.add(target)
elif 'proof:uri-rsa' in relay.contact.lower():
a = aParseContact(relay.contact, relay.fingerprint)
@ -646,6 +718,7 @@ def iMain(lArgs):
if a['url'] in tBAD_URLS:
# The fp is using a contact with a URL we know is bad
LOG.info(f"{relay.fingerprint} skipping in tBAD_URLS {a['url']} {sofar}")
LOG.debug(f"{relay.fingerprint} {a} {sofar}")
iFakeContact += 1
texclude_set.add(relay.fingerprint)
continue
@ -653,6 +726,7 @@ def iMain(lArgs):
if domain in lKNOWN_NODNS:
# The fp is using a contact with a URL we know is bogus
LOG.info(f"{relay.fingerprint} skipping in lKNOWN_NODNS {a['url']} {sofar}")
LOG.debug(f"{relay.fingerprint} {a} {sofar}")
iFakeContact += 1
texclude_set.add(relay.fingerprint)
continue
@ -667,6 +741,7 @@ def iMain(lArgs):
if not b or not 'fps' in b or not b['fps'] or not b['url']:
LOG.warn(f"{relay.fingerprint} did NOT VERIFY {sofar}")
LOG.debug(f"{relay.fingerprint} {b} {sofar}")
# If it's giving contact info that doesnt check out
# it could be a bad exit with fake contact info
texclude_set.add(relay.fingerprint)
@ -692,17 +767,19 @@ def iMain(lArgs):
yaml.dump(aTRUST_DB, indent=2, stream=oFYaml)
oFYaml.close()
LOG.info(f"Filtered {len(twhitelist_set)} whitelisted relays")
texclude_set = texclude_set.difference(twhitelist_set)
LOG.info(f"{len(list(aTRUST_DB.keys()))} good contacts out of {iTotalContacts}")
if oArgs.proof_output and aTRUST_DB:
with open(proof_output_tmp, 'wt') as oFYaml:
yaml.dump(aTRUST_DB, indent=2, stream=oFYaml)
LOG.info(f"Wrote {len(list(aTRUST_DB.keys()))} good contact details to {oArgs.proof_output}")
oFYaml.close()
if os.path.exists(oArgs.proof_output):
bak = oArgs.proof_output +'.bak'
os.rename(oArgs.proof_output, bak)
os.rename(proof_output_tmp, oArgs.proof_output)
LOG.info(f"Wrote {len(list(aTRUST_DB.keys()))} good contact details to {oArgs.proof_output}")
if oArgs.torrc_output and texclude_set:
with open(oArgs.torrc_output, 'wt') as oFTorrc:
@ -725,7 +802,6 @@ def iMain(lArgs):
oGOOD_NODES['GoodNodes']['Relays']['ExitNodes'] = list(aTRUST_DB_INDEX.keys())
# GuardNodes are readonl
vwrite_goodnodes(oArgs, oGOOD_NODES, str(len(ttrust_db_index)))
retval = 0
try:
logging.getLogger('stem').setLevel(30)
@ -762,6 +838,7 @@ def iMain(lArgs):
LOG.errro(f"Failed setting {sINCLUDE_EXIT_KEY} good exit nodes in Tor")
retval += 1
LOG.info("dns-rsa domains:\n{'\n'.join(tdns_contacts)}")
return retval
except InvalidRequest as e: