emdee/exclude_badExits
1
0
Fork 0
Code Issues Pull requests Releases Packages Wiki Activity

add https_adapter.py

Browse source
This commit is contained in:
emdee 2022-11-09 05:43:26 +00:00
parent 0198994486
commit d11d95aafe
3 changed files with 545 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

178
exclude_badExits.py
View file

@ -87,6 +87,7 @@ import time
import argparse
from io import StringIO
from urllib3.util.ssl_match_hostname import CertificateError
from stem import InvalidRequest
from stem.control import Controller
from stem.connection import IncorrectPassword
@ -107,7 +108,9 @@ try:
import coloredlogs
except ImportError as e:
coloredlogs = False
from trustor_poc import lDownloadUrlFps, idns_validate
from trustor_poc import oDownloadUrl, idns_validate, TrustorError
from support_onions import sTorResolve, getaddrinfo, icheck_torrc, bAreWeConnected
global LOG
import logging
@ -115,6 +118,7 @@ import warnings
warnings.filterwarnings('ignore')
LOG = logging.getLogger()
ETC_DIR = '/etc/tor/yaml'
aTRUST_DB = {}
sDETAILS_URL = "https://metrics.torproject.org/rs.html#details/"
# You can call this while bootstrapping
@ -122,6 +126,20 @@ sEXCLUDE_EXIT_KEY = 'ExcludeNodes'
sINCLUDE_EXIT_KEY = 'ExitNodes'
sINCLUDE_GUARD_KEY = 'EntryNodes'
# maybe we should check these each time but we
# got them by sorting bad relays in the wild
lKNOWN_NODNS = [
'0x0.is',
'a9.wtf',
'arvanode.net',
'dodo.pm',
'galtland.network',
'interfesse.net',
'kryptonit.org',
'nx42.de',
'tor-exit-2.aa78i2efsewr0neeknk.xyz',
'tor-exit-3.aa78i2efsewr0neeknk.xyz',
]
def oMakeController(sSock='', port=9051):
import getpass
if sSock and os.path.exists(sSock):
@ -158,15 +176,17 @@ def icheck_torrc(sFile, oArgs):
l = open(sFile, 'rt').readlines()
a = {}
for elt in l:
elt = elt.strip()
if not elt or not ' ' in elt: continue
k,v = elt.split(' ', 1)
a[k] = v
keys = list(a.keys())
keys = a
if 'HashedControlPassword' not in keys:
LOG.info('Add HashedControlPassword for security')
print('run: tor --hashcontrolpassword <TopSecretWord>')
if 'ExcludeNodes' in keys:
elt = 'ExcludeNodes.ExcludeExitNodes.BadExit'
if 'ExcludeExitNodes' in keys:
elt = 'BadNodes.ExcludeExitNodes.BadExit'
LOG.warn(f"Remove ExcludeNodes and move then to {oArgs.bad_nodes}")
print(f"move to the {elt} section as a list")
if 'GuardNodes' in keys:
@ -174,7 +194,7 @@ def icheck_torrc(sFile, oArgs):
LOG.warn(f"Remove GuardNodes and move then to {oArgs.good_nodes}")
print(f"move to the {elt} section as a list")
if 'ExcludeNodes' in keys:
elt = 'ExcludeNodes.ExcludeExitNodes.BadExit'
elt = 'BadNodes.ExcludeNodes.BadExit'
LOG.warn(f"Remove ExcludeNodes and move then to {oArgs.bad_nodes}")
print(f"move to the {elt} section as a list")
if 'ControlSocket' not in keys and os.path.exists('/run/tor/control'):
@ -250,19 +270,20 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
a[elt] = a[elt].replace('[]', '@')
a.update({'fps': []})
keys = list(a.keys())
# test the url for fps and add it to the array
if 'proof' not in a:
LOG.warn(f"{fp} 'proof' not in {list(a.keys())}")
if 'proof' not in keys:
LOG.warn(f"{fp} 'proof' not in {keys}")
return a
if 'url' not in a:
if 'uri' not in a:
if 'url' not in keys:
if 'uri' not in keys:
a['url'] = ''
LOG.warn(f"{fp} url and uri not in {list(a.keys())}")
LOG.warn(f"{fp} url and uri not in {keys}")
return a
a['url'] = a['uri']
LOG.debug(f"{fp} 'uri' but not 'url' in {list(a.keys())}")
LOG.debug(f"{fp} 'uri' but not 'url' in {keys}")
# drop through
if a['url'].startswith('http:'):
@ -270,9 +291,17 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
elif not a['url'].startswith('https:'):
a['url'] = 'https:' +a['url']
# domain should be a unique ket for contacts
# domain should be a unique key for contacts
domain = a['url'][8:]
try:
ip = sTorResolve(domain)
except Exception as e:
lpair = getaddrinfo(domain, 443)
if lpait is None:
LOG.warn(f"TorResolv and getaddrinfo failed for {domain}")
return a
ip = lpair[0]
if a['proof'] not in ['uri-rsa']:
# only support uri for now
if False and ub_ctx:
@ -285,18 +314,33 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
LOG.warn(f"{fp} proof={a['proof']} not supported yet")
return a
LOG.debug(f"{len(list(a.keys()))} contact fields for {fp}")
LOG.debug(f"{len(keys)} contact fields for {fp}")
try:
LOG.debug(f"Downloading from {domain} for {fp}")
l = lDownloadUrlFps(domain, https_cafile,
timeout=timeout, host=host, port=port)
except Exception as e:
LOG.exception(f"Error downloading from {domain} for {fp} {e}")
o = oDownloadUrl(domain, https_cafile,
timeout=timeout, host=host, port=port)
# requests response: text "reason", "status_code"
except AttributeError as e:
LOG.exception(f"AttributeError downloading from {domain} {e}")
except CertificateError as e:
LOG.warn(f"CertificateError downloading from {domain} {e}")
lBAD_URLS += [a['url']]
except TrustorError as e:
LOG.warn(f"TrustorError downloading from {domain} {e.args}")
lBAD_URLS += [a['url']]
except (BaseException ) as e:
LOG.error(f"Exception {type(e)} downloading from {domain} {e}")
else:
if o.status_code >= 300:
LOG.warn(f"Error downloading from {domain} {o.status_code} {o.reason}")
# any reason retry?
lBAD_URLS += [a['url']]
return a
l = o.text.upper().strip().split('\n')
if not l:
# already squacked in lD
LOG.warn(f"Downloading from {domain} failed for {fp}")
LOG.warn(f"Downloading from {domain} empty for {fp}")
lBAD_URLS += [a['url']]
else:
a['fps'] = [elt for elt in l if elt and len(elt) == 40
@ -308,7 +352,6 @@ def aParseContact(contact, fp):
See the Tor ContactInfo Information Sharing Specification v2
https://nusenu.github.io/ContactInfo-Information-Sharing-Specification/
"""
contact = str(contact, 'UTF-8')
l = [line for line in contact.strip().replace('"', '').split(' ')
if ':' in line]
LOG.debug(f"{fp} {len(l)} fields")
@ -319,17 +362,6 @@ def aParseContact(contact, fp):
a = yaml.safe_load(oFd)
return a
def bAreWeConnected():
# FixMe: Linux only
sFile = f"/proc/{os.getpid()}/net/route"
if not os.path.isfile(sFile): return None
i = 0
for elt in open(sFile, "r").readlines():
if elt.startswith('Iface'): continue
if elt.startswith('lo'): continue
i += 1
return i > 0
def vwait_for_controller(controller, wait_boot):
if bAreWeConnected() is False:
raise SystemExit("we are not connected")
@ -412,22 +444,22 @@ def oMainArgparser(_=None):
parser.add_argument('--proxy_port', '--proxy-port', default=9050, type=int,
help='proxy control port')
parser.add_argument('--proxy_ctl', '--proxy-ctl',
default='/run/tor/control',
default='/run/tor/control' if os.path.exists('/run/tor/control') else 9051,
type=str,
help='control socket - or port')
parser.add_argument('--torrc',
default='',
default='/etc/tor/torrc-defaults',
type=str,
help='torrc to check for suggestions')
parser.add_argument('--timeout', default=30, type=int,
parser.add_argument('--timeout', default=60, type=int,
help='proxy download connect timeout')
parser.add_argument('--good_nodes', type=str,
default='/etc/tor/yaml/torrc-goodnodes.yaml',
default=os.path.join(ETC_DIR, '/torrc-goodnodes.yaml'),
help="Yaml file of good nodes that should not be excluded")
parser.add_argument('--bad_nodes', type=str,
default='/etc/tor/yaml/torrc-badnodes.yaml',
default=os.path.join(ETC_DIR, '/torrc-badnodes.yaml'),
help="Yaml file of bad nodes that should also be excluded")
parser.add_argument('--contact', type=str, default='Empty,NoEmail',
help="comma sep list of conditions - Empty,NoEmail")
@ -446,7 +478,7 @@ def oMainArgparser(_=None):
help="comma sep. list of onions to whitelist their introduction points - BROKEN")
parser.add_argument('--torrc_output', type=str, default='',
help="Write the torrc configuration to a file")
parser.add_argument('--proof_output', type=str, default='',
parser.add_argument('--proof_output', type=str, default=os.path.join(ETC_DIR, '/proof.yaml'),
help="Write the proof data of the included nodes to a YAML file")
return parser
@ -457,7 +489,7 @@ def vwrite_badnodes(oArgs):
bak = oArgs.bad_nodes +'.bak'
with open(tmp, 'wt') as oFYaml:
yaml.dump(oBAD_NODES, indent=2, stream=oFYaml)
LOG.info(f"Wrote {len(list(exit_excludelist))} proof details to {oArgs.bad_nodes}")
LOG.info(f"Wrote {len(list(oBAD_NODES.keys()))} to {oArgs.bad_nodes}")
oFYaml.close()
if os.path.exists(oArgs.bad_nodes):
os.rename(oArgs.bad_nodes, bak)
@ -470,7 +502,7 @@ def vwrite_goodnodes(oArgs):
bak = oArgs.good_nodes +'.bak'
with open(tmp, 'wt') as oFYaml:
yaml.dump(oGOOD_NODES, indent=2, stream=oFYaml)
LOG.info(f"Wrote {len(list(exit_excludelist))} proof details to {oArgs.good_nodes}")
LOG.info(f"Wrote {len(list(oGOOD_NODES.keys()))} good nodes to {oArgs.good_nodes}")
oFYaml.close()
if os.path.exists(oArgs.good_nodes):
os.rename(oArgs.good_nodes, bak)
@ -497,7 +529,7 @@ def iMain(lArgs):
except:
aTRUST_DB = {}
if oArgs.proxy_ctl.startswith('/') or os.path.exists(oArgs.proxy_ctl):
if os.path.exists(oArgs.proxy_ctl):
controller = oMakeController(sSock=oArgs.proxy_ctl)
else:
port =int(oArgs.proxy_ctl)
@ -543,18 +575,20 @@ def iMain(lArgs):
lProofGoodFps = []
iDnsContact = 0
iBadContact = 0
lBadContactUrls = []
iFakeContact = 0
aBadContacts = {}
aProofUri = {}
lConds = oArgs.contact.split(',')
iR = 0
for relay in relays:
iR += 1
if not is_valid_fingerprint(relay.fingerprint):
LOG.warn('Invalid Fingerprint: %s' % relay.fingerprint)
continue
relay.fingerprint = relay.fingerprint.upper()
sofar = f"G:{len(list(aProofUri.keys()))} U:{iDnsContact} F:{iFakeContact} BF:{len(exit_excludelist)} GF:{len(lProofGoodFps)}"
sofar = f"G:{len(list(aProofUri.keys()))} U:{iDnsContact} F:{iFakeContact} BF:{len(exit_excludelist)} GF:{len(lProofGoodFps)} #{iR}"
if not relay.exit_policy.is_exiting_allowed():
if sEXCLUDE_EXIT_KEY == 'ExcludeNodes':
LOG.debug(f"{relay.fingerprint} not an exit {sofar}")
@ -573,11 +607,20 @@ def iMain(lArgs):
continue
if relay.contact and b'dns-rsa' in relay.contact.lower():
LOG.info(f"{relay.fingerprint} skipping 'dns-rsa' {sofar}")
relay.contact = str(relay.contact, 'UTF-8')
c = relay.contact.lower()
i = c.find('url:')
if i >=0:
c = c[i+4:]
i = c.find(' ')
if i >=0:
c = c[:i]
LOG.info(f"{relay.fingerprint} skipping 'dns-rsa' {c} {sofar}")
iDnsContact += 1
continue
if relay.contact and b'proof:uri-rsa' in relay.contact.lower():
relay.contact = str(relay.contact, 'UTF-8')
a = aParseContact(relay.contact, relay.fingerprint)
if not a:
LOG.warn(f"{relay.fingerprint} did not parse {sofar}")
@ -588,6 +631,13 @@ def iMain(lArgs):
LOG.info(f"{relay.fingerprint} skipping in lBAD_URLS {a['url']} {sofar}")
exit_excludelist.append(relay.fingerprint)
continue
domain = a['url'][8:]
if domain in lKNOWN_NODNS:
# The fp is using a contact with a URL we know is nonexistent
LOG.info(f"{relay.fingerprint} skipping in lKNOWN_NODNS {a['url']} {sofar}")
exit_excludelist.append(relay.fingerprint)
continue
b = aVerifyContact(list(a.values())[0],
relay.fingerprint,
@ -597,7 +647,7 @@ def iMain(lArgs):
port=oArgs.proxy_port)
if not b['fps'] or not b['url']:
LOG.warn(f"{relay.fingerprint} did not verify {sofar}")
LOG.warn(f"{relay.fingerprint} did NOT VERIFY {sofar}")
# If it's giving contact info that doesnt check out
# it could be a bad exit with fake contact info
exit_excludelist.append(relay.fingerprint)
@ -605,7 +655,7 @@ def iMain(lArgs):
continue
if relay.fingerprint not in b['fps']:
LOG.warn(f"{relay.fingerprint} the fp is not in the list of fps {sofar}")
LOG.warn(f"{relay.fingerprint} the FP IS NOT in the list of fps {sofar}")
# assume a fp is using a bogus contact
exit_excludelist.append(relay.fingerprint)
iFakeContact += 1
@ -657,23 +707,39 @@ def iMain(lArgs):
global oBAD_NODES
oBAD_NODES['BadNodes']['ExcludeNodes']['BadExit'] = exit_excludelist
vwrite_badnodes(oArgs)
# nothing changed vwrite_goodnodes(oArgs)
global oGOOD_NODES
oGOOD_NODES['GoodNodes']['Relays']['ExitNodes'] = lProofGoodFps
vwrite_goodnodes(oArgs)
retval = 0
try:
logging.getLogger('stem').setLevel(30)
if exit_excludelist:
LOG.info(f"{sEXCLUDE_EXIT_KEY} {len(exit_excludelist)} net bad exit nodes")
controller.set_conf(sEXCLUDE_EXIT_KEY, exit_excludelist)
try:
if exit_excludelist:
LOG.info(f"{sEXCLUDE_EXIT_KEY} {len(exit_excludelist)} net bad exit nodes")
controller.set_conf(sEXCLUDE_EXIT_KEY, exit_excludelist)
if lProofGoodFps:
LOG.info(f"{sINCLUDE_EXIT_KEY} {len(lProofGoodFps)} good nodes")
controller.set_conf(sINCLUDE_EXIT_KEY, lProofGoodFps)
except stem.SocketClosed as e:
LOG.error(f"Failed setting {sEXCLUDE_EXIT_KEY} bad exit nodes in Tor")
retval += 1
o = oGOOD_NODES
if 'GuardNodes' in o[oGOOD_ROOT].keys():
LOG.info(f"{sINCLUDE_GUARD_KEY} {len(o[oGOOD_ROOT]['GuardNodes'])} guard nodes")
controller.set_conf(sINCLUDE_GUARD_KEY, o[oGOOD_ROOT]['GuardNodes'])
try:
if lProofGoodFps:
LOG.info(f"{sINCLUDE_EXIT_KEY} {len(lProofGoodFps)} good nodes")
controller.set_conf(sINCLUDE_EXIT_KEY, lProofGoodFps)
except stem.SocketClosed as e:
LOG.error(f"Failed setting {sINCLUDE_EXIT_KEY} good exit nodes in Tor")
retval += 1
try:
o = oGOOD_NODES
if 'GuardNodes' in o[oGOOD_ROOT].keys():
LOG.info(f"{sINCLUDE_GUARD_KEY} {len(o[oGOOD_ROOT]['GuardNodes'])} guard nodes")
controller.set_conf(sINCLUDE_GUARD_KEY, o[oGOOD_ROOT]['GuardNodes'])
except stem.SocketClosed as e:
LOG.errro(f"Failed setting {sINCLUDE_EXIT_KEY} good exit nodes in Tor")
retval += 1
)
return retval
except InvalidRequest as e: