emdee/exclude_badExits
1
0
Fork 0
Code Issues Pull requests Releases Packages Wiki Activity

Added well-known downloading

Browse source
This commit is contained in:
emdee 2022-11-07 11:38:22 +00:00
parent e7e3ddd20d
commit 2be5e6e66e
3 changed files with 564 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

192
exclude_badExits.py
View file

@ -11,7 +11,8 @@ on the ExcludeNodes or ExcludeExitNodes setting of a running Tor.
The basic cut is to exclude Exit nodes that do not have a contact.
That can be extended to nodes that do not have an email in the contact etc.
"""
"""
But there's a problem, and your Tor notice.log will tell you about it:
you could exclude the nodes needed to access hidden services or
directorues. So we need to add to the process the concept of a whitelist.
@ -53,24 +54,35 @@ currently broken in stem 1.8.0: see:
* https://github.com/torproject/stem/issues/96
* https://gitlab.torproject.org/legacy/trac/-/issues/25417
```--bad_output``` will write the torrc configuration to a file.
```--bad_output``` will write the torrc ExcludeNodes configuration to a file.
```--details_output``` will write the lookup URLs of the excluded nodes to a file
```--proof_output``` will write the contact info as a ciiss dictionary
to a YAML file. If the proof is uri-rsa, the well-known file of fingerprints
is downloaded and the fingerprints are added to the on the 'fps' field
of that fingerprint entry of the YAML dictionary. This file is read at the
beginning of the program to start with a trust database, and only new
relays are added to the dictionary. The 'fps' field is emptied if the
host fails to provide the well-known file. You can expect it to take
an hour or two the first time this is run: >700 domains.
For usage, do ```python3 exclude_badExits.py --help`
"""
import sys
from stem.control import Controller
from stem.util.tor_tools import is_valid_fingerprint
import os
import getpass
import re
import time
import argparse
from io import StringIO
from stem.control import Controller
from stem.connection import IncorrectPassword
from stem.util.tor_tools import is_valid_fingerprint
try:
import yaml
except:
@ -83,16 +95,18 @@ try:
# https://pypi.org/project/coloredlogs/
except ImportError as e:
coloredlogs = False
from trustor_poc import lDownloadUrlFps
global LOG
import logging
LOG = logging.getLogger()
aTRUST_DB = {}
sDETAILS_URL = "https://metrics.torproject.org/rs.html#details/"
# You can call this while bootstrapping
def oMakeController(sSock='/run/tor/control', port=9051):
if os.path.exists(sSock):
def oMakeController(sSock='', port=9051):
if sSock and os.path.exists(sSock):
controller = Controller.from_socket_file(path=sSock)
else:
controller = Controller.from_port(port=port)
@ -148,23 +162,85 @@ def lIntroductionPoints(lOnions):
l += [introduction_point.address]
return l
# memory?
lINTS = ['ciissversion', 'uplinkbw', 'signingkeylifetime']
lBOOLS = ['dnssec', 'dnsqname', 'aesni', 'autoupdate', 'dnslocalrootzone'
'sandbox', 'offlinemasterkey']
def aVerifyContact(a, fp, timeout=20, host='127.0.0.1', port=9050):
for elt in lINTS:
if elt in a:
a[elt] = int(a[elt])
for elt in lBOOLS:
if elt in a:
if a[elt] in ['y','yes', 'true', 'True']:
a[elt] = True
else:
a[elt] = False
# just stick fp in for now
a.update({'fps': [fp]})
# test the url for fps and add it to the array
if 'proof' not in a:
# only support uri for now
LOG.warn(f"{fp} 'proof' not in {list(a.keys())}")
return a
if a['proof'] not in ['uri-rsa']:
# only support uri for now
LOG.warn(f"{fp} proof={a['proof']} not supported yet")
return a
if 'url' not in a:
LOG.warn(f"{fp} 'proof' is 'uri-rsa' but url not in {list(a.keys())}")
return a
if a['url'].startswith('http:'):
a['url'] = 'https:' +a['url'][5:]
elif not a['url'].startswith('https:'):
a['url'] = 'https:' +a['url']
domain = a['url'][8:]
LOG.debug(f"{len(list(a.keys()))} contact fields for {fp}")
LOG.info(f"Downloading from {domain} for {fp}")
try:
l = lDownloadUrlFps(domain, timeout=20, host=host, port=port)
except Exception as e:
LOG.exception(f"Error downloading from {domain} for {fp} {e}")
# should we put it's FPs from TRUST_DB on the ExcludeExitNodes?
a['fps'] = []
else:
if not l:
LOG.warn(f"Downloading from {domain} failed for {fp}")
a['fps'] = []
else:
a['fps'] = l
return a
def aParseContact(contact, fp):
contact = str(contact, 'UTF-8')
l = [line for line in contact.strip().replace('"', '').split(' ')
if ':' in line]
LOG.debug(f"{fp} {len(l)} fields")
s = f'"{fp}":\n'
s += '\n'.join([f" {line}\"".replace(':',': \"', 1)
for line in l])
oFd = StringIO(s)
a = yaml.safe_load(oFd)
return a
def oMainArgparser(_=None):
# 'Mode: 0=chat 1=chat+audio 2=chat+audio+video default: 0'
if not os.path.exists('/proc/sys/net/ipv6'):
bIpV6 = 'False'
else:
bIpV6 = 'True'
lIpV6Choices=[bIpV6, 'False']
parser = argparse.ArgumentParser(add_help=True)
parser = argparse.ArgumentParser(add_help=True,
epilog=__doc__)
parser.add_argument('--proxy_host', '--proxy-host', type=str,
default='127.0.0.1',
help='proxy host')
parser.add_argument('--proxy_port', '--proxy-port', default=9051, type=int,
parser.add_argument('--proxy_port', '--proxy-port', default=9050, type=int,
help='proxy control port')
parser.add_argument('--proxy_ctl', '--proxy-ctl',
default='/run/tor/control', type=str,
help='control socket - takes precedence over proxy_port')
default='/run/tor/control',
type=str,
help='control socket - or port')
parser.add_argument('--timeout', default=20, type=int,
help='proxy download timeout')
parser.add_argument('--good_nodes', type=str,
default='/etc/tor/torrc-goodnodes.yaml',
help="Yaml file of good nodes that should not be excluded")
@ -187,10 +263,13 @@ def oMainArgparser(_=None):
help="Write the torrc configuration to a file")
parser.add_argument('--details_output', type=str, default='',
help="Write the lookup URLs of the excluded nodes to a file")
parser.add_argument('--proof_output', type=str, default='',
help="Write the proof data of the included nodes to a YAML file")
return parser
def iMain(lArgs):
global oTOX_OARGS
global aTRUST_DB
parser = oMainArgparser()
oArgs = parser.parse_args(lArgs)
@ -200,14 +279,25 @@ def iMain(lArgs):
force=True)
logging.basicConfig(**aKw)
logging.getLogger('stem').setLevel(oArgs.log_level)
controller = oMakeController(oArgs.proxy_ctl, oArgs.proxy_port)
sFile = oArgs.proof_output
if sFile and os.path.exists(sFile):
with open(sFile, 'rt') as oFd:
aTRUST_DB = yaml.safe_load(oFd)
if oArgs.proxy_ctl.startswith('/') or os.path.exists(oArgs.proxy_ctl):
controller = oMakeController(sSock=oArgs.proxy_ctl)
else:
port =int(oArgs.proxy_ctl)
controller = oMakeController(port=port)
elt = controller.get_conf('UseMicrodescriptors')
if elt != '0' :
LOG.error('"UseMicrodescriptors 0" is required in your /etc/tor/torrc. Exiting.')
return 2
LOG.warn('"UseMicrodescriptors 0" is required in your /etc/tor/torrc. Exiting.')
controller.set_conf('UseMicrodescriptors', 0)
# does it work dynamically?
# return 2
percent = i = 0
# You can call this while boostrapping
while percent < 100 and i < oArgs.wait_boot:
@ -225,7 +315,7 @@ def iMain(lArgs):
LOG.info(f'lYamlGoodNodes {len(lGood)}')
if oArgs.white_onions:
l = lIntroductionPoints(oArgs.white_onions.split(,))
l = lIntroductionPoints(oArgs.white_onions.split(','))
lGood += l
relays = controller.get_server_descriptors()
@ -243,30 +333,63 @@ def iMain(lArgs):
else:
oFd = None
lProofUriFps = []
aProofUri = {}
lConds = oArgs.contact.split(',')
for relay in relays:
if not relay.exit_policy.is_exiting_allowed(): continue
if not is_valid_fingerprint(relay.fingerprint):
LOG.warn('Invalid Fingerprint: %s' % relay.fingerprint)
continue
if relay.fingerprint in aTRUST_DB:
if aTRUST_DB[relay.fingerprint]['fps']:
lProofUriFps += aTRUST_DB[relay.fingerprint]['fps']
if relay.fingerprint in lProofUriFps:
# we already have it.
continue
if relay.contact and b'proof:uri-rsa' in relay.contact.lower():
a = aParseContact(relay.contact, relay.fingerprint)
if not a: continue
b = aVerifyContact(list(a.values())[0], relay.fingerprint,
timeout=oArgs.timeout,
host=oArgs.proxy_host,
port=oArgs.proxy_port)
if not b:
continue
if 'fps' in b and b['fps'] and relay.fingerprint in b['fps']:
lProofUriFps += b['fps']
aProofUri[relay.fingerprint] = b
if ('Empty' in lConds and not relay.contact) or \
('NoEmail' in lConds and relay.contact and not b'@' in relay.contact):
if is_valid_fingerprint(relay.fingerprint):
exit_excludelist.append(relay.fingerprint)
if oFd:
oFd.write(sDETAILS_URL +relay.fingerprint +"\n")
else:
LOG.warn('Invalid Fingerprint: %s' % relay.fingerprint)
if oFd:
LOG.info(f"Wrote details URLs to {oArgs.details_output}")
oFd.close()
exit_excludelist.append(relay.fingerprint)
if oFd:
oFd.write(sDETAILS_URL +relay.fingerprint +"\n")
exit_excludelist = list(set(exit_excludelist).difference(set(lGood)))
LOG.info(f'ExcludeExitNodes {len(exit_excludelist)} net bad exit nodes')
controller.set_conf('ExcludeExitNodes', exit_excludelist)
elt = controller.get_conf('ExcludeExitNodes')
if oArgs.bad_output:
with open(oArgs.bad_output, 'wt') as oFd:
oFd.write(f"ExcludeExitNodes {','.join(exit_excludelist)}\n")
with open(oArgs.bad_output, 'wt') as oFdE:
oFdE.write(f"ExcludeExitNodes {','.join(exit_excludelist)}\n")
LOG.info(f"Wrote tor configuration to {oArgs.bad_output}")
if lProofUriFps:
LOG.info(f'ExitNodes {len(lProofUriFps)} good exit nodes')
controller.set_conf('ExitNodes', lProofUriFps)
if oFd:
LOG.info(f"Wrote details URLs to {oArgs.details_output}")
oFd.close()
if oArgs.proof_output:
with open(oArgs.proof_output, 'wt') as oFdD:
s = yaml.dump_all(aProofUri, indent=2, stream=None)
oFdD.write(s +'\n')
LOG.info(f"Wrote proof details to {oArgs.proof_output}")
oFdD.close()
logging.getLogger('stem').setLevel(40)
for elt in controller._event_listeners:
controller.remove_event_listener(elt)
@ -277,6 +400,9 @@ def iMain(lArgs):
if __name__ == '__main__':
try:
i = iMain(sys.argv[1:])
except IncorrectPassword as e:
LOG.error(e)
i = 1
except Exception as e:
LOG.exception(e)
i = 1