diff --git a/README.md b/README.md index 5d92ee8..8be6c22 100644 --- a/README.md +++ b/README.md @@ -71,3 +71,79 @@ You can expect it to take an hour or two the first time this is run: >700 domains. For usage, do ```python3 exclude_badExits.py --help` + +## Usage +``` +usage: exclude_badExits.py [-h] [--https_cafile HTTPS_CAFILE] + [--proxy_host PROXY_HOST] [--proxy_port PROXY_PORT] + [--proxy_ctl PROXY_CTL] [--torrc TORRC] + [--timeout TIMEOUT] [--good_nodes GOOD_NODES] + [--bad_nodes BAD_NODES] [--contact CONTACT] + [--bad_contacts BAD_CONTACTS] + [--strict_nodes {0,1}] [--wait_boot WAIT_BOOT] + [--points_timeout POINTS_TIMEOUT] + [--log_level LOG_LEVEL] + [--bad_sections BAD_SECTIONS] + [--white_services WHITE_SERVICES] + [--torrc_output TORRC_OUTPUT] + [--proof_output PROOF_OUTPUT] +``` + +### Optional arguments: + +``` + -h, --help show this help message and exit + --https_cafile HTTPS_CAFILE + Certificate Authority file (in PEM) +``` +``` + --proxy_host PROXY_HOST, --proxy-host PROXY_HOST + proxy host + --proxy_port PROXY_PORT, --proxy-port PROXY_PORT + proxy control port + --proxy_ctl PROXY_CTL, --proxy-ctl PROXY_CTL + control socket - or port +``` +``` + --torrc TORRC torrc to check for suggestions + --timeout TIMEOUT proxy download connect timeout +``` +``` + --good_nodes GOOD_NODES + Yaml file of good info that should not be excluded + --bad_nodes BAD_NODES + Yaml file of bad nodes that should also be excluded +``` +``` + --contact CONTACT comma sep list of conditions - Empty,NoEmail + --bad_contacts BAD_CONTACTS + Yaml file of bad contacts that bad FPs are using +``` +``` + --strict_nodes {0,1} Set StrictNodes: 1 is less anonymous but more secure, + although some sites may be unreachable + --wait_boot WAIT_BOOT + Seconds to wait for Tor to booststrap + --points_timeout POINTS_TIMEOUT + Timeout for getting introduction points - must be long + >120sec. 0 means disabled looking for IPs +``` +``` + --log_level LOG_LEVEL + 10=debug 20=info 30=warn 40=error + --bad_sections BAD_SECTIONS + sections of the badnodes.yaml to use, comma separated, + '' BROKEN +``` +``` + --white_services WHITE_SERVICES + comma sep. list of onions to whitelist their + introduction points - BROKEN +``` +``` + --torrc_output TORRC_OUTPUT + Write the torrc configuration to a file + --proof_output PROOF_OUTPUT + Write the proof data of the included nodes to a YAML + file +``` diff --git a/exclude_badExits.bash b/exclude_badExits.bash new file mode 100644 index 0000000..d5d46aa --- /dev/null +++ b/exclude_badExits.bash @@ -0,0 +1,41 @@ +#!/bin/bash +# -*- mode: sh; fill-column: 75; tab-width: 8; coding: utf-8-unix -*- + +PROG=exclude_badExits.py +SOCKS_PORT=9050 +CAFILE=/etc/ssl/certs/ca-certificates.crt + +# an example of running exclude_badExits with full debugging +# expected to take an hour or so +declare -a LARGS +LARGS=( + --log_level 10 + ) +# you may have a special python for installed packages +EXE=`which python3.bash` +LARGS+=( + --strict_nodes 0 + --points_timeout 120 + --proxy-host 127.0.0.1 + --proxy-port $SOCKS_PORT + --https_cafile $CAFILE +) + +if [ -f '/run/tor/control' ] ; then + LARGS+=(--proxy-ctl '/run/tor/control' ) +else + LARGS+=(--proxy-ctl 9051 ) +fi + +ddg=duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad +# for example, whitelist the introduction points to DuckDuckGo +LARGS+=( --white_onions $ddg ) + +# you may need to be the tor user to read /run/tor/control +grep -q ^debian-tor /etc/group && TORU=debian-tor || { + grep -q ^tor /etc/group && TORU=tor +} +sudo -u $TORU $EXE exclude_badExits.py "${LARGS[@]}" \ + 2>&1|tee exclude_badExits6.log + +# The DEBUG statements contain the detail of why the relay was considered bad. diff --git a/exclude_badExits.py b/exclude_badExits.py index 00e7a01..406e65c 100644 --- a/exclude_badExits.py +++ b/exclude_badExits.py @@ -48,7 +48,7 @@ exclusion: the ```--contact``` commandline arg is a comma sep list of conditions More may be added later. Because you don't want to exclude the introduction points to any onion -you want to connect to, ```--white_services``` should whitelist the +you want to connect to, ```--white_onions``` should whitelist the introduction points to a comma sep list of onions, but is currently broken in stem 1.8.0: see: * https://github.com/torproject/stem/issues/96 @@ -65,7 +65,7 @@ not just exclude Exit. If the Contact info is good we add the list of fingerprints to add to ExitNodes, a whitelist of relays to use as exits. -```--proof_output``` will write the contact info as a ciiss dictionary +```--good_contacts``` will write the contact info as a ciiss dictionary to a YAML file. If the proof is uri-rsa, the well-known file of fingerprints is downloaded and the fingerprints are added on a 'fps' field we create of that fingerprint's entry of the YAML dictionary. This file is read at the @@ -88,7 +88,7 @@ import time import argparse import string from io import StringIO -import ipaddr +import ipaddress # list(ipaddress._find_address_range(ipaddress.IPv4Network('172.16.0.0/12')) from urllib3.util.ssl_match_hostname import CertificateError @@ -98,15 +98,24 @@ from stem.control import Controller from stem.connection import IncorrectPassword from stem.util.tor_tools import is_valid_fingerprint try: - import yaml + from ruamel.yaml import YAML + yaml = YAML(typ='rt') + yaml.indent(mapping=2, sequence=2) + safe_load = yaml.load except: yaml = None +if yaml is None: + try: + import yaml + safe_load = yaml.safe_load + except: + yaml = None + try: from unbound import ub_ctx,RR_TYPE_TXT,RR_CLASS_IN except: ub_ctx = RR_TYPE_TXT = RR_CLASS_IN = None - global LOG import logging import warnings @@ -158,7 +167,7 @@ def lYamlBadNodes(sFile, if not yaml: return l if os.path.exists(sFile): with open(sFile, 'rt') as oFd: - oBAD_NODES = yaml.safe_load(oFd) + oBAD_NODES = safe_load(oFd) # BROKEN # root = 'ExcludeNodes' @@ -170,7 +179,7 @@ def lYamlBadNodes(sFile, root = 'ExcludeDomains' if root not in oBAD_NODES[oBAD_ROOT] or not oBAD_NODES[oBAD_ROOT][root]: - lMAYBE_NODNS = yaml.safe_load(StringIO(yKNOWN_NODNS)) + lMAYBE_NODNS = safe_load(StringIO(yKNOWN_NODNS)) else: lMAYBE_NODNS = oBAD_NODES[oBAD_ROOT][root] return l @@ -184,7 +193,7 @@ def lYamlGoodNodes(sFile='/etc/tor/torrc-goodnodes.yaml'): if not yaml: return l if os.path.exists(sFile): with open(sFile, 'rt') as oFd: - o = yaml.safe_load(oFd) + o = safe_load(oFd) oGOOD_NODES = o if 'GuardNodes' in o[oGOOD_ROOT].keys(): l = o[oGOOD_ROOT]['GuardNodes'] @@ -271,7 +280,7 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050) LOG.warn(f"{domain} is bad from {a['url']}") LOG.debug(f"{fp} is bad from {a}") return a - + ip = zResolveDomain(domain) if ip == '': aFP_EMAIL[fp] = a['email'] @@ -373,7 +382,7 @@ def aParseContact(contact, fp): s += '\n'.join([f" {line}\"".replace(':',': \"', 1) for line in l]) oFd = StringIO(s) - a = yaml.safe_load(oFd) + a = safe_load(oFd) return a def oMainArgparser(_=None): @@ -414,10 +423,10 @@ def oMainArgparser(_=None): help='proxy download connect timeout') parser.add_argument('--good_nodes', type=str, - default=os.path.join(ETC_DIR, 'torrc-goodnodes.yaml'), + default=os.path.join(ETC_DIR, 'goodnodes.yaml'), help="Yaml file of good info that should not be excluded") parser.add_argument('--bad_nodes', type=str, - default=os.path.join(ETC_DIR, 'torrc-badnodes.yaml'), + default=os.path.join(ETC_DIR, 'badnodes.yaml'), help="Yaml file of bad nodes that should also be excluded") parser.add_argument('--contact', type=str, default='Empty,NoEmail', help="comma sep list of conditions - Empty,NoEmail") @@ -437,13 +446,13 @@ def oMainArgparser(_=None): parser.add_argument('--bad_sections', type=str, default='MyBadExit', help="sections of the badnodes.yaml to use, comma separated, '' BROKEN") - parser.add_argument('--white_services', type=str, + parser.add_argument('--white_onions', type=str, default='', help="comma sep. list of onions to whitelist their introduction points - BROKEN") parser.add_argument('--torrc_output', type=str, default=os.path.join(ETC_DIR, 'torrc.new'), help="Write the torrc configuration to a file") - parser.add_argument('--proof_output', type=str, default=os.path.join(ETC_DIR, 'proof.yaml'), + parser.add_argument('--good_contacts', type=str, default=os.path.join(ETC_DIR, 'goodcontacts.yaml'), help="Write the proof data of the included nodes to a YAML file") return parser @@ -452,7 +461,7 @@ def vwrite_badnodes(oArgs, oBAD_NODES, slen): tmp = oArgs.bad_nodes +'.tmp' bak = oArgs.bad_nodes +'.bak' with open(tmp, 'wt') as oFYaml: - yaml.dump(oBAD_NODES, indent=2, stream=oFYaml) + yaml.dump(oBAD_NODES, oFYaml) LOG.info(f"Wrote {slen} to {oArgs.bad_nodes}") oFYaml.close() if os.path.exists(oArgs.bad_nodes): @@ -464,7 +473,7 @@ def vwrite_goodnodes(oArgs, oGOOD_NODES, ilen): tmp = oArgs.good_nodes +'.tmp' bak = oArgs.good_nodes +'.bak' with open(tmp, 'wt') as oFYaml: - yaml.dump(oGOOD_NODES, indent=2, stream=oFYaml) + yaml.dump(oGOOD_NODES, oFYaml) LOG.info(f"Wrote {ilen} good relays to {oArgs.good_nodes}") oFYaml.close() if os.path.exists(oArgs.good_nodes): @@ -489,12 +498,11 @@ def iMain(lArgs): icheck_torrc(sFile, oArgs) twhitelist_set = set() - sFile = oArgs.proof_output + sFile = oArgs.good_contacts if sFile and os.path.exists(sFile): try: with open(sFile, 'rt') as oFd: - aTRUST_DB = yaml.safe_load(oFd) - assert type(aTRUST_DB) == dict + aTRUST_DB = safe_load(oFd) LOG.info(f"{len(aTRUST_DB.keys())} trusted contacts from {sFile}") # reverse lookup of fps to contacts # but... @@ -520,8 +528,8 @@ def iMain(lArgs): vwait_for_controller(controller, oArgs.wait_boot) - if oArgs.proof_output: - proof_output_tmp = oArgs.proof_output + '.tmp' + if oArgs.good_contacts: + good_contacts_tmp = oArgs.good_contacts + '.tmp' elt = controller.get_conf('UseMicrodescriptors') if elt != '0' : @@ -541,10 +549,17 @@ def iMain(lArgs): t = set() if 'IntroductionPoints' in oGOOD_NODES[oGOOD_ROOT]['Relays'].keys(): t = set(oGOOD_NODES[oGOOD_ROOT]['Relays']['IntroductionPoints']) - # not working = maybe when stem is updated - w = set(oGOOD_NODES[oGOOD_ROOT]['Services']) - if oArgs.white_services: - w.update(oArgs.white_services.split(',')) + w = set() + if 'Services' in oGOOD_NODES[oGOOD_ROOT].keys(): + # 'Onions' can I use the IntroductionPoints for Services too? + # w = set(oGOOD_NODES[oGOOD_ROOT]['Services']) + pass + if 'Onions' in oGOOD_NODES[oGOOD_ROOT].keys(): + # Provides the descriptor for a hidden service. The **address** is the + # '.onion' address of the hidden service + w = set(oGOOD_NODES[oGOOD_ROOT]['Onions']) + if oArgs.white_onions: + w.update(oArgs.white_onions.split(',')) if oArgs.points_timeout > 0: LOG.info(f"{len(w)} services will be checked from IntroductionPoints") t.update(lIntroductionPoints(controller, w, itimeout=oArgs.points_timeout)) @@ -697,10 +712,10 @@ def iMain(lArgs): aTRUST_DB[relay.fingerprint] = b for elt in b['fps']: aTRUST_DB_INDEX[elt] = b - if oArgs.proof_output and oArgs.log_level <= 20: + if oArgs.good_contacts and oArgs.log_level <= 20: # as we go along then clobber - with open(proof_output_tmp, 'wt') as oFYaml: - yaml.dump(aTRUST_DB, indent=2, stream=oFYaml) + with open(good_contacts_tmp, 'wt') as oFYaml: + yaml.dump(aTRUST_DB, oFYaml) oFYaml.close() LOG.info(f"Filtered {len(twhitelist_set)} whitelisted relays") @@ -709,16 +724,6 @@ def iMain(lArgs): texclude_set = texclude_set.difference(tdns_urls) LOG.info(f"{len(list(aTRUST_DB.keys()))} good contacts out of {iTotalContacts}") - if oArgs.proof_output and aTRUST_DB: - with open(proof_output_tmp, 'wt') as oFYaml: - yaml.dump(aTRUST_DB, indent=2, stream=oFYaml) - oFYaml.close() - if os.path.exists(oArgs.proof_output): - bak = oArgs.proof_output +'.bak' - os.rename(oArgs.proof_output, bak) - os.rename(proof_output_tmp, oArgs.proof_output) - LOG.info(f"Wrote {len(list(aTRUST_DB.keys()))} good contact details to {oArgs.proof_output}") - if oArgs.torrc_output and texclude_set: with open(oArgs.torrc_output, 'wt') as oFTorrc: oFTorrc.write(f"{sEXCLUDE_EXIT_KEY} {','.join(texclude_set)}\n") @@ -730,9 +735,19 @@ def iMain(lArgs): if oArgs.bad_contacts and aBadContacts: # for later analysis with open(oArgs.bad_contacts, 'wt') as oFYaml: - yaml.dump(aBadContacts, indent=2, stream=oFYaml) + yaml.dump(aBadContacts, oFYaml) oFYaml.close() + if oArgs.good_contacts != '' and aTRUST_DB: + with open(good_contacts_tmp, 'wt') as oFYaml: + yaml.dump(aTRUST_DB, oFYaml) + oFYaml.close() + if os.path.exists(oArgs.good_contacts): + bak = oArgs.good_contacts +'.bak' + os.rename(oArgs.good_contacts, bak) + os.rename(good_contacts_tmp, oArgs.good_contacts) + LOG.info(f"Wrote {len(list(aTRUST_DB.keys()))} good contact details to {oArgs.good_contacts}") + oBAD_NODES[oBAD_ROOT]['ExcludeNodes']['BadExit'] = list(texclude_set) oBAD_NODES[oBAD_ROOT]['ExcludeDomains'] = lKNOWN_NODNS vwrite_badnodes(oArgs, oBAD_NODES, str(len(texclude_set))) @@ -740,6 +755,7 @@ def iMain(lArgs): oGOOD_NODES['GoodNodes']['Relays']['ExitNodes'] = list(aTRUST_DB_INDEX.keys()) # GuardNodes are readonl vwrite_goodnodes(oArgs, oGOOD_NODES, len(aTRUST_DB_INDEX.keys())) + retval = 0 try: logging.getLogger('stem').setLevel(30) @@ -776,9 +792,6 @@ def iMain(lArgs): LOG.errro(f"Failed setting {sINCLUDE_EXIT_KEY} good exit nodes in Tor") retval += 1 - sys.stdout.write("dns-rsa domains:\n" +'\n'.join(tdns_urls) +'\n') - return retval - except InvalidRequest as e: # Unacceptable option value: Invalid router list. LOG.error(str(e)) @@ -802,6 +815,9 @@ def iMain(lArgs): controller.close() except Exception as e: LOG.warn(str(e)) + + sys.stdout.write("dns-rsa domains:\n" +'\n'.join(tdns_urls) +'\n') + return retval if __name__ == '__main__': try: diff --git a/support_onions.py b/support_onions.py index 68d3ee0..d42feba 100644 --- a/support_onions.py +++ b/support_onions.py @@ -14,10 +14,12 @@ if False: import cepa as stem from cepa.control import Controller from cepa.connection import MissingPassword + from cepa.util.tor_tools import is_valid_fingerprint else: import stem from stem.control import Controller from stem.connection import MissingPassword + from stem.util.tor_tools import is_valid_fingerprint global LOG import logging @@ -96,7 +98,7 @@ def oGetStemController(log_level=10, sock_or_pair='/run/tor/control'): controller.authenticate(p) oSTEM_CONTROLER = controller LOG.debug(f"{controller}") - return oSTEM_CONTROLER + return oSTEM_CONTROLER def bAreWeConnected(): # FixMe: Linux only @@ -144,7 +146,11 @@ def bin_to_hex(raw_id, length=None): return res.upper() def lIntroductionPoints(controller=None, lOnions=[], itimeout=120, log_level=10): - """now working !!! stem 1.8.x timeout must be huge >120""" + """now working !!! stem 1.8.x timeout must be huge >120 + 'Provides the descriptor for a hidden service. The **address** is the + '.onion' address of the hidden service ' + What about Services? + """ try: from cryptography.utils import int_from_bytes except ImportError: @@ -154,12 +160,12 @@ def lIntroductionPoints(controller=None, lOnions=[], itimeout=120, log_level=10) # this will fai if the trick above didnt work from stem.prereq import is_crypto_available is_crypto_available(ed25519 = True) - + from stem.descriptor.hidden_service import HiddenServiceDescriptorV3 from stem.client.datatype import LinkByFingerprint from stem import Timeout from queue import Empty - + if type(lOnions) not in [set, tuple, list]: lOnions = list(lOnions) if controller is None: @@ -277,7 +283,7 @@ def sTorResolve(target, LOG.info(sLabel +f"{i} on {sHost}:{iPort}" ) sock.close() raise SystemExit(5) - + assert len(data) >= 8 packet_sf = data[1] if packet_sf == 90: @@ -292,7 +298,7 @@ def sTorResolve(target, # os.system("strace tor-resolve -4 "+target+" 2>&1|grep '^sen\|^rec'") return '' - + def getaddrinfo(sHost, sPort): # do this the explicit way = Ive seen the compact connect fail # >>> sHost, sPort = 'l27.0.0.1', 33446 @@ -393,4 +399,4 @@ def lExitExcluder(oArgs, iPort=9051, log_level=10): if __name__ == '__main__': target = 'duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad' controller = oGetStemController(log_level=10) - lIntroductionPoints(controller, [target], itimeout=120) + lIntroductionPoints(controller, [target], itimeout=120) diff --git a/trustor_poc.py b/trustor_poc.py index 0f9406a..4ddc6c1 100644 --- a/trustor_poc.py +++ b/trustor_poc.py @@ -355,7 +355,7 @@ def _my_match_hostname(cert, asserted_hostname): try: my_match_hostname(cert, asserted_hostname) except CertificateError as e: - log.warning( + LOG.warning( "Certificate did not match hostname: %s. Certificate: %s", asserted_hostname, cert,