diff --git a/README.md b/README.md index 8be6c22..5d92ee8 100644 --- a/README.md +++ b/README.md @@ -71,79 +71,3 @@ You can expect it to take an hour or two the first time this is run: >700 domains. For usage, do ```python3 exclude_badExits.py --help` - -## Usage -``` -usage: exclude_badExits.py [-h] [--https_cafile HTTPS_CAFILE] - [--proxy_host PROXY_HOST] [--proxy_port PROXY_PORT] - [--proxy_ctl PROXY_CTL] [--torrc TORRC] - [--timeout TIMEOUT] [--good_nodes GOOD_NODES] - [--bad_nodes BAD_NODES] [--contact CONTACT] - [--bad_contacts BAD_CONTACTS] - [--strict_nodes {0,1}] [--wait_boot WAIT_BOOT] - [--points_timeout POINTS_TIMEOUT] - [--log_level LOG_LEVEL] - [--bad_sections BAD_SECTIONS] - [--white_services WHITE_SERVICES] - [--torrc_output TORRC_OUTPUT] - [--proof_output PROOF_OUTPUT] -``` - -### Optional arguments: - -``` - -h, --help show this help message and exit - --https_cafile HTTPS_CAFILE - Certificate Authority file (in PEM) -``` -``` - --proxy_host PROXY_HOST, --proxy-host PROXY_HOST - proxy host - --proxy_port PROXY_PORT, --proxy-port PROXY_PORT - proxy control port - --proxy_ctl PROXY_CTL, --proxy-ctl PROXY_CTL - control socket - or port -``` -``` - --torrc TORRC torrc to check for suggestions - --timeout TIMEOUT proxy download connect timeout -``` -``` - --good_nodes GOOD_NODES - Yaml file of good info that should not be excluded - --bad_nodes BAD_NODES - Yaml file of bad nodes that should also be excluded -``` -``` - --contact CONTACT comma sep list of conditions - Empty,NoEmail - --bad_contacts BAD_CONTACTS - Yaml file of bad contacts that bad FPs are using -``` -``` - --strict_nodes {0,1} Set StrictNodes: 1 is less anonymous but more secure, - although some sites may be unreachable - --wait_boot WAIT_BOOT - Seconds to wait for Tor to booststrap - --points_timeout POINTS_TIMEOUT - Timeout for getting introduction points - must be long - >120sec. 0 means disabled looking for IPs -``` -``` - --log_level LOG_LEVEL - 10=debug 20=info 30=warn 40=error - --bad_sections BAD_SECTIONS - sections of the badnodes.yaml to use, comma separated, - '' BROKEN -``` -``` - --white_services WHITE_SERVICES - comma sep. list of onions to whitelist their - introduction points - BROKEN -``` -``` - --torrc_output TORRC_OUTPUT - Write the torrc configuration to a file - --proof_output PROOF_OUTPUT - Write the proof data of the included nodes to a YAML - file -``` diff --git a/exclude_badExits.bash b/exclude_badExits.bash deleted file mode 100644 index d5d46aa..0000000 --- a/exclude_badExits.bash +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# -*- mode: sh; fill-column: 75; tab-width: 8; coding: utf-8-unix -*- - -PROG=exclude_badExits.py -SOCKS_PORT=9050 -CAFILE=/etc/ssl/certs/ca-certificates.crt - -# an example of running exclude_badExits with full debugging -# expected to take an hour or so -declare -a LARGS -LARGS=( - --log_level 10 - ) -# you may have a special python for installed packages -EXE=`which python3.bash` -LARGS+=( - --strict_nodes 0 - --points_timeout 120 - --proxy-host 127.0.0.1 - --proxy-port $SOCKS_PORT - --https_cafile $CAFILE -) - -if [ -f '/run/tor/control' ] ; then - LARGS+=(--proxy-ctl '/run/tor/control' ) -else - LARGS+=(--proxy-ctl 9051 ) -fi - -ddg=duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad -# for example, whitelist the introduction points to DuckDuckGo -LARGS+=( --white_onions $ddg ) - -# you may need to be the tor user to read /run/tor/control -grep -q ^debian-tor /etc/group && TORU=debian-tor || { - grep -q ^tor /etc/group && TORU=tor -} -sudo -u $TORU $EXE exclude_badExits.py "${LARGS[@]}" \ - 2>&1|tee exclude_badExits6.log - -# The DEBUG statements contain the detail of why the relay was considered bad. diff --git a/exclude_badExits.py b/exclude_badExits.py index 406e65c..00e7a01 100644 --- a/exclude_badExits.py +++ b/exclude_badExits.py @@ -48,7 +48,7 @@ exclusion: the ```--contact``` commandline arg is a comma sep list of conditions More may be added later. Because you don't want to exclude the introduction points to any onion -you want to connect to, ```--white_onions``` should whitelist the +you want to connect to, ```--white_services``` should whitelist the introduction points to a comma sep list of onions, but is currently broken in stem 1.8.0: see: * https://github.com/torproject/stem/issues/96 @@ -65,7 +65,7 @@ not just exclude Exit. If the Contact info is good we add the list of fingerprints to add to ExitNodes, a whitelist of relays to use as exits. -```--good_contacts``` will write the contact info as a ciiss dictionary +```--proof_output``` will write the contact info as a ciiss dictionary to a YAML file. If the proof is uri-rsa, the well-known file of fingerprints is downloaded and the fingerprints are added on a 'fps' field we create of that fingerprint's entry of the YAML dictionary. This file is read at the @@ -88,7 +88,7 @@ import time import argparse import string from io import StringIO -import ipaddress +import ipaddr # list(ipaddress._find_address_range(ipaddress.IPv4Network('172.16.0.0/12')) from urllib3.util.ssl_match_hostname import CertificateError @@ -98,24 +98,15 @@ from stem.control import Controller from stem.connection import IncorrectPassword from stem.util.tor_tools import is_valid_fingerprint try: - from ruamel.yaml import YAML - yaml = YAML(typ='rt') - yaml.indent(mapping=2, sequence=2) - safe_load = yaml.load + import yaml except: yaml = None -if yaml is None: - try: - import yaml - safe_load = yaml.safe_load - except: - yaml = None - try: from unbound import ub_ctx,RR_TYPE_TXT,RR_CLASS_IN except: ub_ctx = RR_TYPE_TXT = RR_CLASS_IN = None + global LOG import logging import warnings @@ -167,7 +158,7 @@ def lYamlBadNodes(sFile, if not yaml: return l if os.path.exists(sFile): with open(sFile, 'rt') as oFd: - oBAD_NODES = safe_load(oFd) + oBAD_NODES = yaml.safe_load(oFd) # BROKEN # root = 'ExcludeNodes' @@ -179,7 +170,7 @@ def lYamlBadNodes(sFile, root = 'ExcludeDomains' if root not in oBAD_NODES[oBAD_ROOT] or not oBAD_NODES[oBAD_ROOT][root]: - lMAYBE_NODNS = safe_load(StringIO(yKNOWN_NODNS)) + lMAYBE_NODNS = yaml.safe_load(StringIO(yKNOWN_NODNS)) else: lMAYBE_NODNS = oBAD_NODES[oBAD_ROOT][root] return l @@ -193,7 +184,7 @@ def lYamlGoodNodes(sFile='/etc/tor/torrc-goodnodes.yaml'): if not yaml: return l if os.path.exists(sFile): with open(sFile, 'rt') as oFd: - o = safe_load(oFd) + o = yaml.safe_load(oFd) oGOOD_NODES = o if 'GuardNodes' in o[oGOOD_ROOT].keys(): l = o[oGOOD_ROOT]['GuardNodes'] @@ -280,7 +271,7 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050) LOG.warn(f"{domain} is bad from {a['url']}") LOG.debug(f"{fp} is bad from {a}") return a - + ip = zResolveDomain(domain) if ip == '': aFP_EMAIL[fp] = a['email'] @@ -382,7 +373,7 @@ def aParseContact(contact, fp): s += '\n'.join([f" {line}\"".replace(':',': \"', 1) for line in l]) oFd = StringIO(s) - a = safe_load(oFd) + a = yaml.safe_load(oFd) return a def oMainArgparser(_=None): @@ -423,10 +414,10 @@ def oMainArgparser(_=None): help='proxy download connect timeout') parser.add_argument('--good_nodes', type=str, - default=os.path.join(ETC_DIR, 'goodnodes.yaml'), + default=os.path.join(ETC_DIR, 'torrc-goodnodes.yaml'), help="Yaml file of good info that should not be excluded") parser.add_argument('--bad_nodes', type=str, - default=os.path.join(ETC_DIR, 'badnodes.yaml'), + default=os.path.join(ETC_DIR, 'torrc-badnodes.yaml'), help="Yaml file of bad nodes that should also be excluded") parser.add_argument('--contact', type=str, default='Empty,NoEmail', help="comma sep list of conditions - Empty,NoEmail") @@ -446,13 +437,13 @@ def oMainArgparser(_=None): parser.add_argument('--bad_sections', type=str, default='MyBadExit', help="sections of the badnodes.yaml to use, comma separated, '' BROKEN") - parser.add_argument('--white_onions', type=str, + parser.add_argument('--white_services', type=str, default='', help="comma sep. list of onions to whitelist their introduction points - BROKEN") parser.add_argument('--torrc_output', type=str, default=os.path.join(ETC_DIR, 'torrc.new'), help="Write the torrc configuration to a file") - parser.add_argument('--good_contacts', type=str, default=os.path.join(ETC_DIR, 'goodcontacts.yaml'), + parser.add_argument('--proof_output', type=str, default=os.path.join(ETC_DIR, 'proof.yaml'), help="Write the proof data of the included nodes to a YAML file") return parser @@ -461,7 +452,7 @@ def vwrite_badnodes(oArgs, oBAD_NODES, slen): tmp = oArgs.bad_nodes +'.tmp' bak = oArgs.bad_nodes +'.bak' with open(tmp, 'wt') as oFYaml: - yaml.dump(oBAD_NODES, oFYaml) + yaml.dump(oBAD_NODES, indent=2, stream=oFYaml) LOG.info(f"Wrote {slen} to {oArgs.bad_nodes}") oFYaml.close() if os.path.exists(oArgs.bad_nodes): @@ -473,7 +464,7 @@ def vwrite_goodnodes(oArgs, oGOOD_NODES, ilen): tmp = oArgs.good_nodes +'.tmp' bak = oArgs.good_nodes +'.bak' with open(tmp, 'wt') as oFYaml: - yaml.dump(oGOOD_NODES, oFYaml) + yaml.dump(oGOOD_NODES, indent=2, stream=oFYaml) LOG.info(f"Wrote {ilen} good relays to {oArgs.good_nodes}") oFYaml.close() if os.path.exists(oArgs.good_nodes): @@ -498,11 +489,12 @@ def iMain(lArgs): icheck_torrc(sFile, oArgs) twhitelist_set = set() - sFile = oArgs.good_contacts + sFile = oArgs.proof_output if sFile and os.path.exists(sFile): try: with open(sFile, 'rt') as oFd: - aTRUST_DB = safe_load(oFd) + aTRUST_DB = yaml.safe_load(oFd) + assert type(aTRUST_DB) == dict LOG.info(f"{len(aTRUST_DB.keys())} trusted contacts from {sFile}") # reverse lookup of fps to contacts # but... @@ -528,8 +520,8 @@ def iMain(lArgs): vwait_for_controller(controller, oArgs.wait_boot) - if oArgs.good_contacts: - good_contacts_tmp = oArgs.good_contacts + '.tmp' + if oArgs.proof_output: + proof_output_tmp = oArgs.proof_output + '.tmp' elt = controller.get_conf('UseMicrodescriptors') if elt != '0' : @@ -549,17 +541,10 @@ def iMain(lArgs): t = set() if 'IntroductionPoints' in oGOOD_NODES[oGOOD_ROOT]['Relays'].keys(): t = set(oGOOD_NODES[oGOOD_ROOT]['Relays']['IntroductionPoints']) - w = set() - if 'Services' in oGOOD_NODES[oGOOD_ROOT].keys(): - # 'Onions' can I use the IntroductionPoints for Services too? - # w = set(oGOOD_NODES[oGOOD_ROOT]['Services']) - pass - if 'Onions' in oGOOD_NODES[oGOOD_ROOT].keys(): - # Provides the descriptor for a hidden service. The **address** is the - # '.onion' address of the hidden service - w = set(oGOOD_NODES[oGOOD_ROOT]['Onions']) - if oArgs.white_onions: - w.update(oArgs.white_onions.split(',')) + # not working = maybe when stem is updated + w = set(oGOOD_NODES[oGOOD_ROOT]['Services']) + if oArgs.white_services: + w.update(oArgs.white_services.split(',')) if oArgs.points_timeout > 0: LOG.info(f"{len(w)} services will be checked from IntroductionPoints") t.update(lIntroductionPoints(controller, w, itimeout=oArgs.points_timeout)) @@ -712,10 +697,10 @@ def iMain(lArgs): aTRUST_DB[relay.fingerprint] = b for elt in b['fps']: aTRUST_DB_INDEX[elt] = b - if oArgs.good_contacts and oArgs.log_level <= 20: + if oArgs.proof_output and oArgs.log_level <= 20: # as we go along then clobber - with open(good_contacts_tmp, 'wt') as oFYaml: - yaml.dump(aTRUST_DB, oFYaml) + with open(proof_output_tmp, 'wt') as oFYaml: + yaml.dump(aTRUST_DB, indent=2, stream=oFYaml) oFYaml.close() LOG.info(f"Filtered {len(twhitelist_set)} whitelisted relays") @@ -724,6 +709,16 @@ def iMain(lArgs): texclude_set = texclude_set.difference(tdns_urls) LOG.info(f"{len(list(aTRUST_DB.keys()))} good contacts out of {iTotalContacts}") + if oArgs.proof_output and aTRUST_DB: + with open(proof_output_tmp, 'wt') as oFYaml: + yaml.dump(aTRUST_DB, indent=2, stream=oFYaml) + oFYaml.close() + if os.path.exists(oArgs.proof_output): + bak = oArgs.proof_output +'.bak' + os.rename(oArgs.proof_output, bak) + os.rename(proof_output_tmp, oArgs.proof_output) + LOG.info(f"Wrote {len(list(aTRUST_DB.keys()))} good contact details to {oArgs.proof_output}") + if oArgs.torrc_output and texclude_set: with open(oArgs.torrc_output, 'wt') as oFTorrc: oFTorrc.write(f"{sEXCLUDE_EXIT_KEY} {','.join(texclude_set)}\n") @@ -735,19 +730,9 @@ def iMain(lArgs): if oArgs.bad_contacts and aBadContacts: # for later analysis with open(oArgs.bad_contacts, 'wt') as oFYaml: - yaml.dump(aBadContacts, oFYaml) + yaml.dump(aBadContacts, indent=2, stream=oFYaml) oFYaml.close() - if oArgs.good_contacts != '' and aTRUST_DB: - with open(good_contacts_tmp, 'wt') as oFYaml: - yaml.dump(aTRUST_DB, oFYaml) - oFYaml.close() - if os.path.exists(oArgs.good_contacts): - bak = oArgs.good_contacts +'.bak' - os.rename(oArgs.good_contacts, bak) - os.rename(good_contacts_tmp, oArgs.good_contacts) - LOG.info(f"Wrote {len(list(aTRUST_DB.keys()))} good contact details to {oArgs.good_contacts}") - oBAD_NODES[oBAD_ROOT]['ExcludeNodes']['BadExit'] = list(texclude_set) oBAD_NODES[oBAD_ROOT]['ExcludeDomains'] = lKNOWN_NODNS vwrite_badnodes(oArgs, oBAD_NODES, str(len(texclude_set))) @@ -755,7 +740,6 @@ def iMain(lArgs): oGOOD_NODES['GoodNodes']['Relays']['ExitNodes'] = list(aTRUST_DB_INDEX.keys()) # GuardNodes are readonl vwrite_goodnodes(oArgs, oGOOD_NODES, len(aTRUST_DB_INDEX.keys())) - retval = 0 try: logging.getLogger('stem').setLevel(30) @@ -792,6 +776,9 @@ def iMain(lArgs): LOG.errro(f"Failed setting {sINCLUDE_EXIT_KEY} good exit nodes in Tor") retval += 1 + sys.stdout.write("dns-rsa domains:\n" +'\n'.join(tdns_urls) +'\n') + return retval + except InvalidRequest as e: # Unacceptable option value: Invalid router list. LOG.error(str(e)) @@ -815,9 +802,6 @@ def iMain(lArgs): controller.close() except Exception as e: LOG.warn(str(e)) - - sys.stdout.write("dns-rsa domains:\n" +'\n'.join(tdns_urls) +'\n') - return retval if __name__ == '__main__': try: diff --git a/support_onions.py b/support_onions.py index d42feba..68d3ee0 100644 --- a/support_onions.py +++ b/support_onions.py @@ -14,12 +14,10 @@ if False: import cepa as stem from cepa.control import Controller from cepa.connection import MissingPassword - from cepa.util.tor_tools import is_valid_fingerprint else: import stem from stem.control import Controller from stem.connection import MissingPassword - from stem.util.tor_tools import is_valid_fingerprint global LOG import logging @@ -98,7 +96,7 @@ def oGetStemController(log_level=10, sock_or_pair='/run/tor/control'): controller.authenticate(p) oSTEM_CONTROLER = controller LOG.debug(f"{controller}") - return oSTEM_CONTROLER + return oSTEM_CONTROLER def bAreWeConnected(): # FixMe: Linux only @@ -146,11 +144,7 @@ def bin_to_hex(raw_id, length=None): return res.upper() def lIntroductionPoints(controller=None, lOnions=[], itimeout=120, log_level=10): - """now working !!! stem 1.8.x timeout must be huge >120 - 'Provides the descriptor for a hidden service. The **address** is the - '.onion' address of the hidden service ' - What about Services? - """ + """now working !!! stem 1.8.x timeout must be huge >120""" try: from cryptography.utils import int_from_bytes except ImportError: @@ -160,12 +154,12 @@ def lIntroductionPoints(controller=None, lOnions=[], itimeout=120, log_level=10) # this will fai if the trick above didnt work from stem.prereq import is_crypto_available is_crypto_available(ed25519 = True) - + from stem.descriptor.hidden_service import HiddenServiceDescriptorV3 from stem.client.datatype import LinkByFingerprint from stem import Timeout from queue import Empty - + if type(lOnions) not in [set, tuple, list]: lOnions = list(lOnions) if controller is None: @@ -283,7 +277,7 @@ def sTorResolve(target, LOG.info(sLabel +f"{i} on {sHost}:{iPort}" ) sock.close() raise SystemExit(5) - + assert len(data) >= 8 packet_sf = data[1] if packet_sf == 90: @@ -298,7 +292,7 @@ def sTorResolve(target, # os.system("strace tor-resolve -4 "+target+" 2>&1|grep '^sen\|^rec'") return '' - + def getaddrinfo(sHost, sPort): # do this the explicit way = Ive seen the compact connect fail # >>> sHost, sPort = 'l27.0.0.1', 33446 @@ -399,4 +393,4 @@ def lExitExcluder(oArgs, iPort=9051, log_level=10): if __name__ == '__main__': target = 'duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad' controller = oGetStemController(log_level=10) - lIntroductionPoints(controller, [target], itimeout=120) + lIntroductionPoints(controller, [target], itimeout=120) diff --git a/trustor_poc.py b/trustor_poc.py index 4ddc6c1..0f9406a 100644 --- a/trustor_poc.py +++ b/trustor_poc.py @@ -355,7 +355,7 @@ def _my_match_hostname(cert, asserted_hostname): try: my_match_hostname(cert, asserted_hostname) except CertificateError as e: - LOG.warning( + log.warning( "Certificate did not match hostname: %s. Certificate: %s", asserted_hostname, cert,