add exclude_badExits.bash

README.md

@@ -71,3 +71,79 @@ You can expect it to take an hour or two the first time this is run:
 >700 domains.
 
 For usage, do ```python3 exclude_badExits.py --help`
+
+## Usage
+```
+usage: exclude_badExits.py [-h] [--https_cafile HTTPS_CAFILE]
+                           [--proxy_host PROXY_HOST] [--proxy_port PROXY_PORT]
+                           [--proxy_ctl PROXY_CTL] [--torrc TORRC]
+                           [--timeout TIMEOUT] [--good_nodes GOOD_NODES]
+                           [--bad_nodes BAD_NODES] [--contact CONTACT]
+                           [--bad_contacts BAD_CONTACTS]
+                           [--strict_nodes {0,1}] [--wait_boot WAIT_BOOT]
+                           [--points_timeout POINTS_TIMEOUT]
+                           [--log_level LOG_LEVEL]
+                           [--bad_sections BAD_SECTIONS]
+                           [--white_services WHITE_SERVICES]
+                           [--torrc_output TORRC_OUTPUT]
+                           [--proof_output PROOF_OUTPUT]
+```
+
+### Optional arguments:
+
+```
+  -h, --help            show this help message and exit
+  --https_cafile HTTPS_CAFILE
+                        Certificate Authority file (in PEM)
+```
+```
+  --proxy_host PROXY_HOST, --proxy-host PROXY_HOST
+                        proxy host
+  --proxy_port PROXY_PORT, --proxy-port PROXY_PORT
+                        proxy control port
+  --proxy_ctl PROXY_CTL, --proxy-ctl PROXY_CTL
+                        control socket - or port
+```
+```
+  --torrc TORRC         torrc to check for suggestions
+  --timeout TIMEOUT     proxy download connect timeout
+```
+```
+  --good_nodes GOOD_NODES
+                        Yaml file of good info that should not be excluded
+  --bad_nodes BAD_NODES
+                        Yaml file of bad nodes that should also be excluded
+```
+```
+  --contact CONTACT     comma sep list of conditions - Empty,NoEmail
+  --bad_contacts BAD_CONTACTS
+                        Yaml file of bad contacts that bad FPs are using
+```
+```
+  --strict_nodes {0,1}  Set StrictNodes: 1 is less anonymous but more secure,
+                        although some sites may be unreachable
+  --wait_boot WAIT_BOOT
+                        Seconds to wait for Tor to booststrap
+  --points_timeout POINTS_TIMEOUT
+                        Timeout for getting introduction points - must be long
+                        >120sec. 0 means disabled looking for IPs
+```
+```
+  --log_level LOG_LEVEL
+                        10=debug 20=info 30=warn 40=error
+  --bad_sections BAD_SECTIONS
+                        sections of the badnodes.yaml to use, comma separated,
+                        '' BROKEN
+```
+```
+  --white_services WHITE_SERVICES
+                        comma sep. list of onions to whitelist their
+                        introduction points - BROKEN
+```
+```
+  --torrc_output TORRC_OUTPUT
+                        Write the torrc configuration to a file
+  --proof_output PROOF_OUTPUT
+                        Write the proof data of the included nodes to a YAML
+                        file
+```

exclude_badExits.bash

@@ -0,0 +1,41 @@
+#!/bin/bash
+# -*- mode: sh; fill-column: 75; tab-width: 8; coding: utf-8-unix -*-
+
+PROG=exclude_badExits.py
+SOCKS_PORT=9050
+CAFILE=/etc/ssl/certs/ca-certificates.crt
+
+# an example of running exclude_badExits with full debugging
+# expected to take an hour or so
+declare -a LARGS
+LARGS=(
+    --log_level 10
+    )
+# you may have a special python for installed packages
+EXE=`which python3.bash`
+LARGS+=(
+    --strict_nodes 0
+    --points_timeout 120
+    --proxy-host 127.0.0.1
+    --proxy-port $SOCKS_PORT
+    --https_cafile $CAFILE
+)
+
+if [ -f '/run/tor/control' ] ; then
+    LARGS+=(--proxy-ctl '/run/tor/control' )
+else
+    LARGS+=(--proxy-ctl 9051 )
+fi
+
+ddg=duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad
+# for example, whitelist the introduction points to DuckDuckGo
+LARGS+=( --white_onions $ddg )
+
+# you may need to be the tor user to read /run/tor/control
+grep -q ^debian-tor /etc/group && TORU=debian-tor || {
+    grep -q ^tor /etc/group && TORU=tor
+}
+sudo -u $TORU $EXE exclude_badExits.py "${LARGS[@]}" \
+   2>&1|tee exclude_badExits6.log
+
+# The DEBUG statements contain the detail of why the relay was considered bad.

exclude_badExits.py

@@ -48,7 +48,7 @@ exclusion: the ```--contact``` commandline arg is a comma sep list of conditions
 More may be added later.
 
 Because you don't want to exclude the introduction points to any onion
-you want to connect to, ```--white_services``` should whitelist the
+you want to connect to, ```--white_onions``` should whitelist the
 introduction points to a  comma sep list of onions, but is
 currently broken in stem 1.8.0: see:
 * https://github.com/torproject/stem/issues/96
@@ -65,7 +65,7 @@ not just exclude Exit.
 If the Contact info is good we add the list of fingerprints to add
 to ExitNodes, a whitelist of relays to use as exits.
 
-```--proof_output``` will write the contact info as a ciiss dictionary
+```--good_contacts``` will write the contact info as a ciiss dictionary
 to a YAML file. If the proof is uri-rsa, the well-known file of fingerprints
 is downloaded and the fingerprints are added on a 'fps' field we create
 of that fingerprint's entry of the YAML dictionary. This file is read at the
@@ -88,7 +88,7 @@ import time
 import argparse
 import string
 from io import StringIO
-import ipaddr
+import ipaddress
 # list(ipaddress._find_address_range(ipaddress.IPv4Network('172.16.0.0/12'))
 
 from urllib3.util.ssl_match_hostname import CertificateError
@@ -98,15 +98,24 @@ from stem.control import Controller
 from stem.connection import IncorrectPassword
 from stem.util.tor_tools import is_valid_fingerprint
 try:
-    import yaml
+    from ruamel.yaml import YAML
+    yaml = YAML(typ='rt')
+    yaml.indent(mapping=2, sequence=2)
+    safe_load = yaml.load
 except:
     yaml = None
+if yaml is None:
+    try:
+        import yaml
+        safe_load = yaml.safe_load
+    except:
+        yaml = None
+
 try:
     from unbound import ub_ctx,RR_TYPE_TXT,RR_CLASS_IN
 except:
     ub_ctx = RR_TYPE_TXT = RR_CLASS_IN = None
 
-
 global LOG
 import logging
 import warnings
@@ -158,7 +167,7 @@ def lYamlBadNodes(sFile,
     if not yaml: return l
     if os.path.exists(sFile):
         with open(sFile, 'rt') as oFd:
-            oBAD_NODES = yaml.safe_load(oFd)
+            oBAD_NODES = safe_load(oFd)
 
     # BROKEN
 #    root = 'ExcludeNodes'
@@ -170,7 +179,7 @@ def lYamlBadNodes(sFile,
 
     root = 'ExcludeDomains'
     if root not in oBAD_NODES[oBAD_ROOT] or not oBAD_NODES[oBAD_ROOT][root]:
-        lMAYBE_NODNS = yaml.safe_load(StringIO(yKNOWN_NODNS))
+        lMAYBE_NODNS = safe_load(StringIO(yKNOWN_NODNS))
     else:
         lMAYBE_NODNS = oBAD_NODES[oBAD_ROOT][root]
     return l
@@ -184,7 +193,7 @@ def lYamlGoodNodes(sFile='/etc/tor/torrc-goodnodes.yaml'):
     if not yaml: return l
     if os.path.exists(sFile):
         with open(sFile, 'rt') as oFd:
-            o = yaml.safe_load(oFd)
+            o = safe_load(oFd)
             oGOOD_NODES = o
         if 'GuardNodes' in o[oGOOD_ROOT].keys():
             l = o[oGOOD_ROOT]['GuardNodes']
@@ -271,7 +280,7 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
         LOG.warn(f"{domain} is bad from {a['url']}")
         LOG.debug(f"{fp} is bad from {a}")
         return a
-    
+
     ip = zResolveDomain(domain)
     if ip == '':
         aFP_EMAIL[fp] = a['email']
@@ -373,7 +382,7 @@ def aParseContact(contact, fp):
     s += '\n'.join([f"  {line}\"".replace(':',': \"', 1)
                     for line in l])
     oFd = StringIO(s)
-    a = yaml.safe_load(oFd)
+    a = safe_load(oFd)
     return a
 
 def oMainArgparser(_=None):
@@ -414,10 +423,10 @@ def oMainArgparser(_=None):
                         help='proxy download connect timeout')
 
     parser.add_argument('--good_nodes', type=str,
-                        default=os.path.join(ETC_DIR, 'torrc-goodnodes.yaml'),
+                        default=os.path.join(ETC_DIR, 'goodnodes.yaml'),
                         help="Yaml file of good info that should not be excluded")
     parser.add_argument('--bad_nodes', type=str,
-                        default=os.path.join(ETC_DIR, 'torrc-badnodes.yaml'),
+                        default=os.path.join(ETC_DIR, 'badnodes.yaml'),
                         help="Yaml file of bad nodes that should also be excluded")
     parser.add_argument('--contact', type=str, default='Empty,NoEmail',
                         help="comma sep list of conditions - Empty,NoEmail")
@@ -437,13 +446,13 @@ def oMainArgparser(_=None):
     parser.add_argument('--bad_sections', type=str,
                         default='MyBadExit',
                         help="sections of the badnodes.yaml to use, comma separated, '' BROKEN")
-    parser.add_argument('--white_services', type=str,
+    parser.add_argument('--white_onions', type=str,
                         default='',
                         help="comma sep. list of onions to whitelist their introduction points - BROKEN")
     parser.add_argument('--torrc_output', type=str,
                         default=os.path.join(ETC_DIR, 'torrc.new'),
                         help="Write the torrc configuration to a file")
-    parser.add_argument('--proof_output', type=str, default=os.path.join(ETC_DIR, 'proof.yaml'),
+    parser.add_argument('--good_contacts', type=str, default=os.path.join(ETC_DIR, 'goodcontacts.yaml'),
                         help="Write the proof data of the included nodes to a YAML file")
     return parser
 
@@ -452,7 +461,7 @@ def vwrite_badnodes(oArgs, oBAD_NODES, slen):
         tmp = oArgs.bad_nodes +'.tmp'
         bak = oArgs.bad_nodes +'.bak'
         with open(tmp, 'wt') as oFYaml:
-            yaml.dump(oBAD_NODES, indent=2, stream=oFYaml)
+            yaml.dump(oBAD_NODES, oFYaml)
             LOG.info(f"Wrote {slen}  to {oArgs.bad_nodes}")
             oFYaml.close()
         if os.path.exists(oArgs.bad_nodes):
@@ -464,7 +473,7 @@ def vwrite_goodnodes(oArgs, oGOOD_NODES, ilen):
         tmp = oArgs.good_nodes +'.tmp'
         bak = oArgs.good_nodes +'.bak'
         with open(tmp, 'wt') as oFYaml:
-            yaml.dump(oGOOD_NODES, indent=2, stream=oFYaml)
+            yaml.dump(oGOOD_NODES, oFYaml)
             LOG.info(f"Wrote {ilen} good relays to {oArgs.good_nodes}")
             oFYaml.close()
         if os.path.exists(oArgs.good_nodes):
@@ -489,12 +498,11 @@ def iMain(lArgs):
         icheck_torrc(sFile, oArgs)
 
     twhitelist_set = set()
-    sFile = oArgs.proof_output
+    sFile = oArgs.good_contacts
     if sFile and os.path.exists(sFile):
         try:
             with open(sFile, 'rt') as oFd:
-                aTRUST_DB = yaml.safe_load(oFd)
+                aTRUST_DB = safe_load(oFd)
-            assert type(aTRUST_DB) == dict
             LOG.info(f"{len(aTRUST_DB.keys())} trusted contacts from {sFile}")
             # reverse lookup of fps to contacts
             # but...
@@ -520,8 +528,8 @@ def iMain(lArgs):
 
     vwait_for_controller(controller, oArgs.wait_boot)
 
-    if oArgs.proof_output:
+    if oArgs.good_contacts:
-        proof_output_tmp = oArgs.proof_output + '.tmp'
+        good_contacts_tmp = oArgs.good_contacts + '.tmp'
 
     elt = controller.get_conf('UseMicrodescriptors')
     if elt != '0' :
@@ -541,10 +549,17 @@ def iMain(lArgs):
     t = set()
     if 'IntroductionPoints' in oGOOD_NODES[oGOOD_ROOT]['Relays'].keys():
         t = set(oGOOD_NODES[oGOOD_ROOT]['Relays']['IntroductionPoints'])
-    # not working = maybe when stem is updated
+    w = set()
-    w = set(oGOOD_NODES[oGOOD_ROOT]['Services'])
+    if 'Services' in oGOOD_NODES[oGOOD_ROOT].keys():
-    if oArgs.white_services:
+        # 'Onions' can I use the IntroductionPoints for Services too?
-        w.update(oArgs.white_services.split(','))
+        # w = set(oGOOD_NODES[oGOOD_ROOT]['Services'])
+        pass
+    if 'Onions' in oGOOD_NODES[oGOOD_ROOT].keys():
+        # Provides the descriptor for a hidden service. The **address** is the
+        # '.onion' address of the hidden service 
+        w = set(oGOOD_NODES[oGOOD_ROOT]['Onions'])
+    if oArgs.white_onions:
+        w.update(oArgs.white_onions.split(','))
     if oArgs.points_timeout > 0:
         LOG.info(f"{len(w)} services will be checked from IntroductionPoints")
         t.update(lIntroductionPoints(controller, w, itimeout=oArgs.points_timeout))
@@ -697,10 +712,10 @@ def iMain(lArgs):
             aTRUST_DB[relay.fingerprint] = b
             for elt in b['fps']:
                 aTRUST_DB_INDEX[elt] = b
-            if oArgs.proof_output and oArgs.log_level <= 20:
+            if oArgs.good_contacts and oArgs.log_level <= 20:
                 # as we go along then clobber
-                with open(proof_output_tmp, 'wt') as oFYaml:
+                with open(good_contacts_tmp, 'wt') as oFYaml:
-                    yaml.dump(aTRUST_DB, indent=2, stream=oFYaml)
+                    yaml.dump(aTRUST_DB, oFYaml)
                     oFYaml.close()
 
     LOG.info(f"Filtered {len(twhitelist_set)} whitelisted relays")
@@ -709,16 +724,6 @@ def iMain(lArgs):
     texclude_set = texclude_set.difference(tdns_urls)
     LOG.info(f"{len(list(aTRUST_DB.keys()))} good contacts out of {iTotalContacts}")
 
-    if oArgs.proof_output and aTRUST_DB:
-        with open(proof_output_tmp, 'wt') as oFYaml:
-            yaml.dump(aTRUST_DB, indent=2, stream=oFYaml)
-            oFYaml.close()
-        if os.path.exists(oArgs.proof_output):
-            bak = oArgs.proof_output +'.bak'
-            os.rename(oArgs.proof_output, bak)
-        os.rename(proof_output_tmp, oArgs.proof_output)
-        LOG.info(f"Wrote {len(list(aTRUST_DB.keys()))} good contact details to {oArgs.proof_output}")
-
     if oArgs.torrc_output and texclude_set:
         with open(oArgs.torrc_output, 'wt') as oFTorrc:
             oFTorrc.write(f"{sEXCLUDE_EXIT_KEY} {','.join(texclude_set)}\n")
@@ -730,9 +735,19 @@ def iMain(lArgs):
     if oArgs.bad_contacts and aBadContacts:
         # for later analysis
         with open(oArgs.bad_contacts, 'wt') as oFYaml:
-            yaml.dump(aBadContacts, indent=2, stream=oFYaml)
+            yaml.dump(aBadContacts, oFYaml)
             oFYaml.close()
 
+    if oArgs.good_contacts != '' and aTRUST_DB:
+        with open(good_contacts_tmp, 'wt') as oFYaml:
+            yaml.dump(aTRUST_DB, oFYaml)
+            oFYaml.close()
+        if os.path.exists(oArgs.good_contacts):
+            bak = oArgs.good_contacts +'.bak'
+            os.rename(oArgs.good_contacts, bak)
+        os.rename(good_contacts_tmp, oArgs.good_contacts)
+        LOG.info(f"Wrote {len(list(aTRUST_DB.keys()))} good contact details to {oArgs.good_contacts}")
+
     oBAD_NODES[oBAD_ROOT]['ExcludeNodes']['BadExit'] = list(texclude_set)
     oBAD_NODES[oBAD_ROOT]['ExcludeDomains'] = lKNOWN_NODNS
     vwrite_badnodes(oArgs, oBAD_NODES, str(len(texclude_set)))
@@ -740,6 +755,7 @@ def iMain(lArgs):
     oGOOD_NODES['GoodNodes']['Relays']['ExitNodes'] = list(aTRUST_DB_INDEX.keys())
     # GuardNodes are readonl
     vwrite_goodnodes(oArgs, oGOOD_NODES, len(aTRUST_DB_INDEX.keys()))
+    
     retval = 0
     try:
         logging.getLogger('stem').setLevel(30)
@@ -776,9 +792,6 @@ def iMain(lArgs):
             LOG.errro(f"Failed setting {sINCLUDE_EXIT_KEY} good exit nodes in Tor")
             retval += 1
 
-        sys.stdout.write("dns-rsa domains:\n" +'\n'.join(tdns_urls) +'\n')
-        return retval
-
     except InvalidRequest as e:
         # Unacceptable option value: Invalid router list.
         LOG.error(str(e))
@@ -802,6 +815,9 @@ def iMain(lArgs):
             controller.close()
         except Exception as e:
             LOG.warn(str(e))
+            
+    sys.stdout.write("dns-rsa domains:\n" +'\n'.join(tdns_urls) +'\n')
+    return retval
 
 if __name__ == '__main__':
     try:

support_onions.py

@@ -14,10 +14,12 @@ if False:
     import cepa as stem
     from cepa.control import Controller
     from cepa.connection import MissingPassword
+    from cepa.util.tor_tools import is_valid_fingerprint
 else:
     import stem
     from stem.control import Controller
     from stem.connection import MissingPassword
+    from stem.util.tor_tools import is_valid_fingerprint
 
 global LOG
 import logging
@@ -96,7 +98,7 @@ def oGetStemController(log_level=10, sock_or_pair='/run/tor/control'):
         controller.authenticate(p)
     oSTEM_CONTROLER = controller
     LOG.debug(f"{controller}")
-    return oSTEM_CONTROLER 
+    return oSTEM_CONTROLER
 
 def bAreWeConnected():
     # FixMe: Linux only
@@ -144,7 +146,11 @@ def bin_to_hex(raw_id, length=None):
     return res.upper()
 
 def lIntroductionPoints(controller=None, lOnions=[], itimeout=120, log_level=10):
-    """now working !!! stem 1.8.x timeout must be huge >120"""
+    """now working !!! stem 1.8.x timeout must be huge >120
+    'Provides the descriptor for a hidden service. The **address** is the
+    '.onion' address of the hidden service '
+    What about Services?
+    """
     try:
         from cryptography.utils import int_from_bytes
     except ImportError:
@@ -154,12 +160,12 @@ def lIntroductionPoints(controller=None, lOnions=[], itimeout=120, log_level=10)
     # this will fai if the trick above didnt work
     from stem.prereq import is_crypto_available
     is_crypto_available(ed25519 = True)
-    
+
     from stem.descriptor.hidden_service import HiddenServiceDescriptorV3
     from stem.client.datatype import LinkByFingerprint
     from stem import Timeout
     from queue import Empty
-    
+
     if type(lOnions) not in [set, tuple, list]:
         lOnions = list(lOnions)
     if controller is None:
@@ -277,7 +283,7 @@ def sTorResolve(target,
         LOG.info(sLabel +f"{i} on {sHost}:{iPort}" )
         sock.close()
         raise SystemExit(5)
-    
+
     assert len(data) >= 8
     packet_sf = data[1]
     if packet_sf == 90:
@@ -292,7 +298,7 @@ def sTorResolve(target,
 #    os.system("strace tor-resolve -4 "+target+" 2>&1|grep '^sen\|^rec'")
 
     return ''
-    
+
 def getaddrinfo(sHost, sPort):
     # do this the explicit way = Ive seen the compact connect fail
     # >>> sHost, sPort = 'l27.0.0.1', 33446
@@ -393,4 +399,4 @@ def lExitExcluder(oArgs, iPort=9051, log_level=10):
 if __name__ == '__main__':
     target = 'duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad'
     controller = oGetStemController(log_level=10)
-    lIntroductionPoints(controller, [target], itimeout=120)    
+    lIntroductionPoints(controller, [target], itimeout=120)

trustor_poc.py

@@ -355,7 +355,7 @@ def _my_match_hostname(cert, asserted_hostname):
     try:
         my_match_hostname(cert, asserted_hostname)
     except CertificateError as e:
-        log.warning(
+        LOG.warning(
             "Certificate did not match hostname: %s. Certificate: %s",
             asserted_hostname,
             cert,