add fetch method

Author: aagbsn

bwscanner/fetcher.py

@@ -1,16 +1,18 @@
 import warnings
 import hashlib
 
-from twisted.internet import interfaces, reactor, defer, protocol
+from twisted.internet import reactor, defer, protocol
 from twisted.internet.endpoints import TCP4ClientEndpoint
-from twisted.web.client import (SchemeNotSupported, Agent, BrowserLikePolicyForHTTPS,
-                                ResponseDone, PotentialDataLoss, PartialDownloadError)
-from txsocksx.client import SOCKS5ClientFactory
-from txsocksx.tls import TLSWrapClientEndpoint
-from zope.interface import implementer
+from twisted.web.client import (ResponseDone, PotentialDataLoss, PartialDownloadError)
 
 from bwscanner.logger import log
 
+def fetch(tor_state, path, url):
+    d = tor_state.build_circuit(path, False)
+    sport = get_tor_socks_endpoint(tor_state)
+    d.addCallback(lambda c: c.when_built())
+    d.addCallback(lambda c: c.web_agent(reactor, sport))
+    return d.addCallback(lambda a: a.request("GET", url))
 
 def get_tor_socks_endpoint(tor_state):
     proxy_endpoint = tor_state.protocol.get_conf("SocksPort")
@@ -31,78 +33,6 @@ def extract_port_value(result):
     return proxy_endpoint
 
 
-@implementer(interfaces.IStreamClientEndpoint)
-class OnionRoutedTCPClientEndpoint(object):
-    def __init__(self, host, port, state, path):
-        """
-        @param reactor: An L{IReactorTCP} provider
-
-        @param host: A hostname, used when connecting
-        @type host: str
-
-        @param port: The port number, used when connecting
-        @type port: int
-
-        @param path: A list of relay identities.
-        @type path: list
-
-        This endpoint will be routed through Tor over a circuit
-        defined by path.
-        """
-        self.host = host
-        self.port = port
-        self.path = path
-        self.state = state
-
-        self.tor_socks_endpoint = get_tor_socks_endpoint(state)
-
-    def connect(self, protocol_factory):
-        """
-        Implements L{IStreamClientEndpoint.connect} to connect via TCP, after
-        SOCKS5 negotiation and Tor circuit construction is done.
-        """
-        proxy_factory = SOCKS5ClientFactory(self.host, self.port, protocol_factory)
-        self.tor_socks_endpoint.addCallback(lambda end: end.connect(proxy_factory))
-
-        def _create_circ(proto):
-            hp = proto.transport.getHost()
-            d = self.state._attacher.create_circuit(hp.host, hp.port, self.path)
-            d.addErrback(proxy_factory.deferred.errback)
-            return proxy_factory.deferred
-
-        return self.tor_socks_endpoint.addCallback(_create_circ)
-
-
-class OnionRoutedAgent(Agent):
-    _tlsWrapper = TLSWrapClientEndpoint
-    _policyForHTTPS = BrowserLikePolicyForHTTPS
-
-    def __init__(self, *args, **kw):
-        self.path = kw.pop('path')
-        self.state = kw.pop('state')
-        super(OnionRoutedAgent, self).__init__(*args, **kw)
-
-    def _getEndpoint(self, parsedURI, host=None, port=None):
-        try:
-            host, port = parsedURI.host, parsedURI.port
-            scheme = parsedURI.scheme
-        except AttributeError:
-            scheme = parsedURI
-        if scheme not in ('http', 'https'):
-            raise SchemeNotSupported('unsupported scheme', scheme)
-        endpoint = OnionRoutedTCPClientEndpoint(host, port, self.state,
-                                                self.path)
-        if scheme == 'https':
-            if hasattr(self, '_wrapContextFactory'):
-                tls_policy = self._wrapContextFactory(host, port)
-            elif hasattr(self, '_policyForHTTPS'):
-                tls_policy = self._policyForHTTPS().creatorForNetloc(host, port)
-            else:
-                raise NotImplementedError("Cannot create a TLS validation policy.")
-            endpoint = self._tlsWrapper(tls_policy, endpoint)
-        return endpoint
-
-
 class hashingReadBodyProtocol(protocol.Protocol):
     """
     Protocol that collects data sent to it and hashes it.

bwscanner/measurement.py

@@ -7,8 +7,7 @@
 
 from bwscanner.logger import log
 from bwscanner.circuit import TwoHop
-from bwscanner.fetcher import OnionRoutedAgent
-from bwscanner.fetcher import get_tor_socks_endpoint, hashingReadBody
+from bwscanner.fetcher import get_tor_socks_endpoint, hashingReadBody, fetch
 from bwscanner.writer import ResultSink
 
 # defer.setDebugging(True)
@@ -160,10 +159,7 @@ def gotResult(result):
                 return result
             deferred.addBoth(gotResult)
 
-        d = self.state.build_circuit(path, False)
-        d.addCallback(lambda c: c.when_built())
-        d.addCallback(lambda c: c.web_agent(self.clock, self.socks))
-        d.addCallback(lambda a: a.request("GET", url))
+        d = fetch(self.state, path, url)
         d.addCallback(hashingReadBody)
         timeoutDeferred(d, self.request_timeout)
         d.addCallbacks(get_circuit_bw)

scripts/exitip.py

@@ -2,9 +2,7 @@
 import json
 import re
 import sys
-from bwscanner.attacher import SOCKSClientStreamAttacher, start_tor
-from bwscanner.circuit import ExitScan
-from bwscanner.fetcher import OnionRoutedAgent
+from bwscanner.attacher import start_tor
 from twisted.internet import defer, reactor, task
 from twisted.python import log
 from twisted.web.client import readBody
@@ -57,20 +55,20 @@ def pop(circuits):
 def shutdown(ignore):
     reactor.stop()
 
-def add_attacher(state):
-    state.set_attacher(SOCKSClientStreamAttacher(state), reactor)
-    return state
-
 def setup_failed(failure):
     log.err(failure)
 
 def save_results(result, outfile):
     outfile.write(json.dumps(dict([r[1] for r in result if r[1] != None])))
 
+@defer.inlineCallbacks
+def socks(state):
+    s = yield get_tor_socks_endpoint(state)
+    defer.returnValue(s)
+
 def main():
     log.startLogging(sys.stdout)
     tor = start_tor(TorConfig())
-    tor.addCallback(add_attacher)
     tor.addCallback(run_scan)
     tor.addErrback(log.err)
     tor.addBoth(shutdown)

test/test_listener.py

@@ -7,7 +7,7 @@
 from txtorcon.util import available_tcp_port
 
 from bwscanner.listener import CircuitEventListener, StreamBandwidthListener
-from bwscanner.fetcher import OnionRoutedAgent
+from bwscanner.fetcher import fetch
 from test.template import TorTestCase
 
 import random
@@ -133,9 +133,8 @@ def test_circ_avg_bw(self):
     def do_fetch(self):
         time_start = time.time()
         path = self.random_path()
-        agent = OnionRoutedAgent(reactor, path=path, state=self.tor_state)
         url = "http://127.0.0.1:{}".format(self.port)
-        request = yield agent.request("GET", url)
+        request = yield fetch(self.tor_state, path, url)
         body = yield readBody(request)
         assert len(body) == self.fetch_size
         circ = [c for c in self.tor_state.circuits.values() if c.path == path][0]