Skip to content

Instantly share code, notes, and snippets.

@jesstess

jesstess/alexa-top-10000.txt

Created November 25, 2011 04:37
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save jesstess/1392819 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save jesstess/1392819 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. jesstess created this gist Nov 25, 2011.
    10,000 changes: 10,000 additions & 0 deletions alexa-top-10000.txt
    View file
    10,000 additions, 0 deletions not shown because the diff is too large. Please use a local Git client to view these changes.
    42 changes: 42 additions & 0 deletions http_method_test.py
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,42 @@
    import httplib
    import sys

    from multiprocessing import Pool, Lock

    # To see a frequency count for responses:
    # cut -d' ' --complement -f1 outfile | sort | uniq -c

    URL_FILE = "alexa-top-10000.txt"

    if len(sys.argv) != 4:
    print >>sys.stderr, "Please provide an HTTP method, argument, and output filename, e.g."
    print >>sys.stderr, "OPTIONS \* options.txt"
    print >>sys.stderr, "TRACE \* trace.txt"
    print >>sys.stderr, "CONNECT mx1.example.com:25 connect.txt"
    sys.exit(1)

    method, argument, outfile = sys.argv[1:]
    print outfile

    urls = [url.strip() for url in file(URL_FILE).readlines()]

    f = open(outfile, "w")
    lock = Lock()

    def make_request(url):
    conn = httplib.HTTPConnection(url, timeout=5)
    try:
    conn.request(method, argument)
    r1 = conn.getresponse()
    status = "%s %s" % (r1.status, r1.reason)
    except Exception, e:
    status = "Error: %s %s" % (e.__class__.__name__, e.message)

    lock.acquire()
    f.write("%s %s\n" % (url, status,))
    f.flush()
    lock.release()
    conn.close()

    pool = Pool(processes=25)
    pool.map(make_request, urls)