Chirayu Patel chirayu

Golang Workers / Job Queue

A running example of the code from:

This gist creates a working example from blog post, and a alternate example using simple worker pool.

TLDR: if you want simple and controlled concurrency use a worker pool.

Using Python's built-in defaultdict we can easily define a tree data structure:

def tree(): return defaultdict(tree)

That's it!

	local usbWatcher = nil

	-- This is our usbWatcher function
	-- lock when yubikey is removed
	function usbDeviceCallback(data)
	-- this line will let you know the name of each usb device you connect, useful for the string match below
	hs.notify.show("USB", "You just connected", data["productName"])
	-- Replace "Yubikey" with the name of the usb device you want to use.
	if string.match(data["productName"], "Yubikey") then
	if (data["eventType"] == "added") then

	{
	"Envelope" : {
	"WARC-Header-Length" : "578",
	"Block-Digest" : "sha1:YHKQUSBOS4CLYFEKQDVGJ457OAPD6IJO",
	"Format" : "WARC",
	"Actual-Content-Length" : "43428",
	"WARC-Header-Metadata" : {
	"WARC-Record-ID" : "<urn:uuid:ffbfb0c0-6456-42b0-af03-3867be6fc09f>",
	"WARC-Warcinfo-ID" : "<urn:uuid:3169ca8e-39a6-42e9-a4e3-9f001f067bdf>",
	"Content-Length" : "43428",

	#!/usr/bin/env python
	__author__ = 'Kevin Warrick'
	__email__ = 'kwarrick@uga.edu'

	import cPickle
	from functools import wraps

	def redis_lru(capacity=5000, slice=slice(None)):
	"""
	Simple Redis-based LRU cache decorator *.

	from functools import wraps, update_wrapper

	class Task(object):
	"""
	Class that wraps a function to enqueue in pyres
	"""
	_resque_conn = None

	def __init__(self, func, priority):
	self.func = func

	import gevent
	import gevent.pool

	class GroupWithExceptionCatching(gevent.pool.Group):
	def __init__(self, *args):
	super(GroupWithExceptionCatching, self).__init__(*args)
	self._error_handlers = {}

	def _wrap_errors(self, func):
	"""Wrap a callable for triggering error handlers