Last active
August 19, 2025 04:12
-
-
Save diogoos/7a057b554fed7baa33fa8bf2b9f387b3 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from queue import Queue | |
| from concurrent.futures import ThreadPoolExecutor | |
| import threading | |
| def worker(data): | |
| # Pre-process text, create embeddings, or do another compute-intensive tasks | |
| def writer(): | |
| while True: | |
| item = queue.get() | |
| if item is None: | |
| break # sentinel | |
| # Write the item to an output file as needed, or otherwise consume the output | |
| # Start writer thread | |
| writer_thread = threading.Thread(target=writer, daemon=True) | |
| writer_thread.start() | |
| # Loop through each piece of data, and enqueue it to be asynchronously processed | |
| with ThreadPoolExecutor() as executor: | |
| for datum in data: | |
| executor.submit(worker_encode, datum) | |
| # Wait for queue to empty and stop writer | |
| queue.join() | |
| queue.put(None) # sentinel | |
| writer_thread.join() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment