Python Multi-threading

import queue
import threading
import time

# Create a queue
q = queue.Queue()

def worker():
    while True:
        # Retrieve a task from the queue
        item = q.get()
        
        # None is our signal to stop the thread
        if item is None:
            q.task_done()
            break
            
        print(print(f"Working on {item}"))
        time.sleep(0.5) # Simulate work
        
        # Tell the queue that this specific item is fully processed
        q.task_done()

# Start worker threads
threads = []
for i in range(2):
    t = threading.Thread(target=worker)
    t.start()
    threads.append(t)

# Put items into the queue
for item in ["Task A", "Task B", "Task C"]:
    q.put(item)

# Block the main thread until all items are processed
q.join()
print("--- All tasks are finished! ---")

# Clean up threads by sending the stop signal (None)
for i in range(2):
    q.put(None)
for t in threads:
    t.join()

Common Pitfalls to Avoid

• ValueError: task_done() called more times than put(): This happens if you accidentally call task_done() multiple times for a single get(), causing the internal counter to drop below zero.

Forgetting to call it: If a worker crashes or hits an early return/continue before calling task_done(), Queue.join() will wait forever because the counter will never hit zero.

Tip: Use a try...finally block in your worker to ensure q.task_done() is always called, even if an exception occurs during processing:

item = q.get()
try:
    # process item here
finally:
    q.task_done()

 

Producer-Consumer Pattern

import queue
import threading
import time
import random

# 1. Initialize a thread-safe queue
# Setting a maxsize prevents the producer from running away with memory
pipeline = queue.Queue(maxsize=5)

# A flag to signal workers to stop when production is completely done
STOP_SIGNAL = None

# 2. Define the Producer
def producer(q, num_items):
    print("[Producer]: Starting...")
    for i in range(1, num_items + 1):
        # Simulate time taken to generate or fetch data
        time.sleep(random.uniform(0.1, 0.4))
        item = f"Data-Package-{i}"
        
        # Put item into the queue (blocks if queue is full)
        q.put(item)
        print(f"[Producer]: Enqueued {item} (Queue size: {q.qsize()})")
        
    print("[Producer]: Done producing items.")

# 3. Define the Consumer
def consumer(q, consumer_id):
    print(f"[Consumer {consumer_id}]: Starting...")
    while True:
        # Get an item from the queue (blocks if queue is empty)
        item = q.get()
        
        # Check for the poison pill (stop signal)
        if item is STOP_SIGNAL:
            q.task_done()  # Acknowledge the stop signal task
            break
            
        # Simulate processing time
        print(f"[Consumer {consumer_id}]: Processing {item}...")
        time.sleep(random.uniform(0.3, 0.7)) 
        
        # Signal that the specific item is fully processed
        q.task_done()
        print(f"[Consumer {consumer_id}]: Finished {item}")
        
    print(f"[Consumer {consumer_id}]: Shutting down.")

# --- Main Execution ---
if __name__ == "__main__":
    num_consumers = 2
    total_items = 10
    
    # Create and start the producer thread
    producer_thread = threading.Thread(target=producer, args=(pipeline, total_items))
    producer_thread.start()
    
    # Create and start a pool of consumer threads
    consumer_threads = []
    for i in range(num_consumers):
        t = threading.Thread(target=consumer, args=(pipeline, i+1))
        t.start()
        consumer_threads.append(t)
        
    # Wait for the producer to finish generating all items
    producer_thread.join()
    
    # Wait for the queue to become completely empty and all tasks acknowledged
    pipeline.join()
    print("[Main]: All items in the queue have been successfully processed!")
    
    # Tell the consumers they can safely exit by sending "poison pills"
    for _ in range(num_consumers):
        pipeline.put(STOP_SIGNAL)
        
    # Wait for all consumer threads to completely shut down
    for t in consumer_threads:
        t.join()
        
    print("[Main]: Program finished cleanly.")