Real-Life Case Study: The $3,600 Proxy Disaster

1. Implement Smart Proxy Rotation

import requests
import random
from itertools import cycle

# Sequential rotation
def sequential_proxy_rotation():
    proxies_list = open("proxies_list.txt").read().strip().split("\n")
    proxy_pool = cycle(proxies_list)

    for _ in range(4):
        proxy = next(proxy_pool)
        proxies = {"http": f"http://{proxy}", "https": f"http://{proxy}"}
        try:
            response = requests.get("https://httpbin.io/ip", proxies=proxies, timeout=10)
            print(f"Success with proxy: {proxy}" if response.status_code == 200 else f"Failed with status: {response.status_code}")
        except Exception as e:
            print(f"Error with proxy {proxy}: {e}")

# Random rotation
def random_proxy_rotation():
    proxies_list = open("proxies_list.txt").read().strip().split("\n")

    for _ in range(4):
        random_proxy = random.choice(proxies_list)
        proxies = {"http": f"http://{random_proxy}", "https": f"http://{random_proxy}"}
        try:
            response = requests.get("https://httpbin.io/ip", proxies=proxies, timeout=10)
            print(f"Success with proxy: {random_proxy}")
        except Exception as e:
            print(f"Error with proxy {random_proxy}: {e}")

2. Control Request Rate and Timing

import time
import random

def make_request_with_delay(url, proxies):
    time.sleep(random.uniform(10, 15))
    time.sleep(delay)
    try:
        response = requests.get(url, proxies=proxies, timeout=10)
        return response
    except Exception as e:
        print(f"Request failed: {e}")
        return None

3. Implement User-Agent Rotation

import random

user_agents = [
    "Mozilla/5.0 (Windows NT 10.0...)",
    "Mozilla/5.0 (Macintosh; Intel...)",
    "Mozilla/5.0 (Linux; Android...)"
]

def get_random_user_agent():
    return random.choice(user_agents)

def make_request_with_rotation(url, proxies):
    headers = {
        'User-Agent': get_random_user_agent()
    }
    return requests.get(url, proxies=proxies, headers=headers)
    return response

4. Choose the Right Proxy Type

Use residential proxies for stealth and datacenter proxies for speed. Mixing both reduces detection risk by 21%.

5. Implement Session Management

class ProxySession:
    def __init__(self, proxy_list):
        self.proxy_list = proxy_list
        self.sessions = {}

    def get_session(self, session_id):
        if session_id not in self.sessions:
            proxy = random.choice(self.proxy_list)
            session = requests.Session()
            session.proxies = {'http': f'http://{proxy}', 'https': f'http://{proxy}'}
            self.sessions[session_id] = session
        return self.sessions[session_id]

Best Practices for Avoiding Proxy Bans

Monitor and Manage Proxy Health

def monitor_proxy_health(proxy_list):
    healthy_proxies = []
    for proxy in proxy_list:
        try:
            proxies = {'http': f'http://{proxy}', 'https': f'http://{proxy}'}
            response = requests.get('https://httpbin.io/ip', proxies={'http': f'http://{proxy}', 'https': f'http://{proxy}'}, timeout=5)
            if response.status_code == 200:
                healthy_proxies.append(proxy)
                print(f"Proxy {proxy} is healthy") 
            else: 
                print(f"Proxy {proxy} returned status {response.status_code}") 
         except Exception as e: 
                print(f"Proxy {proxy} failed: {e}") 
     return healthy_proxies

Set Usage Thresholds and Alerts

class ProxyUsageMonitor:
    def __init__(self, max_requests=1000, max_bandwidth_gb=1.0):
        self.max_requests = max_requests
        self.max_bandwidth_gb = max_bandwidth_gb
        self.request_count = 0
        self.bandwidth_used = 0.0

    def check_limits(self):
         if self.request_count >= self.max_requests:
            raise Exception(f"Request limit exceeded: {self.request_count}") 
         if self.bandwidth_used >= self.max_bandwidth_gb:
            raise Exception(f"Bandwidth limit exceeded: {self.bandwidth_used} GB")

    def log_request(self, response_size_bytes):
        self.request_count += 1
        self.bandwidth_used += response_size_bytes / (1024**3)
        self.check_limits()
        self.check_limits()Use Separate Credentials

Separate credentials by site to track and manage resource usage precisely.

Block Unnecessary Resources

def setup_resource_blocking(page):
    def route_intercept(route):
        url = route.request.url
        resource_type = route.request.resource_type

        if route.request.resource_type in ["image", "stylesheet"] 
            return route.abort()
        if "target-domain.com" not in url:
            return route.abort()

        return route.continue_()
    
    page.route("**/*", route_intercept)

Real-World Implementation: E-commerce Scraping

Even residential proxies failed on Amazon. Only unblocker services achieved full data extraction. This shows the limits of proxies alone against high-security websites.

Advanced Techniques

• Use machine learning to adapt scraper behavior
• Integrate CAPTCHA-solving services
• Distribute scraping tasks across regions and machines

Conclusion

Proxies are essential but must be used wisely. Key takeaways:

• Use rotation + user-agent + timing
• Block third-party resources
• Set usage limits
• Track proxy health continuously
• Learn from failures and optimize over time