Python3 Web scraping REQUESTS example
Web scraping using REQUESTS Libary and using latest headers.
REQUESTS Web Scraping
import requests
from UserAgentFetcher import UserAgentFetcher
def make_request(url, headers=None):
"""
Makes a GET request to the specified URL and prints the response content.
Args:
url (str): The URL to send the GET request to.
headers (dict, optional): Custom headers for the request.
Returns:
None
"""
# Send a GET request to the specified URL with optional custom headers
response = requests.get(url, headers=headers)
# Check if the response status code indicates success (2xx) or not
if 200 <= response.status_code < 400:
print(f"Request successful: {response.status_code}")
print("Request headers:")
for header, value in headers.items():
print(f"{header}: {value}")
print("Response headers:")
for header, value in response.headers.items():
print(f"{header}: {value}")
print("Response content:")
print(response.text)
else:
print(f"Request failed with status code: {response.status_code}")
if __name__ == "__main__":
# Define the target URL and custom headers
target_url = "https://www.example.com"
ua_fetcher = UserAgentFetcher()
random_user_agent = ua_fetcher.get_random_user_agent()
custom_headers = {
"User-Agent": random_user_agent,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"upgrade-insecure-requests": "1"
}
# Call the make_request function with the defined URL and headers
make_request(target_url, headers=custom_headers)
UserAgentFetcher.py
import requests
import random
class UserAgentFetcher:
def __init__(self):
"""Initialize the UserAgentFetcher class by fetching user agents from a JSON file."""
self.user_agents = self._fetch_user_agents()
def _fetch_user_agents(self):
"""Fetch user agents from the specified URL and return them as a list."""
url = 'https://jnrbsn.github.io/user-agents/user-agents.json'
try:
response = requests.get(url)
response.raise_for_status()
user_agents = response.json()
return user_agents
except requests.exceptions.RequestException as e:
print(f"Error retrieving user agent data: {e}")
return []
def get_random_user_agent(self, search_string='Windows NT 10.0; Win64; x64'):
"""
Get a random user agent that partially matches the given search string.
Args:
search_string (str): The search string to match against user agents.
Returns:
str: A random user agent that matches the search string, or None if no match is found.
"""
filtered_agents = [agent for agent in self.user_agents if search_string in agent]
if filtered_agents:
random_agent = random.choice(filtered_agents)
return random_agent
else:
return None
if __name__ == "__main__":
ua_fetcher = UserAgentFetcher()
random_user_agent = ua_fetcher.get_random_user_agent()
if random_user_agent:
print("Random User Agent:", random_user_agent)
else:
print("No matching user agents found.")
