-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathmain.py
More file actions
90 lines (75 loc) · 3.92 KB
/
Copy pathmain.py
File metadata and controls
90 lines (75 loc) · 3.92 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
import os
import requests
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.common.exceptions import NoSuchElementException, ElementNotInteractableException
from dotenv import load_dotenv
from time import sleep
# https://www.whatismybrowser.com/detect/what-http-headers-is-my-browser-sending/
# Part 1 - Scrape the links, addresses, and prices of the rental properties
header = {
"Accept-Language": "en-US,en;q=0.9",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
}
# Use our Zillow-Clone website (instead of Zillow.com)
response = requests.get("https://appbrewery.github.io/Zillow-Clone", headers=header)
response.raise_for_status()
data = response.text
soup = BeautifulSoup(data, "html.parser")
# Create a list of all the links on the page using a CSS Selector
all_link_elements = soup.select(".StyledPropertyCardDataWrapper a")
all_links = [link["href"] for link in all_link_elements]
print(f"There are {len(all_links)} links to individual listings in total: \n")
print(all_links)
# Create a list of all the addresses on the page using a CSS Selector
# Remove newlines \n, pipe symbols |, and whitespaces to clean up the address data
all_address_elements = soup.select(".StyledPropertyCardDataWrapper address")
all_addresses = [address.get_text().replace(" | ", " ").strip() for address in all_address_elements]
print(f"\n After having been cleaned up, the {len(all_addresses)} addresses now look like this: \n")
print(all_addresses)
# Create a list of all the prices on the page using a CSS Selector
# Get a clean dollar price and strip off any "+" symbols and "per month" /mo abbreviation
all_price_elements = soup.select(".PropertyCardWrapper span")
all_prices = [price.get_text().replace("/mo", "").split("+")[0]
for price in all_price_elements if "$" in price.text]
print(f"\n After having been cleaned up, the {len(all_prices)} prices now look like this: \n")
print(all_prices)
# Part 2 - Fill in the Google Form using Selenium
# Optional - Keep the browser open (helps diagnose issues if the script crashes)
chrome_options = webdriver.ChromeOptions()
chrome_options.add_experimental_option("detach", True)
driver = webdriver.Chrome(options=chrome_options)
load_dotenv()
google_form_url = os.getenv("GOOGLE_FORM_URL")
driver.get(google_form_url)
sleep(3)
for n in range(len(all_links)):
sleep(1)
# Use the xpath to select the "short answer" fields in your Google Form.
# Note, your xpath might be different if you created a different form.
address = driver.find_element(by=By.XPATH,
value='//*[@id="mG61Hd"]/div[2]/div/div[2]/div[1]/div/div/div[2]/div/div[1]'
'/div/div[1]/input')
price = driver.find_element(by=By.XPATH,
value='//*[@id="mG61Hd"]/div[2]/div/div[2]/div[2]/div/div/div[2]/div/div[1]'
'/div/div[1]/input')
link = driver.find_element(by=By.XPATH,
value='//*[@id="mG61Hd"]/div[2]/div/div[2]/div[3]/div/div/div[2]/div/div[1]'
'/div/div[1]/input')
submit_button = driver.find_element(by=By.XPATH,
value='//*[@id="mG61Hd"]/div[2]/div/div[3]/div[1]/div[1]/div')
try:
address.send_keys(all_addresses[n])
price.send_keys(all_prices[n])
link.send_keys(all_links[n])
submit_button.click()
sleep(1)
submit_another_response = driver.find_element(by=By.XPATH,
value="/html/body/div[1]/div[2]/div[1]/div/div[4]/a")
submit_another_response.click()
except (NoSuchElementException, ElementNotInteractableException) as e:
print("Something went wrong...")
print(e)
driver.quit()