import os import time import json import logging import traceback import pymysql import base64 import argparse import srtools import sys import re from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.chrome.service import Service from selenium.webdriver.chrome.options import Options from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.common.exceptions import TimeoutException, NoSuchElementException from selenium.webdriver.common.keys import Keys from selenium.webdriver.common.action_chains import ActionChains from selenium.webdriver.support.ui import Select from webdriver_manager.chrome import ChromeDriverManager #PDFS FOLDER pdfs_dir = 'pdfs' if not os.path.exists(pdfs_dir): os.makedirs(pdfs_dir) #JSON CONFIGURATION AND CREDS with open('credentials.json') as cred_file: credentials = json.load(cred_file) with open('selectors.json') as sel_file: selectors = json.load(sel_file) db_config = { 'host': credentials["db_host"], 'user': credentials["db_user"], 'password': credentials["db_password"], 'database': credentials["db_database"], 'charset': 'utf8mb4' } ################################## #CHROME CONFIGURATION service = Service() chrome_options = Options() chrome_options.add_argument("--headless") chrome_options.add_argument('--disable-gpu') #chrome_options.add_argument("--window-size=1920x1080") # Set window size explicitly chrome_options.add_experimental_option("prefs", { "download.default_directory": os.getcwd(), "download.prompt_for_download": False, "download.directory_upgrade": True, "plugins.always_open_pdf_externally": True }) driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()),options=chrome_options) driver.set_page_load_timeout(30) driver.set_script_timeout(30) driver.implicitly_wait(10) ############### def login(driver=driver): driver.get(credentials['rgz_link']) username_field = driver.find_element(By.ID, selectors['username']) username_field.clear() username_field.send_keys(credentials['rgz_username']) password_field = driver.find_element(By.ID, selectors['password']) password_field.clear() password_field.send_keys(credentials['rgz_password']) driver.find_element(By.ID, selectors['login_button']).click() driver.find_element(By.CSS_SELECTOR, selectors['adrese_link']).click() def get_addresses(opstina): addresses = [] translated_opstina = srtools.cyrillic_to_latin(opstina) try: connection = pymysql.connect(**db_config, cursorclass=pymysql.cursors.DictCursor) with connection.cursor() as cursor: query = "SELECT * FROM ak_location WHERE opstina_ime_lat = %s" cursor.execute(query, (translated_opstina,)) rows = cursor.fetchall() for row in rows: address = { 'opstina': opstina, 'ime_ulice': row['ulica_ime_lat'], 'broj': row['kucni_broj_lat'] } #print(address) addresses.append(address) except pymysql.Error as err: print(f"Error: {err}") finally: if connection and connection.open: connection.close() return addresses def process_address(driver, opstina, ime_ulice, kucni_broj_value, session_id): print("Processing : " + ime_ulice + " broj : " + str(kucni_broj_value)) if not os.path.exists(pdfs_dir + "/" + session_id): os.makedirs(pdfs_dir + "/" + session_id) login() select_element = Select(driver.find_element(By.ID, selectors['opstina_select'])) select_element.select_by_visible_text(opstina) time.sleep(3) for char in ime_ulice: dynamic_input = driver.find_element(By.ID, selectors['ulica_input']) dynamic_input.send_keys(char) time.sleep(0.5) autocomplete_option = WebDriverWait(driver, 10).until(EC.visibility_of_element_located((By.XPATH, selectors["autocomplete"]))) autocomplete_option.click() kucni_broj = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, selectors['kucni_broj_input']))) kucni_broj.send_keys(kucni_broj_value) # Use a different variable name here submit_button = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, selectors['submit_button']))) submit_button.click() rows = WebDriverWait(driver, 20).until(EC.visibility_of_all_elements_located((By.XPATH, "//table[@id='ctl00_ContentPlaceHolder1_Parcele']/tbody/tr")))[1:] current_page = 1 while True: pagination_links = driver.find_elements(By.XPATH, "//tr[@class='pager']//a") for index, row in enumerate(rows): time.sleep(5) rows = WebDriverWait(driver, 20).until(EC.visibility_of_all_elements_located((By.XPATH, "//table[@id='ctl00_ContentPlaceHolder1_Parcele']/tbody/tr")))[1:] row = rows[index] cells = row.find_elements(By.TAG_NAME, "td") if len(cells) < 5 or 'Page$' in cells[-1].find_element(By.TAG_NAME, "a").get_attribute('href'): continue broj_parcele = cells[3].text.strip().replace('/', '-') broj_dela_parcele = cells[4].text.strip() link = cells[-1].find_element(By.TAG_NAME, "a") posebni_delovi_link = cells[0].find_element(By.TAG_NAME, "input") driver.execute_script("arguments[0].click();", posebni_delovi_link) driver.execute_script("arguments[0].click();", link) WebDriverWait(driver, 20).until(EC.number_of_windows_to_be(2)) new_window = [window for window in driver.window_handles if window != driver.current_window_handle][0] driver.switch_to.window(new_window) driver.maximize_window() rows = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.ID, "propNepokretnost_ucLabelPermitionOpstinaNaziv_lblText"))) pdf_data = driver.execute_cdp_cmd("Page.printToPDF", {"printBackground": True}) pdf_filename = f"{pdfs_dir}/{broj_parcele}_{broj_dela_parcele}.pdf" pdf_filename_session = f"{pdfs_dir}/{session_id}/{broj_parcele}_{broj_dela_parcele}.pdf" with open(pdf_filename, "wb") as f: f.write(base64.b64decode(pdf_data['data'])) logging.info(f"File saved: {pdf_filename}") with open(pdf_filename_session, "wb") as f: f.write(base64.b64decode(pdf_data['data'])) logging.info(f"File session saved: {pdf_filename_session}") driver.close() driver.switch_to.window(driver.window_handles[0]) try: table = driver.find_element(By.ID, "ctl00_ContentPlaceHolder1_PosebniDelovi") other_rows = WebDriverWait(driver, 5).until( EC.visibility_of_all_elements_located( (By.XPATH, "//table[@id='ctl00_ContentPlaceHolder1_PosebniDelovi']/tbody/tr")))[1:] for other_row in other_rows: cells = other_row.find_elements(By.TAG_NAME, "td") evid_broj = cells[0].text.strip() broj_stana = cells[2].text.strip() WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.TAG_NAME, "a"))) pdf_button = cells[-1].find_element(By.TAG_NAME, "a") pdf_button.click() WebDriverWait(driver, 10).until(EC.number_of_windows_to_be(2)) new_window = [window for window in driver.window_handles if window != driver.current_window_handle][0] driver.switch_to.window(new_window) driver.maximize_window() pdf_data = driver.execute_cdp_cmd("Page.printToPDF", {"printBackground": True}) pdf_filename = f"{pdfs_dir}/objekat_{broj_parcele}_{broj_dela_parcele}_{evid_broj}.pdf" pdf_filename_session = f"{pdfs_dir}/{session_id}/objekat_{broj_parcele}_{broj_dela_parcele}_{evid_broj}.pdf" with open(pdf_filename, "wb") as f: f.write(base64.b64decode(pdf_data['data'])) logging.info(f"File saved: {pdf_filename}") with open(pdf_filename_session, "wb") as f: f.write(base64.b64decode(pdf_data['data'])) logging.info(f"File saved: {pdf_filename_session}") with open("output.txt", "a") as log_file: log_file.write(pdf_filename + "\n") driver.close() driver.switch_to.window(driver.window_handles[0]) except NoSuchElementException: print("Table does not exist, continuing...") next_page_link = None pagination_links = driver.find_elements(By.XPATH, "//tr[@class='pager']//a") for link in pagination_links: href = link.get_attribute('href') page_match = re.search(r'Page\$(\d+)', href) if page_match: page_number = int(page_match.group(1)) if page_number == current_page + 1: next_page_link = link break '''elif '...' in link.text: #and current_page < len(pagination_links) - 2: pass''' if next_page_link: next_page_link.click() current_page += 1 WebDriverWait(driver, 20).until(EC.visibility_of_all_elements_located((By.XPATH, "//table[@id='ctl00_ContentPlaceHolder1_Parcele']/tbody/tr")))[1:] else: break parser = argparse.ArgumentParser(description='Process some addresses.') group1 = parser.add_argument_group('Single address') group1.add_argument('--opstina', help='Opstina') group1.add_argument('--ulica', help='Street name') group1.add_argument('--broj', help='House number') group1.add_argument('--sessionid', help='Session id') group2 = parser.add_argument_group('Addresses in a municipality') group2.add_argument('--po_opstini', help='Addresses in a certain municipality') group2.add_argument('--sessionido', help='Session id') args = parser.parse_args() if args.ulica and args.broj: print(f"Address: {args.ulica} {args.broj}") process_address(driver, args.opstina, args.ulica, args.broj, args.sessionid) elif args.po_opstini: print(f"Addresses in {args.po_opstini} municipality") addresses_list = get_addresses(args.po_opstini) for address in addresses_list: process_address(driver, args.po_opstini, address['ime_ulice'], address['broj'],args.sessionido)