264 lines
12 KiB
Python
264 lines
12 KiB
Python
from odoo import models, fields, api, _
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
import logging
|
|
from base64 import b64encode
|
|
import re
|
|
|
|
_logger = logging.getLogger(__name__)
|
|
|
|
|
|
class ResPartner(models.Model):
|
|
_inherit = 'res.partner'
|
|
|
|
is_odoo_partner = fields.Boolean(string="Is Odoo Partner", default=False, tracking=True)
|
|
is_odoo_user = fields.Boolean(string="Is Odoo User", default=False, tracking=True)
|
|
odoo_name = fields.Char(string="Name on Odoo")
|
|
odoo_note = fields.Html(string="Note on Odoo")
|
|
odoo_id = fields.Char(string="ID on Odoo")
|
|
odoo_url = fields.Char(string="Odoo Partner URL", tracking=True, index=True)
|
|
odoo_page = fields.Html(string="Odoo Partner/Referral Page")
|
|
odoo_page_update = fields.Date(string="Odoo Page Update")
|
|
odoo_partner_type = fields.Selection(
|
|
[('learning', 'Learning'),
|
|
('ready', 'Ready'),
|
|
('silver', 'Silver'),
|
|
('gold', 'Gold')],
|
|
string="Partner Type",
|
|
tracking=True)
|
|
|
|
@api.model
|
|
def change_color_on_kanban(self):
|
|
for record in self:
|
|
color = 0
|
|
if record.odoo_partner_type == 'learning':
|
|
color = 2
|
|
elif record.odoo_partner_type == 'ready':
|
|
color = 10
|
|
elif record.odoo_partner_type == 'silver':
|
|
color = 7
|
|
elif record.odoo_partner_type == 'gold':
|
|
color = 3
|
|
else:
|
|
if record.is_odoo_user:
|
|
color = 4
|
|
record.color = color
|
|
|
|
color = fields.Integer('Color Index', compute="change_color_on_kanban")
|
|
|
|
def set_image_from_url(self, url):
|
|
response = requests.get(url)
|
|
if response.status_code == 200:
|
|
self.image_1920 = b64encode(response.content).decode('utf-8')
|
|
_logger.info(f"Image for {self.name} set from {url}")
|
|
else:
|
|
raise ValueError('Could not download image: got response code {}'.format(response.status_code))
|
|
|
|
@api.model
|
|
def get_odoo_partner(self):
|
|
|
|
def extract_client_data(soup):
|
|
odoo_name = soup.find(id='partner_name').text
|
|
odoo_note = soup.find('div', class_='col-lg-8 mt32')
|
|
|
|
main_image = soup.find('img', {'class': 'img img-fluid d-block mx-auto mb16'})
|
|
image_link = main_image['src']
|
|
|
|
exist_client = self.env['res.partner'].search([('odoo_name', '=', odoo_name)])
|
|
|
|
if not exist_client:
|
|
exist_client = exist_client.create({
|
|
'name': odoo_name,
|
|
'is_company': True,
|
|
'odoo_name':odoo_name,
|
|
'odoo_note':odoo_note,
|
|
'is_odoo_user': True,
|
|
'odoo_page': soup,
|
|
})
|
|
exist_client.set_image_from_url(image_link)
|
|
_logger.info(f"Client {exist_client.name} created")
|
|
|
|
return exist_client
|
|
|
|
def process_partner_relation(partner, client):
|
|
if partner.id != client.id:
|
|
self.env['res.partner.relation'].create({
|
|
'left_partner_id' : partner.id,
|
|
'right_partner_id' : client.id,
|
|
'type_id' : self.env.ref('bemade_odoo_partner_scrapper.rel_type_odoo_partner').id
|
|
})
|
|
_logger.info(f"Partner {partner.name} Odoo Partner of {client.name}")
|
|
else:
|
|
_logger.info(f"Partner {partner.name} is the same as client {client.name}")
|
|
|
|
def extract_partner_data(url):
|
|
page = requests.get("https://www.odoo.com" + url)
|
|
soup = BeautifulSoup(page.content, 'html.parser')
|
|
|
|
partner_data = {}
|
|
|
|
partner_data['name'] = soup.find(id="partner_name").text
|
|
partner_type = soup.find('h3', class_='col-lg-12 text-center text-muted')
|
|
|
|
if partner_type and partner_type.span:
|
|
partner_data['odoo_partner_type'] = partner_type.span.text.strip().lower()
|
|
|
|
partner_data['odoo_name'] = soup.find(id="partner_name").text
|
|
address_div = soup.find('span', itemprop='streetAddress')
|
|
email_div = soup.find('span', itemprop='email')
|
|
phone_div = soup.find('span', itemprop='telephone')
|
|
website_div = soup.find('span', itemprop='website')
|
|
|
|
main_image = soup.find('img', {'class': 'img img-fluid d-block mx-auto mb16'})
|
|
image_link = main_image['src']
|
|
|
|
clients_url = set()
|
|
for link in soup.find_all('a'):
|
|
href = link.get('href')
|
|
if href is not None:
|
|
# Ignore any URL containing 'country/canada-36'
|
|
if '/customers/' in href:
|
|
# Add to set. Duplicates will be ignored automatically.
|
|
clients_url.add(href)
|
|
|
|
state_data = None
|
|
|
|
if address_div is not None:
|
|
# Extract address components
|
|
address_parts = address_div.get_text(separator="\n").split("\n")
|
|
partner_data['street'] = address_parts[0]
|
|
city_state_postal = address_parts[len(address_parts) - 2].split(', ')
|
|
|
|
if city_state_postal != [',']:
|
|
partner_data['city'] = city_state_postal[0]
|
|
state_postal = city_state_postal[1].split(' ')
|
|
if len(state_postal) == 3 and len(state_postal[2]) == 3:
|
|
partner_data['zip'] = state_postal[1] + state_postal[2]
|
|
state_data = state_postal[0]
|
|
elif len(state_postal) == 2 and state_postal[1]:
|
|
if len(state_postal[1]) == 6:
|
|
partner_data['zip'] = state_postal[1]
|
|
state_data = state_postal[0]
|
|
if len(state_postal[1]) == 3:
|
|
if len(state_postal[0]) == 2:
|
|
partner_data['zip'] = state_postal[1]
|
|
state_data = state_postal[0]
|
|
else:
|
|
partner_data['zip'] = state_postal[0] + state_postal[1]
|
|
else:
|
|
if len(state_postal[0]) == 6:
|
|
partner_data['zip'] = state_postal[0]
|
|
else:
|
|
state_data = state_postal[0]
|
|
|
|
# partner_data['country'] = address_parts[len(address_parts) - 1]
|
|
|
|
partner_data['email'] = email_div.string if email_div and email_div.string else False
|
|
partner_data['phone'] = phone_div.string if phone_div and phone_div.string else False
|
|
partner_data['website'] = website_div.string if website_div and website_div.string else False
|
|
partner_data['is_odoo_partner'] = True
|
|
partner_data['is_odoo_user'] = True
|
|
partner_data['is_company'] = True
|
|
partner_data['image_link'] = image_link
|
|
partner_data['clients_url'] = clients_url
|
|
|
|
if state_data:
|
|
country_id = self.env['res.country'].search([('code', '=', 'CA')]).id
|
|
partner_data['state_id'] = self.env['res.country.state'].search([('code', '=', state_data),('country_id', '=', country_id)]).id
|
|
|
|
partner_data = {k: v for k, v in partner_data.items() if v is not False}
|
|
|
|
return partner_data
|
|
|
|
# Load the webpage at the given url
|
|
page = requests.get("https://www.odoo.com/fr_FR/partners/country/canada-36")
|
|
# Parse the HTML content
|
|
soup = BeautifulSoup(page.content, 'html.parser')
|
|
div = soup.find(id="ref_content")
|
|
partners_url = set()
|
|
other_pages = set()
|
|
|
|
for link in div.find_all('a'):
|
|
href = link.get('href')
|
|
if href is not None:
|
|
# Ignore any URL containing 'country/canada-36'
|
|
if '/page/' in href:
|
|
# Add to set. Duplicates will be ignored automatically.
|
|
other_pages.add(href)
|
|
|
|
for link in div.find_all('a'):
|
|
href = link.get('href')
|
|
if href is not None and href != '':
|
|
# Ignore any URL containing 'country/canada-36'
|
|
if 'country/canada-36' not in href:
|
|
# Add to set. Duplicates will be ignored automatically.
|
|
partners_url.add(href.split('#', 1)[0].split('?', 1)[0])
|
|
|
|
for other_page in other_pages:
|
|
page = requests.get("https://www.odoo.com" + other_page)
|
|
soup = BeautifulSoup(page.content, 'html.parser')
|
|
div = soup.find(id="ref_content")
|
|
for link in div.find_all('a'):
|
|
href = link.get('href')
|
|
if href is not None and href != '':
|
|
# Ignore any URL containing 'country/canada-36'
|
|
if 'country/canada-36' not in href:
|
|
# Add to set. Duplicates will be ignored automatically.
|
|
partners_url.add(href.split('#', 1)[0].split('?', 1)[0])
|
|
|
|
for partner_url in partners_url:
|
|
# page = requests.get("https://www.odoo.com" + partner_url)
|
|
# soup = BeautifulSoup(page.content, 'html.parser')
|
|
|
|
odoo_partner = extract_partner_data(partner_url)
|
|
_logger.info(f"Processing {odoo_partner['name']}")
|
|
|
|
image_url = False
|
|
if odoo_partner['image_link']:
|
|
image_url = odoo_partner['image_link']
|
|
del odoo_partner['image_link']
|
|
_logger.info(f"Image for {odoo_partner['name']} set from {image_url}")
|
|
|
|
if odoo_partner['clients_url']:
|
|
clients_url = odoo_partner['clients_url']
|
|
else:
|
|
clients_url = set()
|
|
|
|
if 'clients_url' in odoo_partner:
|
|
del odoo_partner['clients_url']
|
|
|
|
exist_odoo_name = self.env['res.partner'].search([('odoo_name', '=', odoo_partner['odoo_name'])])
|
|
if exist_odoo_name:
|
|
_logger.info(f"Partner Odoo Name {odoo_partner['name']} already exist")
|
|
new_partner = exist_odoo_name
|
|
else:
|
|
exist_email = self.env['res.partner'].search([('email', '=', odoo_partner['email'])]) if 'email' in odoo_partner else False
|
|
if exist_email:
|
|
_logger.info(f"Partner Email {odoo_partner['name']} already exist")
|
|
new_partner = exist_email
|
|
else:
|
|
exist_phone = self.env['res.partner'].search([('phone', '=', odoo_partner['phone'])]) if 'phone' in odoo_partner else False
|
|
if exist_phone:
|
|
_logger.info(f"Partner Phone {odoo_partner['name']} already exist")
|
|
new_partner = exist_phone
|
|
else:
|
|
_logger.info(f"Partner {odoo_partner['name']} created")
|
|
new_partner = self.env['res.partner'].create(odoo_partner)
|
|
if image_url:
|
|
new_partner.set_image_from_url(image_url)
|
|
|
|
for client_url in clients_url:
|
|
if 'http' not in client_url:
|
|
exist_url = self.env['res.partner'].search([('odoo_url', '=', client_url)])
|
|
# add condition on odoo_date, actually not pulling data if date exist
|
|
if exist_url and exist_url.odoo_page:
|
|
soup = exist_url.odoo_page
|
|
_logger.info(f"Client {exist_url.name} reloading soup from db")
|
|
else:
|
|
print("https://www.odoo.com" + client_url)
|
|
page = requests.get("https://www.odoo.com" + client_url)
|
|
soup = BeautifulSoup(page.content, 'html.parser')
|
|
_logger.info(f"Client {exist_url.name} read from odoo.com")
|
|
|
|
client = extract_client_data(soup)
|
|
process_partner_relation(new_partner, client)
|