Create app.py

app.py

@@ -0,0 +1,594 @@
+
+import streamlit as st
+import requests
+import numpy as np
+from streamlit_lottie import st_lottie
+from PIL import Image
+import warnings
+warnings.filterwarnings("ignore")
+import requests
+import pandas as pd
+import numpy as np
+from bs4 import BeautifulSoup
+import bs4
+from urllib.request import urlopen
+import time
+import re
+import time
+import matplotlib.pyplot as plt 
+import seaborn as sns 
+import matplotlib as mpl
+import plotly 
+import plotly.express as px
+import plotly.graph_objs as go
+import plotly.offline as py
+from plotly.offline import iplot
+from plotly.subplots import make_subplots
+import plotly.figure_factory as ff
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+
+#Settings for using the driver without a UI
+options = webdriver.ChromeOptions()
+options.add_argument('--headless')
+options.add_argument('--no-sandbox')
+options.add_argument('--disable-dev-shm-usage')
+
+options.add_argument("start-maximized")
+options.add_argument("disable-infobars")
+options.add_argument("--disable-extensions")
+driver = webdriver.Chrome('chromedriver',options=options)
+
+
+# wuzzuf function
+def Wuzzuf_scrapping(job_type , job_num):
+    job1 = job_type.split(" ")[0]
+    job2 = job_type.split(" ")[1]
+    link1 = 'https://wuzzuf.net/search/jobs/?a=navbl&q='+job1+'%20'+job1
+    title = []
+    location = []
+    country = []
+    job_description = []
+    Job_Requirements =[]
+    company_name = []
+    links = []
+    Jop_type = []
+    Career_Level = []
+    company_logo = []
+    Job_Categories = []
+    Skills_And_Tools = []
+    Experience_Needed =[]
+    post_time = []
+    Title = []
+    pages_num = np.ceil(job_num/15)
+
+
+    for i in range(int(pages_num) ):
+      link_new = link1 +'&start='+str(i)
+      data  = requests.get(link_new)
+      soup  = BeautifulSoup(data.content)
+      Title = soup.find_all('h2' , {'class': 'css-m604qf'})
+
+  # to get the info about jobs
+
+      for x in range(0,len(Title)):
+        t = re.split('\(|\-',Title[x].find('a').text)
+        title.append(t[0].strip())
+        loc = re.split(',' , soup.find_all('span' , {'class': 'css-5wys0k'})[x].text)
+        r = ""
+        for i in range(len(loc[:-1])):
+          r= r+ ', ' +loc[:-1][i].strip()
+        location.append(r.replace(',', '', 1).strip())
+        country.append(loc[-1].strip())
+        links.append('https://wuzzuf.net' + Title[x].find('a').attrs['href'])
+        m = " ".join(re.findall("[a-zA-Z\d+]+", (soup.find_all('div' , {'class': 'css-d7j1kk'})[x].find('a').text)))
+        company_name.append(m)
+        c = soup.find_all('div' ,{'class':'css-1lh32fc'})[x].find_all('span')
+        if len(c) ==1:
+          Jop_type.append(c[0].text)
+        else:
+          n =[]
+          for i in range(len(c)):
+            n.append(c[i].text)
+          Jop_type.append(n)
+        n =soup.find_all('div' ,{'class':'css-y4udm8'})[x].find_all('div')[1].find_all(['a','span'])
+        Career_Level.append(n[0].text)
+        n =soup.find_all('div' ,{'class':'css-y4udm8'})[x].find_all('div')[1].find_all(['a','span'])
+
+        yy = n[1].text.replace('·',' ').strip()
+        yy = re.findall('[0-9-+]*',yy)
+        y1 =""
+        for i in range(len(yy)):
+        
+          if any(yy[i]):
+            y1 = y1+yy[i]
+        if y1 != "":
+          Experience_Needed.append(y1)
+        else:
+          Experience_Needed.append("Not Specified")
+        time = (soup.find_all('div' ,{'class':'css-d7j1kk'}))[x].find('div')
+        post_time.append(time.text)
+        
+  # to get the logo of the company
+      
+        data1  = requests.get(links[x])
+        soup1 = BeautifulSoup(data1.content)
+        company_logo.append(soup1.find_all('meta',{'property':"og:image"})[0]['content'])
+        #time.sleep(4)
+
+
+  # get Job_Categories , Skills_And_Tools , job_description , and job_requirements from urls
+        driver = webdriver.Chrome('chromedriver',options=options)
+        #driver.implicitly_wait(10)
+        driver.get(links[x])
+        Job_Categories.append(driver.find_element(By.XPATH ,'//*[@id="app"]/div/main/section[2]/div[5]').text.split("\n")[1:])
+        Skills_And_Tools.append(driver.find_element(By.XPATH ,'//*[@id="app"]/div/main/section[2]/div[6]').text.split("\n")[1:])
+        job_description.append(driver.find_element(By.XPATH ,'//*[@id="app"]/div/main/section[3]').text.split("\n")[1:])
+        all =driver.find_elements(By.XPATH ,'//*[@id="app"]/div/main/section[4]/div')
+        dict_other = {}
+    
+        new = all[0].text.split("\n\n")
+
+        if len(new)!=1 :
+          for i in range(len(new)):
+            result =[]
+            for k in (new[i].split('\n')[1:]):  
+              result.append(k.replace("\u202f"," "))
+              dict_other[new[i].split('\n')[0]] = result
+
+            #result = re.sub('[\W_]+', '', ini_string)
+
+          Job_Requirements.append(dict_other)
+
+        else:
+          nn = new[0].replace("\u202f"," ")
+          Job_Requirements.append(nn.split('\n'))
+
+
+  #  create data frame to combine all together
+
+    df = pd.DataFrame({'Title' : title , 'Location' : location ,'country':country,'URLs':links ,'Company_Name' : company_name,'Career_Level':Career_Level,'post_time':post_time,'Experience_Needed':Experience_Needed,'Company_Logo':company_logo,"Job_Categories":Job_Categories , "Skills_And_Tools":Skills_And_Tools , "job_description":job_description,"Job_Requirements":Job_Requirements})
+    
+    df[:job_num].to_excel('WUZZUF_scrapping.xlsx',index=False,encoding='utf-8')
+    return df[:job_num]
+
+
+# linkedin function
+
+
+def LINKEDIN_Scrapping(job_search , num_jobs):
+  job1 = job_search.split(" ")[0]
+  job2 = job_search.split(" ")[1]
+
+  link1 = 'https://www.linkedin.com/jobs/search?keywords='+job1 +'%20' +job2 +'&location=&geoId=&trk=public_jobs_jobs-search-bar_search-submit&position=1&pageNum=0'
+  
+  # FIRST get main informations about jobs
+
+  title = []
+  location = []
+  country = []
+  company_name = []
+  post_time = []
+  links =[]
+  # get the specific numbers of jobs
+  l1 = ""
+  ll =""
+  driver = webdriver.Chrome('chromedriver',options=options)
+  driver.get(link1)
+  SCROLL_PAUSE_TIME = 0.5
+  while True :
+    l1 = driver.find_elements(By.XPATH,'//*[@id="main-content"]/section[2]/ul/li[*]/div')
+    ll= driver.find_elements(By.XPATH ,'//*[@id="main-content"]/section[2]/ul/li[*]/div/a') 
+
+    if len(l1) >= num_jobs:
+      break
+    time.sleep(3)
+    # Get scroll height
+    last_height = driver.execute_script("return document.body.scrollHeight")
+    while True:
+        
+        # Scroll down to bottom
+        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
+
+        # Wait to load page
+        time.sleep(SCROLL_PAUSE_TIME)
+
+        # Calculate new scroll height and compare with last scroll height
+        new_height = driver.execute_script("return document.body.scrollHeight")
+        if new_height == last_height:
+            break
+        last_height = new_height
+        
+    options.add_argument("window-size=1200x600")
+    WebDriverWait(driver, 15).until(EC.element_to_be_clickable((By.XPATH, '//*[@id="main-content"]/section[2]/button'))).click()
+    print(len(l1))
+    time.sleep(2)
+    
+
+
+  l2 = l1[:num_jobs]
+
+  for info in l2:   
+    info_tot = info.text.split("\n")
+    if len(info_tot)==5:
+      title.append(info_tot[1])
+      location.append(info_tot[3])
+      company_name.append(info_tot[2])
+      post_time.append(info_tot[4])
+    else:
+      title.append(info_tot[1])
+      location.append(info_tot[3])
+      company_name.append(info_tot[2])
+      post_time.append(info_tot[5])
+
+  # get links for jobs
+  l3 = ll[:num_jobs]
+  for i in l3:
+    links.append(i.get_attribute('href'))
+  
+  df_ml = pd.DataFrame({'Title' : title , 'Location' : location ,'URLs':links ,'Company_Name' : company_name ,'post_time':post_time})
+
+
+
+
+    # GET DESCRIPTION AND LOGO 
+  def all_description_LOGO(urls):
+    description =[]
+    LOGO =[]
+    for link in urls:         
+      driver = webdriver.Chrome('chromedriver',options=options)
+      driver.get(link)
+      options.add_argument("window-size=1200x600")
+      WebDriverWait(driver, 15).until(EC.element_to_be_clickable((By.XPATH, '//*[@id="main-content"]/section[1]/div/div[1]/section[1]/div/div/section/button[1]'))).click()
+      qqq= 4+444*58/7+65
+      K = driver.find_element(By.XPATH,'//*[@id="main-content"]/section[1]/div/section[2]/div/a/img')
+      LOGO.append(K.get_attribute('src'))
+      time.sleep(3)
+      t = driver.find_element(By.XPATH ,'//*[@id="main-content"]/section[1]/div/div[1]/section[1]/div/div/section/div')
+      t_reverse=t.text[::-1]
+
+      if t_reverse[:9] =="erom wohs":
+        l = len(t.text)
+        strings=t.text[:l-9].split("\n")
+        strings[:] = [x for x in strings if x]
+        description.append(strings)
+      else:
+        strings=t.text.split("\n")
+        strings[:] = [x for x in strings if x]
+        description.append(strings)
+    df_ml = pd.DataFrame({'all_about_job' : description ,'company_logo':LOGO})
+
+    return df_ml
+
+  # apply desc. and logo function
+  E = all_description_LOGO(links)
+
+  # other info function
+  def other(urls):
+    frames =[]
+    for url in urls:
+      data1 = requests.get(url)
+      soup1 = BeautifulSoup(data1.content)
+      j =  soup1.find('ul' , {'class': 'description__job-criteria-list'})
+      time.sleep(4)
+      jj=j.find_all('h3')
+      dic ={}
+      for i in range(len(jj)):
+        dic[jj[i].text.replace('\n',' ').strip()] = j.find_all('span')[i].text.replace('\n',' ').strip()
+      output = pd.DataFrame()
+      output = output.append(dic, ignore_index=True) 
+      frames.append(output)
+    result = pd.concat(frames)
+    return result
+
+  # apply Other function
+  df = other(links)
+  df.fillna('Not_Found',inplace= True)
+  df.reset_index(inplace=True, drop=True)
+ 
+ # combine all together
+  result = pd.concat([df_ml,E, df ], axis=1)
+
+  return result
+
+
+##################### map_bubble #####################
+
+#### function to show map for loaction of the job
+
+
+
+def map_bubble(df):
+  
+  import requests
+  import urllib.parse
+  g =[]
+  for i  in range(len(df.Location)):
+
+    if  df.Location.loc[i].split(","):
+      g.append(df.Location.loc[i].split(",")[0])
+    else:
+      g.append(df.Location.loc[i])
+  df['new_loc']=g 
+  if 'country' in df.columns:
+    df["full_location"] = df["new_loc"] + ", " +df["country"]
+    dict_cities = dict(df.full_location.value_counts())
+  else :
+    dict_cities = dict(df.new_loc.value_counts())
+  lat = []
+  lon = []
+  bubble_df = pd.DataFrame()
+  add=[]
+  val=[]
+  try:
+    for address in dict_cities.keys():
+      url = 'https://nominatim.openstreetmap.org/search/' + urllib.parse.quote(address) +'?format=json'
+
+      response = requests.get(url).json()
+      lat.append(response[0]["lat"])
+      lon.append(response[0]["lon"])
+      add.append(address)
+      val.append(dict_cities[address])
+  except:
+    pass
+
+  bubble_df['address'] =add
+  bubble_df['lat'] = lat
+  bubble_df['lon'] = lon
+  bubble_df['value'] = val
+
+
+  # import the library
+  import folium
+
+  # Make an empty map
+  m = folium.Map(location=[20,0], tiles="OpenStreetMap", zoom_start=2)
+  # add marker one by one on the map
+  for i in range(0,len(bubble_df)):
+    folium.Circle(
+        location=[bubble_df.iloc[i]['lat'], bubble_df.iloc[i]['lon']],
+
+        popup=bubble_df.iloc[i][['address','value']].values,
+        radius=float(bubble_df.iloc[i]['value'])*500,
+        color='#69b3a2',
+        fill=True,
+        fill_color='#69b3a2'
+    ).add_to(m)
+  m
+  # Show the map again
+  return m
+
+
+##########################
+
+
+
+
+
+#########################
+#### wuzzuf analysis
+def wuzzuf_exp(df1):
+  top10_job_title = df1['Title'].value_counts()[:10]
+  fig1 = px.bar(y=top10_job_title.values, 
+              x=top10_job_title.index, 
+              color = top10_job_title.index,
+              color_discrete_sequence=px.colors.sequential.deep,
+              text=top10_job_title.values,
+              title= 'Top 10 Job Titles',
+              template= 'plotly_dark')
+  fig1.update_layout(height=500,width=500, 
+      xaxis_title="Job Titles",
+      yaxis_title="count",
+      font = dict(size=17,family="Franklin Gothic"))
+  st.plotly_chart(fig1)
+
+  type_grouped = df1['Career_Level'].value_counts()
+  #e_type = ['Full-Time','Part-Time','Contract','Freelance']
+  e_type =dict(df1['Career_Level'].value_counts()).keys()
+  fig2 = px.bar(x = e_type, y = type_grouped.values, 
+        color = type_grouped.index, 
+        color_discrete_sequence=px.colors.sequential.dense,
+        template = 'plotly_dark',
+        text = type_grouped.values, title = 'Career Level Distribution')
+  fig2.update_layout( height=500, width=500,
+      xaxis_title="Career Level",
+      yaxis_title="count",
+      font = dict(size=17,family="Franklin Gothic"))
+  fig2.update_traces(width=0.5)
+  st.plotly_chart(fig2)
+  residence = df1['Location'].value_counts()
+  top10_employee_location = residence[:10]
+  fig3 = px.bar(y=top10_employee_location.values, 
+              x=top10_employee_location.index, 
+              color = top10_employee_location.index,
+              color_discrete_sequence=px.colors.sequential.deep,
+              text=top10_employee_location.values,
+              title= 'Top 10 Location of job',
+              template= 'plotly_dark')
+  fig3.update_layout(height=500,width=500,
+      xaxis_title="Location of job",
+      yaxis_title="count",
+      font = dict(size=17,family="Franklin Gothic"))
+  st.plotly_chart(fig3)
+    
+  type_grouped = df1['Experience_Needed'].value_counts()
+  #e_type = ['Full-Time','Part-Time','Contract','Freelance']
+  e_type =dict(df1['Experience_Needed'].value_counts()).keys()
+  fig4 = px.bar(x = e_type, y = type_grouped.values, 
+        color = type_grouped.index, 
+        color_discrete_sequence=px.colors.sequential.dense,
+        template = 'plotly_dark',
+        text = type_grouped.values, title = ' Experience Level Distribution')
+  fig4.update_layout(height=500,width=500,
+      xaxis_title=" Experience Level (years)",
+      yaxis_title="count",
+      font = dict(size=17,family="Franklin Gothic"))
+  fig4.update_traces(width=0.5)
+  st.plotly_chart(fig4)
+  return 
+
+
+
+#########################
+### linkedin analysis
+
+def linkedin_exp(df1):
+  top10_job_title = df1['Title'].value_counts()[:10]
+  fig1 = px.bar(y=top10_job_title.values, 
+              x=top10_job_title.index, 
+              color = top10_job_title.index,
+              color_discrete_sequence=px.colors.sequential.deep,
+              text=top10_job_title.values,
+              title= 'Top 10 Job Titles',
+              template= 'plotly_dark')
+  fig1.update_layout(height=500,width=500, 
+      xaxis_title="Job Titles",
+      yaxis_title="count",
+      font = dict(size=17,family="Franklin Gothic"))
+  st.plotly_chart(fig1)
+
+  type_grouped = df1['Employment type'].value_counts()
+  #e_type = ['Full-Time','Part-Time','Contract','Freelance']
+  e_type =dict(df1['Employment type'].value_counts()).keys()
+  fig2 = px.bar(x = e_type, y = type_grouped.values, 
+        color = type_grouped.index, 
+        color_discrete_sequence=px.colors.sequential.dense,
+        template = 'plotly_dark',
+        text = type_grouped.values, title = 'Employment type Distribution')
+  fig2.update_layout( height=500, width=500,
+      xaxis_title="Employment type",
+      yaxis_title="count",
+      font = dict(size=17,family="Franklin Gothic"))
+  fig2.update_traces(width=0.5)
+  st.plotly_chart(fig2)
+  residence = df1['Location'].value_counts()
+  top10_employee_location = residence[:10]
+  fig3 = px.bar(y=top10_employee_location.values, 
+              x=top10_employee_location.index, 
+              color = top10_employee_location.index,
+              color_discrete_sequence=px.colors.sequential.deep,
+              text=top10_employee_location.values,
+              title= 'Top 10 Location of job',
+              template= 'plotly_dark')
+  fig3.update_layout(height=500,width=500,
+      xaxis_title="Location of job",
+      yaxis_title="count",
+      font = dict(size=17,family="Franklin Gothic"))
+  st.plotly_chart(fig3)
+    
+  type_grouped = df1['Seniority level'].value_counts()
+  #e_type = ['Full-Time','Part-Time','Contract','Freelance']
+  e_type =dict(df1['Seniority level'].value_counts()).keys()
+  fig4 = px.bar(x = e_type, y = type_grouped.values, 
+        color = type_grouped.index, 
+        color_discrete_sequence=px.colors.sequential.dense,
+        template = 'plotly_dark',
+        text = type_grouped.values, title = 'Seniority level Distribution')
+  fig4.update_layout(height=500,width=500,
+      xaxis_title="Seniority level",
+      yaxis_title="count",
+      font = dict(size=17,family="Franklin Gothic"))
+  fig4.update_traces(width=0.5)
+  st.plotly_chart(fig4)
+  return 
+
+
+########################
+
+####################### stream lit app ################################
+
+#site = ""
+#job =""
+#num_jobs = 0
+
+st.set_page_config(page_title="My Web_Scrap Page", page_icon=":tada:", layout="wide")
+
+
+# ---- HEADER SECTION ----
+with st.container():
+    left_column, right_column = st.columns(2)
+    with left_column:
+        st.subheader("Hi! I am Yassmen :wave:")
+        st.title("An Electronics and Communcation Engineer")
+        st.write(
+            "In this app we will scrap jobs from LinkedIn and Wuzzuf websites, let's get it started :boom:"
+        )
+        st.write("[Reach me >](https://www.linkedin.com/in/yassmen-youssef-48439a166/)")
+    with right_column:
+        pass
+       # st_lottie(lottie_coding, height=300, key="coding")
+
+
+
+import streamlit as st
+from streamlit_option_menu import option_menu
+
+#with st.sidebar:
+   # selected = option_menu("Main Menu", ["select website", 'search job','numbers of jobs'], icons=['linkedin', 'search','123'], menu_icon="cast", default_index=1)
+    
+webs =["Wuzzuf","Linkedin"]
+jobs =["Machine Learning","AI Engineer","Data Analysis","Software Testing"]
+nums = np.arange(1,1000)
+
+#with st.sidebar:
+  #if selected == "select website":
+site = st.sidebar.selectbox("select one website", webs)
+  #elif selected == "search job":
+job = st.sidebar.selectbox("select one job", jobs)
+  #elif selected == "numbers of jobs":
+num_jobs = st.sidebar.selectbox("select num of jobs you want to scrap", nums)
+
+
+
+import streamlit.components.v1 as components
+
+import hydralit_components as hc
+n2 = pd.DataFrame()
+
+if st.sidebar.button('Start Scrapping'):
+  if site =="Wuzzuf":
+
+    with st.container():
+        st.write("---")
+        tab1, tab2 ,tab3= st.tabs([" Data", " Bubble Map","Data Exploration"])
+        with tab1 :
+          with hc.HyLoader('✨Now loading' ,hc.Loaders.standard_loaders,index=[3,0,5]):
+            time.sleep(5)
+            n1 = Wuzzuf_scrapping(job ,num_jobs )
+            try:
+              tab1.dataframe(n1)
+            except:
+              try:
+                tab1.write(n1.astype(str).set_index(n1.index.astype(str)))  # Success
+              except:
+                tab1.table(n1)
+        with tab2:
+          map_bubble(n1)
+        with tab3:
+          #tab3.plotly_chart(wuzzuf_exp(n1))
+          wuzzuf_exp(n1)
+
+
+  if site =="Linkedin":
+    with st.container():
+        st.write("---")
+        tab1, tab2 ,tab3= st.tabs([" Data", " Bubble Map","Data Exploration"])
+        with tab1 :
+          with hc.HyLoader('✨Now loading' ,hc.Loaders.standard_loaders,index=[3,0,5]):
+            time.sleep(5)
+            n1 = LINKEDIN_Scrapping(job ,num_jobs )
+            try:
+              tab1.dataframe(n1)
+            except:
+              try:
+                tab1.write(n1.astype(str).set_index(n1.index.astype(str)))  # Success
+              except:
+                tab1.table(n1)
+        with tab2:
+          map_bubble(n1)
+        with tab3:
+          linkedin_exp(n1)  # WILL CHANGE