Upload data_prep.py

data_prep.py

@@ -82,6 +82,89 @@ def extract_div_contents_from_url(url):
     return df
 
 
+import requests
+import pandas as pd
+from bs4 import BeautifulSoup
+
+def extract_div_contents_from_url_new(url, date):
+    response = requests.get(url)
+    if response.status_code != 200:
+        print(f"Error: Received status code {response.status_code} for URL: {url}")
+        return pd.DataFrame(columns=['date', 'title', 'text_url', 'deletion_discussion', 'label', 'confirmation', 'discussion', 'verdict'])
+
+    soup = BeautifulSoup(response.content, 'html.parser')
+    div_classes = ["mw-heading mw-heading3"]
+    divs = []
+
+    for div_class in div_classes:
+        divs.extend(soup.find_all('div', class_=div_class))
+
+    url_fragment = url.split('#')[-1].replace('_', ' ')
+    log_date = url.split('/')[-1]
+
+    data = []
+    for i, div in enumerate(divs):
+        try:
+            title = None
+            text_url = None
+            title_tag = div.find('a')
+            if title_tag:
+                title_span = div.find('span', {'data-mw-comment-start': True})
+                if title_span:
+                    title_anchor = title_span.find_next_sibling('a')
+                    if title_anchor:
+                        title = title_anchor.text
+                        text_url = 'https://en.wikipedia.org' + title_anchor['href']
+                else:
+                    title = title_tag.text
+                    text_url = 'https://en.wikipedia.org' + title_tag['href']
+
+            if title == 'talk page' or title is None:
+                heading_tag = div.find('div', class_='mw-heading mw-heading3')
+                if heading_tag:
+                    title_tag = heading_tag.find('a')
+                    if title_tag:
+                        title = title_tag.text
+                        text_url = 'https://en.wikipedia.org' + title_tag['href']
+
+            if not title:
+                continue
+            if title.lower() != url_fragment.lower():
+                continue  
+
+            next_div = div.find_next('div', class_='mw-heading mw-heading3')
+            deletion_discussion = ''
+            sibling = div.find_next_sibling()
+            while sibling and sibling != next_div:
+                deletion_discussion += str(sibling)
+                sibling = sibling.find_next_sibling()
+
+            label = ''
+            verdict_tag = div.find('p')
+            if verdict_tag:
+                label_b_tag = verdict_tag.find('b')
+                if label_b_tag:
+                    label = label_b_tag.text.strip()
+            confirmation = ''
+            discussion_tag = div.find('dd')
+            if discussion_tag:
+                discussion_tag_i = discussion_tag.find('i')
+                if discussion_tag_i:
+                    confirmation_b_tag = discussion_tag_i.find('b')
+                    if confirmation_b_tag:
+                        confirmation = confirmation_b_tag.text.strip()
+            parts = deletion_discussion.split('<div class="mw-heading mw-heading3">')
+            discussion = parts[-1] if len(parts) > 0 else ''
+            verdict = '<div class="mw-heading mw-heading3">' + parts[1] if len(parts) > 1 else ''
+
+            data.append([date, title, text_url, deletion_discussion, label, confirmation, verdict, discussion])
+        except Exception as e:
+            print(f"Error processing div: {e}")
+            continue
+
+    df = pd.DataFrame(data, columns=['date', 'title', 'text_url', 'deletion_discussion', 'label', 'confirmation', 'discussion', 'verdict'])
+    return df
+
 def extract_post_links_text(discussion_html):
     split_point = '<span class="plainlinks">'
     if split_point in discussion_html:
@@ -123,6 +206,9 @@ def process_split_text_into_sentences(df):
 
 def process_data(url):
     df = extract_div_contents_from_url(url)
+    #check if df is empty
+    if df.empty:
+       df = extract_div_contents_from_url_new(url)
     df = process_discussion(df)
     df = process_html_to_plaintext(df)
     df = process_split_text_into_sentences(df)