Spaces:
Sleeping
Sleeping
File size: 5,110 Bytes
3c77d98 7ba3a06 3c77d98 7ba3a06 3c77d98 7ba3a06 3c77d98 7ba3a06 3c77d98 7ba3a06 3c77d98 7ba3a06 3c77d98 7ba3a06 3c77d98 7ba3a06 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 |
import requests
import pandas as pd
from bs4 import BeautifulSoup
def extract_div_contents_from_url(url):
response = requests.get(url)
if response.status_code != 200:
print(f"Error: Received status code {response.status_code} for URL: {url}")
return pd.DataFrame(columns=['title', 'text_url', 'deletion_discussion', 'label', 'confirmation', 'discussion', 'verdict'])
soup = BeautifulSoup(response.content, 'html.parser')
div_classes = ['boilerplate afd vfd xfd-closed', 'boilerplate afd vfd xfd-closed archived mw-archivedtalk']
divs = []
for div_class in div_classes:
divs.extend(soup.find_all('div', class_=div_class))
url_fragment = url.split('#')[-1].replace('_', ' ')
data = []
for div in divs:
try:
title = None
text_url = None
# Extract title and text_url
title_tag = div.find('a')
if title_tag:
title_span = div.find('span', {'data-mw-comment-start': True})
if title_span:
title_anchor = title_span.find_next_sibling('a')
if title_anchor:
title = title_anchor.text
text_url = 'https://en.wikipedia.org' + title_anchor['href']
else:
title = title_tag.text
text_url = 'https://en.wikipedia.org' + title_tag['href']
if title == 'talk page' or title is None:
heading_tag = div.find('div', class_='mw-heading mw-heading3')
if heading_tag:
title_tag = heading_tag.find('a')
if title_tag:
title = title_tag.text
text_url = 'https://en.wikipedia.org' + title_tag['href']
if not title:
continue
if title.lower() != url_fragment.lower():
continue
deletion_discussion = div.prettify()
# Extract label
label = ''
verdict_tag = div.find('p')
if verdict_tag:
label_b_tag = verdict_tag.find('b')
if label_b_tag:
label = label_b_tag.text.strip()
# Extract confirmation
confirmation = ''
discussion_tag = div.find('dd')
if discussion_tag:
discussion_tag_i = discussion_tag.find('i')
if discussion_tag_i:
confirmation_b_tag = discussion_tag_i.find('b')
if confirmation_b_tag:
confirmation = confirmation_b_tag.text.strip()
# Split deletion_discussion into discussion and verdict
parts = deletion_discussion.split('<div class="mw-heading mw-heading3">')
discussion = parts[0] if len(parts) > 0 else ''
verdict = '<div class="mw-heading mw-heading3">' + parts[1] if len(parts) > 1 else ''
data.append([title, text_url, deletion_discussion, label, confirmation, verdict, discussion])
except Exception as e:
print(f"Error processing div: {e}")
continue
df = pd.DataFrame(data, columns=['title', 'text_url', 'deletion_discussion', 'label', 'confirmation', 'verdict', 'discussion'])
df = df[['title', 'discussion', 'verdict', 'label']]
print(f"DataFrame created with {len(df)} rows")
return df
def extract_post_links_text(discussion_html):
split_point = '<span class="plainlinks">'
if split_point in discussion_html:
parts = discussion_html.split(split_point)
if len(parts) > 1:
return parts[1]
return discussion_html
def process_discussion(df):
df['discussion_cleaned'] = df['verdict'].apply(extract_post_links_text)
return df
def html_to_plaintext(html_content):
soup = BeautifulSoup(html_content, 'html.parser')
for tag in soup.find_all(['p', 'li', 'dd', 'dl']):
tag.insert_before('\n')
tag.insert_after('\n')
for br in soup.find_all('br'):
br.replace_with('\n')
text = soup.get_text(separator=' ', strip=True)
text = '\n'.join([line.strip() for line in text.splitlines() if line.strip() != ''])
return text
def process_html_to_plaintext(df):
df['discussion_cleaned'] = df['discussion_cleaned'].apply(html_to_plaintext)
df = df[['title', 'discussion_cleaned', 'label']]
return df
import pysbd
def split_text_into_sentences(text):
seg = pysbd.Segmenter(language="en", clean=False)
sentences = seg.segment(text)
return ' '.join(sentences[1:])
def process_split_text_into_sentences(df):
df['discussion_cleaned'] = df['discussion_cleaned'].apply(split_text_into_sentences)
return df
def process_data(url):
df = extract_div_contents_from_url(url)
df = process_discussion(df)
df = process_html_to_plaintext(df)
df = process_split_text_into_sentences(df)
#if not empty
if not df.empty:
return df.at[0,'title']+ ' : '+df.at[0, 'discussion_cleaned']
else:
return 'Empty DataFrame'
|