calendar-bot/scraping.py

57 lines
1.9 KiB
Python

from typing import List
from bs4 import BeautifulSoup
import requests
from operator import add
from functools import reduce
import datetime
from event import Event
def get_soup() -> BeautifulSoup:
r = requests.get("http://www.pvv.ntnu.no/hendelser/")
soup = BeautifulSoup(r.text, "html.parser")
return soup
def process_soup(soup: BeautifulSoup) -> List[Event]:
zips = []
events = soup.find_all("ul", "events")
for event in events:
times, locations, organizers = zip(*(list(
map(lambda x: map(lambda y: y.find("strong").text, x),
filter(lambda x: x != [],
map(lambda x: x.find_all("li"),
event.find_all("li")))))))
times = list(map(lambda x: datetime.datetime.strptime(x, "%A %d. %b %H.%M").replace(year=datetime.datetime.now().year), times))
names = list(map(lambda x: x[0].text,
filter(lambda x: x != [],
map(lambda x: x.find_all("a"),
event.find_all("li")))))
descriptions = list(map(lambda x: x.text,
filter(lambda x: x is not None,
map(lambda x: x.find("p"),
event.find_all("li")))))
ids = []
for a in event.find_all("a", href=True):
if a['href'][:10] == '/hendelser':
ids.append(int(a['href'][a['href'].find("=")+1:]))
zips.append(list(zip(ids, times, names, organizers, locations, descriptions)))
events = reduce(add, zips)
events = list(map(lambda x: Event(*x), events))
return events
def get_events_today(events: List[Event]) -> List[Event]:
return list(filter(lambda e: e.start.date() == datetime.datetime.today().date(), events))
if __name__ == "__main__":
print(get_events_today(process_soup(get_soup())))