diff options
| -rw-r--r-- | .travis.yml | 12 | ||||
| -rw-r--r-- | migrations.py | 65 | ||||
| -rw-r--r-- | poller/__init__.py | 202 | ||||
| -rw-r--r-- | poller/db.py | 67 | ||||
| -rw-r--r-- | poller/dedup.py | 71 |
5 files changed, 0 insertions, 417 deletions
diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 9935a5f..0000000 --- a/.travis.yml +++ /dev/null @@ -1,12 +0,0 @@ -language: python -python: - - "3.8" -services: - - "docker" - -install: - - "pip install -r requirements.txt" -script: - - "pylint poller" - - "docker build -t poller ." - diff --git a/migrations.py b/migrations.py deleted file mode 100644 index 5f692cb..0000000 --- a/migrations.py +++ /dev/null @@ -1,65 +0,0 @@ -import os -import json -import sqlite3 - -from poller.db import create_tables, get_latest_from_db - -LATEST_DATA={} - -def update_db(): - db_conn = sqlite3.connect('./data/data.sqlite3') - c = db_conn.cursor() - sql = f'INSERT INTO `alertlevel` VALUES (\'{LATEST_DATA["last_updated"]}\', \'{LATEST_DATA["alert_level"]}\');' - c.execute(sql) - sql = f'INSERT INTO `total` VALUES (\'{LATEST_DATA["last_updated"]}\', {LATEST_DATA["total_students"]}, {LATEST_DATA["total_staff"]});' - c.execute(sql) - sql = f'INSERT INTO `new` VALUES (\'{LATEST_DATA["last_updated"]}\', {LATEST_DATA["new_students"]}, {LATEST_DATA["new_staff"]});' - c.execute(sql) - sql = f'INSERT INTO `quarantine` VALUES (\'{LATEST_DATA["last_updated"]}\', {LATEST_DATA["quarantine_on_campus"]}, {LATEST_DATA["quarantine_off_campus"]});' - c.execute(sql) - sql = f'INSERT INTO `isolation` VALUES (\'{LATEST_DATA["last_updated"]}\', {LATEST_DATA["isolation_on_campus"]}, {LATEST_DATA["isolation_off_campus"]});' - c.execute(sql) - sql = f'INSERT INTO `beds` VALUES (\'{LATEST_DATA["last_updated"]}\', {LATEST_DATA["beds_available"]});' - c.execute(sql) - sql = f'INSERT INTO `tests` VALUES (\'{LATEST_DATA["last_updated"]}\', {LATEST_DATA["tests_administered"]});' - c.execute(sql) - db_conn.commit() - db_conn.close() - -def db_is_same(): - global LATEST_DATA - latest_data = get_latest_from_db() - if latest_data is None or LATEST_DATA is None: - return False - for key in list(latest_data.keys()): - if key != 'last_updated' and LATEST_DATA[key] != latest_data[key]: - return False - return True - -if not os.path.exists('./data'): - os.mkdir('./data') - -create_tables() - -with open('history/history.json', 'r') as fd: - print('importing data...') - data = json.loads(fd.read()) - for day in data: - print(day) - LATEST_DATA = { - 'alert_level': str(day['alert_level']), - 'total_students': int(day['total_students']), - 'total_staff': int(day['total_staff']), - 'new_students': int(day['new_students']), - 'new_staff': int(day['new_staff']), - 'quarantine_on_campus': int(day['quarantine_on_campus']), - 'quarantine_off_campus': int(day['quarantine_off_campus']), - 'isolation_on_campus': int(day['isolation_on_campus']), - 'isolation_off_campus': int(day['isolation_off_campus']), - 'beds_available': int(day['beds_available']), - 'tests_administered': int(day['tests_administered']), - 'last_updated': day['last_updated'] - } - update_db() - print('data imported!') - diff --git a/poller/__init__.py b/poller/__init__.py deleted file mode 100644 index ebb2653..0000000 --- a/poller/__init__.py +++ /dev/null @@ -1,202 +0,0 @@ -""" A small flask Hello World """ - -import os -import threading -import sqlite3 -import atexit -import datetime -import json - -from flask import Flask, jsonify, request, make_response -import requests -from bs4 import BeautifulSoup - -from .dedup import dedup -from .db import create_tables, get_all_from_db, get_latest_from_db - -POOL_TIME = 5 * 60 # Seconds -DASHBOARD_URL = 'https://rit.edu/ready/spring-dashboard' -LATEST_DATA = None -data_thread = threading.Thread() -db_lock = threading.Lock() - -if not os.path.exists('./data'): - os.mkdir('./data') - -def update_db(): - print('updating db') - with db_lock: - db_conn = sqlite3.connect('data/data.sqlite3') - c = db_conn.cursor() - sql = f'INSERT INTO `alertlevel` VALUES (datetime(\'now\'), \'{LATEST_DATA["alert_level"]}\');' - c.execute(sql) - sql = f'INSERT INTO `total` VALUES (datetime(\'now\'), {LATEST_DATA["total_students"]}, {LATEST_DATA["total_staff"]});' - c.execute(sql) - sql = f'INSERT INTO `new` VALUES (datetime(\'now\'), {LATEST_DATA["new_students"]}, {LATEST_DATA["new_staff"]});' - c.execute(sql) - sql = f'INSERT INTO `quarantine` VALUES (datetime(\'now\'), {LATEST_DATA["quarantine_on_campus"]}, {LATEST_DATA["quarantine_off_campus"]});' - c.execute(sql) - sql = f'INSERT INTO `isolation` VALUES (datetime(\'now\'), {LATEST_DATA["isolation_on_campus"]}, {LATEST_DATA["isolation_off_campus"]});' - c.execute(sql) - sql = f'INSERT INTO `beds` VALUES (datetime(\'now\'), {LATEST_DATA["beds_available"]});' - c.execute(sql) - sql = f'INSERT INTO `tests` VALUES (datetime(\'now\'), {LATEST_DATA["tests_administered"]});' - c.execute(sql) - db_conn.commit() - db_conn.close() - dedup() - -def data_is_same(current_data): - global LATEST_DATA - if LATEST_DATA is None or current_data is None: - return False - for key in list(LATEST_DATA.keys()): - if key != 'last_updated' and current_data[key] != LATEST_DATA[key]: - return False - return True - -def db_is_same(current_data): - latest_data = get_latest_from_db() - if latest_data is None or current_data is None: - return False - for key in list(latest_data.keys()): - if key != 'last_updated' and current_data[key] != latest_data[key]: - return False - return True - -def get_data(): - print('fetching data') - global data_thread - data_thread = threading.Timer(POOL_TIME, get_data, ()) - data_thread.start() - create_tables() - page = requests.get(DASHBOARD_URL, headers={'Cache-Control': 'no-cache'}) - soup = BeautifulSoup(page.content, 'html.parser') - total_students = int(soup.find('div', attrs={'class': 'statistic-13872'}).find_all("p", attrs={'class': 'card-header'})[0].text.strip()) - total_staff = int(soup.find('div', attrs={'class': 'statistic-13875'}).find_all("p", attrs={'class': 'card-header'})[0].text.strip()) - new_students = int(soup.find('div', attrs={'class': 'statistic-14332'}).find_all("p", attrs={'class': 'card-header'})[0].text.strip()) - new_staff = int(soup.find('div', attrs={'class': 'statistic-14335'}).find_all("p", attrs={'class': 'card-header'})[0].text.strip()) - quarantine_on_campus = int(soup.find('div', attrs={'class': 'statistic-13893'}).find_all("p", attrs={'class': 'card-header'})[0].text.strip()) - quarantine_off_campus = int(soup.find('div', attrs={'class': 'statistic-13896'}).find_all("p", attrs={'class': 'card-header'})[0].text.strip()) - isolation_on_campus = int(soup.find('div', attrs={'class': 'statistic-13905'}).find_all("p", attrs={'class': 'card-header'})[0].text.strip()) - isolation_off_campus = int(soup.find('div', attrs={'class': 'statistic-13908'}).find_all("p", attrs={'class': 'card-header'})[0].text.strip()) - beds_available = int(soup.find('div', attrs={'class': 'statistic-13935'}).find_all("p", attrs={'class': 'card-header'})[0].text.strip().strip('%')) - tests_administered = int(soup.find('div', attrs={'class': 'statistic-13923'}).find_all("p", attrs={'class': 'card-header'})[0].text.strip().replace("*", " ").replace(",", "")) - container = soup.find('div', attrs={'id': 'pandemic-message-container'}) - alert_level = container.find("a").text - color = "" - if "Green" in alert_level: - color = 'green' - elif "Yellow" in alert_level: - color = 'yellow' - elif "Orange" in alert_level: - color = 'orange' - elif "Red" in alert_level: - color = 'red' - global LATEST_DATA - - fall_data = None - with open('history/fall-2020.json', 'r') as fd: - fall_data = json.loads(fd.read()) - current_data = { - 'alert_level': color, - 'total_students': total_students + fall_data['total_students'], - 'total_staff': total_staff + fall_data['total_staff'], - 'new_students': new_students, - 'new_staff': new_staff, - 'quarantine_on_campus': quarantine_on_campus, - 'quarantine_off_campus': quarantine_off_campus, - 'isolation_on_campus': isolation_on_campus, - 'isolation_off_campus': isolation_off_campus, - 'beds_available': beds_available, - 'tests_administered': tests_administered + fall_data['tests_administered'], - 'last_updated': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') - } - LATEST_DATA = current_data - if not db_is_same(current_data): - update_db() - return current_data - -get_data() - -APP = Flask(__name__) - -# Load file based configuration overrides if present -if os.path.exists(os.path.join(os.getcwd(), 'config.py')): - APP.config.from_pyfile(os.path.join(os.getcwd(), 'config.py')) -else: - APP.config.from_pyfile(os.path.join(os.getcwd(), 'config.env.py')) - -APP.secret_key = APP.config['SECRET_KEY'] - -@APP.route('/api/v0/latest') -def _api_v0_latest(): - return jsonify(LATEST_DATA) - -@APP.route('/api/v0/latestdb') -def _api_v0_latestdb(): - data = get_latest_from_db() - return jsonify(data) - -@APP.route('/api/v0/history') -def _api_v0_history(): - data = get_all_from_db() - return jsonify(data) - -@APP.route('/api/v0/difference') -def _api_v0_difference(): - data = get_all_from_db() - latest = data[-1] - prev = data[-2] - data = { - 'alert_level': f'{prev["alert_level"]} -> {latest["alert_level"]}', - 'total_students': latest["total_students"] - prev["total_students"], - 'total_staff': latest["total_staff"] - prev["total_staff"], - 'new_students': latest["new_students"] - prev["new_students"], - 'new_staff': latest["new_staff"] - prev["new_staff"], - 'quarantine_on_campus': latest["quarantine_on_campus"] - prev["quarantine_on_campus"], - 'quarantine_off_campus': latest["quarantine_off_campus"] - prev["quarantine_off_campus"], - 'isolation_on_campus': latest["isolation_on_campus"] - prev["isolation_on_campus"], - 'isolation_off_campus': latest["isolation_off_campus"] - prev["isolation_off_campus"], - 'beds_available': latest["beds_available"] - prev["beds_available"], - 'tests_administered': latest["tests_administered"] - prev["tests_administered"], - } - return jsonify(data) - -@APP.route('/api/v0/diff') -def _api_v0_diff(): - first = request.args.get('first') - last = request.args.get('last') - data = get_all_from_db() - if first is None: - first = 0 - else: - try: - first = int(first) - except: - first = 0 - if last is None: - last = len(data) - 1 - else: - try: - last = int(last) - except: - last = len(data) - 1 - latest = data[last] - prev = data[first] - data = { - 'alert_level': f'{prev["alert_level"]} -> {latest["alert_level"]}', - 'total_students': latest["total_students"] - prev["total_students"], - 'total_staff': latest["total_staff"] - prev["total_staff"], - 'new_students': latest["new_students"] - prev["new_students"], - 'new_staff': latest["new_staff"] - prev["new_staff"], - 'quarantine_on_campus': latest["quarantine_on_campus"] - prev["quarantine_on_campus"], - 'quarantine_off_campus': latest["quarantine_off_campus"] - prev["quarantine_off_campus"], - 'isolation_on_campus': latest["isolation_on_campus"] - prev["isolation_on_campus"], - 'isolation_off_campus': latest["isolation_off_campus"] - prev["isolation_off_campus"], - 'beds_available': latest["beds_available"] - prev["beds_available"], - 'tests_administered': latest["tests_administered"] - prev["tests_administered"], - 'description': f'day {first} to {last}' - } - return jsonify(data) - diff --git a/poller/db.py b/poller/db.py deleted file mode 100644 index d3836a7..0000000 --- a/poller/db.py +++ /dev/null @@ -1,67 +0,0 @@ -import threading -import sqlite3 - -db_lock = threading.Lock() - -def create_tables(): - with db_lock: - db_conn = sqlite3.connect('data/data.sqlite3') - c = db_conn.cursor() - sql = f'CREATE TABLE IF NOT EXISTS `alertlevel` (time DATETIME PRIMARY KEY NOT NULL, color CHAR(50) NOT NULL);' - c.execute(sql) - sql = f'CREATE TABLE IF NOT EXISTS `total` (time DATETIME PRIMARY KEY NOT NULL, total_students INT NOT NULL, total_staff INT NOT NULL);' - c.execute(sql) - sql = f'CREATE TABLE IF NOT EXISTS `new` (time DATETIME PRIMARY KEY NOT NULL, new_students INT NOT NULL, new_staff INT NOT NULL);' - c.execute(sql) - sql = f'CREATE TABLE IF NOT EXISTS `quarantine` (time DATETIME PRIMARY KEY NOT NULL, quarantine_on_campus INT NOT NULL, quarantine_off_campus INT NOT NULL);' - c.execute(sql) - sql = f'CREATE TABLE IF NOT EXISTS `isolation` (time DATETIME PRIMARY KEY NOT NULL, isolation_on_campus INT NOT NULL, isolation_off_campus INT NOT NULL);' - c.execute(sql) - sql = f'CREATE TABLE IF NOT EXISTS `beds` (time DATETIME PRIMARY KEY NOT NULL, beds_available INT NOT NULL);' - c.execute(sql) - sql = f'CREATE TABLE IF NOT EXISTS `tests` (time DATETIME PRIMARY KEY NOT NULL, tests_administered INT NOT NULL);' - c.execute(sql) - db_conn.commit() - db_conn.close() - - -def get_latest_from_db(): - return get_all_from_db()[-1] - -def get_all_from_db(): - with db_lock: - db_conn = sqlite3.connect('data/data.sqlite3') - c = db_conn.cursor() - sql = 'SELECT alertlevel.time, alertlevel.color, total.total_students, total.total_staff, new.new_students, new.new_staff, ' + \ - 'quarantine.quarantine_on_campus, quarantine.quarantine_off_campus, isolation.isolation_on_campus, isolation.isolation_off_campus, ' + \ - 'beds.beds_available, tests.tests_administered ' + \ - 'FROM `alertlevel` ' + \ - 'INNER JOIN `total` ' + \ - 'ON alertlevel.time = total.time ' + \ - 'INNER JOIN `new` ' + \ - 'ON alertlevel.time = new.time ' + \ - 'INNER JOIN `quarantine` ' + \ - 'ON alertlevel.time = quarantine.time ' + \ - 'INNER JOIN `isolation` ' + \ - 'ON alertlevel.time = isolation.time ' + \ - 'INNER JOIN `beds` ' + \ - 'ON alertlevel.time = beds.time ' + \ - 'INNER JOIN `tests` ' + \ - 'ON alertlevel.time = tests.time' - c.execute(sql) - - data = [{ - 'alert_level': d[1], - 'total_students': d[2], - 'total_staff': d[3], - 'new_students': d[4], - 'new_staff': d[5], - 'quarantine_on_campus': d[6], - 'quarantine_off_campus': d[7], - 'isolation_on_campus': d[8], - 'isolation_off_campus': d[9], - 'beds_available': d[10], - 'tests_administered': d[11], - 'last_updated': d[0] - } for d in c.fetchall()] - return data diff --git a/poller/dedup.py b/poller/dedup.py deleted file mode 100644 index ba68be8..0000000 --- a/poller/dedup.py +++ /dev/null @@ -1,71 +0,0 @@ -import sqlite3 - -def get_all_from_db(): - db_conn = sqlite3.connect('data/data.sqlite3') - c = db_conn.cursor() - sql = 'SELECT alertlevel.time, alertlevel.color, total.total_students, total.total_staff, new.new_students, new.new_staff, ' + \ - 'quarantine.quarantine_on_campus, quarantine.quarantine_off_campus, isolation.isolation_on_campus, isolation.isolation_off_campus, ' + \ - 'beds.beds_available, tests.tests_administered ' + \ - 'FROM `alertlevel` ' + \ - 'INNER JOIN `total` ' + \ - 'ON alertlevel.time = total.time ' + \ - 'INNER JOIN `new` ' + \ - 'ON alertlevel.time = new.time ' + \ - 'INNER JOIN `quarantine` ' + \ - 'ON alertlevel.time = quarantine.time ' + \ - 'INNER JOIN `isolation` ' + \ - 'ON alertlevel.time = isolation.time ' + \ - 'INNER JOIN `beds` ' + \ - 'ON alertlevel.time = beds.time ' + \ - 'INNER JOIN `tests` ' + \ - 'ON alertlevel.time = tests.time' - c.execute(sql) - - data = [{ - 'alert_level': d[1], - 'total_students': d[2], - 'total_staff': d[3], - 'new_students': d[4], - 'new_staff': d[5], - 'quarantine_on_campus': d[6], - 'quarantine_off_campus': d[7], - 'isolation_on_campus': d[8], - 'isolation_off_campus': d[9], - 'beds_available': d[10], - 'tests_administered': d[11], - 'last_updated': d[0] - } for d in c.fetchall()] - return data - - -def drop_by_date(date): - db_conn = sqlite3.connect('data/data.sqlite3') - c = db_conn.cursor() - sql = f'DELETE FROM `alertlevel` WHERE time=\'{date}\';' - c.execute(sql) - sql = f'DELETE FROM `total` WHERE time=\'{date}\';' - c.execute(sql) - sql = f'DELETE FROM `new` WHERE time=\'{date}\';' - c.execute(sql) - sql = f'DELETE FROM `quarantine` WHERE time=\'{date}\';' - c.execute(sql) - sql = f'DELETE FROM `isolation` WHERE time=\'{date}\';' - c.execute(sql) - sql = f'DELETE FROM `beds` WHERE time=\'{date}\';' - c.execute(sql) - sql = f'DELETE FROM `tests` WHERE time=\'{date}\';' - c.execute(sql) - db_conn.commit() - db_conn.close() - - -def dedup(): - data = get_all_from_db() - # get first date - starting_date = data[-1]['last_updated'].split(' ')[0] - for i in range(len(data)-2, 0, -1): - if data[i]['last_updated'].split(' ')[0] != starting_date: - starting_date = data[i]['last_updated'].split(' ')[0] - else: - drop_by_date(data[i]['last_updated']) - print('dropped ' + data[i]['last_updated']) |