aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--.travis.yml12
-rw-r--r--migrations.py65
-rw-r--r--poller/__init__.py202
-rw-r--r--poller/db.py67
-rw-r--r--poller/dedup.py71
5 files changed, 0 insertions, 417 deletions
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 9935a5f..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,12 +0,0 @@
-language: python
-python:
- - "3.8"
-services:
- - "docker"
-
-install:
- - "pip install -r requirements.txt"
-script:
- - "pylint poller"
- - "docker build -t poller ."
-
diff --git a/migrations.py b/migrations.py
deleted file mode 100644
index 5f692cb..0000000
--- a/migrations.py
+++ /dev/null
@@ -1,65 +0,0 @@
-import os
-import json
-import sqlite3
-
-from poller.db import create_tables, get_latest_from_db
-
-LATEST_DATA={}
-
-def update_db():
- db_conn = sqlite3.connect('./data/data.sqlite3')
- c = db_conn.cursor()
- sql = f'INSERT INTO `alertlevel` VALUES (\'{LATEST_DATA["last_updated"]}\', \'{LATEST_DATA["alert_level"]}\');'
- c.execute(sql)
- sql = f'INSERT INTO `total` VALUES (\'{LATEST_DATA["last_updated"]}\', {LATEST_DATA["total_students"]}, {LATEST_DATA["total_staff"]});'
- c.execute(sql)
- sql = f'INSERT INTO `new` VALUES (\'{LATEST_DATA["last_updated"]}\', {LATEST_DATA["new_students"]}, {LATEST_DATA["new_staff"]});'
- c.execute(sql)
- sql = f'INSERT INTO `quarantine` VALUES (\'{LATEST_DATA["last_updated"]}\', {LATEST_DATA["quarantine_on_campus"]}, {LATEST_DATA["quarantine_off_campus"]});'
- c.execute(sql)
- sql = f'INSERT INTO `isolation` VALUES (\'{LATEST_DATA["last_updated"]}\', {LATEST_DATA["isolation_on_campus"]}, {LATEST_DATA["isolation_off_campus"]});'
- c.execute(sql)
- sql = f'INSERT INTO `beds` VALUES (\'{LATEST_DATA["last_updated"]}\', {LATEST_DATA["beds_available"]});'
- c.execute(sql)
- sql = f'INSERT INTO `tests` VALUES (\'{LATEST_DATA["last_updated"]}\', {LATEST_DATA["tests_administered"]});'
- c.execute(sql)
- db_conn.commit()
- db_conn.close()
-
-def db_is_same():
- global LATEST_DATA
- latest_data = get_latest_from_db()
- if latest_data is None or LATEST_DATA is None:
- return False
- for key in list(latest_data.keys()):
- if key != 'last_updated' and LATEST_DATA[key] != latest_data[key]:
- return False
- return True
-
-if not os.path.exists('./data'):
- os.mkdir('./data')
-
-create_tables()
-
-with open('history/history.json', 'r') as fd:
- print('importing data...')
- data = json.loads(fd.read())
- for day in data:
- print(day)
- LATEST_DATA = {
- 'alert_level': str(day['alert_level']),
- 'total_students': int(day['total_students']),
- 'total_staff': int(day['total_staff']),
- 'new_students': int(day['new_students']),
- 'new_staff': int(day['new_staff']),
- 'quarantine_on_campus': int(day['quarantine_on_campus']),
- 'quarantine_off_campus': int(day['quarantine_off_campus']),
- 'isolation_on_campus': int(day['isolation_on_campus']),
- 'isolation_off_campus': int(day['isolation_off_campus']),
- 'beds_available': int(day['beds_available']),
- 'tests_administered': int(day['tests_administered']),
- 'last_updated': day['last_updated']
- }
- update_db()
- print('data imported!')
-
diff --git a/poller/__init__.py b/poller/__init__.py
deleted file mode 100644
index ebb2653..0000000
--- a/poller/__init__.py
+++ /dev/null
@@ -1,202 +0,0 @@
-""" A small flask Hello World """
-
-import os
-import threading
-import sqlite3
-import atexit
-import datetime
-import json
-
-from flask import Flask, jsonify, request, make_response
-import requests
-from bs4 import BeautifulSoup
-
-from .dedup import dedup
-from .db import create_tables, get_all_from_db, get_latest_from_db
-
-POOL_TIME = 5 * 60 # Seconds
-DASHBOARD_URL = 'https://rit.edu/ready/spring-dashboard'
-LATEST_DATA = None
-data_thread = threading.Thread()
-db_lock = threading.Lock()
-
-if not os.path.exists('./data'):
- os.mkdir('./data')
-
-def update_db():
- print('updating db')
- with db_lock:
- db_conn = sqlite3.connect('data/data.sqlite3')
- c = db_conn.cursor()
- sql = f'INSERT INTO `alertlevel` VALUES (datetime(\'now\'), \'{LATEST_DATA["alert_level"]}\');'
- c.execute(sql)
- sql = f'INSERT INTO `total` VALUES (datetime(\'now\'), {LATEST_DATA["total_students"]}, {LATEST_DATA["total_staff"]});'
- c.execute(sql)
- sql = f'INSERT INTO `new` VALUES (datetime(\'now\'), {LATEST_DATA["new_students"]}, {LATEST_DATA["new_staff"]});'
- c.execute(sql)
- sql = f'INSERT INTO `quarantine` VALUES (datetime(\'now\'), {LATEST_DATA["quarantine_on_campus"]}, {LATEST_DATA["quarantine_off_campus"]});'
- c.execute(sql)
- sql = f'INSERT INTO `isolation` VALUES (datetime(\'now\'), {LATEST_DATA["isolation_on_campus"]}, {LATEST_DATA["isolation_off_campus"]});'
- c.execute(sql)
- sql = f'INSERT INTO `beds` VALUES (datetime(\'now\'), {LATEST_DATA["beds_available"]});'
- c.execute(sql)
- sql = f'INSERT INTO `tests` VALUES (datetime(\'now\'), {LATEST_DATA["tests_administered"]});'
- c.execute(sql)
- db_conn.commit()
- db_conn.close()
- dedup()
-
-def data_is_same(current_data):
- global LATEST_DATA
- if LATEST_DATA is None or current_data is None:
- return False
- for key in list(LATEST_DATA.keys()):
- if key != 'last_updated' and current_data[key] != LATEST_DATA[key]:
- return False
- return True
-
-def db_is_same(current_data):
- latest_data = get_latest_from_db()
- if latest_data is None or current_data is None:
- return False
- for key in list(latest_data.keys()):
- if key != 'last_updated' and current_data[key] != latest_data[key]:
- return False
- return True
-
-def get_data():
- print('fetching data')
- global data_thread
- data_thread = threading.Timer(POOL_TIME, get_data, ())
- data_thread.start()
- create_tables()
- page = requests.get(DASHBOARD_URL, headers={'Cache-Control': 'no-cache'})
- soup = BeautifulSoup(page.content, 'html.parser')
- total_students = int(soup.find('div', attrs={'class': 'statistic-13872'}).find_all("p", attrs={'class': 'card-header'})[0].text.strip())
- total_staff = int(soup.find('div', attrs={'class': 'statistic-13875'}).find_all("p", attrs={'class': 'card-header'})[0].text.strip())
- new_students = int(soup.find('div', attrs={'class': 'statistic-14332'}).find_all("p", attrs={'class': 'card-header'})[0].text.strip())
- new_staff = int(soup.find('div', attrs={'class': 'statistic-14335'}).find_all("p", attrs={'class': 'card-header'})[0].text.strip())
- quarantine_on_campus = int(soup.find('div', attrs={'class': 'statistic-13893'}).find_all("p", attrs={'class': 'card-header'})[0].text.strip())
- quarantine_off_campus = int(soup.find('div', attrs={'class': 'statistic-13896'}).find_all("p", attrs={'class': 'card-header'})[0].text.strip())
- isolation_on_campus = int(soup.find('div', attrs={'class': 'statistic-13905'}).find_all("p", attrs={'class': 'card-header'})[0].text.strip())
- isolation_off_campus = int(soup.find('div', attrs={'class': 'statistic-13908'}).find_all("p", attrs={'class': 'card-header'})[0].text.strip())
- beds_available = int(soup.find('div', attrs={'class': 'statistic-13935'}).find_all("p", attrs={'class': 'card-header'})[0].text.strip().strip('%'))
- tests_administered = int(soup.find('div', attrs={'class': 'statistic-13923'}).find_all("p", attrs={'class': 'card-header'})[0].text.strip().replace("*", " ").replace(",", ""))
- container = soup.find('div', attrs={'id': 'pandemic-message-container'})
- alert_level = container.find("a").text
- color = ""
- if "Green" in alert_level:
- color = 'green'
- elif "Yellow" in alert_level:
- color = 'yellow'
- elif "Orange" in alert_level:
- color = 'orange'
- elif "Red" in alert_level:
- color = 'red'
- global LATEST_DATA
-
- fall_data = None
- with open('history/fall-2020.json', 'r') as fd:
- fall_data = json.loads(fd.read())
- current_data = {
- 'alert_level': color,
- 'total_students': total_students + fall_data['total_students'],
- 'total_staff': total_staff + fall_data['total_staff'],
- 'new_students': new_students,
- 'new_staff': new_staff,
- 'quarantine_on_campus': quarantine_on_campus,
- 'quarantine_off_campus': quarantine_off_campus,
- 'isolation_on_campus': isolation_on_campus,
- 'isolation_off_campus': isolation_off_campus,
- 'beds_available': beds_available,
- 'tests_administered': tests_administered + fall_data['tests_administered'],
- 'last_updated': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
- }
- LATEST_DATA = current_data
- if not db_is_same(current_data):
- update_db()
- return current_data
-
-get_data()
-
-APP = Flask(__name__)
-
-# Load file based configuration overrides if present
-if os.path.exists(os.path.join(os.getcwd(), 'config.py')):
- APP.config.from_pyfile(os.path.join(os.getcwd(), 'config.py'))
-else:
- APP.config.from_pyfile(os.path.join(os.getcwd(), 'config.env.py'))
-
-APP.secret_key = APP.config['SECRET_KEY']
-
-@APP.route('/api/v0/latest')
-def _api_v0_latest():
- return jsonify(LATEST_DATA)
-
-@APP.route('/api/v0/latestdb')
-def _api_v0_latestdb():
- data = get_latest_from_db()
- return jsonify(data)
-
-@APP.route('/api/v0/history')
-def _api_v0_history():
- data = get_all_from_db()
- return jsonify(data)
-
-@APP.route('/api/v0/difference')
-def _api_v0_difference():
- data = get_all_from_db()
- latest = data[-1]
- prev = data[-2]
- data = {
- 'alert_level': f'{prev["alert_level"]} -> {latest["alert_level"]}',
- 'total_students': latest["total_students"] - prev["total_students"],
- 'total_staff': latest["total_staff"] - prev["total_staff"],
- 'new_students': latest["new_students"] - prev["new_students"],
- 'new_staff': latest["new_staff"] - prev["new_staff"],
- 'quarantine_on_campus': latest["quarantine_on_campus"] - prev["quarantine_on_campus"],
- 'quarantine_off_campus': latest["quarantine_off_campus"] - prev["quarantine_off_campus"],
- 'isolation_on_campus': latest["isolation_on_campus"] - prev["isolation_on_campus"],
- 'isolation_off_campus': latest["isolation_off_campus"] - prev["isolation_off_campus"],
- 'beds_available': latest["beds_available"] - prev["beds_available"],
- 'tests_administered': latest["tests_administered"] - prev["tests_administered"],
- }
- return jsonify(data)
-
-@APP.route('/api/v0/diff')
-def _api_v0_diff():
- first = request.args.get('first')
- last = request.args.get('last')
- data = get_all_from_db()
- if first is None:
- first = 0
- else:
- try:
- first = int(first)
- except:
- first = 0
- if last is None:
- last = len(data) - 1
- else:
- try:
- last = int(last)
- except:
- last = len(data) - 1
- latest = data[last]
- prev = data[first]
- data = {
- 'alert_level': f'{prev["alert_level"]} -> {latest["alert_level"]}',
- 'total_students': latest["total_students"] - prev["total_students"],
- 'total_staff': latest["total_staff"] - prev["total_staff"],
- 'new_students': latest["new_students"] - prev["new_students"],
- 'new_staff': latest["new_staff"] - prev["new_staff"],
- 'quarantine_on_campus': latest["quarantine_on_campus"] - prev["quarantine_on_campus"],
- 'quarantine_off_campus': latest["quarantine_off_campus"] - prev["quarantine_off_campus"],
- 'isolation_on_campus': latest["isolation_on_campus"] - prev["isolation_on_campus"],
- 'isolation_off_campus': latest["isolation_off_campus"] - prev["isolation_off_campus"],
- 'beds_available': latest["beds_available"] - prev["beds_available"],
- 'tests_administered': latest["tests_administered"] - prev["tests_administered"],
- 'description': f'day {first} to {last}'
- }
- return jsonify(data)
-
diff --git a/poller/db.py b/poller/db.py
deleted file mode 100644
index d3836a7..0000000
--- a/poller/db.py
+++ /dev/null
@@ -1,67 +0,0 @@
-import threading
-import sqlite3
-
-db_lock = threading.Lock()
-
-def create_tables():
- with db_lock:
- db_conn = sqlite3.connect('data/data.sqlite3')
- c = db_conn.cursor()
- sql = f'CREATE TABLE IF NOT EXISTS `alertlevel` (time DATETIME PRIMARY KEY NOT NULL, color CHAR(50) NOT NULL);'
- c.execute(sql)
- sql = f'CREATE TABLE IF NOT EXISTS `total` (time DATETIME PRIMARY KEY NOT NULL, total_students INT NOT NULL, total_staff INT NOT NULL);'
- c.execute(sql)
- sql = f'CREATE TABLE IF NOT EXISTS `new` (time DATETIME PRIMARY KEY NOT NULL, new_students INT NOT NULL, new_staff INT NOT NULL);'
- c.execute(sql)
- sql = f'CREATE TABLE IF NOT EXISTS `quarantine` (time DATETIME PRIMARY KEY NOT NULL, quarantine_on_campus INT NOT NULL, quarantine_off_campus INT NOT NULL);'
- c.execute(sql)
- sql = f'CREATE TABLE IF NOT EXISTS `isolation` (time DATETIME PRIMARY KEY NOT NULL, isolation_on_campus INT NOT NULL, isolation_off_campus INT NOT NULL);'
- c.execute(sql)
- sql = f'CREATE TABLE IF NOT EXISTS `beds` (time DATETIME PRIMARY KEY NOT NULL, beds_available INT NOT NULL);'
- c.execute(sql)
- sql = f'CREATE TABLE IF NOT EXISTS `tests` (time DATETIME PRIMARY KEY NOT NULL, tests_administered INT NOT NULL);'
- c.execute(sql)
- db_conn.commit()
- db_conn.close()
-
-
-def get_latest_from_db():
- return get_all_from_db()[-1]
-
-def get_all_from_db():
- with db_lock:
- db_conn = sqlite3.connect('data/data.sqlite3')
- c = db_conn.cursor()
- sql = 'SELECT alertlevel.time, alertlevel.color, total.total_students, total.total_staff, new.new_students, new.new_staff, ' + \
- 'quarantine.quarantine_on_campus, quarantine.quarantine_off_campus, isolation.isolation_on_campus, isolation.isolation_off_campus, ' + \
- 'beds.beds_available, tests.tests_administered ' + \
- 'FROM `alertlevel` ' + \
- 'INNER JOIN `total` ' + \
- 'ON alertlevel.time = total.time ' + \
- 'INNER JOIN `new` ' + \
- 'ON alertlevel.time = new.time ' + \
- 'INNER JOIN `quarantine` ' + \
- 'ON alertlevel.time = quarantine.time ' + \
- 'INNER JOIN `isolation` ' + \
- 'ON alertlevel.time = isolation.time ' + \
- 'INNER JOIN `beds` ' + \
- 'ON alertlevel.time = beds.time ' + \
- 'INNER JOIN `tests` ' + \
- 'ON alertlevel.time = tests.time'
- c.execute(sql)
-
- data = [{
- 'alert_level': d[1],
- 'total_students': d[2],
- 'total_staff': d[3],
- 'new_students': d[4],
- 'new_staff': d[5],
- 'quarantine_on_campus': d[6],
- 'quarantine_off_campus': d[7],
- 'isolation_on_campus': d[8],
- 'isolation_off_campus': d[9],
- 'beds_available': d[10],
- 'tests_administered': d[11],
- 'last_updated': d[0]
- } for d in c.fetchall()]
- return data
diff --git a/poller/dedup.py b/poller/dedup.py
deleted file mode 100644
index ba68be8..0000000
--- a/poller/dedup.py
+++ /dev/null
@@ -1,71 +0,0 @@
-import sqlite3
-
-def get_all_from_db():
- db_conn = sqlite3.connect('data/data.sqlite3')
- c = db_conn.cursor()
- sql = 'SELECT alertlevel.time, alertlevel.color, total.total_students, total.total_staff, new.new_students, new.new_staff, ' + \
- 'quarantine.quarantine_on_campus, quarantine.quarantine_off_campus, isolation.isolation_on_campus, isolation.isolation_off_campus, ' + \
- 'beds.beds_available, tests.tests_administered ' + \
- 'FROM `alertlevel` ' + \
- 'INNER JOIN `total` ' + \
- 'ON alertlevel.time = total.time ' + \
- 'INNER JOIN `new` ' + \
- 'ON alertlevel.time = new.time ' + \
- 'INNER JOIN `quarantine` ' + \
- 'ON alertlevel.time = quarantine.time ' + \
- 'INNER JOIN `isolation` ' + \
- 'ON alertlevel.time = isolation.time ' + \
- 'INNER JOIN `beds` ' + \
- 'ON alertlevel.time = beds.time ' + \
- 'INNER JOIN `tests` ' + \
- 'ON alertlevel.time = tests.time'
- c.execute(sql)
-
- data = [{
- 'alert_level': d[1],
- 'total_students': d[2],
- 'total_staff': d[3],
- 'new_students': d[4],
- 'new_staff': d[5],
- 'quarantine_on_campus': d[6],
- 'quarantine_off_campus': d[7],
- 'isolation_on_campus': d[8],
- 'isolation_off_campus': d[9],
- 'beds_available': d[10],
- 'tests_administered': d[11],
- 'last_updated': d[0]
- } for d in c.fetchall()]
- return data
-
-
-def drop_by_date(date):
- db_conn = sqlite3.connect('data/data.sqlite3')
- c = db_conn.cursor()
- sql = f'DELETE FROM `alertlevel` WHERE time=\'{date}\';'
- c.execute(sql)
- sql = f'DELETE FROM `total` WHERE time=\'{date}\';'
- c.execute(sql)
- sql = f'DELETE FROM `new` WHERE time=\'{date}\';'
- c.execute(sql)
- sql = f'DELETE FROM `quarantine` WHERE time=\'{date}\';'
- c.execute(sql)
- sql = f'DELETE FROM `isolation` WHERE time=\'{date}\';'
- c.execute(sql)
- sql = f'DELETE FROM `beds` WHERE time=\'{date}\';'
- c.execute(sql)
- sql = f'DELETE FROM `tests` WHERE time=\'{date}\';'
- c.execute(sql)
- db_conn.commit()
- db_conn.close()
-
-
-def dedup():
- data = get_all_from_db()
- # get first date
- starting_date = data[-1]['last_updated'].split(' ')[0]
- for i in range(len(data)-2, 0, -1):
- if data[i]['last_updated'].split(' ')[0] != starting_date:
- starting_date = data[i]['last_updated'].split(' ')[0]
- else:
- drop_by_date(data[i]['last_updated'])
- print('dropped ' + data[i]['last_updated'])