From 73f7f9e567d4e5660ee4137211c2092534b97efd Mon Sep 17 00:00:00 2001 From: binaryatrocity Date: Sun, 13 Jul 2014 15:10:18 -0500 Subject: [PATCH] mv periodic back to cron, add winrate graphs --- .gitignore | 1 + app.py | 24 ------ app/__init__.py | 27 +------ app/analytics.py | 76 ++++++++++++++++++ app/models.py | 6 +- app/static/css/app.css | 13 +++- app/teamspeak.py | 37 +++++---- app/templates/edit_event.html | 10 +++ app/templates/layout.html | 8 +- app/templates/profile.html | 110 +++++++++++++++++++++++---- app/templates/sidenav.html | 2 +- app/templates/teamspeak.html | 2 +- migrations/versions/1c90e0fd276a_.py | 26 +++++++ run.py | 102 +++++++++++++++++++++++++ 14 files changed, 352 insertions(+), 92 deletions(-) delete mode 100755 app.py create mode 100644 app/analytics.py create mode 100644 migrations/versions/1c90e0fd276a_.py create mode 100755 run.py diff --git a/.gitignore b/.gitignore index f0e1281..522490a 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ venv/ *.db *.swp clientlist.txt +*.wsgi diff --git a/app.py b/app.py deleted file mode 100755 index ce0ab83..0000000 --- a/app.py +++ /dev/null @@ -1,24 +0,0 @@ -#!venv/bin/python -from flask import Flask -from flask.ext.script import Manager, Server -from flask.ext.migrate import Migrate, MigrateCommand - -from app import * - -SQLALCHEMY_DATABASE_URI = 'mysql://root:$perwePP@localhost/dotanoobs' - -migrate = Migrate(app, db) -manager = Manager(app) -manager.add_command('db', MigrateCommand) - - -@manager.command -def admin(name): - u = models.User.query.filter_by(nickname=name).first() - if u and not u.admin: - u.admin = True - db.session.commit() - print "User {} has been granted admin access.".format(name) - -if __name__ == '__main__': - manager.run() diff --git a/app/__init__.py b/app/__init__.py index 1e9c73f..0a27603 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -10,29 +10,4 @@ db = SQLAlchemy(app) oid = OpenID(app) cache = Cache(app, config={'CACHE_TYPE': app.config['CACHE_TYPE']}) -import ts3 -from apscheduler.schedulers.background import BackgroundScheduler -from teamspeak import idle_mover, store_active_data, \ - process_ts3_events, award_idle_ts3_points - -def set_voice_server(): - ts3Server = ts3.TS3Server(app.config['TS3_HOST'], app.config['TS3_PORT']) - ts3Server.login(app.config['TS3_USERNAME'], app.config['TS3_PASSWORD']) - ts3Server.use(1) - return ts3Server - -voice = set_voice_server() - -def refresh_voice_server(): - app.logger.info("Refreshing TS3 connection...") - voice = set_voice_server() - -scheduler = BackgroundScheduler(logger=app.logger) -scheduler.add_job(refresh_voice_server, 'interval', hours=6) -scheduler.add_job(idle_mover, 'interval', [voice], minutes=30) -scheduler.add_job(store_active_data, 'interval', [voice], minutes=30) -scheduler.add_job(award_idle_ts3_points, 'interval', [voice], minutes=30) -scheduler.add_job(process_ts3_events, 'interval', [voice], hours=1) -scheduler.start() - -from app import views, models +from app import views diff --git a/app/analytics.py b/app/analytics.py new file mode 100644 index 0000000..bb28d46 --- /dev/null +++ b/app/analytics.py @@ -0,0 +1,76 @@ +import requests +from time import sleep, mktime +from bs4 import BeautifulSoup +from datetime import datetime + +from app import app, db, models + +MODES_TO_SKIP = ['Ability Draft', 'Greeviling', 'Diretide'] + +def collect_match_results(dotabuff_id, num_matches): + results = [] + page = 0 + while True: + page += 1 + url = "http://dotabuff.com/players/{}/matches/?page={}".format(dotabuff_id, page) + data = requests.get(url).text + soup = BeautifulSoup(data).article.table.tbody + # Catch last page + if 'sorry' in soup.tr.td.text.lower(): + break + # Parse the matches on current page + for row in soup.find_all('tr'): + # Pass over bot matches and other 'inactive' games + if 'inactive' in row.get('class', ''): continue + cells = row.find_all('td') + result_cell = cells[2] + match_cell = cells[3] + match_id = int(result_cell.a['href'].split('/')[-1]) + match_type = match_cell.div.text + if match_type in MODES_TO_SKIP: continue + result = True if 'won' in result_cell.a['class'] else False + dt = datetime.strptime(result_cell.time['datetime'], '%Y-%m-%dT%H:%M:%S+00:00') + results.append({'match_id':match_id, 'win':result, 'datetime':dt, 'game_mode':match_type}) + if len(results) > num_matches: + break + if len(results) > num_matches: + break + sleep(60) + results.reverse() + return results + +def apply_window(results, window_size=50): + windows = [] + # Compute the initial window + win_rate = 0.00 + for idx in range(0, window_size): + win_rate += 1 if results[idx]['win'] else 0 + win_rate /= window_size + windows.append(win_rate) + # From here on, modify based on leave/enter data points + fractional_change = 1. / window_size + for idx in range(window_size, len(results)): + if results[idx-window_size]['win'] == results[idx]['win']: + pass + elif results[idx]['win']: + win_rate += fractional_change + else: + win_rate -= fractional_change + windows.append(win_rate) + return windows + +def calculate_winrates(): + users_analyzed = 0 + for user in models.User.query.all(): + db_id = requests.get("http://dotabuff.com/search?q="+user.steam_id).url.split("/")[-1] + result = collect_match_results(db_id, app.config['ANALYTICS_WINRATE_NUM_MATCHES']) + windowed = apply_window(result, app.config['ANALYTICS_WINRATE_WINDOW']) + date_nums = map(lambda x: mktime(x['datetime'].timetuple()),\ + result[app.config['ANALYTICS_WINRATE_WINDOW']-1:]) + winrate = {'total_games': len(result), 'data': zip(date_nums, windowed) } + user.winrate_data = winrate + db.session.commit() + users_analyzed += 1 + sleep(60) + app.logger.info("Calculated win rate numbers for {} doobs.".format(users_analyzed)) + return users_analyzed diff --git a/app/models.py b/app/models.py index c3f5a18..bad7a56 100644 --- a/app/models.py +++ b/app/models.py @@ -81,11 +81,14 @@ class User(db.Model): points_from_events = db.Column(db.Integer) points_from_ts3 = db.Column(db.Integer) points_from_forum = db.Column(db.Integer) + ts3_starttime = db.Column(db.DateTime) ts3_endtime = db.Column(db.DateTime) ts3_rewardtime = db.Column(db.DateTime) ts3_connections = db.Column(MutableDict.as_mutable(Json)) + last_post_reward = db.Column(db.Integer) + winrate_data = db.Column(MutableDict.as_mutable(Json)) @classmethod @@ -96,6 +99,7 @@ class User(db.Model): self.steam_id = steam_id self.random_heroes = {'current':None, 'completed':[]} self.az_completions = 0 + self.ts3_connections = {'list':[]} self.created = datetime.utcnow() self.last_seen = datetime.utcnow() self.bio_text = None @@ -161,7 +165,7 @@ class User(db.Model): db.session.commit(); def finalize_connection(self): - self.ts3_connections.append({'starttime': self.ts3_starttime, 'endtime': self.ts3_endtime}) + self.ts3_connections['list'].append({'starttime': self.ts3_starttime, 'endtime': self.ts3_endtime}) self.ts3_startime = None self.ts3_endtime = None db.session.commit(); diff --git a/app/static/css/app.css b/app/static/css/app.css index 1d858d4..383a4f8 100644 --- a/app/static/css/app.css +++ b/app/static/css/app.css @@ -95,6 +95,15 @@ footer { float:right; } -#profile_links > a { - font-size:14px; +#winrate_graph { + margin: 5em; +} + +.pipemenu > li { + display: inline-block; + list-style: none; +} + +.pipemenu > li + li::before { + content: " | "; } diff --git a/app/teamspeak.py b/app/teamspeak.py index 544bb0d..802cda7 100644 --- a/app/teamspeak.py +++ b/app/teamspeak.py @@ -6,12 +6,12 @@ from datetime import datetime, timedelta from xml.etree import ElementTree from bs4 import BeautifulSoup +import models from app import app, db def getTeamspeakWindow(window=timedelta(weeks=1)): current_time = datetime.utcnow() - from models import TeamspeakData - return TeamspeakData.query.filter(TeamspeakData.time < current_time, TeamspeakData.time > current_time-window).order_by(TeamspeakData.time).all() + return models.TeamspeakData.query.filter(models.TeamspeakData.time < current_time, models.TeamspeakData.time > current_time-window).order_by(models.TeamspeakData.time).all() def registerUserTeamspeakId(user, tsid): server = ts3.TS3Server(app.config['TS3_HOST'], app.config['TS3_PORT']) @@ -173,7 +173,7 @@ def create_teamspeak_viewer(): return "error: %s" % inst def get_ISO3166_mapping(): - with open('app/static/country_codes.xml', mode='r') as d: + with open(path.join(path.dirname(__file__), 'static/country_codes.xml'), mode='r') as d: data = d.read() xml = ElementTree.fromstring(data) d = dict() @@ -187,10 +187,10 @@ ISO3166_MAPPING = get_ISO3166_mapping() # Scheduled functions for TeamspeakServer # -def idle_mover(voice): +def idle_mover(server): """ Checks connected clients idle_time, moving to AFK if over TS3_MAX_IDLETIME. """ - app.logger.info("Running TS3 AFK mover...") + app.logger.debug("Running TS3 AFK mover...") exempt_cids = [] permid_response = server.send_command('permidgetbyname', keys={'permsid': 'i_channel_needed_join_power'}) if permid_response.is_successful: @@ -220,10 +220,8 @@ def idle_mover(voice): clientlist = server.send_command('clientlist', opts=['times']).data for client in clientlist: clientinfo = server.send_command('clientinfo', {'clid':client['clid']}) - if clientinfo.is_successful: - client['client_unique_identifier'] = clientinfo.data[0]['client_unique_identifier'] - else: - raise UserWarning('Could not find the clientinfo for %s' % client['clid']) + #if clientinfo.is_successful: + #client['client_unique_identifier'] = clientinfo.data[0]['client_unique_identifier'] # move idlers to afk channel for client in clientlist: @@ -232,10 +230,10 @@ def idle_mover(voice): # Have TeamSpeak move AFK user to appropriate channel server.send_command('clientmove', keys={'clid': client['clid'], 'cid': afk_channel['cid']}) -def store_active_data(voice): +def store_active_data(server): """ Take a snapshot of Teamspeak (clients, countries, etc) to feed the ts3_stats page """ - app.logger.info("Taking Teamspeak snapshot...") + app.logger.debug("Taking Teamspeak snapshot...") # Get exempt channels (AFK, passworded, join powers) exempt_cids = [] permid_response = server.send_command('permidgetbyname', keys={'permsid': 'i_channel_needed_join_power'}) @@ -276,18 +274,16 @@ def store_active_data(voice): db.session.add(tsdata) db.session.commit() -def process_ts3_events(voice): +def process_ts3_events(server): """ Create Teamspeak channels for upcoming events, delete empty event channels that have expired """ - app.logger.info("Processing Teamspeak events...") + app.logger.debug("Processing Teamspeak events...") # Get list of clients clientlist = server.clientlist() for clid, client in clientlist.iteritems(): clientinfo = server.send_command('clientinfo', {'clid':clid}) if clientinfo.is_successful: client['client_unique_identifier'] = clientinfo.data[0]['client_unique_identifier'] - else: - raise UserWarning('Could not find clientinfo for %s' % clid) # Process any active events for clid, client in clientlist.iteritems(): @@ -327,10 +323,10 @@ def process_ts3_events(voice): server.clientpoke(client['clid'], message) -def award_idle_ts3_points(voice): +def award_idle_ts3_points(server): """ Award points for active time spent in the Teamspeak server. """ - app.logger.info("Awarding Teamspeak idle points") + app.logger.debug("Awarding Teamspeak idle points") # Get exempt channels (AFK, passwords, join power) exempt_cids = [] permid_response = server.send_command('permidgetbyname', keys={'permsid': 'i_channel_needed_join_power'}) @@ -358,14 +354,15 @@ def award_idle_ts3_points(voice): clientinfo = server.send_command('clientinfo', {'clid': clid}) if clientinfo.is_successful: client['client_unique_identifier'] = clientinfo.data[0]['client_unique_identifier'] - else: - raise UserWarning('Could not find the clientinfo for %s' % clid) # Update the data active_users = set() for client in clientlist.values(): if client['cid'] not in exempt_cids: - doob = models.User.query.filter_by(teamspeak_id=client['client_unique_identifier']).first() + try: + doob = models.User.query.filter_by(teamspeak_id=client['client_unique_identifier']).first() + except KeyError: + pass if doob: doob.update_connection() active_users.add(doob) diff --git a/app/templates/edit_event.html b/app/templates/edit_event.html index 459069a..b7a8fd9 100644 --- a/app/templates/edit_event.html +++ b/app/templates/edit_event.html @@ -64,6 +64,16 @@ - +