Bugfixes & Hitbox.tv field in User model

app/analytics.py

@@ -1,7 +1,7 @@
 import requests
 from time import sleep, mktime
 from bs4 import BeautifulSoup
-from datetime import datetime
+from datetime import datetime, timedelta
 
 from app import app, db, models
 
@@ -14,10 +14,15 @@ def collect_match_results(dotabuff_id, num_matches):
         page += 1
         url = "http://dotabuff.com/players/{}/matches/?page={}".format(dotabuff_id, page)
         data = requests.get(url).text
-        soup = BeautifulSoup(data).article.table.tbody
+        try:
-        # Catch last page
+            soup = BeautifulSoup(data).article.table.tbody
-        if 'sorry' in soup.tr.td.text.lower():
+        except:
             break
+        else:
+            # Catch last page
+            if 'sorry' in soup.tr.td.text.lower():
+                break
+
         # Parse the matches on current page
         for row in soup.find_all('tr'):
             # Pass over bot matches and other 'inactive' games
@@ -43,7 +48,7 @@ def apply_window(results, window_size=50):
     windows = []
     # Compute the initial window
     win_rate = 0.00
-    for idx in range(0, window_size):
+    for idx in range(0, window_size-1):
         win_rate += 1 if results[idx]['win'] else 0
     win_rate /= window_size
     windows.append(win_rate)
@@ -59,18 +64,38 @@ def apply_window(results, window_size=50):
         windows.append(win_rate)
     return windows
 
+# Single user alternative for testing/calculating on demand
+def calculate_winrate(user_id):
+    user = models.User.query.get(user_id)
+    db_id = requests.get("http://dotabuff.com/search?q="+user.steam_id).url.split("/")[-1]
+    result = collect_match_results(db_id, app.config['ANALYTICS_WINRATE_NUM_MATCHES'])
+    windowed = apply_window(result, app.config['ANALYTICS_WINRATE_WINDOW'])
+    date_nums = map(lambda x: mktime(x['datetime'].timetuple()),\
+            result[app.config['ANALYTICS_WINRATE_WINDOW']-1:])
+    winrate = {'total_games': len(result), 'data': zip(date_nums, windowed) }
+    user.winrate_data = winrate
+    db.session.commit()
+
 def calculate_winrates():
     users_analyzed = 0
-    for user in models.User.query.all():
+    delta = datetime.utcnow() - timedelta(weeks=4)
+    print "Starting winrate calculation"
+    for user in models.User.query.filter(models.User.last_seen > delta).all():
+        print "Begin calculating winrate for {}".format(user.nickname.encode('utf-8'))
         db_id = requests.get("http://dotabuff.com/search?q="+user.steam_id).url.split("/")[-1]
         result = collect_match_results(db_id, app.config['ANALYTICS_WINRATE_NUM_MATCHES'])
-        windowed = apply_window(result, app.config['ANALYTICS_WINRATE_WINDOW'])
+        if len(result):
-        date_nums = map(lambda x: mktime(x['datetime'].timetuple()),\
+            windowed = apply_window(result, app.config['ANALYTICS_WINRATE_WINDOW'])
-                result[app.config['ANALYTICS_WINRATE_WINDOW']-1:])
+            date_nums = map(lambda x: mktime(x['datetime'].timetuple()),\
-        winrate = {'total_games': len(result), 'data': zip(date_nums, windowed) }
+                    result[app.config['ANALYTICS_WINRATE_WINDOW']-1:])
-        user.winrate_data = winrate
+            winrate = {'total_games': len(result), 'data': zip(date_nums, windowed) }
-        db.session.commit()
+            user.winrate_data = winrate
-        users_analyzed += 1
+            db.session.commit()
-        sleep(60)
+            users_analyzed += 1
+            print "Finished winrate calculation for {} ({} total games)".format(user.nickname.encode('utf-8'), winrate['total_games'])
+            sleep(60)
+        else:
+            print "DotaBuff unable to access match statistics for {}".format(user.nickname.encode('utf-8'))
     app.logger.info("Calculated win rate numbers for {} doobs.".format(users_analyzed))
+    print "Calculated win rate numbers for {} doobs.".format(users_analyzed)
     return users_analyzed

app/forms.py

@@ -6,6 +6,7 @@ class SettingsForm(Form):
     public = BooleanField('public', default=True)
     logo = BooleanField('biglogo', default=True)
     twitch = TextField('twitch')
+    hitbox = TextField('hitbox')
     bio_text = TextAreaField('bio_text')
 
 class EnableStatsForm(Form):

app/models.py

@@ -28,8 +28,11 @@ class Json(db.TypeDecorator):
         return value
 
     def process_result_value(self, value, dialect):
-        if value is not None:
+        try:
-            value = json.loads(value)
+            if value is not None:
+                value = json.loads(value)
+        except ValueError:
+            return {}
         return value
 
 # Mongoness factor - phase 2
@@ -72,6 +75,7 @@ class User(db.Model):
         created = db.Column(db.DateTime)
         last_seen = db.Column(db.DateTime)
         twitch = db.Column(db.String(60))
+        hitbox = db.Column(db.String(60))
         random_heroes = db.Column(MutableDict.as_mutable(Json))
         az_completions = db.Column(db.Integer)
 
@@ -95,6 +99,19 @@ class User(db.Model):
         def get_or_create(self, steam_id):
             return get_or_create_instance(db.session, User, steam_id=steam_id)
 
+        @classmethod
+        def get_streaming_users(self):
+            twitch_streams = []
+            hitbox_streams = []
+            for user in User.query.all():
+		if user.points_from_events + user.points_from_ts3 + user.points_from_forum < 5: continue
+                if user.twitch:
+                    twitch_streams.append(user.twitch)
+                if user.hitbox:
+                    hitbox_streams.append(user.hitbox)
+
+            return {'twitch': twitch_streams, 'hitbox': hitbox_streams}
+
         def __init__(self, steam_id):
             self.steam_id = steam_id
             self.random_heroes = {'current':None, 'completed':[]}

app/teamspeak.py

@@ -361,11 +361,11 @@ def award_idle_ts3_points(server):
             if client['cid'] not in exempt_cids:
                 try:
                     doob = models.User.query.filter_by(teamspeak_id=client['client_unique_identifier']).first()
+                    if doob:
+                        doob.update_connection()
+                        active_users.add(doob)
                 except KeyError:
                     pass
-                if doob:
-                    doob.update_connection()
-                    active_users.add(doob)
         doobs = set(models.User.query.filter(models.User.ts3_starttime != None).all())
         for doob in doobs.intersection(active_users):
             doob.finalize_connection() 

app/templates/index.html

@@ -75,7 +75,8 @@ $(document).ready(function() {
 
 	// Add the twitch streams
 	var stream_url = "https://api.twitch.tv/kraken/streams/";
-        var channels = ["dotanoobs", "bearhugdota", "kreejaffakree", "prettypenguins", "shaneomad"];
+        // var channels = ["dotanoobs", "bearhugdota", "kreejaffakree", "prettypenguins", "shaneomad"];
+        var channels = {{ streamers['twitch']|tojson|safe }};
 	for (var idx in channels) {
 		$.getJSON(stream_url+channels[idx]+"?callback=?", function(data) {
                     if (data.stream) {
@@ -107,7 +108,8 @@ $(document).ready(function() {
         
 	// Add the hitbox streams
 	var stream_url = "http://api.hitbox.tv/media/live/";
-        var channels = ["Bandita", "Gibb3d"];
+        //var channels = ["Bandita", "Gibb3d"];
+        var channels = {{ streamers['hitbox']|tojson|safe }};
 	for (var idx in channels) {
 		$.getJSON(stream_url+channels[idx], function(data) {
                     var livestream = data.livestream[0];

app/templates/settings.html

@@ -25,6 +25,10 @@
                     <label class="uk-form-label">Twitch.tv Username:</label> <br/>
                     {{ form.twitch(placeholder="e.g. shaneomad") }}
                 </div>
+                <div class="uk-form-row">
+                    <label class="uk-form-label">Hitbox.tv Username:</label> <br/>
+                    {{ form.hitbox(placeholder="e.g. shaneomad") }}
+                </div>
                 <div class="uk-form-row">
                     <label class="uk-form-label">Biography text:</label><br/>
                     {{ form.bio_text(rows=14, class='uk-width-1-1', data=g.user.bio_text,  placedholder='What you place here is displayed in your profile when other users view it.') }}

app/views.py

@@ -41,7 +41,8 @@ def flash_form_errors(form):
 def index():
     active = Event.query.filter(Event.start_time <= datetime.utcnow(), Event.end_time > datetime.utcnow()).all()
     upcoming =  Event.query.filter(Event.start_time > datetime.utcnow()).limit(2).all()
-    return render_template("index.html", active_events=active, upcoming_events=upcoming)
+    channels = User.get_streaming_users()
+    return render_template("index.html", active_events=active, upcoming_events=upcoming, streamers=channels)
 	
 @app.route('/login')
 @oid.loginhandler
@@ -79,6 +80,11 @@ def teamspeak():
 def friends():
 	return render_template('friends.html') 
 
+# Stream pages
+@app.route('/shaneomad')
+def twitch_shaneomad():
+    return render_template('potatr.html', twitch_id=app.config.TWITCH_CLIENT_ID)
+
 # User profile page
 @app.route('/user/<int:userid>')
 def user_profile(userid):