#!/usr/bin/env python # -*- coding: utf-8 -*- import sys, os, re, time, urllib2, json LOGFILE = "wm_scorefeed_livescore.log" LIVESCORE_URL = "http://www.livescore.com/~~/m/07/wc/fix/2/" USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko" FAIL_FILE = "failed_data.dat" WM_URL = "http://wm.emphy.de/" WM_Auth_File = "wm_scorefeed_auth.site" # contents of that file: single line with if os.path.exists(WM_Auth_File): WM_Auth = open(WM_Auth_File).read().strip() else: print >>sys.stderr, "WARNING:", WM_Auth_File, "doesn't exist!" print >>sys.stderr, "It shall contain a single line of the form \"username:passwordInPlainText\"" WM_Auth = "" MaxKickoffAge = 3 * 24 * 3600 # maximum "age" of a game to be considered relevant TeamNames = """ ägypten : egypt Ägypten : egypt albanien : albania algerien : algeria argentinien : argentina australien : australia belgien : belgium bosnien-herzegowina : bosnia-herzegovina brasilien : brazil deutschland : germany dänemark : denmark elfenbeinküste : ivory coast frankreich : france griechenland : greece irland : ireland island : iceland italien : italy kamerun : cameroon kolumbien : colombia kroatien : croatia marokko : morocco mexiko : mexico niederlande : netherlands nordirland : n.ireland österreich : austria Österreich : austria polen : poland rumänien : romania russland : russia saudi-arabien : saudi arabia schweden : sweden schweiz : switzerland serbien : serbia slowakei : slovakia spanien : spain südkorea : south korea tschechien : czech republic tunesien : tunisia türkei : turkey ungarn : hungary """ TeamNames = dict(map(str.strip, line.split(':')) for line in TeamNames.split('\n') if line.strip()) C_RED, C_GREEN, C_BROWN, C_BLUE, C_VIOLET, C_CYAN, C_GRAY = ("\x1b[%dm" % x for x in xrange(31,38)) LOG_SUCCESS = C_GREEN LOG_ERROR = C_RED LOG_WARNING = C_BROWN LOG_INFO = C_BLUE _color = ("posix" in os.name.lower()) and sys.stdout.isatty() _logfile = None def log(msg, color=""): global _logfile, LOGFILE, _color prefix = time.strftime("[%Y-%m-%d %H:%M:%S] ", time.localtime()) msg = msg.replace('\n', '\n ') if not(_logfile) and LOGFILE: try: _logfile = open(LOGFILE, "a") except IOError, e: f = LOGFILE LOGFILE = None log("could not open log file %s:\n%s" % (f, e), LOG_ERROR) if _logfile: _logfile.write(prefix + msg + "\n") _logfile.flush() if not _color: color = "" sys.stdout.write(prefix + color + msg + ("\x1b[0m\n" if color else "\n")) sys.stdout.flush() def notags(x): return re.sub(r'<[^>]*>', '', x).strip() class Match(object): def __init__(self): self.valid = False self.minute = 0 self.teamA = None self.teamB = None self.goalsA = None self.goalsB = None self.time = None def __repr__(self): return "Match(valid=%s, minute=%d, teams=(%r, %r), goals=(%r, %r), time=%s)" % \ (self.valid, self.minute, self.teamA, self.teamB, self.goalsA, self.goalsB, (time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(self.time)) if self.time else '0')) ################################################################################ EncodingSubstitutionModes = { # various ways in which CR, LF and space are encoded in the string, # selected by how many colons precede the space before the timestamp 13: (('!', '\n'), ('$', '\r'), ('%', ' ')), 14: (('\xab', '\n'), ('\xa9', '\r'), ('\xbb', ' ')), 15: (('"$!', '\n'), ('&%#', '\r'), ('$#%', ' ')), } def decodeChar(c, offset): if (c < 40) or (c > 126): return c c -= offset return c if (c >= 40) else (c + 126 - 40 + 1) def decodeJSON(s): mode = s.find(' ', 12) # find first space after the "Query-Expiry" bogus text subst = EncodingSubstitutionModes[mode] # look up a substitution set assert all((c == ':') for c in s[12:mode]) # check for preceding colons t = [int(c) for c in s[mode+1:mode+20] if c.isdigit()] # "decode" timestamp if len(t) != 14: code = 27 # invalid timestamp -> fixed code of 27 else: code = sum(t) # code offset is the digit sum of the timestamp # some additional fuzzing code += (t[-1] - t[-2]) if (t[-1] >= t[-2]) else (3 * (t[-6] + 1)) # strip of the header, reverse the string, and decode each position: # if it's between '(' and '~', subtract the code plus the output string # position modulo 10, otherwise keep the character as-is s = ''.join(chr(decodeChar(c, code + ((i + 1) % 10))) for i, c in enumerate(map(ord, s[:mode + 19:-1]))) # apply replacements for special characters for old, new in subst: s = s.replace(old, new) return s def ParseLiveScore(url): while True: try: if url.startswith("http"): req = urllib2.Request(url) req.add_header("User-Agent", USER_AGENT) data = urllib2.urlopen(req).read() # open("cached.dat", "wb").write(data) else: data = open(url).read() break except IOError, e: log("failed to load live score\nURL: %s\nError: %s\ntrying again in 10 seconds" % (url, e), LOG_ERROR) time.sleep(10) continue try: data = json.loads(decodeJSON(data)) except: log("Failed to decode and parse JSON data from LiveScore.") try: f = open(FAIL_FILE, "wb") f.write(data) f.close() except: pass return # json.dump(data, open("livescore.json", 'w'), indent=4) def get_team_name(event, t): for t in event.get(t, []): n = t.get('Nm') if n \ and not(n.lower().startswith(("winner ", "loser "))) \ and not((len(n) == 2) and n[0].isdigit() and n[1].isalpha()): return n for stage in data.get('Stages', []): for event in stage.get('Events', []): m = Match() t = event.get("Esd") try: m.time = time.mktime(((t / 10000000000) % 10000, (t / 100000000) % 100, (t / 1000000) % 100, (t / 10000) % 100, (t / 100) % 100, t % 100, -1, -1, -1)) except (TypeError, ValueError): pass si = int(event.get("Epr", 0)) ss = str(event.get("Eps", "")).upper() m.teamA = get_team_name(event, 'T1') m.teamB = get_team_name(event, 'T2') if not(m.teamA) or not(m.teamB): continue if 'Tr1' in event: m.goalsA = int(event['Tr1']) + int(event.get('Trp1', 0)) if 'Tr2' in event: m.goalsB = int(event['Tr2']) + int(event.get('Trp2', 0)) if ss.endswith("'"): m.minute = int(ss[:-1].split('+', 1)[0].strip()) elif ss == "HT": m.minute = 45 elif ss == "ET": m.minute = 90 elif ss == "PEN": m.minute = 120 elif ss == "FT": m.minute = 90 elif ss == "AET": m.minute = 120 m.valid = (si > 1) and not(m.goalsA is None) and not(m.goalsB is None) yield m ################################################################################ def WMRequest(uri): url = WM_URL + uri try: req = urllib2.Request(url) req.add_header("Authorization", "Basic " + WM_Auth.encode('base64').strip()) data = urllib2.urlopen(req).read() except IOError, e: log("Wettmeister request failed\nURL: %s\nError: %s" % (url, e), LOG_ERROR) return "" return data def GetOpenMatches(): data = WMRequest("adm_score_feed_endpoint.php").split('\n') gmap = {} for line in data: if not line: continue mid, ta, tb = line.split(':') gmap[TeamNames.get(ta, ta) + ':' + TeamNames.get(tb, tb)] = int(mid) return gmap ################################################################################ if __name__ == "__main__": if len(sys.argv) == 2: for m in ParseLiveScore(LIVESCORE_URL if (sys.argv[1] == "--test") else sys.argv[1]): print m sys.exit(0) log("----- %s starting -----" % os.path.basename(sys.argv[0]), C_VIOLET) wait = 0 matches = {} was_short_wait = False active_matches = set() invalid_matches = False while True: if wait <= 60: invalid_matches = False # don't update valid matches too often if wait: if wait >= 90: log("waiting %d minutes (until %s)" % ((wait + 30) / 60, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() + wait))), LOG_INFO) was_short_wait = False else: if not was_short_wait: log("match(es) coming to a close, entering short-wait phase", LOG_INFO) was_short_wait = True try: time.sleep(wait) except KeyboardInterrupt: print break wait = 0 if not(matches) or invalid_matches: matches = GetOpenMatches() if not matches: log("Wettmeister reports no outstanding matches.", LOG_INFO) wait = 3600 continue else: log("Wettmeister reports %d outstanding match(es)" % len(matches), LOG_INFO) invalid_matches = False wait = 90 * 60 n_done = 0 max_age = time.time() - MaxKickoffAge for m in ParseLiveScore(LIVESCORE_URL): if m.time and (m.time < max_age): continue # ignore match if too old a = m.teamA.lower() b = m.teamB.lower() mcode = m.teamA + " - " + m.teamB this_wait = None if m.minute and not(m.valid) and not(mcode in active_matches): log("match %s is now running (minute %d, %s:%s)" % (mcode, m.minute, '?' if (m.goalsA is None) else m.goalsA, '?' if (m.goalsB is None) else m.goalsB), LOG_SUCCESS) active_matches.add(mcode) elif m.valid and (mcode in active_matches): log("match %s has finished" % mcode, LOG_SUCCESS) active_matches.remove(mcode) reverse = False mid = matches.get(a + ':' + b) if not mid: reverse = True mid = matches.get(b + ':' + a) if not mid: if not(m.valid) and not('/' in a) and not('/' in b): # warn about matches not listed by Wettmeister, but ignore # things like 'CountryA/CountryB' for unresolved final # round fixtures log("match %s is not outstanding or invalid" % mcode, LOG_WARNING) invalid_matches = True continue if (mid >= 1000) and m.valid and (m.goalsA == m.goalsB): log("match %s seems to have ended with %s:%s, but needs a clear winner" % (mcode, m.goalsA, m.goalsB), LOG_WARNING) m.valid = False # match must have a clear winner, but there is none yet -> don't enter it! this_wait = 300 if m.valid: a, b = m.goalsA, m.goalsB log("entering result (%s = %d:%d)" % (mcode, a, b), LOG_SUCCESS) if reverse: a, b = b, a WMRequest("adm_score_feed_endpoint.php?m=%d&a=%d&b=%d" % (mid, a, b)) n_done += 1 else: if this_wait: pass elif m.minute < 90: this_wait = 60 * (90 - m.minute) elif (m.minute > 95) and (m.minute < 120): this_wait = 60 * (120 - m.minute) else: this_wait = 10 wait = min(wait, this_wait) if n_done: log("updating scores", LOG_SUCCESS) WMRequest("adm_do_update_scores.php") wait = 0 matches = {} active_matches = set() invalid_matches = False log("----- %s exiting -----" % os.path.basename(sys.argv[0]), C_VIOLET)