From 1676f2fc64880251befdb18ad22f59dae0ae1f4e Mon Sep 17 00:00:00 2001 From: Max Magorsch Date: Mon, 4 May 2020 00:45:07 +0200 Subject: Generate html pages based on the mirmon data A python script is used to generate html pages based on the data produced by mirmon. The html pages visualize the mirror stats using the gentoo tyrian theme. Signed-off-by: Max Magorsch --- html/generate.py | 178 ++++++++++++++++++++++++++++ html/help.jinja2 | 163 ++++++++++++++++++++++++++ html/stats.jinja2 | 339 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 680 insertions(+) create mode 100755 html/generate.py create mode 100644 html/help.jinja2 create mode 100644 html/stats.jinja2 diff --git a/html/generate.py b/html/generate.py new file mode 100755 index 0000000..d6f21c0 --- /dev/null +++ b/html/generate.py @@ -0,0 +1,178 @@ +#!/usr/bin/python + +# +# Generate html pages based on the Mirmon data +# +# This script will parse the json files that contain the mirmon +# data and have been generated before. The json data is used to +# generate html pages that visualize the mirmon data in a human +# readable way. +# +# +# Leaflet is used to visualize the mirrors on a world map. Further- +# more Datatables are used to display the mirrors. +# + +import datetime +import socket +import urllib.request, json +import xml.etree.ElementTree as ET +import jinja2 + +html_folder = "/var/www/mirrorstats.gentoo.org/htdocs/" +cache_path = "/var/www/mirrorstats.gentoo.org/var/html_data_cache.json" + + +mirrorstats = [ 'https://mirrorstats.gentoo.org/rsync/state.json', + 'https://mirrorstats.gentoo.org/distfiles/state.json', + 'https://mirrorstats.gentoo.org/snapshots/state.json', + 'https://mirrorstats.gentoo.org/releases/state.json', + 'https://mirrorstats.gentoo.org/experimental/state.json' + ] + +cache_data = {} + + +# +# Retrieve and parse the JSON at the given URL +# +def getJson(url): + req = urllib.request.Request(url) + r = urllib.request.urlopen(req).read() + return json.loads(r.decode('utf-8')) + + +# +# Retrieve and parse the XML at the given URL +# +def getXML(url): + req = urllib.request.Request(url) + r = urllib.request.urlopen(req).read() + return ET.loads(r.decode('utf-8')).getroot() + + +# +# Get the ip of the given host +# +def getIp(hostname): + try: + ip = socket.gethostbyname(hostname) + except: + ip = "" + return ip + + +# +# Render the stats template for a given page, i.e. all, rsync, distfiles... +# +def renderStatsTemplate(templateEnv, page): + if page == "all": + mirrorstatsList = mirrorstats + else: + mirrorstatsList = ['https://mirrorstats.gentoo.org/' + page + '/state.json'] + + lastUpdate, hostList = getHostList(mirrorstatsList) + template = templateEnv.get_template("stats.jinja2") + template.stream(lastUpdate=lastUpdate, type=page, mirrors=hostList, locations=json.dumps(hostList)).dump(html_folder + page + ".html") + return lastUpdate + + +# read the cache +with open(cache_path) as json_file: + cache_data = json.load(json_file) + +# +# The all mirrors that are present in the given list +# +def getHostList(mirrorstatsList): + hosts = {} + lastUpdate = "" + + # process all mirrors + for mirror_url in mirrorstatsList: + + mirrorData = getJson(mirror_url) + lastUpdate = datetime.datetime.fromtimestamp(int(mirrorData['LastUpdate'])).strftime('%Y-%m-%d %H:%M:%S') + + for mirror in mirrorData['Mirrors']: + if len(mirrorData['Mirrors'][mirror]) >= 0: + for mirrorHost in mirrorData['Mirrors'][mirror]: + hostname = mirrorHost['Host'] + if hostname not in hosts: + hosts[hostname] = {} + + hosts[hostname]['Hostname'] = hostname + + if hostname in cache_data['hosts']: + ip = cache_data['hosts'][hostname] + else: + ip = getIp(hostname) + # populate cache with new value + cache_data['hosts'][hostname] = ip + + if ip != "": + hosts[hostname]['Ip'] = ip + + if 'Stats' not in hosts[hostname]: + hosts[hostname]['Stats'] = [] + + mirrorHost['Type'] = mirror_url.replace("https://mirrorstats.gentoo.org/", "").replace("/state.json", "") + hosts[hostname]['Stats'].append(mirrorHost) + + # compute available protocols + for host in hosts: + protocols = [] + for stat in hosts[host]['Stats']: + protocols.append(stat['Protocol']) + + hosts[host]['Protocols'] = list(set(protocols)) + + # compute mirror locations + for host in hosts: + if 'Ip' not in hosts[host]: + continue + + ip = hosts[host]['Ip'] + if ip in cache_data['ips']: + hosts[host]['Location'] = cache_data['ips'][ip] + else: + mirrorGeoData = getJson("https://ipinfo.io/" + ip + "/json") + hosts[host]['Location'] = mirrorGeoData['loc'] + # populate cache with new value + cache_data['ips'][ip] = mirrorGeoData['loc'] + + return lastUpdate, hosts + + +# +# render jinja2 +# +templateLoader = jinja2.FileSystemLoader(searchpath="./") +templateEnv = jinja2.Environment(loader=templateLoader) + +## stats +lastUpdate = renderStatsTemplate(templateEnv, "all") +renderStatsTemplate(templateEnv, "rsync") +renderStatsTemplate(templateEnv, "distfiles") +renderStatsTemplate(templateEnv, "snapshots") +renderStatsTemplate(templateEnv, "releases") +renderStatsTemplate(templateEnv, "experimental") + +## about +template = templateEnv.get_template("help.jinja2") +template.stream(lastUpdate=lastUpdate).dump(html_folder + "help.html") + + +# +# write the cache +# +with open(cache_path, 'w') as fp: + json.dump(cache_data, fp) + + +# +# finish +# +print("Finished.") + + diff --git a/html/help.jinja2 b/html/help.jinja2 new file mode 100644 index 0000000..6e08363 --- /dev/null +++ b/html/help.jinja2 @@ -0,0 +1,163 @@ + + + + Gentoo Mirrorstats + + + + + + + + +
+
+
+
+ +
+
+ Get Gentoo! +
+ + gentoo.org sites + +
+ gentoo.org + Wiki + Bugs + Forums + Packages +
+ Planet + Archives + Sources +
+ Infra Status +
+
+
+
+
+
+
+ +
+ + +
+
+
+

About Gentoo Mirrors

+
+ If you experience problems with this site (mirrorstats.gentoo.org), please file clearly marked bugs in + Gentoo Bugzilla (product Infrastructure). + Please remember to include mirrorstats in the summary line. +
+

Gentoo rsync mirrors

+

Gentoo rsync mirrors tracked include both official mirrors run by the Gentoo Infrastructure team, as well as community-run mirrors.

+ +

Gentoo source mirrors

+ +
+ If you experience problems with a specific mirror, please file clearly marked bugs in Gentoo Bugzilla (product Mirrors).
+ Please include lots of details about the problem, as the Gentoo + mirror team will have to contact the administrator of individual + mirrors to work on problems. +
    +
  • Hostname of the mirror (in the summary & description)
    • +
  • IP of the mirror: many mirrors have multiple nodes behind a single DNS entry
    • +
  • Timestamp of the problem
    • +
  • Logs of the problem
    • +
+
+
+
+
+ + +
+
+
+
+

Gentoo Mirrorstats

+
+
+ Mirror Data as current of
{{ lastUpdate }} +
+
+
+
+
+
+
+
+

Questions or comments?

+ Please feel free to contact us. +

Gentoo Mirrorstats

+
+
+
+
+
    +
    • +
    • +
    • +
+
+
+ © 2001–2020 Gentoo Foundation, Inc.
+ + Gentoo is a trademark of the Gentoo Foundation, Inc. + The contents of this document, unless otherwise expressly stated, are licensed under the + CC-BY-SA-4.0 license. + The Gentoo Name and Logo Usage Guidelines apply. + +
+
+
+
+ + + + + + + + + diff --git a/html/stats.jinja2 b/html/stats.jinja2 new file mode 100644 index 0000000..e13cd45 --- /dev/null +++ b/html/stats.jinja2 @@ -0,0 +1,339 @@ + + + + Gentoo Mirrorstats + + + + + + + + + + +
+
+
+
+ +
+
+ Get Gentoo! +
+ + gentoo.org sites + +
+ gentoo.org + Wiki + Bugs + Forums + Packages +
+ Planet + Archives + Sources +
+ Infra Status +
+
+
+
+
+
+
+ +
+ + +
+
+
+

Gentoo {% if type != "all" %}{{ type[0]|upper}}{{type[1:]}}{%endif%} Mirrors + {% if type != "all" %} + + + + + {% endif %} +

+ + +

+ Welcome to the Gentoo Linux mirrorstats system. Below you can see an overview of all Gentoo {% if type != "all" %}{{ type[0]|upper}}{{type[1:]}}{%endif%} mirrors.
+ For more information about Gentoo mirrors, how to contribute a mirror or any problems please have a look at the help page. +

+
+
+ +
+ + + + + {% if type == "all" %}{% endif %} + + + + + + + + + {% for mirror in mirrors %} + {% for stat in mirrors[mirror]['Stats'] %} + + + {% if type == "all" %}{% endif %} + + + + + + {% endfor %} + {% endfor %} + + +
HostTypeProtocolDaily StatsProbe StatsLast State
{{ mirror }}{{ stat['Type'] }}{{ stat['Protocol'] }}{{ stat['StateHistory'] }}{{ stat['ProbeHistory'] }} + {% if stat['StatusLastProbe'] == "ok" %} + OK + {% elif stat['StatusLastProbe'] == "hangs" %} + Hangs + {% elif stat['StatusLastProbe'] == "no_time" %} + No Time + {% elif stat['StatusLastProbe'] == "site_not_found" %} + Site Not Found + {% elif stat['StatusLastProbe'] == "'-1'" %} + -1 + {% else %} + {{ stat['StatusLastProbe'] }} + {% endif %} +
+ +
+
+
+ + +
+
+
+
+

Gentoo Mirrorstats

+
+
+ Mirror Data as current of
{{ lastUpdate }} +
+
+ Data powered by:
Mirmon v2.11 +
+
+
+
+
+
+

Questions or comments?

+ Please feel free to contact us. +

Gentoo Mirrorstats

+
+
+
+
+
    +
    • +
    • +
    • +
+
+
+ © 2001–2020 Gentoo Foundation, Inc.
+ + Gentoo is a trademark of the Gentoo Foundation, Inc. + The contents of this document, unless otherwise expressly stated, are licensed under the + CC-BY-SA-4.0 license. + The Gentoo Name and Logo Usage Guidelines apply. + +
+
+
+
+ + + + + + + + + + + + + + + + + + -- cgit v1.2.3-65-gdbad