gnome-shell/src/gnome-shell-test-tool.in

#!@PYTHON@
# -*- mode: Python; indent-tabs-mode: nil; -*-

import datetime
from gi.repository import GLib, GObject, Gio
try:
    import json
except ImportError:
    import simplejson as json
import argparse
import os
import re
import subprocess
import sys
import tempfile
import base64
from configparser import RawConfigParser
import hashlib
import hmac
from http import client
from urllib import parse

def show_version(option, opt_str, value, parser):
    print("GNOME Shell Performance Test @VERSION@")
    sys.exit()

def start_shell(wrap=None, perf_output=None):
    # Set up environment
    env = dict(os.environ)

    filters = ['Gnome-shell-perf-helper'] + options.extra_filter
    env['MUTTER_WM_CLASS_FILTER'] = ','.join(filters)

    if perf_output is not None:
        env['SHELL_PERF_OUTPUT'] = perf_output

    # A fixed background image
    if os.getenv('SHELL_BACKGROUND_IMAGE') is None:
      env['SHELL_BACKGROUND_IMAGE'] = '@pkgdatadir@/perf-background.xml'

    self_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
    args = []
    if wrap:
        args += wrap.split(' ')
    args.append(os.path.join(self_dir, 'gnome-shell'))

    args.append('--automation-script')
    args.append(options.script)

    args.append('--force-animations')

    if options.replace:
        args.append('--replace')

    if options.wayland or options.nested or options.headless:
        args.append('--wayland')
        if options.nested:
            args.append('--nested')
        elif options.headless:
            args.append('--headless')
            if not options.hotplug:
                args.append('--virtual-monitor')
                args.append('1280x720')
        else:
            args.append('--display-server')
        args.append('--wayland-display')
        args.append('gnome-shell-test-display')
    elif options.x11:
        args.append('--x11')

    print("args: {}".format(args))
    return subprocess.Popen(args, env=env)

def run_shell(wrap=None, perf_output=None):
    # we do no additional supervision of gnome-shell,
    # beyond that of wait
    # in particular, we don't kill the shell upon
    # receiving a KeyboardInterrupt, as we expect to be
    # in the same process group
    shell = start_shell(wrap=wrap, perf_output=perf_output)
    shell.wait()
    return shell.returncode == 0

def restore_shell():
    pid = os.fork()
    if (pid == 0):
        os.execlp("gnome-shell", "gnome-shell", "--replace")
    else:
        sys.exit(0)

def upload_performance_report(report_text):
    try:
        config_home = os.environ['XDG_CONFIG_HOME']
    except KeyError:
        config_home = None

    if not config_home:
        config_home = os.path.expanduser("~/.config")

    config_file = os.path.join(config_home, "gnome-shell/perf.ini")

    try:
        config = RawConfigParser()
        f = open(config_file)
        config.readfp(f)
        f.close()

        base_url = config.get('upload', 'url')
        system_name = config.get('upload', 'name')
        secret_key = config.get('upload', 'key')
    except Exception as e:
        print("Can't read upload configuration from %s: %s" % (config_file, str(e)))
        sys.exit(1)

    # Determine host, port and upload URL from provided data, we're
    # a bit extra-careful about normalization since the URL is part
    # of the signature.

    split = parse.urlsplit(base_url)
    scheme = split[0].lower()
    netloc = split[1]
    base_path = split[2]

    m = re.match(r'^(.*?)(?::(\d+))?$', netloc)
    if m.group(2):
        host, port = m.group(1), int(m.group(2))
    else:
        host, port = m.group(1), None

    if scheme != "http":
        print("'%s' is not a HTTP URL" % base_url)
        sys.exit(1)

    if port is None:
        port = 80

    if base_path.endswith('/'):
        base_path = base_path[:-1]

    if port == 80:
        normalized_base = "%s://%s%s" % (scheme, host, base_path)
    else:
        normalized_base = "%s://%s:%d%s" % (scheme, host, port, base_path)

    upload_url = normalized_base + '/system/%s/upload' % system_name
    upload_path = parse.urlsplit(upload_url)[2] # path portion

    # Create signature based on upload URL and the report data

    signature_data = 'POST&' + upload_url + "&&"
    h = hmac.new(secret_key, digestmod=hashlib.sha1)
    h.update(signature_data)
    h.update(report_text)
    signature = parse.quote(base64.b64encode(h.digest()), "~")

    headers = {
        'User-Agent': 'gnome-shell-performance-tool/@VERSION@',
        'Content-Type': 'application/json',
        'X-Shell-Signature': 'HMAC-SHA1 ' + signature
    };

    connection = client.HTTPConnection(host, port)
    connection.request('POST', upload_path, report_text, headers)
    response = connection.getresponse()

    if response.status == 200:
        print("Performance report upload succeeded")
    else:
        print("Performance report upload failed with status %d" % response.status)
        print(response.read())

def gnome_hwtest_log(*args):
    command = ['gnome-hwtest-log', '-t', 'gnome-shell-perf-tool']
    command.extend(args)
    subprocess.check_call(command)

def run_performance_test(wrap=None):
    iters = options.test_iters
    if options.perf_warmup:
        iters += 1

    logs = []
    metric_summaries = {}

    for i in range(0, iters):
        # We create an empty temporary file that the shell will overwrite
        # with the contents.
        handle, output_file = tempfile.mkstemp(".json", "gnome-shell-perf.")
        os.close(handle)

        # Run the performance test and collect the output as JSON
        normal_exit = False
        try:
            normal_exit = run_shell(wrap=wrap, perf_output=output_file)
        except:
            raise
        finally:
            if not normal_exit:
                os.remove(output_file)

        if not normal_exit:
            return False

        try:
            f = open(output_file)
            output = json.load(f)
            f.close()
        except:
            raise
        finally:
            os.remove(output_file)

        # Grab the event definitions and monitor layout the first time around
        if i == 0:
            events = output['events']
            monitors = output['monitors']

        if options.perf_warmup and i == 0:
            continue

        for metric in output['metrics']:
            name = metric['name']
            if not name in metric_summaries:
                summary = {}
                summary['description'] = metric['description']
                summary['units'] = metric['units']
                summary['values'] = []
                metric_summaries[name] = summary
            else:
                summary = metric_summaries[name]

            summary['values'].append(metric['value'])

        logs.append(output['log'])

    if options.perf_output or options.perf_upload:
        # Write a complete report, formatted as JSON. The Javascript/C code that
        # generates the individual reports we are summarizing here is very careful
        # to format them nicely, but we just dump out a compressed no-whitespace
        # version here for simplicity. Using json.dump(indent=0) doesn't real
        # improve the readability of the output much.
        report = {
            'date': datetime.datetime.utcnow().isoformat() + 'Z',
            'events': events,
            'monitors': monitors,
            'metrics': metric_summaries,
            'logs': logs
        }

        # Add the Git revision if available
        self_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
        if os.path.exists(os.path.join(self_dir, 'gnome-shell-jhbuild.in')):
            top_dir = os.path.dirname(self_dir)
            git_dir = os.path.join(top_dir, '.git')
            if os.path.exists(git_dir):
                env = dict(os.environ)
                env['GIT_DIR'] = git_dir
                revision = subprocess.Popen(['git', 'rev-parse', 'HEAD'],
                                            env=env,
                                            stdout=subprocess.PIPE).communicate()[0].strip()
                report['revision'] = revision

        if options.perf_output:
            f = open(options.perf_output, 'w')
            json.dump(report, f)
            f.close()

        if options.perf_upload:
            upload_performance_report(json.dumps(report))
    elif options.hwtest:
        # Log to systemd journal
        for metric in sorted(metric_summaries.keys()):
            summary = metric_summaries[metric]
            gnome_hwtest_log('--metric=' + metric + '=' + str(summary['values'][0]) + summary['units'],
                             '--metric-description=' + summary['description'])
        gnome_hwtest_log('--finished')
    else:
        # Write a human readable summary
        print('------------------------------------------------------------')
        for metric in sorted(metric_summaries.keys()):
            summary = metric_summaries[metric]
            print("#", summary['description'])
            print(metric, ", ".join((str(x) for x in summary['values'])))
        print('------------------------------------------------------------')

    return True

# Main program

parser = argparse.ArgumentParser()
parser.add_argument("script",
                    metavar="AUTOMATION_SCRIPT",
                    help="Automation script to run")
parser.add_argument("--test-iters", type=int, metavar="ITERS",
                    help="Numbers of iterations of the test to run",
                    default=1)
parser.add_argument("--perf-warmup", action="store_true",
                    help="Run a dry run before performance tests")
parser.add_argument("--perf-output", metavar="OUTPUT_FILE",
                    help="Output file to write performance report")
parser.add_argument("--perf-upload", action="store_true",
                    help="Upload performance report to server")
parser.add_argument("--extra-filter", action="append",
                    help="add an extra window class that should be allowed")
parser.add_argument("--hwtest", action="store_true",
                    help="Log results appropriately for GNOME Hardware Testing")
parser.add_argument("--version", action="version",
                    version="GNOME Shell Performance Test @VERSION@")

parser.add_argument("--wrap")

parser.add_argument("-r", "--replace", action="store_true",
                    help="Replace the running window manager")
parser.add_argument("-w", "--wayland", action="store_true",
                    help="Run as a Wayland compositor")
parser.add_argument("-n", "--nested", action="store_true",
                    help="Run as a Wayland nested compositor")
parser.add_argument("-x", "--x11", action="store_true",
                    help="Run as an X11 compositor")
parser.add_argument("--headless", action="store_true",
                    help="Run as a headless Wayland compositor")
parser.add_argument("--hotplug", action="store_true",
                    help="Start without a virtual monitor attached")

options = parser.parse_args()

if options.extra_filter is None:
    options.extra_filter = []

if options.script == 'hwtest':
    options.extra_filter.append('Gedit')

normal_exit = run_performance_test(wrap=options.wrap)
if normal_exit:
    if not options.hwtest and options.replace:
        restore_shell()
else:
    sys.exit(1)