CoCalc -- benchmark

GitHub Repository: emscripten-core/emscripten
Path: blob/main/test/benchmark/benchmark_sse.py
⁴¹³³ views
1
#!/usr/bin/env python3
2
# Copyright 2020 The Emscripten Authors.  All rights reserved.
3
# Emscripten is available under two separate licenses, the MIT license and the
4
# University of Illinois/NCSA Open Source License.  Both these licenses can be
5
# found in the LICENSE file.
6

7
import json
8
import os
9
import re
10
import shutil
11
import sys
12
import tempfile
13
from subprocess import Popen
14

15
__rootpath__ = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
16
sys.path.insert(0, __rootpath__)
17

18
from tools.shared import WINDOWS, CLANG_CXX, EMCC, PIPE
19
from tools.shared import run_process
20
from tools.config import V8_ENGINE
21
from common import EMRUN, test_file
22
import clang_native
23

24
temp_dir = tempfile.mkdtemp()
25

26
# System info
27
system_info = Popen([EMRUN, '--system_info'], stdout=PIPE, stderr=PIPE).communicate()
28

29
# Native info
30
native_info = Popen(['clang', '-v'], stdout=PIPE, stderr=PIPE).communicate()
31

32
# Emscripten info
33
emscripten_info = Popen([EMCC, '-v'], stdout=PIPE, stderr=PIPE).communicate()
34

35

36
def run_benchmark(benchmark_file, results_file, build_args):
37
    # Run native build
38
    out_file = os.path.join(temp_dir, 'benchmark_sse_native')
39
    if WINDOWS:
40
        out_file += '.exe'
41
    cmd = [CLANG_CXX] + clang_native.get_clang_native_args() + [benchmark_file, '-O3', '-o', out_file]
42
    print('Building native version of the benchmark:')
43
    print(' '.join(cmd))
44
    run_process(cmd, env=clang_native.get_clang_native_env())
45

46
    native_results = Popen([out_file], stdout=PIPE, stderr=PIPE).communicate()
47
    print(native_results[0])
48

49
    # Run emscripten build
50
    out_file = os.path.join(temp_dir, 'benchmark_sse_html.js')
51
    cmd = [EMCC, benchmark_file, '-O3', '-sTOTAL_MEMORY=536870912', '-o', out_file] + build_args
52
    print('Building Emscripten version of the benchmark:')
53
    print(' '.join(cmd))
54
    run_process(cmd)
55

56
    cmd = V8_ENGINE + ['--experimental-wasm-simd', os.path.basename(out_file)]
57
    print(' '.join(cmd))
58
    old_dir = os.getcwd()
59
    os.chdir(os.path.dirname(out_file))
60
    wasm_results = Popen(cmd, stdout=PIPE, stderr=PIPE).communicate()
61
    os.chdir(old_dir)
62

63
    if not wasm_results:
64
        raise Exception('Unable to run benchmark in V8!')
65

66
    if not wasm_results[0].strip():
67
        print(wasm_results[1])
68
        sys.exit(1)
69

70
    print(wasm_results[0])
71

72
    def strip_comments(text):
73
        return re.sub('//.*?\n|/\*.*?\*/', '', text, re.S) # noqa
74

75
    benchmark_results = strip_comments(wasm_results[0])
76

77
    # Strip out unwanted print output.
78
    benchmark_results = benchmark_results[benchmark_results.find('{'):].strip()
79
    if '*************************' in benchmark_results:
80
        benchmark_results = benchmark_results[:benchmark_results.find('*************************')].strip()
81

82
    print(benchmark_results)
83

84
    shutil.rmtree(temp_dir)
85

86
    native_results = json.loads(native_results[0])
87
    benchmark_results = benchmark_results[benchmark_results.index('{'):benchmark_results.rindex('}') + 1]
88
    wasm_results = json.loads(benchmark_results)
89

90
    # native_workload = native_results['workload']
91
    # html_workload = wasm_results['workload']
92

93
    html = '''<html><head></head><body><h1>SSE JavaScript Benchmark</h1>
94
    <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.8.2/jquery.min.js"></script>
95
    <script src="https://code.highcharts.com/highcharts.js"></script>
96
    <script src="https://code.highcharts.com/modules/exporting.js"></script><b>System Info:</b><br/>
97
    ''' + system_info[0].replace('\n', '<br/>') + '''
98
    <b>Native Clang Compiler:</b><br/>
99
    ''' + native_info[1].replace('\n', '<br/>') + '''
100
    <b>Emscripten Compiler:</b><br/>
101
    ''' + emscripten_info[0].replace('\n', '<br/>')
102

103
    charts_native = {}
104
    charts_html = {}
105
    for result in native_results['results']:
106
        ch = result['chart']
107
        if ch not in charts_native:
108
            charts_native[ch] = []
109
        charts_native[ch] += [result]
110
    for result in wasm_results['results']:
111
        ch = result['chart']
112
        if ch not in charts_html:
113
            charts_html[ch] = []
114
        charts_html[ch] += [result]
115

116
    def find_result_in_category(results, category):
117
        for result in results:
118
            if result['category'] == category:
119
                return result
120
        return None
121

122
    def format_comparison(a, b):
123
        if a < b and a != 0:
124
            return "<span style='color:green;font-weight:bold;'> {:10.2f}".format(b / a) + 'x FASTER</span>'
125
        elif b != 0:
126
            return "<span style='color:red;font-weight:bold;'> {:10.2f}".format(a / b) + 'x SLOWER</span>'
127
        else:
128
            return "<span style='color:red;font-weight:bold;'> NaN </span>"
129

130
    chartNumber = 0
131

132
    total_time_native_scalar = 0
133
    total_time_native_simd = 0
134
    total_time_html_scalar = 0
135
    total_time_html_simd = 0
136

137
    for chart_name, chart_native_results in charts_native.items():
138
        # Extract data for each chart.
139
        categories = []
140
        nativeScalarResults = []
141
        nativeSimdResults = []
142
        htmlScalarResults = []
143
        htmlSimdResults = []
144
        native_results = chart_native_results
145
        wasm_results = charts_html[chart_name]
146
        textual_results_native = '<p>'
147
        textual_results_html = '<p>'
148
        textual_results_html2 = '<p>'
149
        textual_results_html3 = '<p>'
150
        for result in native_results:
151
            categories += ["'" + result['category'] + "'"]
152
            nsc = result['scalar']
153
            nsi = result['simd']
154
            nativeScalarResults += [str(nsc)]
155
            nativeSimdResults += [str(nsi)]
156
            html_result = find_result_in_category(wasm_results, result['category'])
157
            textual_results_native += 'Native ' + result['category'] + ': ' + "{:10.4f}".format(nsc) + 'ns -> ' + "{:10.4f}".format(nsi) + 'ns. '
158
            textual_results_native += 'Native SSE is ' + format_comparison(nsi, nsc) + ' than native scalar. &nbsp; &nbsp; &nbsp; &nbsp; <br />'
159

160
            if html_result is not None:
161
                hsc = html_result['scalar']
162
                htmlScalarResults += [str(hsc)]
163
                hsi = html_result['simd']
164
                htmlSimdResults += [str(hsi)]
165
                textual_results_html += 'JS ' + result['category'] + ': ' + "{:10.4f}".format(hsc) + 'ns -> ' + "{:10.4f}".format(hsi) + 'ns. '
166
                textual_results_html += 'JS SSE is ' + format_comparison(hsi, hsc) + ' than JS scalar. &nbsp; &nbsp; &nbsp; &nbsp; <br />'
167
                textual_results_html2 += 'JS ' + result['category'] + ': JS scalar is ' + format_comparison(hsc, nsc) + ' than native scalar. &nbsp; &nbsp; &nbsp; &nbsp; <br />'
168
                textual_results_html3 += 'JS ' + result['category'] + ': JS SSE is ' + format_comparison(hsi, nsi) + ' than native SSE. &nbsp; &nbsp; &nbsp; &nbsp; <br />'
169
                total_time_native_scalar += nsc
170
                total_time_native_simd += nsi
171
                total_time_html_scalar += hsc
172
                total_time_html_simd += hsi
173
            else:
174
                htmlScalarResults += [str(-1)]
175
                htmlSimdResults += [str(-1)]
176

177
        chartNumber += 1
178
        html += '<div id="chart' + str(chartNumber) + '" style="width:100%; height:400px; margin-top: 100px;"></div>'
179
        html += '''<script>$(function () {
180
        $('#chart''' + str(chartNumber) + '''').highcharts({
181
            chart: {
182
                type: 'column'
183
            },
184
            title: {
185
                text: "''' + chart_name + '''"
186
            },
187
            subtitle: {
188
                text: 'Time per operation in nanoseconds'
189
            },
190
            xAxis: {
191
                categories: [''' + ','.join(categories) + '''
192
                ]
193
            },
194
            yAxis: {
195
                min: 0,
196
                title: {
197
                    text: 'Time (nanoseconds)'
198
                }
199
            },
200
            tooltip: {
201
                headerFormat: '<span style="font-size:10px">{point.key}</span><table>',
202
                pointFormat: '<tr><td style="color:{series.color};padding:0">{series.name}: </td>' +
203
                    '<td style="padding:0"><b>{point.y:.3f} ns</b></td></tr>',
204
                footerFormat: '</table>',
205
                shared: true,
206
                useHTML: true
207
            },
208
            plotOptions: {
209
                column: {
210
                    pointPadding: 0.2,
211
                    borderWidth: 0
212
                }
213
            },
214
            series: [{
215
                name: 'Native scalar',
216
                data: [''' + ','.join(nativeScalarResults) + ''']
217

218
            }, {
219
                name: 'Native SSE',
220
                data: [''' + ','.join(nativeSimdResults) + ''']
221

222
            }, {
223
                name: 'JS scalar',
224
                data: [''' + ','.join(htmlScalarResults) + ''']
225

226
            }, {
227
                name: 'JS SSE',
228
                data: [''' + ','.join(htmlSimdResults) + ''']
229

230
            }]
231
        });
232
    });</script>''' + '<table><tr><td>' + textual_results_native + '</td><td>' + textual_results_html + '</td></tr><tr><td>' + textual_results_html2 + '</td><td>' + textual_results_html3 + '</td></tr></table>'
233

234
    # Final overall score
235

236
    html += '<div id="overallscore" style="width:100%; height:400px; margin-top: 100px;"></div>'
237
    html += '''<script>$(function () {
238
        $('#overallscore').highcharts({
239
            chart: {
240
                type: 'column'
241
            },
242
            title: {
243
                text: "Overall Execution Time"
244
            },
245
            xAxis: {
246
                categories: ['Total time normalized to native']
247
            },
248
            yAxis: {
249
                min: 0,
250
                title: {
251
                    text: 'Relative time'
252
                }
253
            },
254
            tooltip: {
255
                headerFormat: '<span style="font-size:10px">{point.key}</span><table>',
256
                pointFormat: '<tr><td style="color:{series.color};padding:0">{series.name}: </td>' +
257
                    '<td style="padding:0"><b>{point.y:.3f}x</b></td></tr>',
258
                footerFormat: '</table>',
259
                shared: true,
260
                useHTML: true
261
            },
262
            plotOptions: {
263
                column: {
264
                    pointPadding: 0.2,
265
                    borderWidth: 0
266
                }
267
            },
268
            series: [{
269
                name: 'Native scalar',
270
                data: [''' + str(1.0) + ''']
271

272
            }, {
273
                name: 'Native SSE',
274
                data: [''' + (str(total_time_native_simd / total_time_native_scalar) if total_time_native_scalar != 0 else 'N/A') + ''']
275

276
            }, {
277
                name: 'JS scalar',
278
                data: [''' + (str(total_time_html_scalar / total_time_native_scalar) if total_time_native_scalar != 0 else 'N/A') + ''']
279

280
            }, {
281
                name: 'JS SSE',
282
                data: [''' + (str(total_time_html_simd / total_time_native_scalar) if total_time_native_scalar != 0 else 'N/A') + ''']
283

284
            }]
285
        });
286
    });</script>'''
287

288
    html += '</body></html>'
289

290
    open(results_file, 'w').write(html)
291
    print('Wrote ' + str(len(html)) + ' bytes to file ' + results_file + '.')
292

293

294
if __name__ == '__main__':
295
    suite = sys.argv[1].lower() if len(sys.argv) == 2 else None
296
    if suite in ['sse', 'sse1']:
297
        run_benchmark(test_file('sse/benchmark_sse1.cpp'), 'results_sse1.html', ['-msse'])
298
    elif suite == 'sse2':
299
        run_benchmark(test_file('sse/benchmark_sse2.cpp'), 'results_sse2.html', ['-msse2'])
300
    elif suite == 'sse3':
301
        run_benchmark(test_file('sse/benchmark_sse3.cpp'), 'results_sse3.html', ['-msse3'])
302
    elif suite == 'ssse3':
303
        run_benchmark(test_file('sse/benchmark_ssse3.cpp'), 'results_ssse3.html', ['-mssse3'])
304
    else:
305
        raise Exception('Usage: python test/benchmark_sse.py sse1|sse2|sse3')
306

307
Product

Resources

Company
