CoCalc -- benchmark

GitHub Repository: emscripten-core/emscripten
Path: blob/main/test/benchmark/benchmark_sse.py
⁶¹⁷³ views
1
#!/usr/bin/env python3
2
# Copyright 2020 The Emscripten Authors.  All rights reserved.
3
# Emscripten is available under two separate licenses, the MIT license and the
4
# University of Illinois/NCSA Open Source License.  Both these licenses can be
5
# found in the LICENSE file.
6

7
import json
8
import os
9
import re
10
import subprocess
11
import sys
12

13
__scriptdir__ = os.path.dirname(os.path.abspath(__file__))
14
__testdir__ = os.path.dirname(os.path.dirname(__file__))
15
__rootpath__ = os.path.dirname(__testdir__)
16
print(__rootpath__)
17
sys.path.insert(0, __rootpath__)
18
sys.path.insert(0, __testdir__)
19

20
import clang_native
21
from common import EMRUN, test_file
22

23
from tools.config import V8_ENGINE
24
from tools.shared import CLANG_CXX, EMCC
25
from tools.utils import WINDOWS, run_process
26

27
# System info
28
system_info = subprocess.check_output([EMRUN, '--system_info'], stderr=subprocess.STDOUT, text=True)
29

30
# Native info
31
native_info = subprocess.check_output(['clang', '-v'], stderr=subprocess.STDOUT, text=True)
32

33
# Emscripten info
34
emscripten_info = subprocess.check_output([EMCC, '-v'], stderr=subprocess.STDOUT, text=True)
35

36

37
def run_benchmark(benchmark_file, results_file, build_args):
38
    out_dir = os.path.join(__rootpath__, 'out')
39
    results_file = os.path.join(out_dir, results_file)
40
    # Run native build
41
    out_file = os.path.join(out_dir, 'benchmark_sse_native')
42
    if WINDOWS:
43
        out_file += '.exe'
44
    cmd = [CLANG_CXX] + clang_native.get_clang_native_args() + [benchmark_file, '-O3', '-o', out_file]
45
    print('Building native version of the benchmark:')
46
    print(' '.join(cmd))
47
    run_process(cmd, env=clang_native.get_clang_native_env())
48

49
    native_results = subprocess.check_output(out_file)
50
    print('native_results', native_results)
51

52
    # Run emscripten build
53
    out_file = os.path.join(out_dir, 'benchmark_sse_html.js')
54
    cmd = [EMCC, benchmark_file, '-sENVIRONMENT=web,shell,node', '-msimd128', '-O3', '-sTOTAL_MEMORY=536870912', '-o', out_file] + build_args
55
    print('Building Emscripten version of the benchmark:')
56
    print(' '.join(cmd))
57
    run_process(cmd)
58

59
    cmd = V8_ENGINE + [os.path.basename(out_file)]
60
    print(' '.join(cmd))
61
    wasm_results = subprocess.check_output(cmd, cwd=out_dir, text=True)
62

63
    print('wasm_results', wasm_results)
64

65
    def strip_comments(text):
66
        return re.sub(r'//.*?\n|/\*.*?\*/', '', text, flags=re.S) # noqa
67

68
    benchmark_results = strip_comments(wasm_results)
69
    print('stripped', benchmark_results)
70

71
    # Strip out unwanted print output.
72
    benchmark_results = benchmark_results[benchmark_results.find('{'):].strip()
73
    if '*************************' in benchmark_results:
74
        benchmark_results = benchmark_results[:benchmark_results.find('*************************')].strip()
75

76
    print(benchmark_results)
77

78
    native_results = json.loads(native_results)
79
    benchmark_results = benchmark_results[benchmark_results.index('{'):benchmark_results.rindex('}') + 1]
80
    wasm_results = json.loads(benchmark_results)
81

82
    # native_workload = native_results['workload']
83
    # html_workload = wasm_results['workload']
84

85
    html = '''<html><head></head><body><h1>SSE JavaScript Benchmark</h1>
86
    <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.8.2/jquery.min.js"></script>
87
    <script src="https://code.highcharts.com/highcharts.js"></script>
88
    <script src="https://code.highcharts.com/modules/exporting.js"></script><b>System Info:</b><br/>
89
    ''' + system_info.replace('\n', '<br/>') + '''
90
    <b>Native Clang Compiler:</b><br/>
91
    ''' + native_info.replace('\n', '<br/>') + '''
92
    <b>Emscripten Compiler:</b><br/>
93
    ''' + emscripten_info.replace('\n', '<br/>')
94

95
    charts_native = {}
96
    charts_html = {}
97
    for result in native_results['results']:
98
        ch = result['chart']
99
        if ch not in charts_native:
100
            charts_native[ch] = []
101
        charts_native[ch] += [result]
102
    for result in wasm_results['results']:
103
        ch = result['chart']
104
        if ch not in charts_html:
105
            charts_html[ch] = []
106
        charts_html[ch] += [result]
107

108
    def find_result_in_category(results, category):
109
        for result in results:
110
            if result['category'] == category:
111
                return result
112
        return None
113

114
    def format_comparison(a, b):
115
        if a < b and a != 0:
116
            return "<span style='color:green;font-weight:bold;'> {:10.2f}".format(b / a) + 'x FASTER</span>'
117
        elif b != 0:
118
            return "<span style='color:red;font-weight:bold;'> {:10.2f}".format(a / b) + 'x SLOWER</span>'
119
        else:
120
            return "<span style='color:red;font-weight:bold;'> NaN </span>"
121

122
    chartNumber = 0
123

124
    total_time_native_scalar = 0
125
    total_time_native_simd = 0
126
    total_time_html_scalar = 0
127
    total_time_html_simd = 0
128

129
    for chart_name, chart_native_results in charts_native.items():
130
        # Extract data for each chart.
131
        categories = []
132
        nativeScalarResults = []
133
        nativeSimdResults = []
134
        htmlScalarResults = []
135
        htmlSimdResults = []
136
        native_results = chart_native_results
137
        wasm_results = charts_html[chart_name]
138
        textual_results_native = '<p>'
139
        textual_results_html = '<p>'
140
        textual_results_html2 = '<p>'
141
        textual_results_html3 = '<p>'
142
        for result in native_results:
143
            categories += ["'" + result['category'] + "'"]
144
            nsc = result['scalar']
145
            nsi = result['simd']
146
            nativeScalarResults += [str(nsc)]
147
            nativeSimdResults += [str(nsi)]
148
            html_result = find_result_in_category(wasm_results, result['category'])
149
            textual_results_native += 'Native ' + result['category'] + ': ' + "{:10.4f}".format(nsc) + 'ns -> ' + "{:10.4f}".format(nsi) + 'ns. '
150
            textual_results_native += 'Native SSE is ' + format_comparison(nsi, nsc) + ' than native scalar. &nbsp; &nbsp; &nbsp; &nbsp; <br />'
151

152
            if html_result is not None:
153
                hsc = html_result['scalar']
154
                htmlScalarResults += [str(hsc)]
155
                hsi = html_result['simd']
156
                htmlSimdResults += [str(hsi)]
157
                textual_results_html += 'JS ' + result['category'] + ': ' + "{:10.4f}".format(hsc) + 'ns -> ' + "{:10.4f}".format(hsi) + 'ns. '
158
                textual_results_html += 'JS SSE is ' + format_comparison(hsi, hsc) + ' than JS scalar. &nbsp; &nbsp; &nbsp; &nbsp; <br />'
159
                textual_results_html2 += 'JS ' + result['category'] + ': JS scalar is ' + format_comparison(hsc, nsc) + ' than native scalar. &nbsp; &nbsp; &nbsp; &nbsp; <br />'
160
                textual_results_html3 += 'JS ' + result['category'] + ': JS SSE is ' + format_comparison(hsi, nsi) + ' than native SSE. &nbsp; &nbsp; &nbsp; &nbsp; <br />'
161
                total_time_native_scalar += nsc
162
                total_time_native_simd += nsi
163
                total_time_html_scalar += hsc
164
                total_time_html_simd += hsi
165
            else:
166
                htmlScalarResults += [str(-1)]
167
                htmlSimdResults += [str(-1)]
168

169
        chartNumber += 1
170
        html += '<div id="chart' + str(chartNumber) + '" style="width:100%; height:400px; margin-top: 100px;"></div>'
171
        html += '''<script>$(function () {
172
        $('#chart''' + str(chartNumber) + '''').highcharts({
173
            chart: {
174
                type: 'column'
175
            },
176
            title: {
177
                text: "''' + chart_name + '''"
178
            },
179
            subtitle: {
180
                text: 'Time per operation in nanoseconds'
181
            },
182
            xAxis: {
183
                categories: [''' + ','.join(categories) + '''
184
                ]
185
            },
186
            yAxis: {
187
                min: 0,
188
                title: {
189
                    text: 'Time (nanoseconds)'
190
                }
191
            },
192
            tooltip: {
193
                headerFormat: '<span style="font-size:10px">{point.key}</span><table>',
194
                pointFormat: '<tr><td style="color:{series.color};padding:0">{series.name}: </td>' +
195
                    '<td style="padding:0"><b>{point.y:.3f} ns</b></td></tr>',
196
                footerFormat: '</table>',
197
                shared: true,
198
                useHTML: true
199
            },
200
            plotOptions: {
201
                column: {
202
                    pointPadding: 0.2,
203
                    borderWidth: 0
204
                }
205
            },
206
            series: [{
207
                name: 'Native scalar',
208
                data: [''' + ','.join(nativeScalarResults) + ''']
209

210
            }, {
211
                name: 'Native SSE',
212
                data: [''' + ','.join(nativeSimdResults) + ''']
213

214
            }, {
215
                name: 'JS scalar',
216
                data: [''' + ','.join(htmlScalarResults) + ''']
217

218
            }, {
219
                name: 'JS SSE',
220
                data: [''' + ','.join(htmlSimdResults) + ''']
221

222
            }]
223
        });
224
    });</script>''' + '<table><tr><td>' + textual_results_native + '</td><td>' + textual_results_html + '</td></tr><tr><td>' + textual_results_html2 + '</td><td>' + textual_results_html3 + '</td></tr></table>'
225

226
    # Final overall score
227

228
    html += '<div id="overallscore" style="width:100%; height:400px; margin-top: 100px;"></div>'
229
    html += '''<script>$(function () {
230
        $('#overallscore').highcharts({
231
            chart: {
232
                type: 'column'
233
            },
234
            title: {
235
                text: "Overall Execution Time"
236
            },
237
            xAxis: {
238
                categories: ['Total time normalized to native']
239
            },
240
            yAxis: {
241
                min: 0,
242
                title: {
243
                    text: 'Relative time'
244
                }
245
            },
246
            tooltip: {
247
                headerFormat: '<span style="font-size:10px">{point.key}</span><table>',
248
                pointFormat: '<tr><td style="color:{series.color};padding:0">{series.name}: </td>' +
249
                    '<td style="padding:0"><b>{point.y:.3f}x</b></td></tr>',
250
                footerFormat: '</table>',
251
                shared: true,
252
                useHTML: true
253
            },
254
            plotOptions: {
255
                column: {
256
                    pointPadding: 0.2,
257
                    borderWidth: 0
258
                }
259
            },
260
            series: [{
261
                name: 'Native scalar',
262
                data: [''' + str(1.0) + ''']
263

264
            }, {
265
                name: 'Native SSE',
266
                data: [''' + (str(total_time_native_simd / total_time_native_scalar) if total_time_native_scalar != 0 else 'N/A') + ''']
267

268
            }, {
269
                name: 'JS scalar',
270
                data: [''' + (str(total_time_html_scalar / total_time_native_scalar) if total_time_native_scalar != 0 else 'N/A') + ''']
271

272
            }, {
273
                name: 'JS SSE',
274
                data: [''' + (str(total_time_html_simd / total_time_native_scalar) if total_time_native_scalar != 0 else 'N/A') + ''']
275

276
            }]
277
        });
278
    });</script>'''
279

280
    html += '</body></html>'
281

282
    open(results_file, 'w').write(html)
283
    print('Wrote ' + str(len(html)) + ' bytes to file ' + results_file + '.')
284

285

286
if __name__ == '__main__':
287
    suite = sys.argv[1].lower() if len(sys.argv) == 2 else None
288
    if suite in ['sse', 'sse1']:
289
        run_benchmark(test_file('benchmark/benchmark_sse1.cpp'), 'results_sse1.html', ['-msse'])
290
    elif suite == 'sse2':
291
        run_benchmark(test_file('benchmark/benchmark_sse2.cpp'), 'results_sse2.html', ['-msse2'])
292
    elif suite == 'sse3':
293
        run_benchmark(test_file('benchmark/benchmark_sse3.cpp'), 'results_sse3.html', ['-msse3'])
294
    elif suite == 'ssse3':
295
        run_benchmark(test_file('benchmark/benchmark_ssse3.cpp'), 'results_ssse3.html', ['-mssse3'])
296
    else:
297
        raise Exception('Usage: python test/benchmark/benchmark_sse.py sse1|sse2|sse3')
298

299
Product

Resources

Company
