CoCalc -- check_regression

GitHub Repository: stenzek/duckstation
Path: blob/master/scripts/check_regression_tests.py
⁴²⁰¹ views
1
#!/usr/bin/env python3
2
# SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <[email protected]>
3
# SPDX-License-Identifier: CC-BY-NC-ND-4.0
4

5
import argparse
6
import glob
7
import sys
8
import os
9
import re
10
import hashlib
11

12
from pathlib import Path
13

14
FILE_HEADER = """
15
<!DOCTYPE html>
16
<html>
17
<head>
18
<title>Comparison</title>
19
<style>
20
.modal {
21
  display: none;
22
  position: fixed;
23
  z-index: 1;
24
  padding-top: 100px;
25
  left: 0;
26
  top: 0;
27
  width: 100%;
28
  height: 100%;
29
  overflow: auto;
30
  background-color: black;
31
}
32

33
.modal-content {
34
  position: relative;
35
  margin: auto;
36
  padding: 0;
37
  width: 90%;
38
  max-width: 1200px;
39
}
40

41
.close {
42
  color: white;
43
  position: absolute;
44
  top: 10px;
45
  right: 25px;
46
  font-size: 35px;
47
  font-weight: bold;
48
}
49

50
.close:hover,
51
.close:focus {
52
  color: #999;
53
  text-decoration: none;
54
  cursor: pointer;
55
}
56

57
.prev,
58
.next {
59
  cursor: pointer;
60
  position: absolute;
61
  top: 50%;
62
  width: auto;
63
  padding: 16px;
64
  margin-top: -50px;
65
  color: white;
66
  font-weight: bold;
67
  font-size: 20px;
68
  transition: 0.6s ease;
69
  border-radius: 0 3px 3px 0;
70
  user-select: none;
71
  -webkit-user-select: none;
72
}
73

74
.next {
75
  right: 0;
76
  border-radius: 3px 0 0 3px;
77
}
78

79
.prev:hover,
80
.next:hover {
81
  background-color: rgba(0, 0, 0, 0.8);
82
}
83

84
.item img {
85
    cursor: pointer;
86
}
87

88
#compareTitle {
89
    color: white;
90
    font-size: 20px;
91
    font-family: sans-serif;
92
    margin-bottom: 10px;
93
}
94

95
#compareCaption {
96
    color: white;
97
    font-family: sans-serif;
98
}
99

100
#compareState {
101
    display: block;
102
    position: absolute;
103
    right: 0;
104
    top: 0;
105
    color: red;
106
    font-family: sans-serif;
107
    font-size: 20px;
108
    font-weight: bold;
109
}
110

111
#compareImage {
112
    display: block;
113
    margin: 0 auto;
114
    width: 100%;
115
    height: auto;
116
}
117
</style>
118
</head>
119
<body>
120
"""
121

122
FILE_FOOTER = """
123
<div id="myModal" class="modal" tabindex="0">
124
  <span class="close cursor" onclick="closeModal()">&times;</span>
125
  <div class="modal-content">
126
    <div id="compareTitle"></div>
127
    <img id="compareImage" />
128
    <div id="compareState"></div>
129
    <a class="prev">&#10094;</a>
130
    <a class="next">&#10095;</a>
131
    <div id="compareCaption"></div>
132
  </div>
133
</div>
134
<script>
135
/* Worst script known to man */
136
/* Sources:
137
   https://www.w3schools.com/howto/howto_js_lightbox.asp
138
   https://css-tricks.com/prevent-page-scrolling-when-a-modal-is-open/
139
*/
140

141
function openModal() {
142
  document.body.style.position = 'fixed';
143
  document.body.style.top = `-${window.scrollY}px`;
144
  document.getElementById("myModal").style.display = "block";
145
  document.getElementById("myModal").focus();
146
  setImageIndex(0);
147
}
148

149
function closeModal() {
150
    document.getElementById("myModal").style.display = "none";
151
    const scrollY = document.body.style.top;
152
    document.body.style.position = '';
153
    document.body.style.top = '';
154
    window.scrollTo(0, parseInt(scrollY || '0') * -1);
155
}
156

157
function isModalOpen() {
158
    return (document.getElementById("myModal").style.display == "block");
159
}
160

161
function formatLines(str) {
162
    let lines = str.split("\\n")
163
    lines = lines.filter(line => !line.startsWith("Difference in frames"))
164
    return lines.join("<br>")
165
}
166

167
function extractItem(elem) {
168
    var beforeSel = elem.querySelector(".before");
169
    var afterSel = elem.querySelector(".after");
170
    var preSel = elem.querySelector("pre");
171
    return {
172
        name: elem.querySelector("h1").innerText,
173
        beforeImg: beforeSel ? beforeSel.getAttribute("src") : null,
174
        afterImg: afterSel ? afterSel.getAttribute("src") : null,
175
        details: formatLines(preSel ? preSel.innerText : "")
176
    };
177
}
178

179
const items = [...document.querySelectorAll(".item")].map(extractItem)
180
let currentImage = 0;
181
let currentState = 0;
182

183
function getImageIndexForUri(uri) {
184
    for (let i = 0; i < items.length; i++) {
185
        if (items[i].beforeImg == uri || items[i].afterImg == uri)
186
            return i;
187
    }
188
    return -1;
189
}
190

191
function setImageState(state) {
192
    const item = items[currentImage]
193
    const uri = (state === 0) ? item.beforeImg : item.afterImg;
194
    const stateText = (state === 0) ? "BEFORE" : "AFTER";
195
    const posText = "(" + (currentImage + 1).toString() + "/" + (items.length).toString() + ") ";
196
    document.getElementById("compareImage").setAttribute("src", uri);
197
    document.getElementById("compareState").innerText = stateText;
198
    document.getElementById("compareTitle").innerText = posText + item.name;
199
    document.getElementById("compareCaption").innerHTML = item.details;
200
    currentState = state;
201
}
202

203
function setImageIndex(index) {
204
    if (index < 0 || index > items.length)
205
        return;
206

207
    currentImage = index;
208
    setImageState(0);
209
}
210

211
function handleKey(key) {
212
    if (key == " ") {
213
        setImageState((currentState === 0) ? 1 : 0);
214
        return true;
215
    } else if (key == "ArrowLeft") {
216
        setImageIndex(currentImage - 1);
217
        return true;
218
    } else if (key == "ArrowRight") {
219
        setImageIndex(currentImage + 1);
220
        return true;
221
    } else if (key == "Escape") {
222
        closeModal();
223
        return true;
224
    } else {
225
        console.log(key);
226
        return false;
227
    }
228
}
229

230
document.getElementById("myModal").addEventListener("keydown", function(ev) {
231
    if (ev.defaultPrevented)
232
        return;
233
    
234
    if (handleKey(ev.key))
235
        ev.preventDefault();
236
});
237

238
document.querySelector("#myModal .prev").addEventListener("click", function() {
239
    setImageIndex(currentImage - 1);
240
});
241
document.querySelector("#myModal .next").addEventListener("click", function() {
242
    setImageIndex(currentImage + 1);
243
});
244
document.querySelectorAll(".item img").forEach(elem => elem.addEventListener("click", function() {
245
    if (!isModalOpen())
246
        openModal();
247
    setImageIndex(getImageIndexForUri(this.getAttribute("src")));
248
}));
249
</script>
250
</body>
251
</html>
252
"""
253

254
MAX_DIFF_FRAMES = 9999
255

256
outfile = None
257
def write(line):
258
    outfile.write(line + "\n")
259

260

261
def compare_frames(path1, path2):
262
    try:
263
        if os.stat(path1).st_size != os.stat(path2).st_size:
264
            return False
265

266
        with open(path1, "rb") as f:
267
            hash1 = hashlib.md5(f.read()).digest()
268
        with open(path2, "rb") as f:
269
            hash2 = hashlib.md5(f.read()).digest()
270

271
        #print(hash1, hash2)
272
        return hash1 == hash2
273
    except:
274
        return False
275

276

277
def check_regression_test(baselinedir, testdir, name):
278
    #print("Checking '%s'..." % name)
279

280
    dir1 = os.path.join(baselinedir, name)
281
    dir2 = os.path.join(testdir, name)
282
    if not os.path.isdir(dir2):
283
        #print("*** %s is missing in test set" % name)
284
        return False
285

286
    images = glob.glob(os.path.join(dir1, "frame_*.png"))
287
    diff_frames = []
288
    first_fail = True
289
    has_any = False
290
    
291
    for imagepath in images:
292
        imagename = Path(imagepath).name
293
        matches = re.match("frame_([0-9]+).png", imagename)
294
        if matches is None:
295
            continue
296

297
        framenum = int(matches[1])
298

299
        path1 = os.path.join(dir1, imagename)
300
        path2 = os.path.join(dir2, imagename)
301
        if not os.path.isfile(path2):
302
            print("--- Frame %u for %s is missing in test set" % (framenum, name))
303
            if first_fail:
304
                write("<div class=\"item\">")
305
                write("<h1>{}</h1>".format(name))
306
                write("<table width=\"100%\">")
307
                first_fail = False
308
            write("<p>--- Frame %u for %s is missing in test set</p>" % (framenum, name))
309
            continue
310

311
        has_any = True
312
        if not compare_frames(path1, path2):
313
            diff_frames.append((framenum, path1, path2))
314

315
    for (framenum, path1, path2) in diff_frames[-MAX_DIFF_FRAMES:]:
316
        if first_fail:
317
            write("<div class=\"item\">")
318
            write("<h1>{}</h1>".format(name))
319
            write("<table width=\"100%\">")
320
            first_fail = False
321

322
        imguri1 = Path(path1).as_uri()
323
        imguri2 = Path(path2).as_uri()
324
        write("<tr><td colspan=\"2\">Frame %d</td></tr>" % (framenum))
325
        write("<tr><td><img class=\"before\" src=\"%s\" /></td><td><img class=\"after\" src=\"%s\" /></td></tr>" % (imguri1, imguri2))
326

327
    if not first_fail:
328
        write("</table>")
329
        write("<pre>Difference in frames [%s] for %s</pre>" % (",".join(map(lambda x: str(x[0]), diff_frames)), name))
330
        write("</div>")
331
        print("*** Difference in frames [%s] for %s" % (",".join(map(lambda x: str(x[0]), diff_frames)), name))
332
        #assert has_any
333

334
    return len(diff_frames) == 0
335

336

337
def check_regression_tests(baselinedir, testdir):
338
    gamedirs = glob.glob(baselinedir + "/*", recursive=False)
339
    gamedirs.sort(key=lambda x: os.path.basename(x))
340

341
    success = 0
342
    failure = 0
343

344
    for gamedir in gamedirs:
345
        name = Path(gamedir).name
346
        if check_regression_test(baselinedir, testdir, name):
347
            success += 1
348
        else:
349
            failure += 1
350

351
    return (failure == 0)
352

353

354
if __name__ == "__main__":
355
    parser = argparse.ArgumentParser(description="Check frame dump images for regression tests")
356
    parser.add_argument("-baselinedir", action="store", required=True, help="Directory containing baseline frames to check against")
357
    parser.add_argument("-testdir", action="store", required=True, help="Directory containing frames to check")
358
    parser.add_argument("-maxframes", type=int, action="store", required=False, default=9999, help="Max frames to compare")
359
    parser.add_argument("outfile", action="store", help="The file to write the output to")
360

361
    args = parser.parse_args()
362
    MAX_DIFF_FRAMES = args.maxframes
363

364
    outfile = open(args.outfile, "w")
365
    write(FILE_HEADER)
366

367
    if not check_regression_tests(os.path.realpath(args.baselinedir), os.path.realpath(args.testdir)):
368
        write(FILE_FOOTER)
369
        outfile.close()
370
        sys.exit(1)
371
    else:
372
        outfile.close()
373
        os.remove(args.outfile)
374
        sys.exit(0)
375

376

377
Product

Resources

Company
