Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
REMitchell
GitHub Repository: REMitchell/python-crawling
Path: blob/master/6-Selenium/phantomjs/examples/netsniff.js
164 views
1
if (!Date.prototype.toISOString) {
2
Date.prototype.toISOString = function () {
3
function pad(n) { return n < 10 ? '0' + n : n; }
4
function ms(n) { return n < 10 ? '00'+ n : n < 100 ? '0' + n : n }
5
return this.getFullYear() + '-' +
6
pad(this.getMonth() + 1) + '-' +
7
pad(this.getDate()) + 'T' +
8
pad(this.getHours()) + ':' +
9
pad(this.getMinutes()) + ':' +
10
pad(this.getSeconds()) + '.' +
11
ms(this.getMilliseconds()) + 'Z';
12
}
13
}
14
15
function createHAR(address, title, startTime, resources)
16
{
17
var entries = [];
18
19
resources.forEach(function (resource) {
20
var request = resource.request,
21
startReply = resource.startReply,
22
endReply = resource.endReply;
23
24
if (!request || !startReply || !endReply) {
25
return;
26
}
27
28
// Exclude Data URI from HAR file because
29
// they aren't included in specification
30
if (request.url.match(/(^data:image\/.*)/i)) {
31
return;
32
}
33
34
entries.push({
35
startedDateTime: request.time.toISOString(),
36
time: endReply.time - request.time,
37
request: {
38
method: request.method,
39
url: request.url,
40
httpVersion: "HTTP/1.1",
41
cookies: [],
42
headers: request.headers,
43
queryString: [],
44
headersSize: -1,
45
bodySize: -1
46
},
47
response: {
48
status: endReply.status,
49
statusText: endReply.statusText,
50
httpVersion: "HTTP/1.1",
51
cookies: [],
52
headers: endReply.headers,
53
redirectURL: "",
54
headersSize: -1,
55
bodySize: startReply.bodySize,
56
content: {
57
size: startReply.bodySize,
58
mimeType: endReply.contentType
59
}
60
},
61
cache: {},
62
timings: {
63
blocked: 0,
64
dns: -1,
65
connect: -1,
66
send: 0,
67
wait: startReply.time - request.time,
68
receive: endReply.time - startReply.time,
69
ssl: -1
70
},
71
pageref: address
72
});
73
});
74
75
return {
76
log: {
77
version: '1.2',
78
creator: {
79
name: "PhantomJS",
80
version: phantom.version.major + '.' + phantom.version.minor +
81
'.' + phantom.version.patch
82
},
83
pages: [{
84
startedDateTime: startTime.toISOString(),
85
id: address,
86
title: title,
87
pageTimings: {
88
onLoad: page.endTime - page.startTime
89
}
90
}],
91
entries: entries
92
}
93
};
94
}
95
96
var page = require('webpage').create(),
97
system = require('system');
98
99
if (system.args.length === 1) {
100
console.log('Usage: netsniff.js <some URL>');
101
phantom.exit(1);
102
} else {
103
104
page.address = system.args[1];
105
page.resources = [];
106
107
page.onLoadStarted = function () {
108
page.startTime = new Date();
109
};
110
111
page.onResourceRequested = function (req) {
112
page.resources[req.id] = {
113
request: req,
114
startReply: null,
115
endReply: null
116
};
117
};
118
119
page.onResourceReceived = function (res) {
120
if (res.stage === 'start') {
121
page.resources[res.id].startReply = res;
122
}
123
if (res.stage === 'end') {
124
page.resources[res.id].endReply = res;
125
}
126
};
127
128
page.open(page.address, function (status) {
129
var har;
130
if (status !== 'success') {
131
console.log('FAIL to load the address');
132
phantom.exit(1);
133
} else {
134
page.endTime = new Date();
135
page.title = page.evaluate(function () {
136
return document.title;
137
});
138
har = createHAR(page.address, page.title, page.startTime, page.resources);
139
console.log(JSON.stringify(har, undefined, 4));
140
phantom.exit();
141
}
142
});
143
}
144
145